diff -Nru pmix-3.2.2~rc1/autogen.pl pmix-4.0.0/autogen.pl
--- pmix-3.2.2~rc1/autogen.pl	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/autogen.pl	2021-01-02 08:56:17.000000000 +0000
@@ -72,6 +72,15 @@
 my $hostname;
 my $full_hostname;
 
+# Patch program
+my $patch_prog = "patch";
+# Solaris "patch" doesn't understand unified diffs, and will cause
+# autogen.pl to hang with a "File to patch:" prompt. Default to Linux
+# "patch", but use "gpatch" on Solaris.
+if ($^O eq "solaris") {
+    $patch_prog = "gpatch";
+}
+
 $username = getpwuid($>);
 $full_hostname = `hostname`;
 chomp($full_hostname);
@@ -548,6 +557,129 @@
 
 ##############################################################################
 
+sub patch_autotools_output {
+    my ($topdir) = @_;
+
+    # Set indentation string for verbose output depending on current directory.
+    my $indent_str = "    ";
+    if ($topdir eq ".") {
+        $indent_str = "=== ";
+    }
+
+    # Patch ltmain.sh error for PGI version numbers.  Redirect stderr to
+    # /dev/null because this patch is only necessary for some versions of
+    # Libtool (e.g., 2.2.6b); it'll [rightfully] fail if you have a new
+    # enough Libtool that doesn't need this patch.  But don't alarm the
+    # user and make them think that autogen failed if this patch fails --
+    # make the errors be silent.
+    # Also patch ltmain.sh for NAG compiler
+    if (-f "config/ltmain.sh") {
+        verbose "$indent_str"."Patching PGI compiler version numbers in ltmain.sh\n";
+        system("$patch_prog -N -p0 < $topdir/config/ltmain_pgi_tp.diff >/dev/null 2>&1");
+        unlink("config/ltmain.sh.rej");
+
+        verbose "$indent_str"."Patching \"-pthread\" option for NAG compiler in ltmain.sh\n";
+        system("$patch_prog -N -p0 < $topdir/config/ltmain_nag_pthread.diff >/dev/null 2>&1");
+        unlink("config/ltmain.sh.rej");
+    }
+
+    # If there's no configure script, there's nothing else to do.
+    return
+        if (! -f "configure");
+    my @verbose_out;
+
+    # Total ugh.  We have to patch the configure script itself.  See below
+    # for explanations why.
+    open(IN, "configure") || my_die "Can't open configure";
+    my $c;
+    $c .= $_
+        while(<IN>);
+    close(IN);
+    my $c_orig = $c;
+
+    # The PGI 10 version number broke <=LT
+    # 2.2.6b's version number checking regexps.  We can't fix the
+    # Libtool install; all we can do is patch the resulting configure
+    # script.  :-( The following comes from the upstream patch:
+    # http://lists.gnu.org/archive/html/libtool-patches/2009-11/msg00016.html
+    push(@verbose_out, $indent_str . "Patching configure for Libtool PGI version number regexps\n");
+    $c =~ s/\*pgCC\\ \[1-5\]\* \| \*pgcpp\\ \[1-5\]\*/*pgCC\\ [1-5]\.* | *pgcpp\\ [1-5]\.*/g;
+
+    # See http://git.savannah.gnu.org/cgit/libtool.git/commit/?id=v2.2.6-201-g519bf91 for details
+    # Note that this issue was fixed in LT 2.2.8, however most distros are still using 2.2.6b
+
+    push(@verbose_out, $indent_str . "Patching configure for IBM xlf libtool bug\n");
+    $c =~ s/(\$LD -shared \$libobjs \$deplibs \$)compiler_flags( -soname \$soname)/$1linker_flags$2/g;
+
+    #Check if we are using a recent enough libtool that supports PowerPC little endian
+    if(index($c, 'powerpc64le-*linux*)') == -1) {
+        push(@verbose_out, $indent_str . "Patching configure for PowerPC little endian support\n");
+        my $replace_string = "x86_64-*kfreebsd*-gnu|x86_64-*linux*|powerpc*-*linux*|";
+        $c =~ s/x86_64-\*kfreebsd\*-gnu\|x86_64-\*linux\*\|ppc\*-\*linux\*\|powerpc\*-\*linux\*\|/$replace_string/g;
+        $replace_string =
+        "powerpc64le-*linux*)\n\t    LD=\"\${LD-ld} -m elf32lppclinux\"\n\t    ;;\n\t  powerpc64-*linux*)";
+        $c =~ s/ppc64-\*linux\*\|powerpc64-\*linux\*\)/$replace_string/g;
+        $replace_string =
+        "powerpcle-*linux*)\n\t    LD=\"\${LD-ld} -m elf64lppc\"\n\t    ;;\n\t  powerpc-*linux*)";
+        $c =~ s/ppc\*-\*linux\*\|powerpc\*-\*linux\*\)/$replace_string/g;
+    }
+
+    # Fix consequence of broken libtool.m4
+    # see http://lists.gnu.org/archive/html/bug-libtool/2015-07/msg00002.html and
+    # https://github.com/open-mpi/ompi/issues/751
+    push(@verbose_out, $indent_str . "Patching configure for -L/-R libtool.m4 bug\n");
+    # patch for libtool < 2.4.3
+    $c =~ s/# Some compilers place space between "-\{L,R\}" and the path.\n       # Remove the space.\n       if test \$p = \"-L\" \|\|/# Some compilers place space between "-\{L,-l,R\}" and the path.\n       # Remove the spaces.\n       if test \$p = \"-L\" \|\|\n          test \$p = \"-l\" \|\|/g;
+    # patch for libtool >= 2.4.3
+    $c =~ s/# Some compilers place space between "-\{L,R\}" and the path.\n       # Remove the space.\n       if test x-L = \"\$p\" \|\|\n          test x-R = \"\$p\"\; then/# Some compilers place space between "-\{L,-l,R\}" and the path.\n       # Remove the spaces.\n       if test x-L = \"x\$p\" \|\|\n          test x-l = \"x\$p\" \|\|\n          test x-R = \"x\$p\"\; then/g;
+
+    # Fix OS X Big Sur (11.0.x) support
+    # From https://lists.gnu.org/archive/html/libtool-patches/2020-06/msg00001.html
+    push(@verbose_out, $indent_str . "Patching configure for MacOS Big Sur libtool.m4 bug\n");
+    # Some versions of Libtool use ${wl} consistently, but others did
+    # not (e.g., they used $wl).  Make the regexp be able to handle
+    # both.  Additionally, the case string searching for 10.[012]*
+    # changed over time.  So make sure it can handle both of the case
+    # strings that we're aware of.
+    my $WL = '(\$\{wl\}|\$wl)';
+    my $SOMETIMES = '(\[,.\])*';
+    my $search_string = 'darwin\*\) # darwin 5.x on
+      # if running on 10.5 or later, the deployment target defaults
+      # to the OS version, if on x86, and 10.4, the deployment
+      # target defaults to 10.4. Don\'t you love it\?
+      case \$\{MACOSX_DEPLOYMENT_TARGET-10.0\},\$host in
+    10.0,\*86\*-darwin8\*\|10.0,\*-darwin\[91\]\*\)
+      _lt_dar_allow_undefined=\'' . $WL . '-undefined ' . $WL . 'dynamic_lookup\' ;;
+    10.\[012\]' . $SOMETIMES . '\*\)
+      _lt_dar_allow_undefined=\'' . $WL . '-flat_namespace ' . $WL . '-undefined ' . $WL . 'suppress\' ;;
+    10.\*\)';
+    my $replace_string = 'darwin*)
+      # PMIx patched for Darwin / MacOS Big Sur.  See
+      # http://lists.gnu.org/archive/html/bug-libtool/2015-07/msg00001.html
+      case ${MACOSX_DEPLOYMENT_TARGET},$host in
+      10.[012],*|,*powerpc*)
+      _lt_dar_allow_undefined=\'${wl}-flat_namespace ${wl}-undefined ${wl}suppress\' ;;
+      *)';
+    $c =~ s/$search_string/$replace_string/g;
+
+    # Only write out verbose statements and a new configure if the
+    # configure content actually changed
+    return
+        if ($c eq $c_orig);
+    foreach my $str (@verbose_out) {
+        verbose($str);
+    }
+
+    open(OUT, ">configure.patched") || my_die "Can't open configure.patched";
+    print OUT $c;
+    close(OUT);
+    # Use cp so that we preserve permissions on configure
+    safe_system("cp configure.patched configure");
+    unlink("configure.patched");
+}
+
+##############################################################################
+
 sub in_tarball {
     my $tarball = 0;
     open(IN, "VERSION") || my_die "Can't open VERSION";
@@ -618,7 +750,7 @@
 dnl This file is automatically created by autogen.pl; it should not
 dnl be edited by hand!!
 dnl
-dnl Generated by $username at " . localtime(time) . "
+dnl Generated by $username at " . localtime($ENV{SOURCE_DATE_EPOCH} || time) . "
 dnl on $full_hostname.
 $dnl_line\n\n";
 
@@ -743,6 +875,8 @@
 my $cmd = "autoreconf -ivf --warnings=all,no-obsolete,no-override -I config";
 safe_system($cmd);
 
+patch_autotools_output(".");
+
 #---------------------------------------------------------------------------
 
 verbose "
diff -Nru pmix-3.2.2~rc1/bindings/Makefile.am pmix-4.0.0/bindings/Makefile.am
--- pmix-3.2.2~rc1/bindings/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+SUBDIRS = python
diff -Nru pmix-3.2.2~rc1/bindings/python/construct.py pmix-4.0.0/bindings/python/construct.py
--- pmix-3.2.2~rc1/bindings/python/construct.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/construct.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,518 @@
+#!/usr/bin/env python3
+
+import os, os.path, sys, shutil, signal
+from optparse import OptionParser, OptionGroup
+
+takeconst = False
+takeapis = False
+takedtypes = False
+
+def signal_handler(signal, frame):
+    print("Ctrl-C received")
+    sys.exit(0)
+
+def harvest_constants(options, src, constants, definitions):
+    global takeconst, takeapis, takedtypes
+
+    path = os.path.join(options.src, src)
+    # open the file
+    try:
+        inputfile = open(path, "r")
+    except:
+        print("File", path, "could not be opened")
+        return
+    # read the file - these files aren't too large
+    # so ingest the whole thing at one gulp
+    lines = inputfile.readlines()
+    # cache the string constants, numeric, and typedef constants
+    # in separate lists
+    strconsts = []
+    strconstlen = 0
+    errconsts = []
+    nconsts = []
+    nconstlen = 0
+    typedefs = []
+    apis = []
+    # loop over the lines
+    for n in range(len(lines)):
+        line = lines[n]
+        # remove white space at front and back
+        myline = line.strip()
+        # remove comment lines
+        if "/*" in myline or "*/" in myline or myline.startswith("*"):
+            n += 1
+            continue
+        # if the line starts with #define, then we want it
+        if takeconst and myline.startswith("#define"):
+            value = myline[8:]
+            # skip some well-known unwanted values
+            if value.startswith("PMIx"):
+                continue
+            if value.startswith("PMIX_HAVE_VISIB"):
+                continue
+            tokens = value.split()
+            if len(tokens) >= 2:
+                if tokens[1][0] == '"':
+                    strconsts.append(tokens[0])
+                    if len(tokens[0]) > strconstlen:
+                        strconstlen = len(tokens[0])
+                elif "PMIX_ERR_" in value or tokens[1].startswith("-"):
+                    # numerical constant that looks just like a
+                    # string constant - i.e., PMIX_ERR_FOO...1
+                    # we output them in a separate section, but
+                    # consider them string constants
+                    errconsts.append(tokens[0])
+                    if len(tokens[0]) > strconstlen:
+                        strconstlen = len(tokens[0])
+                elif tokens[1].isdigit() or tokens[1].startswith("UINT") or tokens[1].startswith("0x"):
+                    # values that were defined as UINT32_MAX need to be converted
+                    # to hex values as Python otherwise gets confused
+                    if tokens[1] == "UINT32_MAX":
+                        tokens[1] = "0xffffffff"
+                    elif "UINT32_MAX-1" in tokens[1]:
+                        tokens[1] = "0xfffffffe"
+                    elif "UINT32_MAX-2" in tokens[1]:
+                        tokens[1] = "0xfffffffd"
+                    elif "UINT32_MAX-3" in tokens[1]:
+                        tokens[1] = "0xfffffffc"
+                    elif "UINT8_MAX" in tokens[1]:
+                        tokens[1] = "0xff"
+                    nconsts.append([tokens[0], tokens[1]])
+                    if len(tokens[0]) > nconstlen:
+                        nconstlen = len(tokens[0])
+        elif takeapis and myline.startswith("PMIX_EXPORT"):
+            value = myline[12:].strip()
+            # this is the name of an API - these
+            # are frequently multi-line, so collect
+            # all of them
+            if ";" in value:
+                value = value[:-1]
+                # check for bool type - must be converted to bint
+                value = value.replace("bool ", "bint ")
+                # a single-line API might have a "void" arg
+                # Python doesn't accept "void" as an arg, so
+                # we have to "snip" it out
+                start = value.find("(") + 1
+                end = value.find(")")
+                snip = value[start:end]
+                if snip == "void":
+                    value = value[0:start] + value[end:]
+                newapi = [value]
+                apis.append(newapi)
+            else:
+                value = value.replace("bool ", "bint ")
+                newapi = [value]
+                apirunning = True
+                while apirunning:
+                    n += 1
+                    value = lines[n].strip()
+                    # check for bool type - must be converted to bint
+                    value = value.replace("bool ", "bint ")
+                    if ";" in value:
+                        apirunning = False
+                        value = value[:-1]
+                    newapi.append(value)
+                apis.append(newapi)
+        elif takedtypes and myline.startswith("typedef"):
+            # there are three types of typedef's in PMIx:
+            #
+            # 1. simple one-line typedef - e.g., "typedef int foo"
+            #
+            # 2. multi-line struct types
+            #
+            # 3. function definitions - by PMIx convention, these
+            #    always have an "fn_t" in the name
+            #
+            # 4. enum definitions
+            #
+            # start with the fourth option by looking for "enum"
+            if "enum" in myline:
+                # each line after this one should contain a name
+                # so we just assign a value sequentially.
+                counter = 0
+                n += 1
+                value = lines[n].strip()
+                if ',' in value:
+                    value = value[:-1]
+                while '}' not in value and n < len(lines):
+                    tokens[0] = value
+                    tokens[1] = str(counter)
+                    counter += 1
+                    nconsts.append([tokens[0], tokens[1]])
+                    if len(tokens[0]) > nconstlen:
+                        nconstlen = len(tokens[0])
+                    n += 1
+                    value = lines[n].strip()
+                    if ',' in value:
+                        value = value[:-1]
+                # the termination line contains the type name
+                # for this enum - declare it as integer here
+                value = "typedef int " + value[2:]
+                typedefs.append([value])
+            # now address the first option - detectable by
+            # having a semi-colon at the end of the line and
+            # no "fn_t" or "cbfunc_t" in it
+            elif ";" in myline and not "fn_t" in myline and not "cbfunc_t" in myline:
+                value = myline[:-1]
+                # check for bool type - must be converted to bint
+                if "bool" in value:
+                    value.replace("bool ", "bint ")
+                # check for pre-declaration statements of form
+                # typedef struct foo foo
+                ck = value.split()
+                if len(ck) == 4 and ck[1] == "struct" and ck[2] == ck[3]:
+                    n += 1
+                    continue
+                else:
+                    # check for a typedef that includes a named value
+                    # of either PMIX_MAX_NSLEN or PMIX_MAX_KEYLEN
+                    if "PMIX_MAX_NSLEN+1" in value:
+                        value = value.replace("PMIX_MAX_NSLEN+1", str(256))
+                    elif "PMIX_MAX_NSLEN" in value:
+                        value = value.replace("PMIX_MAX_NSLEN", str(255))
+                    elif "PMIX_MAX_KEYLEN+1" in value:
+                        value = value.replace("PMIX_MAX_KEYLEN+1", str(512))
+                    elif "PMIX_MAX_KEYLEN" in value:
+                        value = value.replace("PMIX_MAX_KEYLEN", str(511))
+                    typedefs.append([value])
+            # now check the third option by looking for
+            # "fn_t" or "cbfunc_t" in it
+            elif "fn_t" in myline or "cbfunc_t" in myline:
+                if ";" in myline:
+                    # this is a one-line function definition
+                    value = myline[:-1]
+                    # check for bool type - must be converted to bint
+                    if "bool" in value:
+                        value.replace("bool ", "bint ")
+                    typedefs.append([value])
+                else:
+                    # this is a multi-line function definition
+                    # check for bool type - must be converted to bint
+                    if "bool" in myline:
+                        myline.replace("bool ", "bint ")
+                    newdef = [myline]
+                    defrunning = True
+                    while defrunning:
+                        n += 1
+                        value = lines[n].strip()
+                        # check for bool type - must be converted to bint
+                        if "bool" in value:
+                            value.replace("bool ", "bint ")
+                        if ";" in value:
+                            defrunning = False
+                            value = value[:-1]
+                        newdef.append(value)
+                    typedefs.append(newdef)
+            # must be a multi-line struct type definition
+            # we capture these in the typedef list
+            # so we output their type definition
+            else:
+                # check for pre-declaration statements of form
+                # typedef struct foo foo
+                value = myline
+                ck = value.split()
+                if len(ck) == 4 and ck[1] == "struct" and ck[2] == ck[3]:
+                    n += 1
+                    continue
+                else:
+                    newdef = []
+                    n += 1
+                    value = lines[n].strip()
+                    nbrk = 1
+                    while nbrk > 0:
+                        # avoid comments
+                        if "/*" in value or "*/" in value or value.startswith("*"):
+                            n += 1
+                            value = lines[n].strip()
+                            continue
+                        if "}" in value:
+                            nbrk -= 1
+                            if nbrk > 0:
+                                n += 1
+                                value = lines[n].strip()
+                            continue
+                        if  "union" in value:
+                            # we have to create another definition that contains
+                            # just the union and then add that back into the
+                            # current typedef
+                            uniondef = []
+                            n += 1
+                            value = lines[n].strip()
+                            while "}" not in value:
+                                # terminate at the semi-colon
+                                idx = value.rfind(';')
+                                value = "    " + value[:idx]
+                                # check for bool type - must be converted to bint
+                                if "bool" in value:
+                                    value = value.replace("bool ", "bint ")
+                                elif "struct timeval " in value:
+                                    value = value.replace("struct timeval", "timeval")
+                                # add it to the union def
+                                uniondef.append(value)
+                                n += 1
+                                value = lines[n].strip()
+                            # terminate at the semi-colon
+                            idx = value.rfind(';')
+                            value = value[:idx]
+                            # extract the name of the union
+                            idx = value.rfind(' ')
+                            value = value[idx+1:]
+                            # save this name
+                            lowname = value
+                            # capitalize the first letter
+                            value = value.title()
+                            # create the union definition - the output
+                            # routine will prepend a 'c'
+                            myvalue = "def union " + value + ":"
+                            uniondef.insert(0, myvalue)
+                            typedefs.append(uniondef)
+                            # add the union to the struct
+                            value = "    " + value + " " + lowname
+                            newdef.append(value)
+                            nbrk -= 1
+                            n += 1
+                            value = lines[n].strip()
+                            continue
+                        # terminate at the semi-colon
+                        idx = value.rfind(';')
+                        value = "    " + value[:idx]
+                        # check for bool type - must be converted to bint
+                        if "bool" in value:
+                            value = value.replace("bool ", "bint ")
+                        elif "struct timeval " in value:
+                            value = value.replace("struct timeval", "timeval")
+                        # we don't want any dimensions - if given, convert
+                        # to the corresponding value
+                        idx = value.find('[')
+                        if idx >= 0:
+                            # find the end of the dimension
+                            dim = value.rfind(']')
+                            dimstr = value[idx+1:dim]
+                            # have to do this manually
+                            if "MAX_KEYLEN" in dimstr:
+                                value = value[:idx] + "[512]" + value[dim+1:]
+                            elif "MAX_NSLEN" in dimstr:
+                                value = value[:idx] + "[256]" + value[dim+1:]
+                            else:
+                                print("BAD DIMENSION " + dimstr)
+                                exit(1)
+                        newdef.append(value)
+                        n += 1
+                        value = lines[n].strip()
+                    # we need to extract the type name
+                    value = "typedef struct " + value[2:]
+                    value = value[:-1] + ":"
+                    newdef.insert(0, value)
+                    typedefs.append(newdef)
+    # only write the data sources once per file
+    defsrc = False
+    constsrc = False
+
+    # start by pretty-printing the string constants
+    # prepended with an underscore to avoid conflicts
+    # with the Python version of the name
+    if takeconst and len(strconsts) > 0:
+        if not defsrc:
+            definitions.write("cdef extern from \"" + src + "\":\n")
+            defsrc = True
+        if not constsrc:
+            constants.write("# " + src + "\n")
+            constsrc = True
+        definitions.write("\n    # STRING CONSTANTS\n")
+        for const in strconsts:
+            defname = "_" + const
+            definitions.write("    cdef const char* " + defname)
+            for i in range (4 + strconstlen - len(const)):
+                definitions.write(" ")
+            definitions.write("\"" + const + "\"\n")
+            # now output it into the constants file
+            constants.write(const)
+            for i in range (4 + strconstlen - len(const)):
+                constants.write(" ")
+            constants.write("= " + defname + "\n")
+        # add some space
+        definitions.write("\n")
+        constants.write("\n")
+
+    if takeconst and len(errconsts) > 0:
+        if not defsrc:
+            definitions.write("cdef extern from \"" + src + "\":\n")
+            defsrc = True
+        if not constsrc:
+            constants.write("# " + src + "\n")
+            constsrc = True
+        definitions.write("\n    # ERROR CONSTANTS\n")
+        for const in errconsts:
+            defname = "_" + const
+            definitions.write("    cdef int " + defname)
+            for i in range (4 + strconstlen - len(const)):
+                definitions.write(" ")
+            definitions.write("\"" + const + "\"\n")
+            # now output it into the constants file
+            constants.write(const)
+            for i in range (4 + strconstlen - len(const)):
+                constants.write(" ")
+            constants.write("= " + defname + "\n")
+        # add some space
+        definitions.write("\n")
+        constants.write("\n")
+
+    if takeconst and len(nconsts) > 0:
+        if not constsrc:
+            constants.write("# " + src + "\n")
+            constsrc = True
+        # pretty-print the numeric constants
+        for num in nconsts:
+            constants.write(num[0])
+            for i in range (4 + strconstlen - len(num[0])):
+                constants.write(" ")
+            constants.write("= " + num[1] + "\n")
+        # add some space
+        constants.write("\n")
+
+    # pretty-print any typedefs
+    if takedtypes and len(typedefs) > 0:
+        if not defsrc:
+            definitions.write("cdef extern from \"" + src + "\":\n")
+            defsrc = True
+        definitions.write("\n    # TYPEDEFS\n")
+        for t in typedefs:
+            definitions.write("    c" + t[0] + "\n")
+            if len(t) > 1:
+                # find the 2nd opening paren
+                idx = t[0].find("(") + 1
+                idx = t[0].find("(", idx) + 2
+                for n in range(1, len(t)):
+                    definitions.write("    ")
+                    for m in range(idx):
+                        definitions.write(" ")
+                    definitions.write(t[n] + "\n")
+            definitions.write("\n")
+        # add some space
+        definitions.write("\n")
+
+    # pretty-print any APIs
+    if takeapis and len(apis) > 0:
+        if not defsrc:
+            definitions.write("cdef extern from \"" + src + "\":\n")
+            defsrc = True
+        definitions.write("\n    # APIS\n")
+        for api in apis:
+            definitions.write("    " + api[0] + "\n")
+            if len(api) > 1:
+                # find the opening paren
+                idx = api[0].find("(") + 1
+                for n in range(1, len(api)):
+                    definitions.write("    ")
+                    for m in range(idx):
+                        definitions.write(" ")
+                    definitions.write(api[n] + "\n")
+            definitions.write("\n")
+    return
+
+def main():
+    global takeconst, takeapis, takedtypes
+    signal.signal(signal.SIGINT, signal_handler)
+
+    parser = OptionParser("usage: %prog [options]")
+    debugGroup = OptionGroup(parser, "Debug Options")
+    debugGroup.add_option("--debug",
+                          action="store_true", dest="debug", default=False,
+                          help="Output lots of debug messages while processing")
+    debugGroup.add_option("--dryrun",
+                          action="store_true", dest="dryrun", default=False,
+                          help="Show commands, but do not execute them")
+    parser.add_option_group(debugGroup)
+
+    execGroup = OptionGroup(parser, "Execution Options")
+    execGroup.add_option("--src", dest="src",
+                         help="The directory where the PMIx header files will be found")
+    execGroup.add_option("--constants",
+                         action="store_true", dest="constants", default=False,
+                         help="Translate constants")
+    execGroup.add_option("--apis",
+                         action="store_true", dest="apis", default=False,
+                         help="Translate APIs")
+    execGroup.add_option("--datatypes",
+                         action="store_true", dest="datatypes", default=False,
+                         help="Translate datatypes")
+    parser.add_option_group(execGroup)
+
+    (options, args) = parser.parse_args()
+
+    if not options.constants and not options.apis and not options.datatypes:
+        takeconst = True
+        takeapis = True
+        takedtypes = True
+
+    if options.constants:
+        takeconst = True
+    if options.apis:
+        takeapis = True
+    if options.datatypes:
+        takedtypes = True
+
+    if options.dryrun or options.debug:
+        debug = True
+    else:
+        debug = False
+
+    if options.src:
+        # see if the source directory exists
+        if not os.path.exists(options.src):
+            print("SOURCE directory",options.src,"does not exist")
+            sys.exit(1)
+
+    if options.dryrun:
+        constants = sys.stdout
+        definitions = sys.stdout
+    else:
+        # open the .pxd file for the definitions
+        # if the output file exists, remove it
+        if os.path.exists("pmix_constants.pxd"):
+            if debug:
+                print("Remove pmix_constants.pxd")
+            if not options.dryrun:
+                os.remove("pmix_constants.pxd")
+        elif debug:
+            print("File pmix_constants.pxd not found")
+        definitions = open("pmix_constants.pxd", "w+")
+        # add the necessary imports
+        definitions.write("from posix.types cimport *\n")
+        definitions.write("from posix.time cimport *\n")
+        definitions.write("from libc.stdint cimport *\n\n")
+
+        # open the .pxi file for the Python-level constants
+        if os.path.exists("pmix_constants.pxi"):
+            if debug:
+                print("Remove pmix_constants.pxi")
+            if not options.dryrun:
+                os.remove("pmix_constants.pxi")
+        elif debug:
+            print("File pmix_constants.pxi not found")
+        constants = open("pmix_constants.pxi", "w+")
+        # add the necessary import and provide a little space for neatness
+        constants.write("from pmix_constants cimport *\n\n")
+
+    # scan across the header files in the src directory
+    # looking for constants and typedefs
+    # add some space
+    harvest_constants(options, "pmix_common.h", constants, definitions)
+    definitions.write("\n\n")
+    constants.write("\n\n")
+    harvest_constants(options, "pmix.h", constants, definitions)
+    # add some space
+    definitions.write("\n\n")
+    constants.write("\n\n")
+    harvest_constants(options, "pmix_server.h", constants, definitions)
+    # add some space
+    definitions.write("\n\n")
+    constants.write("\n\n")
+    harvest_constants(options, "pmix_tool.h", constants, definitions)
+    # close the files to ensure all output is written
+    constants.close()
+    definitions.close()
+    
+if __name__ == '__main__':
+    main()
+
diff -Nru pmix-3.2.2~rc1/bindings/python/Makefile.am pmix-4.0.0/bindings/python/Makefile.am
--- pmix-3.2.2~rc1/bindings/python/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2010 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+# Copyright (c) 2018      Research Organization for Information Science
+#                         and Technology (RIST).  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+helpers = setup.py pmix.pyx pmix.pxi construct.py
+
+EXTRA_DIST = $(helpers)
+
+if WANT_PYTHON_BINDINGS
+
+install-exec-local: $(helpers)
+	$(PYTHON) $(top_srcdir)/bindings/python/construct.py --src="$(top_builddir)/include"
+	$(PYTHON) $(top_srcdir)/bindings/python/setup.py build_ext --include-dirs="$(top_builddir)/include" --library-dirs="$(DESTDIR)$(libdir)" --user
+	$(PYTHON) $(top_srcdir)/bindings/python/setup.py install --prefix="$(DESTDIR)$(prefix)"
+
+uninstall-hook:
+	rm -f $(pythondir)/pmix*.so
+	rm -f $(pythondir)/pypmix-*.egg-info
+
+CLEANFILES += pmix.c
+
+clean-local:
+	rm -rf build
+
+endif
diff -Nru pmix-3.2.2~rc1/bindings/python/pmix.pxi pmix-4.0.0/bindings/python/pmix.pxi
--- pmix-3.2.2~rc1/bindings/python/pmix.pxi	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/pmix.pxi	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1692 @@
+from libc.string cimport memset, strncpy, strcpy, strlen, strdup
+from libc.stdlib cimport malloc, realloc, free
+from libc.string cimport memcpy
+from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
+from cpython.pycapsule cimport PyCapsule_New, PyCapsule_GetPointer
+
+# pull in all the constant definitions - we
+# store them in a separate file for neatness
+include "pmix_constants.pxi"
+
+# provide a lock class for catching information
+# returned in callback functions
+class myLock(threading.Event):
+    def __init__(self):
+        threading.Event.__init__(self)
+        self.event = threading.Event()
+        self.status = PMIX_ERR_NOT_SUPPORTED
+        self.sz = 0
+        self.info = []
+
+    def set(self, status):
+        self.status = status
+        self.event.set()
+
+    def clear(self):
+        self.event.clear()
+
+    def wait(self):
+        self.event.wait()
+
+    def get_status(self):
+        return self.status
+
+    def cache_data(self, data, sz):
+        self.data = array.array('B', data[0])
+        # need to copy the data bytes as the
+        # PMIx server will free it upon return
+        n = 1
+        while n < sz:
+            self.data.append(data[n])
+            n += 1
+        self.sz = sz
+
+    def fetch_data(self):
+        return (self.data, self.sz)
+
+    def cache_info(self, info:list):
+        # need to copy the info array as the
+        # PMIx server will free it upon execing
+        # the callback function
+        self.info = []
+        for x in info:
+            self.info.append(x)
+
+    def fetch_info(self, info:list):
+        for x in self.info:
+            info.append(x)
+
+ctypedef struct pmix_pyshift_t:
+    char *op
+    pmix_byte_object_t *payload
+    size_t idx
+    pmix_modex_cbfunc_t modex
+    pmix_status_t status
+    pmix_byte_object_t bo
+    pmix_byte_object_t *cred
+    pmix_iof_channel_t channel
+    pmix_nspace_t nspace
+    pmix_proc_t source
+    pmix_proc_t *proc
+    pmix_pdata_t *pdata
+    pmix_info_t *results
+    size_t nresults
+    pmix_info_t *info
+    const char *data
+    size_t ndata
+    pmix_op_cbfunc_t op_cbfunc
+    pmix_iof_cbfunc_t iof
+    pmix_info_cbfunc_t query
+    pmix_spawn_cbfunc_t spawn
+    pmix_lookup_cbfunc_t lookup
+    pmix_release_cbfunc_t release_fn
+    pmix_event_notification_cbfunc_fn_t event_handler
+    pmix_tool_connection_cbfunc_t toolconnected
+    pmix_credential_cbfunc_t getcredential
+    pmix_validation_cbfunc_t validationcredential
+    pmix_info_cbfunc_t allocate
+    void *notification_cbdata
+    void *cbdata
+
+cdef void iofhdlr_cache(capsule, ret):
+    cdef pmix_pyshift_t *shifter
+    shifter = <pmix_pyshift_t*>PyCapsule_GetPointer(capsule, "iofhdlr_cache")
+    pyiofhandler(shifter[0].idx, shifter[0].channel, &shifter[0].source,
+                 shifter[0].payload, shifter[0].info, shifter[0].ndata)
+    if 0 < shifter[0].ndata:
+        pmix_free_info(shifter[0].info, shifter[0].ndata)
+    return
+
+cdef void event_cache_cb(capsule, ret):
+    cdef pmix_pyshift_t *shifter
+    shifter = <pmix_pyshift_t*>PyCapsule_GetPointer(capsule, "event_handler")
+    pyeventhandler(shifter[0].idx, shifter[0].status, &shifter[0].source,
+                   shifter[0].info, shifter[0].ndata,
+                   shifter[0].results, shifter[0].nresults,
+                   shifter[0].event_handler, shifter[0].notification_cbdata)
+
+cdef void pmix_convert_locality(pmix_locality_t loc, pyloc:list):
+    if PMIX_LOCALITY_NONLOCAL & loc:
+        pyloc.append(PMIX_LOCALITY_NONLOCAL)
+    if PMIX_LOCALITY_SHARE_HWTHREAD & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_HWTHREAD)
+    if PMIX_LOCALITY_SHARE_CORE & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_CORE)
+    if PMIX_LOCALITY_SHARE_L1CACHE & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_L1CACHE)
+    if PMIX_LOCALITY_SHARE_L2CACHE & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_L2CACHE)
+    if PMIX_LOCALITY_SHARE_L3CACHE & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_L3CACHE)
+    if PMIX_LOCALITY_SHARE_PACKAGE & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_PACKAGE)
+    if PMIX_LOCALITY_SHARE_NUMA & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_NUMA)
+    if PMIX_LOCALITY_SHARE_NODE & loc:
+        pyloc.append(PMIX_LOCALITY_SHARE_NODE)
+    if 0 == pyloc.len():
+        pyloc.append(PMIX_LOCALITY_NONLOCAL)
+    return
+
+cdef void pmix_unload_argv(char **keys, argv:list):
+    n = 0
+    while NULL != keys[n]:
+        mykey = keys[n].decode('ascii')
+        argv.append(mykey)
+        n += 1
+
+cdef int pmix_load_argv(char **keys, argv:list):
+    n = 0
+    for a in argv:
+        pya = a
+        if isinstance(a, str):
+            print("LOADARGV ENCODING")
+            pya = a.encode('ascii')
+        keys[n] = strdup(pya)
+        print("LOADARGV ", n, pya)
+        n += 1
+    keys[n] = NULL
+    return PMIX_SUCCESS
+
+cdef int pmix_load_darray(pmix_data_array_t *array, mytype, mylist:list):
+    cdef pmix_info_t *infoptr;
+    mysize = len(mylist)
+    if PMIX_INFO == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_info_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        infoptr = <pmix_info_t*>array[0].array
+        rc = pmix_load_info(infoptr, mylist)
+    elif PMIX_BOOL == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int*))
+        n = 0
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        boolptr = <int*>array[0].array
+        for item in mylist:
+            int_bool = pmix_bool_convert(item)
+            if int_bool != 0 and int_bool != 1:
+                return PMIX_ERR_BAD_PARAM
+            boolptr[n] = int_bool
+            n += 1
+    elif PMIX_BYTE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(uint8_t*))
+        n = 0
+        # byte val is uint8 type
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        bptr = <uint8_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("uint8 value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            if item > 255:
+                print("uint8 value is out of bounds")
+                return PMIX_ERR_BAD_PARAM
+            bptr[n] = int(item)
+            n += 1
+    elif PMIX_STRING == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(char*))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        strptr = <char**> array[0].array
+        for item in mylist:
+            if isinstance(item, str):
+                pykey = item.encode('ascii')
+            else:
+                pykey = item
+            try:
+                strptr[n] = strdup(pykey)
+            except:
+                print("String value declared but non-string provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            n += 1
+    elif PMIX_SIZE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(size_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        sptr = <size_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("size_t value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            sptr[n] = int(item)
+            n += 1
+    elif PMIX_PID == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pid_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        pidptr = <pid_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("pid_t value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            pidptr[n] = int(item)
+            n += 1
+    elif PMIX_INT == mytype or PMIX_UINT == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        iptr = <int*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("int value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            iptr[n] = int(item)
+            n += 1
+    elif PMIX_INT8 == mytype or PMIX_UINT8 == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int8_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i8ptr = <int8_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("8-bit int value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            i8ptr[n] = int(item)
+            n += 1
+    elif PMIX_INT16 == mytype or PMIX_UINT16 == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int16_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i16ptr = <int16_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("16-bit int value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            i16ptr[n] = int(item)
+            n += 1
+    elif PMIX_INT32 == mytype or PMIX_UINT32 == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int32_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i32ptr = <int32_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("32-bit int value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            i32ptr[n] = int(item)
+            n += 1
+    elif PMIX_INT64 == mytype or PMIX_UINT64 == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int64_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i64ptr = <int64_t*> array[0].array
+        for item in mylist:
+            if not isinstance(item, pmix_int_types):
+                print("64-bit int value declared but non-integer provided")
+                return PMIX_ERR_TYPE_MISMATCH
+            i64ptr[n] = int(item)
+            n += 1
+    elif PMIX_FLOAT == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(float))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        fptr = <float*> array[0].array
+        for item in mylist:
+            fptr[n] = float(item)
+            n += 1
+    elif PMIX_DOUBLE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(double))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        dptr = <double*> array[0].array
+        for item in mylist:
+            dptr[n] = float(item)
+            n += 1
+            n += 1
+    elif PMIX_TIMEVAL == mytype:
+        # TODO: Not clear that "timeval" has the same size as
+        # "struct timeval"
+        array[0].array = PyMem_Malloc(mysize * sizeof(timeval))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        tvptr = <timeval*> array[0].array
+        for item in mylist:
+            tvptr[n].tv_sec  = item['sec']
+            tvptr[n].tv_usec = item['usec']
+            n += 1
+    elif PMIX_TIME == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(time_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        tmptr = <time_t*> array[0].array
+        for item in mylist:
+            tmptr[n] = item
+            n += 1
+    elif PMIX_STATUS == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(int))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        stptr = <int*> array[0].array
+        for item in mylist:
+            stptr[n] = item
+            n += 1
+    elif PMIX_PROC_RANK == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_rank_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        rkptr = <pmix_rank_t*> array[0].array
+        for item in mylist:
+            rkptr[n] = item
+            n += 1
+    elif PMIX_PROC == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_proc_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        prcptr = <pmix_proc_t*> array[0].array
+        for item in mylist:
+            pmix_copy_nspace(prcptr[n].nspace, item['nspace'])
+            prcptr[n].rank = item['rank']
+            n += 1
+    elif PMIX_BYTE_OBJECT == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_byte_object_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        boptr = <pmix_byte_object_t*>array[0].array
+        for item in mylist:
+            boptr[n].size = item['size']
+            boptr[n].bytes = <char*> PyMem_Malloc(boptr[n].size)
+            if not boptr[n].bytes:
+                return PMIX_ERR_NOMEM
+            pyarr = bytes(item['bytes'])
+            pyptr = <const char*> pyarr
+            memcpy(boptr[n].bytes, pyptr, boptr[n].size)
+            n += 1
+    elif PMIX_PERSISTENCE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_persistence_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        perptr = <pmix_persistence_t*> array[0].array
+        for item in mylist:
+            perptr[n] = item
+            n += 1
+    elif PMIX_SCOPE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_scope_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        scptr = <pmix_scope_t*> array[0].array
+        for item in mylist:
+            scptr[n] = item
+            n += 1
+    elif PMIX_RANGE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_data_range_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        rgptr = <pmix_data_range_t*> array[0].array
+        for item in mylist:
+            rgptr[n] = item
+            n += 1
+    elif PMIX_PROC_STATE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_proc_state_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        psptr = <pmix_proc_state_t*> array[0].array
+        for item in mylist:
+            psptr[n] = item
+            n += 1
+    elif PMIX_PROC_INFO == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_proc_info_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        piptr = <pmix_proc_info_t*> array[0].array
+        for item in mylist:
+            pmix_copy_nspace(piptr[n].proc.nspace, item['proc']['nspace'])
+            piptr[n].proc.rank = item['proc']['rank']
+            hostname = item['hostname']
+            if isinstance(hostname, str):
+                pyhostname = hostname.encode('ascii')
+            else:
+                pyhostname = hostname
+            pyhostnameptr = <const char *>(pyhostname)
+            piptr[n].hostname = strdup(pyhostnameptr)
+            executable = item['executable']
+            if isinstance(executable, str):
+                pyexec = executable.encode('ascii')
+            else:
+                pyexec = executable
+            pyexecptr = <const char *>(pyexec)
+            piptr[n].executable_name = strdup(pyexecptr)
+            piptr[n].pid = item['pid']
+            piptr[n].exit_code = item['exitcode']
+            piptr[n].state = item['state']
+            n += 1
+    elif PMIX_DATA_ARRAY == mytype:
+        array[0].array = <pmix_data_array_t*> PyMem_Malloc(sizeof(pmix_data_array_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        daptr = <pmix_data_array_t*>array[0].array
+        n = 0
+        for item in mylist:
+            daptr[n].type = item['val_type']
+            daptr[n].size = len(item['value'])
+            daptr[n].array = <pmix_data_array_t*> PyMem_Malloc(sizeof(pmix_data_array_t))
+            if not daptr[n].array:
+                return PMIX_ERR_NOMEM
+            mydaptr = <pmix_data_array_t*>daptr[n].array
+            try:
+                return pmix_load_darray(mydaptr, daptr[n].type, item['value'])
+            except:
+                return PMIX_ERR_NOT_SUPPORTED
+    elif PMIX_ALLOC_DIRECTIVE == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_alloc_directive_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        aldptr = <pmix_alloc_directive_t*> array[0].array
+        for item in mylist:
+            aldptr[n] = item
+            n += 1
+    elif PMIX_ENVAR == mytype:
+        array[0].array = PyMem_Malloc(mysize * sizeof(pmix_envar_t))
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        envptr = <pmix_envar_t*> array[0].array
+        for item in mylist:
+            enval = item['envar']
+            if isinstance(enval, str):
+                pyns = enval.encode('ascii')
+            else:
+                pyns = enval
+            pynsptr = <const char *>(pyns)
+            envptr[n].envar = strdup(pynsptr)
+            if isinstance(enval, str):
+                pyns = enval.encode('ascii')
+            else:
+                pyns = enval
+            pynsptr = <const char *>(pyns)
+            envptr[n].value = strdup(pynsptr)
+            pyseparator = <char>ord(item['separator'])
+            envptr[n].separator = pyseparator
+            n += 1
+    else:
+        print("UNRECOGNIZED DATA TYPE IN ARRAY")
+        return PMIX_ERR_NOT_SUPPORTED
+    return PMIX_SUCCESS
+
+cdef dict pmix_unload_darray(pmix_data_array_t *array):
+    cdef pmix_info_t *infoptr;
+    if PMIX_INFO == array.type:
+        ilist = []
+        n = 0
+        infoptr = <pmix_info_t*>array[0].array
+        rc = pmix_unload_info(infoptr, array.size, ilist)
+        darray = {'type':array.type, 'array':ilist}
+        pmix_free_info(infoptr, array.size)
+        return darray
+    elif PMIX_BOOL == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        boolptr = <int*>array[0].array
+        list = []
+        n = 0
+        while n < array.size:
+            list.append(boolptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_BYTE == array.type:
+        # byte val is uint8 type
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        bptr = <uint8_t*> array[0].array
+        list = []
+        n = 0
+        while n < array.size:
+            list.append(bptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_STRING == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        strptr = <char**> array[0].array
+        strlist = []
+        while n < array.size:
+            pyb = strptr[n]
+            pystr = pyb.decode("ascii")
+            strlist.append(pystr)
+            free(strptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':strlist}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_SIZE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        sptr = <size_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(sptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_PID == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        pidptr = <pid_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(pidptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_INT == array.type or PMIX_UINT == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        iptr = <int*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(iptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_INT8 == array.type or PMIX_UINT8 == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i8ptr = <int8_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(i8ptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_INT16 == array.type or PMIX_UINT16 == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i16ptr = <int16_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(i16ptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_INT32 == array.type or PMIX_UINT32 == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i32ptr = <int32_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(i32ptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_INT64 == array.type or PMIX_UINT64 == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        i64ptr = <int64_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(i64ptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_FLOAT == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        fptr = <float*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(fptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_DOUBLE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        dptr = <double*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(dptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_TIMEVAL == array.type:
+        # TODO: Not clear that "timeval" has the same size as
+        # "struct timeval"
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        tvptr = <timeval*> array[0].array
+        list = []
+        while n < array.size:
+            d = {'sec': tvptr[n].tv_sec, 'usec': tvptr[n].tv_usec}
+            list.append(d)
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_TIME == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        tmptr = <time_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(tmptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_STATUS == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        stptr = <int*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(stptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_PROC_RANK == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        rkptr = <pmix_rank_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(rkptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_PROC == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        prcptr = <pmix_proc_t*> array[0].array
+        list = []
+        while n < array.size:
+            d = {'nspace': prcptr[n].nspace, 'rank': prcptr[n].rank}
+            list.append(d)
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_BYTE_OBJECT == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        boptr = <pmix_byte_object_t*>array[0].array
+        list = []
+        while n < array.size:
+            if not boptr[n].bytes:
+                return PMIX_ERR_NOMEM
+            d = {'bytes': boptr[n].bytes, 'size': boptr[n].size}
+            list.append(d)
+            PyMem_Free(boptr[n].bytes)
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_PERSISTENCE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        perptr = <pmix_persistence_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(perptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_SCOPE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        scptr = <pmix_scope_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(scptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_RANGE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        rgptr = <pmix_data_range_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(rgptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_PROC_STATE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        psptr = <pmix_proc_state_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(psptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_PROC_INFO == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        piptr = <pmix_proc_info_t*> array[0].array
+        list = []
+        while n < array.size:
+            d = {'proc': {'nspace':piptr[n].proc.nspace,
+            'rank':piptr[n].proc.rank}, 'hostname':piptr[n].hostname,
+            'executable':piptr[n].executable_name, 'pid':piptr[n].pid,
+            'exitcode':piptr[n].exit_code, 'state':piptr[n].state}
+            list.append(d)
+            free(piptr[n].hostname)
+            free(piptr[n].executable_name)
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_DATA_ARRAY == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        daptr = <pmix_data_array_t*>array[0].array
+        n = 0
+        while n < array.size:
+            if not daptr[n].array:
+                return PMIX_ERR_NOMEM
+            mydaptr = <pmix_data_array_t*>daptr[n].array
+            try:
+                rc = pmix_unload_darray(mydaptr)
+                if rc != PMIX_SUCCESS:
+                    return rc
+            except:
+                return PMIX_ERR_NOT_SUPPORTED
+            n += 1
+    elif PMIX_ALLOC_DIRECTIVE == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        aldptr = <pmix_alloc_directive_t*> array[0].array
+        list = []
+        while n < array.size:
+            list.append(aldptr[n])
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    elif PMIX_ENVAR == array.type:
+        if not array[0].array:
+            return PMIX_ERR_NOMEM
+        n = 0
+        envptr = <pmix_envar_t*> array[0].array
+        list = []
+        while n < array.size:
+            pyenv = envptr[n].envar
+            pyval = envptr[n].value
+            pysep = envptr[n].separator
+            d = {'envar':pyenv, 'value':pyval, 'separator':pysep}
+            list.append(d)
+            free(envptr[n].value)
+            free(envptr[n].envar)
+            n += 1
+        darray = {'type':array.type, 'array':list}
+        PyMem_Free(array[0].array)
+        return darray
+    else:
+        print("UNRECOGNIZED DATA TYPE IN ARRAY")
+        return PMIX_ERR_NOT_SUPPORTED
+    return PMIX_SUCCESS
+
+# provide conversion programs that translate incoming
+# PMIx structures into Python dictionaries, and incoming
+# arrays into Python lists of objects
+
+def pmix_bool_convert(f):
+    int_bool = PMIX_ERR_BAD_PARAM
+    if isinstance(f, str):
+        if f.startswith('t') or f.startswith('T'):
+            int_bool = 1
+        elif f.startswith('f') or f.startswith('F'):
+            int_bool = 0
+        else:
+            print("Incorrect boolean value provided")
+            int_bool = PMIX_ERR_BAD_PARAM
+    return int_bool
+
+pmix_int_types = (int, long)
+
+# provide a safe way to copy a Python nspace into
+# the pmix_nspace_t structure that guarantees the
+# array is NULL-terminated
+cdef void pmix_copy_nspace(pmix_nspace_t nspace, ns):
+    nslen = len(ns)
+    if PMIX_MAX_NSLEN < nslen:
+        nslen = PMIX_MAX_NSLEN
+    if isinstance(ns, str):
+        pyns = ns.encode('ascii')
+    else:
+        pyns = ns
+    pynsptr = <const char *>(pyns)
+    memset(nspace, 0, PMIX_MAX_NSLEN+1)
+    memcpy(nspace, pynsptr, nslen)
+
+# provide a safe way to copy a Python key into
+# the pmix_key_t structure that guarantees the
+# array is NULL-terminated
+cdef void pmix_copy_key(pmix_key_t key, ky):
+    klen = len(ky)
+    if PMIX_MAX_KEYLEN < klen:
+        klen = PMIX_MAX_KEYLEN
+    if isinstance(ky, str):
+        pykey = ky.encode('ascii')
+    else:
+        pykey = ky
+    pykeyptr = <const char *>(pykey)
+    memset(key, 0, PMIX_MAX_KEYLEN+1)
+    if 'b' == ky[0]:
+        memcpy(key, &pykeyptr[2], klen-3)
+    else:
+        memcpy(key, pykeyptr, klen)
+
+# loads a python pmix regex into a python bytearray
+cdef bytearray pmix_convert_regex(char *regex):
+    if "pmix" == regex[:4].decode("ascii"):
+        # remove null characters
+        if b'\x00' in regex:
+            regex.replace(b'\x00', '')
+        ba = bytearray(regex)
+    elif "blob" == regex[:4].decode("ascii"):
+        sz_str    = len(regex)
+        sz_prefix = 5
+        # extract length of bytearray
+        regex.split(b'\x00')
+        len_bytearray = regex[1]
+        length = len(len_bytearray) + sz_prefix + sz_str
+        ba = bytearray(length)
+        pyregex = <bytes> regex[:length]
+        index = 0
+        while index < length:
+            ba[index] = pyregex[index]
+            index += 1
+    else:
+        # last case with no ':' in string
+        ba = bytearray(regex)
+    return ba
+
+# provide a function for transferring a Python 'value'
+# object (a dict with value and val_type as keys)
+# to a pmix_value_t
+cdef int pmix_load_value(pmix_value_t *value, val:dict):
+    if not isinstance(val['val_type'], pmix_int_types):
+        return PMIX_ERR_BAD_PARAM
+    value[0].type = val['val_type']
+    if val['val_type'] == PMIX_BOOL:
+        int_bool = pmix_bool_convert(val['value'])
+        if int_bool != 0 and int_bool != 1:
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.flag = int_bool
+    elif val['val_type'] == PMIX_BYTE:
+        # byte val is uint8 type
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.byte = val['value']
+    elif val['val_type'] == PMIX_STRING:
+        if isinstance(val['value'], str):
+            pykey = val['value'].encode('ascii')
+        else:
+            pykey = val['value']
+        try:
+            value[0].data.string = strdup(pykey)
+        except:
+            print("String value declared but non-string provided")
+            return PMIX_ERR_TYPE_MISMATCH
+    elif val['val_type'] == PMIX_SIZE:
+        if not isinstance(val['value'], pmix_int_types):
+            print("size_t value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.size = val['value']
+    elif val['val_type'] == PMIX_PID:
+        if not isinstance(val['value'], pmix_int_types):
+            print("pid value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] < 0:
+            print("pid value is negative")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.pid = val['value']
+    elif val['val_type'] == PMIX_INT:
+        if not isinstance(val['value'], pmix_int_types):
+            print("integer value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.integer = val['value']
+    elif val['val_type'] == PMIX_INT8:
+        if not isinstance(val['value'], pmix_int_types):
+            print("int8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 127 or val['value'] < -128:
+            print("int8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.int8 = val['value']
+    elif val['val_type'] == PMIX_INT16:
+        if not isinstance(val['value'], pmix_int_types):
+            print("int16 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 32767 or val['value'] < -32768:
+            print("int16 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.int16 = val['value']
+    elif val['val_type'] == PMIX_INT32:
+        if not isinstance(val['value'], pmix_int_types):
+            print("int32 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 2147483647 or val['value'] < -2147483648:
+            print("int32 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.int32 = val['value']
+    elif val['val_type'] == PMIX_INT64:
+        if not isinstance(val['value'], pmix_int_types):
+            print("int64 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > (2147483647*2147483647) or val['value'] < -(2147483648*2147483648):
+            print("int64 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.int64 = val['value']
+    elif val['val_type'] == PMIX_UINT:
+        if not isinstance(val['value'], pmix_int_types):
+            print("integer value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] < 0:
+            print("uint value out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.uint = val['value']
+    elif val['val_type'] == PMIX_UINT8:
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255 or val['value'] < 0:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.uint8 = val['value']
+    elif val['val_type'] == PMIX_UINT16:
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint16 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 65536 or val['value'] < 0:
+            print("uint16 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.uint16 = val['value']
+    elif val['val_type'] == PMIX_UINT32:
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint32 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > (65536*65536) or val['value'] < 0:
+            print("uint32 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.uint32 = val['value']
+    elif val['val_type'] == PMIX_UINT64:
+        if not isinstance(val['value'], pmix_int_types):
+            print("int64 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > (2147483648*2147483648):
+            print("uint64 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.uint64 = val['value']
+    elif val['val_type'] == PMIX_FLOAT:
+        float_val = float(val['value'])
+        if not isinstance(float_val, float):
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.fval = float_val
+    elif val['val_type'] == PMIX_DOUBLE:
+        double_val = float(val['value'])
+        if not isinstance(double_val, float):
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.dval = double_val
+    # TODO: need a way to verify usable timevals passed in?
+    elif val['val_type'] == PMIX_TIMEVAL:
+        value[0].data.tv.tv_sec  = val['value']['sec']
+        value[0].data.tv.tv_usec = val['value']['usec']
+    elif val['val_type'] == PMIX_TIME:
+        value[0].data.time = val['val_type']
+    elif val['val_type'] == PMIX_STATUS:
+        if not isinstance(val['value'], int):
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.status = val['value']
+    elif val['val_type'] == PMIX_PROC_RANK:
+        if not isinstance(val['value'], int):
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > (65536*65536) or val['value'] < 0:
+            print("uint32 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.rank = val['value']
+    elif val['val_type'] == PMIX_PROC:
+        value[0].data.proc = <pmix_proc_t*> PyMem_Malloc(sizeof(pmix_proc_t))
+        if not value[0].data.proc:
+            return PMIX_ERR_NOMEM
+        # TODO: check nspace val is a char here
+        pmix_copy_nspace(value[0].data.proc[0].nspace, val['value']['nspace'])
+        # pmix_rank_t is defined as uint32
+        if not isinstance(val['value']['rank'], pmix_int_types):
+            print("uint32 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value']['rank'] > (65536*65536):
+            print("uint32 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.proc[0].rank = val['value']['rank']
+    # TODO: pmix byte object conversion isn't working
+    elif val['val_type'] == PMIX_BYTE_OBJECT:
+        value[0].data.bo.size = val['value']['size']
+        value[0].data.bo.bytes = <char*> PyMem_Malloc(value[0].data.bo.size)
+        if not value[0].data.bo.bytes:
+            return PMIX_ERR_NOMEM
+        pyptr = <char*>val['value']['bytes']
+        memcpy(value[0].data.bo.bytes, pyptr, value[0].data.bo.size)
+    elif val['val_type'] == PMIX_PERSISTENCE:
+        # pmix_persistence_t is defined as uint8
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255 or val['value'] < 0:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.persist = val['value']
+    elif val['val_type'] == PMIX_SCOPE:
+        # pmix_scope_t is defined as uint8
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.scope = val['value']
+    elif val['val_type'] == PMIX_RANGE:
+        # pmix_data_range_t is defined as uint8
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.range = val['value']
+    elif val['val_type'] == PMIX_PROC_STATE:
+        # pmix_proc_state_t is defined as uint8
+        if not isinstance(val['value'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.state = val['value']
+    elif val['val_type'] == PMIX_PROC_INFO:
+        value[0].data.pinfo = <pmix_proc_info_t*> PyMem_Malloc(sizeof(pmix_proc_info_t))
+        if not value[0].data.pinfo:
+            return PMIX_ERR_NOMEM
+        # TODO: verify nspace is copied correctly
+        pmix_copy_nspace(value[0].data.pinfo[0].proc.nspace, val['value']['proc']['nspace'])
+        if not isinstance(val['value']['proc']['rank'], pmix_int_types):
+            print("uint32 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value']['proc']['rank'] > (65536*65536):
+            print("uint32 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.pinfo[0].proc.rank = val['value']['proc']['rank']
+        hostname = val['value']['hostname']
+        if isinstance(hostname, str):
+            pyhostname = hostname.encode('ascii')
+        else:
+            pyhostname = hostname
+        pyhostnameptr = <const char *>(pyhostname)
+        value[0].data.pinfo[0].hostname = strdup(pyhostnameptr)
+        executable = val['value']['executable']
+        if isinstance(executable, str):
+            pyexec = executable.encode('ascii')
+        else:
+            pyexec = executable
+        pyexecptr = <const char *>(pyexec)
+        value[0].data.pinfo[0].executable_name = strdup(pyexecptr)
+        # TODO: check this is a pid type
+        value[0].data.pinfo[0].pid = val['value']['pid']
+        if not isinstance(val['value']['exitcode'], int):
+            print("value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.pinfo[0].exit_code = val['value']['exitcode']
+        if not isinstance(val['value']['state'], pmix_int_types):
+            print("uint8 value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value']['state'] > 255:
+            print("uint8 value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.pinfo[0].state = val['value']['state']
+    elif val['val_type'] == PMIX_DATA_ARRAY:
+        value[0].data.darray = <pmix_data_array_t*> PyMem_Malloc(sizeof(pmix_data_array_t))
+        if not value[0].data.darray:
+            return PMIX_ERR_NOMEM
+        value[0].data.darray[0].type = val['value']['type']
+        value[0].data.darray[0].size = len(val['value']['array'])
+        try:
+            # assume pmix_load_darray does own type checks
+            # it should return with an error code inside that
+            # function if there is one
+            pmix_load_darray(value[0].data.darray, value[0].data.darray[0].type, val['value']['array'])
+        except:
+            return PMIX_ERR_NOT_SUPPORTED
+    elif val['val_type'] == PMIX_ALLOC_DIRECTIVE:
+        if not isinstance(val['value'], pmix_int_types):
+            print("allocdirective value declared but non-integer provided")
+            return PMIX_ERR_TYPE_MISMATCH
+        if val['value'] > 255:
+            print("allocdirective value is out of bounds")
+            return PMIX_ERR_BAD_PARAM
+        value[0].data.adir = val['value']
+    elif val['val_type'] == PMIX_ENVAR:
+        enval = val['value']['envar']
+        if isinstance(enval, str):
+            pyns = enval.encode('ascii')
+        else:
+            pyns = enval
+        pynsptr = <const char *>(pyns)
+        value[0].data.envar.envar = strdup(pynsptr)
+        enval = val['value']['value']
+        if isinstance(enval, str):
+            pyns = enval.encode('ascii')
+        else:
+            pyns = enval
+        pynsptr = <const char *>(pyns)
+        value[0].data.envar.value = strdup(pynsptr)
+        # TODO: way/function to verify char type
+        pyseparator = <char>ord(val['value']['separator'])
+        value[0].data.envar.separator = pyseparator
+    elif val['val_type'] == PMIX_REGEX:
+        regex = val['value']
+        ba = pmix_convert_regex(regex)
+        if not isinstance(ba, bytearray):
+            return PMIX_ERR_TYPE_MISMATCH
+        value[0].data.bo.size  = len(val['value'])
+        value[0].data.bo.bytes = <char*> PyMem_Malloc(value[0].data.bo.size)
+        if not value[0].data.bo.bytes:
+            return PMIX_ERR_NOMEM
+        pyptr = <char*>ba
+        memcpy(value[0].data.bo.bytes, pyptr, value[0].data.bo.size)
+    else:
+        print("UNRECOGNIZED VALUE TYPE")
+        return PMIX_ERR_NOT_SUPPORTED
+    return PMIX_SUCCESS
+
+cdef dict pmix_unload_value(const pmix_value_t *value):
+    if PMIX_BOOL == value[0].type:
+        if value[0].data.flag:
+            return {'value':True, 'val_type':PMIX_BOOL}
+        else:
+            return {'value':False, 'val_type':PMIX_BOOL}
+    elif PMIX_BYTE == value[0].type:
+        return {'value':value[0].data.byte, 'val_type':PMIX_BYTE}
+    elif PMIX_STRING == value[0].type:
+        pyb = value[0].data.string
+        pystr = pyb.decode("ascii")
+        return {'value':pystr, 'val_type':PMIX_STRING}
+    elif PMIX_SIZE == value[0].type:
+        return {'value':value[0].data.size, 'val_type':PMIX_SIZE}
+    elif PMIX_PID == value[0].type:
+        return {'value':value[0].data.pid, 'val_type':PMIX_PID}
+    elif PMIX_INT == value[0].type:
+        return {'value':value[0].data.integer, 'val_type':PMIX_INT}
+    elif PMIX_INT8 == value[0].type:
+        return {'value':value[0].data.int8, 'val_type':PMIX_INT8}
+    elif PMIX_INT16 == value[0].type:
+        return {'value':value[0].data.int16, 'val_type':PMIX_INT16}
+    elif PMIX_INT32 == value[0].type:
+        return {'value':value[0].data.int32, 'val_type':PMIX_INT32}
+    elif PMIX_INT64 == value[0].type:
+        return {'value':value[0].data.int64, 'val_type':PMIX_INT64}
+    elif PMIX_UINT == value[0].type:
+        return {'value':value[0].data.uint, 'val_type':PMIX_UINT}
+    elif PMIX_UINT8 == value[0].type:
+        return {'value':value[0].data.uint8, 'val_type':PMIX_UINT8}
+    elif PMIX_UINT16 == value[0].type:
+        return {'value':value[0].data.uint16, 'val_type':PMIX_UINT16}
+    elif PMIX_UINT32 == value[0].type:
+        return {'value':value[0].data.uint32, 'val_type':PMIX_UINT32}
+    elif PMIX_UINT64 == value[0].type:
+        return {'value':value[0].data.uint64, 'val_type':PMIX_UINT64}
+    elif PMIX_FLOAT == value[0].type:
+        return {'value':value[0].data.fval, 'val_type':PMIX_FLOAT}
+    elif PMIX_DOUBLE == value[0].type:
+        return {'value':value[0].data.dval, 'val_type':PMIX_DOUBLE}
+    elif PMIX_TIMEVAL == value[0].type:
+        return {'value':{'sec':value[0].data.tv.tv_sec, 'usec':value[0].data.tv.tv_usec},
+        'val_type':PMIX_TIMEVAL}
+    elif PMIX_TIME == value[0].type:
+        return {'value':value[0].data.time, 'val_type':PMIX_TIME}
+    elif PMIX_STATUS == value[0].type:
+        return {'value':value[0].data.status, 'val_type':PMIX_STATUS}
+    elif PMIX_PROC_RANK == value[0].type:
+        return {'value':value[0].data.rank, 'val_type':PMIX_PROC_RANK}
+    elif PMIX_PROC == value[0].type:
+        pyns = str(value[0].data.proc[0].nspace)
+        return {'value':{'nspace':pyns, 'rank':value[0].data.proc[0].rank}, 'val_type':PMIX_PROC}
+    elif PMIX_BYTE_OBJECT == value[0].type:
+        mybytes = <char*> PyMem_Malloc(value[0].data.bo.size)
+        if not mybytes:
+            return PMIX_ERR_NOMEM
+        memcpy(mybytes, value[0].data.bo.bytes, value[0].data.bo.size)
+        return {'value':{'bytes':mybytes, 'size':value[0].data.bo.size}, 'val_type':PMIX_BYTE_OBJECT}
+    elif PMIX_PERSISTENCE == value[0].type:
+        return {'value':value[0].data.persist, 'val_type':PMIX_PERSISTENCE}
+    elif PMIX_SCOPE == value[0].type:
+        return {'value':value[0].data.scope, 'val_type':PMIX_SCOPE}
+    elif PMIX_RANGE == value[0].type:
+        return {'value':value[0].data.range, 'val_type':PMIX_RANGE}
+    elif PMIX_PROC_STATE == value[0].type:
+        return {'value':value[0].data.state, 'val_type':PMIX_PROC_STATE}
+    elif PMIX_PROC_INFO == value[0].type:
+        pins = str(value[0].data.pinfo[0].proc.nspace)
+        pirk = value[0].data.pinfo[0].proc.rank
+        pihost = str(value[0].data.pinfo[0].hostname)
+        pexec = str(value[0].data.pinfo[0].executable_name)
+        pipid = value[0].data.pinfo[0].pid
+        piex = value[0].data.pinfo[0].exit_code
+        pist = value[0].data.pinfo[0].state
+        pians = {'proc': {'nspace':pins, 'rank':pirk}, 'hostname': pihost, 'executable': pexec, 'pid': pipid, 'exitcode': piex, 'state': pist}
+        return {'value':pians, 'val_type':PMIX_PROC_INFO}
+    elif PMIX_DATA_ARRAY == value[0].type:
+        try:
+            # assume pmix_unload_darray does own type checks
+            # it should return with an error code inside that
+            # function if there is one
+            darray = pmix_unload_darray(value[0].data.darray)
+            return {'value':darray, 'val_type':PMIX_DATA_ARRAY}
+        except:
+            return PMIX_ERR_NOT_SUPPORTED
+    elif PMIX_ALLOC_DIRECTIVE == value[0].type:
+        return {'value':value[0].data.adir, 'val_type':PMIX_ALLOC_DIRECTIVE}
+    elif PMIX_ENVAR == value[0].type:
+        pyenv = str(value[0].data.envar.envar)
+        pyval = str(value[0].data.envar.value)
+        pysep = value[0].data.envar.separator
+        pyenvans = {'envar': pyenv, 'value': pyval, 'separator': pysep}
+        return {'value':pyenvans, 'val_type':PMIX_ENVAR}
+    elif PMIX_REGEX == value[0].type:
+        return {'value': value[0].data.bo.bytes, 'val_type': PMIX_REGEX}
+    else:
+        print("Unload_value: provided type is unknown", value[0].type)
+        return {'value': None, 'val_type': PMIX_UNDEF}
+
+cdef void pmix_destruct_value(pmix_value_t *value):
+    if value[0].type == PMIX_STRING:
+        free(value[0].data.string);
+
+cdef void pmix_free_value(self, pmix_value_t *value):
+    pmix_destruct_value(value);
+    PyMem_Free(value)
+
+# Convert a dictionary of key-value pairs into an
+# array of pmix_info_t structs
+#
+# @array [INPUT]
+#        - malloc'd array of pmix_info_t structs
+#
+# @dicts [INPUT]
+#          - a list of dictionaries, where each
+#            dictionary has a key, value, and val_type
+#            defined as such:
+#            [{key:y, value:val, val_type:ty}, … ]
+#
+cdef int pmix_load_info(pmix_info_t *array, dicts:list):
+    n = 0
+    for d in dicts:
+        pykey = str(d['key'])
+        pmix_copy_key(array[n].key, pykey)
+        try:
+            array[n].flags = d['flags']
+        except:
+            pass
+        val = {'value':d['value'], 'val_type':d['val_type']}
+        rc = pmix_load_value(&array[n].value, val)
+        if PMIX_SUCCESS != rc:
+            return rc
+        n += 1
+    return PMIX_SUCCESS
+
+# Allocate memory and load pmix info structs
+#
+# @array [INPUT]
+#          - array of pmix_info_t structs
+#
+# @ninfo [INPUT]
+#          - length of the list of dictionaries
+#
+# @dicts [INPUT]
+#          - a list of dictionaries, where each
+#            dictionary has a key, value, and val_type
+#            defined as such:
+#            [{key:y, value:val, val_type:ty}, … ]
+#
+cdef int pmix_alloc_info(pmix_info_t **info_ptr, size_t *ninfo, dicts:list):
+    # Convert any provided dictionary to an array of pmix_info_t
+    if dicts is not None:
+        ninfo[0] = len(dicts)
+        if 0 < ninfo[0]:
+            info_ptr[0] = <pmix_info_t*> PyMem_Malloc(ninfo[0] * sizeof(pmix_info_t))
+            if not info_ptr[0]:
+                return PMIX_ERR_NOMEM
+            rc = pmix_load_info(info_ptr[0], dicts)
+            if PMIX_SUCCESS != rc:
+                pmix_free_info(info_ptr[0], ninfo[0])
+                return rc
+        else:
+            info_ptr[0] = NULL
+            ninfo[0] = 0
+    else:
+        info_ptr[0] = NULL
+        ninfo[0] = 0
+    return PMIX_SUCCESS
+
+cdef int pmix_unload_info(const pmix_info_t *info, size_t ninfo, ilist:list):
+    cdef char* kystr
+    cdef size_t n
+    n = 0
+    while n < ninfo:
+        print("UNLOADING INFO TYPE ", PMIx_Data_type_string(info[n].value.type))
+        # pmix_unload_value returns a python dict of val, val_type
+        val = pmix_unload_value(&info[n].value)
+        if val['val_type'] == PMIX_UNDEF:
+            return PMIX_ERR_NOT_SUPPORTED
+        d     = {}
+        kystr = strdup(info[n].key)
+        pykey = str(kystr.decode("ascii"))
+        free(kystr)
+        d['key']      = pykey
+        # TODO: don't know how to decide if flags was defined or not??
+        d['flags']    = info[n].flags
+        d['value']    = val['value']
+        d['val_type'] = val['val_type']
+        ilist.append(d)
+        n += 1
+    return PMIX_SUCCESS
+
+cdef void pmix_destruct_info(pmix_info_t *info):
+    pmix_destruct_value(&info[0].value)
+
+# Free a malloc'd array of pmix_info_t structures
+#
+# @array [INPUT]
+#        - array of pmix_info_t to be free'd
+#
+# @sz [INPUT]
+#     - number of elements in array
+cdef void pmix_free_info(pmix_info_t *array, size_t sz):
+    n = 0
+    while n < sz:
+        pmix_destruct_info(&array[n])
+        n += 1
+    PyMem_Free(array)
+
+# Convert a dictionary of key-value pairs into an
+# array of pmix_pdata_t structs
+#
+# @array [INPUT]
+#        - malloc'd array of pmix_pdata_t structs
+#
+# @pdata [INPUT]
+#          - a list of dictionaries, where each
+#            dictionary has a key, value, val_type,
+#            and proc keys
+# @proc [INPUT]
+#          - a dictionary with nspace, rank as keys
+cdef int pmix_load_pdata(proc:dict, pmix_pdata_t *array, data:list):
+    n = 0
+    for d in data:
+        pykey = str(d['key'])
+        pmix_copy_key(array[n].key, pykey)
+        val = {'value':d['value'], 'val_type':d['val_type']}
+        rc = pmix_load_value(&array[n].value, val)
+        array[n].proc.rank = proc['rank']
+        pmix_copy_nspace(array[n].proc.nspace, proc['nspace'])
+        if PMIX_SUCCESS != rc:
+            return rc
+        n += 1
+    return PMIX_SUCCESS
+
+cdef int pmix_unload_pdata(const pmix_pdata_t *pdata, size_t npdata, ilist:list):
+    cdef char* kystr
+    cdef size_t n = 0
+    while n < npdata:
+        print("UNLOADING INFO ", pdata[n].key, " TYPE ",
+                PMIx_Data_type_string(pdata[n].value.type))
+        val = pmix_unload_value(&pdata[n].value)
+        if val['val_type'] == PMIX_UNDEF:
+            return PMIX_ERR_NOT_SUPPORTED
+        d     = {}
+        kystr = strdup(pdata[n].key)
+        pykey = kystr.decode("ascii")
+        free(kystr)
+        d['key']      = pykey
+        kystr = strdup(pdata[n].proc.nspace)
+        myns = kystr.decode('ascii')
+        free(kystr)
+        proc = {'nspace':myns, 'rank': pdata[n].proc.rank}
+        d['proc']     = proc
+        d['value']    = val['value']
+        d['val_type'] = val['val_type']
+        ilist.append(d)
+        n += 1
+    return PMIX_SUCCESS
+
+cdef void pmix_destruct_pdata(pmix_pdata_t *pdata):
+    pmix_destruct_value(&pdata[0].value)
+
+# Free a malloc'd array of pmix_pdata_t structures
+#
+# @array [INPUT]
+#        - array of pmix_pdata_t to be free'd
+#
+# @sz [INPUT]
+#     - number of elements in array
+cdef void pmix_free_pdata(pmix_pdata_t *array, size_t sz):
+    n = 0
+    while n < sz:
+        pmix_destruct_pdata(&array[n])
+        n += 1
+    PyMem_Free(array)
+
+cdef int pmix_unload_queries(const pmix_query_t *queries, size_t nqueries, ilist:list):
+    cdef char* kystr
+    cdef size_t n = 0
+    keylist = []
+    qualist = []
+    query = {}
+    while n < nqueries:
+        rc = pmix_unload_argv(queries[n].keys, keylist)
+        pmix_unload_info(queries[n].qualifiers, queries[n].nqual, qualist)
+        query['keys']       = keylist
+        query['qualifiers'] = qualist
+        ilist.append(query)
+        n += 1
+    return PMIX_SUCCESS
+
+
+# Free a malloc'd array of pmix_query_t structs to free
+#
+# @array [INPUT]
+#        - pmix_query_t queries to be free'd
+#
+# @sz [INPUT]
+#     - number of elements in array
+cdef void pmix_free_queries(pmix_query_t *queries, size_t sz):
+    n = 0
+    while n < sz:
+        if queries[n].keys != NULL:
+            j = 0
+            while NULL != queries[n].keys[j]:
+                PyMem_Free(queries[n].keys[j])
+                j += 1
+            PyMem_Free(queries[n].keys)
+        if queries[n].qualifiers != NULL:
+            pmix_free_info(queries[n].qualifiers, queries[n].nqual)
+        n += 1
+    if queries != NULL:
+        PyMem_Free(queries)
+
+# Convert a list of (nspace, rank) tuples into an
+# array of pmix_proc_t structs
+#
+# @proc [INPUT]
+#       - malloc'd array of pmix_proc_t structs
+#
+# @peers [INPUT]
+#       - list of (nspace,rank) tuples
+#
+cdef int pmix_load_procs(pmix_proc_t *proc, peers:list):
+    n = 0
+    for p in peers:
+        pmix_copy_nspace(proc[n].nspace, p['nspace'])
+        proc[n].rank = p['rank']
+        n += 1
+    return PMIX_SUCCESS
+
+cdef int pmix_unload_procs(const pmix_proc_t *procs, size_t nprocs, peers:list):
+    cdef char* kystr
+    n = 0
+    while n < nprocs:
+        kystr = strdup(procs[n].nspace)
+        myns = kystr.decode('ascii')
+        free(kystr)
+        peers.append({'nspace':myns, 'rank':procs[n].rank})
+        n += 1
+    return PMIX_SUCCESS
+
+# Free a malloc'd array of pmix_proc_t structures
+#
+# @array [INPUT]
+#        - array of pmix_proc_t to be free'd
+#
+# @sz [INPUT]
+#     - number of elements in array
+#
+cdef void pmix_free_procs(pmix_proc_t *array, size_t sz):
+    PyMem_Free(array)
+
+cdef void pmix_unload_bytes(char *data, size_t ndata, blist:list):
+    cdef size_t n = 0
+    while n < ndata:
+        blist.append(data[n])
+        n += 1
+
+cdef void pmix_free_apps(pmix_app_t *array, size_t sz):
+    n = 0
+    while n < sz:
+        PyMem_Free(array[n].cmd)
+        # need to free the argv and env arrays
+        PyMem_Free(array[n].cwd)
+        if 0 < array[n].ninfo:
+            pmix_free_info(array[n].info, array[n].ninfo)
+        n += 1
+
+cdef void pmix_unload_apps(const pmix_app_t *apps, size_t napps, pyapps:list):
+    cdef size_t n = 0
+    while n < napps:
+        myapp = {}
+        myapp['cmd'] = str(apps[n].cmd)
+        myargv = []
+        if NULL != apps[n].argv:
+            pmix_unload_argv(apps[n].argv, myargv)
+            myapp['argv'] = myargv
+        myenv = []
+        if NULL != apps[n].env:
+            pmix_unload_argv(apps[n].env, myenv)
+            myapp['env'] = myenv
+        myapp['maxprocs'] = apps[n].maxprocs
+        keyvals = {}
+        if NULL != apps[n].info:
+            pmix_unload_info(apps[n].info, apps[n].ninfo, keyvals)
+            myapp['info'] = keyvals
+        pyapps.append(myapp)
+        n += 1
+
+cdef int pmix_load_apps(pmix_app_t *apps, pyapps:list):
+    cdef size_t m
+    cdef size_t n
+    cdef char** argv
+    n = 0
+    for p in pyapps:
+        pycmd = str(p['cmd']).encode('ascii')
+        try:
+            apps[n].cmd = strdup(pycmd)
+        except:
+            return PMIX_ERR_TYPE_MISMATCH
+
+        try:
+            if p['argv'] is not None:
+                m = len(p['argv']) + 1
+            else:
+                m = 2
+            print("LOADAPPS NUMARGV ", m)
+            argv = <char**> PyMem_Malloc(m * sizeof(char*))
+            if not argv:
+                return PMIX_ERR_NOMEM
+            memset(argv, 0, m)
+            if p['argv'] is not None:
+                print("LOADAPPS ARGV ", p['argv'])
+                # pmix_load_argv(argv, p['argv'])
+                argv[0] = strdup("hostname")
+            else:
+                print("LOADAPPS ARGV ", p['cmd'])
+              #  pmix_load_argv(argv, [p['cmd']])
+                argv[0] = strdup("hostname")
+        except:
+            print("LOADAPPS NO ARGV");
+            argv = <char**> PyMem_Malloc(2 * sizeof(char*))
+            memset(argv, 0, 2)
+            pmix_load_argv(argv, [p['cmd']])
+
+        apps[n].env = NULL
+        try:
+            if p['env'] is not None:
+                m = len(p['env']) + 1
+            else:
+                m = 1
+            env = <char**> PyMem_Malloc(m * sizeof(char*))
+            if not argv:
+                return PMIX_ERR_NOMEM
+            memset(env, 0, m)
+            if p['env'] is not None:
+                pmix_load_argv(env, p['env'])
+        except:
+            pass
+
+        try:
+            pycwd = str(p['cwd']).encode('ascii')
+            apps[n].cwd = strdup(pycwd)
+        except:
+            pycwd = os.getcwd()
+            pycwd = pycwd.encode('ascii')
+            apps[n].cwd = strdup(pycwd)
+
+        apps[n].info = NULL
+        apps[n].ninfo = 0
+        try:
+            if p['info'] is not None:
+                apps[n].ninfo = len(p['info'])
+                apps[n].info =  <pmix_info_t*> PyMem_Malloc(apps[n].ninfo * sizeof(pmix_info_t))
+                if not apps[n].info:
+                    return PMIX_ERR_NOMEM
+                rc = pmix_load_info(apps[n].info, p['info'])
+                if PMIX_SUCCESS != rc:
+                    return rc
+        except:
+            pass
+
+        apps[n].maxprocs = 1
+        try:
+            apps[n].maxprocs = p['maxprocs']
+        except:
+            pass
+
+        n += 1
+    return PMIX_SUCCESS
diff -Nru pmix-3.2.2~rc1/bindings/python/pmix.pyx pmix-4.0.0/bindings/python/pmix.pyx
--- pmix-3.2.2~rc1/bindings/python/pmix.pyx	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/pmix.pyx	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,3404 @@
+#file: pmix.pyx
+
+from libc.string cimport memset, strncpy, strcpy, strlen, strdup
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memcpy
+from libc.stdio cimport printf
+from ctypes import addressof, c_int
+from cython.operator import address
+import signal, time
+import threading, ctypes
+import queue
+import array
+import os
+#import time
+from threading import Timer
+
+# pull in all the constant definitions - we
+# store them in a separate file for neatness
+include "pmix_constants.pxi"
+include "pmix.pxi"
+
+active = myLock()
+myhdlrs = []
+myname = {}
+eventQueue = queue.Queue(maxsize=1000)
+stop_progress = False
+
+def pyevhdlr(stop):
+    global eventQueue
+    global stop_progress
+
+    while True:
+        if stop_progress:
+            break
+        if not eventQueue.empty():
+            capsule = eventQueue.get()
+            try:
+                shifter = <pmix_pyshift_t*>PyCapsule_GetPointer(capsule, NULL)
+            except:
+                # don't beat on the cpu
+                time.sleep(0.001)
+                continue
+            op = shifter[0].op.decode('ascii')
+            if "event_handler" == op:
+                shifter[0].event_handler(shifter[0].status, shifter[0].results, shifter[0].nresults,
+                                         shifter[0].op_cbfunc, shifter[0].cbdata,
+                                         shifter[0].notification_cbdata)
+                if 0 < shifter[0].nresults:
+                    pmix_free_info(shifter[0].results, shifter[0].nresults)
+            elif "fence" == op:
+                shifter = <pmix_pyshift_t*>PyCapsule_GetPointer(capsule, "fence")
+                shifter[0].modex(shifter[0].status, shifter[0].bo.bytes, shifter[0].bo.size,
+                                 shifter[0].cbdata, NULL, NULL)
+            elif "directmodex" == op:
+                shifter[0].modex(shifter[0].status, shifter[0].data, shifter[0].ndata,
+                                 shifter[0].cbdata, NULL, NULL)
+                if 0 < shifter[0].ndata:
+                    PyMem_Free(&(shifter[0].data))
+            elif "lookup" == op:
+                shifter[0].lookup(shifter[0].status, shifter[0].pdata, shifter[0].ndata, shifter[0].cbdata)
+                if 0 < shifter[0].ndata:
+                    pmix_free_pdata(shifter[0].pdata, shifter[0].ndata)
+            elif "spawn" == op:
+                shifter[0].spawn(shifter[0].status, shifter[0].nspace, shifter[0].cbdata)
+            elif "query" == op:
+                shifter[0].query(shifter[0].status, shifter[0].info, shifter[0].ndata, shifter[0].cbdata, NULL, NULL)
+                if 0 < shifter[0].ndata:
+                    pmix_free_info(shifter[0].info, shifter[0].ndata)
+            elif "toolconnected" == op:
+                shifter[0].toolconnected(shifter[0].status, shifter[0].proc, shifter[0].cbdata)
+            elif "allocate" == op:
+                shifter[0].allocate(shifter[0].status, shifter[0].info, shifter[0].ndata,
+                                    shifter[0].cbdata, shifter[0].release_fn,
+                                    shifter[0].notification_cbdata)
+                if 0 < shifter[0].ndata:
+                    pmix_free_info(shifter[0].info, shifter[0].ndata)
+            elif "getcredential" == op:
+                shifter[0].getcredential(shifter[0].status, shifter[0].cred, shifter[0].info,
+                                         shifter[0].ndata, shifter[0].cbdata)
+                if 0 < shifter[0].ndata:
+                    pmix_free_info(shifter[0].info, shifter[0].ndata)
+            elif "validationcredential" == op:
+                shifter[0].validationcredential(shifter[0].status, shifter[0].info, shifter[0].ndata,
+                                                shifter[0].cbdata)
+                if 0 < shifter[0].ndata:
+                    pmix_free_info(shifter[0].info, shifter[0].ndata)
+            else:
+                print("UNSUPPORTED OP", op)
+        # don't beat on the cpu
+        time.sleep(0.001)
+    return
+
+
+# create a progress thread for processing events
+progressThread = threading.Thread(target = pyevhdlr, args =(lambda : stop_progress, ))
+# ensure the thread dies at termination of main so we can exit
+# if we should terminate without finalizing
+progressThread.setDaemon(True)
+
+cdef void dmodx_cbfunc(pmix_status_t status,
+                       char *data, size_t sz,
+                       void *cbdata):
+    global active
+    if PMIX_SUCCESS == status:
+        active.cache_data(data, sz)
+    active.set(status)
+    return
+
+cdef void setupapp_cbfunc(pmix_status_t status,
+                          pmix_info_t info[], size_t ninfo,
+                          void *provided_cbdata,
+                          pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    global active
+    if PMIX_SUCCESS == status:
+        ilist = []
+        rc = pmix_unload_info(info, ninfo, ilist)
+        active.cache_info(ilist)
+        status = rc
+    active.set(status)
+    if (NULL != cbfunc):
+        cbfunc(PMIX_SUCCESS, cbdata)
+    return
+
+cdef void collectinventory_cbfunc(pmix_status_t status, pmix_info_t info[],
+                                  size_t ninfo, void *cbdata,
+                                  pmix_release_cbfunc_t release_fn,
+                                  void *release_cbdata) with gil:
+    global active
+    if PMIX_SUCCESS == status:
+        ilist = []
+        rc = pmix_unload_info(info, ninfo, ilist)
+        active.cache_info(ilist)
+        status = rc
+    active.set(status)
+    if (NULL != release_fn):
+        release_fn(release_cbdata)
+    return
+
+cdef void pyiofhandler(size_t iofhdlr_id, pmix_iof_channel_t channel,
+                       pmix_proc_t *source, pmix_byte_object_t *payload,
+                       pmix_info_t info[], size_t ninfo):
+    cdef char* kystr
+    pychannel = int(channel)
+    pyiof_id  = int(iofhdlr_id)
+
+    # convert the source to python
+    pysource = {}
+    kystr = strdup(source[0].nspace)
+    myns = kystr.decode('ascii')
+    free(kystr)
+    pysource = {'nspace': myns, 'rank': source[0].rank}
+
+    # convert the inbound info to python
+    pyinfo = []
+    pmix_unload_info(info, ninfo, pyinfo)
+
+    # convert payload to python byteobject
+    pybytes = {}
+    if NULL != payload:
+        pybytes['bytes'] = payload[0].bytes
+        pybytes['size']  = payload[0].size
+
+    # find the handler being called
+    found = False
+    rc = PMIX_ERR_NOT_FOUND
+    for h in myhdlrs:
+        try:
+            if iofhdlr_id == h['refid']:
+                found = True
+                # call user iof python handler
+                h['hdlr'](pyiof_id, pychannel, pysource, pybytes, pyinfo)
+        except:
+            pass
+
+    # if we didn't find the handler, cache this event in a timeshift
+    # and try it again
+    if not found:
+        mycaddy    = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("iofhdlr_cache")
+        mycaddy.idx                 = iofhdlr_id
+        mycaddy.channel             = channel
+        memset(mycaddy.source.nspace, 0, PMIX_MAX_NSLEN+1)
+        memcpy(mycaddy.source.nspace, source[0].nspace, PMIX_MAX_NSLEN)
+        mycaddy.source.rank         = source[0].rank
+        memset(mycaddy.payload.bytes, 0, PMIX_MAX_NSLEN+1)
+        memcpy(mycaddy.payload.bytes, payload[0].bytes, PMIX_MAX_NSLEN)
+        mycaddy.payload.size        = payload[0].size
+        mycaddy.info                = info
+        mycaddy.ndata               = ninfo
+        cb = PyCapsule_New(mycaddy, "iofhdlr_cache", NULL)
+        threading.Timer(0.001, iofhdlr_cache, [cb, rc]).start()
+    return
+
+cdef void pyeventhandler(size_t evhdlr_registration_id,
+                         pmix_status_t status,
+                         const pmix_proc_t *source,
+                         pmix_info_t info[], size_t ninfo,
+                         pmix_info_t *results, size_t nresults,
+                         pmix_event_notification_cbfunc_fn_t cbfunc,
+                         void *cbdata):
+    cdef pmix_info_t *myresults
+    cdef pmix_info_t **myresults_ptr
+    cdef size_t nmyresults
+    cdef char* kystr
+    cdef pmix_nspace_t srcnspace
+    global eventQueue
+
+    # convert the source to python
+    pysource = {}
+    memset(srcnspace, 0, PMIX_MAX_NSLEN+1)
+    memcpy(srcnspace, source[0].nspace, PMIX_MAX_NSLEN)
+    kystr = strdup(srcnspace)
+    myns = kystr.decode('ascii')
+    free(kystr)
+    srcrank = int(source[0].rank)
+    print("EVHDLR ", myns, srcrank)
+    pysource = {'nspace': myns, 'rank': srcrank}
+    pyev_id  = int(evhdlr_registration_id)
+
+    # convert the inbound info to python
+    pyinfo = []
+    if 0 < ninfo:
+        rc = pmix_unload_info(info, ninfo, pyinfo)
+        if PMIX_SUCCESS != rc:
+            print("Unable to unload info structs")
+            return
+
+    # convert the inbound results from prior handlers
+    # that serviced this event to python
+    pyresults = []
+    if 0 < nresults:
+        rc = pmix_unload_info(results, nresults, pyresults)
+        if PMIX_SUCCESS != rc:
+            print("Unable to unload prior results")
+            return
+
+    # find the handler being called
+    found = False
+    rc = PMIX_ERR_NOT_FOUND
+    for h in myhdlrs:
+        try:
+            if evhdlr_registration_id == h['refid']:
+                found = True
+                # execute their handler
+                rc, pymyresults = h['hdlr'](pyev_id, status, pysource, pyinfo, pyresults)
+                # allocate and load pmix info structs from python list of dictionaries
+                myresults_ptr = &myresults
+                prc = pmix_alloc_info(myresults_ptr, &nmyresults, pymyresults)
+                if PMIX_SUCCESS != prc:
+                    print("Unable to load new results")
+                mycaddy    = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+                mycaddy.op = strdup("event_handler")
+                mycaddy.status              = rc
+                mycaddy.results             = myresults
+                mycaddy.nresults            = nmyresults
+                mycaddy.op_cbfunc           = NULL
+                mycaddy.cbdata              = NULL
+                mycaddy.notification_cbdata = cbdata
+                mycaddy.event_handler       = cbfunc
+                cb = PyCapsule_New(mycaddy, "event_handler", NULL)
+                # push the results into the queue to return them
+                # to the PMIx library
+                eventQueue.put(cb)
+        except:
+            pass
+
+    # if we didn't find the handler, delay a little and try again
+    if not found:
+        mycaddy    = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("event_handler")
+        mycaddy.idx                 = evhdlr_registration_id
+        mycaddy.status              = status
+        memset(mycaddy.source.nspace, 0, PMIX_MAX_NSLEN+1)
+        memcpy(mycaddy.source.nspace, source[0].nspace, PMIX_MAX_NSLEN)
+        mycaddy.source.rank         = source[0].rank
+        mycaddy.info                = info
+        mycaddy.ndata               = ninfo
+        mycaddy.results             = results
+        mycaddy.nresults            = nresults
+        mycaddy.op_cbfunc           = NULL
+        mycaddy.cbdata              = NULL
+        mycaddy.notification_cbdata = cbdata
+        mycaddy.event_handler       = cbfunc
+        cb = PyCapsule_New(mycaddy, "event_handler", NULL)
+        threading.Timer(0.001, event_cache_cb, [cb, rc]).start()
+    return
+
+cdef class PMIxClient:
+    cdef pmix_proc_t myproc;
+    cdef pmix_fabric_t myfabric;
+    cdef int fabric_set;
+    cdef pmix_topology_t topo
+
+    def __cinit__(self):
+        memset(self.myproc.nspace, 0, sizeof(self.myproc.nspace))
+        self.myproc.rank = PMIX_RANK_UNDEF
+        memset(&self.myfabric, 0, sizeof(self.myfabric))
+        self.fabric_set = 0
+        self.topo.source = NULL
+        self.topo.topology = NULL
+
+    def __init__(self):
+        global myhdlrs, myname
+        memset(self.myproc.nspace, 0, sizeof(self.myproc.nspace))
+        self.myproc.rank = <uint32_t>PMIX_RANK_UNDEF
+        myhdlrs = []
+        myname = {}
+
+    def initialized(self):
+        return PMIx_Initialized()
+
+    def get_version(self):
+        return PMIx_Get_version()
+
+    # Initialize the PMIx client library, connecting
+    # us to the local PMIx server
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where each
+    #            dictionary has a key, value, and val_type
+    #            defined as such:
+    #            [{key:y, value:val, val_type:ty}, … ]
+    #
+    def init(self, dicts:list):
+        cdef size_t klen
+        global myname
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        myname = {}
+        global progressThread
+
+        # start the event handler progress thread
+        print("CLIENT STARTING THREAD")
+        progressThread.start()
+        print("CLIENT THREAD STARTED")
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &klen, dicts)
+        rc = PMIx_Init(&self.myproc, info, klen)
+        if 0 < klen:
+            pmix_free_info(info, klen)
+        if PMIX_SUCCESS == rc:
+            # convert the returned name
+            myname = {'nspace': str(self.myproc.nspace), 'rank': self.myproc.rank}
+        return rc, myname
+
+    # Finalize the client library
+    def finalize(self, dicts:list):
+        cdef size_t klen
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        global stop_progress
+        global progressThread
+
+        # stop progress thread
+        stop_progress = True
+        progressThread.join()
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &klen, dicts)
+        rc = PMIx_Finalize(info, klen)
+        if 0 < klen:
+            pmix_free_info(info, klen)
+        return rc
+
+    def initialized(self):
+        return PMIx_Initialized()
+
+    # Request that the provided array of procs be aborted, returning the
+    # provided _status_ and printing the provided message.
+    #
+    # @status [INPUT]
+    #         - PMIx status to be returned on exit
+    #
+    # @msg [INPUT]
+    #        - string message to be printed
+    #
+    # @procs [INPUT]
+    #        - list of proc nspace,rank dicts
+    def abort(self, status, msg, peers:list):
+        cdef pmix_proc_t *procs
+        cdef size_t sz
+        # convert list of procs to array of pmix_proc_t's
+        if peers is not None:
+            sz = len(peers)
+            if 0 < sz:
+                procs = <pmix_proc_t*> PyMem_Malloc(sz * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                rc = pmix_load_procs(procs, peers)
+                if PMIX_SUCCESS != rc:
+                    pmix_free_procs(procs, sz)
+                    return rc
+            else:
+                # if they didn't give us a set of procs,
+                # then we default to our entire job
+                sz = 1
+                procs = <pmix_proc_t*> PyMem_Malloc(sz * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+                procs[0].rank = PMIX_RANK_WILDCARD
+        else:
+            # if they didn't give us a set of procs,
+            # then we default to our entire job
+            sz = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(sz * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+        if isinstance(msg, str):
+            pymsg = msg.encode('ascii')
+        else:
+            pymsg = msg
+        # pass into PMIx_Abort
+        rc = PMIx_Abort(status, pymsg, procs, sz)
+        if 0 < sz:
+            pmix_free_procs(procs, sz)
+        return rc
+
+    # Store some data locally for retrieval by other areas of the
+    # proc. This is data that has only internal scope - it will
+    # never be "pushed" externally
+    #
+    # @proc [INPUT]
+    #       - namespace and rank of the client (dict)
+    #
+    # @key [INPUT]
+    #      - the key to be stored
+    #
+    # @value [INPUT]
+    #        - a dict to be stored with keys (value, val_type)
+    def store_internal(self, pyproc:dict, pykey:str, pyval:dict):
+        cdef pmix_key_t key
+        cdef pmix_proc_t proc
+        cdef pmix_value_t value
+
+        # convert pyproc to pmix_proc_t
+        if pyproc is None:
+            pmix_copy_nspace(proc.nspace, self.myproc.nspace)
+            proc.rank = self.myproc.rank
+        else:
+            pmix_copy_nspace(proc.nspace, pyproc['nspace'])
+            proc.rank = pyproc['rank']
+
+        # convert key,val to pmix_value_t and pmix_key_t
+        pmix_copy_key(key, pykey)
+
+        # convert the dict to a pmix_value_t
+        rc = pmix_load_value(&value, pyval)
+
+        # call API
+        rc = PMIx_Store_internal(&proc, key, &value)
+        if rc == PMIX_SUCCESS:
+            pmix_free_value(self, &value)
+        return rc
+
+    # put a value into the keystore
+    #
+    # @scope [INPUT]
+    #        - the scope of the data
+    #
+    # @key [INPUT]
+    #      - the key to be stored
+    #
+    # @value [INPUT]
+    #        - a dict to be stored with keys (value, val_type)
+    def put(self, scope, ky, val):
+        cdef pmix_key_t key
+        cdef pmix_value_t value
+        # convert the keyval tuple to a pmix_info_t
+        pmix_copy_key(key, ky)
+        pmix_load_value(&value, val)
+        # pass it into the PMIx_Put function
+        rc = PMIx_Put(scope, key, &value)
+        pmix_destruct_value(&value)
+        return rc
+
+    def commit(self):
+        rc = PMIx_Commit()
+        return rc
+
+    def fence(self, peers:list, dicts:list):
+        cdef pmix_proc_t *procs
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo, nprocs
+        nprocs = 0
+        ninfo = 0
+        # convert list of procs to array of pmix_proc_t's
+        if peers is not None:
+            nprocs = len(peers)
+            if 0 < nprocs:
+                procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                rc = pmix_load_procs(procs, peers)
+                if PMIX_SUCCESS != rc:
+                    pmix_free_procs(procs, nprocs)
+                    return rc
+            else:
+                nprocs = 1
+                procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+                procs[0].rank = PMIX_RANK_WILDCARD
+        else:
+            nprocs = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, dicts)
+        if PMIX_SUCCESS != rc:
+            pmix_free_procs(procs, nprocs)
+            return rc
+
+        # pass it into the fence API
+        rc = PMIx_Fence(procs, nprocs, info, ninfo)
+        if 0 < nprocs:
+            pmix_free_procs(procs, nprocs)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    # retrieve a value from the keystore
+    #
+    # @proc [INPUT]
+    #       - namespace and rank of the client (dict)
+    #
+    # @key [INPUT]
+    #      - the key to be retrieved
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where each
+    #            dictionary has a key, value, and val_type
+    #            defined as such:
+    #            [{key:y, value:val, val_type:ty}, … ]
+    def get(self, proc:dict, ky, dicts:list):
+        cdef pmix_info_t *info;
+        cdef pmix_info_t **info_ptr;
+        cdef size_t ninfo;
+        cdef pmix_key_t key;
+        cdef pmix_value_t *val_ptr;
+        cdef pmix_proc_t p;
+
+        ninfo   = 0
+        val_ptr = NULL
+
+        # convert proc to pmix_proc_t
+        if proc is None:
+            pmix_copy_nspace(p.nspace, self.myproc.nspace)
+            p.rank = self.myproc.rank
+        else:
+            pmix_copy_nspace(p.nspace, proc['nspace'])
+            p.rank = proc['rank']
+
+        # convert key,val to pmix_value_t and pmix_key_t
+        pmix_copy_key(key, ky)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, dicts)
+
+        # pass it into the get API
+        rc = PMIx_Get(&p, key, info, ninfo, &val_ptr)
+        if PMIX_SUCCESS == rc:
+            val = pmix_unload_value(val_ptr)
+            pmix_free_value(self, val_ptr)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc, val
+
+    # Publish the data in the info array for lookup
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where
+    #            a key, flags, value, and val_type
+    #            can be defined as keys
+    def publish(self, dicts:list):
+        cdef pmix_info_t *info;
+        cdef pmix_info_t **info_ptr;
+        cdef size_t ninfo;
+        ninfo = 0
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, dicts)
+
+        # pass it into the publish API
+        rc = PMIx_Publish(info, ninfo)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    # unpublish the data in the data store
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where
+    #            a key, flags, value, and val_type
+    #            can be defined as keys
+    # @pykeys [INPUT]
+    #          - list of python info key strings
+    def unpublish(self, pykeys:list, dicts:list):
+        cdef pmix_info_t *info;
+        cdef pmix_info_t **info_ptr;
+        cdef size_t ninfo;
+        cdef size_t nstrings;
+        cdef char **keys;
+        keys     = NULL
+        ninfo    = 0
+        nstrings = 0
+
+        # load pykeys into char **keys
+        if pykeys is not None:
+            nstrings = len(pykeys)
+            if 0 < nstrings:
+                keys = <char **> PyMem_Malloc(nstrings * sizeof(char*))
+                if not keys:
+                    PMIX_ERR_NOMEM
+            rc = pmix_load_argv(keys, pykeys)
+            if PMIX_SUCCESS != rc:
+                n = 0
+                while keys[n] != NULL:
+                    PyMem_Free(keys[n])
+                    n += 1
+                return rc
+            else:
+                keys = NULL
+        else:
+            keys = NULL
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, dicts)
+
+        # pass it into the unpublish API
+        rc = PMIx_Unpublish(keys, info, ninfo)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    # lookup info published by this or another process
+    # @pdata [INPUT]
+    #          - a list of dictionaries, where key is
+    #            recorded in the pdata dictionary and
+    #            passed to PMIx_Lookup
+    # pdata = {‘proc’: {‘nspace’: mynspace, ‘rank’: myrank}, ‘key’: ky,
+    # ‘value’: v, ‘val_type’: t}
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where
+    #            a key, flags, value, and val_type
+    #            can be defined as keys
+    def lookup(self, data:list, dicts:list):
+        cdef pmix_pdata_t *pdata;
+        cdef pmix_info_t  *info;
+        cdef pmix_info_t  **info_ptr;
+        cdef size_t npdata;
+        cdef size_t ninfo;
+
+        npdata  = 0
+        ninfo   = 0
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, dicts)
+
+        # convert the list of dictionaries to array of
+        # pmix_pdata_t structs
+        if data is not None:
+            npdata = len(data)
+            if 0 < npdata:
+                pdata = <pmix_pdata_t*> PyMem_Malloc(npdata * sizeof(pmix_pdata_t))
+                if not pdata:
+                    return PMIX_ERR_NOMEM
+                n = 0
+                for d in data:
+                    pykey = d['key']
+                    pmix_copy_key(pdata[n].key, pykey)
+                    rc = 0
+                    n += 1
+                if PMIX_SUCCESS != rc:
+                    pmix_free_pdata(pdata, npdata)
+                    return rc
+            else:
+                pdata = NULL
+        else:
+            pdata = NULL
+
+        # pass it into the lookup API
+        rc = PMIx_Lookup(pdata, npdata, info, ninfo)
+        if PMIX_SUCCESS == rc:
+            rc = pmix_unload_pdata(pdata, npdata, data)
+            # remove the first element, which is just the key
+            data.pop(0)
+            pmix_free_info(info, ninfo)
+            pmix_free_pdata(pdata, npdata)
+        return rc, data
+
+    # Spawn a new job
+    #
+    #
+    def spawn(self, jobInfo:list, pyapps:list):
+        cdef pmix_info_t *jinfo;
+        cdef pmix_info_t **jinfo_ptr;
+        cdef pmix_app_t *apps;
+        cdef size_t ninfo
+        cdef size_t napps;
+        cdef pmix_nspace_t nspace;
+
+        # protect against bad input
+        if pyapps is None or len(pyapps) == 0:
+            return PMIX_ERR_BAD_PARAM, None
+
+        # allocate and load pmix info structs from python list of dictionaries
+        if jobInfo is not None:
+            jinfo_ptr = &jinfo
+            rc = pmix_alloc_info(jinfo_ptr, &ninfo, jobInfo)
+        else:
+            jinfo = NULL
+            ninfo = 0
+
+        # convert the list of apps to an array of pmix_app_t
+        napps = len(pyapps)
+        apps = <pmix_app_t*> PyMem_Malloc(napps * sizeof(pmix_app_t))
+        if not apps:
+            pmix_free_info(jinfo, ninfo)
+            return PMIX_ERR_NOMEM, None
+        rc = pmix_load_apps(apps, pyapps)
+        if PMIX_SUCCESS != rc:
+            pmix_free_apps(apps, napps)
+            if 0 < ninfo:
+                pmix_free_info(jinfo, ninfo)
+            return rc, None
+        apps[0].argv[0] = strdup("hostname")
+        rc = PMIx_Spawn(jinfo, ninfo, apps, napps, nspace)
+        pmix_free_apps(apps, napps)
+        if 0 < ninfo:
+            pmix_free_info(jinfo, ninfo)
+        pyns = nspace
+        return rc, pyns.decode('ascii')
+
+    def connect(self, peers:list, pyinfo:list):
+        cdef pmix_proc_t *procs
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+        cdef size_t nprocs
+        nprocs = 0
+        ninfo = 0
+
+        # convert list of procs to array of pmix_proc_t's
+        if peers is not None:
+            nprocs = len(peers)
+            if 0 < nprocs:
+                procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                rc = pmix_load_procs(procs, peers)
+                if PMIX_SUCCESS != rc:
+                    pmix_free_procs(procs, nprocs)
+                    return rc
+            else:
+                nprocs = 1
+                procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+                procs[0].rank = PMIX_RANK_WILDCARD
+        else:
+            nprocs = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Connect(procs, nprocs, info, ninfo)
+        if 0 < nprocs:
+            pmix_free_procs(procs, nprocs)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    def disconnect(self, peers:list, pyinfo:list):
+        cdef pmix_proc_t *procs
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+        cdef size_t nprocs
+        nprocs = 0
+        ninfo = 0
+
+        # convert list of procs to array of pmix_proc_t's
+        if peers is not None:
+            nprocs = len(peers)
+            if 0 < nprocs:
+                procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                rc = pmix_load_procs(procs, peers)
+                if PMIX_SUCCESS != rc:
+                    pmix_free_procs(procs, nprocs)
+                    return rc
+            else:
+                nprocs = 1
+                procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+                if not procs:
+                    return PMIX_ERR_NOMEM
+                pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+                procs[0].rank = PMIX_RANK_WILDCARD
+        else:
+            nprocs = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Disconnect(procs, nprocs, info, ninfo)
+        if 0 < nprocs:
+            pmix_free_procs(procs, nprocs)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    def resolve_peers(self, pynode:str, pyns:str):
+        cdef pmix_nspace_t nspace
+        cdef char *nodename
+        cdef pmix_proc_t *procs
+        cdef size_t nprocs
+        peers = []
+
+        nodename = NULL
+        memset(nspace, 0, sizeof(nspace))
+        procs = NULL
+        if pynode is not None:
+            pyn = pynode.encode('ascii')
+            nodename = strdup(pyn)
+        if pyns is not None:
+            pmix_copy_nspace(nspace, pyns)
+        rc = PMIx_Resolve_peers(nodename, nspace, &procs, &nprocs)
+        if PMIX_SUCCESS == rc and 0 < nprocs:
+            rc = pmix_unload_procs(procs, nprocs, peers)
+            pmix_free_procs(procs, nprocs)
+        return rc, peers
+
+    def resolve_nodes(self, pyns:str):
+        cdef pmix_nspace_t nspace
+        cdef char *nodelist
+
+        nodelist = NULL
+        memset(nspace, 0, sizeof(nspace))
+        if pyns is not None:
+            pmix_copy_nspace(nspace, pyns)
+        rc = PMIx_Resolve_nodes(nspace, &nodelist)
+        if PMIX_SUCCESS == rc:
+            pyn = nodelist
+            pynodes = pyn.decode('ascii')
+            PyMem_Free(nodelist)
+        return rc, pynodes
+
+    def query(self, pyq:list):
+        cdef pmix_query_t *queries
+        cdef size_t nqueries
+        cdef pmix_info_t *results
+        cdef pmix_info_t **results_ptr
+        cdef size_t nresults
+        cdef pmix_info_t **qual_ptr
+        nqueries   = 0
+        nresults   = 0
+        queries    = NULL
+        qual_ptr   = NULL
+
+        pyresults = []
+        if pyq is not None:
+            nqueries = len(pyq)
+            if 0 < nqueries:
+                queries = <pmix_query_t*> PyMem_Malloc(nqueries * sizeof(pmix_query_t))
+                if not queries:
+                    return PMIX_ERR_NOMEM,pyresults
+                n = 0
+                for q in pyq:
+                    queries[n].keys       = NULL
+                    queries[n].qualifiers = NULL
+                    nstrings = len(q['keys'])
+                    if 0 < nstrings:
+                        queries[n].keys = <char **> PyMem_Malloc((nstrings+1) * sizeof(char*))
+                        if not queries[n].keys:
+                            pmix_free_queries(queries, nqueries)
+                            return PMIX_ERR_NOMEM,pyresults
+                        rc = pmix_load_argv(queries[n].keys, q['keys'])
+                        if PMIX_SUCCESS != rc:
+                            pmix_free_queries(queries, nqueries)
+                            return rc,pyresults
+                    # allocate and load pmix info structs from python list of dictionaries
+                    queries[n].nqual = 0
+                    qual_ptr         = &(queries[n].qualifiers)
+                    rc               = pmix_alloc_info(qual_ptr, &(queries[n].nqual), q['qualifiers'])
+                    n += 1
+            else:
+                nqueries = 0
+        else:
+            nqueries = 0
+
+        # pass it into the query_info API
+        rc = PMIx_Query_info(queries, nqueries, &results, &nresults)
+        if PMIX_SUCCESS == rc:
+            rc = pmix_unload_info(results, nresults,  pyresults)
+            # free results info structs
+            pmix_free_info(results, nresults)
+        # free memory for query structs
+        pmix_free_queries(queries, nqueries)
+        return rc, pyresults
+
+    def log(self, pydata:list, pydirs:list):
+        cdef pmix_info_t *data
+        cdef pmix_info_t **data_ptr
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef size_t ndata
+        cdef size_t ndirs
+
+        # allocate and load pmix info structs from python list of dictionaries
+        data_ptr = &data
+        directives_ptr = &directives
+        rc = pmix_alloc_info(data_ptr, &ndata, pydata)
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+
+        # call the API
+        rc = PMIx_Log(data, ndata, directives, ndirs)
+        pmix_free_info(data, ndata)
+        if 0 < ndirs:
+            pmix_free_info(directives, ndirs)
+        return rc
+
+    def allocation_request(self, directive, pyinfo:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef pmix_info_t *results
+        cdef size_t ninfo
+        cdef size_t nresults
+
+        results = NULL
+        nresults = 0
+        pyres = []
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # call the API
+        rc = PMIx_Allocation_request(directive, info, ninfo, &results, &nresults)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        if PMIX_SUCCESS == rc and 0 < nresults:
+            # convert the results
+            rc = pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def job_control(self, pytargets:list, pydirs:list):
+        cdef pmix_proc_t *targets
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef pmix_info_t *results
+        cdef size_t ntargets
+        cdef size_t ndirs
+        cdef size_t nresults
+
+        results = NULL
+        nresults = 0
+        pyres = []
+        # convert list of procs to array of pmix_proc_t's
+        if pytargets is not None:
+            ntargets = len(pytargets)
+            if 0 < ntargets:
+                targets = <pmix_proc_t*> PyMem_Malloc(ntargets * sizeof(pmix_proc_t))
+                if not targets:
+                    return PMIX_ERR_NOMEM, pyres
+                rc = pmix_load_procs(targets, pytargets)
+                if PMIX_SUCCESS != rc:
+                    pmix_free_procs(targets, ntargets)
+                    return rc, pyres
+            else:
+                ntargets = 1
+                targets = <pmix_proc_t*> PyMem_Malloc(ntargets * sizeof(pmix_proc_t))
+                if not targets:
+                    return PMIX_ERR_NOMEM, pyres
+                pmix_copy_nspace(targets[0].nspace, self.myproc.nspace)
+                targets[0].rank = PMIX_RANK_WILDCARD
+        else:
+            ntargets = 1
+            targets = <pmix_proc_t*> PyMem_Malloc(ntargets * sizeof(pmix_proc_t))
+            if not targets:
+                return PMIX_ERR_NOMEM, pyres
+            pmix_copy_nspace(targets[0].nspace, self.myproc.nspace)
+            targets[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+        if PMIX_SUCCESS != rc:
+            if 0 < ntargets:
+                pmix_free_procs(targets, ntargets)
+            return rc
+
+        # call the API
+        rc = PMIx_Job_control(targets, ntargets, directives, ndirs, &results, &nresults)
+        if 0 < ndirs:
+            pmix_free_info(directives, ndirs)
+        if 0 < ntargets:
+            pmix_free_procs(targets, ntargets)
+        if PMIX_SUCCESS == rc and 0 < nresults:
+            # convert the results
+            rc = pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def monitor(self, pymonitor_info:list, code:int, pydirs:list):
+        cdef pmix_info_t *monitor_info
+        cdef pmix_info_t **monitor_info_ptr
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef pmix_info_t *results
+        cdef size_t nmonitor
+        cdef size_t ndirs
+        cdef size_t nresults
+
+        results = NULL
+        nresults = 0
+        pyres = []
+
+        # convert list of info to array of pmix_info_t's
+        monitor_info_ptr = &monitor_info
+        rc = pmix_alloc_info(monitor_info_ptr, &nmonitor, pymonitor_info)
+        if PMIX_SUCCESS != rc:
+            if 0 < nmonitor:
+                pmix_free_info(monitor_info, nmonitor)
+            return rc
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+        if PMIX_SUCCESS != rc:
+            if 0 < ndirs:
+                pmix_free_info(directives, ndirs)
+            return rc
+
+        # call the API
+        rc = PMIx_Process_monitor(monitor_info, code, directives, ndirs, &results, &nresults)
+        if 0 < ndirs:
+            pmix_free_info(directives, ndirs)
+        if 0 < nmonitor:
+            pmix_free_info(monitor_info, nmonitor)
+        if PMIX_SUCCESS == rc and 0 < nresults:
+            # convert the results
+            rc = pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def get_credential(self, pyinfo:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef pmix_byte_object_t bo
+        cdef pmix_byte_object_t *boptr
+        cdef size_t ninfo
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+        if PMIX_SUCCESS != rc:
+            if 0 < ninfo:
+                pmix_free_info(info, ninfo)
+            return rc
+
+        # call the API
+        boptr = &bo
+        rc = PMIx_Get_credential(info, ninfo, boptr)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        blist = []
+        cred = {}
+        if PMIX_SUCCESS == rc and 0 < bo.size:
+            # convert the results
+            pmix_unload_bytes(bo.bytes, bo.size, blist)
+            barray = bytearray(blist)
+            cred['bytes'] = barray
+            cred['size'] = bo.size
+        return rc, cred
+
+    def validate_credential(self, pycred:dict, pyinfo:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef pmix_byte_object_t *bo
+        cdef size_t ninfo
+        cdef pmix_info_t *results
+        cdef size_t nresults
+
+        results = NULL
+        nresults = 0
+        pyres = []
+
+        # convert pycred to pmix_byte_object_t
+        bo = <pmix_byte_object_t*>PyMem_Malloc(sizeof(pmix_byte_object_t))
+        if not bo:
+            return PMIX_ERR_NOMEM
+        cred = bytes(pycred['bytes'], 'ascii')
+        bo.size = sizeof(cred)
+        bo.bytes = <char*> PyMem_Malloc(bo.size)
+        if not bo.bytes:
+            return PMIX_ERR_NOMEM
+        pyptr = <const char*>cred
+        memcpy(bo.bytes, pyptr, bo.size)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+        if PMIX_SUCCESS != rc:
+            if 0 < ninfo:
+                pmix_free_info(info, ninfo)
+            return rc
+
+        # call the API
+        rc = PMIx_Validate_credential(bo, info, ninfo, &results, &nresults)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        if PMIX_SUCCESS == rc and 0 < nresults:
+            # convert the results
+            rc = pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def group_construct(self, group:str, peers:list, pyinfo:list):
+        cdef pmix_proc_t *procs
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef pmix_info_t *results
+        cdef size_t ninfo
+        cdef size_t nprocs
+        cdef size_t nresults
+        nprocs = 0
+        ninfo = 0
+
+        # convert group name
+        pygrp = group.encode('ascii')
+        # convert list of procs to array of pmix_proc_t's
+        if peers is not None:
+            nprocs = len(peers)
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            rc = pmix_load_procs(procs, peers)
+            if PMIX_SUCCESS != rc:
+                pmix_free_procs(procs, nprocs)
+                return rc
+        else:
+            nprocs = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Group_construct(pygrp, procs, nprocs, info, ninfo, &results, &nresults)
+        if 0 < nprocs:
+            pmix_free_procs(procs, nprocs)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        pyres = []
+        if 0 < nresults:
+            # convert results
+            pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def group_invite(self, group:str, peers:list, pyinfo:list):
+        cdef pmix_proc_t *procs
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef pmix_info_t *results
+        cdef size_t ninfo
+        cdef size_t nprocs
+        cdef size_t nresults
+        nprocs = 0
+        ninfo = 0
+
+        # convert group name
+        pygrp = group.encode('ascii')
+        # convert list of procs to array of pmix_proc_t's
+        if peers is not None:
+            nprocs = len(peers)
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            rc = pmix_load_procs(procs, peers)
+            if PMIX_SUCCESS != rc:
+                pmix_free_procs(procs, nprocs)
+                return rc
+        else:
+            nprocs = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Group_invite(pygrp, procs, nprocs, info, ninfo, &results, &nresults)
+        if 0 < nprocs:
+            pmix_free_procs(procs, nprocs)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        pyres = []
+        if 0 < nresults:
+            # convert results
+            pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def group_join(self, group:str, leader:dict, opt:int, pyinfo:list):
+        cdef pmix_proc_t proc
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef pmix_info_t *results
+        cdef size_t ninfo
+        cdef size_t nprocs
+        cdef size_t nresults
+        ninfo = 0
+
+        # convert group name
+        pygrp = group.encode('ascii')
+        # convert leader to proc
+        if leader is not None:
+            pmix_copy_nspace(proc.nspace, leader['nspace'])
+            proc.rank = leader['rank']
+        else:
+            pmix_copy_nspace(proc.nspace, self.myproc.nspace)
+            proc.rank = self.myproc.rank
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Group_join(pygrp, &proc, opt, info, ninfo, &results, &nresults)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        pyres = []
+        if 0 < nresults:
+            # convert results
+            pmix_unload_info(results, nresults, pyres)
+            pmix_free_info(results, nresults)
+        return rc, pyres
+
+    def group_leave(self, group:str, pyinfo:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+        ninfo = 0
+
+        # convert group name
+        pygrp = group.encode('ascii')
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Group_leave(pygrp, info, ninfo)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    def group_destruct(self, group:str, pyinfo:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+        ninfo = 0
+
+        # convert group name
+        pygrp = group.encode('ascii')
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # Call the library
+        rc = PMIx_Group_destruct(pygrp, info, ninfo)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    def register_event_handler(self, pycodes:list, pyinfo:list, hdlr):
+        cdef pmix_status_t *codes
+        cdef size_t ncodes
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+
+        # convert the codes to an array of ints
+        if pycodes is not None:
+            ncodes = len(pycodes)
+            codes = <int*> PyMem_Malloc(ncodes * sizeof(int))
+            if not codes:
+                return PMIX_ERR_NOMEM
+            n = 0
+            for c in pycodes:
+                codes[n] = c
+                n += 1
+        else:
+            codes = NULL
+            ncodes = 0
+        # allocate and load pmix info structs from python list of dictionaries
+        if pyinfo is not None:
+            info_ptr = &info
+            rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+            if PMIX_SUCCESS != rc:
+                print("Error converting info array:", self.error_string(rc))
+                return rc, -1
+        else:
+            info = NULL
+            ninfo = 0
+
+        # pass our hdlr switchyard to the API
+        rc = PMIx_Register_event_handler(codes, ncodes, info, ninfo, pyeventhandler, NULL, NULL)
+
+        # cleanup
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        if 0 < ncodes:
+            PyMem_Free(codes)
+
+        # if rc < 0, then there was an error
+        if 0 > rc:
+            return rc, -1
+
+        # otherwise, this is our ref ID for this hdlr
+        myhdlrs.append({'refid': rc, 'hdlr': hdlr})
+        return PMIX_SUCCESS, rc
+
+    def deregister_event_handler(self, ref:int):
+        rc = PMIx_Deregister_event_handler(ref, NULL, NULL)
+        return rc
+
+    def notify_event(self, status:int, pysrc:dict, range, pyinfo:list):
+        cdef pmix_proc_t proc
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+
+        # convert the proc
+        pmix_copy_nspace(proc.nspace, pysrc['nspace'])
+        proc.rank = pysrc['rank']
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # call the library
+        rc = PMIx_Notify_event(status, &proc, range, info, ninfo, NULL, NULL)
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    def error_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Error_string(pystat)
+        pystr = string
+        val = pystr.decode('ascii')
+        return val
+
+    def proc_state_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Proc_state_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def scope_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Scope_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def persistence_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Persistence_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def data_range_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Data_range_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def info_directives_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Info_directives_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def data_type_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Data_type_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def alloc_directive_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Alloc_directive_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def iof_channel_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_IOF_channel_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def job_state_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Job_state_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def get_attribute_string(self, rep:str):
+        cdef char *string
+        pyrep = rep.encode('ascii')
+        string = <char*>PMIx_Get_attribute_string(pyrep)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def get_attribute_name(self, rep:str):
+        cdef char *string
+        pyrep = rep.encode('ascii')
+        string = <char*>PMIx_Get_attribute_name(pyrep)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def link_state_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Link_state_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def device_type_string(self, pystat:int):
+        cdef char *string
+
+        string = <char*>PMIx_Device_type_string(pystat)
+        pystr = string
+        return pystr.decode('ascii')
+
+    def fabric_register(self, dicts:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        fabricinfo = []
+        if 1 == self.fabric_set:
+            return (PMIX_ERR_RESOURCE_BUSY, None)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+
+        if sz > 0:
+            rc = PMIx_Fabric_register(&self.myfabric, info, sz)
+            pmix_free_info(info, sz)
+        else:
+            rc = PMIx_Fabric_register(&self.myfabric, NULL, 0)
+        if PMIX_SUCCESS == rc:
+            self.fabric_set = 1
+            # convert the fabric info array for return
+            if 0 < self.myfabric.ninfo:
+                pmix_unload_info(self.myfabric.info, self.myfabric.ninfo, fabricinfo)
+        return (rc, fabricinfo)
+
+    def fabric_update(self):
+        fabricinfo = []
+        if 0 == self.fabric_set:
+            return (PMIX_ERR_INIT, None)
+        rc = PMIx_Fabric_update(&self.myfabric)
+        # convert the fabric info array for return
+        if 0 < self.myfabric.ninfo:
+            pmix_unload_info(self.myfabric.info, self.myfabric.ninfo, fabricinfo)
+        return (rc, fabricinfo)
+
+    def fabric_deregister(self):
+        if 0 == self.fabric_set:
+            return PMIX_ERR_INIT
+        rc = PMIx_Fabric_deregister(&self.myfabric)
+        self.fabric_set = 0
+        return rc;
+
+    def load_topology(self):
+        rc = PMIx_Load_topology(&self.topo)
+        return rc
+
+    def get_relative_locality(self, loc1:str, loc2:str):
+        cdef char *string
+        cdef pmix_locality_t locality
+        pyl1 = loc1.encode('ascii')
+        pyl2 = loc2.encode('ascii')
+        pyloc = []
+        rc = PMIx_Get_relative_locality(pyl1, pyl2, &locality)
+        if PMIX_SUCCESS == rc:
+            pmix_convert_locality(locality, pyloc)
+        return (rc, pyloc)
+
+    def parse_cpuset_string(self, csetstr:str):
+        return (PMIX_ERR_NOT_SUPPORTED, None)
+
+    def get_cpuset(self, ref:int):
+        cdef pmix_cpuset_t cpuset
+        cdef char* csetstr
+        pycpus = {}
+        rc = PMIx_Get_cpuset(&cpuset, ref)
+        if PMIX_SUCCESS == rc:
+            rc = PMIx_server_generate_cpuset_string(&cpuset, &csetstr)
+            if PMIX_SUCCESS == rc:
+                pycpus['source'] = strdup(cpuset.source)
+                txt = csetstr.decode('ascii')
+                pycpus['cpus'] = txt.split(",")
+        return (rc, pycpus)
+
+    def compute_distances(self, pycpus:dict, dicts:list):
+        cdef pmix_cpuset_t cpuset
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        cdef pmix_device_distance_t *distances
+        cdef size_t ndist
+
+        results = []
+
+        # check that we loaded our topology
+        if NULL == self.topo.topology:
+            rc = self.load_topology()
+            if PMIX_SUCCESS != rc:
+                return (rc, results)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+        if PMIX_SUCCESS != rc:
+            return (rc, results)
+
+        # convert the cpuset
+        csetstr = pycpus['cpus'].encode('ascii')
+        rc = PMIx_Parse_cpuset_string(csetstr, &cpuset);
+        if PMIX_SUCCESS != rc:
+            return (rc, results)
+
+        # compute distances
+        rc = PMIx_Compute_distances(&self.topo, &cpuset, info, sz, &distances, &ndist)
+        if PMIX_SUCCESS != rc:
+            return (rc, results)
+
+        # convert to Python
+        n = 0
+        while n < ndist:
+            pydist = {}
+            pydist['uuid'] = strdup(distances[n].uuid)
+            pydist['osname'] = strdup(distances[n].osname)
+            pydist['mindist'] = distances[n].mindist
+            pydist['maxdist'] = distances[n].maxdist
+            results.append(pydist)
+            n += 1
+
+        # free the memory
+        PyMem_Free(distances)
+
+        # return result
+        return (rc, results)
+
+    def progress(self):
+        PMIx_Progress()
+        return
+
+pmixservermodule = {}
+def setmodulefn(k, f):
+    global pmixservermodule
+    permitted = ['clientconnected', 'clientfinalized', 'abort',
+                 'fencenb', 'directmodex', 'publish', 'lookup', 'unpublish',
+                 'spawn', 'connect', 'disconnect', 'registerevents',
+                 'deregisterevents', 'listener', 'notify_event', 'query',
+                 'toolconnected', 'log', 'allocate', 'jobcontrol',
+                 'monitor', 'getcredential', 'validatecredential',
+                 'iofpull', 'pushstdin', 'group', 'fabric']
+    if k not in permitted:
+        return PMIX_ERR_BAD_PARAM
+    if not k in pmixservermodule:
+        pmixservermodule[k] = f
+
+cdef class PMIxServer(PMIxClient):
+    cdef pmix_server_module_t myserver
+
+    def __cinit__(self):
+        self.fabric_set = 0
+        memset(self.myproc.nspace, 0, sizeof(self.myproc.nspace))
+        self.myproc.rank = PMIX_RANK_UNDEF
+        # v1.x interfaces
+        self.myserver.client_connected2 = <pmix_server_client_connected2_fn_t>clientconnected
+        self.myserver.client_finalized = <pmix_server_client_finalized_fn_t>clientfinalized
+        self.myserver.abort = <pmix_server_abort_fn_t>clientaborted
+        self.myserver.fence_nb = <pmix_server_fencenb_fn_t>fencenb
+        self.myserver.direct_modex = <pmix_server_dmodex_req_fn_t>directmodex
+        self.myserver.publish = <pmix_server_publish_fn_t>publish
+        self.myserver.lookup = <pmix_server_lookup_fn_t>lookup
+        self.myserver.unpublish = <pmix_server_unpublish_fn_t>unpublish
+        self.myserver.spawn = <pmix_server_spawn_fn_t>spawn
+        self.myserver.connect = <pmix_server_connect_fn_t>connect
+        self.myserver.disconnect = <pmix_server_disconnect_fn_t>disconnect
+        self.myserver.register_events = <pmix_server_register_events_fn_t>registerevents
+        self.myserver.deregister_events = <pmix_server_deregister_events_fn_t>deregisterevents
+        # skip the listener entry as Python servers will never
+        # provide their own socket listener thread
+        #
+        # v2.x interfaces
+        self.myserver.notify_event = <pmix_server_notify_event_fn_t>notifyevent
+        self.myserver.query = <pmix_server_query_fn_t>query
+        self.myserver.tool_connected = <pmix_server_tool_connection_fn_t>toolconnected
+        self.myserver.log = <pmix_server_log_fn_t>log
+        self.myserver.allocate = <pmix_server_alloc_fn_t>allocate
+        self.myserver.job_control = <pmix_server_job_control_fn_t>jobcontrol
+        self.myserver.monitor = <pmix_server_monitor_fn_t>monitor
+        # v3.x interfaces
+        self.myserver.get_credential = <pmix_server_get_cred_fn_t>getcredential
+        self.myserver.validate_credential = <pmix_server_validate_cred_fn_t>validatecredential
+        self.myserver.iof_pull = <pmix_server_iof_fn_t>iofpull
+        self.myserver.push_stdin = <pmix_server_stdin_fn_t>pushstdin
+        # v4.x interfaces
+        self.myserver.group = <pmix_server_grp_fn_t>group
+        self.myserver.fabric = <pmix_server_fabric_fn_t>fabric
+
+    # Initialize the PMIx server library
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where each
+    #            dictionary has a key, value, and val_type
+    #            defined as such:
+    #            [{key:y, value:val, val_type:ty}, … ]
+    #
+    # @map [INPUT]
+    #          - a dictionary of key-function pairs that map
+    #            server module callback functions to provided
+    #            implementations
+    def init(self, dicts:list, map:dict):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        global progressThread
+
+        # start the event handler progress thread
+        progressThread.start()
+
+        # setup server module
+        if map is None or 0 == len(map):
+            print("SERVER REQUIRES AT LEAST ONE MODULE FUNCTION TO OPERATE")
+            return PMIX_ERR_INIT
+        kvkeys = list(map.keys())
+        for key in kvkeys:
+            try:
+                setmodulefn(key, map[key])
+            except KeyError:
+                print("SERVER MODULE FUNCTION ", key, " IS NOT RECOGNIZED")
+                return PMIX_ERR_INIT
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+        if sz > 0:
+            rc = PMIx_server_init(&self.myserver, info, sz)
+        else:
+            rc = PMIx_server_init(&self.myserver, NULL, 0)
+        return rc
+
+    def finalize(self):
+        global stop_progress
+        global progressThread
+
+        # stop progress thread
+        stop_progress = True
+        progressThread.join()
+        # finalize
+        return PMIx_server_finalize()
+
+    def generate_regex(self, hosts:list):
+        cdef char *regex;
+        mycomma = ","
+        myhosts = mycomma.join(hosts)
+        pyhosts = myhosts.encode('ascii')
+        rc = PMIx_generate_regex(pyhosts, &regex)
+        # load regex appropriately into python bytearray
+        ba = pmix_convert_regex(regex)
+        return (rc, ba)
+
+    def generate_ppn(self, procs:list):
+        cdef char *ppn;
+        mysemi = ";"
+        myprocs = mysemi.join(procs)
+        pyprocs = myprocs.encode('ascii')
+        rc = PMIx_generate_ppn(pyprocs, &ppn)
+        if "pmix" == ppn[:4].decode("ascii"):
+            if b'\x00' in ppn:
+                ppn.replace(b'\x00', '')
+            ba = bytearray(ppn)
+        elif "blob" == ppn[:4].decode("ascii"):
+            sz_str    = len(ppn)
+            sz_prefix = 5
+            # extract length of bytearray
+            ppn.split(b'\x00')
+            len_bytearray = ppn[1]
+            length = len(len_bytearray) + sz_prefix + sz_str
+            ba = bytearray(length)
+            index = 0
+            pyppn = <bytes> ppn[:length]
+            while index < length:
+                ba[index] = pyppn[index]
+                index += 1
+        else:
+            # last case with no ':' in string
+            ba = bytearray(ppn)
+        return (rc, ba)
+
+    def generate_cpuset_string(self, cpuset:dict):
+        return (PMIX_ERR_NOT_SUPPORTED, None)
+
+    def generate_locality_string(self, cpuset:dict):
+        return (PMIX_ERR_NOT_SUPPORTED, None)
+
+    # Register a namespace
+    #
+    # @ns [INPUT]
+    #     - Namespace of job (string)
+    #
+    # @nlocalprocs [INPUT]
+    #              - number of local procs for this job (int)
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where each
+    #            dictionary has a key, value, and val_type
+    #            defined as such:
+    #            [{key:y, value:val, val_type:ty}, … ]
+    #
+    def register_nspace(self, ns:str, nlocalprocs:int, dicts:list):
+        cdef pmix_nspace_t nspace
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        global active
+        # convert the args into the necessary C-arguments
+        pmix_copy_nspace(nspace, ns)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+
+        if sz > 0:
+            rc = PMIx_server_register_nspace(nspace, nlocalprocs, info, sz, NULL, NULL)
+        else:
+            rc = PMIx_server_register_nspace(nspace, nlocalprocs, NULL, 0, NULL, NULL)
+        return rc
+
+    # Deregister a namespace
+    #
+    # @ns [INPUT]
+    #     - Namespace of job (string)
+    #
+    def deregister_nspace(self, ns:str):
+        cdef pmix_nspace_t nspace
+        global active
+        # convert the args into the necessary C-arguments
+        pmix_copy_nspace(nspace, ns)
+        PMIx_server_deregister_nspace(nspace, NULL, NULL)
+        return
+
+    # Register resources
+    def register_resources(directives:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, directives)
+        if PMIX_SUCCESS != rc:
+            return rc
+
+        rc = PMIx_server_register_resources(info, sz, NULL, NULL)
+        return rc
+
+    # Deregister resources
+    def deregister_resources(directives:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, directives)
+        if PMIX_SUCCESS != rc:
+            return rc
+
+        rc = PMIx_server_deregister_resources(info, sz, NULL, NULL)
+        return rc
+
+    # Register a client process
+    #
+    # @proc [INPUT]
+    #       - namespace and rank of the client (dict)
+    #
+    # @uid [INPUT]
+    #      - User ID (uid) of the client (int)
+    #
+    # @gid [INPUT]
+    #      - Group ID (gid) of the client (int)
+    #
+    def register_client(self, proc:dict, uid:int, gid:int):
+        global active
+        cdef pmix_proc_t p;
+        pmix_copy_nspace(p.nspace, proc['nspace'])
+        p.rank = proc['rank']
+        rc = PMIx_server_register_client(&p, uid, gid, NULL, NULL, NULL)
+        return rc
+
+    # Deregister a client process
+    #
+    # @proc [INPUT]
+    #       - namespace and rank of the client (dict)
+    #
+    def deregister_client(self, proc:dict):
+        global active
+        cdef pmix_proc_t p;
+        pmix_copy_nspace(p.nspace, proc['nspace'])
+        p.rank = proc['rank']
+        rc = PMIx_server_deregister_client(&p, NULL, NULL)
+        return rc
+
+    # Setup the environment of a child process that is to be forked
+    # by the host
+    #
+    # @proc [INPUT]
+    #       - namespace,rank of client process (tuple)
+    #
+    # @envin [INPUT/OUTPUT]
+    #        - environ of client proc that will be updated
+    #          with PMIx envars (dict)
+    #
+    def setup_fork(self, proc:dict, envin:dict):
+        cdef pmix_proc_t p;
+        cdef char **penv = NULL;
+        cdef unicode pstring
+        pmix_copy_nspace(p.nspace, proc['nspace'])
+        p.rank = proc['rank']
+        # convert the incoming dictionary to an array
+        # of strings
+        rc = PMIx_server_setup_fork(&p, &penv)
+        if PMIX_SUCCESS == rc:
+            # update the incoming dictionary
+            n = 0
+            while NULL != penv[n]:
+                ln = strlen(penv[n])
+                pstring = penv[n].decode('ascii')
+                kv = pstring.split('=')
+                envin[kv[0]] = kv[1]
+                free(penv[n])
+                n += 1
+            free(penv)
+        return rc
+
+    def dmodex_request(self, proc, dataout:dict):
+        global active
+        cdef pmix_proc_t p;
+        pmix_copy_nspace(p.nspace, proc['nspace'])
+        p.rank = proc['rank']
+        active.clear()
+        pybo = (None, 0)
+        rc = PMIx_server_dmodex_request(&p, dmodx_cbfunc, NULL);
+        if PMIX_SUCCESS == rc:
+            active.wait()
+            # transfer the data to the dictionary
+            (data, sz) = active.fetch_data()
+            pybo = (data, sz)
+        return rc, pybo
+
+    def setup_application(self, ns:str, dicts:list):
+        global active
+        cdef pmix_nspace_t nspace;
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        dataout = []
+        pmix_copy_nspace(nspace, ns)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+
+        active.clear()
+        rc = PMIx_server_setup_application(nspace, info, sz, setupapp_cbfunc, NULL);
+        if PMIX_SUCCESS == rc:
+            active.wait()
+            # transfer the data to the dictionary
+            active.fetch_info(dataout)
+        return (rc, dataout)
+
+    def register_attributes(function:str, attrs:list):
+        cdef size_t nattrs
+        cdef char *func
+        cdef char **attarray
+        nattrs    = 0
+        func      = strdup(function)
+
+        if attrs is not None:
+            nattrs = len(attrs)
+            if 0 < nattrs:
+                # allocate and load list of strings into regattrs struct
+                attarray = <char **> PyMem_Malloc((nattrs+1) * sizeof(char*))
+                if not attarray:
+                    return PMIX_ERR_NOMEM
+                rc = pmix_load_argv(attarray, attrs)
+                if PMIX_SUCCESS != rc:
+                    PyMem_Free(attarray)
+                    return rc
+            else:
+                return PMIX_SUCCESS
+        else:
+            return PMIX_SUCCESS
+
+        # call Server API
+        rc = PMIx_Register_attributes(func, attarray)
+
+        if 0 < nattrs:
+            PyMem_Free(attarray)
+        if func != NULL:
+            PyMem_Free(func)
+        return PMIX_SUCCESS
+
+    def collect_inventory(pydirs:list):
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef size_t ndirs
+        ndirs   = 0
+        dataout = []
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+
+        # call the API
+        active.clear()
+        rc = PMIx_server_collect_inventory(directives, ndirs,
+                                           collectinventory_cbfunc, NULL)
+        if PMIX_SUCCESS == rc:
+            active.wait()
+            # transfer the data to the dictionary
+            active.fetch_info(dataout)
+        return (rc, dataout)
+
+    def deliver_inventory(pyinfo:list, pydirs:list):
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ndirs
+        cdef size_t ninfo
+        ndirs   = 0
+        ninfo   = 0
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+        # call the API
+        rc = PMIx_server_deliver_inventory(info, ninfo, directives, ndirs,
+                                           NULL, NULL)
+        return rc
+
+    def setup_local_support(self, ns:str, ilist:list):
+        global active
+        cdef pmix_nspace_t nspace;
+        cdef pmix_info_t *info
+        cdef size_t sz
+        pmix_copy_nspace(nspace, ns)
+        if ilist is not None:
+            sz = len(ilist)
+            info = <pmix_info_t*> PyMem_Malloc(sz * sizeof(pmix_info_t))
+            if not info:
+                return PMIX_ERR_NOMEM
+            n = 0
+            for d in ilist:
+                pykey = str(d['key'])
+                pmix_copy_key(info[n].key, pykey)
+                # the value also needs to be transferred
+                print("SETUP LOCAL ", info[n].key, " TYPE ", PMIx_Data_type_string(d['val_type']))
+                val = {'value':d['value'], 'val_type':d['val_type']}
+                # send dict of value and val_type to pmix_load_value
+                pmix_load_value(&info[n].value, val)
+                n += 1
+                break
+        else:
+            info = NULL
+            sz = 0
+        rc = PMIx_server_setup_local_support(nspace, info, sz, NULL, NULL);
+        if PMIX_SUCCESS == rc:
+            active.wait()
+        return rc
+
+    def iof_deliver(pysrc:dict, pychannel:int, pydata:dict, pydirs:list):
+        cdef pmix_proc_t *source
+        cdef pmix_iof_channel_t channel
+        cdef pmix_byte_object_t *bo
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef size_t ndirs
+        source  = NULL
+        ndirs   = 0
+        channel = pychannel
+
+        # convert pysrc to pmix_proc_t
+        pmix_copy_nspace(source[0].nspace, pysrc['nspace'])
+        source[0].rank = pysrc['rank']
+
+        # convert pydata to pmix_byte_object_t
+        bo = <pmix_byte_object_t*>PyMem_Malloc(sizeof(pmix_byte_object_t))
+        if not bo:
+            return PMIX_ERR_NOMEM
+        data = bytes(pydata['bytes'], 'ascii')
+        bo.size = sizeof(data)
+        bo.bytes = <char*> PyMem_Malloc(bo.size)
+        if not bo.bytes:
+            return PMIX_ERR_NOMEM
+        pyptr = <const char*>data
+        memcpy(bo.bytes, pyptr, bo.size)
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+
+        # call API
+        rc = PMIx_server_IOF_deliver(source, channel, bo, directives, ndirs,
+                                     NULL, NULL)
+        return rc
+
+    def define_process_set(members:list, name:str):
+        cdef pmix_proc_t *procs
+        cdef size_t nprocs
+        nprocs = 0
+
+        # convert set name
+        pyset = name.encode('ascii')
+        # convert list of procs to array of pmix_proc_t's
+        if members is None:
+            return PMIX_ERR_BAD_PARAM
+        nprocs = len(members)
+        procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+        if not procs:
+            return PMIX_ERR_NOMEM
+        rc = pmix_load_procs(procs, members)
+        if PMIX_SUCCESS != rc:
+            pmix_free_procs(procs, nprocs)
+            return rc
+        # define the set
+        rc = PMIx_server_define_process_set(procs, nprocs, pyset)
+        pmix_free_procs(procs, nprocs)
+        return rc
+
+    def delete_process_set(name:str):
+
+        # convert set name
+        pyset = name.encode('ascii')
+        # delete the set
+        rc = PMIx_server_delete_process_set(pyset)
+        return rc
+
+
+cdef int clientconnected(pmix_proc_t *proc, void *server_object,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'clientconnected' in keys:
+        if not proc:
+            return PMIX_ERR_BAD_PARAM
+        myproc = []
+        pmix_unload_procs(proc, 1, myproc)
+        rc = pmixservermodule['clientconnected'](myproc[0])
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int clientfinalized(pmix_proc_t *proc, void *server_object,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'clientfinalized' in keys:
+        if not proc:
+            return PMIX_ERR_BAD_PARAM
+        myproc = []
+        pmix_unload_procs(proc, 1, myproc)
+        rc = pmixservermodule['clientfinalized'](myproc[0])
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int clientaborted(const pmix_proc_t *proc, void *server_object,
+                       int status, const char msg[],
+                       pmix_proc_t procs[], size_t nprocs,
+                       pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'abort' in keys:
+        args = {}
+        myproc = []
+        myprocs = []
+        # convert the caller's name
+        pmix_unload_procs(proc, 1, myproc)
+        args['caller'] = myproc[0]
+        # record the status
+        args['status'] = status
+        # record the msg, if given
+        if NULL != msg:
+            args['msg'] = str(msg)
+        # convert any provided array of procs to be aborted
+        if NULL != procs:
+            pmix_unload_procs(procs, nprocs, myprocs)
+            args['targets'] = myprocs
+        # upcall it
+        rc = pmixservermodule['abort'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int fencenb(const pmix_proc_t procs[], size_t nprocs,
+                 const pmix_info_t info[], size_t ninfo,
+                 char *data, size_t ndata,
+                 pmix_modex_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'fencenb' in keys:
+        args = {}
+        myprocs = []
+        blist = []
+        ilist = []
+        barray = None
+
+        if NULL == procs:
+            myprocs.append({'nspace': myname.nspace, 'rank': PMIX_RANK_WILDCARD})
+        else:
+            pmix_unload_procs(procs, nprocs, myprocs)
+        args['procs'] = myprocs
+        if NULL != info:
+            rc = pmix_unload_info(info, ninfo, ilist)
+            if PMIX_SUCCESS != rc:
+                return rc
+            args['directives'] = ilist
+        if NULL != data:
+            pmix_unload_bytes(data, ndata, blist)
+            barray = bytearray(blist)
+            args['data'] = barray
+        rc, ret_data = pmixservermodule['fencenb'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    data  = strdup(ret_data)
+    ndata = len(ret_data)
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("fence")
+        mycaddy.status = PMIX_SUCCESS
+        mycaddy.bo.bytes = data
+        mycaddy.bo.size = ndata
+        mycaddy.modex = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int directmodex(const pmix_proc_t *proc,
+                     const pmix_info_t info[], size_t ninfo,
+                     pmix_modex_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'directmodex' in keys:
+        args = {}
+        myprocs = []
+        ilist = []
+        pmix_unload_procs(proc, 1, myprocs)
+        args['proc'] = myprocs[0]
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc, ret_data = pmixservermodule['directmodex'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    cdef const char* data
+    cdef size_t ndata
+    data  = strdup(ret_data)
+    ndata = len(ret_data)
+    global eventQueue
+
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("directmodex")
+        mycaddy.status = PMIX_SUCCESS
+        mycaddy.data = data
+        mycaddy.ndata = ndata
+        mycaddy.modex = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int publish(const pmix_proc_t *proc,
+                 const pmix_info_t info[], size_t ninfo,
+                 pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'publish' in keys:
+        args = {}
+        myprocs = []
+        ilist = []
+        pmix_unload_procs(proc, 1, myprocs)
+        args['proc'] = myprocs[0]
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+        args['directives'] = ilist
+        rc = pmixservermodule['publish'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int lookup(const pmix_proc_t *proc, char **keys,
+                const pmix_info_t info[], size_t ninfo,
+                pmix_lookup_cbfunc_t cbfunc, void *cbdata) with gil:
+    srvkeys = pmixservermodule.keys()
+    if 'lookup' in srvkeys:
+        args = {}
+        pdata   = []
+        myprocs = []
+        ilist = []
+        pykeys = []
+        pmix_unload_procs(proc, 1, myprocs)
+        args['proc'] = myprocs[0]
+        n = 0
+        while NULL != keys[n]:
+            pykeys.append(keys[n])
+            n += 1
+        args['keys'] = pykeys
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc, pdata = pmixservermodule['lookup'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+
+    # convert the list of dictionaries to array of
+    # pmix_pdata_t structs
+    cdef pmix_pdata_t *pd;
+    cdef size_t ndata;
+    if pdata is not None:
+        ndata = len(pdata)
+        if 0 < ndata:
+            pd = <pmix_pdata_t*> PyMem_Malloc(ndata * sizeof(pmix_pdata_t))
+            if not pdata:
+                return PMIX_ERR_NOMEM
+            prc = pmix_load_pdata(myprocs[0], pd, pdata)
+            if PMIX_SUCCESS != prc:
+                pmix_free_pdata(pd, ndata)
+                return prc
+        else:
+            pd = NULL
+    else:
+        pd = NULL
+
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("lookup")
+        mycaddy.status = PMIX_SUCCESS
+        mycaddy.pdata = pd
+        mycaddy.ndata = ndata
+        mycaddy.lookup = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int unpublish(const pmix_proc_t *proc, char **keys,
+                   const pmix_info_t info[], size_t ninfo,
+                   pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    srvkeys = pmixservermodule.keys()
+    if 'unpublish' in srvkeys:
+        args = {}
+        myprocs = []
+        ilist = []
+        pykeys = []
+        pmix_unload_procs(proc, 1, myprocs)
+        args['proc'] = myprocs[0]
+        if NULL != keys:
+            pmix_unload_argv(keys, pykeys)
+            args['keys'] = pykeys
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc = pmixservermodule['unpublish'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int spawn(const pmix_proc_t *proc,
+               const pmix_info_t job_info[], size_t ninfo,
+               const pmix_app_t apps[], size_t napps,
+               pmix_spawn_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'spawn' in keys:
+        args = {}
+        myprocs = []
+        ilist = []
+        pyapps = []
+        pmix_unload_procs(proc, 1, myprocs)
+        args['proc'] = myprocs[0]
+        if NULL != job_info:
+            pmix_unload_info(job_info, ninfo, ilist)
+            args['jobinfo'] = ilist
+        pmix_unload_apps(apps, napps, pyapps)
+        args['apps'] = pyapps
+        rc, nspace = pmixservermodule['spawn'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+
+    cdef pmix_nspace_t ns
+    pmix_copy_nspace(ns, nspace)
+
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("spawn")
+        mycaddy.status = rc
+        mycaddy.nspace = ns
+        mycaddy.spawn  = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int connect(const pmix_proc_t procs[], size_t nprocs,
+                 const pmix_info_t info[], size_t ninfo,
+                 pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'connect' in keys:
+        args = {}
+        myprocs = []
+        ilist = []
+        if NULL != procs:
+            pmix_unload_procs(procs, nprocs, myprocs)
+            args['procs'] = myprocs
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc = pmixservermodule['connect'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int disconnect(const pmix_proc_t procs[], size_t nprocs,
+                    const pmix_info_t info[], size_t ninfo,
+                    pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'disconnect' in keys:
+        args = {}
+        myprocs = []
+        ilist = []
+        if NULL != procs:
+            pmix_unload_procs(procs, nprocs, myprocs)
+            args['procs'] = myprocs
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc = pmixservermodule['disconnect'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int registerevents(pmix_status_t *codes, size_t ncodes,
+                        const pmix_info_t info[], size_t ninfo,
+                        pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'registerevents' in keys:
+        args = {}
+        mycodes = []
+        ilist = []
+        if NULL != codes:
+            n = 0
+            while n < ncodes:
+                mycodes.append(codes[n])
+                n += 1
+            args['codes'] = mycodes
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc = pmixservermodule['registerevents'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int deregisterevents(pmix_status_t *codes, size_t ncodes,
+                          pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'deregisterevents' in keys:
+        args = {}
+        mycodes = []
+        if NULL != codes:
+            n = 0
+            while n < ncodes:
+                mycodes.append(codes[n])
+                n += 1
+            args['codes'] = mycodes
+        rc = pmixservermodule['deregisterevents'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int notifyevent(pmix_status_t code,
+                     const pmix_proc_t *source,
+                     pmix_data_range_t drange,
+                     pmix_info_t info[], size_t ninfo,
+                     pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'notifyevent' in keys:
+        args = {}
+        ilist = []
+        myproc = []
+        args['code'] = code
+        pmix_unload_procs(source, 1, myproc)
+        args['source'] = myproc[0]
+        args['range'] = drange
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc = pmixservermodule['notifyevent'](args)
+    else:
+        return PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int query(pmix_proc_t *source,
+               pmix_query_t *queries, size_t nqueries,
+               pmix_info_cbfunc_t cbfunc,
+               void *cbdata) with gil:
+    pyqueries = []
+    keys = pmixservermodule.keys()
+    if 'query' in keys:
+        args = {}
+        myproc = []
+        if NULL == queries or NULL == source:
+            return PMIX_ERR_BAD_PARAM
+        pmix_unload_queries(queries, nqueries, pyqueries)
+        args['queries'] = pyqueries
+        pmix_unload_procs(source, 1, myproc)
+        args['source'] = myproc[0]
+        rc,results = pmixservermodule['query'](args, pyqueries)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+        results = []
+
+    # convert the results list of dictionaries to array of
+    # pmix_info_t structs
+    cdef pmix_info_t *info;
+    cdef size_t ninfo
+    if results is not None and 0 < len(results):
+        ninfo = len(results)
+        if 0 < nqueries:
+            info = <pmix_info_t*> PyMem_Malloc(ninfo * sizeof(pmix_info_t))
+            if not info:
+                return PMIX_ERR_NOMEM
+            prc = pmix_load_info(info, results)
+            if PMIX_SUCCESS != prc:
+                pmix_free_info(info, ninfo)
+                return prc
+        else:
+            info = NULL
+    else:
+        info = NULL
+
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("query")
+        mycaddy.status = rc
+        mycaddy.info = info
+        mycaddy.ndata = nqueries
+        mycaddy.query = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef void toolconnected(pmix_info_t *info, size_t ninfo,
+                        pmix_tool_connection_cbfunc_t cbfunc,
+                        void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'toolconnected' in keys:
+        args = {}
+        ilist = {}
+        if NULL != info:
+            pmix_unload_info(info, ninfo, ilist)
+            args['directives'] = ilist
+        rc, ret_proc = pmixservermodule['toolconnected'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. So we start
+    # a new thread on a timer that should execute a
+    # callback after the funciton returns
+    cdef pmix_proc_t *proc
+    proc = NULL
+    pmix_copy_nspace(proc[0].nspace, ret_proc['nspace'])
+    proc[0].rank = ret_proc['rank']
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("toolconnected")
+        mycaddy.status = rc
+        mycaddy.proc = proc
+        mycaddy.toolconnected = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+    return
+
+cdef void log(const pmix_proc_t *client,
+              const pmix_info_t data[], size_t ndata,
+              const pmix_info_t directives[], size_t ndirs,
+              pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'log' in keys:
+        args = {}
+        ilist = []
+        myproc = []
+        mydirs = []
+        if NULL == client:
+            return
+        pmix_unload_procs(client, 1, myproc)
+        args['source'] = myproc[0]
+        if NULL != data:
+            pmix_unload_info(data, ndata, ilist)
+            args['data'] = ilist
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        rc = pmixservermodule['log'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return
+
+cdef int allocate(const pmix_proc_t *client,
+                  pmix_alloc_directive_t action,
+                  const pmix_info_t directives[], size_t ndirs,
+                  pmix_info_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'allocate' in keys:
+        args = {}
+        myproc = []
+        mydirs = {}
+        keyvals = []
+        if NULL == client:
+            return PMIX_ERR_BAD_PARAM
+        pmix_unload_procs(client, 1, myproc)
+        args['source'] = myproc[0]
+        args['action'] = action
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, keyvals)
+            args['directives'] = keyvals
+        rc, refarginfo = pmixservermodule['allocate'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. So we start
+    # a new thread on a timer that should execute a
+    # callback after the funciton returns
+    cdef pmix_info_t *info
+    cdef pmix_info_t **info_ptr
+    cdef size_t ninfo = 0
+    info              = NULL
+    info_ptr          = &info
+    prc = pmix_alloc_info(info_ptr, &ninfo, refarginfo)
+    if PMIX_SUCCESS != prc:
+        print("Error transferring info to C:", prc)
+        return prc
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("allocate")
+        mycaddy.status = PMIX_SUCCESS
+        mycaddy.info = info
+        mycaddy.ndata = ninfo
+        mycaddy.allocate = cbfunc
+        mycaddy.cbdata = cbdata
+        mycaddy.release_fn = NULL
+        mycaddy.notification_cbdata = NULL
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int jobcontrol(const pmix_proc_t *requestor,
+                    const pmix_proc_t targets[], size_t ntargets,
+                    const pmix_info_t directives[], size_t ndirs,
+                    pmix_info_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'jobcontrol' in keys:
+        args = {}
+        myproc = []
+        mytargets = []
+        mydirs = {}
+        if NULL != requestor:
+            pmix_unload_procs(requestor, 1, myproc)
+            args['source'] = myproc[0]
+        if NULL != targets:
+            pmix_unload_procs(targets, ntargets, mytargets)
+            args['targets'] = mytargets
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        rc = pmixservermodule['jobcontrol'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int monitor(const pmix_proc_t *requestor,
+                 const pmix_info_t *monitor, pmix_status_t error,
+                 const pmix_info_t directives[], size_t ndirs,
+                 pmix_info_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'monitor' in keys:
+        args = {}
+        mymon = {}
+        myproc = []
+        mydirs = {}
+        blist = []
+        if NULL == monitor:
+            return PMIX_ERR_BAD_PARAM
+        if NULL != requestor:
+            pmix_unload_procs(requestor, 1, myproc)
+            args['source'] = myproc[0]
+        pmix_unload_info(monitor, 1, mymon)
+        args['monitor'] = mymon
+        args['error'] = error
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        rc = pmixservermodule['monitor'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int getcredential(const pmix_proc_t *proc,
+                       const pmix_info_t directives[], size_t ndirs,
+                       pmix_credential_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'getcredential' in keys:
+        args = {}
+        myproc = []
+        mydirs = {}
+        if NULL != proc:
+            pmix_unload_procs(proc, 1, myproc)
+            args['source'] = myproc[0]
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        status, pycred, pyinfo = pmixservermodule['getcredential'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. So we start
+    # a new thread on a timer that should execute a
+    # callback after the function returns
+    cdef pmix_info_t *info
+    cdef pmix_info_t **info_ptr
+    cdef size_t ninfo = 0
+    info              = NULL
+    info_ptr          = &info
+    rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+
+    # convert pycred to pmix_byte_object_t
+    bo = <pmix_byte_object_t*>PyMem_Malloc(sizeof(pmix_byte_object_t))
+    if not bo:
+        return PMIX_ERR_NOMEM
+    cred = bytes(pycred['bytes'], 'ascii')
+    bo.size = sizeof(cred)
+    bo.bytes = <char*> PyMem_Malloc(bo.size)
+    if not bo.bytes:
+        return PMIX_ERR_NOMEM
+    pyptr = <const char*>cred
+    memcpy(bo.bytes, pyptr, bo.size)
+
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("getcredential")
+        mycaddy.status = status
+        mycaddy.info = info
+        mycaddy.ndata = ninfo
+        mycaddy.cred = bo
+        mycaddy.getcredential = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int validatecredential(const pmix_proc_t *proc,
+                            const pmix_byte_object_t *cred,
+                            const pmix_info_t directives[], size_t ndirs,
+                            pmix_validation_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'validatecredential' in keys:
+        args = {}
+        keyvals = {}
+        myproc = []
+        mydirs = {}
+        blist = []
+        pycred = {}
+        if NULL != proc:
+            pmix_unload_procs(proc, 1, myproc)
+            args['source'] = myproc[0]
+        if NULL != cred:
+            pmix_unload_bytes(cred[0].bytes, cred[0].size, blist)
+            barray = bytearray(blist)
+            pycred['bytes'] = barray
+            pycred['size'] = cred[0].size
+            args['credential'] = pycred
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        status, pyinfo = pmixservermodule['validatecredential'](args)
+        rc = status
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. So we start
+    # a new thread on a timer that should execute a
+    # callback after the function returns
+    cdef pmix_info_t *info
+    cdef pmix_info_t **info_ptr
+    cdef size_t ninfo = 0
+    info              = NULL
+    info_ptr          = &info
+    prc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+    if PMIX_SUCCESS != prc:
+        print("Error converting info to C:", prc)
+        return prc
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # threadshift so we can generate the callback safely
+    global eventQueue
+    if PMIX_SUCCESS == rc or PMIX_OPERATION_SUCCEEDED == rc:
+        mycaddy = <pmix_pyshift_t*> PyMem_Malloc(sizeof(pmix_pyshift_t))
+        mycaddy.op = strdup("validationcredential")
+        mycaddy.status = status
+        mycaddy.info = info
+        mycaddy.ndata = ninfo
+        mycaddy.validationcredential = cbfunc
+        mycaddy.cbdata = cbdata
+        cb = PyCapsule_New(mycaddy, NULL, NULL)
+        # push the results into the queue to return them
+        # to the PMIx library
+        eventQueue.put(cb)
+        return PMIX_SUCCESS
+    return rc
+
+cdef int iofpull(const pmix_proc_t procs[], size_t nprocs,
+                 const pmix_info_t directives[], size_t ndirs,
+                 pmix_iof_channel_t channels,
+                 pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'iofpull' in keys:
+        args = {}
+        keyvals = {}
+        myprocs = []
+        mydirs = {}
+        pychannels = int(channels)
+        args['channels'] = channels
+        if NULL != procs:
+            pmix_unload_procs(procs, nprocs, myprocs)
+            args['sources'] = myprocs
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        rc = pmixservermodule['iofpull'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+cdef int pushstdin(const pmix_proc_t *source,
+                   const pmix_proc_t targets[], size_t ntargets,
+                   const pmix_info_t directives[], size_t ndirs,
+                   const pmix_byte_object_t *bo,
+                   pmix_op_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'pushstdin' in keys:
+        args = {}
+        keyvals = {}
+        myproc = []
+        mytargets = []
+        mydirs = {}
+        blist = []
+        pyload = {}
+        if NULL != source:
+            pmix_unload_procs(source, 1, myproc)
+            args['source'] = myproc[0]
+        if NULL != targets:
+            pmix_unload_procs(targets, ntargets, mytargets)
+            args['targets'] = mytargets
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        if NULL != bo:
+            pmix_unload_bytes(bo[0].bytes, bo[0].size, blist)
+            barray = bytearray(blist)
+            pyload['bytes'] = barray
+            pyload['size'] = bo[0].size
+            args['payload'] = pyload
+        rc = pmixservermodule['pushstdin'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    # we cannot execute a callback function here as
+    # that would cause PMIx to lockup. Likewise, the
+    # Python function we called can't do it as it
+    # would require them to call a C-function. So
+    # if they succeeded in processing this request,
+    # we return a PMIX_OPERATION_SUCCEEDED status
+    # that let's the underlying PMIx library know
+    # the situation so it can generate its own
+    # callback
+    if PMIX_SUCCESS == rc:
+        rc = PMIX_OPERATION_SUCCEEDED
+    return rc
+
+# TODO: This function requires that the server execute the
+# provided callback function to return the group info, and
+# it is not allowed to do so until _after_ it returns from
+# this upcall. We'll need to figure out a way to 'save' the
+# cbfunc until the server calls us back, possibly by passing
+# an appropriate caddy object in 'cbdata'
+cdef int group(pmix_group_operation_t op, char grp[],
+               const pmix_proc_t procs[], size_t nprocs,
+               const pmix_info_t directives[], size_t ndirs,
+               pmix_info_cbfunc_t cbfunc, void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'group' in keys:
+        args = {}
+        keyvals = {}
+        myprocs = []
+        mydirs = {}
+        args['op'] = op
+        args['group'] = str(grp)
+        pmix_unload_procs(procs, nprocs, myprocs)
+        args['procs'] = myprocs
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        rc = pmixservermodule['group'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    return rc
+
+cdef int fabric(const pmix_proc_t *requestor,
+                pmix_fabric_operation_t op,
+                const pmix_info_t directives[],
+                size_t ndirs,
+                pmix_info_cbfunc_t cbfunc,
+                void *cbdata) with gil:
+    keys = pmixservermodule.keys()
+    if 'fabric' in keys:
+        args = {}
+        keyvals = {}
+        myprocs = []
+        mydirs = {}
+        args['op'] = op
+        if NULL != directives:
+            pmix_unload_info(directives, ndirs, mydirs)
+            args['directives'] = mydirs
+        rc = pmixservermodule['fabric'](args)
+    else:
+        rc = PMIX_ERR_NOT_SUPPORTED
+    return rc
+
+cdef class PMIxTool(PMIxServer):
+    def __cinit__(self):
+        memset(self.myproc.nspace, 0, sizeof(self.myproc.nspace))
+        self.myproc.rank = PMIX_RANK_UNDEF
+
+    # Initialize the PMIx tool library
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where each
+    #            dictionary has a key, value, and val_type
+    #            defined as such:
+    #            [{key:y, value:val, val_type:ty}, … ]
+    def init(self, dicts:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        global myname
+        global progressThread
+
+        # start the event handler progress thread
+        progressThread.start()
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+
+        if sz > 0:
+            rc = PMIx_tool_init(&self.myproc, info, sz)
+            pmix_free_info(info, sz)
+        else:
+            rc = PMIx_tool_init(&self.myproc, NULL, 0)
+        if PMIX_SUCCESS == rc:
+            # convert the returned name
+            myname = {'nspace': str(self.myproc.nspace), 'rank': self.myproc.rank}
+        return rc, myname
+
+    # Finalize the tool library
+    def finalize(self):
+        global stop_progress
+
+        # stop progress thread
+        stop_progress = True
+        progressThread.join(timeout=1)
+        # finalize
+        rc = PMIx_tool_finalize()
+        return rc
+
+    # Disconnect from a server
+    def disconnect(server:dict):
+        cdef pmix_proc_t srvr
+
+        # convert the server name
+        pmix_copy_nspace(srvr.nspace, server['nspace'])
+        srvr.rank = server['rank']
+
+        # perform disconnect
+        rc = PMIx_tool_disconnect(&srvr);
+        return rc
+
+    # Connect to a server
+    #
+    # @dicts [INPUT]
+    #          - a list of dictionaries, where each
+    #            dictionary has a key, value, and val_type
+    #            defined as such:
+    #            [{key:y, value:val, val_type:ty}, … ]
+    def attach_to_server(self, dicts:list):
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t sz
+        cdef pmix_proc_t srvr
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &sz, dicts)
+
+        if sz > 0:
+            rc = PMIx_tool_attach_to_server(&self.myproc, &srvr, info, sz)
+            pmix_free_info(info, sz)
+        else:
+            rc = PMIx_tool_attach_to_server(&self.myproc, &srvr, NULL, 0)
+        if PMIX_SUCCESS == rc:
+            # convert the returned name
+            myname = {'nspace': str(self.myproc.nspace), 'rank': self.myproc.rank}
+            mysrvr = {'nspace': str(srvr.nspace), 'rank': srvr.rank}
+        return rc, myname, mysrvr
+
+    def get_servers(self):
+        cdef pmix_proc_t *servers
+        cdef size_t nservers
+
+        pysrvrs = []
+        rc = PMIx_tool_get_servers(&servers, &nservers)
+        if PMIX_SUCCESS != rc:
+            return rc, pysrvrs
+        rc = pmix_unload_procs(servers, nservers, pysrvrs)
+        PyMem_Free(servers)
+        return rc, pysrvrs
+
+    def set_server(self, server:dict, pyinfo:list):
+        cdef pmix_proc_t srvr
+        cdef pmix_info_t *info
+        cdef pmix_info_t **info_ptr
+        cdef size_t ninfo
+
+        # convert the server name
+        pmix_copy_nspace(srvr.nspace, server['nspace'])
+        srvr.rank = server['rank']
+
+        # allocate and load pmix info structs from python list of dictionaries
+        info_ptr = &info
+        rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+        if PMIX_SUCCESS != rc:
+            if 0 < ninfo:
+                pmix_free_info(info, ninfo)
+            return rc
+
+        # perform op
+        rc = PMIx_tool_set_server(&srvr, info, ninfo);
+        if 0 < ninfo:
+            pmix_free_info(info, ninfo)
+        return rc
+
+    def iof_pull(self, pyprocs:list, iof_channel:int, pydirs:list, hdlr):
+        cdef pmix_proc_t *procs
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef pmix_iof_channel_t channel
+        cdef size_t ndirs
+        cdef size_t nprocs
+        nprocs      = 0
+        ndirs       = 0
+        channel     = iof_channel
+
+        # convert list of procs to array of pmix_proc_t's
+        if pyprocs is not None:
+            nprocs = len(pyprocs)
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM, -1
+            rc = pmix_load_procs(procs, pyprocs)
+            if PMIX_SUCCESS != rc:
+                pmix_free_procs(procs, nprocs)
+                return rc, -1
+        else:
+            nprocs = 1
+            procs = <pmix_proc_t*> PyMem_Malloc(nprocs * sizeof(pmix_proc_t))
+            if not procs:
+                return PMIX_ERR_NOMEM, -1
+            pmix_copy_nspace(procs[0].nspace, self.myproc.nspace)
+            procs[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+
+        # Call the library
+        rc = PMIx_IOF_pull(procs, nprocs, directives, ndirs, channel,
+                           pyiofhandler,
+                           NULL, NULL)
+        if 0 < nprocs:
+            pmix_free_procs(procs, nprocs)
+        if 0 < ndirs:
+            pmix_free_info(directives, ndirs)
+
+        # if rc < 0, then there was an error
+        if 0 > rc:
+            return rc, -1
+
+        # otherwise, this is our ref ID for this hdlr
+        myhdlrs.append({'refid': rc, 'hdlr': hdlr})
+        refid = rc
+        rc = PMIX_SUCCESS
+        return rc, refid
+
+    def iof_deregister(self, regid:int, pydirs:list):
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef size_t ndirs
+        cdef size_t iofhdlr
+        ndirs       = 0
+        iofhdlr     = regid
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+
+        # call the library
+        rc = PMIx_IOF_deregister(iofhdlr, directives, ndirs, NULL, NULL)
+        if 0 < ndirs:
+            pmix_free_info(directives, ndirs)
+        # remove our local hdlr
+        found = False
+        n = 0
+        for h in myhdlrs and not found:
+            try:
+                if iofhdlr == h['refid']:
+                    found = True
+                    del myhdlrs[n]
+                else :
+                    n = n + 1
+            except:
+                pass
+        return rc
+
+    def iof_push(self, pytargets:list, data:dict, pydirs:list):
+        cdef pmix_info_t *directives
+        cdef pmix_info_t **directives_ptr
+        cdef pmix_byte_object_t *bo
+        cdef size_t ndirs
+        cdef pmix_proc_t *targets
+        ntargets    = 0
+        ndirs       = 0
+
+        # convert data to pmix_byte_object_t
+        bo = <pmix_byte_object_t*>PyMem_Malloc(sizeof(pmix_byte_object_t))
+        if not bo:
+            return PMIX_ERR_NOMEM
+        cred = bytes(data['bytes'], 'ascii')
+        bo.size = sizeof(cred)
+        bo.bytes = <char*> PyMem_Malloc(bo.size)
+        if not bo.bytes:
+            return PMIX_ERR_NOMEM
+        pyptr = <const char*>cred
+        memcpy(bo.bytes, pyptr, bo.size)
+
+        # convert list of proc targets to array of pmix_proc_t's
+        if pytargets is not None:
+            ntargets = len(pytargets)
+            targets = <pmix_proc_t*> PyMem_Malloc(ntargets * sizeof(pmix_proc_t))
+            if not targets:
+                return PMIX_ERR_NOMEM
+            rc = pmix_load_procs(targets, pytargets)
+            if PMIX_SUCCESS != rc:
+                pmix_free_procs(targets, ntargets)
+                return rc
+        else:
+            ntargets = 1
+            targets = <pmix_proc_t*> PyMem_Malloc(ntargets * sizeof(pmix_proc_t))
+            if not targets:
+                return PMIX_ERR_NOMEM
+            pmix_copy_nspace(targets[0].nspace, self.myproc.nspace)
+            targets[0].rank = PMIX_RANK_WILDCARD
+
+        # allocate and load pmix info structs from python list of dictionaries
+        directives_ptr = &directives
+        rc = pmix_alloc_info(directives_ptr, &ndirs, pydirs)
+
+        # Call the library
+        rc = PMIx_IOF_push(targets, ntargets, bo, directives, ndirs, NULL, NULL)
+        if 0 < ntargets:
+            pmix_free_procs(targets, ntargets)
+        if 0 < ndirs:
+            pmix_free_info(directives, ndirs)
+        return rc
diff -Nru pmix-3.2.2~rc1/bindings/python/setup.py pmix-4.0.0/bindings/python/setup.py
--- pmix-3.2.2~rc1/bindings/python/setup.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/setup.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,49 @@
+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Build import cythonize
+from sys import platform, maxsize, version_info
+import os
+from Cython.Compiler.Main import default_options
+default_options['emit_linenums'] = True
+from subprocess import check_output, CalledProcessError
+
+def getVersion():
+    dir = os.path.dirname(__file__)
+    vers_path = os.path.join(dir, '../../include', 'pmix_version.h')
+    with open(vers_path) as verFile:
+        lines = verFile.readlines()
+        for l in lines:
+            if 'MAJOR' in l:
+                major = l.split()[2]
+                major = major[:-1]
+            elif 'MINOR' in l:
+                minor = l.split()[2]
+                minor = minor[:-1]
+            elif 'RELEASE' in l:
+                release = l.split()[2]
+                release = release[:-1]
+        vers = [major, minor, release]
+        version = ".".join(vers)
+        return version
+
+setup(
+    name = 'pypmix',
+    version = getVersion(),
+    url = 'https://pmix.org',
+    license = '3-clause BSD',
+    author = 'Ralph H. Castain',
+    author_email = 'ralph.h.castain@intel.com',
+    description = 'Python bindings for PMIx',
+    classifiers = [
+            'Development Status :: 1 - Under Construction',
+            'Intended Audience :: Developers',
+            'Topic :: HPC :: Parallel Programming :: System Management',
+            'License :: 3-clause BSD',
+            'Programming Language :: Python :: 3.4',
+            'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6'],
+    keywords = 'PMI PMIx HPC MPI SHMEM',
+    platforms = 'any',
+    ext_modules = cythonize([Extension("pmix", ["pmix.pyx"], libraries=["pmix"])],
+                            compiler_directives={'language_level': 3})
+)
diff -Nru pmix-3.2.2~rc1/bindings/python/tests/cython/cython_test_functions.pyx pmix-4.0.0/bindings/python/tests/cython/cython_test_functions.pyx
--- pmix-3.2.2~rc1/bindings/python/tests/cython/cython_test_functions.pyx	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/tests/cython/cython_test_functions.pyx	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,105 @@
+# test cython code
+#file: cython_test_functions.pyx
+
+# TODO: add lots more types to load/test
+
+pyinfo = [{'key': PMIX_EVENT_HDLR_NAME, 'value': 'SIMPCLIENT-MODEL',
+           'val_type': PMIX_STRING}]
+darray = {'type':PMIX_INFO, 'array':[{'key':PMIX_ALLOC_FABRIC_ID, 'value':'SIMPSCHED.net', 'val_type':PMIX_STRING},
+                                     {'key':PMIX_ALLOC_FABRIC_SEC_KEY, 'value':'T', 'val_type':PMIX_BOOL},
+                                     {'key':PMIX_SETUP_APP_ENVARS, 'value':'T', 'val_type':PMIX_BOOL}]
+         }
+# test putting PMIX_DATA_ARRAY into a pmix info data array
+darray2 = {'type':PMIX_INFO, 'array':[{'key':PMIX_ALLOC_FABRIC_ID, 'value':'SIMPSCHED.net', 'val_type':PMIX_STRING},
+                                     {'key':PMIX_ALLOC_FABRIC_SEC_KEY, 'value':'T', 'val_type':PMIX_BOOL},
+                                     {'key':'foo', 'value':darray, 'val_type':PMIX_DATA_ARRAY}]
+         }
+str_darray = {'type': PMIX_STRING, 'array':['abc', 'def', 'efg']}
+size_darray = {'type': PMIX_SIZE, 'array':[45, 46, 47]}
+pid_darray = {'type': PMIX_SIZE, 'array':[3, 4, 5]}
+timeval_darray = {'type': PMIX_TIMEVAL, 'array':[{'sec':2, 'usec':3}, {'sec':1, 'usec':2}]}
+bool_darray = {'type': PMIX_BOOL, 'array':['False', 'True']}
+byte_darray = {'type': PMIX_BYTE, 'array':[1, 1]}
+proc_darray = {'type': PMIX_PROC, 'array':[{'nspace':'testnspace', 'rank':0}, {'nspace':'newnspace', 'rank':1}]}
+bo_darray = {'type': PMIX_BYTE_OBJECT, 'array':[{'bytes':bytearray(1), 'size':1}, {'bytes':bytearray(1), 'size':1}]}
+proc_info_darray = {'type': PMIX_PROC_INFO, 'array':[{'proc': {'nspace': 'fakenspace', 'rank': 0}, 'hostname': 'myhostname', 'executable': 'testexec', 'pid': 2, 'exitcode': 0, 'state': 0}]}
+env_darray = {'type': PMIX_ENVAR, 'array': [{'envar': 'TEST_ENVAR', 'value': 'TEST_VAL', 'separator': ':'}]}
+pyregex = 'pmix[test[3:0-2]]'.encode('ascii')
+values = [{'value': 'True', 'val_type': PMIX_BOOL},
+          {'value': 1, 'val_type': PMIX_BYTE},
+          {'value': "foo", 'val_type': PMIX_STRING},
+          {'value': 45, 'val_type': PMIX_SIZE},
+          {'value': 3, 'val_type': PMIX_PID},
+          {'value': 11, 'val_type': PMIX_INT},
+          {'value': 2, 'val_type': PMIX_INT8},
+          {'value': 127, 'val_type': PMIX_INT16},
+          {'value': 100, 'val_type': PMIX_INT32},
+          {'value': 500, 'val_type': PMIX_INT64},
+          {'value': 250, 'val_type': PMIX_UINT},
+          {'value': 201, 'val_type': PMIX_UINT8},
+          {'value': 700, 'val_type': PMIX_UINT16},
+          {'value': 301, 'val_type': PMIX_UINT32},
+          {'value': 301, 'val_type': PMIX_UINT64},
+          {'value': 301.1, 'val_type': PMIX_FLOAT},
+          {'value': 201.2, 'val_type': PMIX_DOUBLE},
+          {'value': {'sec': 2, 'usec': 3}, 'val_type': PMIX_TIMEVAL},
+          {'value': 100, 'val_type': PMIX_TIME},
+          {'value': 34, 'val_type': PMIX_STATUS},
+          {'value': 15, 'val_type': PMIX_PROC_RANK},
+          {'value': {'nspace': 'testnspace', 'rank': 0}, 'val_type': PMIX_PROC},
+          {'value': {'bytes': bytearray(1), 'size': 1}, 'val_type': PMIX_BYTE_OBJECT},
+          {'value': 18, 'val_type': PMIX_PERSISTENCE},
+          {'value': 1, 'val_type': PMIX_SCOPE},
+          {'value': 5, 'val_type': PMIX_RANGE},
+          {'value': 45, 'val_type': PMIX_PROC_STATE},
+          {'value': {'proc': {'nspace': 'fakenspace', 'rank': 0}, 'hostname': 'myhostname', 'executable': 'testexec', 'pid': 2, 'exitcode': 0, 'state': 0}, 'val_type': PMIX_PROC_INFO},
+          {'value': darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': darray2, 'val_type': PMIX_DATA_ARRAY},
+          {'value': str_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': size_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': pid_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': 19, 'val_type': PMIX_ALLOC_DIRECTIVE},
+          {'value': {'envar': 'TEST_ENVAR', 'value': 'TEST_VAL', 'separator':
+                     ':'}, 'val_type': PMIX_ENVAR},
+          {'value': pyregex, 'val_type': PMIX_REGEX},
+          {'value': timeval_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': bool_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': byte_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': proc_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': bo_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': proc_info_darray, 'val_type': PMIX_DATA_ARRAY},
+          {'value': env_darray, 'val_type': PMIX_DATA_ARRAY}
+        ]
+
+def error_string(pystat:int):
+    cdef char *string
+    string = <char*>PMIx_Error_string(pystat)
+    pystr = string
+    val = pystr.decode('ascii')
+    return val
+
+def test_load_value(unload:bool):
+    global values
+    cdef pmix_value_t value
+    for val in values:
+        rc = pmix_load_value(&value, val)
+        if rc == PMIX_SUCCESS:
+            print("LOAD SUCCESS -- VALUE DICT: \t" + str(val) + "\n")
+        if unload and rc == PMIX_SUCCESS:
+            pydict = pmix_unload_value(&value)
+            print("UNLOAD SUCCESS -- VALUE CONVERTED: \t" + str(pydict) + "\n")
+
+def test_alloc_info():
+    global pyinfo, values
+    cdef pmix_info_t *info
+    cdef pmix_info_t **info_ptr
+    cdef size_t ninfo
+
+    # call pmix_alloc_info
+    info_ptr = &info
+    rc = pmix_alloc_info(info_ptr, &ninfo, pyinfo)
+    if rc == PMIX_SUCCESS:
+        print("SUCCESSFULLY LOADED:\n")
+        for i in pyinfo:
+            print("\t" + str(i['key']) + ", " + str(i['value']) + ", " +
+                  str(PMIx_Data_type_string(i['val_type'])) + "\n")
diff -Nru pmix-3.2.2~rc1/bindings/python/tests/cython/test_cython.py pmix-4.0.0/bindings/python/tests/cython/test_cython.py
--- pmix-3.2.2~rc1/bindings/python/tests/cython/test_cython.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/tests/cython/test_cython.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+from pmix import *
+import signal, time
+import os
+import select
+import subprocess
+import argparse
+import sys
+
+global killer
+class GracefulKiller:
+  kill_now = False
+  def __init__(self):
+    signal.signal(signal.SIGINT, self.exit_gracefully)
+    signal.signal(signal.SIGTERM, self.exit_gracefully)
+
+  def exit_gracefully(self,signum, frame):
+    self.kill_now = True
+
+def main():
+    test_count = 0
+    test_fails = 0
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-t", "--test", help="pass string name of test function (i.e test_alloc_info)")
+    parser.add_argument("-a", "--all", action="store_true")
+    args = parser.parse_args()
+    unload = True
+    if len(sys.argv) < 2:
+        print("no arguments provided")
+    if args.test == "test_alloc_info":
+        test_alloc_info()
+    elif args.test == "test_load_value":
+        test_load_value(unload)
+    elif args.all:
+        test_load_value(unload)
+        test_alloc_info()
+
+if __name__ == '__main__':
+    global killer
+    killer = GracefulKiller()
+    main()
diff -Nru pmix-3.2.2~rc1/bindings/python/tests/python/client.py pmix-4.0.0/bindings/python/tests/python/client.py
--- pmix-3.2.2~rc1/bindings/python/tests/python/client.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/tests/python/client.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+import time
+from pmix import *
+
+test_count = 0
+test_fails = 0
+
+def test_put(client, scope, ky, val):
+    print("PUT")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc = client.put(PMIX_GLOBAL, "mykey", {'value':1, 'val_type':PMIX_INT32})
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("PUT TEST FAILED: ", client.error_string(rc))
+
+def test_commit(client):
+    print("COMMIT")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc = client.commit()
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("COMMIT TEST FAILED: ", client.error_string(rc))
+
+def test_fence(client, procs, info):
+    print("FENCE")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc = client.fence(procs, info)
+    # don't count 'not supported' as a test fail
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("FENCE TEST FAILED: ", client.error_string(rc))
+    if rc == -47:
+        print("FENCE call " + client.error_string(rc))
+
+def test_get(client, proc, ky, dicts):
+    print("GET")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc, get_val = client.get(proc, ky, dicts)
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("GET TEST FAILED: ", client.error_string(rc))
+    print("GET_VAL RETURNED: ", get_val)
+
+def test_publish(client, dicts):
+    print("PUBLISH")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc = client.publish(dicts)
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("PUBLISH TEST FAILED: ", client.error_string(rc))
+
+def test_lookup(client, data, dicts):
+    print("LOOKUP")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc, pdata = client.lookup(data, dicts)
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("LOOKUP TEST FAILED: ", client.error_string(rc))
+    print("PDATA RETURNED: ", pdata)
+
+def test_unpublish(client, pykeys, dicts):
+    print("UNPUBLISH")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc = client.unpublish(pykeys, dicts)
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("UNPUBLISH TEST FAILED: ", client.error_string(rc))
+
+def test_query(client, pyqueries):
+    print("QUERY")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc, pyresults = client.query(pyqueries)
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("QUERY TEST FAILED: ", client.error_string(rc))
+    print("QUERY INFO RETURNED: ", pyresults)
+
+def test_pyhandler(st:int, pysource:dict, pyinfo:list, pyresults:list):
+    print("PYHANDLER")
+    status = PMIX_EVENT_ACTION_COMPLETE
+    results = [{'key':'eventkey', 'value':'testevent', 'val_type':PMIX_STRING}]
+    print("PYHANDLER RETURNED: ", results)
+    return status, results
+
+def test_register_event_handler(client, pycodes, info, test_pyhandler):
+    print("REGISTER EVENT HANDLER")
+    global test_count, test_fails
+    test_count = test_count + 1
+    rc = client.register_event_handler(pycodes, info, test_pyhandler)
+    if rc != 0 and rc != -47:
+        test_fails = test_fails + 1
+        print("REGISTER EVENT HANDLER TEST FAILED: ", client.error_string(rc))
+
+def main():
+    foo = PMIxClient()
+    print("Testing PMIx ", foo.get_version())
+    info = [{'key':PMIX_PROGRAMMING_MODEL, 'value':'TEST', 'val_type':PMIX_STRING},
+            {'key':PMIX_MODEL_LIBRARY_NAME, 'value':'PMIX', 'val_type':PMIX_STRING}]
+    my_result,myname = foo.init(info)
+    print("Init result ", my_result)
+    if 0 != my_result:
+        print("FAILED TO INIT")
+        exit(1)
+
+    # put value into keystore
+    test_put(foo, PMIX_GLOBAL, "mykey", {'value':1, 'val_type':PMIX_INT32})
+
+    # commit it
+    test_commit(foo)
+
+    # execute fence
+    procs = []
+    info = []
+    test_fence(foo, procs, info)
+
+    info = []
+    test_get(foo, {'nspace':"testnspace", 'rank': 0}, "mykey", info)
+
+    # test a fence that should return not_supported because
+    # we pass a required attribute that doesn't exist
+    procs = []
+    info = [{'key': 'ARBITRARY', 'flags': PMIX_INFO_REQD, 'value':10, 'val_type':PMIX_INT}]
+    rc = test_fence(foo, procs, info)
+
+    info = [{'key': 'ARBITRARY', 'flags': None, 'value':10, 'val_type':PMIX_INT}]
+    test_publish(foo, info)
+
+    test_fence(foo, procs, info)
+
+    pdata_key = [{'key':'ARBITRARY'}]
+    test_lookup(foo, pdata_key, None)
+
+    pykeys = ['ARBITRARY']
+    info = []
+    test_unpublish(foo, pykeys, info)
+
+    test_fence(foo, procs, info)
+
+    pyq = [{'keys':[PMIX_QUERY_PSET_NAMES], 'qualifiers':[]}]
+    test_query(foo, pyq)
+
+    pycodes = []
+    code = PMIX_MODEL_DECLARED
+    pycodes.append(code)
+    info = [{'key': PMIX_EVENT_HDLR_NAME, 'value': 'SIMPCLIENT-MODEL', 'val_type': PMIX_STRING}]
+    test_register_event_handler(foo, pycodes, info, test_pyhandler)
+
+    time.sleep(2)
+
+    # finalize
+    info = []
+    foo.finalize(info)
+    print("Client finalize complete")
+
+    # count client wrapper tests that returned with no error
+    total_passed = test_count - test_fails
+    print(str(total_passed) + " / " + str(test_count) + " TESTS PASSED")
+if __name__ == '__main__':
+    main()
diff -Nru pmix-3.2.2~rc1/bindings/python/tests/python/sched.py pmix-4.0.0/bindings/python/tests/python/sched.py
--- pmix-3.2.2~rc1/bindings/python/tests/python/sched.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/tests/python/sched.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+from server_upcalls import *
+import os
+import select
+import subprocess
+
+def main():
+    try:
+        foo = PMIxServer()
+    except:
+        print("FAILED TO CREATE SERVER")
+        exit(1)
+    print("Testing server version ", foo.get_version())
+    args = [{'key':PMIX_SERVER_SCHEDULER, 'value':'T', 'val_type':PMIX_BOOL}]
+    map = {'clientconnected': clientconnected,
+           'clientfinalized': clientfinalized,
+           'fencenb': clientfence,
+           'publish': clientpublish,
+           'unpublish': clientunpublish,
+           'lookup': clientlookup,
+           'query': clientquery}
+    my_result = foo.init(args, map)
+    print("Testing PMIx_Initialized")
+    rc = foo.initialized()
+    print("Initialized: ", rc)
+    vers = foo.get_version()
+    print("Version: ", vers)
+
+    # Register a fabric
+    rc = foo.register_fabric(None)
+    print("Fabric registered: ", rc)
+
+    # setup the application
+    (rc, regex) = foo.generate_regex("test000,test001,test002")
+    print("Node regex, rc: ", regex, rc)
+    (rc, ppn) = foo.generate_ppn("0,1,2;3,4,5;6,7")
+    print("PPN, rc: ", ppn, rc)
+    darray = {'type':PMIX_INFO, 'array':[{'key':PMIX_ALLOC_NETWORK_ID,
+                            'value':'SIMPSCHED.net', 'val_type':PMIX_STRING},
+                           {'key':PMIX_ALLOC_NETWORK_SEC_KEY, 'value':'T',
+                            'val_type':PMIX_BOOL},
+                           {'key':PMIX_SETUP_APP_ENVARS, 'value':'T',
+                            'val_type':PMIX_BOOL}]}
+    kyvals = [{'key':PMIX_NODE_MAP, 'value':regex, 'val_type':PMIX_STRING},
+              {'key':PMIX_PROC_MAP, 'value':ppn, 'val_type':PMIX_STRING},
+              {'key':PMIX_ALLOC_NETWORK, 'value':darray, 'val_type':PMIX_DATA_ARRAY}]
+
+    appinfo = []
+    rc, appinfo = foo.setup_application("SIMPSCHED", kyvals)
+    print("SETUPAPP: ", appinfo)
+
+    rc = foo.setup_local_support("SIMPSCHED", appinfo)
+    print("SETUPLOCAL: ", rc)
+
+    # get our environment as a base
+    env = os.environ.copy()
+
+    # register an nspace for the client app
+    kvals = [{'key':PMIX_NODE_MAP, 'value':regex, 'val_type':PMIX_STRING},
+             {'key':PMIX_PROC_MAP, 'value':ppn, 'val_type':PMIX_STRING},
+             {'key':PMIX_UNIV_SIZE, 'value':1, 'val_type':PMIX_UINT32},
+             {'key':PMIX_JOB_SIZE, 'value':1, 'val_type':PMIX_UINT32}]
+    print("REGISTERING NSPACE")
+    rc = foo.register_nspace("testnspace", 1, kvals)
+    print("RegNspace ", rc)
+
+    # register a client
+    uid = os.getuid()
+    gid = os.getgid()
+    print("REGISTERING CLIENT")
+    rc = foo.register_client({'nspace':"testnspace", 'rank':0}, uid, gid)
+    print("RegClient ", rc)
+
+    # setup the fork
+    rc = foo.setup_fork({'nspace':"testnspace", 'rank':0}, env)
+    print("SetupFrk", rc)
+
+    # setup the client argv
+    args = ["./client.py"]
+    # open a subprocess with stdout and stderr
+    # as distinct pipes so we can capture their
+    # output as the process runs
+    p = subprocess.Popen(args, env=env,
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # define storage to catch the output
+    stdout = []
+    stderr = []
+    # loop until the pipes close
+    while True:
+        reads = [p.stdout.fileno(), p.stderr.fileno()]
+        ret = select.select(reads, [], [])
+
+        stdout_done = True
+        stderr_done = True
+
+        for fd in ret[0]:
+            # if the data
+            if fd == p.stdout.fileno():
+                read = p.stdout.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    print('stdout: ' + read)
+                    stdout_done = False
+            elif fd == p.stderr.fileno():
+                read = p.stderr.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    print('stderr: ' + read)
+                    stderr_done = False
+
+        if stdout_done and stderr_done:
+            break
+
+    print("FINALIZING")
+    foo.finalize()
+
+if __name__ == '__main__':
+    global killer
+    killer = GracefulKiller()
+    main()
diff -Nru pmix-3.2.2~rc1/bindings/python/tests/python/server.py pmix-4.0.0/bindings/python/tests/python/server.py
--- pmix-3.2.2~rc1/bindings/python/tests/python/server.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/tests/python/server.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+from server_upcalls import *
+import os
+import select
+import subprocess
+
+def main():
+    try:
+        foo = PMIxServer()
+    except:
+        print("FAILED TO CREATE SERVER")
+        exit(1)
+    print("Testing server version ", foo.get_version())
+    args = [{'key':'FOOBAR', 'value':'VAR', 'val_type':PMIX_STRING},
+            {'key':'BLAST', 'value':7, 'val_type':PMIX_INT32}]
+    map = {'clientconnected': clientconnected,
+           'clientfinalized': clientfinalized,
+           'fencenb': clientfence,
+           'publish': clientpublish,
+           'unpublish': clientunpublish,
+           'lookup': clientlookup,
+           'query': clientquery,
+           'registerevents': client_register_events}
+    my_result = foo.init(args, map)
+    print("Testing PMIx_Initialized")
+    rc = foo.initialized()
+    print("Initialized: ", rc)
+    vers = foo.get_version()
+    print("Version: ", vers)
+
+    # pdata = {‘proc’: {‘nspace’: mynspace, ‘rank’: myrank}, ‘key’: ky,
+    # ‘value’: v, ‘val_type’: t}
+    pmix_locdat = []
+
+    # get our environment as a base
+    env = os.environ.copy()
+    # register an nspace for the client app
+    (rc, regex) = foo.generate_regex(["test000","test001","test002"])
+    print("Node regex, rc: ", regex, rc)
+    (rc, ppn) = foo.generate_ppn(["0,1,2", "3,4,5", "6,7"])
+    print("PPN, rc: ", ppn, rc)
+    kvals = [{'key':PMIX_NODE_MAP, 'value':regex, 'val_type':PMIX_STRING},
+             {'key':PMIX_PROC_MAP, 'value':ppn, 'val_type':PMIX_STRING},
+             {'key':PMIX_UNIV_SIZE, 'value':1, 'val_type':PMIX_UINT32},
+             {'key':PMIX_JOB_SIZE, 'value':1, 'val_type':PMIX_UINT32}]
+    print("REGISTERING NSPACE")
+    rc = foo.register_nspace("testnspace", 1, kvals)
+    print("RegNspace ", rc)
+
+    # register a client
+    uid = os.getuid()
+    gid = os.getgid()
+    rc = foo.register_client({'nspace':"testnspace", 'rank':0}, uid, gid)
+    print("RegClient ", rc)
+    # setup the fork
+    rc = foo.setup_fork({'nspace':"testnspace", 'rank':0}, env)
+    print("SetupFrk", rc)
+
+    # setup the client argv
+    args = ["./client.py"]
+    # open a subprocess with stdout and stderr
+    # as distinct pipes so we can capture their
+    # output as the process runs
+    p = subprocess.Popen(args, env=env,
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # define storage to catch the output
+    stdout = []
+    stderr = []
+    # loop until the pipes close
+    while True:
+        reads = [p.stdout.fileno(), p.stderr.fileno()]
+        ret = select.select(reads, [], [])
+
+        stdout_done = True
+        stderr_done = True
+
+        for fd in ret[0]:
+            # if the data
+            if fd == p.stdout.fileno():
+                read = p.stdout.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    stdout_done = False
+                    print("stdout:", read)
+            if fd == p.stderr.fileno():
+                read = p.stderr.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    stderr_done = False
+                    print("stderr:", read)
+
+        if stdout_done and stderr_done:
+            break
+    print("FINALIZING")
+    foo.finalize()
+
+
+if __name__ == '__main__':
+    global killer
+    killer = GracefulKiller()
+    main()
diff -Nru pmix-3.2.2~rc1/bindings/python/tests/python/server_upcalls.py pmix-4.0.0/bindings/python/tests/python/server_upcalls.py
--- pmix-3.2.2~rc1/bindings/python/tests/python/server_upcalls.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/python/tests/python/server_upcalls.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,97 @@
+from pmix import *
+import signal, time
+global killer
+
+# where local data is published for testing
+pmix_locdata = []
+
+class GracefulKiller:
+  kill_now = False
+  def __init__(self):
+    signal.signal(signal.SIGINT, self.exit_gracefully)
+    signal.signal(signal.SIGTERM, self.exit_gracefully)
+
+  def exit_gracefully(self,signum, frame):
+    self.kill_now = True
+
+def clientconnected(proc:tuple is not None):
+    print("CLIENT CONNECTED", proc)
+    return PMIX_OPERATION_SUCCEEDED
+
+def clientfinalized(proc:tuple is not None):
+    print("CLIENT FINALIZED", proc)
+    return PMIX_OPERATION_SUCCEEDED
+
+def clientfence(args:dict is not None):
+    # check directives
+    print("CLIENTFENCE")
+    output = bytearray(0)
+    try:
+        if args['directives'] is not None:
+            for d in args['directives']:
+                # these are each an info dict
+                if "pmix" not in d['key']:
+                    # we do not support such directives - see if
+                    # it is required
+                    try:
+                        if d['flags'] & PMIX_INFO_REQD:
+                            # return an error
+                            return PMIX_ERR_NOT_SUPPORTED, output
+                    except:
+                        #it can be ignored
+                        pass
+    print("COMPLETE")
+    return PMIX_SUCCESS, output
+
+def clientpublish(args:dict is not None):
+    print("SERVER: PUBLISH")
+    for d in args['directives']:
+        pdata = {}
+        pdata['proc'] = args['proc']
+        pdata['key']            = d['key']
+        pdata['value']          = d['value']
+        pdata['val_type']       = d['val_type']
+        pmix_locdata.append(pdata)
+    return PMIX_OPERATION_SUCCEEDED
+
+def clientunpublish(args:dict is not None):
+    print("SERVER: UNPUBLISH")
+    for k in args['keys']:
+        for d in pmix_locdata:
+            if k.decode('ascii') == d['key']:
+                pmix_locdata.remove(d)
+    return PMIX_OPERATION_SUCCEEDED
+
+def clientlookup(args:dict is not None):
+    print("SERVER: LOOKUP")
+    ret_pdata = []
+    for k in args['keys']:
+        for d in pmix_locdata:
+            if k.decode('ascii') == d['key']:
+                ret_pdata.append(d)
+    # return rc and pdata
+    return PMIX_SUCCESS, ret_pdata
+
+def clientquery(args:dict is not None):
+    print("SERVER: QUERY")
+    # return a python info list of dictionaries
+    info = {}
+    results = []
+    rc = PMIX_ERR_NOT_FOUND
+    # find key we passed in to client, and
+    # if it matches return fake PSET_NAME
+    # since RM actually assigns this, we
+    # just return arbitrary name if key is
+    # found
+    find_str = 'pmix.qry.psets'
+    for q in args['queries']:
+        for k in q['keys']:
+            if k == find_str:
+                info = {'key': find_str, 'value': 'PSET_NAME', 'val_type': PMIX_STRING}
+                results.append(info)
+                rc = PMIX_SUCCESS
+    return rc, results
+
+def client_register_events(args:dict is not None):
+    print("CLIENT REGISTER EVENTS ", args['codes'])
+    return PMIX_OPERATION_SUCCEEDED
diff -Nru pmix-3.2.2~rc1/bindings/README pmix-4.0.0/bindings/README
--- pmix-3.2.2~rc1/bindings/README	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/bindings/README	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,21 @@
+Copyright (c) 2016-2018 Intel, Inc.  All rights reserved.
+
+$COPYRIGHT$
+
+Additional copyrights may follow
+
+$HEADER$
+
+===========================================================================
+
+This is where bindings of PMIx functions to alternative programming languages
+such as Python reside. All functions defined in the public headers have been
+provided with a wrapper. Note that there is no restriction on the number of
+wrappers that can exist, nor on what type of function is wrapped.
+
+There is only one rule to observe: you can wrap a framework, but you cannot wrap a
+specific plugin within that framework. This constraint flows from the fact that
+plugins are only accessed via the framework interface - thus, there is no way to
+guarantee that a particular plugin will be the active selection.
+
+
diff -Nru pmix-3.2.2~rc1/config/getdate.sh pmix-4.0.0/config/getdate.sh
--- pmix-3.2.2~rc1/config/getdate.sh	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/getdate.sh	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,11 @@
+#!/bin/sh
+#
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+
+# Provide a way to override build date for reproducible build results
+# See https://reproducible-builds.org/ for why this is good.
+
+SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date +%s)}"
+date -u -d "@$SOURCE_DATE_EPOCH" "$@" 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" "$@" 2>/dev/null || date -u "$@"
diff -Nru pmix-3.2.2~rc1/config/ltmain_nag_pthread.diff pmix-4.0.0/config/ltmain_nag_pthread.diff
--- pmix-3.2.2~rc1/config/ltmain_nag_pthread.diff	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/ltmain_nag_pthread.diff	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,18 @@
+--- config/ltmain.sh
++++ config/ltmain.sh
+@@ -6417,8 +6417,14 @@
+ 	func_source "$lib"
+
+ 	# Convert "-framework foo" to "foo.ltframework"
++        # and "-pthread" to "-Wl,-pthread" if NAG compiler
+ 	if test -n "$inherited_linker_flags"; then
+-	  tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'`
++          case "$CC" in
++            *nagfor*)
++	      tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g' | $SED 's/-pthread/-Wl,-pthread/g'`;;
++            *)
++	      tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'`;;
++          esac
+ 	  for tmp_inherited_linker_flag in $tmp_inherited_linker_flags; do
+ 	    case " $new_inherited_linker_flags " in
+ 	      *" $tmp_inherited_linker_flag "*) ;;
diff -Nru pmix-3.2.2~rc1/config/ltmain_pgi_tp.diff pmix-4.0.0/config/ltmain_pgi_tp.diff
--- pmix-3.2.2~rc1/config/ltmain_pgi_tp.diff	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/ltmain_pgi_tp.diff	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,11 @@
+--- config/ltmain.sh
++++ config/ltmain.sh
+@@ -4765,7 +4765,7 @@
+       # -p, -pg, --coverage, -fprofile-* pass through profiling flag for GCC
+       # @file GCC response files
+       -64|-mips[0-9]|-r[0-9][0-9]*|-xarch=*|-xtarget=*|+DA*|+DD*|-q*|-m*| \
+-      -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*)
++      -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*|-tp|-tp=*)
+         func_quote_for_eval "$arg"
+        arg="$func_quote_for_eval_result"
+         func_append compile_command " $arg"
diff -Nru pmix-3.2.2~rc1/config/Makefile.am pmix-4.0.0/config/Makefile.am
--- pmix-3.2.2~rc1/config/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -27,6 +27,7 @@
 	c_get_alignment.m4 \
 	pmix_get_version.sh \
 	distscript.sh \
+	getdate.sh \
 	md2nroff.pl \
 	pmix_check_attributes.m4 \
 	pmix_check_broken_qsort.m4 \
diff -Nru pmix-3.2.2~rc1/config/pmix_check_curl.m4 pmix-4.0.0/config/pmix_check_curl.m4
--- pmix-3.2.2~rc1/config/pmix_check_curl.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_check_curl.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,106 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2006 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      QLogic Corp. All rights reserved.
+# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# PMIX_CHECK_CURL
+# --------------------------------------------------------
+# check if CURL support can be found.  sets curl_{CPPFLAGS,
+# LDFLAGS, LIBS}
+AC_DEFUN([PMIX_CHECK_CURL],[
+
+    PMIX_VAR_SCOPE_PUSH(pmix_check_curl_save_CPPFLAGS pmix_check_curl_save_LDFLAGS pmix_check_curl_save_LIBS)
+	AC_ARG_WITH([curl],
+		    [AC_HELP_STRING([--with-curl(=DIR)],
+				    [Build curl support (default=no), optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+
+    AC_ARG_WITH([curl-libdir],
+            [AC_HELP_STRING([--with-curl-libdir=DIR],
+                    [Search for Curl libraries in DIR])])
+
+    pmix_check_curl_happy=no
+    pmix_curl_source=unknown
+    pmix_check_curl_dir=unknown
+    pmix_check_curl_libdir=
+    pmix_check_curl_basedir=
+
+    if test "$with_curl" != "no"; then
+        AC_MSG_CHECKING([where to find curl.h])
+        AS_IF([test "$with_curl" = "yes" || test -z "$with_curl"],
+              [AS_IF([test -f /usr/include/curl.h],
+                     [pmix_check_curl_dir=/usr
+                      pmix_check_curl_basedir=/usr
+                      pmix_curl_source=standard],
+                     [AS_IF([test -f /usr/include/curl/curl.h],
+                            [pmix_check_curl_dir=/usr/include/curl
+                             pmix_check_curl_basedir=/usr
+                             pmix_curl_source=standard])])],
+    	      [AS_IF([test -f $with_curl/include/curl.h],
+                     [pmix_check_curl_dir=$with_curl/include
+                      pmix_check_curl_basedir=$with_curl
+                      pmix_curl_source=$with_curl],
+                      [AS_IF([test -f $with_curl/include/curl/curl.h],
+                             [pmix_check_curl_dir=$with_curl/include/curl
+                              pmix_check_curl_basedir=$with_curl
+                              pmix_curl_source=$with_curl])])])
+        AC_MSG_RESULT([$pmix_check_curl_dir])
+
+        AS_IF([test -z "$with_curl_libdir" || test "$with_curl_libdir" = "yes"],
+              [pmix_check_curl_libdir=$pmix_check_curl_basedir],
+    	      [PMIX_CHECK_WITHDIR([curl-libdir], [$with_curl_libdir], [libcurl.*])
+               pmix_check_curl_libdir=$with_curl_libdir])
+
+    	pmix_check_curl_save_CPPFLAGS="$CPPFLAGS"
+    	pmix_check_curl_save_LDFLAGS="$LDFLAGS"
+    	pmix_check_curl_save_LIBS="$LIBS"
+
+        PMIX_CHECK_PACKAGE([pmix_check_curl],
+		   [curl.h],
+		   [curl],
+		   [curl_easy_getinfo],
+		   [],
+		   [$pmix_check_curl_dir],
+		   [$pmix_check_curl_libdir],
+		   [pmix_check_curl_happy="yes"],
+		   [pmix_check_curl_happy="no"])
+
+    	CPPFLAGS="$pmix_check_curl_save_CPPFLAGS"
+    	LDFLAGS="$pmix_check_curl_save_LDFLAGS"
+    	LIBS="$pmix_check_curl_save_LIBS"
+    fi
+
+    AS_IF([test "$pmix_check_curl_happy" = "no" && test ! -z "$with_curl" && test "$with_curl" != "no"],
+          [AC_MSG_ERROR([curl support requested but not found.  Aborting])])
+
+    AC_MSG_CHECKING([libcurl support available])
+    AC_MSG_RESULT([$pmix_check_curl_happy])
+
+    PMIX_SUMMARY_ADD([[External Packages]],[[Curl]], [pmix_curl], [$pmix_check_curl_happy ($pmix_curl_source)])
+
+    AC_SUBST(pmix_check_curl_CPPFLAGS)
+    AC_SUBST(pmix_check_curl_LDFLAGS)
+    AC_SUBST(pmix_check_curl_LIBS)
+
+    PMIX_VAR_SCOPE_POP
+])
diff -Nru pmix-3.2.2~rc1/config/pmix_check_cython.py pmix-4.0.0/config/pmix_check_cython.py
--- pmix-3.2.2~rc1/config/pmix_check_cython.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_check_cython.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+from Cython.Build import cythonize
+
+def main():
+    print("0")
+
+if __name__ == '__main__':
+    main()
diff -Nru pmix-3.2.2~rc1/config/pmix_check_jansson.m4 pmix-4.0.0/config/pmix_check_jansson.m4
--- pmix-3.2.2~rc1/config/pmix_check_jansson.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_check_jansson.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,113 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2006 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      QLogic Corp. All rights reserved.
+# Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2015      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# PMIX_CHECK_JANSSON
+# --------------------------------------------------------
+# check if JANSSON support can be found.  sets jansson_{CPPFLAGS,
+# LDFLAGS, LIBS}
+AC_DEFUN([PMIX_CHECK_JANSSON],[
+
+    PMIX_VAR_SCOPE_PUSH(pmix_check_jansson_save_CPPFLAGS pmix_check_jansson_save_LDFLAGS pmix_check_jansson_save_LIBS)
+	AC_ARG_WITH([jansson],
+		    [AC_HELP_STRING([--with-jansson(=DIR)],
+				    [Build jansson support (default=no), optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+
+    AC_ARG_WITH([jansson-libdir],
+            [AC_HELP_STRING([--with-jansson-libdir=DIR],
+                    [Search for Jansson libraries in DIR])])
+
+    pmix_check_jansson_happy=no
+    pmix_jansson_source=unknown
+    pmix_check_jansson_dir=
+    pmix_check_jansson_libdir=
+    pmix_check_jansson_basedir=
+
+    if test "$with_jansson" != "no"; then
+        AS_IF([test "$with_jansson" = "yes" || test -z "$with_jansson"],
+              [pmix_check_jansson_dir=/usr
+               pmix_check_jansson_basedir=/usr
+               pmix_jansson_source=standard],
+    	      [PMIX_CHECK_WITHDIR([jansson], [$with_jansson], [include/jansson.h])
+               pmix_check_jansson_dir=$with_jansson/include
+               pmix_check_jansson_basedir=$with_jansson
+               pmix_jansson_source=$with_jansson])
+
+        AS_IF([test -z "$with_jansson_libdir" || test "$with_jansson_libdir" = "yes"],
+              [pmix_check_jansson_libdir=$pmix_check_jansson_basedir/lib],
+    	      [PMIX_CHECK_WITHDIR([jansson-libdir], [$with_jansson_libdir], [libjansson.*])
+               pmix_check_jansson_libdir=$with_jansson_libdir])
+
+    	pmix_check_jansson_save_CPPFLAGS="$CPPFLAGS"
+    	pmix_check_jansson_save_LDFLAGS="$LDFLAGS"
+    	pmix_check_jansson_save_LIBS="$LIBS"
+
+        PMIX_CHECK_PACKAGE([pmix_check_jansson],
+		                   [jansson.h],
+	 	                   [jansson],
+		                   [json_loads],
+		                   [],
+		                   [$pmix_check_jansson_dir],
+		                   [$pmix_check_jansson_libdir],
+		                   [pmix_check_jansson_happy="yes"],
+		                   [pmix_check_jansson_happy="no"])
+
+        if test "$pmix_check_jansson_happy" == "yes"; then
+            AC_MSG_CHECKING([if libjansson version is 2.11 or greater])
+            AS_IF([test "$pmix_jansson_source" != "standard"],
+                  [PMIX_FLAGS_APPEND_UNIQ(CPPFLAGS, $pmix_check_jansson_CPPFLAGS)])
+            AC_COMPILE_IFELSE(
+                  [AC_LANG_PROGRAM([[#include <jansson.h>]],
+                  [[
+        #if JANSSON_VERSION_HEX < 0x00020b00
+        #error "jansson API version is less than 2.11"
+        #endif
+                  ]])],
+                  [AC_MSG_RESULT([yes])],
+                  [AC_MSG_RESULT([no])
+                   pmix_check_jansson_happy=no])
+        fi
+
+    	CPPFLAGS="$pmix_check_jansson_save_CPPFLAGS"
+    	LDFLAGS="$pmix_check_jansson_save_LDFLAGS"
+    	LIBS="$pmix_check_jansson_save_LIBS"
+    fi
+
+    AS_IF([test "$pmix_check_jansson_happy" = "no" && test ! -z "$with_jansson" && test "$with_jansson" != "no"],
+          [AC_MSG_ERROR([Jansson support requested but not found.  Aborting])])
+
+    AC_SUBST(pmix_check_jansson_CPPFLAGS)
+    AC_SUBST(pmix_check_jansson_LDFLAGS)
+    AC_SUBST(pmix_check_jansson_LIBS)
+
+    AC_MSG_CHECKING([Jansson support available])
+    AC_MSG_RESULT([$pmix_check_jansson_happy])
+
+    AM_CONDITIONAL([HAVE_JANSSON], [test "$pmix_check_jansson_happy" = "yes"])
+
+    PMIX_SUMMARY_ADD([[External Packages]],[[Jansson]], [pmix_jansson], [$pmix_check_jansson_happy ($pmix_jansson_source)])
+
+    PMIX_VAR_SCOPE_POP
+])
diff -Nru pmix-3.2.2~rc1/config/pmix_check_lustre.m4 pmix-4.0.0/config/pmix_check_lustre.m4
--- pmix-3.2.2~rc1/config/pmix_check_lustre.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_check_lustre.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,99 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+dnl                         University Research and Technology
+dnl                         Corporation.  All rights reserved.
+dnl Copyright (c) 2004-2005 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+dnl                         University of Stuttgart.  All rights reserved.
+dnl Copyright (c) 2004-2006 The Regents of the University of California.
+dnl                         All rights reserved.
+dnl Copyright (c) 2009-2017 Cisco Systems, Inc.  All rights reserved
+dnl Copyright (c) 2008-2018 University of Houston. All rights reserved.
+dnl Copyright (c) 2015-2018 Research Organization for Information Science
+dnl                         and Technology (RIST).  All rights reserved.
+dnl Copyright (c) 2020      Triad National Security, LLC. All rights
+dnl                         reserved.
+dnl Copyright (c) 2020      Intel, Inc.  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# PMIX_CHECK_LUSTRE(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if LUSTRE support can be found.  sets prefix_{CPPFLAGS,
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+AC_DEFUN([PMIX_CHECK_LUSTRE],[
+
+    check_lustre_CPPFLAGS=
+    check_lustre_LDFLAGS=
+    check_lustre_LIBS=
+
+    check_lustre_save_LIBS="$LIBS"
+    check_lustre_save_LDFLAGS="$LDFLAGS"
+    check_lustre_save_CPPFLAGS="$CPPFLAGS"
+
+    pmix_check_lustre_happy="yes"
+
+    # Get some configuration information
+    AC_ARG_WITH([lustre],
+        [AC_HELP_STRING([--with-lustre(=DIR)],
+             [Build Lustre support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+    PMIX_CHECK_WITHDIR([lustre], [$with_lustre], [include/lustre/lustreapi.h])
+
+    AS_IF([test "$with_lustre" = "no"],
+          [pmix_check_lustre_happy=no])
+
+    AS_IF([test "$pmix_check_lustre_happy" != "no" ],
+          [AC_MSG_CHECKING([looking for lustre libraries and header files in])
+           AS_IF([test "$with_lustre" != "yes"],
+                 [pmix_check_lustre_dir=$with_lustre
+                  AC_MSG_RESULT([($pmix_check_lustre_dir)])],
+                 [AC_MSG_RESULT([(default search paths)])])
+           AS_IF([test -n "$with_lustre_libdir" && \
+                         test "$with_lustre_libdir" != "yes"],
+                 [pmix_check_lustre_libdir=$with_lustre_libdir])
+          ])
+
+    AS_IF([test "$pmix_check_lustre_happy" != "no" ],
+          [PMIX_CHECK_PACKAGE([$1], [lustre/lustreapi.h], [lustreapi], [llapi_file_create],
+                [], [$pmix_check_lustre_dir], [$pmix_check_lustre_libdir],
+                [pmix_check_lustre_happy="yes"],
+                [pmix_check_lustre_happy="no"])])
+
+    AS_IF([test "$pmix_check_lustre_happy" = "yes"],
+          [AC_MSG_CHECKING([for required lustre data structures])
+           cat > conftest.c <<EOF
+#include "lustre/lustreapi.h"
+void alloc_lum()
+{
+  int v1, v3;
+  v1 = sizeof(struct lov_user_md_v1) +
+   LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1);
+  v3 = sizeof(struct lov_user_md_v3) +
+    LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data_v1);
+}
+EOF
+
+           # Try the compile
+           PMIX_LOG_COMMAND(
+               [$CC $CFLAGS -I$pmix_check_lustre_dir/include -c conftest.c],
+               [pmix_check_lustre_struct_happy="yes"],
+               [pmix_check_lustre_struct_happy="no"
+                pmix_check_lustre_happy="no"]
+            )
+            rm -f conftest.c conftest.o
+            AC_MSG_RESULT([$pmix_check_lustre_struct_happy])])
+
+    AS_IF([test "$pmix_check_lustre_happy" = "yes"],
+          [$2],
+          [AS_IF([test -n "$with_lustre" && test "$with_lustre" != "no"],
+                 [AC_MSG_ERROR([Lustre support requested but not found.  Aborting])])
+                  $3])
+])
diff -Nru pmix-3.2.2~rc1/config/pmix_check_tm.m4 pmix-4.0.0/config/pmix_check_tm.m4
--- pmix-3.2.2~rc1/config/pmix_check_tm.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_check_tm.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,175 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+dnl                         University Research and Technology
+dnl                         Corporation.  All rights reserved.
+dnl Copyright (c) 2004-2005 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+dnl                         University of Stuttgart.  All rights reserved.
+dnl Copyright (c) 2004-2005 The Regents of the University of California.
+dnl                         All rights reserved.
+dnl Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved
+dnl Copyright (c) 2015-2017 Research Organization for Information Science
+dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+dnl                         reserved.
+dnl Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# PMIX_CHECK_TM_LIBS_FLAGS(prefix, [LIBS or LDFLAGS])
+# ---------------------------------------------------
+AC_DEFUN([PMIX_CHECK_TM_LIBS_FLAGS],[
+    PMIX_VAR_SCOPE_PUSH([pmix_check_tm_flags])
+    pmix_check_tm_flags=`$pmix_check_tm_pbs_config --libs`
+    for pmix_check_tm_val in $pmix_check_tm_flags; do
+        if test "`echo $pmix_check_tm_val | cut -c1-2`" = "-l"; then
+            if test "$2" = "LIBS"; then
+                $1_$2="$$1_$2 $pmix_check_tm_val"
+            fi
+        else
+            if test "$2" = "LDFLAGS"; then
+                $1_$2="$$1_$2 $pmix_check_tm_val"
+            fi
+        fi
+    done
+    PMIX_VAR_SCOPE_POP
+])
+
+
+# PMIX_CHECK_TM(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+AC_DEFUN([PMIX_CHECK_TM],[
+    if test -z $pmix_check_tm_happy ; then
+	PMIX_VAR_SCOPE_PUSH([pmix_check_tm_found pmix_check_tm_dir pmix_check_tm_libdir pmix_check_tm_pbs_config pmix_check_tm_LDFLAGS_save pmix_check_tm_CPPFLAGS_save pmix_check_tm_LIBS_save])
+
+	AC_ARG_WITH([tm],
+                    [AC_HELP_STRING([--with-tm(=DIR)],
+                                    [Build TM (Torque, PBSPro, and compatible) support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+	PMIX_CHECK_WITHDIR([tm], [$with_tm], [include/tm.h])
+
+	pmix_check_tm_found=no
+	AS_IF([test "$with_tm" = "no"],
+              [pmix_check_tm_happy="no"],
+              [pmix_check_tm_happy="yes"
+               AS_IF([test ! -z "$with_tm" && test "$with_tm" != "yes"],
+                     [pmix_check_tm_dir="$with_tm"
+                      pmix_check_tm_libdir="$with_tm/lib"],
+                     [pmix_check_tm_dir=""])])
+
+	AS_IF([test "$pmix_check_tm_happy" = "yes"],
+              [AC_MSG_CHECKING([for pbs-config])
+               pmix_check_tm_pbs_config="not found"
+               AS_IF([test "$pmix_check_tm_dir" != "" && test -d "$pmix_check_tm_dir" && test -x "$pmix_check_tm_dir/bin/pbs-config"],
+                     [pmix_check_tm_pbs_config="$pmix_check_tm_dir/bin/pbs-config"],
+                     [AS_IF([pbs-config --prefix >/dev/null 2>&1],
+                            [pmix_check_tm_pbs_config="pbs-config"])])
+               AC_MSG_RESULT([$pmix_check_tm_pbs_config])])
+
+	# If we have pbs-config, get the flags we need from there and then
+	# do simplistic tests looking for the tm headers and symbols
+
+	AS_IF([test "$pmix_check_tm_happy" = "yes" && test "$pmix_check_tm_pbs_config" != "not found"],
+              [pmix_check_tm_CPPFLAGS=`$pmix_check_tm_pbs_config --cflags`
+               PMIX_LOG_MSG([pmix_check_tm_CPPFLAGS from pbs-config: $pmix_check_tm_CPPFLAGS], 1)
+
+               PMIX_CHECK_TM_LIBS_FLAGS([pmix_check_tm], [LDFLAGS])
+               PMIX_LOG_MSG([pmix_check_tm_LDFLAGS from pbs-config: $pmix_check_tm_LDFLAGS], 1)
+
+               PMIX_CHECK_TM_LIBS_FLAGS([pmix_check_tm], [LIBS])
+               PMIX_LOG_MSG([pmix_check_tm_LIBS from pbs-config: $pmix_check_tm_LIBS], 1)
+
+               # Now that we supposedly have the right flags, try them out.
+
+               pmix_check_tm_CPPFLAGS_save="$CPPFLAGS"
+               pmix_check_tm_LDFLAGS_save="$LDFLAGS"
+               pmix_check_tm_LIBS_save="$LIBS"
+
+               CPPFLAGS="$CPPFLAGS $pmix_check_tm_CPPFLAGS"
+               LIBS="$LIBS $pmix_check_tm_LIBS"
+               LDFLAGS="$LDFLAGS $pmix_check_tm_LDFLAGS"
+
+               AC_CHECK_HEADER([tm.h],
+			       [AC_CHECK_FUNC([tm_finalize],
+					      [pmix_check_tm_found="yes"])])
+
+               CPPFLAGS="$pmix_check_tm_CPPFLAGS_save"
+               LDFLAGS="$pmix_check_tm_LDFLAGS_save"
+               LIBS="$pmix_check_tm_LIBS_save"])
+
+	# If we don't have pbs-config, then we have to look around
+	# manually.
+
+	# Note that Torque 2.1.0 changed the name of their back-end
+	# library to "libtorque".  So we have to check for both libpbs and
+	# libtorque.  First, check for libpbs.
+
+	pmix_check_package_$1_save_CPPFLAGS="$CPPFLAGS"
+	pmix_check_package_$1_save_LDFLAGS="$LDFLAGS"
+	pmix_check_package_$1_save_LIBS="$LIBS"
+
+	AS_IF([test "$pmix_check_tm_found" = "no"],
+              [AS_IF([test "$pmix_check_tm_happy" = "yes"],
+                     [_PMIX_CHECK_PACKAGE_HEADER([pmix_check_tm],
+						 [tm.h],
+						 [$pmix_check_tm_dir],
+						 [pmix_check_tm_found="yes"],
+						 [pmix_check_tm_found="no"])])
+
+               AS_IF([test "$pmix_check_tm_found" = "yes"],
+                     [_PMIX_CHECK_PACKAGE_LIB([pmix_check_tm],
+					      [pbs],
+					      [tm_init],
+					      [],
+					      [$pmix_check_tm_dir],
+					      [$pmix_check_tm_libdir],
+					      [pmix_check_tm_found="yes"],
+                                              [_PMIX_CHECK_PACKAGE_LIB([pmix_check_tm],
+					                               [pbs],
+					                               [tm_init],
+					                               [-lcrypto],
+					                               [$pmix_check_tm_dir],
+					                               [$pmix_check_tm_libdir],
+					                               [pmix_check_tm_found="yes"],
+					                               [_PMIX_CHECK_PACKAGE_LIB([pmix_check_tm],
+								                                [torque],
+								                                [tm_init],
+								                                [],
+								                                [$pmix_check_tm_dir],
+								                                [$pmix_check_tm_libdir],
+								                                [pmix_check_tm_found="yes"],
+								                                [pmix_check_tm_found="no"])])])])])
+
+	CPPFLAGS="$pmix_check_package_$1_save_CPPFLAGS"
+	LDFLAGS="$pmix_check_package_$1_save_LDFLAGS"
+	LIBS="$pmix_check_package_$1_save_LIBS"
+
+	if test "$pmix_check_tm_found" = "no" ; then
+	    pmix_check_tm_happy=no
+	fi
+
+	PMIX_SUMMARY_ADD([[Resource Managers]],[[Torque]],[$1],[$pmix_check_tm_happy])
+
+	PMIX_VAR_SCOPE_POP
+    fi
+
+    # Did we find the right stuff?
+    AS_IF([test "$pmix_check_tm_happy" = "yes"],
+          [$1_LIBS="[$]$1_LIBS $pmix_check_tm_LIBS"
+	       $1_LDFLAGS="[$]$1_LDFLAGS $pmix_check_tm_LDFLAGS"
+	       $1_CPPFLAGS="[$]$1_CPPFLAGS $pmix_check_tm_CPPFLAGS"
+	       # add the TM libraries to static builds as they are required
+	       $1_WRAPPER_EXTRA_LDFLAGS=[$]$1_LDFLAGS
+	       $1_WRAPPER_EXTRA_LIBS=[$]$1_LIBS
+	       $2],
+          [AS_IF([test ! -z "$with_tm" && test "$with_tm" != "no"],
+                 [AC_MSG_ERROR([TM support requested but not found.  Aborting])])
+	       pmix_check_tm_happy="no"
+	       $3])
+])
diff -Nru pmix-3.2.2~rc1/config/pmix_functions.m4 pmix-4.0.0/config/pmix_functions.m4
--- pmix-3.2.2~rc1/config/pmix_functions.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_functions.m4	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
 dnl Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2009-2020 Cisco Systems, Inc.  All rights reserved.
-dnl Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+dnl Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2017      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
 dnl
@@ -94,16 +94,10 @@
 # Save some stats about this build
 #
 
-DATE_FMT="+%Y-%m-%dT%H:%M:%S"
-if test -n "$SOURCE_DATE_EPOCH" ; then
-  PMIX_CONFIGURE_USER="reproduciblebuild"
-  PMIX_CONFIGURE_HOST="reproduciblebuild"
-  PMIX_CONFIGURE_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "$DATE_FMT" 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" "$DATE_FMT" 2>/dev/null || date -u "$DATE_FMT")
-else
-  PMIX_CONFIGURE_USER="`whoami`"
-  PMIX_CONFIGURE_HOST="`(hostname || uname -n) 2> /dev/null | sed 1q`"
-  PMIX_CONFIGURE_DATE="`date $DATE_FMT`"
-fi
+PMIX_CONFIGURE_USER="${USER:-`whoami`}"
+PMIX_CONFIGURE_HOST="${HOSTNAME:-`(hostname || uname -n) 2> /dev/null | sed 1q`}"
+PMIX_CONFIGURE_DATE="`$top_srcdir/config/getdate.sh`"
+
 
 AC_SUBST([SOURCE_DATE_EPOCH])
 AM_CONDITIONAL([SOURCE_DATE_EPOCH_SET], [test -n "$SOURCE_DATE_EPOCH"])
diff -Nru pmix-3.2.2~rc1/config/pmix_get_version.sh pmix-4.0.0/config/pmix_get_version.sh
--- pmix-3.2.2~rc1/config/pmix_get_version.sh	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_get_version.sh	2021-01-02 08:56:17.000000000 +0000
@@ -10,8 +10,8 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Intel, Inc. All rights reserved
+# Copyright (c) 2008-2020 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -81,18 +81,18 @@
 
             # If we're in a git repo and we found the git command, use
             # git describe to get the repo rev
-            if test -d "$srcdir/.git" && test $git_happy -eq 1; then
+            if test -r "$srcdir/.git" && test $git_happy -eq 1; then
                 if test "$srcdir" != "`pwd`"; then
                     git_save_dir=`pwd`
-                    cd $srcdir
+                    cd "$srcdir"
                     PMIX_REPO_REV=`git describe --tags --always`
-                    cd $git_save_dir
+                    cd "$git_save_dir"
                     unset git_save_dir
                 else
                     PMIX_REPO_REV=`git describe --tags --always`
                 fi
             else
-                PMIX_REPO_REV="date`date '+%Y-%m-%d'`"
+                PMIX_REPO_REV=`$srcdir/config/getdate.sh '+%Y-%m-%d'`
             fi
         fi
 
diff -Nru pmix-3.2.2~rc1/config/pmix_load_platform.m4 pmix-4.0.0/config/pmix_load_platform.m4
--- pmix-3.2.2~rc1/config/pmix_load_platform.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_load_platform.m4	2021-01-02 08:56:17.000000000 +0000
@@ -23,44 +23,44 @@
 # PMIX_LOAD_PLATFORM()
 # --------------------
 AC_DEFUN([PMIX_LOAD_PLATFORM], [
-    AC_ARG_WITH([platform-patches-dir],
-        [AC_HELP_STRING([--with-platform-patches-dir=DIR],
+    AC_ARG_WITH([pmix-platform-patches-dir],
+        [AC_HELP_STRING([--with-pmix-platform-patches-dir=DIR],
                         [Location of the platform patches directory. If you use this option, you must also use --with-platform.])])
 
-    AC_ARG_WITH([platform],
-        [AC_HELP_STRING([--with-platform=FILE],
+    AC_ARG_WITH([pmix-platform],
+        [AC_HELP_STRING([--with-pmix-platform=FILE],
                         [Load options for build from FILE.  Options on the
                          command line not in FILE are used.  Options on the
                          command line and in FILE are replaced by what is
                          in FILE.])])
     m4_ifval([autogen_platform_file], [
-        if test "$with_platform" = "" ; then
-            with_platform=autogen_platform_file
+        if test "$with_pmix_platform" = "" ; then
+            with_pmix_platform=autogen_platform_file
         fi])
-    if test "$with_platform" = "yes" ; then
-        AC_MSG_ERROR([--with-platform argument must include FILE option])
-    elif test "$with_platform" = "no" ; then
-        AC_MSG_ERROR([--without-platform is not a valid argument])
-    elif test "$with_platform" != "" ; then
+    if test "$with_pmix_platform" = "yes" ; then
+        AC_MSG_ERROR([--with-pmix_platform argument must include FILE option])
+    elif test "$with_pmix_platform" = "no" ; then
+        AC_MSG_ERROR([--without-pmix_platform is not a valid argument])
+    elif test "$with_pmix_platform" != "" ; then
         # if not an absolute path, check in contrib/platform
-        if test ! "`echo $with_platform | cut -c1`" = "/" && test ! "`echo $with_platform | cut -c2`" = ".." ; then
-            if test -r "${srcdir}/contrib/platform/$with_platform" ; then
-                with_platform="${srcdir}/contrib/platform/$with_platform"
+        if test ! "`echo $with_pmix_platform | cut -c1`" = "/" && test ! "`echo $with_pmix_platform | cut -c2`" = ".." ; then
+            if test -r "${srcdir}/contrib/platform/$with_pmix_platform" ; then
+                with_pmix_platform="${srcdir}/contrib/platform/$with_pmix_platform"
             fi
         fi
 
         # make sure file exists
-        if test ! -r "$with_platform" ; then
-            AC_MSG_ERROR([platform file $with_platform not found])
+        if test ! -r "$with_pmix_platform" ; then
+            AC_MSG_ERROR([platform file $with_pmix_platform not found])
         fi
 
         # eval into environment
-        PMIX_LOG_MSG([Loading environment file $with_platform, with contents below])
-        PMIX_LOG_FILE([$with_platform])
+        PMIX_LOG_MSG([Loading environment file $with_pmix_platform, with contents below])
+        PMIX_LOG_FILE([$with_pmix_platform])
 
         # setup by getting full pathname for the platform directories
-        platform_base="`dirname $with_platform`"
-        platform_file="`basename $with_platform`"
+        platform_base="`dirname $with_pmix_platform`"
+        platform_file="`basename $with_pmix_platform`"
         # get full pathname of where we are so we can return
         platform_savedir="`pwd`"
         # go to where the platform file is located
@@ -74,7 +74,7 @@
         if test "$PMIX_PLATFORM_LOADED" != "" ; then
            platform_loaded="$PMIX_PLATFORM_LOADED"
         else
-           platform_loaded="$with_platform"
+           platform_loaded="$with_pmix_platform"
         fi
         echo "Loaded platform arguments for $platform_loaded"
         PMIX_LOG_MSG([Loaded platform arguments for $platform_loaded])
@@ -101,17 +101,17 @@
             AC_SUBST(PMIX_PARAM_FROM_PLATFORM, "no")
         fi
 
-        patch_dir="${with_platform}.patches"
-        if test -n "$with_platform_patches_dir"; then
-            if test "$with_platform_patches_dir" = "yes"; then
-                patch_dir="${with_platform}.patches"
-            elif test "$with_platform_patches_dir" = "no"; then
+        patch_dir="${with_pmix_platform}.patches"
+        if test -n "$with_pmix_platform_patches_dir"; then
+            if test "$with_pmix_platform_patches_dir" = "yes"; then
+                patch_dir="${with_pmix_platform}.patches"
+            elif test "$with_pmix_platform_patches_dir" = "no"; then
                 AC_MSG_NOTICE([Disabling platform patches on user request])
                 patch_dir=""
-            elif test -d "$with_platform_patches_dir"; then
-                patch_dir=$with_platform_patches_dir
+            elif test -d "$with_pmix_platform_patches_dir"; then
+                patch_dir=$with_pmix_platform_patches_dir
             else
-                AC_MSG_ERROR([User provided patches directory: $with_platform_patches_dir not found])
+                AC_MSG_ERROR([User provided patches directory: $with_pmix_platform_patches_dir not found])
             fi
         fi
 
diff -Nru pmix-3.2.2~rc1/config/pmix.m4 pmix-4.0.0/config/pmix.m4
--- pmix-3.2.2~rc1/config/pmix.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix.m4	2021-01-02 08:56:17.000000000 +0000
@@ -10,7 +10,7 @@
 dnl                         University of Stuttgart.  All rights reserved.
 dnl Copyright (c) 2004-2005 The Regents of the University of California.
 dnl                         All rights reserved.
-dnl Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
 dnl Copyright (c) 2009-2018 IBM Corporation.  All rights reserved.
 dnl Copyright (c) 2009      Los Alamos National Security, LLC.  All rights
@@ -654,7 +654,7 @@
     # -lrt might be needed for clock_gettime
     PMIX_SEARCH_LIBS_CORE([clock_gettime], [rt])
 
-    AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate tcgetpgrp setpgid ptsname openpty setenv fork execve waitpid])
+    AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate tcgetpgrp setpgid ptsname openpty setenv fork execve waitpid atexit])
 
     # On some hosts, htonl is a define, so the AC_CHECK_FUNC will get
     # confused.  On others, it's in the standard library, but stubbed with
@@ -719,6 +719,9 @@
     LDFLAGS="$LDFLAGS $THREAD_LDFLAGS"
     LIBS="$LIBS $THREAD_LIBS"
 
+    PMIX_WRAPPER_FLAGS_ADD([CFLAGS], [$THREAD_CFLAGS])
+    PMIX_WRAPPER_FLAGS_ADD([LDFLAGS], [$THREAD_LDFLAGS])
+
     #
     # What is the local equivalent of "ln -s"
     #
@@ -762,19 +765,19 @@
 
 
     ##################################
-    # HWLOC
+    # JANSSON
     ##################################
-    pmix_show_title "HWLOC"
+    pmix_show_title "JANSSON"
 
-    PMIX_HWLOC_CONFIG
+    PMIX_CHECK_JANSSON
 
 
     ##################################
-    # ZLIB
+    # CURL
     ##################################
-    pmix_show_title "ZLIB"
+    pmix_show_title "CURL"
 
-    PMIX_ZLIB_CONFIG
+    PMIX_CHECK_CURL
 
 
     ##################################
@@ -860,6 +863,20 @@
     LIBS="$LIBS $PMIX_FINAL_LIBS"
 
     ############################################################################
+    # final wrapper compiler config
+    ############################################################################
+    pmix_show_subtitle "Wrapper compiler final setup"
+
+    # The PMIx wrapper script (i.e., not the C-compiled
+    # executables) need perl.
+    AC_PATH_PROG(PERL, perl, perl)
+
+    # Need the libtool executable before the rpathify stuff
+    LT_OUTPUT
+
+    PMIX_SETUP_WRAPPER_FINAL
+
+    ############################################################################
     # pmixdatadir, pmixlibdir, and pmixinclude are essentially the same as
     # pkg*dir, but will always be */pmix.
     pmixdatadir='${datadir}/pmix'
@@ -894,9 +911,19 @@
     AC_CONFIG_FILES(pmix_config_prefix[test/run_tests13.pl], [chmod +x test/run_tests13.pl])
 #    AC_CONFIG_FILES(pmix_config_prefix[test/run_tests14.pl], [chmod +x test/run_tests14.pl])
 #    AC_CONFIG_FILES(pmix_config_prefix[test/run_tests15.pl], [chmod +x test/run_tests15.pl])
+    if test "$WANT_PYTHON_BINDINGS" = "1"; then
+        AC_CONFIG_FILES(pmix_config_prefix[test/python/run_server.sh], [chmod +x test/python/run_server.sh])
+        AC_CONFIG_FILES(pmix_config_prefix[test/python/run_sched.sh], [chmod +x test/python/run_sched.sh])
+    fi
 
 
     ############################################################################
+    # Check for building man pages
+    ############################################################################
+    pmix_show_subtitle "Man page setup"
+    PMIX_SETUP_MAN_PAGES
+
+    ############################################################################
     # final output
     ############################################################################
 
@@ -906,16 +933,23 @@
 
     AC_CONFIG_FILES(
         pmix_config_prefix[Makefile]
+        pmix_config_prefix[bindings/Makefile]
+        pmix_config_prefix[bindings/python/Makefile]
         pmix_config_prefix[config/Makefile]
         pmix_config_prefix[etc/Makefile]
         pmix_config_prefix[include/Makefile]
         pmix_config_prefix[src/Makefile]
+        pmix_config_prefix[src/include/Makefile]
         pmix_config_prefix[src/util/keyval/Makefile]
         pmix_config_prefix[src/mca/base/Makefile]
         pmix_config_prefix[src/tools/pevent/Makefile]
         pmix_config_prefix[src/tools/pmix_info/Makefile]
         pmix_config_prefix[src/tools/plookup/Makefile]
         pmix_config_prefix[src/tools/pps/Makefile]
+        pmix_config_prefix[src/tools/pattrs/Makefile]
+        pmix_config_prefix[src/tools/pquery/Makefile]
+        pmix_config_prefix[src/tools/wrapper/Makefile]
+        pmix_config_prefix[src/tools/wrapper/pmixcc-wrapper-data.txt]
         )
 
     # publish any embedded flags so external wrappers can use them
@@ -1161,22 +1195,6 @@
 AC_DEFINE_UNQUOTED([PMIX_ENABLE_TIMING], [$WANT_PMIX_TIMING],
                    [Whether we want developer-level timing support or not])
 
-#
-#
-# Install backward compatibility support for PMI-1 and PMI-2
-#
-AC_MSG_CHECKING([if want backward compatibility for PMI-1 and PMI-2])
-AC_ARG_ENABLE(pmi-backward-compatibility,
-              AC_HELP_STRING([--enable-pmi-backward-compatibility],
-                             [enable PMIx support for PMI-1 and PMI-2 (default: enabled)]))
-if test "$enable_pmi_backward_compatibility" = "no"; then
-    AC_MSG_RESULT([no])
-    WANT_PMI_BACKWARD=0
-else
-    AC_MSG_RESULT([yes])
-    WANT_PMI_BACKWARD=1
-fi
-
 AM_CONDITIONAL([WANT_INSTALL_HEADERS], [test $WANT_INSTALL_HEADERS -eq 1])
 
 #
@@ -1196,6 +1214,83 @@
 
 AM_CONDITIONAL([PMIX_INSTALL_BINARIES], [test $WANT_PMIX_BINARIES -eq 1])
 
+#
+# Install Python bindings?
+#
+AC_MSG_CHECKING([if want install Python bindings])
+AC_ARG_ENABLE(python-bindings,
+              AC_HELP_STRING([--enable-python-bindings],
+                             [enable Python bindings (default: disabled)]))
+if test "$enable_python_bindings" != "yes"; then
+    AC_MSG_RESULT([no])
+    WANT_PYTHON_BINDINGS=0
+else
+    AC_MSG_RESULT([yes])
+    WANT_PYTHON_BINDINGS=1
+fi
+
+AM_CONDITIONAL([WANT_PYTHON_BINDINGS], [test $WANT_PYTHON_BINDINGS -eq 1])
+
+AM_PATH_PYTHON([3.4], [pmix_python_good=yes], [pmix_python_good=no])
+
+if test "$WANT_PYTHON_BINDINGS" = "1"; then
+    if test "$pmix_python_good" = "no"; then
+        AC_MSG_WARN([Python bindings were enabled, but no suitable])
+        AC_MSG_WARN([interpreter was found. PMIx requires at least])
+        AC_MSG_WARN([Python v3.4 to provide Python bindings])
+        AC_MSG_ERROR([Cannot continue])
+    fi
+    pyvers=`python3 --version`
+    python_version=${pyvers#"Python "}
+
+    PMIX_SUMMARY_ADD([[Bindings]],[[Python]], [pmix_python], [yes ($python_version)])
+
+    AC_MSG_CHECKING([if Cython package installed as Python package])
+    have_cython=`$srcdir/config/pmix_check_cython.py 2> /dev/null`
+    if test "$have_cython" = "0"; then
+        AC_MSG_RESULT([yes])
+        AC_MSG_CHECKING([Cython version])
+        cython_version=`python -c "from Cython.Compiler.Version import version; print(version)"`
+        AC_MSG_RESULT([$cython_version])
+        PMIX_SUMMARY_ADD([[Bindings]],[[Cython]], [pmix_cython], [yes ($cython_version)])
+    else
+        AC_MSG_RESULT([no])
+        # Cython doesn't have any include or lib files - it is just a binary
+        AC_CHECK_PROG(pmix_cython_rpm, cython, ["found"], ["not found"])
+        if test "$pmix_cython_rpm" = "found"; then
+            AC_MSG_CHECKING([Cython version])
+            cyvers=`cython --version 2>&1`
+            cython_version=${cyvers#"Cython version "}
+            AC_MSG_RESULT([$cython_version])
+            PMIX_SUMMARY_ADD([[Bindings]],[[Cython]], [pmix_cython], [yes ($cython_version)])
+        else
+            AC_MSG_RESULT([no])
+            AC_MSG_WARN([Python bindings were enabled, but the Cython])
+            AC_MSG_WARN([package was not found. PMIx Python bindings])
+            AC_MSG_WARN([require that the Cython package be installed])
+            AC_MSG_ERROR([Cannot continue])
+        fi
+    fi
+
+    pmix_pythondir=`eval echo $pythondir`
+    AC_SUBST([PMIX_PYTHON_EGG_PATH], [$pmix_pythondir], [Path to installed Python egg])
+fi
+
+# If we didn't find a good Python and we don't have dictionary.h, then
+# see if we can find an older Python (because construct_dictionary.py
+# can use an older Python).
+AS_IF([test "$PYTHON" = ":" && test ! -f $srcdir/include/dictionary.h],
+      [PYTHON=
+       AM_PATH_PYTHON
+       # If we still can't find Python (and we don't have
+       # dictionary.h), then give up.
+       AS_IF([test "$PYTHON" = ":"],
+             [AC_MSG_WARN([Could not find a modern enough Python])
+              AC_MSG_WARN([Developer builds (e.g., git clones) of OpenPMIx must have Python available])
+              AC_MSG_ERROR([Cannot continue])
+             ])
+       ])
+
 # see if they want to disable non-RTLD_GLOBAL dlopen
 AC_MSG_CHECKING([if want to support dlopen of non-global namespaces])
 AC_ARG_ENABLE([nonglobal-dlopen],
@@ -1264,6 +1359,8 @@
         AM_CONDITIONAL(WANT_INSTALL_HEADERS, test "$WANT_INSTALL_HEADERS" = 1)
         AM_CONDITIONAL(WANT_PMI_BACKWARD, test "$WANT_PMI_BACKWARD" = 1)
         AM_CONDITIONAL(NEED_LIBPMIX, [test "$pmix_need_libpmix" = "1"])
+        AM_CONDITIONAL([PMIX_HAVE_JANSSON], [test "x$pmix_check_jansson_happy" = "xyes"])
+        AM_CONDITIONAL([PMIX_HAVE_CURL], [test "x$pmix_check_curl_happy" = "xyes"])
     ])
     pmix_did_am_conditionals=yes
 ])dnl
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_cc.m4 pmix-4.0.0/config/pmix_setup_cc.m4
--- pmix-3.2.2~rc1/config/pmix_setup_cc.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_cc.m4	2021-01-02 08:56:17.000000000 +0000
@@ -17,6 +17,9 @@
 dnl Copyright (c) 2015-2019 Research Organization for Information Science
 dnl                         and Technology (RIST).  All rights reserved.
 dnl Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+dnl Copyright (c) 2020      Triad National Security, LLC. All rights
+dnl                         reserved.
+dnl
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -321,7 +324,7 @@
                       pmix_cv_cc_wno_long_double="yes"
                       if test -s conftest.err ; then
                           dnl Yes, it should be "ignor", in order to catch ignoring and ignore
-                          for i in unknown invalid ignor unrecognized ; do
+                          for i in unknown invalid ignor unrecognized 'not supported'; do
                               $GREP -iq $i conftest.err
                               if test "$?" = "0" ; then
                                   pmix_cv_cc_wno_long_double="no"
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_hwloc.m4 pmix-4.0.0/config/pmix_setup_hwloc.m4
--- pmix-3.2.2~rc1/config/pmix_setup_hwloc.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_hwloc.m4	1970-01-01 00:00:00.000000000 +0000
@@ -1,168 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
-# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_hwloc_CONFIG([action-if-found], [action-if-not-found])
-# --------------------------------------------------------------------
-AC_DEFUN([PMIX_HWLOC_CONFIG],[
-    AC_ARG_WITH([hwloc-header],
-                [AC_HELP_STRING([--with-hwloc-header=HEADER],
-                                [The value that should be included in C files to include hwloc.h])])
-
-    AS_IF([test "$pmix_mode" = "embedded"],
-          [_PMIX_HWLOC_EMBEDDED_MODE],
-          [_PMIX_HWLOC_EXTERNAL])
-
-    AC_MSG_CHECKING([hwloc header])
-    AC_DEFINE_UNQUOTED([PMIX_HWLOC_HEADER], [$PMIX_HWLOC_HEADER],
-                       [Location of hwloc.h])
-    AC_MSG_RESULT([$PMIX_HWLOC_HEADER])
-
-    AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC], [$pmix_hwloc_support],
-                   [Whether or not we have hwloc support])
-
-    PMIX_SUMMARY_ADD([[External Packages]],[[HWLOC]], [pmix_hwloc], [$pmix_hwloc_support_will_build ($pmix_hwloc_source)])
-])
-
-AC_DEFUN([_PMIX_HWLOC_EMBEDDED_MODE],[
-    AC_MSG_CHECKING([for hwloc])
-    AC_MSG_RESULT([assumed available (embedded mode)])
-
-    AS_IF([test -z "$with_hwloc_header" || test "$with_hwloc_header" = "yes"],
-          [PMIX_HWLOC_HEADER="<hwloc.h>"],
-          [PMIX_HWLOC_HEADER="$with_hwloc_header"])
-
-    pmix_hwloc_support=1
-    pmix_hwloc_source=embedded
-    pmix_hwloc_support_will_build=yes
- ])
-
-AC_DEFUN([_PMIX_HWLOC_EXTERNAL],[
-    PMIX_VAR_SCOPE_PUSH([pmix_hwloc_dir pmix_hwloc_libdir pmix_hwloc_standard_lib_location pmix_hwloc_standard_header_location pmix_check_hwloc_save_CPPFLAGS pmix_check_hwloc_save_LDFLAGS pmix_check_hwloc_save_LIBS])
-
-    AC_ARG_WITH([hwloc],
-                [AC_HELP_STRING([--with-hwloc=DIR],
-                                [Search for hwloc headers and libraries in DIR ])])
-
-    AC_ARG_WITH([hwloc-libdir],
-                [AC_HELP_STRING([--with-hwloc-libdir=DIR],
-                                [Search for hwloc libraries in DIR ])])
-
-    pmix_hwloc_support=0
-    pmix_check_hwloc_save_CPPFLAGS="$CPPFLAGS"
-    pmix_check_hwloc_save_LDFLAGS="$LDFLAGS"
-    pmix_check_hwloc_save_LIBS="$LIBS"
-    pmix_hwloc_standard_header_location=yes
-    pmix_hwloc_standard_lib_location=yes
-
-    AS_IF([test "$with_hwloc" = "internal" || test "$with_hwloc" = "external"],
-          [with_hwloc=])
-
-    if test "$with_hwloc" != "no"; then
-        AC_MSG_CHECKING([for hwloc in])
-        if test ! -z "$with_hwloc" && test "$with_hwloc" != "yes"; then
-            pmix_hwloc_dir=$with_hwloc/include
-            pmix_hwloc_standard_header_location=no
-            pmix_hwloc_standard_lib_location=no
-            AS_IF([test -z "$with_hwloc_libdir" || test "$with_hwloc_libdir" = "yes"],
-                  [if test -d $with_hwloc/lib; then
-                       pmix_hwloc_libdir=$with_hwloc/lib
-                   elif test -d $with_hwloc/lib64; then
-                       pmix_hwloc_libdir=$with_hwloc/lib64
-                   else
-                       AC_MSG_RESULT([Could not find $with_hwloc/lib or $with_hwloc/lib64])
-                       AC_MSG_ERROR([Can not continue])
-                   fi
-                   AC_MSG_RESULT([$pmix_hwloc_dir and $pmix_hwloc_libdir])],
-                  [AC_MSG_RESULT([$with_hwloc_libdir])])
-        else
-            pmix_hwloc_dir=/usr/include
-            if test -d /usr/lib; then
-                pmix_hwloc_libdir=/usr/lib
-            elif test -d /usr/lib64; then
-                pmix_hwloc_libdir=/usr/lib64
-            else
-                AC_MSG_RESULT([not found])
-                AC_MSG_WARN([Could not find /usr/lib or /usr/lib64 - you may])
-                AC_MSG_WARN([need to specify --with-hwloc_libdir=<path>])
-                AC_MSG_ERROR([Can not continue])
-            fi
-            AC_MSG_RESULT([(default search paths)])
-            pmix_hwloc_standard_header_location=yes
-            pmix_hwloc_standard_lib_location=yes
-        fi
-        AS_IF([test ! -z "$with_hwloc_libdir" && test "$with_hwloc_libdir" != "yes"],
-              [pmix_hwloc_libdir="$with_hwloc_libdir"
-               pmix_hwloc_standard_lib_location=no])
-
-        PMIX_CHECK_PACKAGE([pmix_hwloc],
-                           [hwloc.h],
-                           [hwloc],
-                           [hwloc_topology_init],
-                           [-lhwloc],
-                           [$pmix_hwloc_dir],
-                           [$pmix_hwloc_libdir],
-                           [pmix_hwloc_support=1],
-                           [pmix_hwloc_support=0])
-
-        AS_IF([test "$pmix_hwloc_standard_header_location" != "yes"],
-              [PMIX_FLAGS_APPEND_UNIQ(CPPFLAGS, $pmix_hwloc_CPPFLAGS)])
-
-        AS_IF([test "$pmix_hwloc_standard_lib_location" != "yes"],
-              [PMIX_FLAGS_APPEND_UNIQ(LDFLAGS, $pmix_hwloc_LDFLAGS)])
-        PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_hwloc_LIBS)
-    fi
-
-    if test ! -z "$with_hwloc" && test "$with_hwloc" != "no" && test "$pmix_hwloc_support" != "1"; then
-        AC_MSG_WARN([HWLOC SUPPORT REQUESTED AND NOT FOUND])
-        AC_MSG_ERROR([CANNOT CONTINUE])
-    fi
-
-    if test $pmix_hwloc_support = "1"; then
-        AC_MSG_CHECKING([if external hwloc version is 1.5 or greater])
-        AC_COMPILE_IFELSE(
-              [AC_LANG_PROGRAM([[#include <hwloc.h>]],
-              [[
-    #if HWLOC_API_VERSION < 0x00010500
-    #error "hwloc API version is less than 0x00010500"
-    #endif
-              ]])],
-              [AC_MSG_RESULT([yes])],
-              [AC_MSG_RESULT([no])
-               AC_MSG_ERROR([Cannot continue])])
-    fi
-
-    CPPFLAGS=$pmix_check_hwloc_save_CPPFLAGS
-    LDFLAGS=$pmix_check_hwloc_save_LDFLAGS
-    LIBS=$pmix_check_hwloc_save_LIBS
-
-    AC_MSG_CHECKING([will hwloc support be built])
-    if test "$pmix_hwloc_support" != "1"; then
-        AC_MSG_RESULT([no])
-        pmix_hwloc_source=none
-        pmix_hwloc_support_will_build=no
-    else
-        AC_MSG_RESULT([yes])
-        pmix_hwloc_source=$pmix_hwloc_dir
-        pmix_hwloc_support_will_build=yes
-        AS_IF([test "$pmix_hwloc_standard_header_location" != "yes"],
-              [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_CPPFLAGS, $pmix_hwloc_CPPFLAGS)])
-
-        AS_IF([test "$pmix_hwloc_standard_lib_location" != "yes"],
-              [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LDFLAGS, $pmix_hwloc_LDFLAGS)])
-        PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LIBS, $pmix_hwloc_LIBS)
-    fi
-
-    # Set output variables
-    PMIX_HWLOC_HEADER="<hwloc.h>"
-
-    PMIX_VAR_SCOPE_POP
-])dnl
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_libevent.m4 pmix-4.0.0/config/pmix_setup_libevent.m4
--- pmix-3.2.2~rc1/config/pmix_setup_libevent.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_libevent.m4	2021-01-02 08:56:17.000000000 +0000
@@ -6,6 +6,8 @@
 # Copyright (c) 2017-2019 Research Organization for Information Science
 #                         and Technology (RIST).  All rights reserved.
 # Copyright (c) 2020      IBM Corporation.  All rights reserved.
+# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
+#                         reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -13,19 +15,51 @@
 # $HEADER$
 #
 
+#
+# We have three modes for building libevent.
+#
+# First is an embedded libevent, where PMIx is being built into
+# another library and assumes that libevent is available, that there
+# is a single header (pointed to by --with-libevent-header) which
+# includes all the Libevent bits, and that the right libevent
+# configuration is used.  This mode is used when --enable-embeded-mode
+# is specified to configure.
+#
+# Second is as a co-built libevent.  In this case, PMIx's CPPFLAGS
+# will be set before configure to include the right -Is to pick up
+# libevent headers and LIBS will point to where the .la file for
+# libevent will exist.  When co-building, libevent's configure will be
+# run already, but the library will not yet be built.  It is ok to run
+# any compile-time (not link-time) tests in this mode.  This mode is
+# used when the --with-libevent=cobuild option is specified.
+#
+# Third is an external package.  In this case, all compile and link
+# time tests can be run.  This macro must do any CPPFLAGS/LDFLAGS/LIBS
+# modifications it desires in order to compile and link against
+# libevent.  This mode is used whenever the other modes are not used.
+#
 # MCA_libevent_CONFIG([action-if-found], [action-if-not-found])
 # --------------------------------------------------------------------
 AC_DEFUN([PMIX_LIBEVENT_CONFIG],[
+
+    AC_ARG_WITH([libevent],
+                [AC_HELP_STRING([--with-libevent=DIR],
+                                [Search for libevent headers and libraries in DIR ])])
+
+    AC_ARG_WITH([libevent-libdir],
+                [AC_HELP_STRING([--with-libevent-libdir=DIR],
+                                [Search for libevent libraries in DIR ])])
+
     AC_ARG_WITH([libevent-header],
                 [AC_HELP_STRING([--with-libevent-header=HEADER],
-                                [The value that should be included in C files to include event.h])])
+                                [The value that should be included in C files to include event.h.  This option only has meaning if --enable-embedded-mode is enabled.])])
 
     pmix_libevent_support=0
 
-    AS_IF([test "$pmix_mode" = "embedded"],
-          [_PMIX_LIBEVENT_EMBEDDED_MODE],
-          [AS_IF([test $pmix_libev_support -eq 0],
-                 [_PMIX_LIBEVENT_EXTERNAL])])
+    # figure out our mode...
+    AS_IF([test "$with_libevent" = "cobuild"],
+          [_PMIX_LIBEVENT_EMBEDDED_MODE(cobuild)],
+          [_PMIX_LIBEVENT_EXTERNAL])
 
     if test $pmix_libevent_support -eq 1; then
         AC_MSG_CHECKING([libevent header])
@@ -39,11 +73,12 @@
 
         PMIX_SUMMARY_ADD([[External Packages]],[[Libevent]], [pmix_libevent], [yes ($pmix_libevent_source)])
     fi
+
 ])
 
-AC_DEFUN([_PMIX_LIBEVENT_EMBEDDED_MODE],[
+AC_DEFUN([_PMIX_LIBEVENT_EMBEDDED_MODE], [
     AC_MSG_CHECKING([for libevent])
-    AC_MSG_RESULT([assumed available (embedded mode)])
+    AC_MSG_RESULT([$1])
 
     AS_IF([test -z "$with_libevent_header" || test "$with_libevent_header" = "yes"],
           [PMIX_EVENT_HEADER="<event.h>"
@@ -51,21 +86,24 @@
           [PMIX_EVENT_HEADER="$with_libevent_header"
            PMIX_EVENT2_THREAD_HEADER="$with_libevent_header"])
 
-    pmix_libevent_source=embedded
+    AC_MSG_CHECKING([if co-built libevent includes thread support])
+    AC_TRY_COMPILE([#include <event.h>
+#include <event2/thread.h>
+           ],[
+#if !(EVTHREAD_LOCK_API_VERSION >= 1)
+#error "No threads!"
+#endif
+    ],[AC_MSG_RESULT([yes])],
+    [AC_MSG_RESULT([no])
+    AC_MSG_ERROR([No thread support in co-build libevent.  Aborting])])
+
+    pmix_libevent_source=$1
     pmix_libevent_support=1
 ])
 
 AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[
     PMIX_VAR_SCOPE_PUSH([pmix_event_dir pmix_event_libdir pmix_event_defaults pmix_check_libevent_save_CPPFLAGS pmix_check_libevent_save_LDFLAGS pmix_check_libevent_save_LIBS])
 
-    AC_ARG_WITH([libevent],
-                [AC_HELP_STRING([--with-libevent=DIR],
-                                [Search for libevent headers and libraries in DIR ])])
-
-    AC_ARG_WITH([libevent-libdir],
-                [AC_HELP_STRING([--with-libevent-libdir=DIR],
-                                [Search for libevent libraries in DIR ])])
-
     pmix_check_libevent_save_CPPFLAGS="$CPPFLAGS"
     pmix_check_libevent_save_LDFLAGS="$LDFLAGS"
     pmix_check_libevent_save_LIBS="$LIBS"
@@ -181,8 +219,11 @@
         pmix_libevent_source=$pmix_event_dir
         AS_IF([test "$pmix_event_defaults" = "no"],
               [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_CPPFLAGS, $pmix_libevent_CPPFLAGS)
-               PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LDFLAGS, $pmix_libevent_LDFLAGS)])
+               PMIX_WRAPPER_FLAGS_ADD(CPPFLAGS, $pmix_libevent_CPPFLAGS)
+               PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LDFLAGS, $pmix_libevent_LDFLAGS)
+               PMIX_WRAPPER_FLAGS_ADD(LDFLAGS, $pmix_libevent_LDFLAGS)])
         PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LIBS, $pmix_libevent_LIBS)
+        PMIX_WRAPPER_FLAGS_ADD(LIBS, $pmix_libevent_LIBS)
     else
         AC_MSG_RESULT([no])
     fi
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_libev.m4 pmix-4.0.0/config/pmix_setup_libev.m4
--- pmix-3.2.2~rc1/config/pmix_setup_libev.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_libev.m4	2021-01-02 08:56:17.000000000 +0000
@@ -73,11 +73,14 @@
 
     AS_IF([test $pmix_libev_support -eq 1],
           [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LIBS, $pmix_libev_LIBS)
+           PMIX_WRAPPER_FLAGS_ADD(LIBS, $pmix_libev_LIBS)
 
            AS_IF([test "$pmix_libev_standard_header_location" != "yes"],
-                 [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_CPPFLAGS, $pmix_libev_CPPFLAGS)])
+                 [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_CPPFLAGS, $pmix_libev_CPPFLAGS)
+                  PMIX_WRAPPER_FLAGS_ADD(CPPFLAGS, $pmix_libev_CPPFLAGS)])
            AS_IF([test "$pmix_libev_standard_lib_location" != "yes"],
-                 [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LDFLAGS, $pmix_libev_LDFLAGS)])])
+                 [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LDFLAGS, $pmix_libev_LDFLAGS)
+                  PMIX_WRAPPER_FLAGS_ADD(LDFLAGS, $pmix_libev_LDFLAGS)])])
 
     AC_MSG_CHECKING([will libev support be built])
     if test $pmix_libev_support -eq 1; then
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_man_pages.m4 pmix-4.0.0/config/pmix_setup_man_pages.m4
--- pmix-3.2.2~rc1/config/pmix_setup_man_pages.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_man_pages.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,91 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved.
+dnl
+dnl Copyright (c) 2020      Intel, Inc.  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+dnl
+dnl Just in case someone looks for it here someday, here is a
+dnl conveninent reference for what Markdown pandoc supports:
+dnl
+dnl https://rmarkdown.rstudio.com/authoring_pandoc_markdown.html
+dnl
+
+AC_DEFUN([PMIX_SETUP_MAN_PAGES],[
+    AC_ARG_ENABLE(man-pages,
+                  [AC_HELP_STRING([--disable-man-pages],
+                                  [Do not generate/install man pages (default: enabled)])])
+
+    PANDOC=
+    PMIX_ENABLE_MAN_PAGES=0
+    AC_MSG_CHECKING([if want man pages])
+    AS_IF([test -z "$enable_man_pages" || test "$enable_man_pages" = "yes"],
+          [AC_MSG_RESULT([yes])
+	   PMIX_ENABLE_MAN_PAGES=1
+	   _PMIX_SETUP_PANDOC],
+	  [AC_MSG_RESULT([no])])
+
+    AC_SUBST(PANDOC)
+    AM_CONDITIONAL([PMIX_ENABLE_MAN_PAGES], [test $PMIX_ENABLE_MAN_PAGES -eq 1])
+    AC_DEFINE_UNQUOTED([PMIX_ENABLE_MAN_PAGES], [$PMIX_ENABLE_MAN_PAGES],
+                       [Whether or not we will build manpages])
+
+    AS_IF([test $PMIX_ENABLE_MAN_PAGES -eq 1],
+          [PMIX_SUMMARY_ADD([[Options]],[[Manpages built]], [pmix_manpages], [yes])],
+          [PMIX_SUMMARY_ADD([[Options]],[[Manpages built]], [pmix_manpages], [yes])])
+
+])
+
+dnl Back-end pandoc setup
+AC_DEFUN([_PMIX_SETUP_PANDOC],[
+    PMIX_VAR_SCOPE_PUSH([min_major_version min_minor_version pandoc_version pandoc_major pandoc_minor])
+
+    # If we need to generate man pages, we need pandoc >v1.12.
+    AC_PATH_PROG([PANDOC], [pandoc])
+
+    # If we found Pandoc, check its version.  We need >=v1.12.
+    # To be clear: I know that v1.12 works, and I know that v1.9 does not
+    # work.  I did not test the versions in between to know exactly what
+    # the lowest version is that works.  Someone is free to update this
+    # check someday to be more accurate if they wish.
+    min_major_version=1
+    min_minor_version=12
+    AS_IF([test -n "$PANDOC"],
+          [pandoc_version=`pandoc --version | head -n 1 | awk '{ print $[2] }'`
+           pandoc_major=`echo $pandoc_version | cut -d\. -f1`
+           pandoc_minor=`echo $pandoc_version | cut -d\. -f2`
+           AC_MSG_CHECKING([pandoc version])
+           AC_MSG_RESULT([major: $pandoc_major, minor: $pandoc_minor])
+
+           AC_MSG_CHECKING([if pandoc version is >=v$min_major_version.$min_minor_version])
+           AS_IF([test $pandoc_major -lt $min_major_version], [PANDOC=])
+           AS_IF([test $pandoc_major -eq $min_major_version && test $pandoc_minor -lt $min_minor_version],
+                 [PANDOC=])
+           AS_IF([test -n "$PANDOC"],
+                 [AC_MSG_RESULT([yes])],
+                 [AC_MSG_RESULT([no])])
+          ])
+
+    AS_IF([test -z "$PANDOC" || test -n "`echo $PANDOC | $GREP missing`"],
+          [AS_IF([test "$PMIX_DEVEL" = "1" && test -z "$enable_man_pages"],
+                 [AC_MSG_CHECKING([man pages will be built])
+                  AC_MSG_RESULT([no - adequate pandoc installation not found])
+                  PANDOC=
+                  PMIX_ENABLE_MAN_PAGES=0],
+                 [AS_IF([test ! -f "$srcdir/tools/wrapper/pmixcc.1"],
+                         [AC_MSG_WARN([*** Could not find a suitable pandoc on your system.])
+                          AC_MSG_WARN([*** You need pandoc >=$min_major_version.$min_minor_version to build OpenPMIx man pages.])
+                          AC_MSG_WARN([*** See pandoc.org.])
+                          AC_MSG_WARN([*** NOTE: If you are building from a tarball downloaded from the OpenPMIx GitHub repository, you do not need Pandoc])
+                          AC_MSG_ERROR([Cannot continue])
+                         ])])
+           ])
+
+    PMIX_VAR_SCOPE_POP
+])
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_wrappers.m4 pmix-4.0.0/config/pmix_setup_wrappers.m4
--- pmix-3.2.2~rc1/config/pmix_setup_wrappers.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_wrappers.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,308 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+dnl                         University Research and Technology
+dnl                         Corporation.  All rights reserved.
+dnl Copyright (c) 2004-2005 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+dnl                         University of Stuttgart.  All rights reserved.
+dnl Copyright (c) 2004-2005 The Regents of the University of California.
+dnl                         All rights reserved.
+dnl Copyright (c) 2006-2010 Oracle and/or its affiliates.  All rights reserved.
+dnl Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
+dnl Copyright (c) 2015-2017 Research Organization for Information Science
+dnl                         and Technology (RIST). All rights reserved.
+dnl Copyright (c) 2016      IBM Corporation.  All rights reserved.
+dnl Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+# PMIX_WRAPPER_FLAGS_ADD(variable, new_argument)
+# ----------------------------------------------
+# Add new_argument to the list of arguments for variable in the
+# wrapper compilers, if it's not already there.  For example:
+#   PMIX_WRAPPER_FLAGS_ADD(CFLAGS, "-pthread")
+# will add -pthread to the list of CFLAGS the wrappers use when invoked.
+#
+# This macro MAY NOT be invoked from configure macros for MCA components.
+# See the comment in SETUP_WRAPPER_INIT (below) for more information.
+AC_DEFUN([PMIX_WRAPPER_FLAGS_ADD], [
+    m4_ifdef([mca_component_configure_active],
+        [m4_fatal([PMIX_WRAPPER_FLAGS_ADD can not be called from a component configure])])
+    m4_if([$1], [CPPFLAGS], [PMIX_FLAGS_APPEND_UNIQ([wrapper_extra_cppflags], [$2])],
+          [$1], [CFLAGS], [PMIX_FLAGS_APPEND_UNIQ([wrapper_extra_cflags], [$2])],
+          [$1], [LDFLAGS], [PMIX_FLAGS_APPEND_UNIQ([wrapper_extra_ldflags], [$2])],
+          [$1], [LIBS], [PMIX_FLAGS_APPEND_UNIQ([wrapper_extra_libs], [$2])],
+          [m4_fatal([Unknown wrapper flag type $1])])
+])
+
+
+# PMIX_SETUP_WRAPPER_INIT()
+# -------------------------
+# Setup wrapper compiler configuration information.  Should be called early to
+# prevent lots of calculations and then an abort for a silly user typo.  This
+# macro works in pair with PMIX_SETUP_WRAPPER_FINAL, which should be called
+# almost at the end of configure (after the last call to PMIX_WRAPPER_FLAGS_ADD
+# and after the MCA system has been setup).
+#
+# The wrapper compiler arguments are a little fragile and should NOT
+# be edited by configure directly.  Instead, main configure should use
+# PMIX_WRAPPER_FLAGS_ADD.
+#
+# When building statically, the MCA system will add
+# <framework>_<component>_WRAPPER_EXTRA_{LDFLAGS, LIBS} if set and try
+# to add <framework>_<component>_{LDFLAGS, LIBS} (if not an external
+# configure) to the wrapper LDFLAGS and LIBS.  Any arguments in
+# <framework>_<component>_WRAPPER_EXTRA_CPPFLAGS are passed to the
+# wrapper compilers IF AND ONLY IF the framework was a STOP_AT_FIRST
+# framework, the component is a static component, and devel headers
+# are installed.  Note that MCA components are ONLY allowed to
+# (indirectly) influence the wrapper CPPFLAGS, LDFLAGS, and LIBS.
+# That is, a component may not influence CFLAGS.
+#
+# Notes:
+#   * Keep user flags separate as 1) they should have no influence
+#     over build and 2) they don't go through the uniqification we do
+#     with the other wrapper compiler options
+#   * While the user (the person who runs configure) is allowed to set
+#     <flag>_prefix, configure is not.  There's no known use case for
+#     doing so, and we'd like to force the issue.
+AC_DEFUN([PMIX_SETUP_WRAPPER_INIT],[
+    AC_ARG_WITH([wrapper-cflags],
+                [AC_HELP_STRING([--with-wrapper-cflags],
+                                [Extra flags to add to CFLAGS when using mpicc])])
+    AS_IF([test "$with_wrapper_cflags" = "yes" || test "$with_wrapper_cflags" = "no"],
+          [AC_MSG_ERROR([--with-wrapper-cflags must have an argument.])])
+
+    AC_ARG_WITH([wrapper-cflags-prefix],
+                [AC_HELP_STRING([--with-wrapper-cflags-prefix],
+                                [Extra flags (before user flags) to add to CFLAGS when using mpicc])])
+    AS_IF([test "$with_wrapper_cflags_prefix" = "yes" || test "$with_wrapper_cflags_prefix" = "no"],
+          [AC_MSG_ERROR([--with-wrapper-cflags-prefix must have an argument.])])
+
+    AC_ARG_WITH([wrapper-ldflags],
+                [AC_HELP_STRING([--with-wrapper-ldflags],
+                                [Extra flags to add to LDFLAGS when using wrapper compilers])])
+    AS_IF([test "$with_wrapper_ldflags" = "yes" || test "$with_wrapper_ldflags" = "no"],
+          [AC_MSG_ERROR([--with-wrapper-ldflags must have an argument.])])
+
+    AC_ARG_WITH([wrapper-libs],
+                [AC_HELP_STRING([--with-wrapper-libs],
+                                [Extra flags to add to LIBS when using wrapper compilers])])
+    AS_IF([test "$with_wrapper_libs" = "yes" || test "$with_wrapper_libs" = "no"],
+          [AC_MSG_ERROR([--with-wrapper-libs must have an argument.])])
+
+    AC_MSG_CHECKING([if want wrapper compiler rpath support])
+    AC_ARG_ENABLE([wrapper-rpath],
+                  [AS_HELP_STRING([--enable-wrapper-rpath],
+                  [enable rpath/runpath support in the wrapper compilers (default=yes)])])
+    AS_IF([test "$enable_wrapper_rpath" != "no"], [enable_wrapper_rpath=yes])
+    AC_MSG_RESULT([$enable_wrapper_rpath])
+
+    AC_MSG_CHECKING([if want wrapper compiler runpath support])
+    AC_ARG_ENABLE([wrapper-runpath],
+                  [AS_HELP_STRING([--enable-wrapper-runpath],
+                  [enable runpath in the wrapper compilers if linker supports it (default: enabled,  unless wrapper-rpath is disabled).])])
+    AS_IF([test "$enable_wrapper_runpath" != "no"], [enable_wrapper_runpath=yes])
+    AC_MSG_RESULT([$enable_wrapper_runpath])
+
+    AS_IF([test "$enable_wrapper_rpath" = "no" && test "$enable_wrapper_runpath" = "yes"],
+          [AC_MSG_ERROR([--enable-wrapper-runpath cannot be selected with --disable-wrapper-rpath])])
+])
+
+# PMIX_LIBTOOL_CONFIG(libtool-variable, result-variable,
+#                     libtool-tag, extra-code)
+# Retrieve information from the generated libtool
+AC_DEFUN([PMIX_LIBTOOL_CONFIG],[
+    PMIX_VAR_SCOPE_PUSH([rpath_script rpath_outfile])
+    # Output goes into globally-visible variable.  Run this in a
+    # sub-process so that we don't pollute the current process
+    # environment.
+    rpath_script=conftest.$$.sh
+    rpath_outfile=conftest.$$.out
+    rm -f $rpath_script $rpath_outfile
+    cat > $rpath_script <<EOF
+#!/bin/sh
+
+# Slurp in the libtool config into my environment
+
+# Apparently, "libtoool --config" calls "exit", so we can't source it
+# (because if script A sources script B, and B calls "exit", then both
+# B and A will exit).  Instead, we have to send the output to a file
+# and then source that.
+$PMIX_TOP_BUILDDIR/libtool $3 --config > $rpath_outfile
+
+chmod +x $rpath_outfile
+. ./$rpath_outfile
+rm -f $rpath_outfile
+
+# Evaluate \$$1, and substitute in LIBDIR for \$libdir
+$4
+flags="\`eval echo \$$1\`"
+echo \$flags
+
+# Done
+exit 0
+EOF
+    chmod +x $rpath_script
+    $2=`./$rpath_script`
+    rm -f $rpath_script
+    PMIX_VAR_SCOPE_POP
+])
+
+# Check to see whether the linker supports DT_RPATH.  We'll need to
+# use config.rpath to find the flags that it needs, if it does (see
+# comments in config.rpath for an explanation of where it came from).
+AC_DEFUN([PMIX_SETUP_RPATH],[
+    PMIX_VAR_SCOPE_PUSH([rpath_libdir_save])
+    AC_MSG_CHECKING([if linker supports RPATH])
+    PMIX_LIBTOOL_CONFIG([hardcode_libdir_flag_spec],[rpath_args],[],[libdir=LIBDIR])
+
+    AS_IF([test -n "$rpath_args"],
+          [WRAPPER_RPATH_SUPPORT=rpath
+           AC_MSG_RESULT([yes ($rpath_args)])],
+          [WRAPPER_RPATH_SUPPORT=unnecessary
+           AC_MSG_RESULT([yes (no extra flags needed)])])
+
+    PMIX_VAR_SCOPE_POP
+
+    # If we found RPATH support, check for RUNPATH support, too
+    AS_IF([test "$WRAPPER_RPATH_SUPPORT" = "rpath"],
+          [PMIX_SETUP_RUNPATH])
+])
+
+# Check to see if the linker supports the DT_RUNPATH flags via
+# --enable-new-dtags (a GNU ld-specific option).  These flags are more
+# social than DT_RPATH -- they can be overridden by LD_LIBRARY_PATH
+# (where a regular DT_RPATH cannot).
+#
+# If DT_RUNPATH is supported, then we'll use *both* the RPATH and
+# RUNPATH flags in the LDFLAGS.
+AC_DEFUN([PMIX_SETUP_RUNPATH],[
+    PMIX_VAR_SCOPE_PUSH([LDFLAGS_save])
+
+    # Set the output in $runpath_args
+    runpath_args=
+    LDFLAGS_save=$LDFLAGS
+    LDFLAGS="$LDFLAGS -Wl,--enable-new-dtags"
+    AS_IF([test x"$enable_wrapper_runpath" = x"yes"],
+           [AC_LANG_PUSH([C])
+            AC_MSG_CHECKING([if linker supports RUNPATH])
+            AC_LINK_IFELSE([AC_LANG_PROGRAM([], [return 7;])],
+                           [WRAPPER_RPATH_SUPPORT=runpath
+                            runpath_args="-Wl,--enable-new-dtags"
+                            AC_MSG_RESULT([yes (-Wl,--enable-new-dtags)])],
+                           [AC_MSG_RESULT([no])])
+            AC_LANG_POP([C])])
+    LDFLAGS=$LDFLAGS_save
+
+    PMIX_VAR_SCOPE_POP
+])
+
+# Called to find all -L arguments in the LDFLAGS and add in RPATH args
+# for each of them.  Then also add in an RPATH for @{libdir} (which
+# will be replaced by the wrapper compile to the installdir libdir at
+# runtime), and the RUNPATH args, if we have them.
+AC_DEFUN([RPATHIFY_LDFLAGS_INTERNAL],[
+    PMIX_VAR_SCOPE_PUSH([rpath_out rpath_dir rpath_tmp])
+    AS_IF([test "$enable_wrapper_rpath" = "yes" && test "$WRAPPER_RPATH_SUPPORT" != "disabled" && test "$WRAPPER_RPATH_SUPPORT" != "unnecessary"], [
+           rpath_out=""
+           for val in ${$1}; do
+               case $val in
+               -L*)
+                   rpath_dir=`echo $val | cut -c3-`
+                   rpath_tmp=`echo ${$2} | sed -e s@LIBDIR@$rpath_dir@`
+                   rpath_out="$rpath_out $rpath_tmp"
+                   ;;
+               esac
+           done
+
+           # Now add in the RPATH args for @{libdir}, and the RUNPATH args
+           rpath_tmp=`echo ${$2} | sed -e s/LIBDIR/@{libdir}/`
+           $1="${$1} $rpath_out $rpath_tmp ${$3}"
+          ])
+    PMIX_VAR_SCOPE_POP
+])
+
+AC_DEFUN([RPATHIFY_LDFLAGS],[RPATHIFY_LDFLAGS_INTERNAL([$1], [rpath_args], [runpath_args])])
+
+dnl
+dnl Avoid some repetitive code below
+dnl
+AC_DEFUN([_PMIX_SETUP_WRAPPER_FINAL_PKGCONFIG],[
+    AC_MSG_CHECKING([for $1 pkg-config LDFLAGS])
+    $1_PKG_CONFIG_LDFLAGS=`echo "$$1_WRAPPER_EXTRA_LDFLAGS" | sed -e 's/@{libdir}/\${libdir}/g'`
+    AC_SUBST([$1_PKG_CONFIG_LDFLAGS])
+    AC_MSG_RESULT([$$1_PKG_CONFIG_LDFLAGS])
+])
+
+
+# PMIX_SETUP_WRAPPER_FINAL()
+# ---------------------------
+AC_DEFUN([PMIX_SETUP_WRAPPER_FINAL],[
+
+    # Setup RPATH support, if desired
+    WRAPPER_RPATH_SUPPORT=disabled
+    AS_IF([test "$enable_wrapper_rpath" = "yes"],
+          [PMIX_SETUP_RPATH])
+    AS_IF([test "$enable_wrapper_rpath" = "yes" && test "$WRAPPER_RPATH_SUPPORT" = "disabled"],
+          [AC_MSG_WARN([RPATH support requested but not available])
+           AC_MSG_ERROR([Cannot continue])])
+
+    # Note that we have to setup <package>_PKG_CONFIG_LDFLAGS for the
+    # pkg-config files to parallel the
+    # <package>_WRAPPER_EXTRA_LDFLAGS.  This is because pkg-config
+    # will not understand the @{libdir} notation in
+    # *_WRAPPER_EXTRA_LDFLAGS; we have to translate it to ${libdir}.
+
+    # We now have all relevant flags.  Substitute them in everywhere.
+   AC_MSG_CHECKING([for PMIX CPPFLAGS])
+   if test "$WANT_INSTALL_HEADERS" = "1" ; then
+       PMIX_WRAPPER_EXTRA_CPPFLAGS='-I${includedir}'
+   fi
+   PMIX_WRAPPER_EXTRA_CPPFLAGS="$PMIX_WRAPPER_EXTRA_CPPFLAGS $pmix_mca_wrapper_extra_cppflags $wrapper_extra_cppflags $with_wrapper_cppflags"
+   PMIX_FLAGS_UNIQ(PMIX_WRAPPER_EXTRA_CPPFLAGS)
+   AC_SUBST([PMIX_WRAPPER_EXTRA_CPPFLAGS])
+   AC_MSG_RESULT([$PMIX_WRAPPER_EXTRA_CPPFLAGS])
+
+   AC_MSG_CHECKING([for PMIX CFLAGS])
+   PMIX_WRAPPER_EXTRA_CFLAGS="$wrapper_extra_cflags $with_wrapper_cflags"
+   PMIX_FLAGS_UNIQ(PMIX_WRAPPER_EXTRA_CFLAGS)
+   AC_SUBST([PMIX_WRAPPER_EXTRA_CFLAGS])
+   AC_MSG_RESULT([$PMIX_WRAPPER_EXTRA_CFLAGS])
+
+   AC_MSG_CHECKING([for PMIX CFLAGS_PREFIX])
+   PMIX_WRAPPER_EXTRA_CFLAGS_PREFIX="$with_wrapper_cflags_prefix"
+   PMIX_FLAGS_UNIQ(PMIX_WRAPPER_EXTRA_CFLAGS_PREFIX)
+   AC_SUBST([PMIX_WRAPPER_EXTRA_CFLAGS_PREFIX])
+   AC_MSG_RESULT([$PMIX_WRAPPER_EXTRA_CFLAGS_PREFIX])
+
+   AC_MSG_CHECKING([for PMIX LDFLAGS])
+   PMIX_WRAPPER_EXTRA_LDFLAGS="$pmix_mca_wrapper_extra_ldflags $wrapper_extra_ldflags $with_wrapper_ldflags"
+   PMIX_FLAGS_UNIQ(PMIX_WRAPPER_EXTRA_LDFLAGS)
+   RPATHIFY_LDFLAGS([PMIX_WRAPPER_EXTRA_LDFLAGS])
+   AC_SUBST([PMIX_WRAPPER_EXTRA_LDFLAGS])
+   AC_MSG_RESULT([$PMIX_WRAPPER_EXTRA_LDFLAGS])
+
+   # Convert @{libdir} to ${libdir} for pkg-config
+   _PMIX_SETUP_WRAPPER_FINAL_PKGCONFIG([PMIX])
+
+   # wrapper_extra_libs doesn't really get populated until after the mca system runs
+   # since most of the libs come from libtool.  So this is the first time we can
+   # uniq them.  ROMIO in particular adds lots of things already in wrapper_extra_libs,
+   # and this cleans the duplication up a bunch.  Always add everything the user
+   # asked for, as they know better than us.
+   AC_MSG_CHECKING([for PMIX LIBS])
+   PMIX_WRAPPER_EXTRA_LIBS="$pmix_mca_wrapper_extra_libs"
+   PMIX_FLAGS_APPEND_UNIQ([PMIX_WRAPPER_EXTRA_LIBS], [$wrapper_extra_libs])
+   PMIX_WRAPPER_EXTRA_LIBS="$PMIX_WRAPPER_EXTRA_LIBS $with_wrapper_libs"
+   PMIX_FLAGS_UNIQ(PMIX_WRAPPER_EXTRA_LIBS)
+   AC_SUBST([PMIX_WRAPPER_EXTRA_LIBS])
+   AC_MSG_RESULT([$PMIX_WRAPPER_EXTRA_LIBS])
+])
diff -Nru pmix-3.2.2~rc1/config/pmix_setup_zlib.m4 pmix-4.0.0/config/pmix_setup_zlib.m4
--- pmix-3.2.2~rc1/config/pmix_setup_zlib.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_setup_zlib.m4	1970-01-01 00:00:00.000000000 +0000
@@ -1,92 +0,0 @@
-# -*- shell-script -*-
-#
-# Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
-# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# MCA_zlib_CONFIG([action-if-found], [action-if-not-found])
-# --------------------------------------------------------------------
-AC_DEFUN([PMIX_ZLIB_CONFIG],[
-    PMIX_VAR_SCOPE_PUSH([pmix_zlib_dir pmix_zlib_libdir pmix_zlib_standard_lib_location pmix_zlib_standard_header_location])
-
-    AC_ARG_WITH([zlib],
-                [AC_HELP_STRING([--with-zlib=DIR],
-                                [Search for zlib headers and libraries in DIR ])])
-
-    AC_ARG_WITH([zlib-libdir],
-                [AC_HELP_STRING([--with-zlib-libdir=DIR],
-                                [Search for zlib libraries in DIR ])])
-
-    pmix_zlib_support=0
-
-    if test "$with_zlib" != "no"; then
-        AC_MSG_CHECKING([for zlib in])
-        if test ! -z "$with_zlib" && test "$with_zlib" != "yes"; then
-            pmix_zlib_dir=$with_zlib
-            pmix_zlib_standard_header_location=no
-            pmix_zlib_standard_lib_location=no
-            AS_IF([test -z "$with_zlib_libdir" || test "$with_zlib_libdir" = "yes"],
-                  [if test -d $with_zlib/lib; then
-                       pmix_zlib_libdir=$with_zlib/lib
-                   elif test -d $with_zlib/lib64; then
-                       pmix_zlib_libdir=$with_zlib/lib64
-                   else
-                       AC_MSG_RESULT([Could not find $with_zlib/lib or $with_zlib/lib64])
-                       AC_MSG_ERROR([Can not continue])
-                   fi
-                   AC_MSG_RESULT([$pmix_zlib_dir and $pmix_zlib_libdir])],
-                  [AC_MSG_RESULT([$with_zlib_libdir])])
-        else
-            AC_MSG_RESULT([(default search paths)])
-            pmix_zlib_standard_header_location=yes
-            pmix_zlib_standard_lib_location=yes
-        fi
-        AS_IF([test ! -z "$with_zlib_libdir" && test "$with_zlib_libdir" != "yes"],
-              [pmix_zlib_libdir="$with_zlib_libdir"
-               pmix_zlib_standard_lib_location=no])
-
-        PMIX_CHECK_PACKAGE([pmix_zlib],
-                           [zlib.h],
-                           [z],
-                           [deflate],
-                           [-lz],
-                           [$pmix_zlib_dir],
-                           [$pmix_zlib_libdir],
-                           [pmix_zlib_support=1],
-                           [pmix_zlib_support=0])
-        if test $pmix_zlib_support = "1"; then
-            LIBS="$LIBS -lz"
-            PMIX_EMBEDDED_LIBS="$PMIX_EMBEDDED_LIBS -lz"
-            if test "$pmix_zlib_standard_header_location" != "yes"; then
-                PMIX_EMBEDDED_CPPFLAGS="$PMIX_EMBEDDED_CPPFLAGS $pmix_zlib_CPPFLAGS"
-                CPPFLAGS="$CPPFLAGS $pmix_zlib_CPPFLAGS"
-            fi
-            if test "$pmix_zlib_standard_lib_location" != "yes"; then
-                PMIX_EMBEDDED_LDFLAGS="$PMIX_EMBEDDED_LDFLAGS $pmix_zlib_LDFLAGS"
-                LDFLAGS="$LDFLAGS $pmix_zlib_LDFLAGS"
-            fi
-        fi
-    fi
-
-    if test ! -z "$with_zlib" && test "$with_zlib" != "no" && test "$pmix_zlib_support" != "1"; then
-        AC_MSG_WARN([ZLIB SUPPORT REQUESTED AND NOT FOUND])
-        AC_MSG_ERROR([CANNOT CONTINUE])
-    fi
-
-    AC_MSG_CHECKING([will zlib support be built])
-    if test "$pmix_zlib_support" != "1"; then
-        AC_MSG_RESULT([no])
-    else
-        AC_MSG_RESULT([yes])
-    fi
-
-    AC_DEFINE_UNQUOTED([PMIX_HAVE_ZLIB], [$pmix_zlib_support],
-                       [Whether or not we have zlib support])
-    PMIX_VAR_SCOPE_POP
-])dnl
diff -Nru pmix-3.2.2~rc1/config/pmix_summary.m4 pmix-4.0.0/config/pmix_summary.m4
--- pmix-3.2.2~rc1/config/pmix_summary.m4	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/config/pmix_summary.m4	2021-01-02 08:56:17.000000000 +0000
@@ -5,7 +5,7 @@
 dnl Copyright (c) 2016-2018 Cisco Systems, Inc.  All rights reserved
 dnl Copyright (c) 2016      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2018      Intel, Inc.  All rights reserved.
+dnl Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
 dnl $COPYRIGHT$
 dnl
 dnl Additional copyrights may follow
@@ -50,8 +50,8 @@
         echo "Debug build: yes"
     fi
 
-    if test ! -z $with_platform ; then
-        echo "Platform file: $with_platform"
+    if test ! -z $with_pmix_platform ; then
+        echo "Platform file: $with_pmix_platform"
     else
         echo "Platform file: (none)"
     fi
diff -Nru pmix-3.2.2~rc1/configure.ac pmix-4.0.0/configure.ac
--- pmix-3.2.2~rc1/configure.ac	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/configure.ac	2021-01-02 08:56:17.000000000 +0000
@@ -76,9 +76,9 @@
 top_buildir=`pwd`
 cd "$srcdir"
 PMIX_TOP_SRCDIR="`pwd`"
+top_srcdir=$PMIX_TOP_SRCDIR
 AC_SUBST(PMIX_TOP_SRCDIR)
 cd "$PMIX_TOP_BUILDDIR"
-top_srcdir=`pwd`
 
 AC_MSG_NOTICE([builddir: $PMIX_TOP_BUILDDIR])
 AC_MSG_NOTICE([srcdir: $PMIX_TOP_SRCDIR])
@@ -88,6 +88,7 @@
 
 # setup configure options (e.g., show_title and friends)
 PMIX_CONFIGURE_SETUP
+
 pmix_show_title "Configuring PMIx"
 
 # This must be before AM_INIT_AUTOMAKE
@@ -114,6 +115,46 @@
 AS_IF([test -z "$CFLAGS_save"], [CFLAGS=])
 PMIX_VAR_SCOPE_POP
 
+# Sanity checks
+AC_DEFUN([PMIX_CHECK_DIR_FOR_SPACES],[
+    dir="$1"
+    article="$2"
+    label="$3"
+
+    AC_MSG_CHECKING([directory of $label])
+    AC_MSG_RESULT([$dir])
+    AS_IF([test -n "`echo $dir | grep ' '`"],
+          [AC_MSG_WARN([This version of OpenPMIx does not support $article $label])
+           AC_MSG_WARN([with a path that contains spaces])
+           AC_MSG_ERROR([Cannot continue.])])
+])
+
+AC_DEFUN([PMIX_CANONICALIZE_PATH],[
+    case $host_os in
+    darwin*)
+        # MacOS does not have "readlink -f" or realpath (at least as
+        # of MacOS Cataline / 10.15).  Instead, use Python, because we
+        # know MacOS comes with a /usr/bin/python that has
+        # os.path.realpath.
+        $2=`/usr/bin/python -c 'import os; print os.path.realpath("'$1'")'`
+        ;;
+    *)
+        $2=`readlink -f $1`
+        ;;
+    esac
+])
+
+PMIX_VAR_SCOPE_PUSH(pmix_checkdir)
+PMIX_CHECK_DIR_FOR_SPACES([$srcdir], [a], [source tree])
+PMIX_CANONICALIZE_PATH([$srcdir], [pmix_checkdir])
+PMIX_CHECK_DIR_FOR_SPACES([$pmix_checkdir], [an], [absolute source tree])
+PMIX_CANONICALIZE_PATH([.], [pmix_checkdir])
+PMIX_CHECK_DIR_FOR_SPACES([$pmix_checkdir], [a], [build tree])
+PMIX_CHECK_DIR_FOR_SPACES([$prefix], [a], [prefix])
+PMIX_CANONICALIZE_PATH([$prefix], [pmix_checkdir])
+PMIX_CHECK_DIR_FOR_SPACES([$pmix_checkdir], [an], [absolute prefix])
+PMIX_VAR_SCOPE_POP
+
 ####################################################################
 # Setup the configure header files
 ####################################################################
@@ -176,6 +217,7 @@
 
 AM_ENABLE_SHARED
 AM_DISABLE_STATIC
+PMIX_SETUP_WRAPPER_INIT
 
 # This did not exist pre AM 1.11.x (where x is somewhere >0 and <3),
 # but it is necessary in AM 1.12.x.
@@ -266,9 +308,9 @@
 # Dependencies that themselves have a pkg-config file available.
 #
 PC_REQUIRES=""
-AS_IF([test "$pmix_hwloc_support_will_build" = "yes" && test "$pmix_hwloc_source" != "embedded"],
+AS_IF([test $pmix_hwloc_support -eq 1 && test "$pmix_hwloc_source" != "cobuild"],
       [PC_REQUIRES="$PC_REQUIRES hwloc"])
-AS_IF([test $pmix_libevent_support -eq 1 && test "$pmix_libevent_source" != "embedded"],
+AS_IF([test $pmix_libevent_support -eq 1 && test "$pmix_libevent_source" != "cobuild"],
       [PC_REQUIRES="$PC_REQUIRES libevent"])
 AS_IF([test "$pmix_zlib_support" = "1"],
       [PC_REQUIRES="$PC_REQUIRES zlib"])
@@ -309,11 +351,14 @@
 AC_SUBST([libpmi_so_version])
 AC_SUBST([libpmi2_so_version])
 AC_SUBST([libmca_common_dstore_so_version])
+AC_SUBST([libmca_common_sse_so_version])
 
 AC_CONFIG_FILES(pmix_config_prefix[contrib/Makefile]
                 pmix_config_prefix[examples/Makefile]
                 pmix_config_prefix[test/Makefile]
+                pmix_config_prefix[test/python/Makefile]
                 pmix_config_prefix[test/simple/Makefile]
+                pmix_config_prefix[test/sshot/Makefile]
                 pmix_config_prefix[maint/pmix.pc])
 
 pmix_show_title "Configuration complete"
diff -Nru pmix-3.2.2~rc1/contrib/construct_dictionary.py pmix-4.0.0/contrib/construct_dictionary.py
--- pmix-3.2.2~rc1/contrib/construct_dictionary.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/contrib/construct_dictionary.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,315 @@
+#
+# Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
+# $COPYRIGHT$
+#
+# Construct a dictionary for translating attributes to/from
+# their defined name and their string representation - used
+# by tools to interpret user input
+#
+
+from __future__ import print_function
+import os, os.path, sys, shutil
+from optparse import OptionParser, OptionGroup
+
+def harvest_constants(options, path, constants):
+    # open the file
+    try:
+        inputfile = open(path, "r")
+    except Exception as e:
+        print("File {path} could not be opened: {e}"
+              .format(path=path, e=e))
+        return 1
+
+    # read the file - these files aren't too large
+    # so ingest the whole thing at one gulp
+    try:
+        lines = inputfile.readlines()
+    except Exception as e:
+        print("Error reading file {path}: {e}"
+              .format(path=path, e=e))
+        inputfile.close()
+        return 1
+
+    inputfile.close()  # we read everything, so done with the file
+
+    firstline = True
+    preamble = "                               \""
+    linesize = 53
+    # loop over the lines
+    for n in range(len(lines)):
+        line = lines[n]
+        # remove white space at front and back
+        myline = line.strip()
+        # remove comment lines
+        if "/*" in myline or "*/" in myline or myline.startswith("*"):
+            continue
+        # if the line starts with #define, then we want it
+        if myline.startswith("#define"):
+            value = myline[8:]
+            # skip some well-known unwanted values
+            if value.startswith("PMIx"):
+                continue
+            if value.startswith("PMIX_HAVE_VISIB"):
+                continue
+            if value.startswith("PMIX_LAUNCHER_RNDZ_FILE"):
+                continue
+            if value.startswith("PMIX_LAUNCHER_RNDZ_URI"):
+                continue
+            if value.startswith("PMIX_KEEPALIVE_PIPE"):
+                continue
+            tokens = value.split()
+            if len(tokens) >= 2:
+                if tokens[1][0] == '"':
+                    if not firstline:
+                        constants.write(",\n\n")
+                    firstline = False
+                    constants.write("    {.name = \"" + tokens[0] + "\", .string = " + tokens[1])
+                    # only one attribute violates the one-word rule for type
+                    if tokens[0] == "PMIX_EVENT_BASE":
+                        dstart = 3
+                        datatype = "PMIX_POINTER"
+                    elif tokens[0] == "PMIX_ATTR_UNDEF":
+                        constants.write(", .type = PMIX_POINTER,\n     .description = (char *[]){\"NONE\"}}")
+                        continue
+                    else:
+                        dstart = 4
+                        if tokens[3] == "(bool)":
+                            datatype = "PMIX_BOOL"
+                        elif tokens[3] == "(char*)":
+                            datatype = "PMIX_STRING"
+                        elif tokens[3] == "(pmix_rank_t)":
+                            datatype = "PMIX_PROC_RANK"
+                        elif tokens[3] == "(uint32_t)":
+                            datatype = "PMIX_UINT32"
+                        elif tokens[3] == "(int32_t)":
+                            datatype = "PMIX_INT32"
+                        elif tokens[3] == "(uint64_t)":
+                            datatype = "PMIX_UINT64"
+                        elif tokens[3] == "(int)":
+                            datatype = "PMIX_INT"
+                        elif tokens[3] == "(uint16_t)":
+                            datatype = "PMIX_UINT16"
+                        elif tokens[3] == "(pmix_data_array_t)" or tokens[3] == "(pmix_data_array_t*)":
+                            datatype = "PMIX_DATA_ARRAY"
+                        elif tokens[3] == "(pmix_proc_t*)":
+                            datatype = "PMIX_PROC"
+                        elif tokens[3] == "(pmix_coord_t*)":
+                            datatype = "PMIX_COORD"
+                        elif tokens[3] == "(pmix_coord_view_t)":
+                            datatype = "PMIX_UINT8"
+                        elif tokens[3] == "(float)":
+                            datatype = "PMIX_FLOAT"
+                        elif tokens[3] == "(pmix_byte_object_t)":
+                            datatype = "PMIX_BYTE_OBJECT"
+                        elif tokens[3] == "(hwloc_topology_t)":
+                            datatype = "PMIX_POINTER"
+                        elif tokens[3] == "(size_t)":
+                            datatype = "PMIX_SIZE"
+                        elif tokens[3] == "(pmix_data_range_t)":
+                            datatype = "PMIX_UINT8"
+                        elif tokens[3] == "(pmix_persistence_t)":
+                            datatype = "PMIX_UINT8"
+                        elif tokens[3] == "(pmix_scope_t)":
+                            datatype = "PMIX_UINT8"
+                        elif tokens[3] == "(pmix_status_t)":
+                            datatype = "PMIX_STATUS"
+                        elif tokens[3] == "(void*)":
+                            datatype = "PMIX_POINTER"
+                        elif tokens[3] == "(TBD)":
+                            datatype = "TBD"
+                        elif tokens[3] == "(time_t)":
+                            datatype = "PMIX_TIME"
+                        elif tokens[3] == "(pmix_envar_t*)":
+                            datatype = "PMIX_ENVAR"
+                        elif tokens[3] == "(pid_t)":
+                            datatype = "PMIX_PID"
+                        elif tokens[3] == "(pmix_proc_state_t)":
+                            datatype = "PMIX_PROC_STATE"
+                        elif tokens[3] == "(pmix_link_state_t)":
+                            datatype = "PMIX_LINK_STATE"
+                        elif tokens[3] == "(pointer)":
+                            datatype = "PMIX_POINTER"
+                        elif tokens[3] == "(pmix_topology_t)" or tokens[3] == "(pmix_topology_t*)":
+                            datatype = "PMIX_TOPO"
+                        elif tokens[3] == "(pmix_cpuset_t)" or tokens[3] == "(pmix_cpuset_t*)":
+                            datatype = "PMIX_PROC_CPUSET"
+                        elif tokens[3] == "(pmix_geometry_t)" or tokens[3] == "(pmix_geometry_t*)":
+                            datatype = "PMIX_GEOMETRY"
+                        elif tokens[3] == "(pmix_device_distance_t)" or tokens[3] == "(pmix_device_distance_t*)":
+                            datatype = "PMIX_DEVICE_DIST"
+                        elif tokens[3] == "(pmix_endpoint_t)" or tokens[3] == "(pmix_endpoint_t*)":
+                            datatype = "PMIX_ENDPOINT"
+                        elif tokens[3] == "(pmix_device_type_t)":
+                            datatype = "PMIX_DEVTYPE"
+                        elif tokens[3] == "(varies)":
+                            datatype = "PMIX_INT"
+                        else:
+                            print("UNKNOWN TOKEN: {tok}".format(tok=tokens[3]))
+                            return 1
+                    constants.write(", .type = " + datatype + ",\n     .description = (char *[]){\"")
+                    # the description consists of at least all remaining tokens
+                    m = dstart + 1
+                    desc = tokens[dstart].replace("\"", "\\\"")
+                    if "DEPRECATED" in desc:
+                        constants.write(desc + "\"")
+                        constants.write(", NULL}}")
+                        continue
+                    firstout = True
+                    while m < len(tokens):
+                        tmp = tokens[m].replace("\"", "\\\"")
+                        if (len(tmp) + len(desc) + 1) > linesize:
+                            if firstout:
+                                constants.write(desc + "\"")
+                            else:
+                                constants.write(",\n" + preamble + desc + "\"")
+                            firstout = False
+                            desc = tmp
+                        else:
+                            desc += " " + tmp
+                        m += 1
+                    # if the next line starts with '/', then it is a continuation
+                    # of the description
+                    line = lines[n+1]
+                    # remove white space at front and back
+                    myline = line.strip()
+                    while len(myline) > 0 and myline[0] == '/':
+                        # step over until we see the beginning of text
+                        m = 1
+                        while myline[m] == '/' or myline[m] == ' ':
+                            m += 1
+                        # the rest of the line is part of the description
+                        k = m
+                        tmp = myline[k]
+                        if tmp == '"':
+                            tmp = "\\\""
+                        k += 1
+                        while k < len(myline):
+                            while k < len(myline) and ' ' != myline[k]:
+                                if myline[k] == '"':
+                                    tmp += "\\\""
+                                else:
+                                    tmp += myline[k]
+                                k += 1
+                            if (len(tmp) + len(desc) + 1) > linesize:
+                                if firstout:
+                                    constants.write(desc + "\"")
+                                else:
+                                    constants.write(",\n" + preamble + desc + "\"")
+                                firstout = False
+                                desc = tmp
+                            else:
+                                if len(desc) > 0:
+                                    desc += " " + tmp
+                                else:
+                                    desc = tmp
+                            tmp = ""
+                            k += 1
+                        n += 1
+                        line = lines[n+1]
+                        myline = line.strip()
+                    if len(desc) > 0:
+                        if firstout:
+                            constants.write(desc + "\"")
+                        else:
+                            constants.write(",\n" + preamble + desc + "\"")
+                    # finish it up by closing the definition
+                    constants.write(", NULL}}")
+
+    inputfile.close()
+    return 0
+
+def main():
+    parser = OptionParser("usage: %prog [options]")
+    debugGroup = OptionGroup(parser, "Debug Options")
+    debugGroup.add_option("--dryrun",
+                          action="store_true", dest="dryrun", default=False,
+                          help="Show output to screen")
+    parser.add_option_group(debugGroup)
+
+    (options, args) = parser.parse_args()
+
+    # Find the top-level PMIx source tree dir.
+    # Start with the location of this script, which we know is in
+    # $top_srcdir/contrib.
+    top_src_dir = os.path.dirname(sys.argv[0])
+    top_src_dir = os.path.join(top_src_dir, "..")
+    top_src_dir = os.path.abspath(top_src_dir)
+
+    # Sanity check
+    checkfile = os.path.join(top_src_dir, "VERSION")
+    if not os.path.exists(checkfile):
+        print("ERROR: Could not find top source directory for Open PMIx")
+        return 1
+
+    source_include_dir = os.path.join(top_src_dir, "include")
+
+    # This script is invoked from src/include/Makefile.am, and
+    # therefore the cwd will be $(builddir)/src/include.  Verify this
+    # by checking for a file that we know should be in there.
+    build_src_include_dir = os.getcwd()
+    checkfile = os.path.join(build_src_include_dir, "pmix_config.h")
+    if not os.path.exists(checkfile):
+        print("ERROR: Could not find build directory for Open PMIx")
+        return 1
+
+    if options.dryrun:
+        constants = sys.stdout
+        outpath = None
+    else:
+        outpath = os.path.join(build_src_include_dir, "dictionary.h")
+        try:
+            constants = open(outpath, "w+")
+        except Exception as e:
+            print("{outpath} CANNOT BE OPENED - DICTIONARY COULD NOT BE CONSTRUCTED: {e}"
+                  .format(outpath=outpath, e=e))
+            return 1
+
+    # write the header
+    constants.write("""/*
+ * This file is autogenerated by construct_dictionary.py.
+ * Do not edit this file by hand.
+ */
+
+#include "src/include/pmix_config.h"
+#include "src/include/pmix_globals.h"
+
+pmix_regattr_input_t dictionary[] = {
+""")
+
+    # scan across the header files in the src directory
+    # looking for constants and typedefs
+
+    # pmix_common.h.in is in the src tree
+    rc = harvest_constants(options,
+                           os.path.join(source_include_dir, "pmix_common.h.in"),
+                           constants)
+    if 0 != rc:
+        constants.close()
+        if outpath:
+            os.remove(outpath)
+        print("HARVEST PMIX_COMMON FAILED - DICTIONARY COULD NOT BE CONSTRUCTED")
+        return 1
+    constants.write(",\n")
+
+    # pmix_deprecated.h is in the source tree
+    rc = harvest_constants(options,
+                           os.path.join(source_include_dir, "pmix_deprecated.h"),
+                           constants)
+    if 0 != rc:
+        constants.close()
+        if outpath:
+            os.remove(outpath)
+        print("HARVEST PMIX_DEPRECATED FAILED - DICTIONARY COULD NOT BE CONSTRUCTED")
+        return 1
+
+    # mark the end of the array
+    constants.write(""",
+    {.name = ""}
+};
+""")
+    constants.close()
+    return 0
+
+if __name__ == '__main__':
+    exit(main())
diff -Nru pmix-3.2.2~rc1/contrib/Makefile.am pmix-4.0.0/contrib/Makefile.am
--- pmix-3.2.2~rc1/contrib/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/contrib/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -25,13 +25,17 @@
 EXTRA_DIST = \
 	make_dist_tarball \
 	buildrpm.sh \
+    construct_dictionary.py \
     pmix_jenkins.sh \
     pmix-release.sh \
     pmix.spec \
     update-my-copyright.pl \
     whitespace-purge.sh \
     make_manpage.pl \
-	platform/optimized
+    platform/intel/bend/mac \
+    platform/intel/bend/mac.conf \
+    platform/intel/bend/linux \
+    platform/intel/bend/linux.conf
 
 include perf_tools/Makefile.include
 
diff -Nru pmix-3.2.2~rc1/contrib/perf_tools/pmi2_utils.c pmix-4.0.0/contrib/perf_tools/pmi2_utils.c
--- pmix-3.2.2~rc1/contrib/perf_tools/pmi2_utils.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/contrib/perf_tools/pmi2_utils.c	2021-01-02 08:56:17.000000000 +0000
@@ -58,9 +58,14 @@
 
 static inline void pmi_base64_encode_block (const unsigned char in[3], char out[4], int len) {
     out[0] = pmi_base64_encsym (in[0] >> 2);
-    out[1] = pmi_base64_encsym (((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4));
+
+    /* len is the length of in[] - we need to make sure we don't reference uninitialized data, hence the conditionals */
+    out[1] = 1 < len ? pmi_base64_encsym(((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4)) :
+                       pmi_base64_encsym((in[0] & 0x03) << 4);
     /* Cray PMI doesn't allow = in PMI attributes so pad with spaces */
-    out[2] = 1 < len ? pmi_base64_encsym(((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6)) : ' ';
+    out[2] = 1 < len ? pmi_base64_encsym((in[1] & 0x0f) << 2) : ' ';
+    out[2] = 2 < len ? pmi_base64_encsym(((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6)) : out[2];
+
     out[3] = 2 < len ? pmi_base64_encsym(in[2] & 0x3f) : ' ';
 }
 
diff -Nru pmix-3.2.2~rc1/contrib/platform/intel/bend/linux pmix-4.0.0/contrib/platform/intel/bend/linux
--- pmix-3.2.2~rc1/contrib/platform/intel/bend/linux	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/contrib/platform/intel/bend/linux	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,8 @@
+enable_debug_symbols=yes
+enable_binaries=yes
+enable_picky=yes
+enable_debug=yes
+enable_shared=yes
+enable_static=no
+enable_ipv6=no
+with_devel_headers=yes
diff -Nru pmix-3.2.2~rc1/contrib/platform/intel/bend/linux.conf pmix-4.0.0/contrib/platform/intel/bend/linux.conf
--- pmix-3.2.2~rc1/contrib/platform/intel/bend/linux.conf	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/contrib/platform/intel/bend/linux.conf	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is the default system-wide MCA parameters defaults file.
+# Specifically, the MCA parameter "mca_param_files" defaults to a
+# value of
+# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
+# (this file is the latter of the two).  So if the default value of
+# mca_param_files is not changed, this file is used to set system-wide
+# MCA parameters.  This file can therefore be used to set system-wide
+# default MCA parameters for all users.  Of course, users can override
+# these values if they want, but this file is an excellent location
+# for setting system-specific MCA parameters for those users who don't
+# know / care enough to investigate the proper values for them.
+
+# Note that this file is only applicable where it is visible (in a
+# filesystem sense).  Specifically, MPI processes each read this file
+# during their startup to determine what default values for MCA
+# parameters should be used.  mpirun does not bundle up the values in
+# this file from the node where it was run and send them to all nodes;
+# the default value decisions are effectively distributed.  Hence,
+# these values are only applicable on nodes that "see" this file.  If
+# $sysconf is a directory on a local disk, it is likely that changes
+# to this file will need to be propagated to other nodes.  If $sysconf
+# is a directory that is shared via a networked filesystem, changes to
+# this file will be visible to all nodes that share this $sysconf.
+
+# The format is straightforward: one per line, mca_param_name =
+# rvalue.  Quoting is ignored (so if you use quotes or escape
+# characters, they'll be included as part of the value).  For example:
+
+# Disable run-time MPI parameter checking
+#   mpi_param_check = 0
+
+# Note that the value "~/" will be expanded to the current user's home
+# directory.  For example:
+
+# Change component loading path
+#   component_path = /usr/local/lib/openmpi:~/my_openmpi_components
+
+# See "ompi_info --param all all" for a full listing of Open MPI MCA
+# parameters available and their default values.
+#
+
+# Basic behavior to smooth startup
+mca_base_component_show_load_errors = 1
diff -Nru pmix-3.2.2~rc1/contrib/platform/intel/bend/mac pmix-4.0.0/contrib/platform/intel/bend/mac
--- pmix-3.2.2~rc1/contrib/platform/intel/bend/mac	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/contrib/platform/intel/bend/mac	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,8 @@
+enable_debug_symbols=yes
+enable_binaries=yes
+enable_picky=yes
+enable_debug=yes
+enable_shared=yes
+enable_static=no
+enable_ipv6=no
+with_devel_headers=yes
diff -Nru pmix-3.2.2~rc1/contrib/platform/intel/bend/mac.conf pmix-4.0.0/contrib/platform/intel/bend/mac.conf
--- pmix-3.2.2~rc1/contrib/platform/intel/bend/mac.conf	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/contrib/platform/intel/bend/mac.conf	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This is the default system-wide MCA parameters defaults file.
+# Specifically, the MCA parameter "mca_param_files" defaults to a
+# value of
+# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf"
+# (this file is the latter of the two).  So if the default value of
+# mca_param_files is not changed, this file is used to set system-wide
+# MCA parameters.  This file can therefore be used to set system-wide
+# default MCA parameters for all users.  Of course, users can override
+# these values if they want, but this file is an excellent location
+# for setting system-specific MCA parameters for those users who don't
+# know / care enough to investigate the proper values for them.
+
+# Note that this file is only applicable where it is visible (in a
+# filesystem sense).  Specifically, MPI processes each read this file
+# during their startup to determine what default values for MCA
+# parameters should be used.  mpirun does not bundle up the values in
+# this file from the node where it was run and send them to all nodes;
+# the default value decisions are effectively distributed.  Hence,
+# these values are only applicable on nodes that "see" this file.  If
+# $sysconf is a directory on a local disk, it is likely that changes
+# to this file will need to be propagated to other nodes.  If $sysconf
+# is a directory that is shared via a networked filesystem, changes to
+# this file will be visible to all nodes that share this $sysconf.
+
+# The format is straightforward: one per line, mca_param_name =
+# rvalue.  Quoting is ignored (so if you use quotes or escape
+# characters, they'll be included as part of the value).  For example:
+
+# Disable run-time MPI parameter checking
+#   mpi_param_check = 0
+
+# Note that the value "~/" will be expanded to the current user's home
+# directory.  For example:
+
+# Change component loading path
+#   component_path = /usr/local/lib/openmpi:~/my_openmpi_components
+
+# See "ompi_info --param all all" for a full listing of Open MPI MCA
+# parameters available and their default values.
+#
+
+# Basic behavior to smooth startup
+mca_base_component_show_load_errors = 1
diff -Nru pmix-3.2.2~rc1/contrib/platform/optimized pmix-4.0.0/contrib/platform/optimized
--- pmix-3.2.2~rc1/contrib/platform/optimized	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/contrib/platform/optimized	1970-01-01 00:00:00.000000000 +0000
@@ -1,3 +0,0 @@
-enable_mem_debug=no
-enable_mem_profile=no
-enable_debug=no
diff -Nru pmix-3.2.2~rc1/contrib/pmix.spec pmix-4.0.0/contrib/pmix.spec
--- pmix-3.2.2~rc1/contrib/pmix.spec	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/contrib/pmix.spec	2021-01-02 08:56:17.000000000 +0000
@@ -12,7 +12,7 @@
 # Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
-# Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
+# Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
 # Copyright (c) 2015      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
@@ -190,7 +190,7 @@
 #
 #############################################################################
 
-Summary: An extended/exascale implementation of PMI
+Summary: An extended/exascale implementation of the PMIx Standard
 Name: %{?_name:%{_name}}%{!?_name:pmix}
 Version: $VERSION
 Release: 1%{?dist}
@@ -222,11 +222,9 @@
 
 PMI Exascale (PMIx) represents an attempt to resolve these questions by providing
 an extended version of the PMI standard specifically designed to support clusters
-up to and including exascale sizes. The overall objective of the project is not to
-branch the existing pseudo-standard definitions - in fact, PMIx fully supports both
-of the existing PMI-1 and PMI-2 APIs - but rather to (a) augment and extend those
-APIs to eliminate some current restrictions that impact scalability, and (b) provide
-a reference implementation of the PMI-server that demonstrates the desired level of
+up to and including exascale sizes. The overall objective of the project is to
+eliminate some current restrictions that impact scalability, and provide
+a reference implementation of the PMIx-server that demonstrates the desired level of
 scalability.
 
 %if %{build_all_in_one_rpm}
@@ -236,35 +234,12 @@
 # if build_all_in_one_rpm = 0, build split packages
 %if !%{build_all_in_one_rpm}
 %package devel
-Summary: PMIx developpement packages
+Summary: PMIx development packages
 Requires: %{name}%{?_isa} = %{version}-%{release}
 
 %description devel
 This RPM contains headers and shared objects symbolic links necessary to compile
 and link against PMIx.
-
-%package libpmi
-Summary: PMI-1 and PMI-2 compatibility libraries
-Requires: %{name}%{?_isa} = %{version}-%{release}
-Conflicts: slurm-libpmi
-
-%description libpmi
-The %{name}-libpmi package contains libpmi and libpmi2 libraries that provide
-the respective APIs and a copy of the PMIx library – each API is translated
-into its PMIx equivalent. This is especially targeted at apps/libs that are
-hardcoded to dlopen “libpmi” or “libpmi2”.
-This package conflicts sith slurm-libpmi, which provides its own, incompatible 
-versions of libpmi.so and libpmi2.so.
-
-%package libpmi-devel
-Summary: PMI-1 and PMI-2 compatibility developpement libraryes
-Requires: %{name}%{?_isa} = %{version}-%{release}
-Requires: %{name}%{?_isa}-devel = %{version}-%{release}
-Conflicts: slurm-libpmi
-
-%description libpmi-devel
-The %{name}-libpmi-devel package contains headers and shared objects
-symbolic links of libpmi and libpmi2 libraries.
 %endif
 
 #############################################################################
@@ -383,10 +358,6 @@
 # We don't need that in an RPM.
 find $RPM_BUILD_ROOT -name config.log -exec rm -f {} \;
 
-# If we build separate RPMs, then move the libpmi.* and libpmi2.* compat libs 
-# out of the way
-find $RPM_BUILD_ROOT -name 'libpmi.' | xargs rm -f
-
 # First, the [optional] modulefile
 
 %if %{install_modulefile}
@@ -536,29 +507,12 @@
 %exclude %{_includedir}
 %exclude %{_libdir}/*.so
 %exclude %{_libdir}/*.la
-%exclude %{_libdir}/libpmi.*
-%exclude %{_libdir}/libpmi2.*
 
 %files devel
 %{_includedir}
 %{_libdir}/*.so
 %{_libdir}/*.la
 
-%exclude %{_libdir}/libpmi.*
-%exclude %{_libdir}/libpmi2.*
-%exclude %{_includedir}/pmi.*
-%exclude %{_includedir}/pmi2.*
-
-%files libpmi
-%{_libdir}/libpmi.so.*
-%{_libdir}/libpmi2.so.*
-
-%files libpmi-devel
-%{_libdir}/libpmi.so
-%{_libdir}/libpmi2.so
-%{_includedir}/pmi.h
-%{_includedir}/pmi2.h
-
 %endif # build_all_in_one_rpm
 
 
@@ -569,6 +523,9 @@
 #
 #############################################################################
 %changelog
+* Thu Oct 1 2020 Ralph Castain <rhc@pmix.org>
+- Remove all references to PMI-1 and PMI-2 libraries
+
 * Mon Sep 21 2020 Piotr Lesnicki <piotr.lesnicki@atos.net>
 - Enable separate -devel rpms
 
diff -Nru pmix-3.2.2~rc1/contrib/symbol-hiding.pl pmix-4.0.0/contrib/symbol-hiding.pl
--- pmix-3.2.2~rc1/contrib/symbol-hiding.pl	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/contrib/symbol-hiding.pl	1970-01-01 00:00:00.000000000 +0000
@@ -1,217 +0,0 @@
-#!/usr/bin/env perl
-#
-# Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-
-use strict;
-use Getopt::Long;
-
-# globals
-my $myfile;
-my $mylib;
-my $myprefix;
-my $mysuffix;
-my $mycapprefix;
-my $mymacro;
-my $mycapmacro;
-my $mystrip;
-
-# Set to true if the script should merely check for symbols in
-# the library that are not in the provided output file - useful
-# for determining if something has changed prior to doing an update
-my $CHECK_ONLY = 0;
-# Set to true to suppress most informational messages. Only missing
-# symbols will be printed.
-my $QUIET = 0;
-# Set to true if we just want to see the help message
-my $HELP = 0;
-# Set to true if we want to reverse the hiding direction
-my $REVERSE = 0;
-
-
-GetOptions(
-    "help" => \$HELP,
-    "quiet" => \$QUIET,
-    "check-only" => \$CHECK_ONLY,
-    "prefix=s" => \$myprefix,
-    "suffix=s" => \$mysuffix,
-    "lib=s" => \$mylib,
-    "file=s" => \$myfile,
-    "reverse" => \$REVERSE,
-    "macro=s" => \$mymacro,
-    "capmacro=s" => \$mycapmacro,
-    "strip=s" => \$mystrip,
-) or die "unable to parse options, stopped";
-
-if ($HELP) {
-    print <<EOT;
-$0 [options]
-
---help | -h          This help message
---quiet | -q         Only output critical messages to stdout
---check-only         Output the symbols we would have prefixed, but don't do anything
---prefix=NAME        Add NAME to the front of all found symbols
---suffix=NAME        Add NAME to the end of all found symbols
---lib=NAME           Library containing symbols that are to be "hidden"
---file=NAME          Output file for results, or existing file to be updated
---reverse            Reverse the direction of hiding (i.e., #define prefix_foo to be foo)
---macro=NAME         Embed each lowercase symbol in the provided macro
---capmacro=NAME      Embed each uppercase symbol in the provided macro
---strip=NAME         Strip any leading string that matches the given (ignore case)
-EOT
-    exit(0);
-}
-
-#-------------------------------------------------------------------------------
-# predeclare sub for print-like syntax
-sub quiet_print {
-    unless ($QUIET) {
-        print @_;
-    }
-}
-
-#-------------------------------------------------------------------------------
-
-$mycapprefix = uc $myprefix;
-
-# get the symbol output for this lib
-my $output = qx(nm $mylib);
-
-# cycle across each line and tokenize it
-my @symbols;
-my @rawsymbols;
-my @capsymbols;
-my @rawcapsymbols;
-my $len = 0;
-my $rawval;
-foreach my $line (split /[\r\n]+/, $output) {
-    my @values = split(' ',$line);
-    my $val = shift(@values);
-    # if the first character isn't a number, then
-    # we can ignore the line
-    my $str = substr($val,0,1);
-    if ("0" ne $str) {
-        quiet_print "skipping\n";
-        next;
-    }
-    # this is a line of interest - see if the
-    # next token indicates a public symbol by
-    # being a 'T' or a 'B'
-    $val = shift(@values);
-    if ("T" eq $val || "B" eq $val || "D" eq $val) {
-        $val = shift(@values);
-        # if this symbol contains a '.', then we
-        # need to ignore it
-        if (index($val, ".") != -1) {
-            quiet_print "skipping $val\n";
-            next;
-        }
-        quiet_print "GOT: " . $val . "\n";
-        my $c = substr($val, 0, 1);
-        # if the first character is '_', then check the second
-        if ($c eq '_') {
-            $c = substr($val, 1, 1);
-            $val = substr($val, 1);
-        }
-        # if we are supposed to strip the symbols, then do so here
-        $rawval = $val;
-        if ($mystrip) {
-            my $test = substr($val, 0, length($mystrip));
-            if (lc $test eq lc $mystrip) {
-                $val = substr($val, length($mystrip));
-            }
-        }
-        if ($c =~ /^[[:upper:]]/) {
-            quiet_print "Adding to upper symbols\n";
-            push @capsymbols, $val;
-            push @rawcapsymbols, $rawval;
-        } else {
-            quiet_print("Adding to lower symbols\n");
-            push @symbols, $val;
-            push @rawsymbols, $rawval;
-        }
-        if ($len < length($val)) {
-            $len = length($val);
-        }
-    }
-}
-
-$len = $len + 5;
-if ($myfile ne "") {
-    open FILE, ">$myfile" || die "file could not be opened";
-}
-
-# handle all the capitalized symbols first
-my $index = 0;
-foreach my $sym (@capsymbols) {
-    my $out;
-    if ($REVERSE) {
-        if ($mycapmacro) {
-            $out = "#define " . $mycapmacro . "(" . @rawcapsymbols[$index] . ")";
-        } else {
-            $out = "#define " . $mycapprefix . @rawcapsymbols[$index] . $mysuffix;
-        }
-    } else {
-        $out = "#define " . @rawcapsymbols[$index];
-    }
-    my $diff = $len - length($sym);
-    for (my $i=0; $i < $diff; $i++) {
-       $out = $out . " ";
-    }
-    if ($REVERSE) {
-        $out = $out . $sym . "\n";
-    } else {
-        if ($mycapmacro) {
-            $out = $out . $mycapmacro . "(" . $sym . ")";
-        } else {
-            $out =  $out . $mycapprefix . $sym . $mysuffix;
-        }
-    }
-    $out = $out . "\n";
-    if ($myfile ne "") {
-        print FILE $out;
-    } else {
-        print $out;
-    }
-    $index = $index + 1;
-}
-
-# now do the lowercase symbols
-$index = 0;
-foreach my $sym (@symbols) {
-    my $out;
-    if ($REVERSE) {
-        if ($mymacro) {
-            $out = "#define " . $mymacro . "(" . @rawsymbols[$index] . ")";
-        } else {
-            $out = "#define " . $myprefix . @rawsymbols[$index] . $mysuffix;
-        }
-    } else {
-        $out = "#define " . @rawsymbols[$index];
-    }
-    my $diff = $len - length($sym);
-    for (my $i=0; $i < $diff; $i++) {
-       $out = $out . " ";
-    }
-    if ($REVERSE) {
-        $out = $out . $sym . "\n";
-    } else {
-        if ($mymacro) {
-            $out = $out . $mymacro . "(" . $sym . ")";
-        } else {
-            $out =  $out . $myprefix . $sym . $mysuffix;
-        }
-    }
-    $out = $out . "\n";
-    if ($myfile ne "") {
-        print FILE $out;
-    } else {
-        print $out;
-    }
-    $index = $index + 1;
-}
-if ($myfile ne "") {
-    close FILE;
-}
-
diff -Nru pmix-3.2.2~rc1/debian/changelog pmix-4.0.0/debian/changelog
--- pmix-3.2.2~rc1/debian/changelog	2020-12-27 15:28:23.000000000 +0000
+++ pmix-4.0.0/debian/changelog	2021-01-22 20:23:31.000000000 +0000
@@ -1,8 +1,65 @@
-pmix (3.2.2~rc1-1ubuntu1) hirsute; urgency=medium
+pmix (4.0.0-4ubuntu1) hirsute; urgency=medium
 
-  * PMIx should only be merged when OpenMPI is merged
+  * Don't build manpages on i386.
 
- -- Graham Inggs <ginggs@ubuntu.com>  Sun, 27 Dec 2020 15:28:23 +0000
+ -- Matthias Klose <doko@ubuntu.com>  Fri, 22 Jan 2021 21:23:31 +0100
+
+pmix (4.0.0-4) unstable; urgency=medium
+
+  * Add Breaks: libopenmpi3 (<< 4.1.0-3) to libpmix2 to get necessary
+    config file when using openmpi. Closes: #978066
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Wed, 20 Jan 2021 17:30:05 +0000
+
+pmix (4.0.0-3) unstable; urgency=medium
+
+  * Regression: ensure pmix-mca-params.conf is installed in share directory
+    to provide correct mca_base_component_path. Closes: #978066, #979744
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Tue, 12 Jan 2021 15:11:53 +0000
+
+pmix (4.0.0-2) unstable; urgency=medium
+
+  * Build-Depend on Cython, python3-distuils
+  * python3.patch: change refs from python -> python3
+  * Close bugs fixed in exp release. Closes: #920906, #978066
+  * Revert build to include both shared, static builds
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Fri, 08 Jan 2021 10:15:47 +0000
+
+pmix (4.0.0-1) experimental; urgency=medium
+
+  * New upstream release
+  * Patch merged upstream:
+    - python-bindings.patch
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Sat, 02 Jan 2021 08:56:03 +0000
+
+pmix (4.0.0~rc2-1) experimental; urgency=medium
+
+  * New upstream release
+  * Add json, python binding support
+    - Needs python-bindings.patch
+  * Be explicit in optional dependencies
+  * Fix pmix share install path, for help messages
+  * Add build-dep on pandoc for man pages
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Sun, 27 Dec 2020 16:40:30 +0000
+
+pmix (4.0.0~rc1-2) unstable; urgency=medium
+
+  * Update d/watch to openmpix github site
+  * Fix broken symlinks (libpmix2.links regression). Closes: #977873
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Wed, 23 Dec 2020 13:11:09 +0000
+
+pmix (4.0.0~rc1-1) unstable; urgency=medium
+
+  * New upstream release
+  * Drop libpmi1, libpmi2 packages
+  * Build-depend on python3
+
+ -- Alastair McKinstry <mckinstry@debian.org>  Mon, 21 Dec 2020 08:01:53 +0000
 
 pmix (3.2.2~rc1-1) unstable; urgency=medium
 
diff -Nru pmix-3.2.2~rc1/debian/control pmix-4.0.0/debian/control
--- pmix-3.2.2~rc1/debian/control	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/control	2021-01-22 20:23:31.000000000 +0000
@@ -3,11 +3,18 @@
 Priority: optional
 Maintainer: Alastair McKinstry <mckinstry@debian.org>
 Build-Depends: debhelper-compat (= 13), 
+ dh-sequence-python3,
  flex,
+ python3-all-dev,
+ cython3,
+ python3-distutils,
+ pandoc [!i386],
  libpsm-infinipath1-dev [amd64 i386],
  libpsm2-dev [amd64],
+ libhwloc-dev,
  zlib1g-dev,
  libevent-dev,
+ libjansson-dev,
  libltdl-dev,
  libmunge-dev [ !hurd-i386]
 Standards-Version: 4.5.1
@@ -27,49 +34,12 @@
  the Instant On initiative for rapid startup of applications at exascale
  and beyond.
  
-Package: libpmi-pmix-dev
-Section: libdevel
-Architecture: any
-Multi-Arch: same
-Breaks: libpmix-dev (<< 2.1.0~rc1-2)
-Replaces: libpmix-dev (<< 2.1.0~rc1-2)
-Depends: ${shlibs:Depends}, ${misc:Depends}, 
- libpmi1-pmix (= ${binary:Version}), libpmi2-pmix (= ${binary:Version})
-Description: Development files for the PMI library (OpenMPI )
- This is the OpenMPI implementation of the Process Management Interface (PMI)
- Exascale API. PMIx aims to retain transparent compatibility with the existing
- PMI-1 and PMI-2 definitions, and any future PMI releases; Support 
- the Instant On initiative for rapid startup of applications at exascale
- and beyond.
-
-Package: libpmi1-pmix
-Section: libs
-Architecture: any
-Multi-Arch: same
-Breaks: libpmix2 (<< 2.1.0~rc1-2), libpmi-pmix, libpmix-dev (<< 3.2.0~rc1-5)
-Replaces: libpmix2 (<< 2.1.0~rc1-2), libpmi-pmix, libpmix-dev (<< 3.2.0~rc1-5)
-Depends: ${shlibs:Depends}, ${misc:Depends}, libhwloc-plugins
-Description: OpenMPI implementation of the PMI v1 library
- This is the OpenMPI implementation of the Process Management Interface (PMI)
- API, v1. 
-
-Package: libpmi2-pmix
-Section: libs
-Architecture: any
-Multi-Arch: same
-Depends: ${shlibs:Depends}, ${misc:Depends}, libhwloc-plugins
-Breaks: libpmix2 (<< 2.1.0~rc1-2)
-Replaces: libpmix2 (<< 2.1.0~rc1-2)
-Description: OpenMPI implementation of the PMI v2 library
- This is the OpenMPI implementation of the Process Management Interface (PMI)
- API, v2. 
-
 Package: libpmix2
 Section: libs
 Architecture: any
 Multi-Arch: same
-Breaks: libpmix-dev (<< 3.2.0~rc1-2), libopenmpi3 (<< 3.1.2)
-Replaces: libpmix-dev (<< 3.2.0~rc1-2), libopenmpi3 (<< 3.1.2)
+Breaks: libpmix-dev (<< 3.2.0~rc1-2), libopenmpi3 (<< 4.1.0-3)
+Replaces: libpmix-dev (<< 3.2.0~rc1-2), libopenmpi3 (<< 4.1.0-3)
 Depends: ${shlibs:Depends}, ${misc:Depends}, libhwloc-plugins
 Description: Process Management Interface (Exascale) library
  This is the OpenMPI implementation of the Process Management Interface (PMI)
@@ -87,3 +57,13 @@
 Description: Process Management Interface (Exascale) library - tools
  PMIX is the OpenMPI implementation of the Process Management Interface (PMI)
  Exascale API. This package provides utilities for working with pmix.
+
+Package: python3-pmix
+Depends: python3, ${misc:Depends}, ${shlibs:Depends}, libpmix2 (= ${binary:Version})
+Architecture: any
+Multi-Arch: foreign
+Recommends: libpmix-dev
+Description: Process Management Interface (Exascale) library - Python wrapper
+ PMIX is the OpenMPI implementation of the Process Management Interface (PMI)
+ Exascale API. This package provides Python3 bindings.
+
diff -Nru pmix-3.2.2~rc1/debian/libpmi1-pmix.install pmix-4.0.0/debian/libpmi1-pmix.install
--- pmix-3.2.2~rc1/debian/libpmi1-pmix.install	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmi1-pmix.install	1970-01-01 00:00:00.000000000 +0000
@@ -1,2 +0,0 @@
-usr/lib/${DEB_HOST_MULTIARCH}/libpmi.so.1.0.1	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-usr/lib/${DEB_HOST_MULTIARCH}/libpmi.so.1	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
diff -Nru pmix-3.2.2~rc1/debian/libpmi1-pmix.links pmix-4.0.0/debian/libpmi1-pmix.links
--- pmix-3.2.2~rc1/debian/libpmi1-pmix.links	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmi1-pmix.links	1970-01-01 00:00:00.000000000 +0000
@@ -1,2 +0,0 @@
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi.so.1.0.1  /usr/lib/${DEB_HOST_MULTIARCH}/libpmi.so.1.0.1
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi.so.1.0.1  /usr/lib/${DEB_HOST_MULTIARCH}/libpmi.so.1
diff -Nru pmix-3.2.2~rc1/debian/libpmi2-pmix.install pmix-4.0.0/debian/libpmi2-pmix.install
--- pmix-3.2.2~rc1/debian/libpmi2-pmix.install	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmi2-pmix.install	1970-01-01 00:00:00.000000000 +0000
@@ -1 +0,0 @@
-usr/lib/${DEB_HOST_MULTIARCH}/libpmi2.so.1*	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
diff -Nru pmix-3.2.2~rc1/debian/libpmi2-pmix.links pmix-4.0.0/debian/libpmi2-pmix.links
--- pmix-3.2.2~rc1/debian/libpmi2-pmix.links	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmi2-pmix.links	1970-01-01 00:00:00.000000000 +0000
@@ -1,2 +0,0 @@
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi2.so.1.0.0  /usr/lib/${DEB_HOST_MULTIARCH}/libpmi2.so.1
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi2.so.1.0.0  /usr/lib/${DEB_HOST_MULTIARCH}/libpmi2.so.1.0.0
diff -Nru pmix-3.2.2~rc1/debian/libpmi-pmix-dev.install pmix-4.0.0/debian/libpmi-pmix-dev.install
--- pmix-3.2.2~rc1/debian/libpmi-pmix-dev.install	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmi-pmix-dev.install	1970-01-01 00:00:00.000000000 +0000
@@ -1,6 +0,0 @@
-/usr/include/pmi.h	/usr/include/${DEB_HOST_MULTIARCH}/pmix2/include
-/usr/include/pmi2.h	/usr/include/${DEB_HOST_MULTIARCH}/pmix2/include
-/usr/lib/${DEB_HOST_MULTIARCH}/libpmi.a /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-/usr/lib/${DEB_HOST_MULTIARCH}/libpmi2.a /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-/usr/lib/${DEB_HOST_MULTIARCH}/libpmi2.so /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-/usr/lib/${DEB_HOST_MULTIARCH}/libpmi.so /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
diff -Nru pmix-3.2.2~rc1/debian/libpmi-pmix-dev.links pmix-4.0.0/debian/libpmi-pmix-dev.links
--- pmix-3.2.2~rc1/debian/libpmi-pmix-dev.links	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmi-pmix-dev.links	1970-01-01 00:00:00.000000000 +0000
@@ -1,2 +0,0 @@
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi2.so.1.0.0  /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi2.so
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi.so.1.0.1  /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmi.so
diff -Nru pmix-3.2.2~rc1/debian/libpmix2.install pmix-4.0.0/debian/libpmix2.install
--- pmix-3.2.2~rc1/debian/libpmix2.install	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmix2.install	2021-01-20 17:30:05.000000000 +0000
@@ -1,7 +1,7 @@
 usr/lib/${DEB_HOST_MULTIARCH}/libpmix.so.2*	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
 usr/lib/${DEB_HOST_MULTIARCH}/libpmix.so /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
+#usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
 usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so.*	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
 usr/lib/${DEB_HOST_MULTIARCH}/pmix		/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-usr/share/pmix		/usr/share/pmix2
+usr/lib/${DEB_HOST_MULTIARCH}/pmix2/share/*
 debian/pmix-mca-params.conf	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/share
diff -Nru pmix-3.2.2~rc1/debian/libpmix2.links pmix-4.0.0/debian/libpmix2.links
--- pmix-3.2.2~rc1/debian/libpmix2.links	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmix2.links	2021-01-20 17:30:05.000000000 +0000
@@ -1,4 +1,4 @@
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmix.so.2.2.32  /usr/lib/${DEB_HOST_MULTIARCH}/libpmix.so.2
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmix.so.2.2.32  /usr/lib/${DEB_HOST_MULTIARCH}/libpmix.so.2.2.2.32
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libmca_common_dstore.so.1.0.2 /usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so.1
-/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libmca_common_dstore.so.1.0.2 /usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so.1.0.2
+/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmix.so.2.3.0  /usr/lib/${DEB_HOST_MULTIARCH}/libpmix.so.2
+/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libpmix.so.2.3.0  /usr/lib/${DEB_HOST_MULTIARCH}/libpmix.so.2.3.0
+/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libmca_common_dstore.so.1.0.3 /usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so.1
+/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib/libmca_common_dstore.so.1.0.3 /usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so.1.0.3
diff -Nru pmix-3.2.2~rc1/debian/libpmix-bin.install pmix-4.0.0/debian/libpmix-bin.install
--- pmix-3.2.2~rc1/debian/libpmix-bin.install	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmix-bin.install	2021-01-20 17:30:05.000000000 +0000
@@ -1 +1,2 @@
 /usr/bin/*
+# src/tools/wrapper/pmixcc.1	usr/share/man/man1
diff -Nru pmix-3.2.2~rc1/debian/libpmix-dev.install pmix-4.0.0/debian/libpmix-dev.install
--- pmix-3.2.2~rc1/debian/libpmix-dev.install	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/libpmix-dev.install	2021-01-20 17:30:05.000000000 +0000
@@ -1,4 +1,5 @@
 /usr/include/pmix*	/usr/lib/${DEB_HOST_MULTIARCH}/pmix2/include
 #/usr/share/man/*
 /usr/lib/${DEB_HOST_MULTIARCH}/libpmix.a /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
-usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig/pmix.pc
+/usr/lib/${DEB_HOST_MULTIARCH}/libmca_common_dstore.so /usr/lib/${DEB_HOST_MULTIARCH}/pmix2/lib
+/usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig/pmix.pc
diff -Nru pmix-3.2.2~rc1/debian/patches/hurd-fix.patch pmix-4.0.0/debian/patches/hurd-fix.patch
--- pmix-3.2.2~rc1/debian/patches/hurd-fix.patch	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/patches/hurd-fix.patch	2021-01-20 17:30:05.000000000 +0000
@@ -4,18 +4,3 @@
 Forwarded: no
 Last-Updated: 2017-11-25
 
-Index: pmix-3.1.4~rc1/src/mca/base/pmix_mca_base_var.c
-===================================================================
---- pmix-3.1.4~rc1.orig/src/mca/base/pmix_mca_base_var.c
-+++ pmix-3.1.4~rc1/src/mca/base/pmix_mca_base_var.c
-@@ -50,6 +50,10 @@
- #include "src/util/output.h"
- #include "src/util/pmix_environ.h"
- 
-+#ifndef MAXPATHLEN /* Hurd */
-+#define MAXPATHLEN 1024
-+#endif
-+
- /*
-  * local variables
-  */
diff -Nru pmix-3.2.2~rc1/debian/patches/missing.patch pmix-4.0.0/debian/patches/missing.patch
--- pmix-3.2.2~rc1/debian/patches/missing.patch	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/patches/missing.patch	1970-01-01 00:00:00.000000000 +0000
@@ -1,1356 +0,0 @@
-Description: Taken from git HEAD (2020-09-07) to include header definitions
- needed for 3.2.0-rc1 to compile
-Author: Alastair McKinstry <mckinstry@debian.org>
-Last-Updated: 2020-09-08
-Forwarded: no
-
-Index: pmix-3.2.0/include/pmix_common.h.in
-===================================================================
---- pmix-3.2.0.orig/include/pmix_common.h.in
-+++ pmix-3.2.0/include/pmix_common.h.in
-@@ -105,11 +105,15 @@ typedef uint32_t pmix_rank_t;
- /* other special rank values will be used to define
-  * groups of ranks for use in collectives */
- #define PMIX_RANK_LOCAL_NODE    UINT32_MAX-2        // all ranks on local node
-+#define PMIX_RANK_LOCAL_PEERS   UINT32_MAX-4        // all peers (i.e., all procs within the same nspace) on local node
- /* define an invalid value */
- #define PMIX_RANK_INVALID   UINT32_MAX-3
- /* define a boundary for valid ranks */
- #define PMIX_RANK_VALID         UINT32_MAX-50
- 
-+/* define a value to indicate that data applies
-+ * to all apps in a job */
-+#define PMIX_APP_WILDCARD  UINT32_MAX
- 
- /****  PMIX ENVIRONMENTAL PARAMETERS  ****/
- /* There are a few environmental parameters used by PMIx for
-@@ -147,17 +151,25 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_SERVER_SYSTEM_SUPPORT          "pmix.srvr.sys"         // (bool) The host RM wants to declare itself as being
-                                                                     //        the local system server for PMIx connection
-                                                                     //        requests
-+#define PMIX_SERVER_SESSION_SUPPORT         "pmix.srvr.sess"        // (bool) The host RM wants to declare itself as being
-+                                                                    //        the local session server for PMIx connection
-+                                                                    //        requests
- #define PMIX_SERVER_TMPDIR                  "pmix.srvr.tmpdir"      // (char*) temp directory where PMIx server will place
-                                                                     //        client rendezvous points and contact info
- #define PMIX_SYSTEM_TMPDIR                  "pmix.sys.tmpdir"       // (char*) temp directory for this system, where PMIx
-                                                                     //        server will place tool rendezvous points and
-                                                                     //        contact info
-+#define PMIX_SERVER_SHARE_TOPOLOGY          "pmix.srvr.share"       // (bool) server is to share its copy of the local node
-+                                                                    // topology (whether given to it or self-discovered) with any clients.
- #define PMIX_SERVER_ENABLE_MONITORING       "pmix.srv.monitor"      // (bool) Enable PMIx internal monitoring by server
- #define PMIX_SERVER_NSPACE                  "pmix.srv.nspace"       // (char*) Name of the nspace to use for this server
- #define PMIX_SERVER_RANK                    "pmix.srv.rank"         // (pmix_rank_t) Rank of this server
- #define PMIX_SERVER_GATEWAY                 "pmix.srv.gway"         // (bool) Server is acting as a gateway for PMIx requests
-                                                                     //        that cannot be serviced on backend nodes
-                                                                     //        (e.g., logging to email)
-+#define PMIX_SERVER_SCHEDULER               "pmix.srv.sched"        // (bool) Server supports system scheduler
-+#define PMIX_SERVER_START_TIME              "pmix.srv.strtime"      // (char*) Time when the server started - i.e., when the server created
-+                                                                    //         it's rendezvous file (given in ctime string format)
- 
- /* tool-related attributes */
- #define PMIX_TOOL_NSPACE                    "pmix.tool.nspace"      // (char*) Name of the nspace to use for this tool
-@@ -173,17 +185,27 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_TOOL_DO_NOT_CONNECT            "pmix.tool.nocon"       // (bool) the tool wants to use internal PMIx support, but does
-                                                                     //        not want to connect to a PMIx server
-                                                                     //        from the specified processes to this tool
--#define PMIX_RECONNECT_SERVER               "pmix.cnct.recon"       // (bool) tool is requesting to change server connections
-+#define PMIX_TOOL_CONNECT_OPTIONAL          "pmix.tool.conopt"      // (bool) tool shall connect to a server if available, but otherwise
-+                                                                    //        continue to operate unconnected
- #define PMIX_LAUNCHER                       "pmix.tool.launcher"    // (bool) tool is a launcher and needs rendezvous files created
- #define PMIX_LAUNCHER_RENDEZVOUS_FILE       "pmix.tool.lncrnd"      // (char*) Pathname of file where connection info is to be stored
-+#define PMIX_TOOL_ATTACHMENT_FILE           "pmix.tool.attach"      // (char*) File containing connection info to be used for attaching to server
-+
- 
- /* identification attributes */
- #define PMIX_USERID                         "pmix.euid"             // (uint32_t) effective user id
- #define PMIX_GRPID                          "pmix.egid"             // (uint32_t) effective group id
--#define PMIX_DSTPATH                        "pmix.dstpath"          // (char*) path to dstore files
- #define PMIX_VERSION_INFO                   "pmix.version"          // (char*) PMIx version of contactor
- #define PMIX_REQUESTOR_IS_TOOL              "pmix.req.tool"         // (bool) requesting process is a tool
- #define PMIX_REQUESTOR_IS_CLIENT            "pmix.req.client"       // (bool) requesting process is a client process
-+#define PMIX_PSET_NAME                      "pmix.pset.nm"          // (char*) assigned name for the process
-+                                                                    //          set containing the given process
-+#define PMIX_PSET_NAMES                     "pmix.pset.nms"         // (pmix_data_array_t*) Returns an array of string names of the
-+                                                                    //          process sets in which the given process is a member.
-+#define PMIX_PSET_MEMBERS                   "pmix.pset.mems"        // (pmix_data_array_t*) An array of pmix_proc_t containing
-+                                                                    //          the members of the newly defined process set.
-+#define PMIX_REINCARNATION                  "pmix.reinc"            // (uint32_t) number of times this process has been instantiated - i.e.,
-+                                                                    //            tracks the number of times it has been restarted
- 
- /* model attributes */
- #define PMIX_PROGRAMMING_MODEL              "pmix.pgm.model"        // (char*) programming model being initialized (e.g., "MPI" or "OpenMP")
-@@ -226,7 +248,6 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_CPUSET                         "pmix.cpuset"           // (char*) hwloc bitmap applied to proc upon launch
- #define PMIX_CREDENTIAL                     "pmix.cred"             // (char*) security credential assigned to proc
- #define PMIX_SPAWNED                        "pmix.spawned"          // (bool) true if this proc resulted from a call to PMIx_Spawn
--#define PMIX_ARCH                           "pmix.arch"             // (uint32_t) datatype architecture flag
- 
- /* scratch directory locations for use by applications */
- #define PMIX_TMPDIR                         "pmix.tmpdir"           // (char*) top-level tmp dir assigned to session
-@@ -246,6 +267,7 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_NPROC_OFFSET                   "pmix.offset"           // (pmix_rank_t) starting global rank of this job
- #define PMIX_LOCAL_RANK                     "pmix.lrank"            // (uint16_t) rank on this node within this job
- #define PMIX_NODE_RANK                      "pmix.nrank"            // (uint16_t) rank on this node spanning all jobs
-+#define PMIX_PACKAGE_RANK                   "pmix.pkgrank"          // (uint16_t) rank within this job on the package where this proc resides
- #define PMIX_LOCALLDR                       "pmix.lldr"             // (pmix_rank_t) lowest rank on this node within this job
- #define PMIX_APPLDR                         "pmix.aldr"             // (pmix_rank_t) lowest rank in this app within this job
- #define PMIX_PROC_PID                       "pmix.ppid"             // (pid_t) pid of specified proc
-@@ -261,8 +283,6 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_LOCAL_PEERS                    "pmix.lpeers"           // (char*) comma-delimited string of ranks on this node within the specified nspace
- #define PMIX_LOCAL_PROCS                    "pmix.lprocs"           // (pmix_data_array_t*) array of pmix_proc_t of procs on the specified node
- #define PMIX_LOCAL_CPUSETS                  "pmix.lcpus"            // (char*) colon-delimited cpusets of local peers within the specified nspace
--#define PMIX_PROC_URI                       "pmix.puri"             // (char*) URI containing contact info for proc
--#define PMIX_LOCALITY                       "pmix.loc"              // (uint16_t) relative locality of two procs
- #define PMIX_PARENT_ID                      "pmix.parent"           // (pmix_proc_t*) identifier of the process that called PMIx_Spawn
-                                                                     //                to launch this proc's application
- #define PMIX_EXIT_CODE                      "pmix.exit.code"        // (int) exit code returned when proc terminated
-@@ -286,20 +306,8 @@ typedef uint32_t pmix_rank_t;
- 
- 
- /* topology info */
--#define PMIX_NET_TOPO                       "pmix.ntopo"            // (char*) xml-representation of network topology
--#define PMIX_LOCAL_TOPO                     "pmix.ltopo"            // (char*) xml-representation of local node topology
--#define PMIX_TOPOLOGY                       "pmix.topo"             // (hwloc_topology_t) pointer to the PMIx client's internal topology object
--#define PMIX_TOPOLOGY_XML                   "pmix.topo.xml"         // (char*) XML-based description of topology
--#define PMIX_TOPOLOGY_FILE                  "pmix.topo.file"        // (char*) full path to file containing XML topology description
--#define PMIX_TOPOLOGY_SIGNATURE             "pmix.toposig"          // (char*) topology signature string
-+#define PMIX_TOPOLOGY2                      "pmix.topo2"            // (pmix_topology_t) pointer to a PMIx topology object
- #define PMIX_LOCALITY_STRING                "pmix.locstr"           // (char*) string describing a proc's location
--#define PMIX_HWLOC_SHMEM_ADDR               "pmix.hwlocaddr"        // (size_t) address of HWLOC shared memory segment
--#define PMIX_HWLOC_SHMEM_SIZE               "pmix.hwlocsize"        // (size_t) size of HWLOC shared memory segment
--#define PMIX_HWLOC_SHMEM_FILE               "pmix.hwlocfile"        // (char*) path to HWLOC shared memory file
--#define PMIX_HWLOC_XML_V1                   "pmix.hwlocxml1"        // (char*) XML representation of local topology using HWLOC v1.x format
--#define PMIX_HWLOC_XML_V2                   "pmix.hwlocxml2"        // (char*) XML representation of local topology using HWLOC v2.x format
--#define PMIX_HWLOC_SHARE_TOPO               "pmix.hwlocsh"          // (bool) Share the HWLOC topology via shared memory
--#define PMIX_HWLOC_HOLE_KIND                "pmix.hwlocholek"       // (char*) Kind of VM "hole" HWLOC should use for shared memory
- 
- 
- /* request-related info */
-@@ -310,8 +318,6 @@ typedef uint32_t pmix_rank_t;
-                                                                     //        the host RM
- #define PMIX_WAIT                           "pmix.wait"             // (int) caller requests that the server wait until at least the specified
-                                                                     //       #values are found (0 => all and is the default)
--#define PMIX_COLLECTIVE_ALGO                "pmix.calgo"            // (char*) comma-delimited list of algorithms to use for collective
--#define PMIX_COLLECTIVE_ALGO_REQD           "pmix.calreqd"          // (bool) if true, indicates that the requested choice of algo is mandatory
- #define PMIX_NOTIFY_COMPLETION              "pmix.notecomp"         // (bool) notify parent process upon termination of child job
- #define PMIX_RANGE                          "pmix.range"            // (pmix_data_range_t) value for calls to publish/lookup/unpublish or for
-                                                                     //        monitoring event notifications
-@@ -319,18 +325,23 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_DATA_SCOPE                     "pmix.scope"            // (pmix_scope_t) scope of the data to be found in a PMIx_Get call
- #define PMIX_OPTIONAL                       "pmix.optional"         // (bool) look only in the client's local data store for the requested value - do
-                                                                     //        not request data from the server if not found
-+#define PMIX_GET_STATIC_VALUES              "pmix.get.static"       // (bool) Request that any pointers in the returned value point directly
-+                                                                    //       to values in the key-value store
- #define PMIX_EMBED_BARRIER                  "pmix.embed.barrier"    // (bool) execute a blocking fence operation before executing the
-                                                                     //        specified operation
- #define PMIX_JOB_TERM_STATUS                "pmix.job.term.status"  // (pmix_status_t) status returned upon job termination
-+#define PMIX_PROC_TERM_STATUS               "pmix.proc.term.status" // (pmix_status_t) status returned upon process termination
- #define PMIX_PROC_STATE_STATUS              "pmix.proc.state"       // (pmix_proc_state_t) process state
-+#define PMIX_NOTIFY_LAUNCH                  "pmix.note.lnch"        // (bool) notify the requestor upon launch of the child job and return
-+                                                                    //        its namespace in the event
- #define PMIX_GET_REFRESH_CACHE              "pmix.get.refresh"      // (bool) when retrieving data for a remote process, refresh the existing
-                                                                     //        local data cache for the process in case new values have been
-                                                                     //        put and committed by it since the last refresh
- 
-+
- /* attributes used by host server to pass data to/from the server convenience library - the
-  * data will then be parsed and provided to the local clients. Not generally accessible by users */
- #define PMIX_REGISTER_NODATA                "pmix.reg.nodata"       // (bool) Registration is for nspace only, do not copy job data
--#define PMIX_PROC_DATA                      "pmix.pdata"            // (pmix_data_array_t*) starts with rank, then contains more data
- #define PMIX_NODE_MAP                       "pmix.nmap"             // (char*) regex of nodes containing procs for this job
- #define PMIX_PROC_MAP                       "pmix.pmap"             // (char*) regex describing procs on each node within this job
- #define PMIX_ANL_MAP                        "pmix.anlmap"           // (char*) process mapping in ANL notation (used in PMI-1/PMI-2)
-@@ -339,11 +350,6 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_REQUIRED_KEY                   "pmix.req.key"          // (char*) key the user needs prior to responding from a dmodex request
- 
- 
--/* attributes used internally to communicate data from the server to the client */
--#define PMIX_PROC_BLOB                      "pmix.pblob"            // (pmix_byte_object_t) packed blob of process data
--#define PMIX_MAP_BLOB                       "pmix.mblob"            // (pmix_byte_object_t) packed blob of process location
--
--
- /* event handler registration and notification info keys */
- #define PMIX_EVENT_HDLR_NAME                "pmix.evname"           // (char*) string name identifying this handler
- #define PMIX_EVENT_HDLR_FIRST               "pmix.evfirst"          // (bool) invoke this event handler before any other handlers
-@@ -387,7 +393,6 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_ADD_HOSTFILE                   "pmix.addhostfile"      // (char*) hostfile to add to existing allocation
- #define PMIX_PREFIX                         "pmix.prefix"           // (char*) prefix to use for starting spawned procs
- #define PMIX_WDIR                           "pmix.wdir"             // (char*) working directory for spawned procs
--#define PMIX_MAPPER                         "pmix.mapper"           // (char*) mapper to use for placing spawned procs
- #define PMIX_DISPLAY_MAP                    "pmix.dispmap"          // (bool) display process map upon spawn
- #define PMIX_PPR                            "pmix.ppr"              // (char*) #procs to spawn on each identified resource
- #define PMIX_MAPBY                          "pmix.mapby"            // (char*) mapping policy
-@@ -395,7 +400,6 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_BINDTO                         "pmix.bindto"           // (char*) binding policy
- #define PMIX_PRELOAD_BIN                    "pmix.preloadbin"       // (bool) preload binaries
- #define PMIX_PRELOAD_FILES                  "pmix.preloadfiles"     // (char*) comma-delimited list of files to pre-position
--#define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) spawned procs will not call PMIx_Init
- #define PMIX_STDIN_TGT                      "pmix.stdin"            // (pmix_proc_t*) proc that is to receive stdin
-                                                                     //                (PMIX_RANK_WILDCARD = all in given nspace)
- #define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"         // (bool) spawned app consists of debugger daemons
-@@ -408,6 +412,8 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_MERGE_STDERR_STDOUT            "pmix.mergeerrout"      // (bool) merge stdout and stderr streams from application procs
- #define PMIX_OUTPUT_TO_FILE                 "pmix.outfile"          // (char*) direct application output into files of form
-                                                                     //         "<filename>.rank" with both stdout and stderr redirected into it
-+#define PMIX_OUTPUT_TO_DIRECTORY            "pmix.outdir"           // (char*) direct application output into files of form
-+                                                                    //         "<directory>/<jobid>/rank.<rank>/stdout[err]"
- #define PMIX_INDEX_ARGV                     "pmix.indxargv"         // (bool) mark the argv with the rank of the proc
- #define PMIX_CPUS_PER_PROC                  "pmix.cpuperproc"       // (uint32_t) #cpus to assign to each rank
- #define PMIX_NO_PROCS_ON_HEAD               "pmix.nolocal"          // (bool) do not place procs on the head node
-@@ -425,14 +431,22 @@ typedef uint32_t pmix_rank_t;
-                                                                     //        from the spawned processes to this process (typically used by a tool)
- #define PMIX_SPAWN_TOOL                     "pmix.spwn.tool"        // (bool) job being spawned is a tool
- #define PMIX_CMD_LINE                       "pmix.cmd.line"         // (char*) command line executing in the specified nspace
-+#define PMIX_FORK_EXEC_AGENT                "pmix.fe.agnt"          // (char*) command line of fork/exec agent to be used for starting
-+                                                                    //         local processes
-+#define PMIX_TIMEOUT_STACKTRACES            "pmix.tim.stack"        // (bool) include process stacktraces in timeout report from a job
-+#define PMIX_TIMEOUT_REPORT_STATE           "pmix.tim.state"        // (bool) report process states in timeout report from a job
-+#define PMIX_APP_ARGV                       "pmix.app.argv"         // (char*) consolidated argv passed to the spawn command for the given app
- 
--/* query attributes */
--#define PMIX_QUERY_REFRESH_CACHE            "pmix.qry.rfsh"         // (bool) retrieve updated information from server
--                                                                    //        to update local cache
--#define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"           // (char*) return a comma-delimited list of active namespaces
--#define PMIX_QUERY_NAMESPACE_INFO           "pmix.qry.nsinfo"       // (pmix_data_array_t) request an array of active nspace information - each
-+
-+/* query keys - value type shown is the type of the value that will be RETURNED by that key  */
-+#define PMIX_QUERY_SUPPORTED_KEYS           "pmix.qry.keys"         // (char*) returns comma-delimited list of keys supported by the query
-+                                                                    //         function. NO QUALIFIERS
-+#define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"           // (char*) returns a comma-delimited list of active namespaces. NO QUALIFIERS
-+#define PMIX_QUERY_NAMESPACE_INFO           "pmix.qry.nsinfo"       // (pmix_data_array_t*) returns an array of active nspace information - each
-                                                                     //        element will contain an array including the namespace plus the
-                                                                     //        command line of the application executing within it
-+                                                                    //        SUPPORTED QUALIFIERS: PMIX_NSPACE of specific nspace whose info
-+                                                                    //        is being requested
- #define PMIX_QUERY_JOB_STATUS               "pmix.qry.jst"          // (pmix_status_t) returns status of a specified currently executing job
-                                                                     //         REQUIRES a PMIX_NSPACE qualifier indicating the nspace being queried
- #define PMIX_QUERY_QUEUE_LIST               "pmix.qry.qlst"         // (char*) request a comma-delimited list of scheduler queues. NO QUALIFIERS
-@@ -452,14 +466,52 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_QUERY_MEMORY_USAGE             "pmix.qry.mem"          // (pmix_data_array_t*) return info on memory usage for the procs indicated in the qualifiers
-                                                                     //        SUPPORTED QUALIFIERS: PMIX_NSPACE/PMIX_RANK, or PMIX_PROCID of specific proc(s)
-                                                                     //        whose info is being requested
--#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"        // (bool) constrain the query to local information only
--#define PMIX_QUERY_REPORT_AVG               "pmix.qry.avg"          // (bool) report average values
--#define PMIX_QUERY_REPORT_MINMAX            "pmix.qry.minmax"       // (bool) report minimum and maximum value
- #define PMIX_QUERY_ALLOC_STATUS             "pmix.query.alloc"      // (char*) return a string reporting status of an allocation request
-                                                                     //         REQUIRES a PMIX_ALLOC_ID qualifier indicating the allocation request being queried
- #define PMIX_TIME_REMAINING                 "pmix.time.remaining"   // (uint32_t) returns number of seconds remaining in allocation
-                                                                     //         for the specified nspace (defaults to allocation containing the caller)
-                                                                     //         SUPPORTED QUALIFIERS: PMIX_NSPACE of the nspace whose info is being requested
-+#define PMIX_QUERY_NUM_PSETS                "pmix.qry.psetnum"      // (size_t) returns the number of psets defined
-+                                                                    //          in the specified range (defaults to session)
-+                                                                    //         SUPPORTED QUALIFIERS: PMIX_RANGE whose info is being requested
-+#define PMIX_QUERY_PSET_NAMES               "pmix.qry.psets"        // (char*) returns a comma-delimited list of the names of the
-+                                                                    //         psets defined in the specified range (defaults to session)
-+                                                                    //         SUPPORTED QUALIFIERS: PMIX_RANGE whose info is being requested
-+#define PMIX_QUERY_ATTRIBUTE_SUPPORT        "pmix.qry.attrs"        // (pmix_data_array_t*) returns array of pmix_info_t where each element consists
-+                                                                    //         of a key containing the name of the function, and an array of pmix_regattr_t
-+                                                                    //         detailing the attribute support for that function
-+                                                                    //         SUPPORTED QUALIFIERS: PMIX_CLIENT_FUNCTIONS, PMIX_SERVER_FUNCTIONS,
-+                                                                    //         PMIX_TOOL_FUNCTIONS, and/or PMIX_HOST_FUNCTIONS
-+#define PMIX_CLIENT_FUNCTIONS               "pmix.client.fns"       // (char*) returns a comma-delimited list of supported PMIx client functions. NO QUALIFIERS
-+#define PMIX_SERVER_FUNCTIONS               "pmix.srvr.fns"         // (char*) returns a comma-delimited list of supported PMIx server functions. NO QUALIFIERS
-+#define PMIX_TOOL_FUNCTIONS                 "pmix.tool.fns"         // (char*) returns a comma-delimited list of supported PMIx tool functions. NO QUALIFIERS
-+#define PMIX_HOST_FUNCTIONS                 "pmix.host.fns"         // (char*) returns a comma-delimited list of PMIx functions supported by the host environment
-+#define PMIX_QUERY_AVAIL_SERVERS            "pmix.qry.asrvrs"       // (pmix_data_array_t*) array of pmix_info_t, each element containing an array of
-+                                                                    //         pmix_info_t of available data for servers on this node
-+                                                                    //         to which the caller might be able to connect. NO QUALIFIERS
-+
-+/* query qualifiers - these are used to provide information to narrow/modify the query. Value type shown is the type of data expected
-+ * to be provided with the key */
-+#define PMIX_QUERY_REFRESH_CACHE            "pmix.qry.rfsh"         // (bool) retrieve updated information from server
-+                                                                    //        to update local cache
-+#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"        // (bool) constrain the query to local information only
-+#define PMIX_QUERY_REPORT_AVG               "pmix.qry.avg"          // (bool) report average values
-+#define PMIX_QUERY_REPORT_MINMAX            "pmix.qry.minmax"       // (bool) report minimum and maximum value
-+#define PMIX_CLIENT_ATTRIBUTES              "pmix.client.attrs"     // (char*) comma-delimited list of functions, including "all"
-+                                                                    //        when used in a query, indicates whether or not to include
-+                                                                    //        attributes supported by the PMIx client library
-+#define PMIX_SERVER_ATTRIBUTES              "pmix.srvr.attrs"       // (char*) comma-delimited list of functions, including "all"
-+                                                                    //        when used in a query, indicates whether or not to include
-+                                                                    //        attributes supported by the PMIx server library
-+#define PMIX_HOST_ATTRIBUTES                "pmix.host.attrs"       // (char*) comma-delimited list of functions, including "all"
-+                                                                    //        when used in a query, indicates whether or not to include
-+                                                                    //        attributes supported by the host environment
-+#define PMIX_TOOL_ATTRIBUTES                "pmix.tool.attrs"       // (char*) comma-delimited list of functions, including "all"
-+                                                                    //        when used in a query, indicates whether or not to include
-+                                                                    //        attributes supported by the PMIx tool library
-+#define PMIX_QUERY_SUPPORTED_QUALIFIERS     "pmix.qry.quals"        // (bool) return comma-delimited list of qualifiers supported by
-+                                                                    //        a query on the provided key, instead of actually performing
-+                                                                    //        the query on the key.
- 
- 
- /* PMIx_Get information retrieval attributes */
-@@ -505,6 +557,7 @@ typedef uint32_t pmix_rank_t;
-                                                                     //        information. At a minimum, either the PMIX_NODEID or PMIX_HOSTNAME
-                                                                     //        attribute is required to be included in the array, though both may be
-                                                                     //        included.
-+#define PMIX_SERVER_INFO_ARRAY              "pmix.srv.arr"          // (pmix_data_array_t*) array of data on a given server, starting with its nspace
- 
- 
- /* log attributes */
-@@ -542,12 +595,10 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_DEBUG_STOP_ON_EXEC             "pmix.dbg.exec"         // (bool) job is being spawned under debugger - instruct it to pause on start
- #define PMIX_DEBUG_STOP_IN_INIT             "pmix.dbg.init"         // (bool) instruct job to stop during PMIx init
- #define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"       // (bool) block at desired point until receiving debugger release notification
--#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job assigned to this debugger to be debugged. Note
--                                                                    //         that id's, pids, and other info on the procs is available
--                                                                    //         via a query for the nspace's local or global proctable
- #define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"      // (bool) job to be debugged is waiting for a release
- #define PMIX_DEBUG_JOB_DIRECTIVES           "pmix.dbg.jdirs"        // (pmix_data_array_t*) array of job-level directives
- #define PMIX_DEBUG_APP_DIRECTIVES           "pmix.dbg.adirs"        // (pmix_data_array_t*) array of app-level directives
-+#define PMIX_DEBUG_TARGET                   "pmix.dbg.tgt"          // (pmix_proc_t*) Identifier of proc(s) to be debugged
- 
- /* Resource Manager identification */
- #define PMIX_RM_NAME                        "pmix.rm.name"          // (char*) string name of the resource manager
-@@ -564,8 +615,13 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_APPEND_ENVAR                   "pmix.envar.appnd"        // (pmix_envar_t*) append the given value to the specified
-                                                                       //                 envar using the separator character,
-                                                                       //                 creating the envar if it doesn't already exist
-+#define PMIX_FIRST_ENVAR                    "pmix.envar.first"        // (pmix_envar_t*) ensure the given value appears first in the
-+                                                                      //                 specified envar using the separator
-+                                                                      //                 character, creating the envar if it doesn't already exist
- 
- /* attributes relating to allocations */
-+#define PMIX_ALLOC_REQ_ID                   "pmix.alloc.reqid"      // (char*) User-provided string identifier for this allocation request
-+                                                                    //         which can later be used to query status of the request.
- #define PMIX_ALLOC_ID                       "pmix.alloc.id"         // (char*) A string identifier (provided by the host environment) for
-                                                                     //         the resulting allocation which can later be used to reference
-                                                                     //         the allocated resources in, for example, a call to PMIx_Spawn
-@@ -575,40 +631,41 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_ALLOC_NUM_CPU_LIST             "pmix.alloc.ncpulist"   // (char*) regex of #cpus for each node
- #define PMIX_ALLOC_CPU_LIST                 "pmix.alloc.cpulist"    // (char*) regex of specific cpus indicating the cpus involved.
- #define PMIX_ALLOC_MEM_SIZE                 "pmix.alloc.msize"      // (float) number of Mbytes
--#define PMIX_ALLOC_NETWORK                  "pmix.alloc.net"        // (pmix_data_array_t*) Array of pmix_info_t describing
--                                                                    //         network resource request. This must include at least:
--                                                                    //           * PMIX_ALLOC_NETWORK_ID
--                                                                    //           * PMIX_ALLOC_NETWORK_TYPE
--                                                                    //           * PMIX_ALLOC_NETWORK_ENDPTS
-+#define PMIX_ALLOC_FABRIC                   "pmix.alloc.net"        // (pmix_data_array_t*) Array of pmix_info_t describing
-+                                                                    //         fabric resource request. This must include at least:
-+                                                                    //           * PMIX_ALLOC_FABRIC_ID
-+                                                                    //           * PMIX_ALLOC_FABRIC_TYPE
-+                                                                    //           * PMIX_ALLOC_FABRIC_ENDPTS
-                                                                     //         plus whatever other descriptors are desired
--#define PMIX_ALLOC_NETWORK_ID               "pmix.alloc.netid"      // (char*) key to be used when accessing this requested network allocation. The
-+#define PMIX_ALLOC_FABRIC_ID                "pmix.alloc.netid"      // (char*) key to be used when accessing this requested fabric allocation. The
-                                                                     //         allocation will be returned/stored as a pmix_data_array_t of
-                                                                     //         pmix_info_t indexed by this key and containing at least one
-                                                                     //         entry with the same key and the allocated resource description.
--                                                                    //         The type of the included value depends upon the network
-+                                                                    //         The type of the included value depends upon the fabric
-                                                                     //         support. For example, a TCP allocation might consist of a
-                                                                     //         comma-delimited string of socket ranges such as
-                                                                     //         "32000-32100,33005,38123-38146". Additional entries will consist
-                                                                     //         of any provided resource request directives, along with their
-                                                                     //         assigned values. Examples include:
--                                                                    //           * PMIX_ALLOC_NETWORK_TYPE - the type of resources provided
--                                                                    //           * PMIX_ALLOC_NETWORK_PLANE - if applicable, what plane the
-+                                                                    //           * PMIX_ALLOC_FABRIC_TYPE - the type of resources provided
-+                                                                    //           * PMIX_ALLOC_FABRIC_PLANE - if applicable, what plane the
-                                                                     //               resources were assigned from
--                                                                    //           * PMIX_ALLOC_NETWORK_QOS - the assigned QoS
-+                                                                    //           * PMIX_ALLOC_FABRIC_QOS - the assigned QoS
-                                                                     //           * PMIX_ALLOC_BANDWIDTH - the allocated bandwidth
--                                                                    //           * PMIX_ALLOC_NETWORK_SEC_KEY - a security key for the requested
--                                                                    //               network allocation
-+                                                                    //           * PMIX_ALLOC_FABRIC_SEC_KEY - a security key for the requested
-+                                                                    //               fabric allocation
-                                                                     //         NOTE: the assigned values may differ from those requested,
-                                                                     //         especially if the "required" flag was not set in the request
- #define PMIX_ALLOC_BANDWIDTH                "pmix.alloc.bw"         // (float) Mbits/sec
--#define PMIX_ALLOC_NETWORK_QOS              "pmix.alloc.netqos"     // (char*) quality of service level
-+#define PMIX_ALLOC_FABRIC_QOS               "pmix.alloc.netqos"     // (char*) quality of service level
- #define PMIX_ALLOC_TIME                     "pmix.alloc.time"       // (uint32_t) time in seconds that the allocation shall remain valid
--#define PMIX_ALLOC_NETWORK_TYPE             "pmix.alloc.nettype"    // (char*) type of desired transport (e.g., tcp, udp)
--#define PMIX_ALLOC_NETWORK_PLANE            "pmix.alloc.netplane"   // (char*) id string for the NIC (aka plane) to be used for this allocation
-+#define PMIX_ALLOC_FABRIC_TYPE              "pmix.alloc.nettype"    // (char*) type of desired transport (e.g., tcp, udp)
-+#define PMIX_ALLOC_FABRIC_PLANE             "pmix.alloc.netplane"   // (char*) id string for the NIC (aka plane) to be used for this allocation
-                                                                     //         (e.g., CIDR for Ethernet)
--#define PMIX_ALLOC_NETWORK_ENDPTS           "pmix.alloc.endpts"     // (size_t) number of endpoints to allocate per process
--#define PMIX_ALLOC_NETWORK_ENDPTS_NODE      "pmix.alloc.endpts.nd"  // (size_t) number of endpoints to allocate per node
--#define PMIX_ALLOC_NETWORK_SEC_KEY          "pmix.alloc.nsec"       // (pmix_byte_object_t) network security key
-+#define PMIX_ALLOC_FABRIC_ENDPTS            "pmix.alloc.endpts"     // (size_t) number of endpoints to allocate per process
-+#define PMIX_ALLOC_FABRIC_ENDPTS_NODE       "pmix.alloc.endpts.nd"  // (size_t) number of endpoints to allocate per node
-+#define PMIX_ALLOC_FABRIC_SEC_KEY           "pmix.alloc.nsec"       // (pmix_byte_object_t) fabric security key
-+#define PMIX_ALLOC_QUEUE                    "pmix.alloc.queue"      // (char*) name of queue being referenced
- 
- 
- /* job control attributes */
-@@ -701,6 +758,115 @@ typedef uint32_t pmix_rank_t;
- #define PMIX_SETUP_APP_NONENVARS            "pmix.setup.nenv"       // (bool) include all non-envar data
- #define PMIX_SETUP_APP_ALL                  "pmix.setup.all"        // (bool) include all relevant data
- 
-+/* Attributes supporting the PMIx Groups APIs */
-+#define PMIX_GROUP_ID                       "pmix.grp.id"           // (char*) user-provided group identifier
-+#define PMIX_GROUP_LEADER                   "pmix.grp.ldr"          // (bool) this process is the leader of the group
-+#define PMIX_GROUP_OPTIONAL                 "pmix.grp.opt"          // (bool) participation is optional - do not return an error if any of the
-+                                                                    //        specified processes terminate without having joined. The default
-+                                                                    //        is false
-+#define PMIX_GROUP_NOTIFY_TERMINATION       "pmix.grp.notterm"      // (bool) notify remaining members when another member terminates without first
-+                                                                    //        leaving the group. The default is false
-+#define PMIX_GROUP_INVITE_DECLINE           "pmix.grp.decline"      // (bool) notify the inviting process that this process does not wish to
-+                                                                    //        participate in the proposed group The default is true
-+#define PMIX_GROUP_FT_COLLECTIVE            "pmix.grp.ftcoll"       // (bool) adjust internal tracking for terminated processes. Default is false
-+#define PMIX_GROUP_MEMBERSHIP               "pmix.grp.mbrs"         // (pmix_data_array_t*) array of group member ID's
-+#define PMIX_GROUP_ASSIGN_CONTEXT_ID        "pmix.grp.actxid"       // (bool) request that the RM assign a unique numerical (size_t) ID to this group
-+#define PMIX_GROUP_CONTEXT_ID               "pmix.grp.ctxid"        // (size_t) context ID assigned to group
-+#define PMIX_GROUP_LOCAL_ONLY               "pmix.grp.lcl"          // (bool) group operation only involves local procs
-+#define PMIX_GROUP_ENDPT_DATA               "pmix.grp.endpt"        // (pmix_byte_object_t) data collected to be shared during construction
-+
-+
-+/* Storage-Related Attributes */
-+#define PMIX_QUERY_STORAGE_LIST             "pmix.strg.list"       // (char*) return comma-delimited list of identifiers for all available storage systems
-+#define PMIX_STORAGE_CAPACITY_LIMIT         "pmix.strg.cap"        // (uint64_t) return overall capacity (in Megabytes[base2]) of specified storage system
-+#define PMIX_STORAGE_CAPACITY_FREE          "pmix.strg.free"       // (uint64_t) return free capacity (in Megabytes[base2]) of specified storage system
-+#define PMIX_STORAGE_CAPACITY_AVAIL         "pmix.strg.avail"      // (uint64_t) return capacity (in Megabytes[[base2]]) of specified storage
-+                                                                   //            system that is available for use by the calling program
-+
-+#define PMIX_STORAGE_OBJECT_LIMIT           "pmix.strg.obj"        // (uint64_t) return overall limit on number of objects (e.g., inodes) of specified storage system
-+#define PMIX_STORAGE_OBJECTS_FREE           "pmix.strg.objf"       // (uint64_t) return number of free objects (e.g., inodes) of specified storage system
-+#define PMIX_STORAGE_OBJECTS_AVAIL          "pmix.strg.obja"       // (uint64_t) return number of objects (e.g., inodes) of specified storage system
-+                                                                   //            that are available for use by the calling program
-+
-+#define PMIX_STORAGE_BW                     "pmix.strg.bw"         // (float) return overall bandwidth (in Megabytes[base2]/sec) of specified storage system
-+#define PMIX_STORAGE_AVAIL_BW               "pmix.strg.availbw"    // (float) return overall bandwidth (in Megabytes[base2]/sec) of specified storage system
-+                                                                   //         that is available for use by the calling program
-+
-+#define PMIX_STORAGE_ID                     "pmix.strg.id"         // (char*) identifier of the storage system being referenced
-+#define PMIX_STORAGE_PATH                   "pmix.strg.path"       // (char*) Mount point corresponding to a specified storage ID
-+#define PMIX_STORAGE_TYPE                   "pmix.strg.type"       // (char*) Qualifier indicating the type of storage being referenced by a query
-+                                                                   //         (e.g., lustre, gpfs, online, fabric-attached, ...)
-+
-+/* Fabric-related Attributes */
-+#define PMIX_FABRIC_COST_MATRIX             "pmix.fab.cm"          // (pointer) Pointer to a two-dimensional array of point-to-point relative
-+                                                                   //           communication costs expressed as uint16_t values
-+#define PMIX_FABRIC_GROUPS                  "pmix.fab.grps"        // (char*) A string delineating the group membership of nodes in the system,
-+                                                                   //         where each fabric group consists of the group number followed by
-+                                                                   //         a colon and a comma-delimited list of nodes in that group, with the
-+                                                                   //         groups delimited by semi-colons (e.g.,
-+                                                                   //         0:node000,node002,node004,node006;1:node001,node003,node005,node007)
-+#define PMIX_FABRIC_VENDOR                  "pmix.fab.vndr"        // (char*) Name of fabric vendor (e.g., Amazon, Mellanox, Cray, Intel)
-+#define PMIX_FABRIC_IDENTIFIER              "pmix.fab.id"          // (char*) An identifier for the fabric (e.g., MgmtEthernet, Slingshot-11,
-+                                                                   //         OmniPath-1)
-+#define PMIX_FABRIC_INDEX                   "pmix.fab.idx"         // (size_t) The index of the fabric as returned in pmix_fabric_t
-+#define PMIX_FABRIC_NUM_VERTICES            "pmix.fab.nverts"      // (size_t) Total number of NICs in the system - corresponds to the number
-+                                                                   //          of vertices (i.e., rows and columns) in the cost matrix
-+#define PMIX_FABRIC_COORDINATE              "pmix.fab.coord"       // (pmix_coord_t*) Fabric coordinate of the specified process in
-+                                                                   //       the given view type (e.g., logical vs physical)
-+#define PMIX_FABRIC_VIEW                    "pmix.fab.view"        // (pmix_coord_view_t) Requested view type (e.g., logical vs physical)
-+#define PMIX_FABRIC_DIMS                    "pmix.fab.dims"        // (uint32_t) Number of dimensions in the specified fabric plane/view
-+#define PMIX_FABRIC_PLANE                   "pmix.fab.plane"       // (char*) string ID of a fabric plane
-+#define PMIX_FABRIC_SWITCH                  "pmix.fab.switch"      // (char*) string ID of a fabric switch
-+#define PMIX_FABRIC_ENDPT                   "pmix.fab.endpt"       // (pmix_data_array_t*) array of fabric endpts for process
-+#define PMIX_FABRIC_SHAPE                   "pmix.fab.shape"       // (pmix_data_array_t*) number of interfaces (uint32_t) on each dimension of the
-+                                                                   //        specified fabric plane in the requested view
-+#define PMIX_FABRIC_SHAPE_STRING            "pmix.fab.shapestr"    // (char*) fabric shape expressed as a string (e.g., "10x12x2")
-+#define PMIX_SWITCH_PEERS                   "pmix.speers"          // (char*) comma-delimited string of peers that share the same switch as the proc
-+                                                                   //         specified in the call to PMIx_Get. Multi-NIC environments will return
-+                                                                   //         an array of results, each element containing the NIC and the list of
-+                                                                   //         peers sharing the switch to which that NIC is connected.
-+#define PMIX_FABRIC_DEVICE                  "pmix.fabdev"          // (pmix_data_array_t*) An array of pmix_info_t describing a particular
-+                                                                   //         fabric device (NIC).
-+#define PMIX_FABRIC_DEVICE_NAME             "pmix.fabdev.nm"       // (char*) The operating system name associated with the device. This may be
-+                                                                   //         a logical fabric interface name (e.g. eth0 or eno1) or an absolute
-+                                                                   //         filename.
-+#define PMIX_FABRIC_DEVICE_INDEX            "pmix.fabdev.idx"      // (uint32_t) The fabric-wide index of the specified NIC within a given fabric
-+#define PMIX_FABRIC_DEVICE_VENDOR           "pmix.fabdev.vndr"     // (char*) Indicates the name of the vendor that distributes the NIC.
-+#define PMIX_FABRIC_DEVICE_BUS_TYPE         "pmix.fabdev.btyp"     // (char*) The type of bus to which the device is attached (e.g., "PCI", "GEN-Z").
-+#define PMIX_FABRIC_DEVICE_ID               "pmix.fabdev.devid"    // (char*) A vendor-provided identifier for the device or product
-+#define PMIX_FABRIC_DEVICE_DRIVER           "pmix.fabdev.driver"   // (char*) The name of the driver associated with the device
-+#define PMIX_FABRIC_DEVICE_FIRMWARE         "pmix.fabdev.fmwr"     // (char*) The device’s firmware version
-+#define PMIX_FABRIC_DEVICE_ADDRESS          "pmix.fabdev.addr"     // (char*) The primary link-level address associated with the NIC, such as a
-+                                                                   //         MAC address. If multiple addresses are available, only one will be
-+                                                                   //         reported.
-+#define PMIX_FABRIC_DEVICE_MTU              "pmix.fabdev.mtu"      // (size_t) The maximum transfer unit of link level frames or packets, in bytes.
-+#define PMIX_FABRIC_DEVICE_SPEED            "pmix.fabdev.speed"    // (size_t) The active link data rate, given in bits per second.
-+#define PMIX_FABRIC_DEVICE_STATE            "pmix.fabdev.state"    // (pmix_link_state_t) The last available physical port state. Possible values
-+                                                                   //          are PMIX_LINK_STATE_UNKNOWN, PMIX_LINK_DOWN, and PMIX_LINK_UP, to
-+                                                                   //          indicate if the port state is unknown or not applicable (unknown),
-+                                                                   //          inactive (down), or active (up).
-+#define PMIX_FABRIC_DEVICE_TYPE             "pmix.fabdev.type"     // (char*) Specifies the type of fabric interface currently active on the device,
-+                                                                   //         such as Ethernet or InfiniBand.
-+#define PMIX_FABRIC_DEVICE_PCI_DEVID        "pmix.fabdev.pcidevid" // (char*) A node-level unique identifier for a PCI device. Provided only if the
-+                                                                   //         device is located on a \ac{PCI} bus. The identifier is constructed as
-+                                                                   //         a four-part tuple delimited by colons comprised of the \ac{PCI} 16-bit
-+                                                                   //         domain, 8-bit bus, 8-bit device, and 8-bit function IDs, each expressed
-+                                                                   //         in zero-extended hexadecimal form. Thus, an example identifier might be
-+                                                                   //         "abc1:0f:23:01". The combination of node identifier PMIX_HOSTNAME or
-+                                                                   //         PMIX_NODEID and PMIX_FABRIC_DEVICE_PCI_DEVID shall be unique within the
-+                                                                   //         system.
-+
-+/* Descriptive Attributes */
-+#define PMIX_MAX_VALUE                      "pmix.descr.maxval"    // (varies) Used in pmix_regattr_t to describe the maximum valid value
-+                                                                   //          for the associated attribute.
-+#define PMIX_MIN_VALUE                      "pmix.descr.minval"    // (varies) Used in pmix_regattr_t to describe the minimum valid value
-+                                                                   //          for the associated attribute.
-+#define PMIX_ENUM_VALUE                     "pmix.descr.enum"      // (char*) Used in pmix_regattr_t to describe accepted values for the
-+                                                                   //         associated attribute. Numerical values shall be presented in
-+                                                                   //         a form convertible to the attribute's declared data type.
-+                                                                   //         Named values (i.e., values defined by constant names via a
-+                                                                   //         typical C-language enum declaration) must be provided as
-+                                                                   //         their numerical equivalent.
- 
- /****    PROCESS STATE DEFINITIONS    ****/
- typedef uint8_t pmix_proc_state_t;
-@@ -740,10 +906,35 @@ typedef uint8_t pmix_proc_state_t;
- #define PMIX_PROC_STATE_FAILED_TO_LAUNCH        (PMIX_PROC_STATE_ERROR + 13)  /* unable to launch process */
- 
- 
-+/****    JOB STATE DEFINITIONS    ****/
-+typedef uint8_t pmix_job_state_t;
-+#define PMIX_JOB_STATE_UNDEF                     0  // undefined process state
-+#define PMIX_JOB_STATE_PREPPED                   1  // job is ready to be launched
-+#define PMIX_JOB_STATE_LAUNCH_UNDERWAY           2  // job launch underway
-+#define PMIX_JOB_STATE_RUNNING                   3  // all procs have been spawned
-+#define PMIX_JOB_STATE_SUSPENDED                 4  // job has been suspended
-+#define PMIX_JOB_STATE_CONNECTED                 5  // all procs have connected to their PMIx server
-+
-+/*
-+* Define a "boundary" so users can easily and quickly determine
-+* if a job is still running or not - any value less than
-+* this one means that the job has not terminated
-+*/
-+#define PMIX_JOB_STATE_UNTERMINATED             15
-+
-+#define PMIX_JOB_STATE_TERMINATED               20  // job has terminated and is no longer running - typically will
-+                                                    // be accompanied by the job exit status in response to a query
-+
-+/* Define a boundary so users can easily and quickly determine
-+* if a job abnormally terminated - leave a little room
-+* for future expansion
-+*/
-+#define PMIX_JOB_STATE_TERMINATED_WITH_ERROR    50  // job has terminated and is no longer running - typically will
-+                                                    // be accompanied by a job-related error code in response to a query
-+
-+
- /****    PMIX ERROR CONSTANTS    ****/
- /* PMIx errors are always negative, with 0 reserved for success */
--#define PMIX_ERR_BASE                   0
--
- typedef int pmix_status_t;
- 
- /* v1.x error values - must be fixed in place for backward
-@@ -753,72 +944,39 @@ typedef int pmix_status_t;
-  * at least defined to ensure older codes will compile */
- #define PMIX_SUCCESS                                 0
- #define PMIX_ERROR                                  -1          // general error
--#define PMIX_ERR_SILENT                             -2
--/* debugger release flag */
--#define PMIX_ERR_DEBUGGER_RELEASE                   -3
- /* fault tolerance */
- #define PMIX_ERR_PROC_RESTART                       -4
- #define PMIX_ERR_PROC_CHECKPOINT                    -5
- #define PMIX_ERR_PROC_MIGRATE                       -6
--/* abort */
--#define PMIX_ERR_PROC_ABORTED                       -7
--#define PMIX_ERR_PROC_REQUESTED_ABORT               -8
--#define PMIX_ERR_PROC_ABORTING                      -9
- /* communication failures */
--#define PMIX_ERR_SERVER_FAILED_REQUEST              -10
--#define PMIX_EXISTS                                 -11
- #define PMIX_ERR_INVALID_CRED                       -12
--#define PMIX_ERR_HANDSHAKE_FAILED                   -13
--#define PMIX_ERR_READY_FOR_HANDSHAKE                -14
- #define PMIX_ERR_WOULD_BLOCK                        -15
- #define PMIX_ERR_UNKNOWN_DATA_TYPE                  -16
--#define PMIX_ERR_PROC_ENTRY_NOT_FOUND               -17
- #define PMIX_ERR_TYPE_MISMATCH                      -18
- #define PMIX_ERR_UNPACK_INADEQUATE_SPACE            -19
- #define PMIX_ERR_UNPACK_FAILURE                     -20
- #define PMIX_ERR_PACK_FAILURE                       -21
--#define PMIX_ERR_PACK_MISMATCH                      -22
- #define PMIX_ERR_NO_PERMISSIONS                     -23
- #define PMIX_ERR_TIMEOUT                            -24
- #define PMIX_ERR_UNREACH                            -25
--#define PMIX_ERR_IN_ERRNO                           -26
- #define PMIX_ERR_BAD_PARAM                          -27
- #define PMIX_ERR_RESOURCE_BUSY                      -28
- #define PMIX_ERR_OUT_OF_RESOURCE                    -29
--#define PMIX_ERR_DATA_VALUE_NOT_FOUND               -30
- #define PMIX_ERR_INIT                               -31
- #define PMIX_ERR_NOMEM                              -32
--#define PMIX_ERR_INVALID_ARG                        -33
--#define PMIX_ERR_INVALID_KEY                        -34
--#define PMIX_ERR_INVALID_KEY_LENGTH                 -35
--#define PMIX_ERR_INVALID_VAL                        -36
--#define PMIX_ERR_INVALID_VAL_LENGTH                 -37
--#define PMIX_ERR_INVALID_LENGTH                     -38
--#define PMIX_ERR_INVALID_NUM_ARGS                   -39
--#define PMIX_ERR_INVALID_ARGS                       -40
--#define PMIX_ERR_INVALID_NUM_PARSED                 -41
--#define PMIX_ERR_INVALID_KEYVALP                    -42
--#define PMIX_ERR_INVALID_SIZE                       -43
--#define PMIX_ERR_INVALID_NAMESPACE                  -44
--#define PMIX_ERR_SERVER_NOT_AVAIL                   -45
- #define PMIX_ERR_NOT_FOUND                          -46
- #define PMIX_ERR_NOT_SUPPORTED                      -47
--#define PMIX_ERR_NOT_IMPLEMENTED                    -48
- #define PMIX_ERR_COMM_FAILURE                       -49
- #define PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER     -50
- #define PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES     -51
--
--/* define a starting point for v2.x error values */
--#define PMIX_ERR_V2X_BASE                   -100
-+#define PMIX_ERR_PARTIAL_SUCCESS                    -52
-+#define PMIX_ERR_DUPLICATE_KEY                      -53
-+#define PMIX_ERR_GET_MALLOC_REQD                    -54
-+#define PMIX_PROCESS_SET_DEFINE                     -55
-+#define PMIX_PROCESS_SET_DELETE                     -56
- 
- /* v2.x communication errors */
--#define PMIX_ERR_LOST_CONNECTION_TO_SERVER          -101
--#define PMIX_ERR_LOST_PEER_CONNECTION               -102
--#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT          -103
--/* used by the query system */
- #define PMIX_QUERY_PARTIAL_SUCCESS                  -104
--/* request responses */
--#define PMIX_NOTIFY_ALLOC_COMPLETE                  -105
- /* job control */
- #define PMIX_JCTRL_CHECKPOINT                       -106    // monitored by client to trigger checkpoint operation
- #define PMIX_JCTRL_CHECKPOINT_COMPLETE              -107    // sent by client and monitored by server to notify that requested
-@@ -829,16 +987,11 @@ typedef int pmix_status_t;
- #define PMIX_MONITOR_HEARTBEAT_ALERT                -109
- #define PMIX_MONITOR_FILE_ALERT                     -110
- #define PMIX_PROC_TERMINATED                        -111
--#define PMIX_ERR_INVALID_TERMINATION                -112
- 
- /* operational */
- #define PMIX_ERR_EVENT_REGISTRATION                 -144
--#define PMIX_ERR_JOB_TERMINATED                     -145
- #define PMIX_ERR_UPDATE_ENDPOINTS                   -146
- #define PMIX_MODEL_DECLARED                         -147
--#define PMIX_GDS_ACTION_COMPLETE                    -148
--#define PMIX_PROC_HAS_CONNECTED                     -149
--#define PMIX_CONNECT_REQUESTED                      -150
- #define PMIX_MODEL_RESOURCES                        -151     // model resource usage has changed
- #define PMIX_OPENMP_PARALLEL_ENTERED                -152     // an OpenMP parallel region has been entered
- #define PMIX_OPENMP_PARALLEL_EXITED                 -153     // an OpenMP parallel region has completed
-@@ -847,15 +1000,53 @@ typedef int pmix_status_t;
- #define PMIX_OPERATION_IN_PROGRESS                  -156
- #define PMIX_OPERATION_SUCCEEDED                    -157
- #define PMIX_ERR_INVALID_OPERATION                  -158
-+#define PMIX_GROUP_INVITED                          -159
-+#define PMIX_GROUP_LEFT                             -160
-+#define PMIX_GROUP_INVITE_ACCEPTED                  -161
-+#define PMIX_GROUP_INVITE_DECLINED                  -162
-+#define PMIX_GROUP_INVITE_FAILED                    -163
-+#define PMIX_GROUP_MEMBERSHIP_UPDATE                -164
-+#define PMIX_GROUP_CONSTRUCT_ABORT                  -165
-+#define PMIX_GROUP_CONSTRUCT_COMPLETE               -166
-+#define PMIX_GROUP_LEADER_SELECTED                  -167
-+#define PMIX_GROUP_LEADER_FAILED                    -168
-+#define PMIX_GROUP_CONTEXT_ID_ASSIGNED              -169
-+#define PMIX_GROUP_MEMBER_FAILED                    -170
-+#define PMIX_ERR_REPEAT_ATTR_REGISTRATION           -171
-+#define PMIX_ERR_IOF_FAILURE                        -172
-+#define PMIX_ERR_IOF_COMPLETE                       -173
-+#define PMIX_LAUNCH_COMPLETE                        -174     // include nspace of the launched job with notification
-+#define PMIX_FABRIC_UPDATED                         -175
-+#define PMIX_FABRIC_UPDATE_PENDING                  -176
-+
-+/* job-related errors */
-+#define PMIX_ERR_JOB_APP_NOT_EXECUTABLE             -177
-+#define PMIX_ERR_JOB_NO_EXE_SPECIFIED               -178
-+#define PMIX_ERR_JOB_FAILED_TO_MAP                  -179
-+#define PMIX_ERR_JOB_CANCELED                       -180
-+#define PMIX_ERR_JOB_FAILED_TO_LAUNCH               -181
-+#define PMIX_ERR_JOB_ABORTED                        -182
-+#define PMIX_ERR_JOB_KILLED_BY_CMD                  -183
-+#define PMIX_ERR_JOB_ABORTED_BY_SIG                 -184
-+#define PMIX_ERR_JOB_TERM_WO_SYNC                   -185
-+#define PMIX_ERR_JOB_SENSOR_BOUND_EXCEEDED          -186
-+#define PMIX_ERR_JOB_NON_ZERO_TERM                  -187
-+#define PMIX_ERR_JOB_ALLOC_FAILED                   -188
-+#define PMIX_ERR_JOB_ABORTED_BY_SYS_EVENT           -189
-+
-+/* job-related non-error events */
-+#define PMIX_EVENT_JOB_START                        -191
-+#define PMIX_EVENT_JOB_END                          -145
-+#define PMIX_EVENT_SESSION_START                    -192
-+#define PMIX_EVENT_SESSION_END                      -193
- 
- /* system failures */
--#define PMIX_ERR_NODE_DOWN                          -231
--#define PMIX_ERR_NODE_OFFLINE                       -232
--#define PMIX_ERR_SYS_OTHER                          -330
-+#define PMIX_ERR_SYS_BASE                           -230
-+
- 
- /* define a macro for identifying system event values */
- #define PMIX_SYSTEM_EVENT(a)   \
--    ((a) <= PMIX_ERR_NODE_DOWN && PMIX_ERR_SYS_OTHER <= (a))
-+    ((a) <= PMIX_ERR_SYS_BASE && PMIX_ERR_SYS_OTHER <= (a))
- 
- /* used by event handlers */
- #define PMIX_EVENT_NO_ACTION_TAKEN                  -331
-@@ -863,10 +1054,6 @@ typedef int pmix_status_t;
- #define PMIX_EVENT_ACTION_DEFERRED                  -333
- #define PMIX_EVENT_ACTION_COMPLETE                  -334
- 
--/* define a starting point for PMIx internal error codes
-- * that are never exposed outside the library */
--#define PMIX_INTERNAL_ERR_BASE                      -1330
--
- /* define a starting point for user-level defined error
-  * constants - negative values larger than this are guaranteed
-  * not to conflict with PMIx values. Definitions should always
-@@ -925,7 +1112,11 @@ typedef uint16_t pmix_data_type_t;
- // Hole left by deprecation/removal of PMIX_INFO_ARRAY
- #define PMIX_IOF_CHANNEL        45
- #define PMIX_ENVAR              46
--#define PMIX_REGEX              49 // numerical ID matches v4
-+#define PMIX_COORD              47
-+#define PMIX_REGATTR            48
-+#define PMIX_REGEX              49
-+#define PMIX_JOB_STATE          50
-+#define PMIX_LINK_STATE         51
- /********************/
- 
- /* define a boundary for implementers so they can add their own data types */
-@@ -1009,6 +1200,20 @@ typedef uint16_t pmix_iof_channel_t;
- #define PMIX_FWD_STDDIAG_CHANNEL    0x0008
- #define PMIX_FWD_ALL_CHANNELS       0x00ff
- 
-+/* define values associated with PMIx_Group_join
-+ * to indicate accept and decline - this is
-+ * done for readability of user code */
-+typedef enum {
-+    PMIX_GROUP_DECLINE,
-+    PMIX_GROUP_ACCEPT
-+} pmix_group_opt_t;
-+
-+typedef enum {
-+    PMIX_GROUP_CONSTRUCT,
-+    PMIX_GROUP_DESTRUCT
-+} pmix_group_operation_t;
-+
-+
- /* define some "hooks" external libraries can use to
-  * intercept memory allocation/release operations */
- static inline void* pmix_malloc(size_t n)
-@@ -1030,10 +1235,10 @@ static inline void* pmix_calloc(size_t n
- #define PMIX_CHECK_KEY(a, b) \
-     (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN))
- 
--#define PMIX_LOAD_KEY(a, b) \
--    do {                                            \
--        memset((a), 0, PMIX_MAX_KEYLEN+1);          \
--        pmix_strncpy((a), (b), PMIX_MAX_KEYLEN);    \
-+#define PMIX_LOAD_KEY(a, b)                                             \
-+    do {                                                                \
-+        memset((a), 0, PMIX_MAX_KEYLEN+1);                              \
-+        pmix_strncpy((char*)(a), (const char*)(b), PMIX_MAX_KEYLEN);    \
-     }while(0)
- 
- /* define a convenience macro for loading nspaces */
-@@ -1061,6 +1266,112 @@ static inline void* pmix_calloc(size_t n
-     (PMIX_CHECK_NSPACE((a)->nspace, (b)->nspace) && ((a)->rank == (b)->rank || (PMIX_RANK_WILDCARD == (a)->rank || PMIX_RANK_WILDCARD == (b)->rank)))
- 
- 
-+/****    PMIX COORD    ****/
-+/* define coordinate system views */
-+typedef uint8_t pmix_coord_view_t;
-+#define PMIX_COORD_VIEW_UNDEF       0x00
-+#define PMIX_COORD_LOGICAL_VIEW     0x01
-+#define PMIX_COORD_PHYSICAL_VIEW    0x02
-+
-+/* define a structure for a proc's fabric coordinate */
-+typedef struct pmix_coord {
-+    char *fabric;
-+    char *plane;
-+    pmix_coord_view_t view;
-+    int *coord;
-+    size_t dims;
-+} pmix_coord_t;
-+
-+#define PMIX_COORD_CREATE(m, d, n)                                      \
-+    do {                                                                \
-+        (m) = (pmix_coord_t*)pmix_calloc((n), sizeof(pmix_coord_t));    \
-+        if (NULL != (m)) {                                              \
-+            (m)->fabric = NULL;                                         \
-+            (m)->plane = NULL;                                          \
-+            (m)->view = PMIX_COORD_VIEW_UNDEF;                          \
-+            (m)->dims = (d);                                            \
-+            (m)->coord = (int*)pmix_calloc((m)->dims, sizeof(int));     \
-+        }                                                               \
-+    } while(0)
-+
-+#define PMIX_COORD_CONSTRUCT(m)             \
-+    do {                                    \
-+        (m)->fabric = NULL;                 \
-+        (m)->plane = NULL;                  \
-+        (m)->view = PMIX_COORD_VIEW_UNDEF;  \
-+        (m)->coord = NULL;                  \
-+        (m)->dims = 0;                      \
-+    } while(0)
-+
-+#define PMIX_COORD_DESTRUCT(m)              \
-+    do {                                    \
-+        (m)->view = PMIX_COORD_VIEW_UNDEF;  \
-+        if (NULL != (m)->coord) {           \
-+            if (NULL != (m)->fabric) {      \
-+                free((m)->fabric);          \
-+            }                               \
-+            if (NULL != (m)->plane) {       \
-+                free((m)->plane);           \
-+            };                              \
-+            pmix_free((m)->coord);          \
-+            (m)->coord = NULL;              \
-+            (m)->dims = 0;                  \
-+        }                                   \
-+    } while(0)
-+
-+#define PMIX_COORD_FREE(m, n)                       \
-+    do {                                            \
-+        size_t _nc_;                                \
-+        if (NULL != (m)) {                          \
-+            for (_nc_ = 0; _nc_ < (n); _nc_++) {    \
-+                PMIX_COORD_DESTRUCT(&(m)[_nc_]);    \
-+            }                                       \
-+            free((m));                              \
-+            (m) = NULL;                             \
-+        }                                           \
-+    } while(0)
-+
-+
-+/****    PMIX LINK STATES    ****/
-+typedef uint8_t pmix_link_state_t;
-+#define PMIX_LINK_STATE_UNKNOWN     0  // The port state is unknown or not applicable
-+#define PMIX_LINK_DOWN              1  // The port is inactive.
-+#define PMIX_LINK_UP                2  // The port is active.
-+
-+
-+/****   PMIX CPUSET    ****/
-+typedef struct{
-+    char *source;
-+    void *bitmap;
-+} pmix_cpuset_t;
-+
-+#define PMIX_CPUSET_CONSTRUCT(m) \
-+    memset((m), 0, sizeof(pmix_cpuset_t))
-+
-+
-+/****    PMIX TOPOLOGY    ****/
-+typedef struct {
-+    char *source;
-+    void *topology;
-+} pmix_topology_t;
-+
-+#define PMIX_TOPOLOGY_CONSTRUCT(m) \
-+    memset((m), 0, sizeof(pmix_topology_t))
-+
-+/**** PMIX RELATIVE LOCALITY    ****/
-+typedef uint16_t pmix_locality_t;
-+#define PMIX_LOCALITY_UNKNOWN           0x0000
-+#define PMIX_LOCALITY_NONLOCAL          0x8000
-+#define PMIX_LOCALITY_SHARE_HWTHREAD    0x0001
-+#define PMIX_LOCALITY_SHARE_CORE        0x0002
-+#define PMIX_LOCALITY_SHARE_L1CACHE     0x0004
-+#define PMIX_LOCALITY_SHARE_L2CACHE     0x0008
-+#define PMIX_LOCALITY_SHARE_L3CACHE     0x0010
-+#define PMIX_LOCALITY_SHARE_PACKAGE     0x0020
-+#define PMIX_LOCALITY_SHARE_NUMA        0x0040
-+#define PMIX_LOCALITY_SHARE_NODE        0x4000
-+
-+
- /****    PMIX BYTE OBJECT    ****/
- typedef struct pmix_byte_object {
-     char *bytes;
-@@ -1352,6 +1663,11 @@ typedef struct pmix_data_array {
- /**** AVOID CIRCULAR DEPENDENCIES                    ****/
- 
- 
-+/* we cannot forward-declare the pmix_regattr_t struct
-+ * as Cython doesn't know what to do with it. Thus, we
-+ * will utilize the void* entry of the pmix_value_t to
-+ * hold the point to pmix_regattr_t */
-+
- /****    PMIX VALUE STRUCT    ****/
- 
- /* NOTE: operations can supply a collection of values under
-@@ -1394,6 +1710,9 @@ typedef struct pmix_value {
-         void *ptr;
-         pmix_alloc_directive_t adir;
-         pmix_envar_t envar;
-+        pmix_coord_t *coord;
-+        pmix_link_state_t linkstate;
-+        pmix_job_state_t jstate;
-     } data;
- } pmix_value_t;
- /* allocate and initialize a specified number of value structs */
-@@ -1496,29 +1815,6 @@ typedef struct pmix_value {
-         }                                               \
-     } while(0)
- 
--#if 0
--    // This macro is no longer supported in the v3.2 and later series
--#define PMIX_VALUE_COMPRESSED_STRING_UNPACK(s)                              \
--    do {                                                                    \
--        char *tmp;                                                          \
--        /* if this is a compressed string, then uncompress it */            \
--        if (PMIX_COMPRESSED_STRING == (s)->type) {                          \
--            pmix_util_uncompress_string(&tmp, (uint8_t*)(s)->data.bo.bytes, \
--                (s)->data.bo.size);                                         \
--            if (NULL == tmp) {                                              \
--                PMIX_ERROR_LOG(PMIX_ERR_NOMEM);                             \
--                rc = PMIX_ERR_NOMEM;                                        \
--                PMIX_VALUE_RELEASE(s);                                      \
--                val = NULL;                                                 \
--            } else {                                                        \
--                PMIX_VALUE_DESTRUCT(s);                                     \
--                (s)->data.string = tmp;                                     \
--                (s)->type = PMIX_STRING;                                    \
--            }                                                               \
--        }                                                                   \
--    } while(0)
--#endif
--
- /****    PMIX INFO STRUCT    ****/
- typedef struct pmix_info {
-     pmix_key_t key;
-@@ -1555,7 +1851,7 @@ typedef struct pmix_info {
-             for (_is=0; _is < (n); _is++) {         \
-                 PMIX_INFO_DESTRUCT(&((m)[_is]));    \
-             }                                       \
--            pmix_free((m));                              \
-+            pmix_free((m));                         \
-             (m) = NULL;                             \
-         }                                           \
-     } while (0)
-@@ -1809,6 +2105,109 @@ typedef struct pmix_query {
-         }                                           \
-     } while (0)
- 
-+/****    ATTRIBUTE REGISTRATION STRUCT   ****/
-+typedef struct pmix_regattr_t {
-+    char *name;
-+    pmix_key_t string;
-+    pmix_data_type_t type;
-+    char **description;
-+} pmix_regattr_t;
-+
-+#define PMIX_REGATTR_CONSTRUCT(a)                       \
-+    do {                                                \
-+        if (NULL != (a)) {                              \
-+            (a)->name = NULL;                           \
-+            memset((a)->string, 0, PMIX_MAX_KEYLEN+1);  \
-+            (a)->type = PMIX_UNDEF;                     \
-+            (a)->description = NULL;                    \
-+        }                                               \
-+    } while(0)
-+
-+#define PMIX_REGATTR_LOAD(a, n, k, t, v)                        \
-+    do {                                                        \
-+        pmix_status_t _rgl;                                     \
-+        if (NULL != (n)) {                                      \
-+            (a)->name = strdup((n));                            \
-+        }                                                       \
-+        if (NULL != (k)) {                                      \
-+            PMIX_LOAD_KEY((a)->string, (k));                    \
-+        }                                                       \
-+        (a)->type = (t);                                        \
-+        if (NULL != (v)) {                                      \
-+            PMIX_ARGV_APPEND(_rgl, &(a)->description, (v));     \
-+        }                                                       \
-+    } while(0)
-+
-+#define PMIX_REGATTR_DESTRUCT(a)                    \
-+    do {                                            \
-+        if (NULL != (a)) {                          \
-+            if (NULL != (a)->name) {                \
-+                pmix_free((a)->name);                    \
-+            }                                       \
-+            if (NULL != (a)->description) {         \
-+                PMIX_ARGV_FREE((a)->description);   \
-+            }                                       \
-+        }                                           \
-+    } while(0)
-+
-+#define PMIX_REGATTR_CREATE(m, n)                                       \
-+    do {                                                                \
-+        (m) = (pmix_regattr_t*)pmix_calloc((n) , sizeof(pmix_regattr_t));    \
-+    } while (0)
-+
-+#define PMIX_REGATTR_FREE(m, n)                         \
-+    do {                                                \
-+        size_t _ra;                                     \
-+        if (NULL != (m)) {                              \
-+            for (_ra=0; _ra < (n); _ra++) {             \
-+                PMIX_REGATTR_DESTRUCT(&((m)[_ra]));     \
-+            }                                           \
-+            pmix_free((m));                                  \
-+            (m) = NULL;                                 \
-+        }                                               \
-+    } while (0)
-+
-+#define PMIX_REGATTR_XFER(a, b)                                         \
-+    do {                                                                \
-+        size_t _n;                                                      \
-+        PMIX_REGATTR_CONSTRUCT((a));                                    \
-+        if (NULL != ((b)->name)) {                                      \
-+            (a)->name = strdup((b)->name);                              \
-+        }                                                               \
-+        PMIX_LOAD_KEY((a)->string, (b)->string);                        \
-+        (a)->type = (b)->type;                                          \
-+        if (NULL != (b)->description) {                                 \
-+            PMIX_ARGV_COPY((a)->description, (b)->description);         \
-+        }                                                               \
-+    } while(0)
-+
-+
-+/****    FABRIC STRUCT    ****/
-+/* Define a pmix_fabric_t struct for
-+ * interacting with fabric-related interfaces */
-+typedef struct pmix_fabric_s {
-+    /* user-supplied name for this fabric */
-+    char *name;
-+    /* a PMIx-supplied index identifying this registration object */
-+    size_t index;
-+    /* array containing information (provided by the PMIx library)
-+     * about the fabric */
-+    pmix_info_t *info;
-+    size_t ninfo;
-+    /* object pointer for use by the PMIx library */
-+    void *module;
-+} pmix_fabric_t;
-+
-+/* convenience macros to support pmix_fabric_t */
-+#define PMIX_FABRIC_CONSTRUCT(x) \
-+    memset(x, 0, sizeof(pmix_fabric_t))
-+
-+typedef enum {
-+    PMIX_FABRIC_REQUEST_INFO,
-+    PMIX_FABRIC_UPDATE_INFO,
-+    PMIX_FABRIC_GET_VERTEX_INFO,
-+    PMIX_FABRIC_GET_DEVICE_INDEX
-+} pmix_fabric_operation_t;
- 
- /****    GENERIC HELPER MACROS    ****/
- 
-@@ -1845,7 +2244,7 @@ typedef struct pmix_query {
-  * argv array - same as above only prepend
-  */
- #define PMIX_ARGV_PREPEND(r, a, b) \
--    (r) = pmix_argv_prepend_nosize(a, b)
-+    (r) = pmix_argv_prepend_nosize(&(a), b)
- 
- /* Append to an argv-style array, but only if the provided argument
-  * doesn't already exist somewhere in the array. Ignore the size of the array.
-@@ -1858,10 +2257,10 @@ typedef struct pmix_query {
-  *
-  * This function is identical to the pmix_argv_append_nosize() function
-  * except that it only appends the provided argument if it does not already
-- * exist in the provided array, or overwrites it if it is.
-+ * exist in the provided array.
-  */
--#define PMIX_ARGV_APPEND_UNIQUE(r, a, b, c) \
--    (r) = pmix_argv_append_unique_nosize(a, b, c)
-+#define PMIX_ARGV_APPEND_UNIQUE(r, a, b) \
-+    (r) = pmix_argv_append_unique_nosize(a, b)
- 
- /* Free a NULL-terminated argv array.
-  *
-@@ -2091,9 +2490,9 @@ typedef void (*pmix_notification_fn_t)(s
- typedef void (*pmix_hdlr_reg_cbfunc_t)(pmix_status_t status,
-                                        size_t refid,
-                                        void *cbdata);
--/* maintain backward compatibility with v2 definition - change of name */
-+/* retain the deprecated form */
- typedef void (*pmix_evhdlr_reg_cbfunc_t)(pmix_status_t status,
--                                         size_t evhdlr_ref,
-+                                         size_t refid,
-                                          void *cbdata);
- 
- /* define a callback function for calls to PMIx_Get_nb. The status
-@@ -2544,11 +2943,12 @@ PMIX_EXPORT pmix_status_t PMIx_Data_prin
-  * NOTE: This is NOT a destructive procedure - the
-  * source buffer's payload will remain intact, as will any pre-existing
-  * payload in the destination's buffer.
-- */
-+*/
- PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest,
-                                                  pmix_data_buffer_t *src);
- 
- 
-+
- /********    STANDARD MACROS FOR DARRAY AND VALUE SUPPORT     ********/
- static inline void pmix_darray_destruct(pmix_data_array_t *m);
- 
-@@ -2606,7 +3006,8 @@ static inline void pmix_darray_destruct(
-         } else if (PMIX_APP == m->type) {
-             pmix_app_t *_a = (pmix_app_t*)m->array;
-             PMIX_APP_FREE(_a, m->size);
--        } else if (PMIX_BYTE_OBJECT == m->type) {
-+        } else if (PMIX_BYTE_OBJECT == m->type ||
-+                   PMIX_COMPRESSED_STRING == m->type) {
-             pmix_byte_object_t *_b = (pmix_byte_object_t*)m->array;
-             PMIX_BYTE_OBJECT_FREE(_b, m->size);
-         } else if (PMIX_STRING == m->type) {
-@@ -2644,7 +3045,8 @@ static inline void pmix_darray_destruct(
-                 PMIX_QUERY_CREATE((m)->array, (n));                 \
-             } else if (PMIX_APP == (t)) {                           \
-                 PMIX_APP_CREATE((m)->array, (n));                   \
--            } else if (PMIX_BYTE_OBJECT == (t)) {                   \
-+            } else if (PMIX_BYTE_OBJECT == (t) ||                   \
-+                       PMIX_COMPRESSED_STRING == (t)) {             \
-                 PMIX_BYTE_OBJECT_CREATE((m)->array, (n));           \
-             } else if (PMIX_ALLOC_DIRECTIVE == (t) ||               \
-                        PMIX_PROC_STATE == (t) ||                    \
-@@ -2653,10 +3055,9 @@ static inline void pmix_darray_destruct(
-                        PMIX_DATA_RANGE == (t) ||                    \
-                        PMIX_BYTE == (t) ||                          \
-                        PMIX_INT8 == (t) ||                          \
--                       PMIX_UINT8 == (t)) {                         \
--                (m)->array = pmix_calloc((n), sizeof(int8_t));      \
--            } else if (PMIX_POINTER == (t)) {                       \
--		(m)->array = pmix_calloc((n), sizeof(void*));       \
-+                       PMIX_UINT8 == (t) ||                         \
-+                       PMIX_POINTER == (t)) {                       \
-+                (m)->array = pmix_calloc((n), sizeof(int8_t));           \
-             } else if (PMIX_STRING == (t)) {                        \
-                 (m)->array = pmix_calloc((n), sizeof(char*));            \
-             } else if (PMIX_SIZE == (t)) {                          \
-@@ -2688,6 +3089,14 @@ static inline void pmix_darray_destruct(
-                 (m)->array = pmix_calloc((n), sizeof(struct timeval));   \
-             } else if (PMIX_TIME == (t)) {                          \
-                 (m)->array = pmix_calloc((n), sizeof(time_t));           \
-+            } else if (PMIX_REGATTR == (t)) {                       \
-+                PMIX_REGATTR_CREATE((m)->array, (n));               \
-+            } else if (PMIX_BOOL == (t)) {                          \
-+                (m)->array = pmix_calloc((n), sizeof(bool));             \
-+            } else if (PMIX_COORD == (t)) {                         \
-+                (m)->array = pmix_calloc((n), sizeof(pmix_coord_t));     \
-+            } else if (PMIX_LINK_STATE == (t)) {                    \
-+                (m)->array = pmix_calloc((n), sizeof(pmix_link_state_t));  \
-             }                                                       \
-         } else {                                                    \
-             (m)->array = NULL;                                      \
-@@ -2728,22 +3137,21 @@ static inline void pmix_strncpy(char *de
-                                 const char *src,
-                                 size_t len)
- {
--    size_t i, k;
--    char *new_dest = dest;
-+    size_t i;
- 
-     /* use an algorithm that also protects against
-      * non-NULL-terminated src strings */
--    for (i=0, k=0; i <= len; ++i, ++src, ++new_dest) {
--        ++k;
--        *new_dest = *src;
-+    for (i=0; i < len; ++i, ++src, ++dest) {
-+        *dest = *src;
-         if ('\0' == *src) {
-             break;
-         }
-     }
--    dest[k-1] = '\0';
-+    *dest = '\0';
- }
- 
- #include <pmix_extend.h>
-+#include <pmix_deprecated.h>
- 
- #if defined(c_plusplus) || defined(__cplusplus)
- }
-Index: pmix-3.2.0/include/pmix_deprecated.h
-===================================================================
---- /dev/null
-+++ pmix-3.2.0/include/pmix_deprecated.h
-@@ -0,0 +1,156 @@
-+/*
-+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
-+ * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
-+ *                         All rights reserved.
-+ * Copyright (c) 2015      Research Organization for Information Science
-+ *                         and Technology (RIST). All rights reserved.
-+ * $COPYRIGHT$
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions are
-+ * met:
-+ *
-+ * - Redistributions of source code must retain the above copyright
-+ *   notice, this list of conditions and the following disclaimer.
-+ *
-+ * - Redistributions in binary form must reproduce the above copyright
-+ *   notice, this list of conditions and the following disclaimer listed
-+ *   in this license in the documentation and/or other materials
-+ *   provided with the distribution.
-+ *
-+ * - Neither the name of the copyright holders nor the names of its
-+ *   contributors may be used to endorse or promote products derived from
-+ *   this software without specific prior written permission.
-+ *
-+ * The copyright holders provide no reassurances that the source code
-+ * provided does not infringe any patent, copyright, or any other
-+ * intellectual property rights of third parties.  The copyright holders
-+ * disclaim any liability to any recipient for claims brought against
-+ * recipient by any third party for infringement of that parties
-+ * intellectual property rights.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ * $HEADER$
-+ *
-+ * PMIx provides a "function-shipping" approach to support for
-+ * implementing the server-side of the protocol. This method allows
-+ * resource managers to implement the server without being burdened
-+ * with PMIx internal details. Accordingly, each PMIx API is mirrored
-+ * here in a function call to be provided by the server. When a
-+ * request is received from the client, the corresponding server function
-+ * will be called with the information.
-+ *
-+ * Any functions not supported by the RM can be indicated by a NULL for
-+ * the function pointer. Client calls to such functions will have a
-+ * "not supported" error returned.
-+ */
-+
-+#ifndef PMIx_DEPRECATED_H
-+#define PMIx_DEPRECATED_H
-+
-+#if defined(c_plusplus) || defined(__cplusplus)
-+extern "C" {
-+#endif
-+
-+/***** v4 DECPRECATIONS/REMOVALS *****/
-+/* APIs */
-+PMIX_EXPORT pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc,
-+                                                      pmix_info_t info[], size_t ninfo);
-+
-+
-+/* CONSTANTS */
-+#define PMIX_ERR_SILENT                             -2
-+#define PMIX_ERR_DEBUGGER_RELEASE                   -3
-+#define PMIX_ERR_PROC_ABORTED                       -7
-+#define PMIX_ERR_PROC_REQUESTED_ABORT               -8
-+#define PMIX_ERR_PROC_ABORTING                      -9
-+#define PMIX_ERR_SERVER_FAILED_REQUEST              -10
-+#define PMIX_EXISTS                                 -11
-+#define PMIX_ERR_HANDSHAKE_FAILED                   -13
-+#define PMIX_ERR_READY_FOR_HANDSHAKE                -14
-+#define PMIX_ERR_PROC_ENTRY_NOT_FOUND               -17
-+#define PMIX_ERR_PACK_MISMATCH                      -22
-+#define PMIX_ERR_IN_ERRNO                           -26
-+#define PMIX_ERR_DATA_VALUE_NOT_FOUND               -30
-+#define PMIX_ERR_INVALID_ARG                        -33
-+#define PMIX_ERR_INVALID_KEY                        -34
-+#define PMIX_ERR_INVALID_KEY_LENGTH                 -35
-+#define PMIX_ERR_INVALID_VAL                        -36
-+#define PMIX_ERR_INVALID_VAL_LENGTH                 -37
-+#define PMIX_ERR_INVALID_LENGTH                     -38
-+#define PMIX_ERR_INVALID_NUM_ARGS                   -39
-+#define PMIX_ERR_INVALID_ARGS                       -40
-+#define PMIX_ERR_INVALID_NUM_PARSED                 -41
-+#define PMIX_ERR_INVALID_KEYVALP                    -42
-+#define PMIX_ERR_INVALID_SIZE                       -43
-+#define PMIX_ERR_INVALID_NAMESPACE                  -44
-+#define PMIX_ERR_SERVER_NOT_AVAIL                   -45
-+#define PMIX_ERR_NOT_IMPLEMENTED                    -48
-+#define PMIX_ERR_LOST_CONNECTION_TO_SERVER          -101
-+#define PMIX_ERR_LOST_PEER_CONNECTION               -102
-+#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT          -103
-+#define PMIX_NOTIFY_ALLOC_COMPLETE                  -105
-+#define PMIX_ERR_INVALID_TERMINATION                -112
-+#define PMIX_ERR_JOB_TERMINATED                     -145    // DEPRECATED NAME - non-error termination
-+#define PMIX_GDS_ACTION_COMPLETE                    -148
-+#define PMIX_PROC_HAS_CONNECTED                     -149
-+#define PMIX_CONNECT_REQUESTED                      -150
-+#define PMIX_ERR_NODE_DOWN                          -231
-+#define PMIX_ERR_NODE_OFFLINE                       -232
-+#define PMIX_ERR_SYS_OTHER                          -330
-+
-+/* ATTRIBUTES */
-+#define PMIX_TOPOLOGY                       "pmix.topo"             // (hwloc_topology_t) pointer to the PMIx client's internal
-+                                                                    //         topology object
-+#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job assigned to this debugger to be debugged. Note
-+                                                                    //         that id's, pids, and other info on the procs is available
-+                                                                    //         via a query for the nspace's local or global proctable
-+#define PMIX_RECONNECT_SERVER               "pmix.cnct.recon"       // (bool) tool is requesting to change server connections
-+#define PMIX_ALLOC_NETWORK                  "pmix.alloc.net"        // (pmix_data_array_t*) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_ID               "pmix.alloc.netid"      // (char*) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_QOS              "pmix.alloc.netqos"     // (char*) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_TYPE             "pmix.alloc.nettype"    // (char*) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_PLANE            "pmix.alloc.netplane"   // (char*) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_ENDPTS           "pmix.alloc.endpts"     // (size_t) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_ENDPTS_NODE      "pmix.alloc.endpts.nd"  // (size_t) ***** DEPRECATED *****
-+#define PMIX_ALLOC_NETWORK_SEC_KEY          "pmix.alloc.nsec"       // (pmix_byte_object_t) ***** DEPRECATED *****
-+#define PMIX_PROC_DATA                      "pmix.pdata"            // (pmix_data_array_t*) starts with rank, then contains more data
-+#define PMIX_LOCALITY                       "pmix.loc"              // (uint16_t) relative locality of two procs
-+
-+#define PMIX_LOCAL_TOPO                     "pmix.ltopo"            // (char*) xml-representation of local node topology
-+#define PMIX_TOPOLOGY_XML                   "pmix.topo.xml"         // (char*) XML-based description of topology
-+#define PMIX_TOPOLOGY_FILE                  "pmix.topo.file"        // (char*) full path to file containing XML topology description
-+#define PMIX_TOPOLOGY_SIGNATURE             "pmix.toposig"          // (char*) topology signature string
-+#define PMIX_HWLOC_SHMEM_ADDR               "pmix.hwlocaddr"        // (size_t) address of HWLOC shared memory segment
-+#define PMIX_HWLOC_SHMEM_SIZE               "pmix.hwlocsize"        // (size_t) size of HWLOC shared memory segment
-+#define PMIX_HWLOC_SHMEM_FILE               "pmix.hwlocfile"        // (char*) path to HWLOC shared memory file
-+#define PMIX_HWLOC_XML_V1                   "pmix.hwlocxml1"        // (char*) XML representation of local topology using HWLOC v1.x format
-+#define PMIX_HWLOC_XML_V2                   "pmix.hwlocxml2"        // (char*) XML representation of local topology using HWLOC v2.x format
-+#define PMIX_HWLOC_SHARE_TOPO               "pmix.hwlocsh"          // (bool) Share the HWLOC topology via shared memory
-+#define PMIX_HWLOC_HOLE_KIND                "pmix.hwlocholek"       // (char*) Kind of VM "hole" HWLOC should use for shared memory
-+#define PMIX_DSTPATH                        "pmix.dstpath"          // (char*) path to dstore files
-+#define PMIX_COLLECTIVE_ALGO                "pmix.calgo"            // (char*) comma-delimited list of algorithms to use for collective
-+#define PMIX_COLLECTIVE_ALGO_REQD           "pmix.calreqd"          // (bool) if true, indicates that the requested choice of algo is mandatory
-+#define PMIX_PROC_BLOB                      "pmix.pblob"            // (pmix_byte_object_t) packed blob of process data
-+#define PMIX_MAP_BLOB                       "pmix.mblob"            // (pmix_byte_object_t) packed blob of process location
-+#define PMIX_MAPPER                         "pmix.mapper"           // (char*) mapper to use for placing spawned procs
-+#define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) spawned procs will not call PMIx_Init
-+#define PMIX_PROC_URI                       "pmix.puri"             // (char*) URI containing contact info for proc
-+#define PMIX_ARCH                           "pmix.arch"             // (uint32_t) datatype architecture flag
-+
-+#if defined(c_plusplus) || defined(__cplusplus)
-+}
-+#endif
-+
-+#endif
-Index: pmix-3.2.0/src/util/error.h
-===================================================================
---- pmix-3.2.0.orig/src/util/error.h
-+++ pmix-3.2.0/src/util/error.h
-@@ -28,6 +28,11 @@
- 
-  BEGIN_C_DECLS
- 
-+/* define a starting point for PMIx internal error codes
-+ * that are never exposed outside the library */
-+#define PMIX_INTERNAL_ERR_BASE                      -1330
-+
-+
- /* internal error codes - never exposed outside of the library */
- #define PMIX_ERR_NOT_AVAILABLE                          (PMIX_INTERNAL_ERR_BASE - 28)
- #define PMIX_ERR_FATAL                                  (PMIX_INTERNAL_ERR_BASE - 29)
-Index: pmix-3.2.0/include/Makefile.am
-===================================================================
---- pmix-3.2.0.orig/include/Makefile.am
-+++ pmix-3.2.0/include/Makefile.am
-@@ -15,7 +15,8 @@ include_HEADERS = \
-         pmix.h \
-         pmix_server.h \
-         pmix_tool.h \
--        pmix_extend.h
-+        pmix_extend.h \
-+	pmix_deprecated.h
- 
- if WANT_PMI_BACKWARD
- include_HEADERS += \
-@@ -25,6 +26,6 @@ endif
- 
- nodist_include_HEADERS = \
-     pmix_common.h \
--    pmix_version.h
-+    pmix_version.h 
- 
- endif
diff -Nru pmix-3.2.2~rc1/debian/patches/python3.patch pmix-4.0.0/debian/patches/python3.patch
--- pmix-3.2.2~rc1/debian/patches/python3.patch	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/debian/patches/python3.patch	2021-01-20 17:30:05.000000000 +0000
@@ -0,0 +1,15 @@
+Description: Change refs from python to python3 
+Author: Alastair McKinstry <mckinstry@debian.org>
+Last-Updated: 2021-04-05
+Forwarded: yes
+
+Index: pmix/config/pmix_check_cython.py
+===================================================================
+--- pmix.orig/config/pmix_check_cython.py
++++ pmix/config/pmix_check_cython.py
+@@ -1,4 +1,4 @@
+-#!/usr/bin/env python
++#!/usr/bin/python3
+ from __future__ import print_function
+ 
+ from Cython.Build import cythonize
diff -Nru pmix-3.2.2~rc1/debian/patches/series pmix-4.0.0/debian/patches/series
--- pmix-3.2.2~rc1/debian/patches/series	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/patches/series	2021-01-20 17:30:05.000000000 +0000
@@ -1,2 +1,2 @@
-hurd-fix.patch
-missing.patch
+# hurd-fix.patch
+# python3.patch
diff -Nru pmix-3.2.2~rc1/debian/python3-pmix.install pmix-4.0.0/debian/python3-pmix.install
--- pmix-3.2.2~rc1/debian/python3-pmix.install	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/debian/python3-pmix.install	2021-01-20 17:30:05.000000000 +0000
@@ -0,0 +1 @@
+usr/lib/python3*
diff -Nru pmix-3.2.2~rc1/debian/rules pmix-4.0.0/debian/rules
--- pmix-3.2.2~rc1/debian/rules	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/rules	2021-01-22 20:23:31.000000000 +0000
@@ -18,9 +18,21 @@
   export DEB_LDFLAGS_MAINT_APPEND += -Wl,--no-as-needed -latomic -Wl,--as-needed
 endif
 
+export LD_LIBRARY_PATH:=$(CURDIR)/debian/tmp/$(LIBDIR):$(LD_LIBRARY_PATH)
+
 %:
 	dh $@ 
 
+BUILD_FLAGS:= \
+	$(PSM2) \
+	--sysconfdir=$(LIBDIR)/pmix2/share \
+	--datarootdir=$(LIBDIR)/pmix2/share \
+	--with-zlib=/usr \
+	--with-jansson=/usr \
+	--with-hwloc=/usr \
+	--enable-python-bindings \
+	$(if $(filter $(DEB_HOST_ARCH), i386),--disable-man-pages)
+
 override_dh_auto_clean:
 	dh_auto_clean
 	rm -f $(patsubst %, debian/%, ${AUTOGENERATED})
@@ -33,12 +45,20 @@
                 done
 
 override_dh_auto_configure:
-	dh_auto_configure --builddirectory=debian/static-build -- $(PSM2) \
-		--enable-static --sysconfdir=$(LIBDIR)/pmix2/share --disable-man-pages
-	dh_auto_configure --builddirectory=debian/shared-build -- $(PSM2) \
-		--enable-shared --sysconfdir=$(LIBDIR)/pmix2/share --disable-man-pages
+	# Hack to fix quilt issue in 4.0.0
+	patch -p1 < debian/patches/python3.patch
+	dh_auto_configure --builddirectory=debian/static-build -- $(BUILD_FLAGS) \
+		--enable-static 
+	dh_auto_configure --builddirectory=debian/shared-build -- $(BUILD_FLAGS) \
+		--enable-shared
 
 override_dh_auto_build:
+	# Hack - build system doesn't work well with builddirs
+	cp include/*.h debian/static-build/include
+	cp include/*.h debian/shared-build/include
+	cp debian/static-build/include/pmix_version.h include
+	cp bindings/python/pmix.* debian/static-build/bindings/python
+	cp bindings/python/pmix.* debian/shared-build/bindings/python
 	dh_auto_build  --builddirectory=debian/static-build
 	dh_auto_build  --builddirectory=debian/shared-build
 
diff -Nru pmix-3.2.2~rc1/debian/watch pmix-4.0.0/debian/watch
--- pmix-3.2.2~rc1/debian/watch	2020-12-05 15:21:44.000000000 +0000
+++ pmix-4.0.0/debian/watch	2021-01-20 17:30:05.000000000 +0000
@@ -1,3 +1,3 @@
 version=3
-opts="uversionmangle=s/rc/~rc/, filenamemangle=s/.+\/(\d\S+)\.tar\.gz/$1\.tar\.gz/" \
-  https://github.com/pmix/pmix/tags .*/v?(\d\S+)\.tar\.gz
+opts="uversionmangle=s/-rc/~rc/, filenamemangle=s/.+\/(\d\S+)\.tar\.gz/$1\.tar\.gz/" \
+  https://github.com/openpmix/openpmix/tags .*/v?(\d\S+)\.tar\.gz
diff -Nru pmix-3.2.2~rc1/examples/asyncgroup.c pmix-4.0.0/examples/asyncgroup.c
--- pmix-3.2.2~rc1/examples/asyncgroup.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/examples/asyncgroup.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <pthread.h>
+
+#include <pmix.h>
+
+typedef struct {
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    volatile bool active;
+    pmix_status_t status;
+} mylock_t;
+
+#define DEBUG_CONSTRUCT_LOCK(l)                     \
+    do {                                            \
+        pthread_mutex_init(&(l)->mutex, NULL);      \
+        pthread_cond_init(&(l)->cond, NULL);        \
+        (l)->active = true;                         \
+        (l)->status = PMIX_SUCCESS;                 \
+    } while(0)
+
+#define DEBUG_DESTRUCT_LOCK(l)              \
+    do {                                    \
+        pthread_mutex_destroy(&(l)->mutex); \
+        pthread_cond_destroy(&(l)->cond);   \
+    } while(0)
+
+#define DEBUG_WAIT_THREAD(lck)                                      \
+    do {                                                            \
+        pthread_mutex_lock(&(lck)->mutex);                          \
+        while ((lck)->active) {                                     \
+            pthread_cond_wait(&(lck)->cond, &(lck)->mutex);         \
+        }                                                           \
+        pthread_mutex_unlock(&(lck)->mutex);                        \
+    } while(0)
+
+#define DEBUG_WAKEUP_THREAD(lck)                        \
+    do {                                                \
+        pthread_mutex_lock(&(lck)->mutex);              \
+        (lck)->active = false;                          \
+        pthread_cond_broadcast(&(lck)->cond);           \
+        pthread_mutex_unlock(&(lck)->mutex);            \
+    } while(0)
+
+
+static pmix_proc_t myproc;
+static mylock_t invitedlock;
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    fprintf(stderr, "Client %s:%d NOTIFIED with status %d\n", myproc.nspace, myproc.rank, status);
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+static void op_callbk(pmix_status_t status,
+                      void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+static void errhandler_reg_callbk(pmix_status_t status,
+                                  size_t errhandler_ref,
+                                  void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+static void grpcomplete(pmix_status_t status,
+                        pmix_info_t *info, size_t ninfo,
+                        void *cbdata,
+                        pmix_release_cbfunc_t release_fn,
+                        void *release_cbdata)
+{
+    fprintf(stderr, "%s:%d GRPCOMPLETE\n", myproc.nspace, myproc.rank);
+    DEBUG_WAKEUP_THREAD(&invitedlock);
+}
+
+static void invitefn(size_t evhdlr_registration_id,
+                     pmix_status_t status,
+                     const pmix_proc_t *source,
+                     pmix_info_t info[], size_t ninfo,
+                     pmix_info_t results[], size_t nresults,
+                     pmix_event_notification_cbfunc_fn_t cbfunc,
+                     void *cbdata)
+{
+    size_t n;
+    char *grp = NULL;
+    pmix_status_t rc;
+
+    /* if I am the leader, I can ignore this event */
+    if (PMIX_CHECK_PROCID(source, &myproc)) {
+        fprintf(stderr, "%s:%d INVITED, BUT LEADER\n", myproc.nspace, myproc.rank);
+        /* mark the event chain as complete */
+        if (NULL != cbfunc) {
+            cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+        }
+        return;
+    }
+
+    /* search for grp id */
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_CHECK_KEY(&info[n], PMIX_GROUP_ID)) {
+            grp = info[n].value.data.string;
+            break;
+        }
+    }
+    fprintf(stderr, "Client %s:%d INVITED by source %s:%d\n",
+            myproc.nspace, myproc.rank,
+            source->nspace, source->rank);
+    invitedlock.status = status;
+    fprintf(stderr, "%s:%d ACCEPTING INVITE\n", myproc.nspace, myproc.rank);
+    rc = PMIx_Group_join_nb(grp, source, PMIX_GROUP_ACCEPT, NULL, 0, grpcomplete, NULL);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "%s:%d Error in Group_join_nb: %sn", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    }
+
+    /* mark the event chain as complete */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+
+int main(int argc, char **argv)
+{
+    int rc;
+    pmix_value_t *val = NULL;
+    pmix_proc_t proc, *procs;
+    uint32_t nprocs;
+    mylock_t lock;
+    pmix_status_t code;
+    pmix_info_t *results;
+    size_t nresults;
+
+    /* init us */
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(0);
+    }
+    fprintf(stderr, "[%d] Client ns %s rank %d: Running\n", (int)getpid(), myproc.nspace, myproc.rank);
+
+    DEBUG_CONSTRUCT_LOCK(&invitedlock);
+
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
+
+    /* get our universe size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    nprocs = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+    if (nprocs < 4) {
+        if (0 == myproc.rank) {
+            fprintf(stderr, "This example requires a minimum of 4 processes\n");
+        }
+        goto done;
+    }
+    fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs);
+
+    /* register our default errhandler */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, errhandler_reg_callbk, (void*)&lock);
+    DEBUG_WAIT_THREAD(&lock);
+    rc = lock.status;
+    DEBUG_DESTRUCT_LOCK(&lock);
+    if (PMIX_SUCCESS != rc) {
+        goto done;
+    }
+
+    /* we need to register handlers for invitations */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    code = PMIX_GROUP_INVITED;
+    PMIx_Register_event_handler(&code, 1, NULL, 0,
+                                invitefn, errhandler_reg_callbk, (void*)&lock);
+    DEBUG_WAIT_THREAD(&lock);
+    rc = lock.status;
+    DEBUG_DESTRUCT_LOCK(&lock);
+    if (PMIX_SUCCESS != rc) {
+        goto done;
+    }
+
+    /* call fence to sync */
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc);
+        goto done;
+    }
+
+    /* rank=0 constructs a new group */
+    if (0 == myproc.rank) {
+        fprintf(stderr, "%d executing Group_invite\n", myproc.rank);
+        nprocs = 3;
+        PMIX_PROC_CREATE(procs, nprocs);
+        PMIX_PROC_LOAD(&procs[0], myproc.nspace, 0);
+        PMIX_PROC_LOAD(&procs[1], myproc.nspace, 2);
+        PMIX_PROC_LOAD(&procs[2], myproc.nspace, 3);
+        rc = PMIx_Group_invite("ourgroup", procs, nprocs, NULL, 0, &results, &nresults);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Group_invite failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+        PMIX_PROC_FREE(procs, nprocs);
+        fprintf(stderr, "%s:%d Execute fence across group\n", myproc.nspace, myproc.rank);
+        PMIX_PROC_LOAD(&proc, "ourgroup", PMIX_RANK_WILDCARD);
+        rc = PMIx_Fence(&proc, 1, NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Fence across group failed: %d\n", myproc.nspace, myproc.rank, rc);
+            goto done;
+        }
+        fprintf(stderr, "%d executing Group_destruct\n", myproc.rank);
+        rc = PMIx_Group_destruct("ourgroup", NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Group_destruct failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+    } else if (2 == myproc.rank || 3 == myproc.rank) {
+        /* wait to be invited */
+        fprintf(stderr, "%s:%d waiting to be invited\n", myproc.nspace, myproc.rank);
+        DEBUG_WAIT_THREAD(&invitedlock);
+        DEBUG_DESTRUCT_LOCK(&invitedlock);
+        fprintf(stderr, "%s:%d Execute fence across group\n", myproc.nspace, myproc.rank);
+        PMIX_PROC_LOAD(&proc, "ourgroup", PMIX_RANK_WILDCARD);
+        rc = PMIx_Fence(&proc, 1, NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Fence across group failed: %d\n", myproc.nspace, myproc.rank, rc);
+            goto done;
+        }
+        fprintf(stderr, "%d executing Group_destruct\n", myproc.rank);
+        rc = PMIx_Group_destruct("ourgroup", NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Group_destruct failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+    }
+
+    /* call fence to sync */
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+
+ done:
+    /* finalize us */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    PMIx_Deregister_event_handler(1, op_callbk, &lock);
+    DEBUG_WAIT_THREAD(&lock);
+    DEBUG_DESTRUCT_LOCK(&lock);
+
+    fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fprintf(stderr, "%s:%d COMPLETE\n", myproc.nspace, myproc.rank);
+    fflush(stderr);
+    return(0);
+}
diff -Nru pmix-3.2.2~rc1/examples/client.c pmix-4.0.0/examples/client.c
--- pmix-3.2.2~rc1/examples/client.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/examples/client.c	2021-01-02 08:56:17.000000000 +0000
@@ -135,6 +135,7 @@
     myrel_t myrel;
     pmix_status_t dbg = PMIX_ERR_DEBUGGER_RELEASE;
     pid_t pid;
+    pmix_topology_t mytopo;
 
     pid = getpid();
     fprintf(stderr, "Client %lu: Running\n", (unsigned long)pid);
@@ -201,6 +202,15 @@
         PMIX_VALUE_RELEASE(val);
     }
 
+    /* check for local topology info */
+    PMIX_TOPOLOGY_CONSTRUCT(&mytopo);
+    if (PMIX_SUCCESS != (rc = PMIx_Load_topology(&mytopo))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Load_topology failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    fprintf(stderr, "Client %s:%d topology loaded\n", myproc.nspace, myproc.rank);
+
+
     /* get our universe size */
     if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %d\n", myproc.nspace, myproc.rank, rc);
diff -Nru pmix-3.2.2~rc1/examples/debugger.c pmix-4.0.0/examples/debugger.c
--- pmix-3.2.2~rc1/examples/debugger.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/examples/debugger.c	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -204,13 +204,12 @@
     debugger[0].cwd = strdup(cwd);
     /* provide directives so the daemons go where we want, and
      * let the RM know these are debugger daemons */
-    dninfo = 5;
+    dninfo = 4;
     PMIX_INFO_CREATE(dinfo, dninfo);
     PMIX_INFO_LOAD(&dinfo[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING);  // instruct the RM to launch one copy of the executable on each node
     PMIX_INFO_LOAD(&dinfo[1], PMIX_DEBUGGER_DAEMONS, NULL, PMIX_BOOL); // these are debugger daemons
     PMIX_INFO_LOAD(&dinfo[2], PMIX_DEBUG_JOB, appspace, PMIX_STRING); // the nspace being debugged
     PMIX_INFO_LOAD(&dinfo[3], PMIX_NOTIFY_COMPLETION, NULL, PMIX_BOOL); // notify us when the debugger job completes
-    PMIX_INFO_LOAD(&dinfo[4], PMIX_DEBUG_WAITING_FOR_NOTIFY, NULL, PMIX_BOOL);  // tell the daemon that the proc is waiting to be released
     /* spawn the daemons */
     fprintf(stderr, "Debugger: spawning %s\n", debugger[0].cmd);
     if (PMIX_SUCCESS != (rc = PMIx_Spawn(dinfo, dninfo, debugger, 1, dspace))) {
diff -Nru pmix-3.2.2~rc1/examples/group.c pmix-4.0.0/examples/group.c
--- pmix-3.2.2~rc1/examples/group.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/examples/group.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <pthread.h>
+
+#include <pmix.h>
+
+typedef struct {
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    volatile bool active;
+    pmix_status_t status;
+} mylock_t;
+
+#define DEBUG_CONSTRUCT_LOCK(l)                     \
+    do {                                            \
+        pthread_mutex_init(&(l)->mutex, NULL);      \
+        pthread_cond_init(&(l)->cond, NULL);        \
+        (l)->active = true;                         \
+        (l)->status = PMIX_SUCCESS;                 \
+    } while(0)
+
+#define DEBUG_DESTRUCT_LOCK(l)              \
+    do {                                    \
+        pthread_mutex_destroy(&(l)->mutex); \
+        pthread_cond_destroy(&(l)->cond);   \
+    } while(0)
+
+#define DEBUG_WAIT_THREAD(lck)                                      \
+    do {                                                            \
+        pthread_mutex_lock(&(lck)->mutex);                          \
+        while ((lck)->active) {                                     \
+            pthread_cond_wait(&(lck)->cond, &(lck)->mutex);         \
+        }                                                           \
+        pthread_mutex_unlock(&(lck)->mutex);                        \
+    } while(0)
+
+#define DEBUG_WAKEUP_THREAD(lck)                        \
+    do {                                                \
+        pthread_mutex_lock(&(lck)->mutex);              \
+        (lck)->active = false;                          \
+        pthread_cond_broadcast(&(lck)->cond);           \
+        pthread_mutex_unlock(&(lck)->mutex);            \
+    } while(0)
+
+
+static pmix_proc_t myproc;
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    fprintf(stderr, "Client %s:%d NOTIFIED with status %d\n", myproc.nspace, myproc.rank, status);
+}
+
+static void op_callbk(pmix_status_t status,
+                      void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    fprintf(stderr, "Client %s:%d OP CALLBACK CALLED WITH STATUS %d\n", myproc.nspace, myproc.rank, status);
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+static void errhandler_reg_callbk(pmix_status_t status,
+                                  size_t errhandler_ref,
+                                  void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    fprintf(stderr, "Client %s:%d ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu\n",
+               myproc.nspace, myproc.rank, status, (unsigned long)errhandler_ref);
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+int main(int argc, char **argv)
+{
+    int rc;
+    pmix_value_t *val = NULL;
+    pmix_proc_t proc, *procs;
+    uint32_t nprocs;
+    mylock_t lock;
+    pmix_info_t *results, info;
+    size_t nresults, cid;
+
+    /* init us */
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(0);
+    }
+    fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
+
+    PMIX_PROC_CONSTRUCT(&proc);
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
+
+    /* get our universe size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get universe size failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    nprocs = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+    if (nprocs < 4) {
+        if (0 == myproc.rank) {
+            fprintf(stderr, "This example requires a minimum of 4 processes\n");
+        }
+        goto done;
+    }
+    fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs);
+
+    /* register our default errhandler */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, errhandler_reg_callbk, (void*)&lock);
+    DEBUG_WAIT_THREAD(&lock);
+    rc = lock.status;
+    DEBUG_DESTRUCT_LOCK(&lock);
+    if (PMIX_SUCCESS != rc) {
+        goto done;
+    }
+
+    /* call fence to sync */
+    PMIX_PROC_CONSTRUCT(&proc);
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc);
+        goto done;
+    }
+
+    /* rank=0,2,3 construct a new group */
+    if (0 == myproc.rank || 2 == myproc.rank || 3 == myproc.rank) {
+        fprintf(stderr, "%d executing Group_construct\n", myproc.rank);
+        nprocs = 3;
+        PMIX_PROC_CREATE(procs, nprocs);
+        PMIX_PROC_LOAD(&procs[0], myproc.nspace, 0);
+        PMIX_PROC_LOAD(&procs[1], myproc.nspace, 2);
+        PMIX_PROC_LOAD(&procs[2], myproc.nspace, 3);
+        PMIX_INFO_LOAD(&info, PMIX_GROUP_ASSIGN_CONTEXT_ID, NULL, PMIX_BOOL);
+        rc = PMIx_Group_construct("ourgroup", procs, nprocs, &info, 1, &results, &nresults);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Group_construct failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+        /* we should have a single results object */
+        if (NULL != results) {
+            cid = 0;
+            PMIX_VALUE_GET_NUMBER(rc, &results[0].value, cid, size_t);
+            fprintf(stderr, "%d Group construct complete with status %s KEY %s CID %d\n",
+                    myproc.rank, PMIx_Error_string(rc), results[0].key, (int)cid);
+        } else {
+            fprintf(stderr, "%d Group construct complete, but no CID returned\n", myproc.rank);
+        }
+        PMIX_PROC_FREE(procs, nprocs);
+        fprintf(stderr, "%d executing Group_destruct\n", myproc.rank);
+        rc = PMIx_Group_destruct("ourgroup", NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Group_destruct failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+            goto done;
+        }
+    }
+
+ done:
+    /* finalize us */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    PMIx_Deregister_event_handler(1, op_callbk, &lock);
+    DEBUG_WAIT_THREAD(&lock);
+    DEBUG_DESTRUCT_LOCK(&lock);
+
+    fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fprintf(stderr, "%s:%d COMPLETE\n", myproc.nspace, myproc.rank);
+    fflush(stderr);
+    return(0);
+}
diff -Nru pmix-3.2.2~rc1/examples/Makefile.am pmix-4.0.0/examples/Makefile.am
--- pmix-3.2.2~rc1/examples/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/examples/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -22,7 +22,7 @@
 
 AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix
 
-noinst_PROGRAMS = client client2 dmodex dynamic fault pub pubi tool debugger debuggerd alloc jctrl
+noinst_PROGRAMS = client client2 dmodex dynamic fault pub pubi tool debugger debuggerd alloc jctrl group asyncgroup
 if !WANT_HIDDEN
 # these examples use internal symbols
 # use --disable-visibility
@@ -77,6 +77,14 @@
 tool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
 tool_LDADD = $(top_builddir)/src/libpmix.la
 
+group_SOURCES = group.c examples.h
+group_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+group_LDADD = $(top_builddir)/src/libpmix.la
+
+asyncgroup_SOURCES = asyncgroup.c examples.h
+asyncgroup_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+asyncgroup_LDADD = $(top_builddir)/src/libpmix.la
+
 if !WANT_HIDDEN
 server_SOURCES = server.c examples.h
 server_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
diff -Nru pmix-3.2.2~rc1/examples/tool.c pmix-4.0.0/examples/tool.c
--- pmix-3.2.2~rc1/examples/tool.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/examples/tool.c	2021-01-02 08:56:17.000000000 +0000
@@ -15,6 +15,8 @@
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2019      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +33,8 @@
 #include <pmix_tool.h>
 #include "examples.h"
 
+static pmix_proc_t myproc = {"UNDEF", PMIX_RANK_UNDEF};
+
 static void cbfunc(pmix_status_t status,
                    pmix_info_t *info, size_t ninfo,
                    void *cbdata,
@@ -61,10 +65,94 @@
     DEBUG_WAKEUP_THREAD(&mq->lock);
 }
 
+/* this is an event notification function that we explicitly request
+ * be called when the PMIX_ERR_JOB_TERMINATED notification is issued.
+ * We could catch it in the general event notification function and test
+ * the status to see if it was "job terminated", but it often is simpler
+ * to declare a use-specific notification callback point. In this case,
+ * we are asking to know whenever a job terminates, and we will then
+ * know we can exit */
+static void release_fn(size_t evhdlr_registration_id,
+                       pmix_status_t status,
+                       const pmix_proc_t *source,
+                       pmix_info_t info[], size_t ninfo,
+                       pmix_info_t results[], size_t nresults,
+                       pmix_event_notification_cbfunc_fn_t cbfunc,
+                       void *cbdata)
+{
+    myrel_t *lock;
+    bool found;
+    int exit_code = 0;
+    size_t n;
+    pmix_proc_t *affected = NULL;
+
+    /* find the return object */
+    lock = NULL;
+    found = false;
+    for (n=0; n < ninfo; n++) {
+        if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
+            lock = (myrel_t*)info[n].value.data.ptr;
+            /* not every RM will provide an exit code, but check if one was given */
+        } else if (0 == strncmp(info[n].key, PMIX_EXIT_CODE, PMIX_MAX_KEYLEN)) {
+            exit_code = info[n].value.data.integer;
+            found = true;
+        } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) {
+            affected = info[n].value.data.proc;
+        }
+    }
+    /* if the object wasn't returned, then that is an error */
+    if (NULL == lock) {
+        fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n");
+        /* let the event handler progress */
+        if (NULL != cbfunc) {
+            cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+        }
+        return;
+    }
+
+    fprintf(stderr, "TOOL NOTIFIED THAT ALL CHILD PROCS %s TERMINATED \n",
+            (NULL == affected) ? "NULL" : affected->nspace);
+    if (found) {
+        if (!lock->exit_code_given) {
+            lock->exit_code = exit_code;
+            lock->exit_code_given = true;
+        }
+    }
+    DEBUG_WAKEUP_THREAD(&lock->lock);
+
+    /* tell the event handler state machine that we are the last step */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+    return;
+}
+
+/* event handler registration is done asynchronously because it
+ * may involve the PMIx server registering with the host RM for
+ * external events. So we provide a callback function that returns
+ * the status of the request (success or an error), plus a numerical index
+ * to the registered event. The index is used later on to deregister
+ * an event handler - if we don't explicitly deregister it, then the
+ * PMIx server will do so when it see us exit */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    if (PMIX_SUCCESS != status) {
+        fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+    }
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+
 int main(int argc, char **argv)
 {
     pmix_status_t rc;
-    pmix_proc_t myproc;
+    pmix_proc_t proc;
     pmix_query_t *query;
     size_t nq, ninfo = 0, n, m;
     myquery_data_t mydata;
@@ -75,6 +163,12 @@
     pmix_data_array_t *darray, *dptr;
     bool geturi = false;
     char hostname[1024];
+    char *spawn = NULL;
+    pmix_app_t app;
+    pmix_nspace_t child;
+    pmix_status_t code = PMIX_ERR_JOB_TERMINATED;
+    myrel_t myrel;
+    mylock_t mylock;
 
     gethostname(hostname, 1024);
     for (n=1; n < (size_t)argc; n++) {
@@ -84,24 +178,43 @@
                 exit(1);
             }
             server_uri = argv[n+1];
+            ++n;
+            ++ninfo;
         } else if (0 == strcmp("-nspace", argv[n]) || 0 == strcmp("--nspace", argv[n])) {
             if (NULL == argv[n+1]) {
                 fprintf(stderr, "Must provide nspace argument to %s option\n", argv[n]);
                 exit(1);
             }
             nspace = argv[n+1];
+            ++n;
         } else if (0 == strcmp("-uri", argv[n]) || 0 == strcmp("--uri", argv[n])) {
             /* retrieve the PMIx server's uri from the indicated node */
             nodename = argv[n+1];
             geturi = true;
+            ++n;
+        } else if (0 == strcmp("-spawn", argv[n]) || 0 == strcmp("--spawn", argv[n])) {
+            if (NULL == argv[n+1]) {
+                fprintf(stderr, "Must provide executable argument to %s option\n", argv[n]);
+                exit(1);
+            }
+            spawn = argv[n+1];
+            ++n;
+            ninfo += 2;
         }
     }
 
+    PMIX_INFO_CREATE(info, ninfo);
+    n = 0;
     if (NULL != server_uri) {
-        ninfo = 1;
-        PMIX_INFO_CREATE(info, ninfo);
-        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_URI, server_uri, PMIX_STRING);
+        PMIX_INFO_LOAD(&info[n], PMIX_SERVER_URI, server_uri, PMIX_STRING);
         fprintf(stderr, "Connecting to %s\n", server_uri);
+        ++n;
+    }
+    if (NULL != spawn) {
+        PMIX_INFO_LOAD(&info[n], PMIX_TOOL_CONNECT_OPTIONAL, NULL, PMIX_BOOL);
+        ++n;
+        PMIX_INFO_LOAD(&info[n], PMIX_LAUNCHER, NULL, PMIX_BOOL);
+        ++n;
     }
 
     /* init us */
@@ -144,6 +257,39 @@
         goto done;
     }
 
+    /* if we want to spawn a proc, then do so */
+    if (NULL != spawn) {
+        DEBUG_CONSTRUCT_LOCK(&myrel.lock);
+        /* setup notification so we know when the child has terminated */
+        PMIX_INFO_CREATE(info, 2);
+        PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER);
+        /* only call me back when this specific job terminates - the
+         * children will have our nspace */
+        PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
+        PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &proc, PMIX_PROC);
+
+        DEBUG_CONSTRUCT_LOCK(&mylock);
+        PMIx_Register_event_handler(&code, 1, info, 2,
+                                    release_fn, evhandler_reg_callbk, (void*)&mylock);
+        DEBUG_WAIT_THREAD(&mylock);
+        rc = mylock.status;
+        DEBUG_DESTRUCT_LOCK(&mylock);
+        PMIX_INFO_FREE(info, 2);
+
+        PMIX_APP_CONSTRUCT(&app);
+        PMIX_ARGV_SPLIT(app.argv, spawn, ' ');
+        app.cmd = strdup(app.argv[0]);
+        app.maxprocs = 1;
+        rc = PMIx_Spawn(NULL, 0, &app, 1, child);
+        if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
+            fprintf(stderr, "Failed to spawn %s\n", PMIx_Error_string(rc));
+            goto done;
+        }
+        /* wait here */
+        DEBUG_WAIT_THREAD(&myrel.lock);
+        goto done;
+    }
+
     if (NULL == nspace) {
         /* query the list of active nspaces */
         nq = 1;
@@ -218,6 +364,6 @@
 
  done:
     /* finalize us */
-    PMIx_Finalize(NULL, 0);
+    PMIx_tool_finalize();
     return(rc);
 }
diff -Nru pmix-3.2.2~rc1/include/Makefile.am pmix-4.0.0/include/Makefile.am
--- pmix-3.2.2~rc1/include/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -15,13 +15,8 @@
         pmix.h \
         pmix_server.h \
         pmix_tool.h \
-        pmix_extend.h
-
-if WANT_PMI_BACKWARD
-include_HEADERS += \
-        pmi.h \
-        pmi2.h
-endif
+        pmix_extend.h \
+        pmix_deprecated.h
 
 nodist_include_HEADERS = \
     pmix_common.h \
diff -Nru pmix-3.2.2~rc1/include/pmi2.h pmix-4.0.0/include/pmi2.h
--- pmix-3.2.2~rc1/include/pmi2.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmi2.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,565 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *  (C) 2007 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-
-#ifndef PMI2_H_INCLUDED
-#define PMI2_H_INCLUDED
-
-/* Structure and constant definitions */
-#include <pmix_common.h>
-
-#define PMI2_MAX_KEYLEN 64
-#define PMI2_MAX_VALLEN 1024
-#define PMI2_MAX_ATTRVALUE 1024
-#define PMI2_ID_NULL -1
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-/*D
-PMI2_CONSTANTS - PMI2 definitions
-
-Error Codes:
-+ PMI2_SUCCESS - operation completed successfully
-. PMI2_FAIL - operation failed
-. PMI2_ERR_NOMEM - input buffer not large enough
-. PMI2_ERR_INIT - PMI not initialized
-. PMI2_ERR_INVALID_ARG - invalid argument
-. PMI2_ERR_INVALID_KEY - invalid key argument
-. PMI2_ERR_INVALID_KEY_LENGTH - invalid key length argument
-. PMI2_ERR_INVALID_VAL - invalid val argument
-. PMI2_ERR_INVALID_VAL_LENGTH - invalid val length argument
-. PMI2_ERR_INVALID_LENGTH - invalid length argument
-. PMI2_ERR_INVALID_NUM_ARGS - invalid number of arguments
-. PMI2_ERR_INVALID_ARGS - invalid args argument
-. PMI2_ERR_INVALID_NUM_PARSED - invalid num_parsed length argument
-. PMI2_ERR_INVALID_KEYVALP - invalid keyvalp argument
-. PMI2_ERR_INVALID_SIZE - invalid size argument
-- PMI2_ERR_OTHER - other unspecified error
-
-D*/
-#define PMI2_SUCCESS                0
-#define PMI2_FAIL                   -1
-#define PMI2_ERR_INIT               1
-#define PMI2_ERR_NOMEM              2
-#define PMI2_ERR_INVALID_ARG        3
-#define PMI2_ERR_INVALID_KEY        4
-#define PMI2_ERR_INVALID_KEY_LENGTH 5
-#define PMI2_ERR_INVALID_VAL        6
-#define PMI2_ERR_INVALID_VAL_LENGTH 7
-#define PMI2_ERR_INVALID_LENGTH     8
-#define PMI2_ERR_INVALID_NUM_ARGS   9
-#define PMI2_ERR_INVALID_ARGS       10
-#define PMI2_ERR_INVALID_NUM_PARSED 11
-#define PMI2_ERR_INVALID_KEYVALP    12
-#define PMI2_ERR_INVALID_SIZE       13
-#define PMI2_ERR_OTHER              14
-
-/* This is here to allow spawn multiple functions to compile.  This
-   needs to be removed once those functions are fixed for pmi2 */
-typedef struct PMI_keyval_t
-{
-    char * key;
-    char * val;
-} PMI_keyval_t;
-
-
-/*@
-  PMI2_Connect_comm_t - connection structure used when connecting to other jobs
-
-  Fields:
-  + read - Read from a connection to the leader of the job to which
-    this process will be connecting. Returns 0 on success or an MPI
-    error code on failure.
-  . write - Write to a connection to the leader of the job to which
-    this process will be connecting. Returns 0 on success or an MPI
-    error code on failure.
-  . ctx - An anonymous pointer to data that may be used by the read
-    and write members.
-  - isMaster - Indicates which process is the "master"; may have the
-    values 1 (is the master), 0 (is not the master), or -1 (neither is
-    designated as the master). The two processes must agree on which
-    process is the master, or both must select -1 (neither is the
-    master).
-
-  Notes:
-  A typical implementation of these functions will use the read and
-  write calls on a pre-established file descriptor (fd) between the
-  two leading processes. This will be needed only if the PMI server
-  cannot access the KVS spaces of another job (this may happen, for
-  example, if each mpiexec creates the KVS spaces for the processes
-  that it manages).
-
-@*/
-typedef struct PMI2_Connect_comm {
-    int (*read)( void *buf, int maxlen, void *ctx );
-    int (*write)( const void *buf, int len, void *ctx );
-    void *ctx;
-    int  isMaster;
-} PMI2_Connect_comm_t;
-
-
-/*@
-  PMI2_Init - initialize the Process Manager Interface
-
-  Output Parameter:
-  + spawned - spawned flag
-  . size - number of processes in the job
-  . rank - rank of this process in the job
-  - appnum - which executable is this on the mpiexec commandline
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  Initialize PMI for this process group. The value of spawned indicates whether
-  this process was created by 'PMI2_Spawn_multiple'.  'spawned' will be non-zero
-  iff this process group has a parent.
-
-@*/
-PMIX_EXPORT int PMI2_Init(int *spawned, int *size, int *rank, int *appnum);
-
-/*@
-  PMI2_Finalize - finalize the Process Manager Interface
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  Finalize PMI for this job.
-
-@*/
-PMIX_EXPORT int PMI2_Finalize(void);
-
-/*@
-  PMI2_Initialized - check if PMI has been initialized
-
-  Return values:
-  Non-zero if PMI2_Initialize has been called successfully, zero otherwise.
-
-@*/
-PMIX_EXPORT int PMI2_Initialized(void);
-
-/*@
-  PMI2_Abort - abort the process group associated with this process
-
-  Input Parameters:
-  + flag - non-zero if all processes in this job should abort, zero otherwise
-  - error_msg - error message to be printed
-
-  Return values:
-  If the abort succeeds this function will not return.  Returns an MPI
-  error code otherwise.
-
-@*/
-PMIX_EXPORT int PMI2_Abort(int flag, const char msg[]);
-
-/*@
-  PMI2_Spawn - spawn a new set of processes
-
-  Input Parameters:
-  + count - count of commands
-  . cmds - array of command strings
-  . argcs - size of argv arrays for each command string
-  . argvs - array of argv arrays for each command string
-  . maxprocs - array of maximum processes to spawn for each command string
-  . info_keyval_sizes - array giving the number of elements in each of the
-    'info_keyval_vectors'
-  . info_keyval_vectors - array of keyval vector arrays
-  . preput_keyval_size - Number of elements in 'preput_keyval_vector'
-  . preput_keyval_vector - array of keyvals to be pre-put in the spawned keyval space
-  - jobIdSize - size of the buffer provided in jobId
-
-  Output Parameter:
-  + jobId - job id of the spawned processes
-  - errors - array of errors for each command
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  This function spawns a set of processes into a new job.  The 'count'
-  field refers to the size of the array parameters - 'cmd', 'argvs', 'maxprocs',
-  'info_keyval_sizes' and 'info_keyval_vectors'.  The 'preput_keyval_size' refers
-  to the size of the 'preput_keyval_vector' array.  The 'preput_keyval_vector'
-  contains keyval pairs that will be put in the keyval space of the newly
-  created job before the processes are started.  The 'maxprocs' array
-  specifies the desired number of processes to create for each 'cmd' string.
-  The actual number of processes may be less than the numbers specified in
-  maxprocs.  The acceptable number of processes spawned may be controlled by
-  ``soft'' keyvals in the info arrays.  The ``soft'' option is specified by
-  mpiexec in the MPI-2 standard.  Environment variables may be passed to the
-  spawned processes through PMI implementation specific 'info_keyval' parameters.
-@*/
-PMIX_EXPORT int PMI2_Job_Spawn(int count, const char * cmds[],
-                               int argcs[], const char ** argvs[],
-                               const int maxprocs[],
-                               const int info_keyval_sizes[],
-                               const PMI_keyval_t *info_keyval_vectors[],
-                               int preput_keyval_size,
-                               const PMI_keyval_t *preput_keyval_vector[],
-                               char jobId[], int jobIdSize,
-                               int errors[]);
-
-/*@
-  PMI2_Job_GetId - get job id of this job
-
-  Input parameters:
-  . jobid_size - size of buffer provided in jobid
-
-  Output parameters:
-  . jobid - the job id of this job
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_Job_GetId(char jobid[], int jobid_size);
-
-/*@
-  PMI2_Job_GetRank - get rank of this job
-  Output parameters:
-  . rank - the rank of this job
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-@*/
-PMIX_EXPORT int PMI2_Job_GetRank(int* rank);
-
-/*@
-  PMI2_Info_GetSize - get the number of processes on the node
-  Output parameters:
-  . rank - the rank of this job
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-@*/
-PMIX_EXPORT int PMI2_Info_GetSize(int* size);
-
-/*@
-  PMI2_Job_Connect - connect to the parallel job with ID jobid
-
-  Input parameters:
-  . jobid - job id of the job to connect to
-
-  Output parameters:
-  . conn - connection structure used to exteblish communication with
-    the remote job
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  This just "registers" the other parallel job as part of a parallel
-  program, and is used in the PMI2_KVS_xxx routines (see below). This
-  is not a collective call and establishes a connection between all
-  processes that are connected to the calling processes (on the one
-  side) and that are connected to the named jobId on the other
-  side. Processes that are already connected may call this routine.
-
-@*/
-PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn);
-
-/*@
-  PMI2_Job_Disconnect - disconnects from the job with ID jobid
-
-  Input parameters:
-  . jobid - job id of the job to connect to
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[]);
-
-/*@
-  PMI2_KVS_Put - put a key/value pair in the keyval space for this job
-
-  Input Parameters:
-  + key - key
-  - value - value
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  If multiple PMI2_KVS_Put calls are made with the same key between
-  calls to PMI2_KVS_Fence, the behavior is undefined. That is, the
-  value returned by PMI2_KVS_Get for that key after the PMI2_KVS_Fence
-  is not defined.
-
-@*/
-PMIX_EXPORT int PMI2_KVS_Put(const char key[], const char value[]);
-/*@
-  PMI2_KVS_Fence - commit all PMI2_KVS_Put calls made before this fence
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  This is a collective call across the job.  It has semantics that are
-  similar to those for MPI_Win_fence and hence is most easily
-  implemented as a barrier across all of the processes in the job.
-  Specifically, all PMI2_KVS_Put operations performed by any process in
-  the same job must be visible to all processes (by using PMI2_KVS_Get)
-  after PMI2_KVS_Fence completes.  However, a PMI implementation could
-  make this a lazy operation by not waiting for all processes to enter
-  their corresponding PMI2_KVS_Fence until some process issues a
-  PMI2_KVS_Get. This might be appropriate for some wide-area
-  implementations.
-
-@*/
-PMIX_EXPORT int PMI2_KVS_Fence(void);
-
-/*@
-  PMI2_KVS_Get - returns the value associated with key in the key-value
-      space associated with the job ID jobid
-
-  Input Parameters:
-  + jobid - the job id identifying the key-value space in which to look
-    for key.  If jobid is NULL, look in the key-value space of this job.
-  . src_pmi_id - the pmi id of the process which put this keypair.  This
-    is just a hint to the server.  PMI2_ID_NULL should be passed if no
-    hint is provided.
-  . key - key
-  - maxvalue - size of the buffer provided in value
-
-  Output Parameters:
-  + value - value associated with key
-  - vallen - length of the returned value, or, if the length is longer
-    than maxvalue, the negative of the required length is returned
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_KVS_Get(const char *jobid, int src_pmi_id, const char key[], char value [], int maxvalue, int *vallen);
-
-/*@
-  PMI2_Info_GetNodeAttr - returns the value of the attribute associated
-      with this node
-
-  Input Parameters:
-  + name - name of the node attribute
-  . valuelen - size of the buffer provided in value
-  - waitfor - if non-zero, the function will not return until the
-    attribute is available
-
-  Output Parameters:
-  + value - value of the attribute
-  - found - non-zero indicates that the attribute was found
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  This provides a way, when combined with PMI2_Info_PutNodeAttr, for
-  processes on the same node to share information without requiring a
-  more general barrier across the entire job.
-
-  If waitfor is non-zero, the function will never return with found
-  set to zero.
-
-  Predefined attributes:
-  + memPoolType - If the process manager allocated a shared memory
-    pool for the MPI processes in this job and on this node, return
-    the type of that pool. Types include sysv, anonmmap and ntshm.
-  . memSYSVid - Return the SYSV memory segment id if the memory pool
-    type is sysv. Returned as a string.
-  . memAnonMMAPfd - Return the FD of the anonymous mmap segment. The
-    FD is returned as a string.
-  - memNTName - Return the name of the Windows NT shared memory
-    segment, file mapping object backed by system paging
-    file.  Returned as a string.
-
-@*/
-PMIX_EXPORT int PMI2_Info_GetNodeAttr(const char name[], char value[], int valuelen, int *found, int waitfor);
-
-/*@
-  PMI2_Info_GetNodeAttrIntArray - returns the value of the attribute associated
-      with this node.  The value must be an array of integers.
-
-  Input Parameters:
-  + name - name of the node attribute
-  - arraylen - number of elements in array
-
-  Output Parameters:
-  + array - value of attribute
-  . outlen - number of elements returned
-  - found - non-zero if attribute was found
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  Notice that, unlike PMI2_Info_GetNodeAttr, this function does not
-  have a waitfor parameter, and will return immediately with found=0
-  if the attribute was not found.
-
-  Predefined array attribute names:
-  + localRanksCount - Return the number of local ranks that will be
-    returned by the key localRanks.
-  . localRanks - Return the ranks in MPI_COMM_WORLD of the processes
-    that are running on this node.
-  - cartCoords - Return the Cartesian coordinates of this process in
-    the underlying network topology. The coordinates are indexed from
-    zero. Value only if the Job attribute for physTopology includes
-    cartesian.
-
-@*/
-PMIX_EXPORT int PMI2_Info_GetNodeAttrIntArray(const char name[], int array[], int arraylen, int *outlen, int *found);
-
-/*@
-  PMI2_Info_PutNodeAttr - stores the value of the named attribute
-  associated with this node
-
-  Input Parameters:
-  + name - name of the node attribute
-  - value - the value of the attribute
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Notes:
-  For example, it might be used to share segment ids with other
-  processes on the same SMP node.
-
-@*/
-PMIX_EXPORT int PMI2_Info_PutNodeAttr(const char name[], const char value[]);
-
-/*@
-  PMI2_Info_GetJobAttr - returns the value of the attribute associated
-  with this job
-
-  Input Parameters:
-  + name - name of the job attribute
-  - valuelen - size of the buffer provided in value
-
-  Output Parameters:
-  + value - value of the attribute
-  - found - non-zero indicates that the attribute was found
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_Info_GetJobAttr(const char name[], char value[], int valuelen, int *found);
-
-/*@
-  PMI2_Info_GetJobAttrIntArray - returns the value of the attribute associated
-      with this job.  The value must be an array of integers.
-
-  Input Parameters:
-  + name - name of the job attribute
-  - arraylen - number of elements in array
-
-  Output Parameters:
-  + array - value of attribute
-  . outlen - number of elements returned
-  - found - non-zero if attribute was found
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-  Predefined array attribute names:
-
-  + universeSize - The size of the "universe" (defined for the MPI
-    attribute MPI_UNIVERSE_SIZE
-
-  . hasNameServ - The value hasNameServ is true if the PMI2 environment
-    supports the name service operations (publish, lookup, and
-    unpublish).
-
-  . physTopology - Return the topology of the underlying network. The
-    valid topology types include cartesian, hierarchical, complete,
-    kautz, hypercube; additional types may be added as necessary. If
-    the type is hierarchical, then additional attributes may be
-    queried to determine the details of the topology. For example, a
-    typical cluster has a hierarchical physical topology, consisting
-    of two levels of complete networks - the switched Ethernet or
-    Infiniband and the SMP nodes. Other systems, such as IBM BlueGene,
-    have one level that is cartesian (and in virtual node mode, have a
-    single-level physical topology).
-
-  . physTopologyLevels - Return a string describing the topology type
-    for each level of the underlying network. Only valid if the
-    physTopology is hierarchical. The value is a comma-separated list
-    of physical topology types (except for hierarchical). The levels
-    are ordered starting at the top, with the network closest to the
-    processes last. The lower level networks may connect only a subset
-    of processes. For example, for a cartesian mesh of SMPs, the value
-    is cartesian,complete. All processes are connected by the
-    cartesian part of this, but for each complete network, only the
-    processes on the same node are connected.
-
-  . cartDims - Return a string of comma-separated values describing
-    the dimensions of the Cartesian topology. This must be consistent
-    with the value of cartCoords that may be returned by
-    PMI2_Info_GetNodeAttrIntArray.
-
-    These job attributes are just a start, but they provide both an
-    example of the sort of external data that is available through the
-    PMI interface and how extensions can be added within the same API
-    and wire protocol. For example, adding more complex network
-    topologies requires only adding new keys, not new routines.
-
-  . isHeterogeneous - The value isHeterogeneous is true if the
-    processes belonging to the job are running on nodes with different
-    underlying data models.
-
-@*/
-PMIX_EXPORT int PMI2_Info_GetJobAttrIntArray(const char name[], int array[], int arraylen, int *outlen, int *found);
-
-/*@
-  PMI2_Nameserv_publish - publish a name
-
-  Input parameters:
-  + service_name - string representing the service being published
-  . info_ptr -
-  - port - string representing the port on which to contact the service
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_Nameserv_publish(const char service_name[], const PMI_keyval_t *info_ptr, const char port[]);
-
-/*@
-  PMI2_Nameserv_lookup - lookup a service by name
-
-  Input parameters:
-  + service_name - string representing the service being published
-  . info_ptr -
-  - portLen - size of buffer provided in port
-
-  Output parameters:
-  . port - string representing the port on which to contact the service
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_Nameserv_lookup(const char service_name[], const PMI_keyval_t *info_ptr,
-                                     char port[], int portLen);
-/*@
-  PMI2_Nameserv_unpublish - unpublish a name
-
-  Input parameters:
-  + service_name - string representing the service being unpublished
-  - info_ptr -
-
-  Return values:
-  Returns 'PMI2_SUCCESS' on success and an PMI error code on failure.
-
-@*/
-PMIX_EXPORT int PMI2_Nameserv_unpublish(const char service_name[],
-                                        const PMI_keyval_t *info_ptr);
-
-
-
-#if defined(__cplusplus)
-}
-#endif
-
-#endif /* PMI2_H_INCLUDED */
diff -Nru pmix-3.2.2~rc1/include/pmi.h pmix-4.0.0/include/pmi.h
--- pmix-3.2.2~rc1/include/pmi.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmi.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,832 +0,0 @@
-/*****************************************************************************\
- *  pmi.h - Process Management Interface for MPICH2
- *  See http://www-unix.mcs.anl.gov/mpi/mpich2/
- *
- *  NOTE: Dynamic Process Management functions (PMI part 2) are not supported
- *  at this time. Functions required for MPI-1 (PMI part 1) are supported.
- *****************************************************************************
- *  COPYRIGHT
- *
- *  The following is a notice of limited availability of the code, and
- *  disclaimer which must be included in the prologue of the code and in all
- *  source listings of the code.
- *
- *  Copyright Notice + 2002 University of Chicago
- *
- *  Permission is hereby granted to use, reproduce, prepare derivative
- *  works, and to redistribute to others. This software was authored by:
- *
- *  Argonne National Laboratory Group
- *  W. Gropp: (630) 252-4318; FAX: (630) 252-5986; e-mail: gropp@mcs.anl.gov
- *  E. Lusk: (630) 252-7852; FAX: (630) 252-5986; e-mail: lusk@mcs.anl.gov
- *  Mathematics and Computer Science Division Argonne National Laboratory,
- *  Argonne IL 60439
- *
- *  GOVERNMENT LICENSE
- *
- *  Portions of this material resulted from work developed under a U.S.
- *  Government Contract and are subject to the following license: the
- *  Government is granted for itself and others acting on its behalf a
- *  paid-up, nonexclusive, irrevocable worldwide license in this computer
- *  software to reproduce, prepare derivative works, and perform publicly
- *  and display publicly.
- *
- *  DISCLAIMER
- *
- *  This computer code material was prepared, in part, as an account of work
- *  sponsored by an agency of the United States Government. Neither the
- *  United States, nor the University of Chicago, nor any of their
- *  employees, makes any warranty express or implied, or assumes any legal
- *  liability or responsibility for the accuracy, completeness, or
- *  usefulness of any information, apparatus, product, or process disclosed,
- *  or represents that its use would not infringe privately owned rights.
- *
- *  MCS Division <http://www.mcs.anl.gov>        Argonne National Laboratory
- *  <http://www.anl.gov>     University of Chicago <http://www.uchicago.edu>
-\*****************************************************************************/
-
-#ifndef PMI_H
-#define PMI_H
-
-/* Structure and constant definitions */
-#include <pmix_common.h>
-
-/* prototypes for the PMI interface in MPICH2 */
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-/*D
-PMI_CONSTANTS - PMI definitions
-
-Error Codes:
-+ PMI_SUCCESS - operation completed successfully
-. PMI_FAIL - operation failed
-. PMI_ERR_NOMEM - input buffer not large enough
-. PMI_ERR_INIT - PMI not initialized
-. PMI_ERR_INVALID_ARG - invalid argument
-. PMI_ERR_INVALID_KEY - invalid key argument
-. PMI_ERR_INVALID_KEY_LENGTH - invalid key length argument
-. PMI_ERR_INVALID_VAL - invalid val argument
-. PMI_ERR_INVALID_VAL_LENGTH - invalid val length argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-. PMI_ERR_INVALID_NUM_ARGS - invalid number of arguments
-. PMI_ERR_INVALID_ARGS - invalid args argument
-. PMI_ERR_INVALID_NUM_PARSED - invalid num_parsed length argument
-. PMI_ERR_INVALID_KEYVALP - invalid keyvalp argument
-- PMI_ERR_INVALID_SIZE - invalid size argument
-
-Booleans:
-+ PMI_TRUE - true
-- PMI_FALSE - false
-
-D*/
-#define PMI_SUCCESS                  0
-#define PMI_FAIL                    -1
-#define PMI_ERR_INIT                 1
-#define PMI_ERR_NOMEM                2
-#define PMI_ERR_INVALID_ARG          3
-#define PMI_ERR_INVALID_KEY          4
-#define PMI_ERR_INVALID_KEY_LENGTH   5
-#define PMI_ERR_INVALID_VAL          6
-#define PMI_ERR_INVALID_VAL_LENGTH   7
-#define PMI_ERR_INVALID_LENGTH       8
-#define PMI_ERR_INVALID_NUM_ARGS     9
-#define PMI_ERR_INVALID_ARGS        10
-#define PMI_ERR_INVALID_NUM_PARSED  11
-#define PMI_ERR_INVALID_KEYVALP     12
-#define PMI_ERR_INVALID_SIZE        13
-#define PMI_ERR_INVALID_KVS         14
-
-typedef int PMI_BOOL;
-#define PMI_TRUE     1
-#define PMI_FALSE    0
-
-/* PMI Group functions */
-
-/*@
-PMI_Init - initialize the Process Manager Interface
-
-Output Parameter:
-. spawned - spawned flag
-
-Return values:
-+ PMI_SUCCESS - initialization completed successfully
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - initialization failed
-
-Notes:
-Initialize PMI for this process group. The value of spawned indicates whether
-this process was created by 'PMI_Spawn_multiple'.  'spawned' will be 'PMI_TRUE' if
-this process group has a parent and 'PMI_FALSE' if it does not.
-
-@*/
-PMIX_EXPORT int PMI_Init( int *spawned );
-
-/*@
-PMI_Initialized - check if PMI has been initialized
-
-Output Parameter:
-. initialized - boolean value
-
-Return values:
-+ PMI_SUCCESS - initialized successfully set
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to set the variable
-
-Notes:
-On successful output, initialized will either be 'PMI_TRUE' or 'PMI_FALSE'.
-
-+ PMI_TRUE - initialize has been called.
-- PMI_FALSE - initialize has not been called or previously failed.
-
-@*/
-PMIX_EXPORT int PMI_Initialized( PMI_BOOL *initialized );
-
-/*@
-PMI_Finalize - finalize the Process Manager Interface
-
-Return values:
-+ PMI_SUCCESS - finalization completed successfully
-- PMI_FAIL - finalization failed
-
-Notes:
- Finalize PMI for this process group.
-
-@*/
-PMIX_EXPORT int PMI_Finalize( void );
-
-/*@
-PMI_Get_size - obtain the size of the process group
-
-Output Parameters:
-. size - pointer to an integer that receives the size of the process group
-
-Return values:
-+ PMI_SUCCESS - size successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to return the size
-
-Notes:
-This function returns the size of the process group to which the local process
-belongs.
-
-@*/
-PMIX_EXPORT int PMI_Get_size( int *size );
-
-/*@
-PMI_Get_rank - obtain the rank of the local process in the process group
-
-Output Parameters:
-. rank - pointer to an integer that receives the rank in the process group
-
-Return values:
-+ PMI_SUCCESS - rank successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to return the rank
-
-Notes:
-This function returns the rank of the local process in its process group.
-
-@*/
-PMIX_EXPORT int PMI_Get_rank( int *rank );
-
-/*@
-PMI_Get_universe_size - obtain the universe size
-
-Output Parameters:
-. size - pointer to an integer that receives the size
-
-Return values:
-+ PMI_SUCCESS - size successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to return the size
-
-
-@*/
-PMIX_EXPORT int PMI_Get_universe_size( int *size );
-
-/*@
-PMI_Get_appnum - obtain the application number
-
-Output parameters:
-. appnum - pointer to an integer that receives the appnum
-
-Return values:
-+ PMI_SUCCESS - appnum successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to return the size
-
-
-@*/
-PMIX_EXPORT int PMI_Get_appnum( int *appnum );
-
-/*@
-PMI_Publish_name - publish a name
-
-Input parameters:
-. service_name - string representing the service being published
-. port - string representing the port on which to contact the service
-
-Return values:
-+ PMI_SUCCESS - port for service successfully published
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to publish service
-
-
-@*/
-PMIX_EXPORT int PMI_Publish_name( const char service_name[], const char port[] );
-
-/*@
-PMI_Unpublish_name - unpublish a name
-
-Input parameters:
-. service_name - string representing the service being unpublished
-
-Return values:
-+ PMI_SUCCESS - port for service successfully published
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to unpublish service
-
-
-@*/
-PMIX_EXPORT int PMI_Unpublish_name( const char service_name[] );
-
-/*@
-PMI_Lookup_name - lookup a service by name
-
-Input parameters:
-. service_name - string representing the service being published
-
-Output parameters:
-. port - string representing the port on which to contact the service
-
-Return values:
-+ PMI_SUCCESS - port for service successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to lookup service
-
-
-@*/
-PMIX_EXPORT int PMI_Lookup_name( const char service_name[], char port[] );
-
-/*@
-PMI_Get_id - obtain the id of the process group
-
-Input Parameter:
-. length - length of the id_str character array
-
-Output Parameter:
-. id_str - character array that receives the id of the process group
-
-Return values:
-+ PMI_SUCCESS - id successfully obtained
-. PMI_ERR_INVALID_ARG - invalid rank argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-- PMI_FAIL - unable to return the id
-
-Notes:
-This function returns a string that uniquely identifies the process group
-that the local process belongs to.  The string passed in must be at least
-as long as the number returned by 'PMI_Get_id_length_max()'.
-
-@*/
-PMIX_EXPORT int PMI_Get_id( char id_str[], int length );
-
-/*@
-PMI_Get_kvs_domain_id - obtain the id of the PMI domain
-
-Input Parameter:
-. length - length of id_str character array
-
-Output Parameter:
-. id_str - character array that receives the id of the PMI domain
-
-Return values:
-+ PMI_SUCCESS - id successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-- PMI_FAIL - unable to return the id
-
-Notes:
-This function returns a string that uniquely identifies the PMI domain
-where keyval spaces can be shared.  The string passed in must be at least
-as long as the number returned by 'PMI_Get_id_length_max()'.
-
-@*/
-PMIX_EXPORT int PMI_Get_kvs_domain_id( char id_str[], int length );
-
-/*@
-PMI_Get_id_length_max - obtain the maximum length of an id string
-
-Output Parameters:
-. length - the maximum length of an id string
-
-Return values:
-+ PMI_SUCCESS - length successfully set
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to return the maximum length
-
-Notes:
-This function returns the maximum length of a process group id string.
-
-@*/
-PMIX_EXPORT int PMI_Get_id_length_max( int *length );
-
-/*@
-PMI_Barrier - barrier across the process group
-
-Return values:
-+ PMI_SUCCESS - barrier successfully finished
-- PMI_FAIL - barrier failed
-
-Notes:
-This function is a collective call across all processes in the process group
-the local process belongs to.  It will not return until all the processes
-have called 'PMI_Barrier()'.
-
-@*/
-PMIX_EXPORT int PMI_Barrier( void );
-
-/*@
-PMI_Get_clique_size - obtain the number of processes on the local node
-
-Output Parameters:
-. size - pointer to an integer that receives the size of the clique
-
-Return values:
-+ PMI_SUCCESS - size successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to return the clique size
-
-Notes:
-This function returns the number of processes in the local process group that
-are on the local node along with the local process.  This is a simple topology
-function to distinguish between processes that can communicate through IPC
-mechanisms (e.g., shared memory) and other network mechanisms.
-
-@*/
-PMIX_EXPORT int PMI_Get_clique_size( int *size );
-
-/*@
-PMI_Get_clique_ranks - get the ranks of the local processes in the process group
-
-Input Parameters:
-. length - length of the ranks array
-
-Output Parameters:
-. ranks - pointer to an array of integers that receive the local ranks
-
-Return values:
-+ PMI_SUCCESS - ranks successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-- PMI_FAIL - unable to return the ranks
-
-Notes:
-This function returns the ranks of the processes on the local node.  The array
-must be at least as large as the size returned by 'PMI_Get_clique_size()'.  This
-is a simple topology function to distinguish between processes that can
-communicate through IPC mechanisms (e.g., shared memory) and other network
-mechanisms.
-
-@*/
-PMIX_EXPORT int PMI_Get_clique_ranks( int ranks[], int length);
-
-/*@
-PMI_Abort - abort the process group associated with this process
-
-Input Parameters:
-+ exit_code - exit code to be returned by this process
-- error_msg - error message to be printed
-
-Return values:
-. none - this function should not return
-@*/
-PMIX_EXPORT int PMI_Abort(int exit_code, const char error_msg[]);
-
-/* PMI Keymap functions */
-/*@
-PMI_KVS_Get_my_name - obtain the name of the keyval space the local process group has access to
-
-Input Parameters:
-. length - length of the kvsname character array
-
-Output Parameters:
-. kvsname - a string that receives the keyval space name
-
-Return values:
-+ PMI_SUCCESS - kvsname successfully obtained
-. PMI_ERR_INVALID_ARG - invalid argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-- PMI_FAIL - unable to return the kvsname
-
-Notes:
-This function returns the name of the keyval space that this process and all
-other processes in the process group have access to.  The output parameter,
-kvsname, must be at least as long as the value returned by
-'PMI_KVS_Get_name_length_max()'.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Get_my_name( char kvsname[], int length );
-
-/*@
-PMI_KVS_Get_name_length_max - obtain the length necessary to store a kvsname
-
-Output Parameter:
-. length - maximum length required to hold a keyval space name
-
-Return values:
-+ PMI_SUCCESS - length successfully set
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to set the length
-
-Notes:
-This function returns the string length required to store a keyval space name.
-
-A routine is used rather than setting a maximum value in 'pmi.h' to allow
-different implementations of PMI to be used with the same executable.  These
-different implementations may allow different maximum lengths; by using a
-routine here, we can interface with a variety of implementations of PMI.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Get_name_length_max( int *length );
-
-/*@
-PMI_KVS_Get_key_length_max - obtain the length necessary to store a key
-
-Output Parameter:
-. length - maximum length required to hold a key string.
-
-Return values:
-+ PMI_SUCCESS - length successfully set
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to set the length
-
-Notes:
-This function returns the string length required to store a key.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Get_key_length_max( int *length );
-
-/*@
-PMI_KVS_Get_value_length_max - obtain the length necessary to store a value
-
-Output Parameter:
-. length - maximum length required to hold a keyval space value
-
-Return values:
-+ PMI_SUCCESS - length successfully set
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to set the length
-
-Notes:
-This function returns the string length required to store a value from a
-keyval space.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Get_value_length_max( int *length );
-
-/*@
-PMI_KVS_Create - create a new keyval space
-
-Input Parameter:
-. length - length of the kvsname character array
-
-Output Parameters:
-. kvsname - a string that receives the keyval space name
-
-Return values:
-+ PMI_SUCCESS - keyval space successfully created
-. PMI_ERR_INVALID_ARG - invalid argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-- PMI_FAIL - unable to create a new keyval space
-
-Notes:
-This function creates a new keyval space.  Everyone in the same process group
-can access this keyval space by the name returned by this function.  The
-function is not collective.  Only one process calls this function.  The output
-parameter, kvsname, must be at least as long as the value returned by
-'PMI_KVS_Get_name_length_max()'.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Create( char kvsname[], int length );
-
-/*@
-PMI_KVS_Destroy - destroy keyval space
-
-Input Parameters:
-. kvsname - keyval space name
-
-Return values:
-+ PMI_SUCCESS - keyval space successfully destroyed
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - unable to destroy the keyval space
-
-Notes:
-This function destroys a keyval space created by 'PMI_KVS_Create()'.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Destroy( const char kvsname[] );
-
-/*@
-PMI_KVS_Put - put a key/value pair in a keyval space
-
-Input Parameters:
-+ kvsname - keyval space name
-. key - key
-- value - value
-
-Return values:
-+ PMI_SUCCESS - keyval pair successfully put in keyval space
-. PMI_ERR_INVALID_KVS - invalid kvsname argument
-. PMI_ERR_INVALID_KEY - invalid key argument
-. PMI_ERR_INVALID_VAL - invalid val argument
-- PMI_FAIL - put failed
-
-Notes:
-This function puts the key/value pair in the specified keyval space.  The
-value is not visible to other processes until 'PMI_KVS_Commit()' is called.
-The function may complete locally.  After 'PMI_KVS_Commit()' is called, the
-value may be retrieved by calling 'PMI_KVS_Get()'.  All keys put to a keyval
-space must be unique to the keyval space.  You may not put more than once
-with the same key.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Put( const char kvsname[], const char key[], const char value[]);
-
-/*@
-PMI_KVS_Commit - commit all previous puts to the keyval space
-
-Input Parameters:
-. kvsname - keyval space name
-
-Return values:
-+ PMI_SUCCESS - commit succeeded
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - commit failed
-
-Notes:
-This function commits all previous puts since the last 'PMI_KVS_Commit()' into
-the specified keyval space. It is a process local operation.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Commit( const char kvsname[] );
-
-/*@
-PMI_KVS_Get - get a key/value pair from a keyval space
-
-Input Parameters:
-+ kvsname - keyval space name
-. key - key
-- length - length of value character array
-
-Output Parameters:
-. value - value
-
-Return values:
-+ PMI_SUCCESS - get succeeded
-. PMI_ERR_INVALID_KVS - invalid kvsname argument
-. PMI_ERR_INVALID_KEY - invalid key argument
-. PMI_ERR_INVALID_VAL - invalid val argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-- PMI_FAIL - get failed
-
-Notes:
-This function gets the value of the specified key in the keyval space.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[], int length);
-
-/*@
-PMI_KVS_Iter_first - initialize the iterator and get the first value
-
-Input Parameters:
-+ kvsname - keyval space name
-. key_len - length of key character array
-- val_len - length of val character array
-
-Output Parameters:
-+ key - key
-- value - value
-
-Return values:
-+ PMI_SUCCESS - keyval pair successfully retrieved from the keyval space
-. PMI_ERR_INVALID_KVS - invalid kvsname argument
-. PMI_ERR_INVALID_KEY - invalid key argument
-. PMI_ERR_INVALID_KEY_LENGTH - invalid key length argument
-. PMI_ERR_INVALID_VAL - invalid val argument
-. PMI_ERR_INVALID_VAL_LENGTH - invalid val length argument
-- PMI_FAIL - failed to initialize the iterator and get the first keyval pair
-
-Notes:
-This function initializes the iterator for the specified keyval space and
-retrieves the first key/val pair.  The end of the keyval space is specified
-by returning an empty key string.  key and val must be at least as long as
-the values returned by 'PMI_KVS_Get_key_length_max()' and
-'PMI_KVS_Get_value_length_max()'.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Iter_first(const char kvsname[], char key[], int key_len, char val[], int val_len);
-
-/*@
-PMI_KVS_Iter_next - get the next keyval pair from the keyval space
-
-Input Parameters:
-+ kvsname - keyval space name
-. key_len - length of key character array
-- val_len - length of val character array
-
-Output Parameters:
-+ key - key
-- value - value
-
-Return values:
-+ PMI_SUCCESS - keyval pair successfully retrieved from the keyval space
-. PMI_ERR_INVALID_KVS - invalid kvsname argument
-. PMI_ERR_INVALID_KEY - invalid key argument
-. PMI_ERR_INVALID_KEY_LENGTH - invalid key length argument
-. PMI_ERR_INVALID_VAL - invalid val argument
-. PMI_ERR_INVALID_VAL_LENGTH - invalid val length argument
-- PMI_FAIL - failed to get the next keyval pair
-
-Notes:
-This function retrieves the next keyval pair from the specified keyval space.
-'PMI_KVS_Iter_first()' must have been previously called.  The end of the keyval
-space is specified by returning an empty key string.  The output parameters,
-key and val, must be at least as long as the values returned by
-'PMI_KVS_Get_key_length_max()' and 'PMI_KVS_Get_value_length_max()'.
-
-@*/
-PMIX_EXPORT int PMI_KVS_Iter_next(const char kvsname[], char key[], int key_len, char val[], int val_len);
-
-/* PMI Process Creation functions */
-
-/*S
-PMI_keyval_t - keyval structure used by PMI_Spawn_mulitiple
-
-Fields:
-+ key - name of the key
-- val - value of the key
-
-S*/
-typedef struct PMI_keyval_t
-{
-    char * key;
-    char * val;
-} PMI_keyval_t;
-
-/*@
-PMI_Spawn_multiple - spawn a new set of processes
-
-Input Parameters:
-+ count - count of commands
-. cmds - array of command strings
-. argvs - array of argv arrays for each command string
-. maxprocs - array of maximum processes to spawn for each command string
-. info_keyval_sizes - array giving the number of elements in each of the
-  'info_keyval_vectors'
-. info_keyval_vectors - array of keyval vector arrays
-. preput_keyval_size - Number of elements in 'preput_keyval_vector'
-- preput_keyval_vector - array of keyvals to be pre-put in the spawned keyval space
-
-Output Parameter:
-. errors - array of errors for each command
-
-Return values:
-+ PMI_SUCCESS - spawn successful
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - spawn failed
-
-Notes:
-This function spawns a set of processes into a new process group.  The 'count'
-field refers to the size of the array parameters - 'cmd', 'argvs', 'maxprocs',
-'info_keyval_sizes' and 'info_keyval_vectors'.  The 'preput_keyval_size' refers
-to the size of the 'preput_keyval_vector' array.  The 'preput_keyval_vector'
-contains keyval pairs that will be put in the keyval space of the newly
-created process group before the processes are started.  The 'maxprocs' array
-specifies the desired number of processes to create for each 'cmd' string.
-The actual number of processes may be less than the numbers specified in
-maxprocs.  The acceptable number of processes spawned may be controlled by
-``soft'' keyvals in the info arrays.  The ``soft'' option is specified by
-mpiexec in the MPI-2 standard.  Environment variables may be passed to the
-spawned processes through PMI implementation specific 'info_keyval' parameters.
-@*/
-PMIX_EXPORT int PMI_Spawn_multiple(int count,
-                                   const char * cmds[],
-                                   const char ** argvs[],
-                                   const int maxprocs[],
-                                   const int info_keyval_sizesp[],
-                                   const PMI_keyval_t * info_keyval_vectors[],
-                                   int preput_keyval_size,
-                                   const PMI_keyval_t preput_keyval_vector[],
-                                   int errors[]);
-
-
-/*@
-PMI_Parse_option - create keyval structures from a single command line argument
-
-Input Parameters:
-+ num_args - length of args array
-- args - array of command line arguments starting with the argument to be parsed
-
-Output Parameters:
-+ num_parsed - number of elements of the argument array parsed
-. keyvalp - pointer to an array of keyvals
-- size - size of the allocated array
-
-Return values:
-+ PMI_SUCCESS - success
-. PMI_ERR_INVALID_NUM_ARGS - invalid number of arguments
-. PMI_ERR_INVALID_ARGS - invalid args argument
-. PMI_ERR_INVALID_NUM_PARSED - invalid num_parsed length argument
-. PMI_ERR_INVALID_KEYVALP - invalid keyvalp argument
-. PMI_ERR_INVALID_SIZE - invalid size argument
-- PMI_FAIL - fail
-
-Notes:
-This function removes one PMI specific argument from the command line and
-creates the corresponding 'PMI_keyval_t' structure for it.  It returns
-an array and size to the caller.  The array must be freed by 'PMI_Free_keyvals()'.
-If the first element of the args array is not a PMI specific argument, the function
-returns success and sets num_parsed to zero.  If there are multiple PMI specific
-arguments in the args array, this function may parse more than one argument as long
-as the options are contiguous in the args array.
-
-@*/
-PMIX_EXPORT int PMI_Parse_option(int num_args, char *args[], int *num_parsed, PMI_keyval_t **keyvalp, int *size);
-
-/*@
-PMI_Args_to_keyval - create keyval structures from command line arguments
-
-Input Parameters:
-+ argcp - pointer to argc
-- argvp - pointer to argv
-
-Output Parameters:
-+ keyvalp - pointer to an array of keyvals
-- size - size of the allocated array
-
-Return values:
-+ PMI_SUCCESS - success
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - fail
-
-Notes:
-This function removes PMI specific arguments from the command line and
-creates the corresponding 'PMI_keyval_t' structures for them.  It returns
-an array and size to the caller that can then be passed to 'PMI_Spawn_multiple()'.
-The array can be freed by 'PMI_Free_keyvals()'.  The routine 'free()' should
-not be used to free this array as there is no requirement that the array be
-allocated with 'malloc()'.
-
-@*/
-PMIX_EXPORT int PMI_Args_to_keyval(int *argcp, char *((*argvp)[]), PMI_keyval_t **keyvalp, int *size);
-
-/*@
-PMI_Free_keyvals - free the keyval structures created by PMI_Args_to_keyval
-
-Input Parameters:
-+ keyvalp - array of keyvals
-- size - size of the array
-
-Return values:
-+ PMI_SUCCESS - success
-. PMI_ERR_INVALID_ARG - invalid argument
-- PMI_FAIL - fail
-
-Notes:
- This function frees the data returned by 'PMI_Args_to_keyval' and 'PMI_Parse_option'.
- Using this routine instead of 'free' allows the PMI package to track
- allocation of storage or to use interal storage as it sees fit.
-@*/
-PMIX_EXPORT int PMI_Free_keyvals(PMI_keyval_t keyvalp[], int size);
-
-/*@
-PMI_Get_options - get a string of command line argument descriptions that may be printed to the user
-
-Input Parameters:
-. length - length of str
-
-Output Parameters:
-+ str - description string
-- length - length of string or necessary length if input is not large enough
-
-Return values:
-+ PMI_SUCCESS - success
-. PMI_ERR_INVALID_ARG - invalid argument
-. PMI_ERR_INVALID_LENGTH - invalid length argument
-. PMI_ERR_NOMEM - input length too small
-- PMI_FAIL - fail
-
-Notes:
- This function returns the command line options specific to the pmi implementation
-@*/
-PMIX_EXPORT int PMI_Get_options(char *str, int *length);
-
-#if defined(__cplusplus)
-}
-#endif
-
-#endif
diff -Nru pmix-3.2.2~rc1/include/pmix_common.h.in pmix-4.0.0/include/pmix_common.h.in
--- pmix-3.2.2~rc1/include/pmix_common.h.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmix_common.h.in	2021-01-02 08:56:17.000000000 +0000
@@ -3,7 +3,7 @@
  * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016-2019 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016-2020 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * Copyright (c) 2016-2019 Mellanox Technologies, Inc.
  *                         All rights reserved.
  *
@@ -42,6 +42,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
+ * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -105,26 +106,34 @@
 /* other special rank values will be used to define
  * groups of ranks for use in collectives */
 #define PMIX_RANK_LOCAL_NODE    UINT32_MAX-2        // all ranks on local node
+#define PMIX_RANK_LOCAL_PEERS   UINT32_MAX-4        // all peers (i.e., all procs within the same nspace) on local node
 /* define an invalid value */
 #define PMIX_RANK_INVALID   UINT32_MAX-3
 /* define a boundary for valid ranks */
 #define PMIX_RANK_VALID         UINT32_MAX-50
-
+/* define a macro for testing for valid ranks */
+#define PMIX_RANK_IS_VALID(r)   \
+    ((r) < PMIX_RANK_VALID)
+
+/* define a value to indicate that data applies
+ * to all apps in a job */
+#define PMIX_APP_WILDCARD  UINT32_MAX
 
 /****  PMIX ENVIRONMENTAL PARAMETERS  ****/
-/* There are a few environmental parameters used by PMIx for
- * various operations. While there is no "definition" of them
- * as values, we do record them here for informational purposes.
- *
- * PMIX_LAUNCHER_PAUSE_FOR_TOOL - if set to non-zero value, instructs
- * launchers (e.g., "prun") to stop prior to spawning the application until
- * a tool can connect with further instructions. This envar will be
- * set by the tool and is _not_ intended for the direct use of users.
- *
- * PMIX_LAUNCHER_RENDEZVOUS_FILE - if set, contains the full pathname
+/* URI of tool waiting for launcher to rendezvous back to it */
+#define PMIX_LAUNCHER_RNDZ_URI "PMIX_LAUNCHER_RNDZ_URI"
+
+/* PMIX_LAUNCHER_RNDZ_FILE - if set, contains the full pathname
  * of a file the launcher is to write that contains its connection info.
  * Works in addition to anything else the launcher may output.
  */
+#define PMIX_LAUNCHER_RNDZ_FILE "PMIX_LAUNCHER_RNDZ_FILE"
+
+/* pipe to be monitored that indicates when the parent process
+ * terminates - used by fork'd tools to identify when the tool
+ * that started them has died */
+#define PMIX_KEEPALIVE_PIPE "PMIX_KEEPALIVE_PIPE"
+
 
 /* define a set of "standard" PMIx attributes that can
  * be queried. Implementations (and users) are free to extend as
@@ -135,11 +144,11 @@
  * might choose to expose - i.e., they are values provided
  * by the resource manager as opposed to the application. Thus,
  * these keys are RESERVED */
-#define PMIX_ATTR_UNDEF      NULL
+#define PMIX_ATTR_UNDEF      "pmix.undef"
 
 /* initialization attributes */
-#define PMIX_EVENT_BASE                     "pmix.evbase"           // (struct event_base *) pointer to libevent event_base
-                                                                    // to use in place of the internal progress thread
+#define PMIX_EXTERNAL_PROGRESS              "pmix.evext"            // (bool) The host shall progress the PMIx library via
+                                                                    //        calls to PMIx_Progress
 #define PMIX_SERVER_TOOL_SUPPORT            "pmix.srvr.tool"        // (bool) The host RM wants to declare itself as willing
                                                                     //        to accept tool connection requests
 #define PMIX_SERVER_REMOTE_CONNECTIONS      "pmix.srvr.remote"      // (bool) Allow connections from remote tools (do not use
@@ -147,17 +156,29 @@
 #define PMIX_SERVER_SYSTEM_SUPPORT          "pmix.srvr.sys"         // (bool) The host RM wants to declare itself as being
                                                                     //        the local system server for PMIx connection
                                                                     //        requests
+#define PMIX_SERVER_SESSION_SUPPORT         "pmix.srvr.sess"        // (bool) The host RM wants to declare itself as being
+                                                                    //        the local session server for PMIx connection
+                                                                    //        requests
 #define PMIX_SERVER_TMPDIR                  "pmix.srvr.tmpdir"      // (char*) temp directory where PMIx server will place
                                                                     //        client rendezvous points and contact info
 #define PMIX_SYSTEM_TMPDIR                  "pmix.sys.tmpdir"       // (char*) temp directory for this system, where PMIx
                                                                     //        server will place tool rendezvous points and
                                                                     //        contact info
+#define PMIX_SERVER_SHARE_TOPOLOGY          "pmix.srvr.share"       // (bool) server is to share its copy of the local node
+                                                                    // topology (whether given to it or self-discovered) with any clients.
 #define PMIX_SERVER_ENABLE_MONITORING       "pmix.srv.monitor"      // (bool) Enable PMIx internal monitoring by server
 #define PMIX_SERVER_NSPACE                  "pmix.srv.nspace"       // (char*) Name of the nspace to use for this server
 #define PMIX_SERVER_RANK                    "pmix.srv.rank"         // (pmix_rank_t) Rank of this server
 #define PMIX_SERVER_GATEWAY                 "pmix.srv.gway"         // (bool) Server is acting as a gateway for PMIx requests
                                                                     //        that cannot be serviced on backend nodes
                                                                     //        (e.g., logging to email)
+#define PMIX_SERVER_SCHEDULER               "pmix.srv.sched"        // (bool) Server supports system scheduler
+#define PMIX_SERVER_START_TIME              "pmix.srv.strtime"      // (char*) Time when the server started - i.e., when the server created
+                                                                    //         it's rendezvous file (given in ctime string format)
+#define PMIX_HOMOGENEOUS_SYSTEM             "pmix.homo"             // (bool) The nodes comprising the session are homogeneous - i.e., they
+                                                                    //        each contain the same number of identical packages, fabric interfaces,
+                                                                    //        GPU, and other devices
+
 
 /* tool-related attributes */
 #define PMIX_TOOL_NSPACE                    "pmix.tool.nspace"      // (char*) Name of the nspace to use for this tool
@@ -173,17 +194,46 @@
 #define PMIX_TOOL_DO_NOT_CONNECT            "pmix.tool.nocon"       // (bool) the tool wants to use internal PMIx support, but does
                                                                     //        not want to connect to a PMIx server
                                                                     //        from the specified processes to this tool
-#define PMIX_RECONNECT_SERVER               "pmix.cnct.recon"       // (bool) tool is requesting to change server connections
+#define PMIX_TOOL_CONNECT_OPTIONAL          "pmix.tool.conopt"      // (bool) tool shall connect to a server if available, but otherwise
+                                                                    //        continue to operate unconnected
 #define PMIX_LAUNCHER                       "pmix.tool.launcher"    // (bool) tool is a launcher and needs rendezvous files created
 #define PMIX_LAUNCHER_RENDEZVOUS_FILE       "pmix.tool.lncrnd"      // (char*) Pathname of file where connection info is to be stored
+#define PMIX_TOOL_ATTACHMENT_FILE           "pmix.tool.attach"      // (char*) File containing connection info to be used for attaching to server
+#define PMIX_PRIMARY_SERVER                 "pmix.pri.srvr"         // (bool) The server to which the tool is connecting shall be designated
+                                                                    //        the primary server once connection has been accomplished.
+#define PMIX_NOHUP                          "pmix.nohup"            // (bool) Any processes started on behalf of the calling tool (or the
+                                                                    //        specified namespace, if such specification is included in the
+                                                                    //        list of attributes) should continue after the tool disconnects
+                                                                    //        from its server
+#define PMIX_LAUNCHER_DAEMON                "pmix.lnch.dmn"         // (char*) Path to executable that is to be used as the backend daemon
+                                                                    //        for the launcher. This replaces the launcher's own daemon with
+                                                                    //        the specified executable. Note that the user is therefore
+                                                                    //        responsible for ensuring compatibility of the specified
+                                                                    //        executable and the host launcher.
+#define PMIX_EXEC_AGENT                     "pmix.exec.agnt"        // (char*) Path to executable that the launcher's backend daemons are to
+                                                                    //        fork/exec in place of the actual application processes. The
+                                                                    //        launcher's daemon shall pass the full command line of the
+                                                                    //        application on the command line of the exec agent, which shall
+                                                                    //        not connect back to the launcher's daemon. The exec agent is
+                                                                    //        responsible for exec'ing the specified application process in
+                                                                    //        its own place.
+#define PMIX_LAUNCH_DIRECTIVES              "pmix.lnch.dirs"        // (pmix_data_array_t*) Array of pmix_info_t containing directives for
+                                                                    //        the launcher - a convenience attribute for retrieving all
+                                                                    //        directives with a single call to PMIx_Get
 
 /* identification attributes */
 #define PMIX_USERID                         "pmix.euid"             // (uint32_t) effective user id
 #define PMIX_GRPID                          "pmix.egid"             // (uint32_t) effective group id
-#define PMIX_DSTPATH                        "pmix.dstpath"          // (char*) path to dstore files
 #define PMIX_VERSION_INFO                   "pmix.version"          // (char*) PMIx version of contactor
 #define PMIX_REQUESTOR_IS_TOOL              "pmix.req.tool"         // (bool) requesting process is a tool
 #define PMIX_REQUESTOR_IS_CLIENT            "pmix.req.client"       // (bool) requesting process is a client process
+#define PMIX_PSET_NAME                      "pmix.pset.nm"          // (char*) The name of the newly defined process set.
+#define PMIX_PSET_NAMES                     "pmix.pset.nms"         // (pmix_data_array_t*) Returns an array of string names of the
+                                                                    //          process sets in which the given process is a member.
+#define PMIX_PSET_MEMBERS                   "pmix.pset.mems"        // (pmix_data_array_t*) An array of pmix_proc_t containing
+                                                                    //          the members of the newly defined process set.
+#define PMIX_REINCARNATION                  "pmix.reinc"            // (uint32_t) number of times this process has been instantiated - i.e.,
+                                                                    //            tracks the number of times it has been restarted
 
 /* model attributes */
 #define PMIX_PROGRAMMING_MODEL              "pmix.pgm.model"        // (char*) programming model being initialized (e.g., "MPI" or "OpenMP")
@@ -218,15 +268,11 @@
 #define PMIX_TCP_DISABLE_IPV6               "pmix.tcp.disipv6"      // (bool) true to disable IPv6 family
 
 
-/* attributes for GDS */
-#define PMIX_GDS_MODULE                     "pmix.gds.mod"          // (char*) comma-delimited string of desired modules
-
-
 /* general proc-level attributes */
-#define PMIX_CPUSET                         "pmix.cpuset"           // (char*) hwloc bitmap applied to proc upon launch
+#define PMIX_CPUSET                         "pmix.cpuset"           // (char*) String representation of bitmap applied to process upon launch
+#define PMIX_CPUSET_BITMAP                  "pmix.bitmap"           // (pmix_cpuset_t*) Bitmap applied to process at launch
 #define PMIX_CREDENTIAL                     "pmix.cred"             // (char*) security credential assigned to proc
 #define PMIX_SPAWNED                        "pmix.spawned"          // (bool) true if this proc resulted from a call to PMIx_Spawn
-#define PMIX_ARCH                           "pmix.arch"             // (uint32_t) datatype architecture flag
 
 /* scratch directory locations for use by applications */
 #define PMIX_TMPDIR                         "pmix.tmpdir"           // (char*) top-level tmp dir assigned to session
@@ -246,6 +292,7 @@
 #define PMIX_NPROC_OFFSET                   "pmix.offset"           // (pmix_rank_t) starting global rank of this job
 #define PMIX_LOCAL_RANK                     "pmix.lrank"            // (uint16_t) rank on this node within this job
 #define PMIX_NODE_RANK                      "pmix.nrank"            // (uint16_t) rank on this node spanning all jobs
+#define PMIX_PACKAGE_RANK                   "pmix.pkgrank"          // (uint16_t) rank within this job on the package where this proc resides
 #define PMIX_LOCALLDR                       "pmix.lldr"             // (pmix_rank_t) lowest rank on this node within this job
 #define PMIX_APPLDR                         "pmix.aldr"             // (pmix_rank_t) lowest rank in this app within this job
 #define PMIX_PROC_PID                       "pmix.ppid"             // (pid_t) pid of specified proc
@@ -261,14 +308,12 @@
 #define PMIX_LOCAL_PEERS                    "pmix.lpeers"           // (char*) comma-delimited string of ranks on this node within the specified nspace
 #define PMIX_LOCAL_PROCS                    "pmix.lprocs"           // (pmix_data_array_t*) array of pmix_proc_t of procs on the specified node
 #define PMIX_LOCAL_CPUSETS                  "pmix.lcpus"            // (char*) colon-delimited cpusets of local peers within the specified nspace
-#define PMIX_PROC_URI                       "pmix.puri"             // (char*) URI containing contact info for proc
-#define PMIX_LOCALITY                       "pmix.loc"              // (uint16_t) relative locality of two procs
 #define PMIX_PARENT_ID                      "pmix.parent"           // (pmix_proc_t*) identifier of the process that called PMIx_Spawn
                                                                     //                to launch this proc's application
 #define PMIX_EXIT_CODE                      "pmix.exit.code"        // (int) exit code returned when proc terminated
 
 /* size info */
-#define PMIX_UNIV_SIZE                      "pmix.univ.size"        // (uint32_t) #procs in this nspace
+#define PMIX_UNIV_SIZE                      "pmix.univ.size"        // (uint32_t) #slots in this session
 #define PMIX_JOB_SIZE                       "pmix.job.size"         // (uint32_t) #procs in this job
 #define PMIX_JOB_NUM_APPS                   "pmix.job.napps"        // (uint32_t) #apps in this job
 #define PMIX_APP_SIZE                       "pmix.app.size"         // (uint32_t) #procs in this application
@@ -276,8 +321,9 @@
 #define PMIX_NODE_SIZE                      "pmix.node.size"        // (uint32_t) #procs across all jobs on this node
 #define PMIX_MAX_PROCS                      "pmix.max.size"         // (uint32_t) max #procs for this job
 #define PMIX_NUM_SLOTS                      "pmix.num.slots"        // (uint32_t) #slots allocated
-#define PMIX_NUM_NODES                      "pmix.num.nodes"        // (uint32_t) #nodes in this nspace
-
+#define PMIX_NUM_NODES                      "pmix.num.nodes"        // (uint32_t) #nodes currently hosting processes in the specified realm.
+#define PMIX_NUM_ALLOCATED_NODES            "pmix.num.anodes"       // (uint32_t) #nodes in the specified realm regardless of whether or
+                                                                    //            not they currently host processes.
 
 /* Memory info */
 #define PMIX_AVAIL_PHYS_MEMORY              "pmix.pmem"             // (uint64_t) total available physical memory on this node
@@ -286,32 +332,26 @@
 
 
 /* topology info */
-#define PMIX_NET_TOPO                       "pmix.ntopo"            // (char*) xml-representation of network topology
-#define PMIX_LOCAL_TOPO                     "pmix.ltopo"            // (char*) xml-representation of local node topology
-#define PMIX_TOPOLOGY                       "pmix.topo"             // (hwloc_topology_t) pointer to the PMIx client's internal topology object
-#define PMIX_TOPOLOGY_XML                   "pmix.topo.xml"         // (char*) XML-based description of topology
-#define PMIX_TOPOLOGY_FILE                  "pmix.topo.file"        // (char*) full path to file containing XML topology description
-#define PMIX_TOPOLOGY_SIGNATURE             "pmix.toposig"          // (char*) topology signature string
+#define PMIX_TOPOLOGY2                      "pmix.topo2"            // (pmix_topology_t) pointer to a PMIx topology object
 #define PMIX_LOCALITY_STRING                "pmix.locstr"           // (char*) string describing a proc's location
-#define PMIX_HWLOC_SHMEM_ADDR               "pmix.hwlocaddr"        // (size_t) address of HWLOC shared memory segment
-#define PMIX_HWLOC_SHMEM_SIZE               "pmix.hwlocsize"        // (size_t) size of HWLOC shared memory segment
-#define PMIX_HWLOC_SHMEM_FILE               "pmix.hwlocfile"        // (char*) path to HWLOC shared memory file
-#define PMIX_HWLOC_XML_V1                   "pmix.hwlocxml1"        // (char*) XML representation of local topology using HWLOC v1.x format
-#define PMIX_HWLOC_XML_V2                   "pmix.hwlocxml2"        // (char*) XML representation of local topology using HWLOC v2.x format
-#define PMIX_HWLOC_SHARE_TOPO               "pmix.hwlocsh"          // (bool) Share the HWLOC topology via shared memory
-#define PMIX_HWLOC_HOLE_KIND                "pmix.hwlocholek"       // (char*) Kind of VM "hole" HWLOC should use for shared memory
 
 
 /* request-related info */
 #define PMIX_COLLECT_DATA                   "pmix.collect"          // (bool) collect data and return it at the end of the operation
+#define PMIX_ALL_CLONES_PARTICIPATE         "pmix.clone.part"       // (bool) All clones of the calling process must participate in the collective operation.
+#define PMIX_COLLECT_GENERATED_JOB_INFO     "pmix.collect.gen"      // (bool) Collect all job-level information (i.e., reserved keys) that was locally
+                                                                    //        generated by PMIx servers. Some job-level information (e.g., distance between
+                                                                    //        processes and fabric devices) is best determined on a distributed basis as it
+                                                                    //        primarily pertains to local processes. Should remote processes need to access
+                                                                    //        the information, it can either be obtained collectively using the PMIx_Fence
+                                                                    //        operation with this directive, or can be retrieved one peer at a time using
+                                                                    //        PMIx_Get without first having performed the job-wide collection.
 #define PMIX_TIMEOUT                        "pmix.timeout"          // (int) time in sec before specified operation should time out (0 => infinite)
 #define PMIX_IMMEDIATE                      "pmix.immediate"        // (bool) specified operation should immediately return an error from the PMIx
                                                                     //        server if requested data cannot be found - do not request it from
                                                                     //        the host RM
 #define PMIX_WAIT                           "pmix.wait"             // (int) caller requests that the server wait until at least the specified
                                                                     //       #values are found (0 => all and is the default)
-#define PMIX_COLLECTIVE_ALGO                "pmix.calgo"            // (char*) comma-delimited list of algorithms to use for collective
-#define PMIX_COLLECTIVE_ALGO_REQD           "pmix.calreqd"          // (bool) if true, indicates that the requested choice of algo is mandatory
 #define PMIX_NOTIFY_COMPLETION              "pmix.notecomp"         // (bool) notify parent process upon termination of child job
 #define PMIX_RANGE                          "pmix.range"            // (pmix_data_range_t) value for calls to publish/lookup/unpublish or for
                                                                     //        monitoring event notifications
@@ -319,31 +359,41 @@
 #define PMIX_DATA_SCOPE                     "pmix.scope"            // (pmix_scope_t) scope of the data to be found in a PMIx_Get call
 #define PMIX_OPTIONAL                       "pmix.optional"         // (bool) look only in the client's local data store for the requested value - do
                                                                     //        not request data from the server if not found
+#define PMIX_GET_STATIC_VALUES              "pmix.get.static"       // (bool) Request that the data be returned in the provided storage location
+#define PMIX_GET_POINTER_VALUES             "pmix.get.pntrs"        // (bool) Request that any pointers in the returned value point directly
+                                                                    //       to values in the key-value store
 #define PMIX_EMBED_BARRIER                  "pmix.embed.barrier"    // (bool) execute a blocking fence operation before executing the
                                                                     //        specified operation
 #define PMIX_JOB_TERM_STATUS                "pmix.job.term.status"  // (pmix_status_t) status returned upon job termination
+#define PMIX_PROC_TERM_STATUS               "pmix.proc.term.status" // (pmix_status_t) status returned upon process termination
 #define PMIX_PROC_STATE_STATUS              "pmix.proc.state"       // (pmix_proc_state_t) process state
 #define PMIX_GET_REFRESH_CACHE              "pmix.get.refresh"      // (bool) when retrieving data for a remote process, refresh the existing
                                                                     //        local data cache for the process in case new values have been
                                                                     //        put and committed by it since the last refresh
+#define PMIX_ACCESS_PERMISSIONS             "pmix.aperms"           // (pmix_data_array_t*) Define access permissions for the published
+                                                                    //        data. The value shall contain an array of pmix_info_t structs
+                                                                    //        containing the specified permissions.
+#define PMIX_ACCESS_USERIDS                 "pmix.auids"            // (pmix_data_array_t*) Array of effective UIDs that are allowed to
+                                                                    //        access the published data
+#define PMIX_ACCESS_GRPIDS                  "pmix.agids"            // (pmix_data_array_t*) Array of effective GIDs that are allowed to
+                                                                    //        access the published data
+#define PMIX_WAIT_FOR_CONNECTION            "pmix.wait.conn"        // (bool) wait until the specified connection has been made
+
 
 /* attributes used by host server to pass data to/from the server convenience library - the
  * data will then be parsed and provided to the local clients. Not generally accessible by users */
 #define PMIX_REGISTER_NODATA                "pmix.reg.nodata"       // (bool) Registration is for nspace only, do not copy job data
-#define PMIX_PROC_DATA                      "pmix.pdata"            // (pmix_data_array_t*) starts with rank, then contains more data
 #define PMIX_NODE_MAP                       "pmix.nmap"             // (char*) regex of nodes containing procs for this job
+#define PMIX_NODE_MAP_RAW                   "pmix.nmap.raw"         // (char*) comma-delimited list of nodes containing procs for this job
 #define PMIX_PROC_MAP                       "pmix.pmap"             // (char*) regex describing procs on each node within this job
+#define PMIX_PROC_MAP_RAW                   "pmix.pmap.raw"         // (char*) semi-colon delimited list of strings, each string containing
+                                                                    //         a comma-delimited list of ranks on the corresponding node
 #define PMIX_ANL_MAP                        "pmix.anlmap"           // (char*) process mapping in ANL notation (used in PMI-1/PMI-2)
 #define PMIX_APP_MAP_TYPE                   "pmix.apmap.type"       // (char*) type of mapping used to layout the application (e.g., cyclic)
 #define PMIX_APP_MAP_REGEX                  "pmix.apmap.regex"      // (char*) regex describing the result of the mapping
 #define PMIX_REQUIRED_KEY                   "pmix.req.key"          // (char*) key the user needs prior to responding from a dmodex request
 
 
-/* attributes used internally to communicate data from the server to the client */
-#define PMIX_PROC_BLOB                      "pmix.pblob"            // (pmix_byte_object_t) packed blob of process data
-#define PMIX_MAP_BLOB                       "pmix.mblob"            // (pmix_byte_object_t) packed blob of process location
-
-
 /* event handler registration and notification info keys */
 #define PMIX_EVENT_HDLR_NAME                "pmix.evname"           // (char*) string name identifying this handler
 #define PMIX_EVENT_HDLR_FIRST               "pmix.evfirst"          // (bool) invoke this event handler before any other handlers
@@ -366,6 +416,8 @@
 #define PMIX_EVENT_PROXY                    "pmix.evproxy"          // (pmix_proc_t*) PMIx server that sourced the event
 #define PMIX_EVENT_TEXT_MESSAGE             "pmix.evtext"           // (char*) text message suitable for output by recipient - e.g., describing
                                                                     //         the cause of the event
+#define PMIX_EVENT_TIMESTAMP                "pmix.evtstamp"         // (time_t) System time when the associated event occurred.
+
 
 /* fault tolerance-related events */
 #define PMIX_EVENT_TERMINATE_SESSION        "pmix.evterm.sess"      // (bool) RM intends to terminate session
@@ -373,11 +425,6 @@
 #define PMIX_EVENT_TERMINATE_NODE           "pmix.evterm.node"      // (bool) RM intends to terminate all procs on this node
 #define PMIX_EVENT_TERMINATE_PROC           "pmix.evterm.proc"      // (bool) RM intends to terminate just this process
 #define PMIX_EVENT_ACTION_TIMEOUT           "pmix.evtimeout"        // (int) time in sec before RM will execute error response
-#define PMIX_EVENT_NO_TERMINATION           "pmix.evnoterm"         // (bool) indicates that the handler has satisfactorily handled
-                                                                    //        the event and believes termination of the application is not required
-#define PMIX_EVENT_WANT_TERMINATION         "pmix.evterm"           // (bool) indicates that the handler has determined that the
-                                                                    //        application should be terminated
-
 
 /* attributes used to describe "spawn" directives */
 #define PMIX_PERSONALITY                    "pmix.pers"             // (char*) name of personality to use
@@ -387,7 +434,6 @@
 #define PMIX_ADD_HOSTFILE                   "pmix.addhostfile"      // (char*) hostfile to add to existing allocation
 #define PMIX_PREFIX                         "pmix.prefix"           // (char*) prefix to use for starting spawned procs
 #define PMIX_WDIR                           "pmix.wdir"             // (char*) working directory for spawned procs
-#define PMIX_MAPPER                         "pmix.mapper"           // (char*) mapper to use for placing spawned procs
 #define PMIX_DISPLAY_MAP                    "pmix.dispmap"          // (bool) display process map upon spawn
 #define PMIX_PPR                            "pmix.ppr"              // (char*) #procs to spawn on each identified resource
 #define PMIX_MAPBY                          "pmix.mapby"            // (char*) mapping policy
@@ -395,7 +441,6 @@
 #define PMIX_BINDTO                         "pmix.bindto"           // (char*) binding policy
 #define PMIX_PRELOAD_BIN                    "pmix.preloadbin"       // (bool) preload binaries
 #define PMIX_PRELOAD_FILES                  "pmix.preloadfiles"     // (char*) comma-delimited list of files to pre-position
-#define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) spawned procs will not call PMIx_Init
 #define PMIX_STDIN_TGT                      "pmix.stdin"            // (pmix_proc_t*) proc that is to receive stdin
                                                                     //                (PMIX_RANK_WILDCARD = all in given nspace)
 #define PMIX_DEBUGGER_DAEMONS               "pmix.debugger"         // (bool) spawned app consists of debugger daemons
@@ -408,6 +453,8 @@
 #define PMIX_MERGE_STDERR_STDOUT            "pmix.mergeerrout"      // (bool) merge stdout and stderr streams from application procs
 #define PMIX_OUTPUT_TO_FILE                 "pmix.outfile"          // (char*) direct application output into files of form
                                                                     //         "<filename>.rank" with both stdout and stderr redirected into it
+#define PMIX_OUTPUT_TO_DIRECTORY            "pmix.outdir"           // (char*) direct application output into files of form
+                                                                    //         "<directory>/<jobid>/rank.<rank>/stdout[err]"
 #define PMIX_INDEX_ARGV                     "pmix.indxargv"         // (bool) mark the argv with the rank of the proc
 #define PMIX_CPUS_PER_PROC                  "pmix.cpuperproc"       // (uint32_t) #cpus to assign to each rank
 #define PMIX_NO_PROCS_ON_HEAD               "pmix.nolocal"          // (bool) do not place procs on the head node
@@ -425,14 +472,29 @@
                                                                     //        from the spawned processes to this process (typically used by a tool)
 #define PMIX_SPAWN_TOOL                     "pmix.spwn.tool"        // (bool) job being spawned is a tool
 #define PMIX_CMD_LINE                       "pmix.cmd.line"         // (char*) command line executing in the specified nspace
-
-/* query attributes */
-#define PMIX_QUERY_REFRESH_CACHE            "pmix.qry.rfsh"         // (bool) retrieve updated information from server
-                                                                    //        to update local cache
-#define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"           // (char*) return a comma-delimited list of active namespaces
-#define PMIX_QUERY_NAMESPACE_INFO           "pmix.qry.nsinfo"       // (pmix_data_array_t) request an array of active nspace information - each
+#define PMIX_FORKEXEC_AGENT                 "pmix.fe.agnt"          // (char*) command line of fork/exec agent to be used for starting
+                                                                    //         local processes
+#define PMIX_TIMEOUT_STACKTRACES            "pmix.tim.stack"        // (bool) include process stacktraces in timeout report from a job
+#define PMIX_TIMEOUT_REPORT_STATE           "pmix.tim.state"        // (bool) report process states in timeout report from a job
+#define PMIX_APP_ARGV                       "pmix.app.argv"         // (char*) consolidated argv passed to the spawn command for the given app
+#define PMIX_NOTIFY_JOB_EVENTS              "pmix.note.jev"         // (bool) Requests that the launcher generate the PMIX_EVENT_JOB_START,
+                                                                    //        PMIX_LAUNCH_COMPLETE, and PMIX_EVENT_JOB_END events. Each event is to
+                                                                    //        include at least the namespace of the corresponding job and a
+                                                                    //        PMIX_EVENT_TIMESTAMP indicating the time the event occurred.
+#define PMIX_NOTIFY_PROC_TERMINATION        "pmix.noteproc"         // (bool) Requests that the launcher generate the PMIX_EVENT_PROC_TERMINATED
+                                                                    //        event whenever a process either normally or abnormally terminates.
+#define PMIX_NOTIFY_PROC_ABNORMAL_TERMINATION   "pmix.noteabproc"   // (bool) Requests that the launcher generate the PMIX_EVENT_PROC_TERMINATED
+                                                                    //        event only when a process abnormally terminates.
+
+/* query keys - value type shown is the type of the value that will be RETURNED by that key  */
+#define PMIX_QUERY_SUPPORTED_KEYS           "pmix.qry.keys"         // (char*) returns comma-delimited list of keys supported by the query
+                                                                    //         function. NO QUALIFIERS
+#define PMIX_QUERY_NAMESPACES               "pmix.qry.ns"           // (char*) returns a comma-delimited list of active namespaces. NO QUALIFIERS
+#define PMIX_QUERY_NAMESPACE_INFO           "pmix.qry.nsinfo"       // (pmix_data_array_t*) returns an array of active nspace information - each
                                                                     //        element will contain an array including the namespace plus the
                                                                     //        command line of the application executing within it
+                                                                    //        SUPPORTED QUALIFIERS: PMIX_NSPACE of specific nspace whose info
+                                                                    //        is being requested
 #define PMIX_QUERY_JOB_STATUS               "pmix.qry.jst"          // (pmix_status_t) returns status of a specified currently executing job
                                                                     //         REQUIRES a PMIX_NSPACE qualifier indicating the nspace being queried
 #define PMIX_QUERY_QUEUE_LIST               "pmix.qry.qlst"         // (char*) request a comma-delimited list of scheduler queues. NO QUALIFIERS
@@ -452,14 +514,73 @@
 #define PMIX_QUERY_MEMORY_USAGE             "pmix.qry.mem"          // (pmix_data_array_t*) return info on memory usage for the procs indicated in the qualifiers
                                                                     //        SUPPORTED QUALIFIERS: PMIX_NSPACE/PMIX_RANK, or PMIX_PROCID of specific proc(s)
                                                                     //        whose info is being requested
-#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"        // (bool) constrain the query to local information only
-#define PMIX_QUERY_REPORT_AVG               "pmix.qry.avg"          // (bool) report average values
-#define PMIX_QUERY_REPORT_MINMAX            "pmix.qry.minmax"       // (bool) report minimum and maximum value
 #define PMIX_QUERY_ALLOC_STATUS             "pmix.query.alloc"      // (char*) return a string reporting status of an allocation request
                                                                     //         REQUIRES a PMIX_ALLOC_ID qualifier indicating the allocation request being queried
 #define PMIX_TIME_REMAINING                 "pmix.time.remaining"   // (uint32_t) returns number of seconds remaining in allocation
                                                                     //         for the specified nspace (defaults to allocation containing the caller)
                                                                     //         SUPPORTED QUALIFIERS: PMIX_NSPACE of the nspace whose info is being requested
+#define PMIX_QUERY_NUM_PSETS                "pmix.qry.psetnum"      // (size_t) returns the number of psets defined
+                                                                    //          in the specified range (defaults to session)
+                                                                    //         SUPPORTED QUALIFIERS: PMIX_RANGE whose info is being requested
+#define PMIX_QUERY_PSET_NAMES               "pmix.qry.psets"        // (char*) returns a comma-delimited list of the names of the
+                                                                    //         psets defined in the specified range (defaults to session)
+                                                                    //         SUPPORTED QUALIFIERS: PMIX_RANGE whose info is being requested
+#define PMIX_QUERY_PSET_MEMBERSHIP          "pmix.qry.pmems"        // (pmix_data_array_t*) Return an array of pmix_proc_t containing the members of
+                                                                    //         the specified process set.
+#define PMIX_QUERY_NUM_GROUPS               "pmix.qry.pgrpnum"      // (size_t) Return the number of process groups defined in the specified range
+                                                                    //         (defaults to session). OPTIONAL QUALIFERS: PMIX_RANGE.
+#define PMIX_QUERY_GROUP_NAMES              "pmix.qry.pgrp"         // (pmix_data_array_t*) Return a pmix_data_array_t containing an array of string
+                                                                    //         names of the process groups defined in the specified range (defaults
+                                                                    //         to session). OPTIONAL QUALIFERS: PMIX_RANGE
+#define PMIX_QUERY_GROUP_MEMBERSHIP         "pmix.qry.pgrpmems"     // (pmix_data_array_t*) Return a pmix_data_array_t of pmix_proc_t containing
+                                                                    //         the members of the specified process group. REQUIRED QUALIFIERS:
+                                                                    //         PMIX_GROUP_ID.
+#define PMIX_QUERY_ATTRIBUTE_SUPPORT        "pmix.qry.attrs"        // (pmix_data_array_t*) returns array of pmix_info_t where each element consists
+                                                                    //         of a key containing the name of the function, and an array of pmix_regattr_t
+                                                                    //         detailing the attribute support for that function
+                                                                    //         SUPPORTED QUALIFIERS: PMIX_CLIENT_FUNCTIONS, PMIX_SERVER_FUNCTIONS,
+                                                                    //         PMIX_TOOL_FUNCTIONS, and/or PMIX_HOST_FUNCTIONS
+#define PMIX_CLIENT_FUNCTIONS               "pmix.client.fns"       // (char*) returns a comma-delimited list of supported PMIx client functions. NO QUALIFIERS
+#define PMIX_SERVER_FUNCTIONS               "pmix.srvr.fns"         // (char*) returns a comma-delimited list of supported PMIx server functions. NO QUALIFIERS
+#define PMIX_TOOL_FUNCTIONS                 "pmix.tool.fns"         // (char*) returns a comma-delimited list of supported PMIx tool functions. NO QUALIFIERS
+#define PMIX_HOST_FUNCTIONS                 "pmix.host.fns"         // (char*) returns a comma-delimited list of PMIx functions supported by the host environment
+#define PMIX_QUERY_AVAIL_SERVERS            "pmix.qry.asrvrs"       // (pmix_data_array_t*) array of pmix_info_t, each element containing an array of
+                                                                    //         pmix_info_t of available data for servers on this node
+                                                                    //         to which the caller might be able to connect. NO QUALIFIERS
+#define PMIX_QUERY_QUALIFIERS               "pmix.qry.quals"        // (pmix_data_array_t*) Contains an array of qualifiers that were included in the
+                                                                    //         query that produced the provided results. This attribute is solely for
+                                                                    //         reporting purposes and cannot be used in PMIx_Get or other query
+                                                                    //         operations
+#define PMIX_QUERY_RESULTS                  "pmix.qry.res"          // (pmix_data_array_t*) Contains an array of query results for a given pmix_query_t passed to the
+                                                                    //         PMIx_Query_info APIs. If qualifiers were included in the query, then the first element
+                                                                    //         of the array shall be the PMIX_QUERY_QUALIFIERS attribute containing those qualifiers.
+                                                                    //         Each of the remaining elements of the array is a pmix_info_t containing the query key
+                                                                    //         and the corresponding value returned by the query. This attribute is solely for
+                                                                    //         reporting purposes and cannot be used in PMIx_Get or other query operations
+
+
+/* query qualifiers - these are used to provide information to narrow/modify the query. Value type shown is the type of data expected
+ * to be provided with the key */
+#define PMIX_QUERY_REFRESH_CACHE            "pmix.qry.rfsh"         // (bool) retrieve updated information from server
+                                                                    //        to update local cache
+#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"        // (bool) constrain the query to local information only
+#define PMIX_QUERY_REPORT_AVG               "pmix.qry.avg"          // (bool) report average values
+#define PMIX_QUERY_REPORT_MINMAX            "pmix.qry.minmax"       // (bool) report minimum and maximum value
+#define PMIX_CLIENT_ATTRIBUTES              "pmix.client.attrs"     // (char*) comma-delimited list of functions, including "all"
+                                                                    //        when used in a query, indicates whether or not to include
+                                                                    //        attributes supported by the PMIx client library
+#define PMIX_SERVER_ATTRIBUTES              "pmix.srvr.attrs"       // (char*) comma-delimited list of functions, including "all"
+                                                                    //        when used in a query, indicates whether or not to include
+                                                                    //        attributes supported by the PMIx server library
+#define PMIX_HOST_ATTRIBUTES                "pmix.host.attrs"       // (char*) comma-delimited list of functions, including "all"
+                                                                    //        when used in a query, indicates whether or not to include
+                                                                    //        attributes supported by the host environment
+#define PMIX_TOOL_ATTRIBUTES                "pmix.tool.attrs"       // (char*) comma-delimited list of functions, including "all"
+                                                                    //        when used in a query, indicates whether or not to include
+                                                                    //        attributes supported by the PMIx tool library
+#define PMIX_QUERY_SUPPORTED_QUALIFIERS     "pmix.qry.quals"        // (bool) return comma-delimited list of qualifiers supported by
+                                                                    //        a query on the provided key, instead of actually performing
+                                                                    //        the query on the key.
 
 
 /* PMIx_Get information retrieval attributes */
@@ -501,10 +622,20 @@
                                                                     //        information. The PMIX_NSPACE or PMIX_JOBID attributes of the job containing
                                                                     //        the appplication, plus its PMIX_APPNUM attribute, are required to be
                                                                     //        included in the array.
+#define PMIX_PROC_INFO_ARRAY                "pmix.pdata"            // (pmix_data_array_t*) Provide an array of pmix_info_t containing process-realm
+                                                                    //        information. The PMIX_RANK and PMIX_NSPACE attributes, or the
+                                                                    //        PMIX_PROCID attribute, are required to be included in the array when
+                                                                    //        the array is not included as part of a call to
+                                                                    //        PMIx_server_register_nspace - i.e., when the job containing the process
+                                                                    //        is ambiguous. All three may be included if desired. When the array is
+                                                                    //        included in some broader structure that identifies the job, then only
+                                                                    //        the PMIX_RANK or the PMIX_PROCID attribute must be included (the others
+                                                                    //        are optional).
 #define PMIX_NODE_INFO_ARRAY                "pmix.node.arr"         // (pmix_data_array_t*) Provide an array of pmix_info_t containing node-level
                                                                     //        information. At a minimum, either the PMIX_NODEID or PMIX_HOSTNAME
                                                                     //        attribute is required to be included in the array, though both may be
                                                                     //        included.
+#define PMIX_SERVER_INFO_ARRAY              "pmix.srv.arr"          // (pmix_data_array_t*) array of data on a given server, starting with its nspace
 
 
 /* log attributes */
@@ -536,18 +667,44 @@
 
 #define PMIX_LOG_GLOBAL_DATASTORE           "pmix.log.gstore"       // (bool) log the provided data to a global datastore
 #define PMIX_LOG_JOB_RECORD                 "pmix.log.jrec"         // (bool) log the provided information to the RM's job record
+#define PMIX_LOG_PROC_TERMINATION           "pmix.logproc"          // (bool) Requests that the launcher log the PMIX_EVENT_PROC_TERMINATED event
+                                                                    //        whenever a process either normally or abnormally terminates.
+#define PMIX_LOG_PROC_ABNORMAL_TERMINATION  "pmix.logabproc"        // (bool) Requests that the launcher log the PMIX_EVENT_PROC_TERMINATED event
+                                                                    //        only when a process abnormally terminates.
+#define PMIX_LOG_JOB_EVENTS                 "pmix.log.jev"          // (bool) Requests that the launcher log the PMIX_EVENT_JOB_START,
+                                                                    //        PMIX_LAUNCH_COMPLETE, and PMIX_EVENT_JOB_END events using PMIx_Log
+#define PMIX_LOG_COMPLETION                 "pmix.logcomp"          // (bool) Requests that the launcher log the PMIX_EVENT_JOB_END event
+                                                                    //        for normal or abnormal termination of the spawned job using
+                                                                    //        PMIx_Log. The event shall include the returned status code
+                                                                    //        (PMIX_JOB_TERM_STATUS) for the corresponding job; the identity
+                                                                    //        (PMIX_PROCID) and exit status (PMIX_EXIT_CODE) of the first failed
+                                                                    //        process, if applicable; and a PMIX_EVENT_TIMESTAMP indicating the time
+                                                                    //        the termination occurred.
 
 
 /* debugger attributes */
 #define PMIX_DEBUG_STOP_ON_EXEC             "pmix.dbg.exec"         // (bool) job is being spawned under debugger - instruct it to pause on start
 #define PMIX_DEBUG_STOP_IN_INIT             "pmix.dbg.init"         // (bool) instruct job to stop during PMIx init
 #define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"       // (bool) block at desired point until receiving debugger release notification
-#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) nspace of the job assigned to this debugger to be debugged. Note
-                                                                    //         that id's, pids, and other info on the procs is available
-                                                                    //         via a query for the nspace's local or global proctable
-#define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"      // (bool) job to be debugged is waiting for a release
-#define PMIX_DEBUG_JOB_DIRECTIVES           "pmix.dbg.jdirs"        // (pmix_data_array_t*) array of job-level directives
-#define PMIX_DEBUG_APP_DIRECTIVES           "pmix.dbg.adirs"        // (pmix_data_array_t*) array of app-level directives
+#define PMIX_DEBUG_TARGET                   "pmix.dbg.tgt"          // (pmix_proc_t*) Identifier of proc(s) to be debugged
+#define PMIX_DEBUG_DAEMONS_PER_PROC         "pmix.dbg.dpproc"       // (uint16_t) Number of debugger daemons to be spawned per application
+                                                                    //        process. The launcher is to pass the identifier of the namespace to
+                                                                    //        be debugged by including the PMIX_DEBUG_TARGET attribute in the
+                                                                    //        daemon's job-level information. The debugger daemons spawned on a
+                                                                    //        given node are responsible for self-determining their specific
+                                                                    //        target process(es) - e.g., by referencing their own PMIX_LOCAL_RANK
+                                                                    //        in the daemon debugger job versus the corresponding PMIX_LOCAL_RANK
+                                                                    //        of the target processes on the node.
+#define PMIX_DEBUG_DAEMONS_PER_NODE         "pmix.dbg.dpnd"         // (uint16_t) Number of debugger daemons to be spawned on each node where the
+                                                                    //        target job is executing. The launcher is to pass the identifier of
+                                                                    //        the namespace to be debugged by including the PMIX_DEBUG_TARGET
+                                                                    //        attribute in the daemon's job-level information. The debugger
+                                                                    //        daemons spawned on a given node are responsible for
+                                                                    //        self-determining their specific target process(es) - e.g., by
+                                                                    //        referencing their own PMIX_LOCAL_RANK in the daemon debugger job
+                                                                    //        versus the corresponding PMIX_LOCAL_RANK of the target processes on
+                                                                    //        the node.
+
 
 /* Resource Manager identification */
 #define PMIX_RM_NAME                        "pmix.rm.name"          // (char*) string name of the resource manager
@@ -564,8 +721,13 @@
 #define PMIX_APPEND_ENVAR                   "pmix.envar.appnd"        // (pmix_envar_t*) append the given value to the specified
                                                                       //                 envar using the separator character,
                                                                       //                 creating the envar if it doesn't already exist
+#define PMIX_FIRST_ENVAR                    "pmix.envar.first"        // (pmix_envar_t*) ensure the given value appears first in the
+                                                                      //                 specified envar using the separator
+                                                                      //                 character, creating the envar if it doesn't already exist
 
 /* attributes relating to allocations */
+#define PMIX_ALLOC_REQ_ID                   "pmix.alloc.reqid"      // (char*) User-provided string identifier for this allocation request
+                                                                    //         which can later be used to query status of the request.
 #define PMIX_ALLOC_ID                       "pmix.alloc.id"         // (char*) A string identifier (provided by the host environment) for
                                                                     //         the resulting allocation which can later be used to reference
                                                                     //         the allocated resources in, for example, a call to PMIx_Spawn
@@ -575,40 +737,41 @@
 #define PMIX_ALLOC_NUM_CPU_LIST             "pmix.alloc.ncpulist"   // (char*) regex of #cpus for each node
 #define PMIX_ALLOC_CPU_LIST                 "pmix.alloc.cpulist"    // (char*) regex of specific cpus indicating the cpus involved.
 #define PMIX_ALLOC_MEM_SIZE                 "pmix.alloc.msize"      // (float) number of Mbytes
-#define PMIX_ALLOC_NETWORK                  "pmix.alloc.net"        // (pmix_data_array_t*) Array of pmix_info_t describing
-                                                                    //         network resource request. This must include at least:
-                                                                    //           * PMIX_ALLOC_NETWORK_ID
-                                                                    //           * PMIX_ALLOC_NETWORK_TYPE
-                                                                    //           * PMIX_ALLOC_NETWORK_ENDPTS
+#define PMIX_ALLOC_FABRIC                   "pmix.alloc.net"        // (pmix_data_array_t*) Array of pmix_info_t describing
+                                                                    //         fabric resource request. This must include at least:
+                                                                    //           * PMIX_ALLOC_FABRIC_ID
+                                                                    //           * PMIX_ALLOC_FABRIC_TYPE
+                                                                    //           * PMIX_ALLOC_FABRIC_ENDPTS
                                                                     //         plus whatever other descriptors are desired
-#define PMIX_ALLOC_NETWORK_ID               "pmix.alloc.netid"      // (char*) key to be used when accessing this requested network allocation. The
+#define PMIX_ALLOC_FABRIC_ID                "pmix.alloc.netid"      // (char*) key to be used when accessing this requested fabric allocation. The
                                                                     //         allocation will be returned/stored as a pmix_data_array_t of
                                                                     //         pmix_info_t indexed by this key and containing at least one
                                                                     //         entry with the same key and the allocated resource description.
-                                                                    //         The type of the included value depends upon the network
+                                                                    //         The type of the included value depends upon the fabric
                                                                     //         support. For example, a TCP allocation might consist of a
                                                                     //         comma-delimited string of socket ranges such as
                                                                     //         "32000-32100,33005,38123-38146". Additional entries will consist
                                                                     //         of any provided resource request directives, along with their
                                                                     //         assigned values. Examples include:
-                                                                    //           * PMIX_ALLOC_NETWORK_TYPE - the type of resources provided
-                                                                    //           * PMIX_ALLOC_NETWORK_PLANE - if applicable, what plane the
+                                                                    //           * PMIX_ALLOC_FABRIC_TYPE - the type of resources provided
+                                                                    //           * PMIX_ALLOC_FABRIC_PLANE - if applicable, what plane the
                                                                     //               resources were assigned from
-                                                                    //           * PMIX_ALLOC_NETWORK_QOS - the assigned QoS
+                                                                    //           * PMIX_ALLOC_FABRIC_QOS - the assigned QoS
                                                                     //           * PMIX_ALLOC_BANDWIDTH - the allocated bandwidth
-                                                                    //           * PMIX_ALLOC_NETWORK_SEC_KEY - a security key for the requested
-                                                                    //               network allocation
+                                                                    //           * PMIX_ALLOC_FABRIC_SEC_KEY - a security key for the requested
+                                                                    //               fabric allocation
                                                                     //         NOTE: the assigned values may differ from those requested,
                                                                     //         especially if the "required" flag was not set in the request
 #define PMIX_ALLOC_BANDWIDTH                "pmix.alloc.bw"         // (float) Mbits/sec
-#define PMIX_ALLOC_NETWORK_QOS              "pmix.alloc.netqos"     // (char*) quality of service level
+#define PMIX_ALLOC_FABRIC_QOS               "pmix.alloc.netqos"     // (char*) quality of service level
 #define PMIX_ALLOC_TIME                     "pmix.alloc.time"       // (uint32_t) time in seconds that the allocation shall remain valid
-#define PMIX_ALLOC_NETWORK_TYPE             "pmix.alloc.nettype"    // (char*) type of desired transport (e.g., tcp, udp)
-#define PMIX_ALLOC_NETWORK_PLANE            "pmix.alloc.netplane"   // (char*) id string for the NIC (aka plane) to be used for this allocation
+#define PMIX_ALLOC_FABRIC_TYPE              "pmix.alloc.nettype"    // (char*) type of desired transport (e.g., tcp, udp)
+#define PMIX_ALLOC_FABRIC_PLANE             "pmix.alloc.netplane"   // (char*) id string for the NIC (aka plane) to be used for this allocation
                                                                     //         (e.g., CIDR for Ethernet)
-#define PMIX_ALLOC_NETWORK_ENDPTS           "pmix.alloc.endpts"     // (size_t) number of endpoints to allocate per process
-#define PMIX_ALLOC_NETWORK_ENDPTS_NODE      "pmix.alloc.endpts.nd"  // (size_t) number of endpoints to allocate per node
-#define PMIX_ALLOC_NETWORK_SEC_KEY          "pmix.alloc.nsec"       // (pmix_byte_object_t) network security key
+#define PMIX_ALLOC_FABRIC_ENDPTS            "pmix.alloc.endpts"     // (size_t) number of endpoints to allocate per process
+#define PMIX_ALLOC_FABRIC_ENDPTS_NODE       "pmix.alloc.endpts.nd"  // (size_t) number of endpoints to allocate per node
+#define PMIX_ALLOC_FABRIC_SEC_KEY           "pmix.alloc.nsec"       // (pmix_byte_object_t) fabric security key
+#define PMIX_ALLOC_QUEUE                    "pmix.alloc.queue"      // (char*) name of queue being referenced
 
 
 /* job control attributes */
@@ -694,13 +857,175 @@
 #define PMIX_IOF_TAG_OUTPUT                 "pmix.iof.tag"          // (bool) Tag output with the channel it comes from
 #define PMIX_IOF_TIMESTAMP_OUTPUT           "pmix.iof.ts"           // (bool) Timestamp output
 #define PMIX_IOF_XML_OUTPUT                 "pmix.iof.xml"          // (bool) Format output in XML
-#define PMIX_IOF_STOP                       "pmix.iof.stop"         // (bool) Stop forwarding the specified channel(s)
+#define PMIX_IOF_COPY                       "pmix.iof.cpy"          // (bool) Requests that the host environment deliver a copy of the
+                                                                    //        specified output stream(s) to the tool, letting the stream(s)
+                                                                    //        continue to also be delivered to the default location. This
+                                                                    //        allows the tool to tap into the output stream(s) without
+                                                                    //        redirecting it from its current final destination.
+#define PMIX_IOF_REDIRECT                   "pmix.iof.redir"        // (bool) Requests that the host environment intercept the specified
+                                                                    //        output stream(s) and deliver it to the requesting tool instead
+                                                                    //        of its current final destination. This might be used, for
+                                                                    //        example, during a debugging procedure to avoid injection of
+                                                                    //        debugger-related output into the application's results file.
+                                                                    //        The original output stream(s) destination is restored upon
+                                                                    //        termination of the tool.
 
 /* Attributes for controlling contents of application setup data */
 #define PMIX_SETUP_APP_ENVARS               "pmix.setup.env"        // (bool) harvest and include relevant envars
 #define PMIX_SETUP_APP_NONENVARS            "pmix.setup.nenv"       // (bool) include all non-envar data
 #define PMIX_SETUP_APP_ALL                  "pmix.setup.all"        // (bool) include all relevant data
 
+/* Attributes supporting the PMIx Groups APIs */
+#define PMIX_GROUP_ID                       "pmix.grp.id"           // (char*) user-provided group identifier
+#define PMIX_GROUP_LEADER                   "pmix.grp.ldr"          // (bool) this process is the leader of the group
+#define PMIX_GROUP_OPTIONAL                 "pmix.grp.opt"          // (bool) participation is optional - do not return an error if any of the
+                                                                    //        specified processes terminate without having joined. The default
+                                                                    //        is false
+#define PMIX_GROUP_NOTIFY_TERMINATION       "pmix.grp.notterm"      // (bool) notify remaining members when another member terminates without first
+                                                                    //        leaving the group. The default is false
+#define PMIX_GROUP_FT_COLLECTIVE            "pmix.grp.ftcoll"       // (bool) adjust internal tracking for terminated processes. Default is false
+#define PMIX_GROUP_MEMBERSHIP               "pmix.grp.mbrs"         // (pmix_data_array_t*) array of group member ID's
+#define PMIX_GROUP_ASSIGN_CONTEXT_ID        "pmix.grp.actxid"       // (bool) request that the RM assign a unique numerical (size_t) ID to this group
+#define PMIX_GROUP_CONTEXT_ID               "pmix.grp.ctxid"        // (size_t) context ID assigned to group
+#define PMIX_GROUP_LOCAL_ONLY               "pmix.grp.lcl"          // (bool) group operation only involves local procs
+#define PMIX_GROUP_ENDPT_DATA               "pmix.grp.endpt"        // (pmix_byte_object_t) data collected to be shared during construction
+#define PMIX_GROUP_NAMES                    "pmix.pgrp.nm"          // (pmix_data_array_t*) Returns an array of string names of the process groups
+                                                                    //        in which the given process is a member.
+
+
+/* Fabric-related Attributes */
+#define PMIX_FABRIC_COST_MATRIX             "pmix.fab.cm"          // (pointer) Pointer to a two-dimensional array of point-to-point relative
+                                                                   //           communication costs expressed as uint16_t values
+#define PMIX_FABRIC_GROUPS                  "pmix.fab.grps"        // (char*) A string delineating the group membership of nodes in the system,
+                                                                   //         where each fabric group consists of the group number followed by
+                                                                   //         a colon and a comma-delimited list of nodes in that group, with the
+                                                                   //         groups delimited by semi-colons (e.g.,
+                                                                   //         0:node000,node002,node004,node006;1:node001,node003,node005,node007)
+#define PMIX_FABRIC_VENDOR                  "pmix.fab.vndr"        // (char*) Name of fabric vendor (e.g., Amazon, Mellanox, HPE, Intel)
+#define PMIX_FABRIC_IDENTIFIER              "pmix.fab.id"          // (char*) An identifier for the fabric (e.g., MgmtEthernet, Slingshot-11,
+                                                                   //         OmniPath-1)
+#define PMIX_FABRIC_INDEX                   "pmix.fab.idx"         // (size_t) The index of the fabric as returned in pmix_fabric_t
+#define PMIX_FABRIC_COORDINATES             "pmix.fab.coord"       // (pmix_data_array_t*) Array of pmix_geometry_t fabric coordinates for
+                                                                   //          devices on the specified node. The array will contain the
+                                                                   //          coordinates of all devices on the node, including values for
+                                                                   //          all supported coordinate views. The information for devices
+                                                                   //          on the local node shall be provided if the node is not
+                                                                   //          specified in the request.
+#define PMIX_FABRIC_DEVICE_VENDORID         "pmix.fabdev.vendid"   // (char*) This is a vendor-provided identifier for the device or product.
+#define PMIX_FABRIC_NUM_DEVICES             "pmix.fab.nverts"      // (size_t) Total number of fabric devices in the system - corresponds to
+                                                                   //          the number of rows or columns in the cost matrix
+#define PMIX_FABRIC_DIMS                    "pmix.fab.dims"        // (uint32_t) Number of dimensions in the specified fabric plane/view. If no
+                                                                   //          plane is specified in a request, then the dimensions of all planes
+                                                                   //          in the overall system will be returned as a pmix_data_array_t
+                                                                   //          containing an array of uint32_t values. Default is to
+                                                                   //          provide dimensions in logical view.
+
+#define PMIX_FABRIC_PLANE                   "pmix.fab.plane"       // (char*) ID string of a fabric plane (e.g., CIDR for Ethernet). When used as
+                                                                   //          a modifier in a request for information, specifies the plane whose
+                                                                   //          information is to be returned. When used directly as a key in a
+                                                                   //          request, returns a pmix_data_array_t of string
+                                                                   //          identifiers for all fabric planes in the overall system.
+
+#define PMIX_FABRIC_SWITCH                  "pmix.fab.switch"      // (char*) ID string of a fabric switch. When used as a modifier in a request
+                                                                   //         for information, specifies the switch whose information is to be
+                                                                   //         returned. When used directly as a key in a request, returns a
+                                                                   //         pmix_data_array_t of string identifiers for all fabric switches in
+                                                                   //         the overall system.
+
+#define PMIX_FABRIC_ENDPT                   "pmix.fab.endpt"       // (pmix_data_array_t*) Fabric endpoints for a specified process. As multiple
+                                                                   //         endpoints may be assigned to a given process (e.g., in the case
+                                                                   //         where multiple devices are associated with a package to which the
+                                                                   //         process is bound), the returned values will be provided in a
+                                                                   //         pmix_data_array_t of pmix_endpoint_t elements.
+
+#define PMIX_FABRIC_SHAPE                   "pmix.fab.shape"       // (pmix_data_array_t*) The size of each dimension in the specified fabric
+                                                                   //         plane/view, returned in a pmix_data_array_t containing an array of
+                                                                   //         uint32_t values. The size is defined as the number of elements
+                                                                   //         present in that dimension - e.g., the number of devices in one
+                                                                   //         dimension of a physical view of a fabric plane. If no plane is
+                                                                   //         specified, then the shape of each plane in the overall system will
+                                                                   //         be returned in a pmix_data_array_t array where each element is
+                                                                   //         itself a two-element array containing the PMIX_FABRIC_PLANE
+                                                                   //         followed by that plane's fabric shape. Default is to provide the
+                                                                   //         shape in logical view.
+
+#define PMIX_FABRIC_SHAPE_STRING            "pmix.fab.shapestr"    // (char*) Network shape expressed as a string (e.g., "10x12x2"). If no plane
+                                                                   //         is specified, then the shape of each plane in the overall system
+                                                                   //         will be returned in a pmix_data_array_t array where
+                                                                   //         each element is itself a two-element array containing the
+                                                                   //         PMIX_FABRIC_PLANE followed by that plane's fabric shape string.
+                                                                   //         Default is to provide the shape in logical view.
+
+#define PMIX_SWITCH_PEERS                   "pmix.speers"          // (char*) Peer ranks that share the same switch as the process specified in
+                                                                   //         the call to PMIx_Get. Returns a pmix_data_array_t array of
+                                                                   //         pmix_info_t results, each element containing the PMIX_SWITCH_PEERS
+                                                                   //         key with a three-element pmix_data_array_t array of pmix_info_t
+                                                                   //         containing the PMIX_FABRIC_DEVICE_ID of the local fabric device,
+                                                                   //         the PMIX_FABRIC_SWITCH identifying the switch to which it is
+                                                                   //         connected, and a comma-delimited string of peer ranks sharing the
+                                                                   //         switch to which that device is connected.
+#define PMIX_FABRIC_DEVICE                  "pmix.fabdev"          // (pmix_data_array_t*) An array of pmix_info_t describing a particular
+                                                                   //         fabric device. The first element in the array shall be the
+                                                                   //         PMIX_FABRIC_DEVICE_ID of the device
+#define PMIX_FABRIC_DEVICES                 "pmix.fab.devs"        // (pmix_data_array_t*) Array of pmix_info_t containing information for all
+                                                                   //         devices on the specified node. Each element of the array will contain
+                                                                   //         a PMIX_FABRIC_DEVICE entry, which in turn will contain an array of
+                                                                   //         information on a given device.
+#define PMIX_FABRIC_DEVICE_NAME             "pmix.fabdev.nm"       // (char*) The operating system name associated with the device. This may be
+                                                                   //         a logical fabric interface name (e.g. eth0 or eno1) or an absolute
+                                                                   //         filename.
+#define PMIX_FABRIC_DEVICE_INDEX            "pmix.fabdev.idx"      // (uint32_t) Index of the device within an associated communication cost
+                                                                   //         matrix.
+
+#define PMIX_FABRIC_DEVICE_VENDOR           "pmix.fabdev.vndr"     // (char*) Indicates the name of the vendor that distributes the NIC.
+#define PMIX_FABRIC_DEVICE_BUS_TYPE         "pmix.fabdev.btyp"     // (char*) The type of bus to which the device is attached (e.g., "PCI",
+                                                                   //         "GEN-Z").
+#define PMIX_FABRIC_DEVICE_DRIVER           "pmix.fabdev.driver"   // (char*) The name of the driver associated with the device
+#define PMIX_FABRIC_DEVICE_FIRMWARE         "pmix.fabdev.fmwr"     // (char*) The device's firmware version
+#define PMIX_FABRIC_DEVICE_ADDRESS          "pmix.fabdev.addr"     // (char*) The primary link-level address associated with the device, such as a
+                                                                   //         MAC address. If multiple addresses are available, only one will be
+                                                                   //         reported.
+#define PMIX_FABRIC_DEVICE_COORDINATES      "pmix.fab.coord"       // (pmix_geometry_t) The pmix_geometry_t fabric coordinates for the device, including
+                                                                   //          values for all supported coordinate views.
+#define PMIX_FABRIC_DEVICE_MTU              "pmix.fabdev.mtu"      // (size_t) The maximum transfer unit of link level frames or packets,
+                                                                   //          in bytes.
+#define PMIX_FABRIC_DEVICE_SPEED            "pmix.fabdev.speed"    // (size_t) The active link data rate, given in bits per second.
+#define PMIX_FABRIC_DEVICE_STATE            "pmix.fabdev.state"    // (pmix_link_state_t) The last available physical port state. Possible values
+                                                                   //          are PMIX_LINK_STATE_UNKNOWN, PMIX_LINK_DOWN, and PMIX_LINK_UP, to
+                                                                   //          indicate if the port state is unknown or not applicable (unknown),
+                                                                   //          inactive (down), or active (up).
+#define PMIX_FABRIC_DEVICE_TYPE             "pmix.fabdev.type"     // (char*) Specifies the type of fabric interface currently active on the
+                                                                   //         device, such as Ethernet or InfiniBand.
+#define PMIX_FABRIC_DEVICE_PCI_DEVID        "pmix.fabdev.pcidevid" // (char*) A node-level unique identifier for a PCI device. Provided only if the
+                                                                   //         device is located on a \ac{PCI} bus. The identifier is constructed as
+                                                                   //         a four-part tuple delimited by colons comprised of the \ac{PCI} 16-bit
+                                                                   //         domain, 8-bit bus, 8-bit device, and 8-bit function IDs, each expressed
+                                                                   //         in zero-extended hexadecimal form. Thus, an example identifier might be
+                                                                   //         "abc1:0f:23:01". The combination of node identifier PMIX_HOSTNAME or
+                                                                   //         PMIX_NODEID and PMIX_FABRIC_DEVICE_PCI_DEVID shall be unique within the
+                                                                   //         system.
+
+
+/* Distance Attributes */
+#define PMIX_DEVICE_DISTANCES               "pmix.dev.dist"        // (pmix_data_array_t*) Return an array of pmix_device_dist_t containing the
+                                                                   //         minimum and maximum distances of the given process location to all
+                                                                   //         devices of the specified type on the local node.
+#define PMIX_DEVICE_TYPE                    "pmix.dev.type"        // (pmix_device_type_t) Bitmask specifying the type(s) of device(s) whose
+                                                                   //         information is being requested. Only used as a directive/qualifier.
+#define PMIX_DEVICE_ID                      "pmix.dev.id"          // (char*) System-wide UUID or node-local OS name of a particular device.
+
+
+/* Descriptive Attributes */
+#define PMIX_MAX_VALUE                      "pmix.descr.maxval"    // (varies) Used in pmix_regattr_t to describe the maximum valid value
+                                                                   //          for the associated attribute.
+#define PMIX_MIN_VALUE                      "pmix.descr.minval"    // (varies) Used in pmix_regattr_t to describe the minimum valid value
+                                                                   //          for the associated attribute.
+#define PMIX_ENUM_VALUE                     "pmix.descr.enum"      // (char*) Used in pmix_regattr_t to describe accepted values for the
+                                                                   //         associated attribute. Numerical values shall be presented in
+                                                                   //         a form convertible to the attribute's declared data type.
+                                                                   //         Named values (i.e., values defined by constant names via a
+                                                                   //         typical C-language enum declaration) must be provided as
+                                                                   //         their numerical equivalent.
 
 /****    PROCESS STATE DEFINITIONS    ****/
 typedef uint8_t pmix_proc_state_t;
@@ -740,10 +1065,35 @@
 #define PMIX_PROC_STATE_FAILED_TO_LAUNCH        (PMIX_PROC_STATE_ERROR + 13)  /* unable to launch process */
 
 
+/****    JOB STATE DEFINITIONS    ****/
+typedef uint8_t pmix_job_state_t;
+#define PMIX_JOB_STATE_UNDEF                     0  // undefined process state
+#define PMIX_JOB_STATE_AWAITING_ALLOC            1  // Job is waiting for resources to be allocated to it
+#define PMIX_JOB_STATE_LAUNCH_UNDERWAY           2  // job launch underway
+#define PMIX_JOB_STATE_RUNNING                   3  // all procs have been spawned
+#define PMIX_JOB_STATE_SUSPENDED                 4  // job has been suspended
+#define PMIX_JOB_STATE_CONNECTED                 5  // all procs have connected to their PMIx server
+
+/*
+* Define a "boundary" so users can easily and quickly determine
+* if a job is still running or not - any value less than
+* this one means that the job has not terminated
+*/
+#define PMIX_JOB_STATE_UNTERMINATED             15
+
+#define PMIX_JOB_STATE_TERMINATED               20  // job has terminated and is no longer running - typically will
+                                                    // be accompanied by the job exit status in response to a query
+
+/* Define a boundary so users can easily and quickly determine
+* if a job abnormally terminated - leave a little room
+* for future expansion
+*/
+#define PMIX_JOB_STATE_TERMINATED_WITH_ERROR    50  // job has terminated and is no longer running - typically will
+                                                    // be accompanied by a job-related error code in response to a query
+
+
 /****    PMIX ERROR CONSTANTS    ****/
 /* PMIx errors are always negative, with 0 reserved for success */
-#define PMIX_ERR_BASE                   0
-
 typedef int pmix_status_t;
 
 /* v1.x error values - must be fixed in place for backward
@@ -753,109 +1103,126 @@
  * at least defined to ensure older codes will compile */
 #define PMIX_SUCCESS                                 0
 #define PMIX_ERROR                                  -1          // general error
-#define PMIX_ERR_SILENT                             -2
-/* debugger release flag */
-#define PMIX_ERR_DEBUGGER_RELEASE                   -3
 /* fault tolerance */
 #define PMIX_ERR_PROC_RESTART                       -4
 #define PMIX_ERR_PROC_CHECKPOINT                    -5
 #define PMIX_ERR_PROC_MIGRATE                       -6
-/* abort */
-#define PMIX_ERR_PROC_ABORTED                       -7
-#define PMIX_ERR_PROC_REQUESTED_ABORT               -8
-#define PMIX_ERR_PROC_ABORTING                      -9
+#define PMIX_ERR_EXISTS                             -11
 /* communication failures */
-#define PMIX_ERR_SERVER_FAILED_REQUEST              -10
-#define PMIX_EXISTS                                 -11
 #define PMIX_ERR_INVALID_CRED                       -12
-#define PMIX_ERR_HANDSHAKE_FAILED                   -13
-#define PMIX_ERR_READY_FOR_HANDSHAKE                -14
 #define PMIX_ERR_WOULD_BLOCK                        -15
 #define PMIX_ERR_UNKNOWN_DATA_TYPE                  -16
-#define PMIX_ERR_PROC_ENTRY_NOT_FOUND               -17
 #define PMIX_ERR_TYPE_MISMATCH                      -18
 #define PMIX_ERR_UNPACK_INADEQUATE_SPACE            -19
 #define PMIX_ERR_UNPACK_FAILURE                     -20
 #define PMIX_ERR_PACK_FAILURE                       -21
-#define PMIX_ERR_PACK_MISMATCH                      -22
 #define PMIX_ERR_NO_PERMISSIONS                     -23
 #define PMIX_ERR_TIMEOUT                            -24
 #define PMIX_ERR_UNREACH                            -25
-#define PMIX_ERR_IN_ERRNO                           -26
 #define PMIX_ERR_BAD_PARAM                          -27
 #define PMIX_ERR_RESOURCE_BUSY                      -28
 #define PMIX_ERR_OUT_OF_RESOURCE                    -29
-#define PMIX_ERR_DATA_VALUE_NOT_FOUND               -30
 #define PMIX_ERR_INIT                               -31
 #define PMIX_ERR_NOMEM                              -32
-#define PMIX_ERR_INVALID_ARG                        -33
-#define PMIX_ERR_INVALID_KEY                        -34
-#define PMIX_ERR_INVALID_KEY_LENGTH                 -35
-#define PMIX_ERR_INVALID_VAL                        -36
-#define PMIX_ERR_INVALID_VAL_LENGTH                 -37
-#define PMIX_ERR_INVALID_LENGTH                     -38
-#define PMIX_ERR_INVALID_NUM_ARGS                   -39
-#define PMIX_ERR_INVALID_ARGS                       -40
-#define PMIX_ERR_INVALID_NUM_PARSED                 -41
-#define PMIX_ERR_INVALID_KEYVALP                    -42
-#define PMIX_ERR_INVALID_SIZE                       -43
-#define PMIX_ERR_INVALID_NAMESPACE                  -44
-#define PMIX_ERR_SERVER_NOT_AVAIL                   -45
 #define PMIX_ERR_NOT_FOUND                          -46
 #define PMIX_ERR_NOT_SUPPORTED                      -47
-#define PMIX_ERR_NOT_IMPLEMENTED                    -48
+#define PMIX_ERR_PARAM_VALUE_NOT_SUPPORTED          -59
 #define PMIX_ERR_COMM_FAILURE                       -49
 #define PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER     -50
 #define PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES     -51
+#define PMIX_ERR_PARTIAL_SUCCESS                    -52
+#define PMIX_ERR_DUPLICATE_KEY                      -53
+#define PMIX_ERR_EMPTY                              -60
+#define PMIX_ERR_LOST_CONNECTION                    -61
+#define PMIX_ERR_EXISTS_OUTSIDE_SCOPE               -62
+
+/* Process set */
+#define PMIX_PROCESS_SET_DEFINE                     -55
+#define PMIX_PROCESS_SET_DELETE                     -56
+
+/* Debugger ops */
+#define PMIX_DEBUGGER_RELEASE                       -57
+#define PMIX_DEBUG_WAITING_FOR_NOTIFY               -58
 
-/* define a starting point for v2.x error values */
-#define PMIX_ERR_V2X_BASE                   -100
-
-/* v2.x communication errors */
-#define PMIX_ERR_LOST_CONNECTION_TO_SERVER          -101
-#define PMIX_ERR_LOST_PEER_CONNECTION               -102
-#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT          -103
-/* used by the query system */
+/* query errors */
 #define PMIX_QUERY_PARTIAL_SUCCESS                  -104
-/* request responses */
-#define PMIX_NOTIFY_ALLOC_COMPLETE                  -105
+
 /* job control */
 #define PMIX_JCTRL_CHECKPOINT                       -106    // monitored by client to trigger checkpoint operation
 #define PMIX_JCTRL_CHECKPOINT_COMPLETE              -107    // sent by client and monitored by server to notify that requested
-                                                                            //     checkpoint operation has completed
+                                                            //     checkpoint operation has completed
 #define PMIX_JCTRL_PREEMPT_ALERT                    -108    // monitored by client to detect RM intends to preempt
 
 /* monitoring */
 #define PMIX_MONITOR_HEARTBEAT_ALERT                -109
 #define PMIX_MONITOR_FILE_ALERT                     -110
 #define PMIX_PROC_TERMINATED                        -111
-#define PMIX_ERR_INVALID_TERMINATION                -112
 
 /* operational */
 #define PMIX_ERR_EVENT_REGISTRATION                 -144
-#define PMIX_ERR_JOB_TERMINATED                     -145
-#define PMIX_ERR_UPDATE_ENDPOINTS                   -146
 #define PMIX_MODEL_DECLARED                         -147
-#define PMIX_GDS_ACTION_COMPLETE                    -148
-#define PMIX_PROC_HAS_CONNECTED                     -149
-#define PMIX_CONNECT_REQUESTED                      -150
 #define PMIX_MODEL_RESOURCES                        -151     // model resource usage has changed
 #define PMIX_OPENMP_PARALLEL_ENTERED                -152     // an OpenMP parallel region has been entered
 #define PMIX_OPENMP_PARALLEL_EXITED                 -153     // an OpenMP parallel region has completed
-#define PMIX_LAUNCH_DIRECTIVE                       -154
 #define PMIX_LAUNCHER_READY                         -155
 #define PMIX_OPERATION_IN_PROGRESS                  -156
 #define PMIX_OPERATION_SUCCEEDED                    -157
 #define PMIX_ERR_INVALID_OPERATION                  -158
+#define PMIX_GROUP_INVITED                          -159
+#define PMIX_GROUP_LEFT                             -160
+#define PMIX_GROUP_INVITE_ACCEPTED                  -161
+#define PMIX_GROUP_INVITE_DECLINED                  -162
+#define PMIX_GROUP_INVITE_FAILED                    -163
+#define PMIX_GROUP_MEMBERSHIP_UPDATE                -164
+#define PMIX_GROUP_CONSTRUCT_ABORT                  -165
+#define PMIX_GROUP_CONSTRUCT_COMPLETE               -166
+#define PMIX_GROUP_LEADER_SELECTED                  -167
+#define PMIX_GROUP_LEADER_FAILED                    -168
+#define PMIX_GROUP_CONTEXT_ID_ASSIGNED              -169
+#define PMIX_GROUP_MEMBER_FAILED                    -170
+#define PMIX_ERR_REPEAT_ATTR_REGISTRATION           -171
+#define PMIX_ERR_IOF_FAILURE                        -172
+#define PMIX_ERR_IOF_COMPLETE                       -173
+#define PMIX_LAUNCH_COMPLETE                        -174     // include nspace of the launched job with notification
+#define PMIX_FABRIC_UPDATED                         -175
+#define PMIX_FABRIC_UPDATE_PENDING                  -176
+#define PMIX_FABRIC_UPDATE_ENDPOINTS                 -57
+
+/* job-related errors */
+#define PMIX_ERR_JOB_APP_NOT_EXECUTABLE             -177
+#define PMIX_ERR_JOB_NO_EXE_SPECIFIED               -178
+#define PMIX_ERR_JOB_FAILED_TO_MAP                  -179
+#define PMIX_ERR_JOB_CANCELED                       -180
+#define PMIX_ERR_JOB_FAILED_TO_LAUNCH               -181
+#define PMIX_ERR_JOB_ABORTED                        -182
+#define PMIX_ERR_JOB_KILLED_BY_CMD                  -183
+#define PMIX_ERR_JOB_ABORTED_BY_SIG                 -184
+#define PMIX_ERR_JOB_TERM_WO_SYNC                   -185
+#define PMIX_ERR_JOB_SENSOR_BOUND_EXCEEDED          -186
+#define PMIX_ERR_JOB_NON_ZERO_TERM                  -187
+#define PMIX_ERR_JOB_ALLOC_FAILED                   -188
+#define PMIX_ERR_JOB_ABORTED_BY_SYS_EVENT           -189
+
+/* job-related non-error events */
+#define PMIX_EVENT_JOB_START                        -191
+#define PMIX_EVENT_JOB_END                          -145
+#define PMIX_EVENT_SESSION_START                    -192
+#define PMIX_EVENT_SESSION_END                      -193
+
+/* process-related events */
+#define PMIX_ERR_PROC_TERM_WO_SYNC                  -200
+#define PMIX_EVENT_PROC_TERMINATED                  -201
 
 /* system failures */
-#define PMIX_ERR_NODE_DOWN                          -231
-#define PMIX_ERR_NODE_OFFLINE                       -232
-#define PMIX_ERR_SYS_OTHER                          -330
+#define PMIX_EVENT_SYS_BASE                         -230
+#define PMIX_EVENT_NODE_DOWN                        -231
+#define PMIX_EVENT_NODE_OFFLINE                     -232
+#define PMIX_EVENT_SYS_OTHER                        -330
+
 
 /* define a macro for identifying system event values */
 #define PMIX_SYSTEM_EVENT(a)   \
-    ((a) <= PMIX_ERR_NODE_DOWN && PMIX_ERR_SYS_OTHER <= (a))
+    ((a) <= PMIX_EVENT_SYS_BASE && PMIX_EVENT_SYS_OTHER <= (a))
 
 /* used by event handlers */
 #define PMIX_EVENT_NO_ACTION_TAKEN                  -331
@@ -863,10 +1230,6 @@
 #define PMIX_EVENT_ACTION_DEFERRED                  -333
 #define PMIX_EVENT_ACTION_COMPLETE                  -334
 
-/* define a starting point for PMIx internal error codes
- * that are never exposed outside the library */
-#define PMIX_INTERNAL_ERR_BASE                      -1330
-
 /* define a starting point for user-level defined error
  * constants - negative values larger than this are guaranteed
  * not to conflict with PMIx values. Definitions should always
@@ -876,56 +1239,68 @@
 
 /****    PMIX DATA TYPES    ****/
 typedef uint16_t pmix_data_type_t;
-#define PMIX_UNDEF               0
-#define PMIX_BOOL                1  // converted to/from native true/false to uint8 for pack/unpack
-#define PMIX_BYTE                2  // a byte of data
-#define PMIX_STRING              3  // NULL-terminated string
-#define PMIX_SIZE                4  // size_t
-#define PMIX_PID                 5  // OS-pid
-#define PMIX_INT                 6
-#define PMIX_INT8                7
-#define PMIX_INT16               8
-#define PMIX_INT32               9
-#define PMIX_INT64              10
-#define PMIX_UINT               11
-#define PMIX_UINT8              12
-#define PMIX_UINT16             13
-#define PMIX_UINT32             14
-#define PMIX_UINT64             15
-#define PMIX_FLOAT              16
-#define PMIX_DOUBLE             17
-#define PMIX_TIMEVAL            18
-#define PMIX_TIME               19
-#define PMIX_STATUS             20  // needs to be tracked separately from integer for those times
-                                    // when we are embedded and it needs to be converted to the
-                                    // host error definitions
-#define PMIX_VALUE              21
-#define PMIX_PROC               22
-#define PMIX_APP                23
-#define PMIX_INFO               24
-#define PMIX_PDATA              25
-#define PMIX_BUFFER             26
-#define PMIX_BYTE_OBJECT        27
-#define PMIX_KVAL               28
+#define PMIX_UNDEF                       0
+#define PMIX_BOOL                        1  // converted to/from native true/false to uint8 for pack/unpack
+#define PMIX_BYTE                        2  // a byte of data
+#define PMIX_STRING                      3  // NULL-terminated string
+#define PMIX_SIZE                        4  // size_t
+#define PMIX_PID                         5  // OS-pid
+#define PMIX_INT                         6
+#define PMIX_INT8                        7
+#define PMIX_INT16                       8
+#define PMIX_INT32                       9
+#define PMIX_INT64                      10
+#define PMIX_UINT                       11
+#define PMIX_UINT8                      12
+#define PMIX_UINT16                     13
+#define PMIX_UINT32                     14
+#define PMIX_UINT64                     15
+#define PMIX_FLOAT                      16
+#define PMIX_DOUBLE                     17
+#define PMIX_TIMEVAL                    18
+#define PMIX_TIME                       19
+#define PMIX_STATUS                     20  // needs to be tracked separately from integer for those times
+                                            // when we are embedded and it needs to be converted to the
+                                            // host error definitions
+#define PMIX_VALUE                      21
+#define PMIX_PROC                       22
+#define PMIX_APP                        23
+#define PMIX_INFO                       24
+#define PMIX_PDATA                      25
+#define PMIX_BUFFER                     26
+#define PMIX_BYTE_OBJECT                27
+#define PMIX_KVAL                       28
 // Hole left by deprecation/removal of PMIX_MODEX
-#define PMIX_PERSIST            30
-#define PMIX_POINTER            31
-#define PMIX_SCOPE              32
-#define PMIX_DATA_RANGE         33
-#define PMIX_COMMAND            34
-#define PMIX_INFO_DIRECTIVES    35
-#define PMIX_DATA_TYPE          36
-#define PMIX_PROC_STATE         37
-#define PMIX_PROC_INFO          38
-#define PMIX_DATA_ARRAY         39
-#define PMIX_PROC_RANK          40
-#define PMIX_QUERY              41
-#define PMIX_COMPRESSED_STRING  42  // string compressed with zlib
-#define PMIX_ALLOC_DIRECTIVE    43
+#define PMIX_PERSIST                    30
+#define PMIX_POINTER                    31
+#define PMIX_SCOPE                      32
+#define PMIX_DATA_RANGE                 33
+#define PMIX_COMMAND                    34
+#define PMIX_INFO_DIRECTIVES            35
+#define PMIX_DATA_TYPE                  36
+#define PMIX_PROC_STATE                 37
+#define PMIX_PROC_INFO                  38
+#define PMIX_DATA_ARRAY                 39
+#define PMIX_PROC_RANK                  40
+#define PMIX_QUERY                      41
+#define PMIX_COMPRESSED_STRING          42  // string compressed with zlib
+#define PMIX_ALLOC_DIRECTIVE            43
 // Hole left by deprecation/removal of PMIX_INFO_ARRAY
-#define PMIX_IOF_CHANNEL        45
-#define PMIX_ENVAR              46
-#define PMIX_REGEX              49 // numerical ID matches v4
+#define PMIX_IOF_CHANNEL                45
+#define PMIX_ENVAR                      46
+#define PMIX_COORD                      47
+#define PMIX_REGATTR                    48
+#define PMIX_REGEX                      49
+#define PMIX_JOB_STATE                  50
+#define PMIX_LINK_STATE                 51
+#define PMIX_PROC_CPUSET                52
+#define PMIX_GEOMETRY                   53
+#define PMIX_DEVICE_DIST                54
+#define PMIX_ENDPOINT                   55
+#define PMIX_TOPO                       56
+#define PMIX_DEVTYPE                    57
+#define PMIX_LOCTYPE                    58
+#define PMIX_COMPRESSED_BYTE_OBJECT     59
 /********************/
 
 /* define a boundary for implementers so they can add their own data types */
@@ -1009,6 +1384,20 @@
 #define PMIX_FWD_STDDIAG_CHANNEL    0x0008
 #define PMIX_FWD_ALL_CHANNELS       0x00ff
 
+/* define values associated with PMIx_Group_join
+ * to indicate accept and decline - this is
+ * done for readability of user code */
+typedef enum {
+    PMIX_GROUP_DECLINE,
+    PMIX_GROUP_ACCEPT
+} pmix_group_opt_t;
+
+typedef enum {
+    PMIX_GROUP_CONSTRUCT,
+    PMIX_GROUP_DESTRUCT
+} pmix_group_operation_t;
+
+
 /* define some "hooks" external libraries can use to
  * intercept memory allocation/release operations */
 static inline void* pmix_malloc(size_t n)
@@ -1030,10 +1419,13 @@
 #define PMIX_CHECK_KEY(a, b) \
     (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN))
 
-#define PMIX_LOAD_KEY(a, b) \
-    do {                                            \
-        memset((a), 0, PMIX_MAX_KEYLEN+1);          \
-        pmix_strncpy((a), (b), PMIX_MAX_KEYLEN);    \
+#define PMIX_CHECK_RESERVED_KEY(a) \
+    (0 == strncmp((a), "pmix", 4))
+
+#define PMIX_LOAD_KEY(a, b)                                             \
+    do {                                                                \
+        memset((a), 0, PMIX_MAX_KEYLEN+1);                              \
+        pmix_strncpy((char*)(a), (const char*)(b), PMIX_MAX_KEYLEN);    \
     }while(0)
 
 /* define a convenience macro for loading nspaces */
@@ -1060,6 +1452,234 @@
 #define PMIX_CHECK_PROCID(a, b) \
     (PMIX_CHECK_NSPACE((a)->nspace, (b)->nspace) && ((a)->rank == (b)->rank || (PMIX_RANK_WILDCARD == (a)->rank || PMIX_RANK_WILDCARD == (b)->rank)))
 
+#define PMIX_CHECK_RANK(a, b) \
+    ((a) == (b) || (PMIX_RANK_WILDCARD == (a) || PMIX_RANK_WILDCARD == (b)))
+
+/****    PMIX COORD    ****/
+/* define coordinate system views */
+typedef uint8_t pmix_coord_view_t;
+#define PMIX_COORD_VIEW_UNDEF       0x00
+#define PMIX_COORD_LOGICAL_VIEW     0x01
+#define PMIX_COORD_PHYSICAL_VIEW    0x02
+
+/* define a structure for a proc's fabric coordinate */
+typedef struct pmix_coord {
+    pmix_coord_view_t view;
+    uint32_t *coord;
+    size_t dims;
+} pmix_coord_t;
+
+#define PMIX_COORD_CREATE(m, d, n)                                      \
+    do {                                                                \
+        pmix_coord_t *_m;                                               \
+        _m = (pmix_coord_t*)pmix_calloc((d), sizeof(pmix_coord_t));     \
+        if (NULL != _m) {                                               \
+            _m->view = PMIX_COORD_VIEW_UNDEF;                           \
+            _m->dims = (n);                                             \
+            _m->coord = (uint32_t*)pmix_calloc((n), sizeof(uint32_t));  \
+            (m) = _m;                                                   \
+        }                                                               \
+    } while(0)
+
+#define PMIX_COORD_CONSTRUCT(m)             \
+    do {                                    \
+        (m)->view = PMIX_COORD_VIEW_UNDEF;  \
+        (m)->coord = NULL;                  \
+        (m)->dims = 0;                      \
+    } while(0)
+
+#define PMIX_COORD_DESTRUCT(m)              \
+    do {                                    \
+        (m)->view = PMIX_COORD_VIEW_UNDEF;  \
+        if (NULL != (m)->coord) {           \
+            pmix_free((m)->coord);          \
+            (m)->coord = NULL;              \
+            (m)->dims = 0;                  \
+        }                                   \
+    } while(0)
+
+#define PMIX_COORD_FREE(m, n)                       \
+    do {                                            \
+        size_t _nc_;                                \
+        if (NULL != (m)) {                          \
+            for (_nc_ = 0; _nc_ < (n); _nc_++) {    \
+                PMIX_COORD_DESTRUCT(&(m)[_nc_]);    \
+            }                                       \
+            free((m));                              \
+            (m) = NULL;                             \
+        }                                           \
+    } while(0)
+
+
+/****    PMIX LINK STATES    ****/
+typedef uint8_t pmix_link_state_t;
+#define PMIX_LINK_STATE_UNKNOWN     0  // The port state is unknown or not applicable
+#define PMIX_LINK_DOWN              1  // The port is inactive.
+#define PMIX_LINK_UP                2  // The port is active.
+
+
+/****   PMIX CPUSET    ****/
+typedef struct{
+    char *source;
+    void *bitmap;
+} pmix_cpuset_t;
+
+#define PMIX_CPUSET_CONSTRUCT(m) \
+    memset((m), 0, sizeof(pmix_cpuset_t))
+
+#define PMIX_CPUSET_DESTRUCT(m) \
+    pmix_ploc_base_destruct_cpuset((m))
+
+#define PMIX_CPUSET_CREATE(m, n)    \
+    (m) = (pmix_cpuset_t*)calloc((n), sizeof(pmix_cpuset_t));
+
+#define PMIX_CPUSET_FREE(m, n) \
+    pmix_ploc_base_release_cpuset((m), (n))
+
+
+/****    PMIX BIND ENVELOPE    ****/
+typedef uint8_t pmix_bind_envelope_t;
+#define PMIX_CPUBIND_PROCESS    0
+#define PMIX_CPUBIND_THREAD     1
+
+
+/****    PMIX TOPOLOGY    ****/
+typedef struct {
+    char *source;
+    void *topology;
+} pmix_topology_t;
+
+#define PMIX_TOPOLOGY_CONSTRUCT(m) \
+    memset((m), 0, sizeof(pmix_topology_t))
+
+#define PMIX_TOPOLOGY_DESTRUCT(m) \
+    pmix_ploc_base_destruct_topology((m))
+
+#define PMIX_TOPOLOGY_CREATE(m, n) \
+    (m) = (pmix_topology_t*)calloc(n, sizeof(pmix_topology_t))
+
+#define PMIX_TOPOLOGY_FREE(m, n) \
+    pmix_ploc_base_release_topology((m), (n))
+
+/**** PMIX RELATIVE LOCALITY    ****/
+typedef uint16_t pmix_locality_t;
+#define PMIX_LOCALITY_UNKNOWN           0x0000
+#define PMIX_LOCALITY_NONLOCAL          0x8000
+#define PMIX_LOCALITY_SHARE_HWTHREAD    0x0001
+#define PMIX_LOCALITY_SHARE_CORE        0x0002
+#define PMIX_LOCALITY_SHARE_L1CACHE     0x0004
+#define PMIX_LOCALITY_SHARE_L2CACHE     0x0008
+#define PMIX_LOCALITY_SHARE_L3CACHE     0x0010
+#define PMIX_LOCALITY_SHARE_PACKAGE     0x0020
+#define PMIX_LOCALITY_SHARE_NUMA        0x0040
+#define PMIX_LOCALITY_SHARE_NODE        0x4000
+
+
+/****    PMIX GEOMETRY     ****/
+typedef struct pmix_geometry {
+    size_t fabric;
+    char *uuid;
+    char *osname;
+    pmix_coord_t *coordinates;
+    size_t ncoords;
+} pmix_geometry_t;
+
+#define PMIX_GEOMETRY_CONSTRUCT(m) \
+    memset((m), 0, sizeof(pmix_geometry_t));
+
+#define PMIX_GEOMETRY_DESTRUCT(m)                               \
+    do {                                                        \
+        if (NULL != (m)->uuid) {                                \
+            free((m)->uuid);                                    \
+            (m)->uuid = NULL;                                   \
+        }                                                       \
+        if (NULL != (m)->osname) {                              \
+            free((m)->osname);                                  \
+            (m)->osname = NULL;                                 \
+        }                                                       \
+        if (NULL != (m)->coordinates) {                         \
+            PMIX_COORD_FREE((m)->coordinates, (m)->ncoords);    \
+        }                                                       \
+    } while(0)
+
+#define PMIX_GEOMETRY_CREATE(m, n)                              \
+    (m) = (pmix_geometry_t*)calloc((n), sizeof(pmix_geometry_t))
+
+#define PMIX_GEOMETRY_FREE(m, n)                    \
+    do {                                            \
+        size_t _i;                                  \
+        if (NULL != (m)) {                          \
+            for (_i=0; _i < (n); _i++) {            \
+                PMIX_GEOMETRY_DESTRUCT(&(m)[_i]);   \
+            }                                       \
+            free((m));                              \
+            (m) = NULL;                             \
+        }                                           \
+    } while(0)
+
+
+/****    PMIX_DEVICE_TYPE    ****/
+typedef uint64_t pmix_device_type_t;
+#define PMIX_DEVTYPE_UNKNOWN        0x0
+#define PMIX_DEVTYPE_BLOCK          0x1
+#define PMIX_DEVTYPE_GPU            0x2
+#define PMIX_DEVTYPE_NETWORK        0x4
+#define PMIX_DEVTYPE_OPENFABRICS    0x8
+#define PMIX_DEVTYPE_DMA            0x10
+#define PMIX_DEVTYPE_COPROC         0x20
+
+/****    PMIX_DISTANCE     ****/
+typedef struct pmix_device_distance {
+    char *uuid;
+    char *osname;
+    pmix_device_type_t type;
+    uint16_t mindist;
+    uint16_t maxdist;
+} pmix_device_distance_t;
+
+#define PMIX_DEVICE_DIST_CONSTRUCT(m)                       \
+    do {                                                    \
+        memset((m), 0, sizeof(pmix_device_distance_t));     \
+        (m)->mindist = UINT16_MAX;                          \
+        (m)->maxdist = UINT16_MAX;                          \
+    } while(0);
+
+#define PMIX_DEVICE_DIST_DESTRUCT(m)    \
+    do {                                \
+        if (NULL != ((m)->uuid)) {      \
+            free((m)->uuid);            \
+        }                               \
+        if (NULL != ((m)->osname)) {    \
+            free((m)->osname);          \
+        }                               \
+    } while(0)
+
+#define PMIX_DEVICE_DIST_CREATE(m, n)                                                       \
+    do {                                                                                    \
+        size_t _i;                                                                          \
+        pmix_device_distance_t *_m;                                                         \
+        _m = (pmix_device_distance_t*)pmix_calloc((n), sizeof(pmix_device_distance_t));     \
+        if (NULL != _m) {                                                                   \
+            for (_i=0; _i < (n); _i++) {                                                    \
+                _m[_i].mindist = UINT16_MAX;                                                \
+                _m[_i].maxdist = UINT16_MAX;                                                \
+            }                                                                               \
+        }                                                                                   \
+        (m) = _m;                                                                           \
+    } while(0)
+
+#define PMIX_DEVICE_DIST_FREE(m, n)                     \
+    do {                                                \
+        size_t _i;                                      \
+        if (NULL != (m)) {                              \
+            for (_i=0; _i < (n); _i++) {                \
+                PMIX_DEVICE_DIST_DESTRUCT(&(m)[_i]);    \
+            }                                           \
+            free((m));                                  \
+            (m) = NULL;                                 \
+        }                                               \
+    } while(0)
+
 
 /****    PMIX BYTE OBJECT    ****/
 typedef struct pmix_byte_object {
@@ -1111,6 +1731,46 @@
     } while(0)
 
 
+/****    PMIX ENDPOINT    ****/
+typedef struct pmix_endpoint {
+    char *uuid;
+    char *osname;
+    pmix_byte_object_t endpt;
+} pmix_endpoint_t;
+
+#define PMIX_ENDPOINT_CONSTRUCT(m)      \
+    memset((m), 0, sizeof(pmix_endpoint_t))
+
+#define PMIX_ENDPOINT_DESTRUCT(m)       \
+    do {                                \
+        if (NULL != (m)->uuid) {        \
+            free((m)->uuid);            \
+        }                               \
+        if (NULL != (m)->osname) {      \
+            free((m)->osname);          \
+        }                               \
+        if (NULL != (m)->endpt.bytes) { \
+            free((m)->endpt.bytes);     \
+        }                               \
+    } while(0)
+
+#define PMIX_ENDPOINT_CREATE(m, n)      \
+    (m) = (pmix_endpoint_t*)calloc((n), sizeof(pmix_endpoint_t))
+
+#define PMIX_ENDPOINT_FREE(m, n)                    \
+    do {                                            \
+        size_t _n;                                  \
+        if (NULL != (m)) {                          \
+            for (_n=0; _n < (n); _n++) {            \
+                PMIX_ENDPOINT_DESTRUCT(&((m)[_n])); \
+            }                                       \
+            free((m));                              \
+            (m) = NULL;                             \
+        }                                           \
+    } while(0)
+
+
+
 /****    PMIX ENVAR STRUCT   ****/
 /* Provide a structure for specifying environment variable modifications
  * Standard environment variables (e.g., PATH, LD_LIBRARY_PATH, and LD_PRELOAD)
@@ -1352,6 +2012,11 @@
 /**** AVOID CIRCULAR DEPENDENCIES                    ****/
 
 
+/* we cannot forward-declare the pmix_regattr_t struct
+ * as Cython doesn't know what to do with it. Thus, we
+ * will utilize the void* entry of the pmix_value_t to
+ * hold the point to pmix_regattr_t */
+
 /****    PMIX VALUE STRUCT    ****/
 
 /* NOTE: operations can supply a collection of values under
@@ -1394,6 +2059,16 @@
         void *ptr;
         pmix_alloc_directive_t adir;
         pmix_envar_t envar;
+        pmix_coord_t *coord;
+        pmix_link_state_t linkstate;
+        pmix_job_state_t jstate;
+        pmix_topology_t *topo;
+        pmix_cpuset_t *cpuset;
+        pmix_locality_t locality;
+        pmix_geometry_t *geometry;
+        pmix_device_type_t devtype;
+        pmix_device_distance_t *devdist;
+        pmix_endpoint_t *endpoint;
     } data;
 } pmix_value_t;
 /* allocate and initialize a specified number of value structs */
@@ -1496,29 +2171,6 @@
         }                                               \
     } while(0)
 
-#if 0
-    // This macro is no longer supported in the v3.2 and later series
-#define PMIX_VALUE_COMPRESSED_STRING_UNPACK(s)                              \
-    do {                                                                    \
-        char *tmp;                                                          \
-        /* if this is a compressed string, then uncompress it */            \
-        if (PMIX_COMPRESSED_STRING == (s)->type) {                          \
-            pmix_util_uncompress_string(&tmp, (uint8_t*)(s)->data.bo.bytes, \
-                (s)->data.bo.size);                                         \
-            if (NULL == tmp) {                                              \
-                PMIX_ERROR_LOG(PMIX_ERR_NOMEM);                             \
-                rc = PMIX_ERR_NOMEM;                                        \
-                PMIX_VALUE_RELEASE(s);                                      \
-                val = NULL;                                                 \
-            } else {                                                        \
-                PMIX_VALUE_DESTRUCT(s);                                     \
-                (s)->data.string = tmp;                                     \
-                (s)->type = PMIX_STRING;                                    \
-            }                                                               \
-        }                                                                   \
-    } while(0)
-#endif
-
 /****    PMIX INFO STRUCT    ****/
 typedef struct pmix_info {
     pmix_key_t key;
@@ -1555,7 +2207,7 @@
             for (_is=0; _is < (n); _is++) {         \
                 PMIX_INFO_DESTRUCT(&((m)[_is]));    \
             }                                       \
-            pmix_free((m));                              \
+            pmix_free((m));                         \
             (m) = NULL;                             \
         }                                           \
     } while (0)
@@ -1577,6 +2229,46 @@
         pmix_value_xfer(&(d)->value, (pmix_value_t*)&(s)->value);   \
     } while(0)
 
+/* macros for constructing an array of pmix_info_t
+ * on-the-fly, without knowing in advance the final
+ * number of elements it will hold */
+
+/* Start a list of pmix_info_t
+ * p - a void* pointer */
+#define PMIX_INFO_LIST_START(p)    \
+    (p) = pmix_info_list_start()
+
+/* Add a pmix_info_t to the list
+ * r - a returned pmix_status_t
+ * p - the void* pointer provided from PMIX_INFO_LIST_START
+ * a - an attribute key
+ * v - pointer to the value to be stored
+ * t - the PMIx data type of the value */
+#define PMIX_INFO_LIST_ADD(r, p, a, v, t)     \
+    (r) = pmix_info_list_add((p), (a), (v), (t))
+
+/* Transfer a pmix_info_t to the list
+ * r - a returned pmix_status_t
+ * p - the void* pointer provided from PMIX_INFO_LIST_START
+ * a - pointer to the pmix_info_t whose values are to be transferred
+ */
+#define PMIX_INFO_LIST_XFER(r, p, a)     \
+    (r) = pmix_info_list_xfer((p), (a))
+
+/* Convert a list of pmix_info_t into an array of pmix_info_t
+ * r - a returned pmix_status_t. PMIX_ERR_EMPTY if list was empty,
+ *     otherwise either success or error constant
+ * p - the void* pointer provided from PMIX_INFO_LIST_START
+ * m - address of a pmix_data_array_t into which
+ *     the array of info is to be placed */
+#define PMIX_INFO_LIST_CONVERT(r, p, m)   \
+    (r) = pmix_info_list_convert((p), (m))
+
+/* Release any memory associated with the list
+ * p - the void* pointer provided from PMIX_INFO_LIST_START */
+#define PMIX_INFO_LIST_RELEASE(p) \
+    pmix_info_list_release((p))
+
 
 /* macros for setting and unsetting the "reqd" flag
  * in a pmix_info_t */
@@ -1809,6 +2501,107 @@
         }                                           \
     } while (0)
 
+/****    ATTRIBUTE REGISTRATION STRUCT   ****/
+typedef struct pmix_regattr_t {
+    char *name;
+    pmix_key_t string;
+    pmix_data_type_t type;
+    char **description;
+} pmix_regattr_t;
+
+#define PMIX_REGATTR_CONSTRUCT(a)                       \
+    do {                                                \
+        if (NULL != (a)) {                              \
+            (a)->name = NULL;                           \
+            memset((a)->string, 0, PMIX_MAX_KEYLEN+1);  \
+            (a)->type = PMIX_UNDEF;                     \
+            (a)->description = NULL;                    \
+        }                                               \
+    } while(0)
+
+#define PMIX_REGATTR_LOAD(a, n, k, t, v)                        \
+    do {                                                        \
+        pmix_status_t _rgl;                                     \
+        if (NULL != (n)) {                                      \
+            (a)->name = strdup((n));                            \
+        }                                                       \
+        if (NULL != (k)) {                                      \
+            PMIX_LOAD_KEY((a)->string, (k));                    \
+        }                                                       \
+        (a)->type = (t);                                        \
+        if (NULL != (v)) {                                      \
+            PMIX_ARGV_APPEND(_rgl, &(a)->description, (v));     \
+        }                                                       \
+    } while(0)
+
+#define PMIX_REGATTR_DESTRUCT(a)                    \
+    do {                                            \
+        if (NULL != (a)) {                          \
+            if (NULL != (a)->name) {                \
+                pmix_free((a)->name);                    \
+            }                                       \
+            if (NULL != (a)->description) {         \
+                PMIX_ARGV_FREE((a)->description);   \
+            }                                       \
+        }                                           \
+    } while(0)
+
+#define PMIX_REGATTR_CREATE(m, n)                                       \
+    do {                                                                \
+        (m) = (pmix_regattr_t*)pmix_calloc((n) , sizeof(pmix_regattr_t));    \
+    } while (0)
+
+#define PMIX_REGATTR_FREE(m, n)                         \
+    do {                                                \
+        size_t _ra;                                     \
+        if (NULL != (m)) {                              \
+            for (_ra=0; _ra < (n); _ra++) {             \
+                PMIX_REGATTR_DESTRUCT(&((m)[_ra]));     \
+            }                                           \
+            pmix_free((m));                                  \
+            (m) = NULL;                                 \
+        }                                               \
+    } while (0)
+
+#define PMIX_REGATTR_XFER(a, b)                                         \
+    do {                                                                \
+        size_t _n;                                                      \
+        PMIX_REGATTR_CONSTRUCT((a));                                    \
+        if (NULL != ((b)->name)) {                                      \
+            (a)->name = strdup((b)->name);                              \
+        }                                                               \
+        PMIX_LOAD_KEY((a)->string, (b)->string);                        \
+        (a)->type = (b)->type;                                          \
+        if (NULL != (b)->description) {                                 \
+            PMIX_ARGV_COPY((a)->description, (b)->description);         \
+        }                                                               \
+    } while(0)
+
+
+/****    FABRIC STRUCT    ****/
+/* Define a pmix_fabric_t struct for
+ * interacting with fabric-related interfaces */
+typedef struct pmix_fabric_s {
+    /* user-supplied name for this fabric */
+    char *name;
+    /* a PMIx-supplied index identifying this registration object */
+    size_t index;
+    /* array containing information (provided by the PMIx library)
+     * about the fabric */
+    pmix_info_t *info;
+    size_t ninfo;
+    /* object pointer for use by the PMIx library */
+    void *module;
+} pmix_fabric_t;
+
+/* convenience macros to support pmix_fabric_t */
+#define PMIX_FABRIC_CONSTRUCT(x) \
+    memset(x, 0, sizeof(pmix_fabric_t))
+
+typedef enum {
+    PMIX_FABRIC_REQUEST_INFO,
+    PMIX_FABRIC_UPDATE_INFO
+} pmix_fabric_operation_t;
 
 /****    GENERIC HELPER MACROS    ****/
 
@@ -1845,7 +2638,7 @@
  * argv array - same as above only prepend
  */
 #define PMIX_ARGV_PREPEND(r, a, b) \
-    (r) = pmix_argv_prepend_nosize(a, b)
+    (r) = pmix_argv_prepend_nosize(&(a), b)
 
 /* Append to an argv-style array, but only if the provided argument
  * doesn't already exist somewhere in the array. Ignore the size of the array.
@@ -1858,10 +2651,10 @@
  *
  * This function is identical to the pmix_argv_append_nosize() function
  * except that it only appends the provided argument if it does not already
- * exist in the provided array, or overwrites it if it is.
+ * exist in the provided array.
  */
-#define PMIX_ARGV_APPEND_UNIQUE(r, a, b, c) \
-    (r) = pmix_argv_append_unique_nosize(a, b, c)
+#define PMIX_ARGV_APPEND_UNIQUE(r, a, b) \
+    (r) = pmix_argv_append_unique_nosize(a, b)
 
 /* Free a NULL-terminated argv array.
  *
@@ -2091,9 +2884,9 @@
 typedef void (*pmix_hdlr_reg_cbfunc_t)(pmix_status_t status,
                                        size_t refid,
                                        void *cbdata);
-/* maintain backward compatibility with v2 definition - change of name */
+/* retain the deprecated form */
 typedef void (*pmix_evhdlr_reg_cbfunc_t)(pmix_status_t status,
-                                         size_t evhdlr_ref,
+                                         size_t refid,
                                          void *cbdata);
 
 /* define a callback function for calls to PMIx_Get_nb. The status
@@ -2181,372 +2974,14 @@
                                          pmix_info_t info[], size_t ninfo,
                                          void *cbdata);
 
-/****    COMMON SUPPORT FUNCTIONS    ****/
-/* Note that these are moved to pmix.h in the v4.x series and later */
-/******     EVENT NOTIFICATION SUPPORT      ******/
-/* Register an event handler to report events. Three types of events
- * can be reported:
- *
- * (a) those that occur within the client library, but are not
- *     reportable via the API itself (e.g., loss of connection to
- *     the server). These events typically occur during behind-the-scenes
- *     non-blocking operations.
- *
- * (b) job-related events such as the failure of another process in
- *     the job or in any connected job, impending failure of hardware
- *     within the job's usage footprint, etc.
- *
- * (c) system notifications that are made available by the local
- *     administrators
- *
- * By default, only events that directly affect the process and/or
- * any process to which it is connected (via the PMIx_Connect call)
- * will be reported. Options to modify that behavior can be provided
- * in the info array
- *
- * Both the client application and the resource manager can register
- * err handlers for specific events. PMIx client/server calls the registered
- * err handler upon receiving event notify notification (via PMIx_Notify_event)
- * from the other end (Resource Manager/Client application).
- *
- * Multiple err handlers can be registered for different events. PMIX returns
- * an integer reference to each register handler in the callback fn. The caller
- * must retain the reference in order to deregister the evhdlr.
- * Modification of the notification behavior can be accomplished by
- * deregistering the current evhdlr, and then registering it
- * using a new set of info values.
- *
- * If cbfunc is NULL, then this is treated as a BLOCKING call - a positive
- * return value represents the reference ID for the request, while
- * negative values indicate the corresponding error
- *
- * See pmix_common.h for a description of the notification function */
-PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes,
-                                             pmix_info_t info[], size_t ninfo,
-                                             pmix_notification_fn_t evhdlr,
-                                             pmix_hdlr_reg_cbfunc_t cbfunc,
-                                             void *cbdata);
-
-/* Deregister an event handler
- * evhdlr_ref is the reference returned by PMIx from the call to
- * PMIx_Register_event_handler. If non-NULL, the provided cbfunc
- * will be called to confirm removal of the designated handler */
-PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref,
-                                               pmix_op_cbfunc_t cbfunc,
-                                               void *cbdata);
-
-/* Report an event for notification via any
- * registered evhdlr.
- *
- * This function allows the host server to direct the server
- * convenience library to notify all registered local procs of
- * an event. The event can be local, or anywhere in the cluster.
- * The status indicates the event being reported.
- *
- * The client application can also call this function to notify the
- * resource manager and/or other processes of an event it encountered.
- * It can also be used to asynchronously notify other parts of its
- * own internal process - e.g., for one library to notify another
- * when initialized inside the process.
- *
- * status - status code indicating the event being reported
- *
- * source - the process that generated the event
- *
- * range - the range in which the event is to be reported. For example,
- *         a value of PMIX_RANGE_LOCAL would instruct the system
- *         to only notify procs on the same local node as the
- *         event generator.
- *
- * info - an array of pmix_info_t structures provided by the event
- *        generator to pass any additional information about the
- *        event. This can include an array of pmix_proc_t structs
- *        describing the processes impacted by the event, the nature
- *        of the event and its severity, etc. The precise contents
- *        of the array will depend on the event generator.
- *
- * ninfo - number of elements in the info array
- *
- * cbfunc - callback function to be called upon completion of the
- *          notify_event function's actions. Note that any messages
- *          will have been queued, but may not have been transmitted
- *          by this time. Note that the caller is required to maintain
- *          the input data until the callback function has been executed!
- *          If cbfunc is NULL, then this is treated as a BLOCKING call and
- *          the result of the operation is provided in the returned
- *          status
- *
- * cbdata - the caller's provided void* object
- */
-PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
-                                            const pmix_proc_t *source,
-                                            pmix_data_range_t range,
-                                            const pmix_info_t info[], size_t ninfo,
-                                            pmix_op_cbfunc_t cbfunc, void *cbdata);
-
-
-/******    PRETTY-PRINT DEFINED VALUE TYPES     ******/
-/* Provide a string representation for several types of value. Note
- * that the provided string is statically defined and must NOT be
- * free'd. Supported value types:
- *
- * - pmix_status_t (PMIX_STATUS)
- * - pmix_scope_t   (PMIX_SCOPE)
- * - pmix_persistence_t  (PMIX_PERSIST)
- * - pmix_data_range_t   (PMIX_DATA_RANGE)
- * - pmix_info_directives_t   (PMIX_INFO_DIRECTIVES)
- * - pmix_data_type_t   (PMIX_DATA_TYPE)
- * - pmix_alloc_directive_t  (PMIX_ALLOC_DIRECTIVE)
- * - pmix_iof_channel_t  (PMIX_IOF_CHANNEL)
- */
-PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t status);
-PMIX_EXPORT const char* PMIx_Proc_state_string(pmix_proc_state_t state);
-PMIX_EXPORT const char* PMIx_Scope_string(pmix_scope_t scope);
-PMIX_EXPORT const char* PMIx_Persistence_string(pmix_persistence_t persist);
-PMIX_EXPORT const char* PMIx_Data_range_string(pmix_data_range_t range);
-PMIX_EXPORT const char* PMIx_Info_directives_string(pmix_info_directives_t directives);
-PMIX_EXPORT const char* PMIx_Data_type_string(pmix_data_type_t type);
-PMIX_EXPORT const char* PMIx_Alloc_directive_string(pmix_alloc_directive_t directive);
-PMIX_EXPORT const char* PMIx_IOF_channel_string(pmix_iof_channel_t channel);
-
-/* Get the PMIx version string. Note that the provided string is
- * statically defined and must NOT be free'd  */
-PMIX_EXPORT const char* PMIx_Get_version(void);
-
-/* Store some data locally for retrieval by other areas of the
- * proc. This is data that has only internal scope - it will
- * never be "pushed" externally */
-PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc,
-                                              const pmix_key_t key, pmix_value_t *val);
-
-
-/******    DATA BUFFER PACK/UNPACK SUPPORT    ******/
-/**
- * Top-level interface function to pack one or more values into a
- * buffer.
- *
- * The pack function packs one or more values of a specified type into
- * the specified buffer.  The buffer must have already been
- * initialized via the PMIX_DATA_BUFFER_CREATE or PMIX_DATA_BUFFER_CONSTRUCT
- * call - otherwise, the pack_value function will return an error.
- * Providing an unsupported type flag will likewise be reported as an error.
- *
- * Note that any data to be packed that is not hard type cast (i.e.,
- * not type cast to a specific size) may lose precision when unpacked
- * by a non-homogeneous recipient.  The PACK function will do its best to deal
- * with heterogeneity issues between the packer and unpacker in such
- * cases. Sending a number larger than can be handled by the recipient
- * will return an error code (generated upon unpacking) -
- * the error cannot be detected during packing.
- *
- * The identity of the intended recipient of the packed buffer (i.e., the
- * process that will be unpacking it) is used solely to resolve any data type
- * differences between PMIx versions. The recipient must, therefore, be
- * known to the user prior to calling the pack function so that the
- * PMIx library is aware of the version the recipient is using.
- *
- * @param *target Pointer to a pmix_proc_t structure containing the
- * nspace/rank of the process that will be unpacking the final buffer.
- * A NULL value may be used to indicate that the target is based on
- * the same PMIx version as the caller.
- *
- * @param *buffer A pointer to the buffer into which the value is to
- * be packed.
- *
- * @param *src A void* pointer to the data that is to be packed. Note
- * that strings are to be passed as (char **) - i.e., the caller must
- * pass the address of the pointer to the string as the void*. This
- * allows PMIx to use a single pack function, but still allow
- * the caller to pass multiple strings in a single call.
- *
- * @param num_values An int32_t indicating the number of values that are
- * to be packed, beginning at the location pointed to by src. A string
- * value is counted as a single value regardless of length. The values
- * must be contiguous in memory. Arrays of pointers (e.g., string
- * arrays) should be contiguous, although (obviously) the data pointed
- * to need not be contiguous across array entries.
- *
- * @param type The type of the data to be packed - must be one of the
- * PMIX defined data types.
- *
- * @retval PMIX_SUCCESS The data was packed as requested.
- *
- * @retval PMIX_ERROR(s) An appropriate PMIX error code indicating the
- * problem encountered. This error code should be handled
- * appropriately.
- *
- * @code
- * pmix_data_buffer_t *buffer;
- * int32_t src;
- *
- * PMIX_DATA_BUFFER_CREATE(buffer);
- * status_code = PMIx_Data_pack(buffer, &src, 1, PMIX_INT32);
- * @endcode
- */
-PMIX_EXPORT pmix_status_t PMIx_Data_pack(const pmix_proc_t *target,
-                                         pmix_data_buffer_t *buffer,
-                                         void *src, int32_t num_vals,
-                                         pmix_data_type_t type);
-
-/**
- * Unpack values from a buffer.
- *
- * The unpack function unpacks the next value (or values) of a
- * specified type from the specified buffer.
- *
- * The buffer must have already been initialized via an PMIX_DATA_BUFFER_CREATE or
- * PMIX_DATA_BUFFER_CONSTRUCT call (and assumedly filled with some data) -
- * otherwise, the unpack_value function will return an
- * error. Providing an unsupported type flag will likewise be reported
- * as an error, as will specifying a data type that DOES NOT match the
- * type of the next item in the buffer. An attempt to read beyond the
- * end of the stored data held in the buffer will also return an
- * error.
- *
- * NOTE: it is possible for the buffer to be corrupted and that
- * PMIx will *think* there is a proper variable type at the
- * beginning of an unpack region - but that the value is bogus (e.g., just
- * a byte field in a string array that so happens to have a value that
- * matches the specified data type flag). Therefore, the data type error check
- * is NOT completely safe. This is true for ALL unpack functions.
- *
- *
- * Unpacking values is a "nondestructive" process - i.e., the values are
- * not removed from the buffer. It is therefore possible for the caller
- * to re-unpack a value from the same buffer by resetting the unpack_ptr.
- *
- * Warning: The caller is responsible for providing adequate memory
- * storage for the requested data. As noted below, the user
- * must provide a parameter indicating the maximum number of values that
- * can be unpacked into the allocated memory. If more values exist in the
- * buffer than can fit into the memory storage, then the function will unpack
- * what it can fit into that location and return an error code indicating
- * that the buffer was only partially unpacked.
- *
- * Note that any data that was not hard type cast (i.e., not type cast
- * to a specific size) when packed may lose precision when unpacked by
- * a non-homogeneous recipient.  PMIx will do its best to deal with
- * heterogeneity issues between the packer and unpacker in such
- * cases. Sending a number larger than can be handled by the recipient
- * will return an error code generated upon unpacking - these errors
- * cannot be detected during packing.
- *
- * The identity of the source of the packed buffer (i.e., the
- * process that packed it) is used solely to resolve any data type
- * differences between PMIx versions. The source must, therefore, be
- * known to the user prior to calling the unpack function so that the
- * PMIx library is aware of the version the source used.
- *
- * @param *source Pointer to a pmix_proc_t structure containing the
- * nspace/rank of the process that packed the provided buffer.
- * A NULL value may be used to indicate that the source is based on
- * the same PMIx version as the caller.
- *
- * @param *buffer A pointer to the buffer from which the value will be
- * extracted.
- *
- * @param *dest A void* pointer to the memory location into which the
- * data is to be stored. Note that these values will be stored
- * contiguously in memory. For strings, this pointer must be to (char
- * **) to provide a means of supporting multiple string
- * operations. The unpack function will allocate memory for each
- * string in the array - the caller must only provide adequate memory
- * for the array of pointers.
- *
- * @param type The type of the data to be unpacked - must be one of
- * the BFROP defined data types.
- *
- * @retval *max_num_values The number of values actually unpacked. In
- * most cases, this should match the maximum number provided in the
- * parameters - but in no case will it exceed the value of this
- * parameter.  Note that if you unpack fewer values than are actually
- * available, the buffer will be in an unpackable state - the function will
- * return an error code to warn of this condition.
- *
- * @note The unpack function will return the actual number of values
- * unpacked in this location.
- *
- * @retval PMIX_SUCCESS The next item in the buffer was successfully
- * unpacked.
- *
- * @retval PMIX_ERROR(s) The unpack function returns an error code
- * under one of several conditions: (a) the number of values in the
- * item exceeds the max num provided by the caller; (b) the type of
- * the next item in the buffer does not match the type specified by
- * the caller; or (c) the unpack failed due to either an error in the
- * buffer or an attempt to read past the end of the buffer.
- *
- * @code
- * pmix_data_buffer_t *buffer;
- * int32_t dest;
- * char **string_array;
- * int32_t num_values;
- *
- * num_values = 1;
- * status_code = PMIx_Data_unpack(buffer, (void*)&dest, &num_values, PMIX_INT32);
- *
- * num_values = 5;
- * string_array = pmix_malloc(num_values*sizeof(char *));
- * status_code = PMIx_Data_unpack(buffer, (void*)(string_array), &num_values, PMIX_STRING);
- *
- * @endcode
- */
-PMIX_EXPORT pmix_status_t PMIx_Data_unpack(const pmix_proc_t *source,
-                                           pmix_data_buffer_t *buffer, void *dest,
-                                           int32_t *max_num_values,
-                                           pmix_data_type_t type);
-
-/**
- * Copy a data value from one location to another.
- *
- * Since registered data types can be complex structures, the system
- * needs some way to know how to copy the data from one location to
- * another (e.g., for storage in the registry). This function, which
- * can call other copy functions to build up complex data types, defines
- * the method for making a copy of the specified data type.
- *
- * @param **dest The address of a pointer into which the
- * address of the resulting data is to be stored.
- *
- * @param *src A pointer to the memory location from which the
- * data is to be copied.
- *
- * @param type The type of the data to be copied - must be one of
- * the PMIx defined data types.
- *
- * @retval PMIX_SUCCESS The value was successfully copied.
- *
- * @retval PMIX_ERROR(s) An appropriate error code.
- *
+/* Define a callback function to return device distance arrays
  */
-PMIX_EXPORT pmix_status_t PMIx_Data_copy(void **dest, void *src,
-                                         pmix_data_type_t type);
-
-/**
- * Print a data value.
- *
- * Since registered data types can be complex structures, the system
- * needs some way to know how to print them (i.e., convert them to a string
- * representation). Provided for debug purposes.
- *
- * @retval PMIX_SUCCESS The value was successfully printed.
- *
- * @retval PMIX_ERROR(s) An appropriate error code.
- */
-PMIX_EXPORT pmix_status_t PMIx_Data_print(char **output, char *prefix,
-                                          void *src, pmix_data_type_t type);
-
-/**
- * Copy a payload from one buffer to another
- *
- * This function will append a copy of the payload in one buffer into
- * another buffer.
- * NOTE: This is NOT a destructive procedure - the
- * source buffer's payload will remain intact, as will any pre-existing
- * payload in the destination's buffer.
- */
-PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest,
-                                                 pmix_data_buffer_t *src);
+typedef void (*pmix_device_dist_cbfunc_t)(pmix_status_t status,
+                                          pmix_device_distance_t *dist,
+                                          size_t ndist,
+                                          void *cbdata,
+                                          pmix_release_cbfunc_t release_fn,
+                                          void *release_cbdata);
 
 
 /********    STANDARD MACROS FOR DARRAY AND VALUE SUPPORT     ********/
@@ -2606,7 +3041,8 @@
         } else if (PMIX_APP == m->type) {
             pmix_app_t *_a = (pmix_app_t*)m->array;
             PMIX_APP_FREE(_a, m->size);
-        } else if (PMIX_BYTE_OBJECT == m->type) {
+        } else if (PMIX_BYTE_OBJECT == m->type ||
+                   PMIX_COMPRESSED_STRING == m->type) {
             pmix_byte_object_t *_b = (pmix_byte_object_t*)m->array;
             PMIX_BYTE_OBJECT_FREE(_b, m->size);
         } else if (PMIX_STRING == m->type) {
@@ -2644,7 +3080,8 @@
                 PMIX_QUERY_CREATE((m)->array, (n));                 \
             } else if (PMIX_APP == (t)) {                           \
                 PMIX_APP_CREATE((m)->array, (n));                   \
-            } else if (PMIX_BYTE_OBJECT == (t)) {                   \
+            } else if (PMIX_BYTE_OBJECT == (t) ||                   \
+                       PMIX_COMPRESSED_STRING == (t)) {             \
                 PMIX_BYTE_OBJECT_CREATE((m)->array, (n));           \
             } else if (PMIX_ALLOC_DIRECTIVE == (t) ||               \
                        PMIX_PROC_STATE == (t) ||                    \
@@ -2653,10 +3090,9 @@
                        PMIX_DATA_RANGE == (t) ||                    \
                        PMIX_BYTE == (t) ||                          \
                        PMIX_INT8 == (t) ||                          \
-                       PMIX_UINT8 == (t)) {                         \
-                (m)->array = pmix_calloc((n), sizeof(int8_t));      \
-            } else if (PMIX_POINTER == (t)) {                       \
-		(m)->array = pmix_calloc((n), sizeof(void*));       \
+                       PMIX_UINT8 == (t) ||                         \
+                       PMIX_POINTER == (t)) {                       \
+                (m)->array = pmix_calloc((n), sizeof(int8_t));           \
             } else if (PMIX_STRING == (t)) {                        \
                 (m)->array = pmix_calloc((n), sizeof(char*));            \
             } else if (PMIX_SIZE == (t)) {                          \
@@ -2688,6 +3124,14 @@
                 (m)->array = pmix_calloc((n), sizeof(struct timeval));   \
             } else if (PMIX_TIME == (t)) {                          \
                 (m)->array = pmix_calloc((n), sizeof(time_t));           \
+            } else if (PMIX_REGATTR == (t)) {                       \
+                PMIX_REGATTR_CREATE((m)->array, (n));               \
+            } else if (PMIX_BOOL == (t)) {                          \
+                (m)->array = pmix_calloc((n), sizeof(bool));             \
+            } else if (PMIX_COORD == (t)) {                         \
+                (m)->array = pmix_calloc((n), sizeof(pmix_coord_t));     \
+            } else if (PMIX_LINK_STATE == (t)) {                    \
+                (m)->array = pmix_calloc((n), sizeof(pmix_link_state_t));  \
             }                                                       \
         } else {                                                    \
             (m)->array = NULL;                                      \
@@ -2712,7 +3156,6 @@
         }                                   \
     } while(0)
 
-
 /**
  * Provide a safe version of strncpy that doesn't generate
  * a ton of spurious warnings. Note that not every environment
@@ -2728,22 +3171,21 @@
                                 const char *src,
                                 size_t len)
 {
-    size_t i, k;
-    char *new_dest = dest;
+    size_t i;
 
     /* use an algorithm that also protects against
      * non-NULL-terminated src strings */
-    for (i=0, k=0; i <= len; ++i, ++src, ++new_dest) {
-        ++k;
-        *new_dest = *src;
+    for (i=0; i < len; ++i, ++src, ++dest) {
+        *dest = *src;
         if ('\0' == *src) {
             break;
         }
     }
-    dest[k-1] = '\0';
+    *dest = '\0';
 }
 
 #include <pmix_extend.h>
+#include <pmix_deprecated.h>
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
diff -Nru pmix-3.2.2~rc1/include/pmix_deprecated.h pmix-4.0.0/include/pmix_deprecated.h
--- pmix-3.2.2~rc1/include/pmix_deprecated.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/include/pmix_deprecated.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer listed
+ *   in this license in the documentation and/or other materials
+ *   provided with the distribution.
+ *
+ * - Neither the name of the copyright holders nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * The copyright holders provide no reassurances that the source code
+ * provided does not infringe any patent, copyright, or any other
+ * intellectual property rights of third parties.  The copyright holders
+ * disclaim any liability to any recipient for claims brought against
+ * recipient by any third party for infringement of that parties
+ * intellectual property rights.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $HEADER$
+ *
+ * PMIx provides a "function-shipping" approach to support for
+ * implementing the server-side of the protocol. This method allows
+ * resource managers to implement the server without being burdened
+ * with PMIx internal details. Accordingly, each PMIx API is mirrored
+ * here in a function call to be provided by the server. When a
+ * request is received from the client, the corresponding server function
+ * will be called with the information.
+ *
+ * Any functions not supported by the RM can be indicated by a NULL for
+ * the function pointer. Client calls to such functions will have a
+ * "not supported" error returned.
+ */
+
+#ifndef PMIx_DEPRECATED_H
+#define PMIx_DEPRECATED_H
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/***** v4 DECPRECATIONS/REMOVALS *****/
+/* APIs */
+PMIX_EXPORT pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc,
+                                                      pmix_info_t info[], size_t ninfo);
+
+
+/* CONSTANTS */
+#define PMIX_ERR_SILENT                             -2
+#define PMIX_ERR_DEBUGGER_RELEASE                   -3
+#define PMIX_ERR_PROC_ABORTED                       -7
+#define PMIX_ERR_PROC_REQUESTED_ABORT               -8
+#define PMIX_ERR_PROC_ABORTING                      -9
+#define PMIX_ERR_SERVER_FAILED_REQUEST              -10
+#define PMIX_EXISTS                                 -11
+#define PMIX_ERR_HANDSHAKE_FAILED                   -13
+#define PMIX_ERR_READY_FOR_HANDSHAKE                -14
+#define PMIX_ERR_PROC_ENTRY_NOT_FOUND               -17
+#define PMIX_ERR_PACK_MISMATCH                      -22
+#define PMIX_ERR_IN_ERRNO                           -26
+#define PMIX_ERR_DATA_VALUE_NOT_FOUND               -30
+#define PMIX_ERR_INVALID_ARG                        -33
+#define PMIX_ERR_INVALID_KEY                        -34
+#define PMIX_ERR_INVALID_KEY_LENGTH                 -35
+#define PMIX_ERR_INVALID_VAL                        -36
+#define PMIX_ERR_INVALID_VAL_LENGTH                 -37
+#define PMIX_ERR_INVALID_LENGTH                     -38
+#define PMIX_ERR_INVALID_NUM_ARGS                   -39
+#define PMIX_ERR_INVALID_ARGS                       -40
+#define PMIX_ERR_INVALID_NUM_PARSED                 -41
+#define PMIX_ERR_INVALID_KEYVALP                    -42
+#define PMIX_ERR_INVALID_SIZE                       -43
+#define PMIX_ERR_INVALID_NAMESPACE                  -44
+#define PMIX_ERR_SERVER_NOT_AVAIL                   -45
+#define PMIX_ERR_NOT_IMPLEMENTED                    -48
+#define PMIX_ERR_LOST_CONNECTION_TO_SERVER          -101
+#define PMIX_ERR_LOST_PEER_CONNECTION               -102
+#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT          -103
+#define PMIX_NOTIFY_ALLOC_COMPLETE                  -105
+#define PMIX_ERR_INVALID_TERMINATION                -112
+#define PMIX_ERR_JOB_TERMINATED                     -145    // DEPRECATED NAME - non-error termination
+#define PMIX_ERR_UPDATE_ENDPOINTS                   -146
+#define PMIX_GDS_ACTION_COMPLETE                    -148
+#define PMIX_PROC_HAS_CONNECTED                     -149
+#define PMIX_CONNECT_REQUESTED                      -150
+#define PMIX_ERR_NODE_DOWN                          -231
+#define PMIX_ERR_NODE_OFFLINE                       -232
+
+#define PMIX_ERR_SYS_BASE                           PMIX_EVENT_SYS_BASE
+#define PMIX_ERR_SYS_OTHER                          PMIX_EVENT_SYS_OTHER
+
+#define PMIX_JOB_STATE_PREPPED                   1  // job is ready to be launched
+
+/* ATTRIBUTES */
+#define PMIX_EVENT_BASE                     "pmix.evbase"           // (struct event_base *) pointer to libevent event_base
+                                                                    // to use in place of the internal progress thread ***** DEPRECATED *****
+#define PMIX_TOPOLOGY                       "pmix.topo"             // (hwloc_topology_t) ***** DEPRECATED ***** pointer to the PMIx client's internal
+                                                                    //         topology object
+#define PMIX_DEBUG_JOB                      "pmix.dbg.job"          // (char*) ***** DEPRECATED ***** nspace of the job assigned to this debugger to be debugged. Note
+                                                                    //         that id's, pids, and other info on the procs is available
+                                                                    //         via a query for the nspace's local or global proctable
+#define PMIX_RECONNECT_SERVER               "pmix.cnct.recon"       // (bool) tool is requesting to change server connections
+#define PMIX_ALLOC_NETWORK                  "pmix.alloc.net"        // (pmix_data_array_t*) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_ID               "pmix.alloc.netid"      // (char*) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_QOS              "pmix.alloc.netqos"     // (char*) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_TYPE             "pmix.alloc.nettype"    // (char*) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_PLANE            "pmix.alloc.netplane"   // (char*) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_ENDPTS           "pmix.alloc.endpts"     // (size_t) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_ENDPTS_NODE      "pmix.alloc.endpts.nd"  // (size_t) ***** DEPRECATED *****
+#define PMIX_ALLOC_NETWORK_SEC_KEY          "pmix.alloc.nsec"       // (pmix_byte_object_t) ***** DEPRECATED *****
+#define PMIX_PROC_DATA                      "pmix.pdata"            // (pmix_data_array_t*) ***** DEPRECATED ***** starts with rank, then contains more data
+#define PMIX_LOCALITY                       "pmix.loc"              // (uint16_t)  ***** DEPRECATED *****relative locality of two procs
+
+#define PMIX_LOCAL_TOPO                     "pmix.ltopo"            // (char*)  ***** DEPRECATED *****xml-representation of local node topology
+#define PMIX_TOPOLOGY_XML                   "pmix.topo.xml"         // (char*)  ***** DEPRECATED *****XML-based description of topology
+#define PMIX_TOPOLOGY_FILE                  "pmix.topo.file"        // (char*)  ***** DEPRECATED *****full path to file containing XML topology description
+#define PMIX_TOPOLOGY_SIGNATURE             "pmix.toposig"          // (char*)  ***** DEPRECATED *****topology signature string
+#define PMIX_HWLOC_SHMEM_ADDR               "pmix.hwlocaddr"        // (size_t)  ***** DEPRECATED *****address of HWLOC shared memory segment
+#define PMIX_HWLOC_SHMEM_SIZE               "pmix.hwlocsize"        // (size_t)  ***** DEPRECATED *****size of HWLOC shared memory segment
+#define PMIX_HWLOC_SHMEM_FILE               "pmix.hwlocfile"        // (char*)  ***** DEPRECATED *****path to HWLOC shared memory file
+#define PMIX_HWLOC_XML_V1                   "pmix.hwlocxml1"        // (char*)  ***** DEPRECATED ***** XML representation of local topology using HWLOC v1.x format
+#define PMIX_HWLOC_XML_V2                   "pmix.hwlocxml2"        // (char*) ***** DEPRECATED ***** XML representation of local topology using HWLOC v2.x format
+#define PMIX_HWLOC_SHARE_TOPO               "pmix.hwlocsh"          // (bool) ***** DEPRECATED ***** Share the HWLOC topology via shared memory
+#define PMIX_HWLOC_HOLE_KIND                "pmix.hwlocholek"       // (char*) ***** DEPRECATED ***** Kind of VM "hole" HWLOC should use for shared memory
+#define PMIX_DSTPATH                        "pmix.dstpath"          // (char*) ***** DEPRECATED ***** path to dstore files
+#define PMIX_COLLECTIVE_ALGO                "pmix.calgo"            // (char*) ***** DEPRECATED ***** comma-delimited list of algorithms to use for collective
+#define PMIX_COLLECTIVE_ALGO_REQD           "pmix.calreqd"          // (bool) ***** DEPRECATED ***** if true, indicates that the requested choice of algo is mandatory
+#define PMIX_PROC_BLOB                      "pmix.pblob"            // (pmix_byte_object_t) ***** DEPRECATED ***** packed blob of process data
+#define PMIX_MAP_BLOB                       "pmix.mblob"            // (pmix_byte_object_t) ***** DEPRECATED ***** packed blob of process location
+#define PMIX_MAPPER                         "pmix.mapper"           // (char*) ***** DEPRECATED ***** mapper to use for placing spawned procs
+#define PMIX_NON_PMI                        "pmix.nonpmi"           // (bool) ***** DEPRECATED ***** spawned procs will not call PMIx_Init
+#define PMIX_PROC_URI                       "pmix.puri"             // (char*) ***** DEPRECATED ***** URI containing contact info for proc
+#define PMIX_ARCH                           "pmix.arch"             // (uint32_t) ***** DEPRECATED ***** datatype architecture flag
+
+#define PMIX_DEBUG_JOB_DIRECTIVES           "pmix.dbg.jdirs"        // (pmix_data_array_t*) ***** DEPRECATED ***** array of job-level directives
+#define PMIX_DEBUG_APP_DIRECTIVES           "pmix.dbg.adirs"        // (pmix_data_array_t*) ***** DEPRECATED ***** array of app-level directives
+#define PMIX_EVENT_NO_TERMINATION           "pmix.evnoterm"         // (bool) ***** DEPRECATED ***** indicates that the handler has satisfactorily handled
+                                                                    //        the event and believes termination of the application is not required
+#define PMIX_EVENT_WANT_TERMINATION         "pmix.evterm"           // (bool) ***** DEPRECATED ***** indicates that the handler has determined that the
+                                                                    //        application should be terminated
+
+/* attributes for GDS */
+#define PMIX_GDS_MODULE                     "pmix.gds.mod"          // (char*) ***** DEPRECATED ***** comma-delimited string of desired modules
+
+
+#define PMIX_IOF_STOP                       "pmix.iof.stop"         // (bool) ***** DEPRECATED ***** Stop forwarding the specified channel(s)
+#define PMIX_NOTIFY_LAUNCH                  "pmix.note.lnch"        // (bool) ***** DEPRECATED ***** notify the requestor upon launch of the child job and return
+                                                                    //        its namespace in the event
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif
diff -Nru pmix-3.2.2~rc1/include/pmix_extend.h pmix-4.0.0/include/pmix_extend.h
--- pmix-3.2.2~rc1/include/pmix_extend.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmix_extend.h	2021-01-02 08:56:17.000000000 +0000
@@ -42,6 +42,18 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $HEADER$
+ *
+ * PMIx provides a "function-shipping" approach to support for
+ * implementing the server-side of the protocol. This method allows
+ * resource managers to implement the server without being burdened
+ * with PMIx internal details. Accordingly, each PMIx API is mirrored
+ * here in a function call to be provided by the server. When a
+ * request is received from the client, the corresponding server function
+ * will be called with the information.
+ *
+ * Any functions not supported by the RM can be indicated by a NULL for
+ * the function pointer. Client calls to such functions will have a
+ * "not supported" error returned.
  */
 
 #ifndef PMIx_EXTEND_H
@@ -54,9 +66,8 @@
 /* expose some functions that are resolved in the
  * PMIx library, but part of a header that
  * includes internal functions - we don't
- * want to expose the entire header here. These
- * back the associated macros included in the
- * PMIx Standard
+ * want to expose the entire header here. For
+ * consistency, we provide macro versions as well
  */
 void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type);
 
@@ -68,7 +79,7 @@
 
 pmix_status_t pmix_argv_prepend_nosize(char ***argv, const char *arg);
 
-pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite);
+pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg);
 
 void pmix_argv_free(char **argv);
 
@@ -83,70 +94,23 @@
 pmix_status_t pmix_setenv(const char *name, const char *value,
                           bool overwrite, char ***env);
 
+void pmix_ploc_base_destruct_cpuset(pmix_cpuset_t *cpuset);
+
+void pmix_ploc_base_release_cpuset(pmix_cpuset_t *cpuset, size_t n);
+
+void pmix_ploc_base_destruct_topology(pmix_topology_t *topo);
+
+void pmix_ploc_base_release_topology(pmix_topology_t *topo, size_t n);
+
+void* pmix_info_list_start(void);
 
-/* the following are a set of legacy macros not included in the
- * PMIx Standard, but used in some codes (e.g., the Slurm plugin).
- * These should be considered "deprecated" and will be removed
- * in the next major release of the PRI */
-#define PMIX_VAL_FIELD_int(x)       ((x)->data.integer)
-#define PMIX_VAL_FIELD_uint32_t(x)  ((x)->data.uint32)
-#define PMIX_VAL_FIELD_uint16_t(x)  ((x)->data.uint16)
-#define PMIX_VAL_FIELD_string(x)    ((x)->data.string)
-#define PMIX_VAL_FIELD_float(x)     ((x)->data.fval)
-#define PMIX_VAL_FIELD_byte(x)      ((x)->data.byte)
-#define PMIX_VAL_FIELD_flag(x)      ((x)->data.flag)
-
-#define PMIX_VAL_TYPE_int      PMIX_INT
-#define PMIX_VAL_TYPE_uint32_t PMIX_UINT32
-#define PMIX_VAL_TYPE_uint16_t PMIX_UINT16
-#define PMIX_VAL_TYPE_string   PMIX_STRING
-#define PMIX_VAL_TYPE_float    PMIX_FLOAT
-#define PMIX_VAL_TYPE_byte     PMIX_BYTE
-#define PMIX_VAL_TYPE_flag     PMIX_BOOL
-
-#define PMIX_VAL_set_assign(_v, _field, _val )   \
-    do {                                                            \
-        (_v)->type = PMIX_VAL_TYPE_ ## _field;                      \
-        PMIX_VAL_FIELD_ ## _field((_v)) = _val;                     \
-    } while (0)
-
-#define PMIX_VAL_set_strdup(_v, _field, _val )       \
-    do {                                                                \
-        (_v)->type = PMIX_VAL_TYPE_ ## _field;                          \
-        PMIX_VAL_FIELD_ ## _field((_v)) = strdup(_val);                 \
-    } while (0)
-
-#define PMIX_VAL_SET_int        PMIX_VAL_set_assign
-#define PMIX_VAL_SET_uint32_t   PMIX_VAL_set_assign
-#define PMIX_VAL_SET_uint16_t   PMIX_VAL_set_assign
-#define PMIX_VAL_SET_string     PMIX_VAL_set_strdup
-#define PMIX_VAL_SET_float      PMIX_VAL_set_assign
-#define PMIX_VAL_SET_byte       PMIX_VAL_set_assign
-#define PMIX_VAL_SET_flag       PMIX_VAL_set_assign
-
-#define PMIX_VAL_SET(_v, _field, _val )   \
-    PMIX_VAL_SET_ ## _field(_v, _field, _val)
-
-#define PMIX_VAL_cmp_val(_val1, _val2)      ((_val1) != (_val2))
-#define PMIX_VAL_cmp_float(_val1, _val2)    (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001))
-#define PMIX_VAL_cmp_ptr(_val1, _val2)      strncmp(_val1, _val2, strlen(_val1)+1)
-
-#define PMIX_VAL_CMP_int        PMIX_VAL_cmp_val
-#define PMIX_VAL_CMP_uint32_t   PMIX_VAL_cmp_val
-#define PMIX_VAL_CMP_uint16_t   PMIX_VAL_cmp_val
-#define PMIX_VAL_CMP_float      PMIX_VAL_cmp_float
-#define PMIX_VAL_CMP_string     PMIX_VAL_cmp_ptr
-#define PMIX_VAL_CMP_byte       PMIX_VAL_cmp_val
-#define PMIX_VAL_CMP_flag       PMIX_VAL_cmp_val
+pmix_status_t pmix_info_list_add(void *ptr, pmix_key_t key, void *value, pmix_data_type_t type);
 
-#define PMIX_VAL_ASSIGN(_v, _field, _val) \
-    PMIX_VAL_set_assign(_v, _field, _val)
+pmix_status_t pmix_info_list_xfer(void *ptr, const pmix_info_t *info);
 
-#define PMIX_VAL_CMP(_field, _val1, _val2) \
-    PMIX_VAL_CMP_ ## _field(_val1, _val2)
+pmix_status_t pmix_info_list_convert(void *ptr, pmix_data_array_t *array);
 
-#define PMIX_VAL_FREE(_v) \
-     PMIx_free_value_data(_v)
+void pmix_info_list_release(void *ptr);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
diff -Nru pmix-3.2.2~rc1/include/pmix.h pmix-4.0.0/include/pmix.h
--- pmix-3.2.2~rc1/include/pmix.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmix.h	2021-01-02 08:56:17.000000000 +0000
@@ -200,7 +200,7 @@
  *     an error. The timeout parameter can help avoid "hangs" due to programming
  *     errors that prevent the target proc from ever exposing its data.
  */
-PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const pmix_key_t key,
+PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[],
                                    const pmix_info_t info[], size_t ninfo,
                                    pmix_value_t **val);
 
@@ -411,9 +411,6 @@
  * executing on a given node. We assume that the host RM will
  * exercise appropriate access control on the information.
  *
- * NOTE: there is no blocking form of this API as the structures
- * passed to query info differ from those for receiving the results
- *
  * The following return status codes are provided in the callback:
  *
  * PMIX_SUCCESS - all data has been returned
@@ -421,6 +418,9 @@
  * PMIX_ERR_PARTIAL_SUCCESS - some of the data has been returned
  * PMIX_ERR_NOT_SUPPORTED - the host RM does not support this function
  */
+PMIX_EXPORT pmix_status_t PMIx_Query_info(pmix_query_t queries[], size_t nqueries,
+                                          pmix_info_t **results, size_t *nresults);
+
 PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nqueries,
                                              pmix_info_cbfunc_t cbfunc, void *cbdata);
 
@@ -496,7 +496,8 @@
  * any provided pmix_info_t array.
  */
 PMIX_EXPORT pmix_status_t PMIx_Job_control(const pmix_proc_t targets[], size_t ntargets,
-                                           const pmix_info_t directives[], size_t ndirs);
+                                           const pmix_info_t directives[], size_t ndirs,
+                                           pmix_info_t **results, size_t *nresults);
 
 PMIX_EXPORT pmix_status_t PMIx_Job_control_nb(const pmix_proc_t targets[], size_t ntargets,
                                               const pmix_info_t directives[], size_t ndirs,
@@ -530,7 +531,8 @@
  * Note: a process can send a heartbeat to the server using the PMIx_Heartbeat
  * macro provided below*/
 PMIX_EXPORT pmix_status_t PMIx_Process_monitor(const pmix_info_t *monitor, pmix_status_t error,
-                                               const pmix_info_t directives[], size_t ndirs);
+                                               const pmix_info_t directives[], size_t ndirs,
+                                               pmix_info_t **results, size_t *nresults);
 
 PMIX_EXPORT pmix_status_t PMIx_Process_monitor_nb(const pmix_info_t *monitor, pmix_status_t error,
                                                   const pmix_info_t directives[], size_t ndirs,
@@ -574,7 +576,10 @@
  * will _not_ be called in such cases.
  */
 PMIX_EXPORT pmix_status_t PMIx_Get_credential(const pmix_info_t info[], size_t ninfo,
-                                              pmix_credential_cbfunc_t cbfunc, void *cbdata);
+                                              pmix_byte_object_t *credential);
+
+PMIX_EXPORT pmix_status_t PMIx_Get_credential_nb(const pmix_info_t info[], size_t ninfo,
+                                                 pmix_credential_cbfunc_t cbfunc, void *cbdata);
 
 /* Request validation of a credential by the PMIx server/SMS
  * Input values include:
@@ -607,126 +612,765 @@
  */
 PMIX_EXPORT pmix_status_t PMIx_Validate_credential(const pmix_byte_object_t *cred,
                                                    const pmix_info_t info[], size_t ninfo,
-                                                   pmix_validation_cbfunc_t cbfunc, void *cbdata);
+                                                   pmix_info_t **results, size_t *nresults);
 
-/* Define a callback function for delivering forwarded IO to a process
- * This function will be called whenever data becomes available, or a
- * specified buffering size and/or time has been met. The function
- * will be passed the following values:
+PMIX_EXPORT pmix_status_t PMIx_Validate_credential_nb(const pmix_byte_object_t *cred,
+                                                      const pmix_info_t info[], size_t ninfo,
+                                                      pmix_validation_cbfunc_t cbfunc, void *cbdata);
+
+
+/* Construct a new group composed of the specified processes and identified with
+ * the provided group identifier. Both blocking and non-blocking versions
+ * are provided (the callback function for the non-blocking form will be called
+ * once all specified processes have joined the group). The group identifier is
+ * a user-defined, NULL-terminated character array of length less than or equal
+ * to PMIX_MAX_NSLEN. Only characters accepted by standard string comparison
+ * functions (e.g., strncmp) are supported.
+ *
+ * Processes may engage in multiple simultaneous group construct operations as
+ * desired so long as each is provided with a unique group ID. The info array
+ * can be used to pass user-level directives regarding timeout constraints and
+ * other options available from the PMIx server.
+ *
+ * The construct leader (if PMIX_GROUP_LEADER is provided) or all participants
+ * will receive events (if registered for the PMIX_GROUP_MEMBER_FAILED event)
+ * whenever a process fails or terminates prior to calling
+ * PMIx_Group_construct(_nb) – the events will contain the identifier of the
+ * process that failed to join plus any other information that the resource
+ * manager provided. This provides an opportunity for the leader to react to
+ * the event – e.g., to invite an alternative member to the group or to decide
+ * to proceed with a smaller group. The decision to proceed with a smaller group
+ * is communicated to the PMIx library in the results array at the end of the
+ * event handler. This allows PMIx to properly adjust accounting for procedure
+ * completion. When construct is complete, the participating PMIx servers will
+ * be alerted to any change in participants and each group member will (if
+ * registered) receive a PMIX_GROUP_MEMBERSHIP_UPDATE event updating the group
+ * membership.
+ *
+ * Processes in a group under construction are not allowed to leave the group
+ * until group construction is complete. Upon completion of the construct
+ * procedure, each group member will have access to the job-level information
+ * of all nspaces represented in the group and the contact information for
+ * every group member.
+ *
+ * Failure of the leader at any time will cause a PMIX_GROUP_LEADER_FAILED event
+ * to be delivered to all participants so they can optionally declare a new leader.
+ * A new leader is identified by providing the PMIX_GROUP_LEADER attribute in
+ * the results array in the return of the event handler. Only one process is
+ * allowed to return that attribute, declaring itself as the new leader. Results
+ * of the leader selection will be communicated to all participants via a
+ * PMIX_GROUP_LEADER_SELECTED event identifying the new leader. If no leader
+ * was selected, then the status code provided in the event handler will provide
+ * an error value so the participants can take appropriate action.
+ *
+ * Any participant that returns PMIX_GROUP_CONSTRUCT_ABORT from the leader failed
+ * event handler will cause the construct process to abort. Those processes
+ * engaged in the blocking construct will return from the call with the
+ * PMIX_GROUP_CONSTRUCT_ABORT status. Non-blocking participants will have
+ * their callback function executed with that status.
+ *
+ * Some relevant attributes for this operation:
+ *    PMIX_GROUP_LEADER - declare this process to be the leader of the construction
+ *                        procedure. If a process provides this attribute, then
+ *                        failure notification for any participating process will
+ *                        go only to that one process. In the absence of a
+ *                        declared leader, failure events go to all participants.
+ *    PMIX_GROUP_OPTIONAL - participation is optional - do not return an error if
+ *                          any of the specified processes terminate
+ *                          without having joined (default=false)
+ *    PMIX_GROUP_NOTIFY_TERMINATION - notify remaining members when another member
+ *                                    terminates without first leaving the
+ *                                    group (default=false)
+ *    PMIX_GROUP_ASSIGN_CONTEXT_ID - requests that the RM assign a unique context
+ *                                   ID (size_t) to the group. The value is returned
+ *                                   in the PMIX_GROUP_CONSTRUCT_COMPLETE event
+ *    PMIX_TIMEOUT - return an error if the group doesn't assemble within the
+ *                   specified number of seconds. Targets the scenario where a
+ *                   process fails to call PMIx_Group_connect due to hanging
  *
- * iofhdlr - the returned registration number of the handler being invoked.
- *           This is required when deregistering the handler.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Group_construct(const char grp[],
+                                               const pmix_proc_t procs[], size_t nprocs,
+                                               const pmix_info_t directives[], size_t ndirs,
+                                               pmix_info_t **results, size_t *nresults);
+
+PMIX_EXPORT pmix_status_t PMIx_Group_construct_nb(const char grp[],
+                                                  const pmix_proc_t procs[], size_t nprocs,
+                                                  const pmix_info_t info[], size_t ninfo,
+                                                  pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Explicitly invite specified processes to join a group.
  *
- * channel - a bitmask identifying the channel the data arrived on
+ * Each invited process will be notified of the invitation via the PMIX_GROUP_INVITED
+ * event. The processes being invited must have registered for the PMIX_GROUP_INVITED
+ * event in order to be notified of the invitation. When ready to respond, each invited
+ * process provides a response using the appropriate form of PMIx_Group_join. This will
+ * notify the inviting process that the invitation was either accepted (via the
+ * PMIX_GROUP_INVITE_ACCEPTED event) or declined (via the PMIX_GROUP_INVITE_DECLINED event).
+ * The inviting process will also receive PMIX_GROUP_MEMBER_FAILED events whenever a
+ * process fails or terminates prior to responding to the invitation.
+ *
+ * Upon accepting the invitation, both the inviting and invited process will receive
+ * access to the job-level information of each other’s nspaces and the contact
+ * information of the other process.
+ *
+ * Some relevant attributes for this operation:
+ *    PMIX_GROUP_ASSIGN_CONTEXT_ID - requests that the RM assign a unique context
+ *                                   ID (size_t) to the group. The value is returned
+ *                                   in the PMIX_GROUP_CONSTRUCT_COMPLETE event
+ *    PMIX_TIMEOUT (int): return an error if the group doesn’t assemble within the
+ *                        specified number of seconds. Targets the scenario where a
+ *                        process fails to call PMIx_Group_connect due to hanging
+ *
+ * The inviting process is automatically considered the leader of the asynchronous
+ * group construction procedure and will receive all failure or termination events
+ * for invited members prior to completion. The inviting process is required to
+ * provide a PMIX_GROUP_CONSTRUCT_COMPLETE event once the group has been fully
+ * assembled – this event will be distributed to all participants along with the
+ * final membership.
+ *
+ * Failure of the leader at any time will cause a PMIX_GROUP_LEADER_FAILED event
+ * to be delivered to all participants so they can optionally declare a new leader.
+ * A new leader is identified by providing the PMIX_GROUP_LEADER attribute in
+ * the results array in the return of the event handler. Only one process is
+ * allowed to return that attribute, declaring itself as the new leader. Results
+ * of the leader selection will be communicated to all participants via a
+ * PMIX_GROUP_LEADER_SELECTED event identifying the new leader. If no leader
+ * was selected, then the status code provided in the event handler will provide
+ * an error value so the participants can take appropriate action.
+ *
+ * Any participant that returns PMIX_GROUP_CONSTRUCT_ABORT from the event
+ * handler will cause all participants to receive an event notifying them
+ * of that status.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Group_invite(const char grp[],
+                                            const pmix_proc_t procs[], size_t nprocs,
+                                            const pmix_info_t info[], size_t ninfo,
+                                            pmix_info_t **results, size_t *nresult);
+
+PMIX_EXPORT pmix_status_t PMIx_Group_invite_nb(const char grp[],
+                                               const pmix_proc_t procs[], size_t nprocs,
+                                               const pmix_info_t info[], size_t ninfo,
+                                               pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Respond to an invitation to join a group that is being asynchronously constructed.
+ *
+ * The process must have registered for the PMIX_GROUP_INVITED event in order to be
+ * notified of the invitation. When ready to respond, the process provides a response
+ * using the appropriate form of PMIx_Group_join.
+ *
+ * Critical Note: Since the process is alerted to the invitation in a PMIx event handler,
+ * the process must not use the blocking form of this call unless it first “thread shifts”
+ * out of the handler and into its own thread context. Likewise, while it is safe to call
+ * the non-blocking form of the API from the event handler, the process must not block
+ * in the handler while waiting for the callback function to be called.
+ *
+ * Calling this function causes the group “leader” to be notified that the process has
+ * either accepted or declined the request. The blocking form of the API will return
+ * once the group has been completely constructed or the group’s construction has failed
+ * (as determined by the leader) – likewise, the callback function of the non-blocking
+ * form will be executed upon the same conditions.
+ *
+ * Failure of the leader at any time will cause a PMIX_GROUP_LEADER_FAILED event
+ * to be delivered to all participants so they can optionally declare a new leader.
+ * A new leader is identified by providing the PMIX_GROUP_LEADER attribute in
+ * the results array in the return of the event handler. Only one process is
+ * allowed to return that attribute, declaring itself as the new leader. Results
+ * of the leader selection will be communicated to all participants via a
+ * PMIX_GROUP_LEADER_SELECTED event identifying the new leader. If no leader
+ * was selected, then the status code provided in the event handler will provide
+ * an error value so the participants can take appropriate action.
+ *
+ * Any participant that returns PMIX_GROUP_CONSTRUCT_ABORT from the leader failed
+ * event handler will cause all participants to receive an event notifying them
+ * of that status. Similarly, the leader may elect to abort the procedure
+ * by either returning PMIX_GROUP_CONSTRUCT_ABORT from the handler assigned
+ * to the PMIX_GROUP_INVITE_ACCEPTED or PMIX_GROUP_INVITE_DECLINED codes, or
+ * by generating an event for the abort code. Abort events will be sent to
+ * all invited participants.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Group_join(const char grp[],
+                                          const pmix_proc_t *leader,
+                                          pmix_group_opt_t opt,
+                                          const pmix_info_t info[], size_t ninfo,
+                                          pmix_info_t **results, size_t *nresult);
+
+PMIX_EXPORT pmix_status_t PMIx_Group_join_nb(const char grp[],
+                                             const pmix_proc_t *leader,
+                                             pmix_group_opt_t opt,
+                                             const pmix_info_t info[], size_t ninfo,
+                                             pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Leave a PMIx Group. Calls to PMIx_Group_leave (or its non-blocking form) will cause
+ * a PMIX_GROUP_LEFT event to be generated notifying all members of the group of the
+ * caller’s departure. The function will return (or the non-blocking function will
+ * execute the specified callback function) once the event has been locally generated
+ * and is not indicative of remote receipt. All PMIx-based collectives such as
+ * PMIx_Fence in action across the group will automatically be adjusted if the
+ * collective was called with the PMIX_GROUP_FT_COLLECTIVE attribute (default is
+ * false) – otherwise, the standard error return behavior will be provided.
+ *
+ * Critical Note: The PMIx_Group_leave API is intended solely for asynchronous
+ * departures of individual processes from a group as it is not a scalable
+ * operation – i.e., when a process determines it should no longer be a part of a
+ * defined group, but the remainder of the group retains a valid reason to continue
+ * in existence. Developers are advised to use PMIx_Group_destruct (or its
+ * non-blocking form) for all other scenarios as it represents a more scalable
+ * operation.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Group_leave(const char grp[],
+                                           const pmix_info_t info[], size_t ninfo);
+
+PMIX_EXPORT pmix_status_t PMIx_Group_leave_nb(const char grp[],
+                                              const pmix_info_t info[], size_t ninfo,
+                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/* Destruct a group identified by the provided group identifier. Both blocking and
+ * non-blocking versions are provided (the callback function for the non-blocking
+ * form will be called once all members of the group have called “destruct”).
+ * Processes may engage in multiple simultaneous group destruct operations as
+ * desired so long as each involves a unique group ID. The info array can be used
+ * to pass user-level directives regarding timeout constraints and other options
+ * available from the PMIx server.
+ *
+ * Some relevant attributes for this operation:
+ *
+ *    PMIX_TIMEOUT (int): return an error if the group doesn’t destruct within the
+ *                        specified number of seconds. Targets the scenario where
+ *                        a process fails to call PMIx_Group_destruct due to hanging
+ *
+ * The destruct API will return an error if any group process fails or terminates
+ * prior to calling PMIx_Group_destruct or its non-blocking version unless the
+ * PMIX_GROUP_NOTIFY_TERMINATION attribute was provided (with a value of true) at
+ * time of group construction. If notification was requested, then a event will
+ * be delivered (using PMIX_GROUP_MEMBER_FAILED) for each process that fails to
+ * call destruct and the destruct tracker updated to account for the lack of
+ * participation. The PMIx_Group_destruct operation will subsequently return
+ * PMIX_SUCCESS when the remaining processes have all called destruct – i.e., the
+ * event will serve in place of return of an error.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Group_destruct(const char grp[],
+                                              const pmix_info_t info[], size_t ninfo);
+
+PMIX_EXPORT pmix_status_t PMIx_Group_destruct_nb(const char grp[],
+                                                 const pmix_info_t info[], size_t ninfo,
+                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/****************************************/
+/****    COMMON SUPPORT FUNCTIONS    ****/
+/****************************************/
+
+/******     EVENT NOTIFICATION SUPPORT      ******/
+/* Register an event handler to report events. Three types of events
+ * can be reported:
+ *
+ * (a) those that occur within the client library, but are not
+ *     reportable via the API itself (e.g., loss of connection to
+ *     the server). These events typically occur during behind-the-scenes
+ *     non-blocking operations.
+ *
+ * (b) job-related events such as the failure of another process in
+ *     the job or in any connected job, impending failure of hardware
+ *     within the job's usage footprint, etc.
+ *
+ * (c) system notifications that are made available by the local
+ *     administrators
+ *
+ * By default, only events that directly affect the process and/or
+ * any process to which it is connected (via the PMIx_Connect call)
+ * will be reported. Options to modify that behavior can be provided
+ * in the info array
+ *
+ * Both the client application and the resource manager can register
+ * err handlers for specific events. PMIx client/server calls the registered
+ * err handler upon receiving event notify notification (via PMIx_Notify_event)
+ * from the other end (Resource Manager/Client application).
+ *
+ * Multiple err handlers can be registered for different events. PMIX returns
+ * an integer reference to each register handler in the callback fn. The caller
+ * must retain the reference in order to deregister the evhdlr.
+ * Modification of the notification behavior can be accomplished by
+ * deregistering the current evhdlr, and then registering it
+ * using a new set of info values.
+ *
+ * If cbfunc is NULL, then this is treated as a BLOCKING call - a positive
+ * return value represents the reference ID for the request, while
+ * negative values indicate the corresponding error
+ *
+ * See pmix_common.h for a description of the notification function */
+PMIX_EXPORT pmix_status_t PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes,
+                                                      pmix_info_t info[], size_t ninfo,
+                                                      pmix_notification_fn_t evhdlr,
+                                                      pmix_hdlr_reg_cbfunc_t cbfunc,
+                                                      void *cbdata);
+
+/* Deregister an event handler
+ * evhdlr_ref is the reference returned by PMIx from the call to
+ * PMIx_Register_event_handler. If non-NULL, the provided cbfunc
+ * will be called to confirm removal of the designated handler */
+PMIX_EXPORT pmix_status_t PMIx_Deregister_event_handler(size_t evhdlr_ref,
+                                                        pmix_op_cbfunc_t cbfunc,
+                                                        void *cbdata);
+
+/* Report an event for notification via any
+ * registered evhdlr.
+ *
+ * This function allows the host server to direct the server
+ * convenience library to notify all registered local procs of
+ * an event. The event can be local, or anywhere in the cluster.
+ * The status indicates the event being reported.
+ *
+ * The client application can also call this function to notify the
+ * resource manager and/or other processes of an event it encountered.
+ * It can also be used to asynchronously notify other parts of its
+ * own internal process - e.g., for one library to notify another
+ * when initialized inside the process.
+ *
+ * status - status code indicating the event being reported
+ *
+ * source - the process that generated the event
+ *
+ * range - the range in which the event is to be reported. For example,
+ *         a value of PMIX_RANGE_LOCAL would instruct the system
+ *         to only notify procs on the same local node as the
+ *         event generator.
+ *
+ * info - an array of pmix_info_t structures provided by the event
+ *        generator to pass any additional information about the
+ *        event. This can include an array of pmix_proc_t structs
+ *        describing the processes impacted by the event, the nature
+ *        of the event and its severity, etc. The precise contents
+ *        of the array will depend on the event generator.
  *
- * source - the nspace/rank of the process that generated the data
+ * ninfo - number of elements in the info array
+ *
+ * cbfunc - callback function to be called upon completion of the
+ *          notify_event function's actions. Note that any messages
+ *          will have been queued, but may not have been transmitted
+ *          by this time. Note that the caller is required to maintain
+ *          the input data until the callback function has been executed!
+ *          If cbfunc is NULL, then this is treated as a BLOCKING call and
+ *          the result of the operation is provided in the returned
+ *          status
+ *
+ * cbdata - the caller's provided void* object
+ */
+PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
+                                            const pmix_proc_t *source,
+                                            pmix_data_range_t range,
+                                            const pmix_info_t info[], size_t ninfo,
+                                            pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+
+/******    FABRIC-RELATED APIS    ******/
+/* Register for access to fabric-related information, including
+ * communication cost matrix. This call must be made prior to
+ * requesting information from a fabric.
+ *
+ * fabric - address of a pmix_fabric_t (backed by storage). User
+ *          may populate the "name" field at will - PMIx does not
+ *          utilize this field
+ *
+ * directives - an optional array of values indicating desired
+ *              behaviors and/or fabric to be accessed. If NULL,
+ *              then the highest priority available fabric will
+ *              be used
  *
- * payload - pointer to a PMIx byte object containing the data. Note that
- *           multiple strings may be included, and that the data may
- *           _not_ be NULL terminated
+ * ndirs - number of elements in the directives array
  *
- * info - an optional array of info provided by the source containing
- *        metadata about the payload. This could include PMIX_IOF_COMPLETE
+ * Return values include:
  *
- * ninfo - number of elements in the optional info array
+ * PMIX_SUCCESS - indicates success
  */
- typedef void (*pmix_iof_cbfunc_t)(size_t iofhdlr, pmix_iof_channel_t channel,
-                                   pmix_proc_t *source, pmix_byte_object_t *payload,
-                                   pmix_info_t info[], size_t ninfo);
+PMIX_EXPORT pmix_status_t PMIx_Fabric_register(pmix_fabric_t *fabric,
+                                               const pmix_info_t directives[],
+                                               size_t ndirs);
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_register_nb(pmix_fabric_t *fabric,
+												  const pmix_info_t directives[],
+											      size_t ndirs,
+												  pmix_op_cbfunc_t cbfunc, void *cbdata);
 
 
-/* Register to receive output forwarded from a remote process.
+/* Update fabric-related information. This call can be made at any time to request an update of the
+ * fabric information contained in the provided pmix_fabric_t object. The caller is not allowed
+ * to access the provided pmix_fabric_t until the call has returned.
  *
- * procs - array of identifiers for sources whose IO is being
- *         requested. Wildcard rank indicates that all procs
- *         in the specified nspace are included in the request
+ * fabric - pointer to the pmix_fabric_t struct provided to
+ *          the registration function
  *
- * nprocs - number of identifiers in the procs array
+ * Return values include:
  *
- * directives - optional array of attributes to control the
- *              behavior of the request. For example, this
- *              might include directives on buffering IO
- *              before delivery, and/or directives to include
- *              or exclude any backlogged data
+ * PMIX_SUCCESS - indicates successful update
+ */
+PMIX_EXPORT pmix_status_t PMIx_Fabric_update(pmix_fabric_t *fabric);
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_update_nb(pmix_fabric_t *fabric,
+											    pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+
+/* Deregister a fabric object, providing an opportunity for
+ * the PMIx server library to cleanup any information
+ * (e.g., cost matrix) associated with it
  *
- * ndirs - number of elements in the directives array
+ * fabric - pointer to the pmix_fabric_t struct provided
+ *          to the registration function
+ */
+PMIX_EXPORT pmix_status_t PMIx_Fabric_deregister(pmix_fabric_t *fabric);
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_deregister_nb(pmix_fabric_t *fabric,
+												    pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+
+/* Compute the distance information for the current process
+ * Returns an array of distances from the current process
+ * location to each of the local devices of the specified type(s)
+ *
+ * distances - pointer to location where the array of
+ *             distances is to be returned
  *
- * channel - bitmask of IO channels included in the request.
- *           NOTE: STDIN is not supported as it will always
- *           be delivered to the stdin file descriptor
- *
- * cbfunc - function to be called when relevant IO is received. A
- *          NULL indicates that the IO is to be written to stdout
- *          or stderr as per the originating channel
- *
- * regcbfunc - since registration is async, this is the
- *             function to be called when registration is
- *             completed. The function itself will return
- *             a non-success error if the registration cannot
- *             be submitted - in this case, the regcbfunc
- *             will _not_ be called.
- *
- * cbdata - pointer to object to be returned in regcbfunc
- */
-PMIX_EXPORT pmix_status_t PMIx_IOF_pull(const pmix_proc_t procs[], size_t nprocs,
-                                        const pmix_info_t directives[], size_t ndirs,
-                                        pmix_iof_channel_t channel, pmix_iof_cbfunc_t cbfunc,
-                                        pmix_hdlr_reg_cbfunc_t regcbfunc, void *regcbdata);
-
-/* Deregister from output forwarded from a remote process.
- *
- * iofhdlr - the registration number returned from the
- *           call to PMIx_IOF_pull
- *
- * directives - optional array of attributes to control the
- *              behavior of the request. For example, this
- *              might include directives regarding what to
- *              do with any data currently in the IO buffer
- *              for this process
- *
- * cbfunc - function to be called when deregistration has
- *          been completed. Note that any IO to be flushed
- *          may continue to be received after deregistration
- *          has completed.
+ * ndist - number of elements in the distances array
  *
- * cbdata - pointer to object to be returned in cbfunc
+ * Return values include:
+ *
+ * PMIX_SUCCESS - distance array was successfully returned
+ * Other error
  */
-PMIX_EXPORT pmix_status_t PMIx_IOF_deregister(size_t iofhdlr,
-                                              const pmix_info_t directives[], size_t ndirs,
-                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
+PMIX_EXPORT pmix_status_t PMIx_Compute_distances(pmix_topology_t *topo,
+                                                 pmix_cpuset_t *cpuset,
+                                                 pmix_info_t info[], size_t ninfo,
+                                                 pmix_device_distance_t *distances[],
+                                                 size_t *ndist);
+
+PMIX_EXPORT pmix_status_t PMIx_Compute_distances_nb(pmix_topology_t *topo,
+                                                    pmix_cpuset_t *cpuset,
+                                                    pmix_info_t info[], size_t ninfo,
+                                                    pmix_device_dist_cbfunc_t cbfunc,
+                                                    void *cbdata);
+
+/* Load the local hwardware topology description
+ *
+ * topo - pointer to a pmix_topology_t object. This object
+ *        must be initialized! If the a particular "source"
+ *        for the topology is required (e.g., "hwloc"), then
+ *        the "source" field of the object must be set to
+ *        that value
+ *
+ * Return values include:
+ * PMIX_SUCCESS - indicates return of a valid value
+ * PMIX_ERR_NOT_FOUND - provided source is not available
+ * PMIX_ERR_NOT_SUPPORTED - current implementation does not support this option
+ */
+PMIX_EXPORT pmix_status_t PMIx_Load_topology(pmix_topology_t *topo);
+
 
-/* Push data collected locally (typically from stdin) to
- * stdin of target recipients.
+/* Get the PU binding bitmap from its string representation
  *
- * targets - array of process identifiers to which the data is to be delivered. Note
- *           that a WILDCARD rank indicates that all procs in the given nspace are
- *           to receive a copy of the data
- *
- * ntargets - number of procs in the targets array
- *
- * directives - optional array of attributes to control the
- *              behavior of the request. For example, this
- *              might include directives on buffering IO
- *              before delivery, and/or directives to include
- *              or exclude any backlogged data
+ * cpuset_string - string representation of the binding bitmap
+ *                 (as returned by PMIx_Get using the PMIX_CPUSET key)
  *
- * ndirs - number of elements in the directives array
+ * cpuset - pointer to a pmix_cpuset_t object where the result
+ *          is to be stored
+ *
+ * Return values include:
+ * PMIX_SUCCESS - indicates return of a valid value
+ * PMIX_ERR_NOT_FOUND - provided source is not available
+ * PMIX_ERR_NOT_SUPPORTED - current implementation does not support this option
+ */
+PMIX_EXPORT pmix_status_t PMIx_Parse_cpuset_string(const char *cpuset_string,
+	                                               pmix_cpuset_t *cpuset);
+
+PMIX_EXPORT pmix_status_t PMIx_Get_cpuset(pmix_cpuset_t *cpuset, pmix_bind_envelope_t ref);
+
+/* Get the relative locality of two local processes given their locality strings.
+ *
+ * locality1 - String returned by the PMIx_server_generate_locality_string API
+ *
+ * locality2 - String returned by the PMIx_server_generate_locality_string API
+ *
+ * locality - Pointer to the location where the relative locality bitmask is
+ *            to be constructed
+ *
+ * Return values include:
+ * PMIX_SUCCESS - indicates return of a valid value
+ * other error constant
+ */
+PMIX_EXPORT pmix_status_t PMIx_Get_relative_locality(const char *locality1,
+	                                                 const char *locality2,
+	                                                 pmix_locality_t *locality);
+
+PMIX_EXPORT void PMIx_Progress(void);
+
+/******    PRETTY-PRINT DEFINED VALUE TYPES     ******/
+/* Provide a string representation for several types of value. Note
+ * that the provided string is statically defined and must NOT be
+ * free'd. Supported value types:
+ *
+ * - pmix_status_t (PMIX_STATUS)
+ * - pmix_scope_t   (PMIX_SCOPE)
+ * - pmix_persistence_t  (PMIX_PERSIST)
+ * - pmix_data_range_t   (PMIX_DATA_RANGE)
+ * - pmix_info_directives_t   (PMIX_INFO_DIRECTIVES)
+ * - pmix_data_type_t   (PMIX_DATA_TYPE)
+ * - pmix_alloc_directive_t  (PMIX_ALLOC_DIRECTIVE)
+ * - pmix_iof_channel_t  (PMIX_IOF_CHANNEL)
+ * - pmix_job_state_t  (PMIX_JOB_STATE)
+ * - pmix_proc_state_t  (PMIX_PROC_STATE)
+ */
+PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t status);
+PMIX_EXPORT const char* PMIx_Proc_state_string(pmix_proc_state_t state);
+PMIX_EXPORT const char* PMIx_Scope_string(pmix_scope_t scope);
+PMIX_EXPORT const char* PMIx_Persistence_string(pmix_persistence_t persist);
+PMIX_EXPORT const char* PMIx_Data_range_string(pmix_data_range_t range);
+PMIX_EXPORT const char* PMIx_Info_directives_string(pmix_info_directives_t directives);
+PMIX_EXPORT const char* PMIx_Data_type_string(pmix_data_type_t type);
+PMIX_EXPORT const char* PMIx_Alloc_directive_string(pmix_alloc_directive_t directive);
+PMIX_EXPORT const char* PMIx_IOF_channel_string(pmix_iof_channel_t channel);
+PMIX_EXPORT const char* PMIx_Job_state_string(pmix_job_state_t state);
+PMIX_EXPORT const char* PMIx_Get_attribute_string(char *attribute);
+PMIX_EXPORT const char* PMIx_Get_attribute_name(char *attrstring);
+PMIX_EXPORT const char* PMIx_Link_state_string(pmix_link_state_t state);
+PMIX_EXPORT const char* PMIx_Device_type_string(pmix_device_type_t type);
+
+/* Get the PMIx version string. Note that the provided string is
+ * statically defined and must NOT be free'd  */
+PMIX_EXPORT const char* PMIx_Get_version(void);
+
+/* Store some data locally for retrieval by other areas of the
+ * proc. This is data that has only internal scope - it will
+ * never be "pushed" externally */
+PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc,
+                                              const pmix_key_t key, pmix_value_t *val);
+
+
+/******    DATA BUFFER PACK/UNPACK SUPPORT    ******/
+/**
+ * Top-level interface function to pack one or more values into a
+ * buffer.
+ *
+ * The pack function packs one or more values of a specified type into
+ * the specified buffer.  The buffer must have already been
+ * initialized via the PMIX_DATA_BUFFER_CREATE or PMIX_DATA_BUFFER_CONSTRUCT
+ * call - otherwise, the pack_value function will return an error.
+ * Providing an unsupported type flag will likewise be reported as an error.
+ *
+ * Note that any data to be packed that is not hard type cast (i.e.,
+ * not type cast to a specific size) may lose precision when unpacked
+ * by a non-homogeneous recipient.  The PACK function will do its best to deal
+ * with heterogeneity issues between the packer and unpacker in such
+ * cases. Sending a number larger than can be handled by the recipient
+ * will return an error code (generated upon unpacking) -
+ * the error cannot be detected during packing.
+ *
+ * The identity of the intended recipient of the packed buffer (i.e., the
+ * process that will be unpacking it) is used solely to resolve any data type
+ * differences between PMIx versions. The recipient must, therefore, be
+ * known to the user prior to calling the pack function so that the
+ * PMIx library is aware of the version the recipient is using.
+ *
+ * @param *target Pointer to a pmix_proc_t structure containing the
+ * nspace/rank of the process that will be unpacking the final buffer.
+ * A NULL value may be used to indicate that the target is based on
+ * the same PMIx version as the caller.
+ *
+ * @param *buffer A pointer to the buffer into which the value is to
+ * be packed.
+ *
+ * @param *src A void* pointer to the data that is to be packed. Note
+ * that strings are to be passed as (char **) - i.e., the caller must
+ * pass the address of the pointer to the string as the void*. This
+ * allows PMIx to use a single pack function, but still allow
+ * the caller to pass multiple strings in a single call.
+ *
+ * @param num_values An int32_t indicating the number of values that are
+ * to be packed, beginning at the location pointed to by src. A string
+ * value is counted as a single value regardless of length. The values
+ * must be contiguous in memory. Arrays of pointers (e.g., string
+ * arrays) should be contiguous, although (obviously) the data pointed
+ * to need not be contiguous across array entries.
+ *
+ * @param type The type of the data to be packed - must be one of the
+ * PMIX defined data types.
+ *
+ * @retval PMIX_SUCCESS The data was packed as requested.
+ *
+ * @retval PMIX_ERROR(s) An appropriate PMIX error code indicating the
+ * problem encountered. This error code should be handled
+ * appropriately.
+ *
+ * @code
+ * pmix_data_buffer_t *buffer;
+ * int32_t src;
+ *
+ * PMIX_DATA_BUFFER_CREATE(buffer);
+ * status_code = PMIx_Data_pack(buffer, &src, 1, PMIX_INT32);
+ * @endcode
+ */
+PMIX_EXPORT pmix_status_t PMIx_Data_pack(const pmix_proc_t *target,
+                                         pmix_data_buffer_t *buffer,
+                                         void *src, int32_t num_vals,
+                                         pmix_data_type_t type);
+
+/**
+ * Unpack values from a buffer.
+ *
+ * The unpack function unpacks the next value (or values) of a
+ * specified type from the specified buffer.
+ *
+ * The buffer must have already been initialized via an PMIX_DATA_BUFFER_CREATE or
+ * PMIX_DATA_BUFFER_CONSTRUCT call (and assumedly filled with some data) -
+ * otherwise, the unpack_value function will return an
+ * error. Providing an unsupported type flag will likewise be reported
+ * as an error, as will specifying a data type that DOES NOT match the
+ * type of the next item in the buffer. An attempt to read beyond the
+ * end of the stored data held in the buffer will also return an
+ * error.
+ *
+ * NOTE: it is possible for the buffer to be corrupted and that
+ * PMIx will *think* there is a proper variable type at the
+ * beginning of an unpack region - but that the value is bogus (e.g., just
+ * a byte field in a string array that so happens to have a value that
+ * matches the specified data type flag). Therefore, the data type error check
+ * is NOT completely safe. This is true for ALL unpack functions.
+ *
+ *
+ * Unpacking values is a "nondestructive" process - i.e., the values are
+ * not removed from the buffer. It is therefore possible for the caller
+ * to re-unpack a value from the same buffer by resetting the unpack_ptr.
+ *
+ * Warning: The caller is responsible for providing adequate memory
+ * storage for the requested data. As noted below, the user
+ * must provide a parameter indicating the maximum number of values that
+ * can be unpacked into the allocated memory. If more values exist in the
+ * buffer than can fit into the memory storage, then the function will unpack
+ * what it can fit into that location and return an error code indicating
+ * that the buffer was only partially unpacked.
+ *
+ * Note that any data that was not hard type cast (i.e., not type cast
+ * to a specific size) when packed may lose precision when unpacked by
+ * a non-homogeneous recipient.  PMIx will do its best to deal with
+ * heterogeneity issues between the packer and unpacker in such
+ * cases. Sending a number larger than can be handled by the recipient
+ * will return an error code generated upon unpacking - these errors
+ * cannot be detected during packing.
+ *
+ * The identity of the source of the packed buffer (i.e., the
+ * process that packed it) is used solely to resolve any data type
+ * differences between PMIx versions. The source must, therefore, be
+ * known to the user prior to calling the unpack function so that the
+ * PMIx library is aware of the version the source used.
+ *
+ * @param *source Pointer to a pmix_proc_t structure containing the
+ * nspace/rank of the process that packed the provided buffer.
+ * A NULL value may be used to indicate that the source is based on
+ * the same PMIx version as the caller.
+ *
+ * @param *buffer A pointer to the buffer from which the value will be
+ * extracted.
+ *
+ * @param *dest A void* pointer to the memory location into which the
+ * data is to be stored. Note that these values will be stored
+ * contiguously in memory. For strings, this pointer must be to (char
+ * **) to provide a means of supporting multiple string
+ * operations. The unpack function will allocate memory for each
+ * string in the array - the caller must only provide adequate memory
+ * for the array of pointers.
+ *
+ * @param type The type of the data to be unpacked - must be one of
+ * the BFROP defined data types.
+ *
+ * @retval *max_num_values The number of values actually unpacked. In
+ * most cases, this should match the maximum number provided in the
+ * parameters - but in no case will it exceed the value of this
+ * parameter.  Note that if you unpack fewer values than are actually
+ * available, the buffer will be in an unpackable state - the function will
+ * return an error code to warn of this condition.
+ *
+ * @note The unpack function will return the actual number of values
+ * unpacked in this location.
+ *
+ * @retval PMIX_SUCCESS The next item in the buffer was successfully
+ * unpacked.
+ *
+ * @retval PMIX_ERROR(s) The unpack function returns an error code
+ * under one of several conditions: (a) the number of values in the
+ * item exceeds the max num provided by the caller; (b) the type of
+ * the next item in the buffer does not match the type specified by
+ * the caller; or (c) the unpack failed due to either an error in the
+ * buffer or an attempt to read past the end of the buffer.
+ *
+ * @code
+ * pmix_data_buffer_t *buffer;
+ * int32_t dest;
+ * char **string_array;
+ * int32_t num_values;
+ *
+ * num_values = 1;
+ * status_code = PMIx_Data_unpack(buffer, (void*)&dest, &num_values, PMIX_INT32);
+ *
+ * num_values = 5;
+ * string_array = pmix_malloc(num_values*sizeof(char *));
+ * status_code = PMIx_Data_unpack(buffer, (void*)(string_array), &num_values, PMIX_STRING);
+ *
+ * @endcode
+ */
+PMIX_EXPORT pmix_status_t PMIx_Data_unpack(const pmix_proc_t *source,
+                                           pmix_data_buffer_t *buffer, void *dest,
+                                           int32_t *max_num_values,
+                                           pmix_data_type_t type);
+
+/**
+ * Copy a data value from one location to another.
  *
- * bo - pointer to a byte object containing the stdin data
+ * Since registered data types can be complex structures, the system
+ * needs some way to know how to copy the data from one location to
+ * another (e.g., for storage in the registry). This function, which
+ * can call other copy functions to build up complex data types, defines
+ * the method for making a copy of the specified data type.
  *
- * cbfunc - callback function when the data has been forwarded.
+ * @param **dest The address of a pointer into which the
+ * address of the resulting data is to be stored.
+ *
+ * @param *src A pointer to the memory location from which the
+ * data is to be copied.
+ *
+ * @param type The type of the data to be copied - must be one of
+ * the PMIx defined data types.
+ *
+ * @retval PMIX_SUCCESS The value was successfully copied.
+ *
+ * @retval PMIX_ERROR(s) An appropriate error code.
  *
- * cbdata - object to be returned in cbfunc
  */
-PMIX_EXPORT pmix_status_t PMIx_IOF_push(const pmix_proc_t targets[], size_t ntargets,
-                                        pmix_byte_object_t *bo,
-                                        const pmix_info_t directives[], size_t ndirs,
-                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
+PMIX_EXPORT pmix_status_t PMIx_Data_copy(void **dest, void *src,
+                                         pmix_data_type_t type);
+
+/**
+ * Print a data value.
+ *
+ * Since registered data types can be complex structures, the system
+ * needs some way to know how to print them (i.e., convert them to a string
+ * representation). Provided for debug purposes.
+ *
+ * @retval PMIX_SUCCESS The value was successfully printed.
+ *
+ * @retval PMIX_ERROR(s) An appropriate error code.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Data_print(char **output, char *prefix,
+                                          void *src, pmix_data_type_t type);
+
+/**
+ * Copy a payload from one buffer to another
+ *
+ * This function will append a copy of the payload in one buffer into
+ * another buffer.
+ * NOTE: This is NOT a destructive procedure - the
+ * source buffer's payload will remain intact, as will any pre-existing
+ * payload in the destination's buffer.
+ */
+PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest,
+                                                 pmix_data_buffer_t *src);
 
-/****************************************/
-/****    COMMON SUPPORT FUNCTIONS    ****/
-/* Found in pmix_common.h.in in the v3.x and earlier series */
-/****************************************/
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
diff -Nru pmix-3.2.2~rc1/include/pmix_rename.h.in pmix-4.0.0/include/pmix_rename.h.in
--- pmix-3.2.2~rc1/include/pmix_rename.h.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmix_rename.h.in	1970-01-01 00:00:00.000000000 +0000
@@ -1,618 +0,0 @@
-#define PMI2_Abort                                              @PMIX_RENAME@PMI2_Abort
-#define PMI2_Finalize                                           @PMIX_RENAME@PMI2_Finalize
-#define PMI2_Info_GetJobAttr                                    @PMIX_RENAME@PMI2_Info_GetJobAttr
-#define PMI2_Info_GetJobAttrIntArray                            @PMIX_RENAME@PMI2_Info_GetJobAttrIntArray
-#define PMI2_Info_GetNodeAttr                                   @PMIX_RENAME@PMI2_Info_GetNodeAttr
-#define PMI2_Info_GetNodeAttrIntArray                           @PMIX_RENAME@PMI2_Info_GetNodeAttrIntArray
-#define PMI2_Info_GetSize                                       @PMIX_RENAME@PMI2_Info_GetSize
-#define PMI2_Info_PutNodeAttr                                   @PMIX_RENAME@PMI2_Info_PutNodeAttr
-#define PMI2_Init                                               @PMIX_RENAME@PMI2_Init
-#define PMI2_Initialized                                        @PMIX_RENAME@PMI2_Initialized
-#define PMI2_Job_Connect                                        @PMIX_RENAME@PMI2_Job_Connect
-#define PMI2_Job_Disconnect                                     @PMIX_RENAME@PMI2_Job_Disconnect
-#define PMI2_Job_GetId                                          @PMIX_RENAME@PMI2_Job_GetId
-#define PMI2_Job_GetRank                                        @PMIX_RENAME@PMI2_Job_GetRank
-#define PMI2_Job_Spawn                                          @PMIX_RENAME@PMI2_Job_Spawn
-#define PMI2_KVS_Fence                                          @PMIX_RENAME@PMI2_KVS_Fence
-#define PMI2_KVS_Get                                            @PMIX_RENAME@PMI2_KVS_Get
-#define PMI2_KVS_Put                                            @PMIX_RENAME@PMI2_KVS_Put
-#define PMI2_Nameserv_lookup                                    @PMIX_RENAME@PMI2_Nameserv_lookup
-#define PMI2_Nameserv_publish                                   @PMIX_RENAME@PMI2_Nameserv_publish
-#define PMI2_Nameserv_unpublish                                 @PMIX_RENAME@PMI2_Nameserv_unpublish
-#define PMI_Abort                                               @PMIX_RENAME@PMI_Abort
-#define PMI_Args_to_keyval                                      @PMIX_RENAME@PMI_Args_to_keyval
-#define PMI_Barrier                                             @PMIX_RENAME@PMI_Barrier
-#define PMI_Finalize                                            @PMIX_RENAME@PMI_Finalize
-#define PMI_Free_keyvals                                        @PMIX_RENAME@PMI_Free_keyvals
-#define PMI_Get_appnum                                          @PMIX_RENAME@PMI_Get_appnum
-#define PMI_Get_clique_ranks                                    @PMIX_RENAME@PMI_Get_clique_ranks
-#define PMI_Get_clique_size                                     @PMIX_RENAME@PMI_Get_clique_size
-#define PMI_Get_id                                              @PMIX_RENAME@PMI_Get_id
-#define PMI_Get_id_length_max                                   @PMIX_RENAME@PMI_Get_id_length_max
-#define PMI_Get_kvs_domain_id                                   @PMIX_RENAME@PMI_Get_kvs_domain_id
-#define PMI_Get_options                                         @PMIX_RENAME@PMI_Get_options
-#define PMI_Get_rank                                            @PMIX_RENAME@PMI_Get_rank
-#define PMI_Get_size                                            @PMIX_RENAME@PMI_Get_size
-#define PMI_Get_universe_size                                   @PMIX_RENAME@PMI_Get_universe_size
-#define PMI_Init                                                @PMIX_RENAME@PMI_Init
-#define PMI_Initialized                                         @PMIX_RENAME@PMI_Initialized
-#define PMI_KVS_Commit                                          @PMIX_RENAME@PMI_KVS_Commit
-#define PMI_KVS_Create                                          @PMIX_RENAME@PMI_KVS_Create
-#define PMI_KVS_Destroy                                         @PMIX_RENAME@PMI_KVS_Destroy
-#define PMI_KVS_Get                                             @PMIX_RENAME@PMI_KVS_Get
-#define PMI_KVS_Get_key_length_max                              @PMIX_RENAME@PMI_KVS_Get_key_length_max
-#define PMI_KVS_Get_my_name                                     @PMIX_RENAME@PMI_KVS_Get_my_name
-#define PMI_KVS_Get_name_length_max                             @PMIX_RENAME@PMI_KVS_Get_name_length_max
-#define PMI_KVS_Get_value_length_max                            @PMIX_RENAME@PMI_KVS_Get_value_length_max
-#define PMI_KVS_Iter_first                                      @PMIX_RENAME@PMI_KVS_Iter_first
-#define PMI_KVS_Iter_next                                       @PMIX_RENAME@PMI_KVS_Iter_next
-#define PMI_KVS_Put                                             @PMIX_RENAME@PMI_KVS_Put
-#define PMI_Lookup_name                                         @PMIX_RENAME@PMI_Lookup_name
-#define PMI_Parse_option                                        @PMIX_RENAME@PMI_Parse_option
-#define PMI_Publish_name                                        @PMIX_RENAME@PMI_Publish_name
-#define PMI_Spawn_multiple                                      @PMIX_RENAME@PMI_Spawn_multiple
-#define PMI_Unpublish_name                                      @PMIX_RENAME@PMI_Unpublish_name
-#define PMIx_Abort                                              @PMIX_RENAME@PMIx_Abort
-#define PMIx_Allocation_request_nb                              @PMIX_RENAME@PMIx_Allocation_request_nb
-#define PMIx_Alloc_directive_string                             @PMIX_RENAME@PMIx_Alloc_directive_string
-#define pmix_argv_append                                        @PMIX_RENAME@pmix_argv_append
-#define pmix_argv_append_nosize                                 @PMIX_RENAME@pmix_argv_append_nosize
-#define pmix_argv_append_unique_nosize                          @PMIX_RENAME@pmix_argv_append_unique_nosize
-#define pmix_argv_copy                                          @PMIX_RENAME@pmix_argv_copy
-#define pmix_argv_count                                         @PMIX_RENAME@pmix_argv_count
-#define pmix_argv_delete                                        @PMIX_RENAME@pmix_argv_delete
-#define pmix_argv_free                                          @PMIX_RENAME@pmix_argv_free
-#define pmix_argv_insert                                        @PMIX_RENAME@pmix_argv_insert
-#define pmix_argv_insert_element                                @PMIX_RENAME@pmix_argv_insert_element
-#define pmix_argv_join                                          @PMIX_RENAME@pmix_argv_join
-#define pmix_argv_join_range                                    @PMIX_RENAME@pmix_argv_join_range
-#define pmix_argv_len                                           @PMIX_RENAME@pmix_argv_len
-#define pmix_argv_prepend_nosize                                @PMIX_RENAME@pmix_argv_prepend_nosize
-#define pmix_argv_split                                         @PMIX_RENAME@pmix_argv_split
-#define pmix_argv_split_with_empty                              @PMIX_RENAME@pmix_argv_split_with_empty
-#define pmix_asprintf                                           @PMIX_RENAME@pmix_asprintf
-#define pmix_basename                                           @PMIX_RENAME@pmix_basename
-#define pmix_bfrop_base_copy_persist                            @PMIX_RENAME@pmix_bfrop_base_copy_persist
-#define pmix_bfrop_base_select                                  @PMIX_RENAME@pmix_bfrop_base_select
-#define pmix_bfrop_buffer_extend                                @PMIX_RENAME@pmix_bfrop_buffer_extend
-#define pmix_bfrop_get_data_type                                @PMIX_RENAME@pmix_bfrop_get_data_type
-#define pmix_bfrops_base_assign_module                          @PMIX_RENAME@pmix_bfrops_base_assign_module
-#define pmix_bfrops_base_copy                                   @PMIX_RENAME@pmix_bfrops_base_copy
-#define pmix_bfrops_base_copy_app                               @PMIX_RENAME@pmix_bfrops_base_copy_app
-#define pmix_bfrops_base_copy_array                             @PMIX_RENAME@pmix_bfrops_base_copy_array
-#define pmix_bfrops_base_copy_bo                                @PMIX_RENAME@pmix_bfrops_base_copy_bo
-#define pmix_bfrops_base_copy_buf                               @PMIX_RENAME@pmix_bfrops_base_copy_buf
-#define pmix_bfrops_base_copy_darray                            @PMIX_RENAME@pmix_bfrops_base_copy_darray
-#define pmix_bfrops_base_copy_info                              @PMIX_RENAME@pmix_bfrops_base_copy_info
-#define pmix_bfrops_base_copy_kval                              @PMIX_RENAME@pmix_bfrops_base_copy_kval
-#define pmix_bfrops_base_copy_modex                             @PMIX_RENAME@pmix_bfrops_base_copy_modex
-#define pmix_bfrops_base_copy_payload                           @PMIX_RENAME@pmix_bfrops_base_copy_payload
-#define pmix_bfrops_base_copy_pdata                             @PMIX_RENAME@pmix_bfrops_base_copy_pdata
-#define pmix_bfrops_base_copy_pinfo                             @PMIX_RENAME@pmix_bfrops_base_copy_pinfo
-#define pmix_bfrops_base_copy_proc                              @PMIX_RENAME@pmix_bfrops_base_copy_proc
-#define pmix_bfrops_base_copy_query                             @PMIX_RENAME@pmix_bfrops_base_copy_query
-#define pmix_bfrops_base_copy_string                            @PMIX_RENAME@pmix_bfrops_base_copy_string
-#define pmix_bfrops_base_copy_value                             @PMIX_RENAME@pmix_bfrops_base_copy_value
-#define pmix_bfrops_base_data_type_string                       @PMIX_RENAME@pmix_bfrops_base_data_type_string
-#define pmix_bfrops_base_framework                              @PMIX_RENAME@pmix_bfrops_base_framework
-#define pmix_bfrops_base_get_available_modules                  @PMIX_RENAME@pmix_bfrops_base_get_available_modules
-#define pmix_bfrops_base_output                                 @PMIX_RENAME@pmix_bfrops_base_output
-#define pmix_bfrops_base_pack                                   @PMIX_RENAME@pmix_bfrops_base_pack
-#define pmix_bfrops_base_pack_alloc_directive                   @PMIX_RENAME@pmix_bfrops_base_pack_alloc_directive
-#define pmix_bfrops_base_pack_app                               @PMIX_RENAME@pmix_bfrops_base_pack_app
-#define pmix_bfrops_base_pack_array                             @PMIX_RENAME@pmix_bfrops_base_pack_array
-#define pmix_bfrops_base_pack_bo                                @PMIX_RENAME@pmix_bfrops_base_pack_bo
-#define pmix_bfrops_base_pack_bool                              @PMIX_RENAME@pmix_bfrops_base_pack_bool
-#define pmix_bfrops_base_pack_buf                               @PMIX_RENAME@pmix_bfrops_base_pack_buf
-#define pmix_bfrops_base_pack_buffer                            @PMIX_RENAME@pmix_bfrops_base_pack_buffer
-#define pmix_bfrops_base_pack_byte                              @PMIX_RENAME@pmix_bfrops_base_pack_byte
-#define pmix_bfrops_base_pack_cmd                               @PMIX_RENAME@pmix_bfrops_base_pack_cmd
-#define pmix_bfrops_base_pack_darray                            @PMIX_RENAME@pmix_bfrops_base_pack_darray
-#define pmix_bfrops_base_pack_datatype                          @PMIX_RENAME@pmix_bfrops_base_pack_datatype
-#define pmix_bfrops_base_pack_double                            @PMIX_RENAME@pmix_bfrops_base_pack_double
-#define pmix_bfrops_base_pack_float                             @PMIX_RENAME@pmix_bfrops_base_pack_float
-#define pmix_bfrops_base_pack_info                              @PMIX_RENAME@pmix_bfrops_base_pack_info
-#define pmix_bfrops_base_pack_info_directives                   @PMIX_RENAME@pmix_bfrops_base_pack_info_directives
-#define pmix_bfrops_base_pack_int                               @PMIX_RENAME@pmix_bfrops_base_pack_int
-#define pmix_bfrops_base_pack_int16                             @PMIX_RENAME@pmix_bfrops_base_pack_int16
-#define pmix_bfrops_base_pack_int32                             @PMIX_RENAME@pmix_bfrops_base_pack_int32
-#define pmix_bfrops_base_pack_int64                             @PMIX_RENAME@pmix_bfrops_base_pack_int64
-#define pmix_bfrops_base_pack_kval                              @PMIX_RENAME@pmix_bfrops_base_pack_kval
-#define pmix_bfrops_base_pack_modex                             @PMIX_RENAME@pmix_bfrops_base_pack_modex
-#define pmix_bfrops_base_pack_pdata                             @PMIX_RENAME@pmix_bfrops_base_pack_pdata
-#define pmix_bfrops_base_pack_persist                           @PMIX_RENAME@pmix_bfrops_base_pack_persist
-#define pmix_bfrops_base_pack_pid                               @PMIX_RENAME@pmix_bfrops_base_pack_pid
-#define pmix_bfrops_base_pack_pinfo                             @PMIX_RENAME@pmix_bfrops_base_pack_pinfo
-#define pmix_bfrops_base_pack_proc                              @PMIX_RENAME@pmix_bfrops_base_pack_proc
-#define pmix_bfrops_base_pack_pstate                            @PMIX_RENAME@pmix_bfrops_base_pack_pstate
-#define pmix_bfrops_base_pack_ptr                               @PMIX_RENAME@pmix_bfrops_base_pack_ptr
-#define pmix_bfrops_base_pack_query                             @PMIX_RENAME@pmix_bfrops_base_pack_query
-#define pmix_bfrops_base_pack_range                             @PMIX_RENAME@pmix_bfrops_base_pack_range
-#define pmix_bfrops_base_pack_rank                              @PMIX_RENAME@pmix_bfrops_base_pack_rank
-#define pmix_bfrops_base_pack_scope                             @PMIX_RENAME@pmix_bfrops_base_pack_scope
-#define pmix_bfrops_base_pack_sizet                             @PMIX_RENAME@pmix_bfrops_base_pack_sizet
-#define pmix_bfrops_base_pack_status                            @PMIX_RENAME@pmix_bfrops_base_pack_status
-#define pmix_bfrops_base_pack_string                            @PMIX_RENAME@pmix_bfrops_base_pack_string
-#define pmix_bfrops_base_pack_time                              @PMIX_RENAME@pmix_bfrops_base_pack_time
-#define pmix_bfrops_base_pack_timeval                           @PMIX_RENAME@pmix_bfrops_base_pack_timeval
-#define pmix_bfrops_base_pack_val                               @PMIX_RENAME@pmix_bfrops_base_pack_val
-#define pmix_bfrops_base_pack_value                             @PMIX_RENAME@pmix_bfrops_base_pack_value
-#define pmix_bfrops_base_print                                  @PMIX_RENAME@pmix_bfrops_base_print
-#define pmix_bfrops_base_print_alloc_directive                  @PMIX_RENAME@pmix_bfrops_base_print_alloc_directive
-#define pmix_bfrops_base_print_app                              @PMIX_RENAME@pmix_bfrops_base_print_app
-#define pmix_bfrops_base_print_array                            @PMIX_RENAME@pmix_bfrops_base_print_array
-#define pmix_bfrops_base_print_bo                               @PMIX_RENAME@pmix_bfrops_base_print_bo
-#define pmix_bfrops_base_print_bool                             @PMIX_RENAME@pmix_bfrops_base_print_bool
-#define pmix_bfrops_base_print_buf                              @PMIX_RENAME@pmix_bfrops_base_print_buf
-#define pmix_bfrops_base_print_byte                             @PMIX_RENAME@pmix_bfrops_base_print_byte
-#define pmix_bfrops_base_print_cmd                              @PMIX_RENAME@pmix_bfrops_base_print_cmd
-#define pmix_bfrops_base_print_darray                           @PMIX_RENAME@pmix_bfrops_base_print_darray
-#define pmix_bfrops_base_print_datatype                         @PMIX_RENAME@pmix_bfrops_base_print_datatype
-#define pmix_bfrops_base_print_double                           @PMIX_RENAME@pmix_bfrops_base_print_double
-#define pmix_bfrops_base_print_float                            @PMIX_RENAME@pmix_bfrops_base_print_float
-#define pmix_bfrops_base_print_info                             @PMIX_RENAME@pmix_bfrops_base_print_info
-#define pmix_bfrops_base_print_info_directives                  @PMIX_RENAME@pmix_bfrops_base_print_info_directives
-#define pmix_bfrops_base_print_int                              @PMIX_RENAME@pmix_bfrops_base_print_int
-#define pmix_bfrops_base_print_int16                            @PMIX_RENAME@pmix_bfrops_base_print_int16
-#define pmix_bfrops_base_print_int32                            @PMIX_RENAME@pmix_bfrops_base_print_int32
-#define pmix_bfrops_base_print_int64                            @PMIX_RENAME@pmix_bfrops_base_print_int64
-#define pmix_bfrops_base_print_int8                             @PMIX_RENAME@pmix_bfrops_base_print_int8
-#define pmix_bfrops_base_print_kval                             @PMIX_RENAME@pmix_bfrops_base_print_kval
-#define pmix_bfrops_base_print_modex                            @PMIX_RENAME@pmix_bfrops_base_print_modex
-#define pmix_bfrops_base_print_pdata                            @PMIX_RENAME@pmix_bfrops_base_print_pdata
-#define pmix_bfrops_base_print_persist                          @PMIX_RENAME@pmix_bfrops_base_print_persist
-#define pmix_bfrops_base_print_pid                              @PMIX_RENAME@pmix_bfrops_base_print_pid
-#define pmix_bfrops_base_print_pinfo                            @PMIX_RENAME@pmix_bfrops_base_print_pinfo
-#define pmix_bfrops_base_print_proc                             @PMIX_RENAME@pmix_bfrops_base_print_proc
-#define pmix_bfrops_base_print_pstate                           @PMIX_RENAME@pmix_bfrops_base_print_pstate
-#define pmix_bfrops_base_print_ptr                              @PMIX_RENAME@pmix_bfrops_base_print_ptr
-#define pmix_bfrops_base_print_query                            @PMIX_RENAME@pmix_bfrops_base_print_query
-#define pmix_bfrops_base_print_range                            @PMIX_RENAME@pmix_bfrops_base_print_range
-#define pmix_bfrops_base_print_rank                             @PMIX_RENAME@pmix_bfrops_base_print_rank
-#define pmix_bfrops_base_print_scope                            @PMIX_RENAME@pmix_bfrops_base_print_scope
-#define pmix_bfrops_base_print_size                             @PMIX_RENAME@pmix_bfrops_base_print_size
-#define pmix_bfrops_base_print_status                           @PMIX_RENAME@pmix_bfrops_base_print_status
-#define pmix_bfrops_base_print_string                           @PMIX_RENAME@pmix_bfrops_base_print_string
-#define pmix_bfrops_base_print_time                             @PMIX_RENAME@pmix_bfrops_base_print_time
-#define pmix_bfrops_base_print_timeval                          @PMIX_RENAME@pmix_bfrops_base_print_timeval
-#define pmix_bfrops_base_print_uint                             @PMIX_RENAME@pmix_bfrops_base_print_uint
-#define pmix_bfrops_base_print_uint16                           @PMIX_RENAME@pmix_bfrops_base_print_uint16
-#define pmix_bfrops_base_print_uint32                           @PMIX_RENAME@pmix_bfrops_base_print_uint32
-#define pmix_bfrops_base_print_uint64                           @PMIX_RENAME@pmix_bfrops_base_print_uint64
-#define pmix_bfrops_base_print_uint8                            @PMIX_RENAME@pmix_bfrops_base_print_uint8
-#define pmix_bfrops_base_print_value                            @PMIX_RENAME@pmix_bfrops_base_print_value
-#define pmix_bfrops_base_std_copy                               @PMIX_RENAME@pmix_bfrops_base_std_copy
-#define pmix_bfrops_base_unpack                                 @PMIX_RENAME@pmix_bfrops_base_unpack
-#define pmix_bfrops_base_unpack_alloc_directive                 @PMIX_RENAME@pmix_bfrops_base_unpack_alloc_directive
-#define pmix_bfrops_base_unpack_app                             @PMIX_RENAME@pmix_bfrops_base_unpack_app
-#define pmix_bfrops_base_unpack_array                           @PMIX_RENAME@pmix_bfrops_base_unpack_array
-#define pmix_bfrops_base_unpack_bo                              @PMIX_RENAME@pmix_bfrops_base_unpack_bo
-#define pmix_bfrops_base_unpack_bool                            @PMIX_RENAME@pmix_bfrops_base_unpack_bool
-#define pmix_bfrops_base_unpack_buf                             @PMIX_RENAME@pmix_bfrops_base_unpack_buf
-#define pmix_bfrops_base_unpack_byte                            @PMIX_RENAME@pmix_bfrops_base_unpack_byte
-#define pmix_bfrops_base_unpack_cmd                             @PMIX_RENAME@pmix_bfrops_base_unpack_cmd
-#define pmix_bfrops_base_unpack_darray                          @PMIX_RENAME@pmix_bfrops_base_unpack_darray
-#define pmix_bfrops_base_unpack_datatype                        @PMIX_RENAME@pmix_bfrops_base_unpack_datatype
-#define pmix_bfrops_base_unpack_double                          @PMIX_RENAME@pmix_bfrops_base_unpack_double
-#define pmix_bfrops_base_unpack_float                           @PMIX_RENAME@pmix_bfrops_base_unpack_float
-#define pmix_bfrops_base_unpack_info                            @PMIX_RENAME@pmix_bfrops_base_unpack_info
-#define pmix_bfrops_base_unpack_info_directives                 @PMIX_RENAME@pmix_bfrops_base_unpack_info_directives
-#define pmix_bfrops_base_unpack_int                             @PMIX_RENAME@pmix_bfrops_base_unpack_int
-#define pmix_bfrops_base_unpack_int16                           @PMIX_RENAME@pmix_bfrops_base_unpack_int16
-#define pmix_bfrops_base_unpack_int32                           @PMIX_RENAME@pmix_bfrops_base_unpack_int32
-#define pmix_bfrops_base_unpack_int64                           @PMIX_RENAME@pmix_bfrops_base_unpack_int64
-#define pmix_bfrops_base_unpack_kval                            @PMIX_RENAME@pmix_bfrops_base_unpack_kval
-#define pmix_bfrops_base_unpack_modex                           @PMIX_RENAME@pmix_bfrops_base_unpack_modex
-#define pmix_bfrops_base_unpack_pdata                           @PMIX_RENAME@pmix_bfrops_base_unpack_pdata
-#define pmix_bfrops_base_unpack_persist                         @PMIX_RENAME@pmix_bfrops_base_unpack_persist
-#define pmix_bfrops_base_unpack_pid                             @PMIX_RENAME@pmix_bfrops_base_unpack_pid
-#define pmix_bfrops_base_unpack_pinfo                           @PMIX_RENAME@pmix_bfrops_base_unpack_pinfo
-#define pmix_bfrops_base_unpack_proc                            @PMIX_RENAME@pmix_bfrops_base_unpack_proc
-#define pmix_bfrops_base_unpack_pstate                          @PMIX_RENAME@pmix_bfrops_base_unpack_pstate
-#define pmix_bfrops_base_unpack_ptr                             @PMIX_RENAME@pmix_bfrops_base_unpack_ptr
-#define pmix_bfrops_base_unpack_query                           @PMIX_RENAME@pmix_bfrops_base_unpack_query
-#define pmix_bfrops_base_unpack_range                           @PMIX_RENAME@pmix_bfrops_base_unpack_range
-#define pmix_bfrops_base_unpack_rank                            @PMIX_RENAME@pmix_bfrops_base_unpack_rank
-#define pmix_bfrops_base_unpack_scope                           @PMIX_RENAME@pmix_bfrops_base_unpack_scope
-#define pmix_bfrops_base_unpack_sizet                           @PMIX_RENAME@pmix_bfrops_base_unpack_sizet
-#define pmix_bfrops_base_unpack_status                          @PMIX_RENAME@pmix_bfrops_base_unpack_status
-#define pmix_bfrops_base_unpack_string                          @PMIX_RENAME@pmix_bfrops_base_unpack_string
-#define pmix_bfrops_base_unpack_time                            @PMIX_RENAME@pmix_bfrops_base_unpack_time
-#define pmix_bfrops_base_unpack_timeval                         @PMIX_RENAME@pmix_bfrops_base_unpack_timeval
-#define pmix_bfrops_base_unpack_val                             @PMIX_RENAME@pmix_bfrops_base_unpack_val
-#define pmix_bfrops_base_unpack_value                           @PMIX_RENAME@pmix_bfrops_base_unpack_value
-#define pmix_bfrops_base_value_cmp                              @PMIX_RENAME@pmix_bfrops_base_value_cmp
-#define pmix_bfrops_base_value_load                             @PMIX_RENAME@pmix_bfrops_base_value_load
-#define pmix_bfrops_base_value_unload                           @PMIX_RENAME@pmix_bfrops_base_value_unload
-#define pmix_bfrops_base_value_xfer                             @PMIX_RENAME@pmix_bfrops_base_value_xfer
-#define pmix_bfrops_globals                                     @PMIX_RENAME@pmix_bfrops_globals
-#define pmix_bfrop_store_data_type                              @PMIX_RENAME@pmix_bfrop_store_data_type
-#define pmix_bfrop_too_small                                    @PMIX_RENAME@pmix_bfrop_too_small
-#define pmix_bfrop_type_info_t_class                            @PMIX_RENAME@pmix_bfrop_type_info_t_class
-#define pmix_buffer_t_class                                     @PMIX_RENAME@pmix_buffer_t_class
-#define pmix_cb_t_class                                         @PMIX_RENAME@pmix_cb_t_class
-#define pmix_class_finalize                                     @PMIX_RENAME@pmix_class_finalize
-#define pmix_class_init_epoch                                   @PMIX_RENAME@pmix_class_init_epoch
-#define pmix_class_initialize                                   @PMIX_RENAME@pmix_class_initialize
-#define pmix_client_globals                                     @PMIX_RENAME@pmix_client_globals
-#define pmix_cmd_line_add                                       @PMIX_RENAME@pmix_cmd_line_add
-#define pmix_cmd_line_create                                    @PMIX_RENAME@pmix_cmd_line_create
-#define pmix_cmd_line_get_argc                                  @PMIX_RENAME@pmix_cmd_line_get_argc
-#define pmix_cmd_line_get_argv                                  @PMIX_RENAME@pmix_cmd_line_get_argv
-#define pmix_cmd_line_get_ninsts                                @PMIX_RENAME@pmix_cmd_line_get_ninsts
-#define pmix_cmd_line_get_param                                 @PMIX_RENAME@pmix_cmd_line_get_param
-#define pmix_cmd_line_get_tail                                  @PMIX_RENAME@pmix_cmd_line_get_tail
-#define pmix_cmd_line_get_usage_msg                             @PMIX_RENAME@pmix_cmd_line_get_usage_msg
-#define pmix_cmd_line_is_taken                                  @PMIX_RENAME@pmix_cmd_line_is_taken
-#define pmix_cmd_line_make_opt3                                 @PMIX_RENAME@pmix_cmd_line_make_opt3
-#define pmix_cmd_line_make_opt_mca                              @PMIX_RENAME@pmix_cmd_line_make_opt_mca
-#define pmix_cmd_line_parse                                     @PMIX_RENAME@pmix_cmd_line_parse
-#define pmix_cmd_line_t_class                                   @PMIX_RENAME@pmix_cmd_line_t_class
-#define pmix_command_string                                     @PMIX_RENAME@pmix_command_string
-#define PMIx_Commit                                             @PMIX_RENAME@PMIx_Commit
-#define PMIx_Connect                                            @PMIX_RENAME@PMIx_Connect
-#define PMIx_Connect_nb                                         @PMIX_RENAME@PMIx_Connect_nb
-#define PMIx_Data_copy                                          @PMIX_RENAME@PMIx_Data_copy
-#define PMIx_Data_copy_payload                                  @PMIX_RENAME@PMIx_Data_copy_payload
-#define PMIx_Data_pack                                          @PMIX_RENAME@PMIx_Data_pack
-#define PMIx_Data_print                                         @PMIX_RENAME@PMIx_Data_print
-#define PMIx_Data_range_string                                  @PMIX_RENAME@PMIx_Data_range_string
-#define PMIx_Data_type_string                                   @PMIX_RENAME@PMIx_Data_type_string
-#define PMIx_Data_unpack                                        @PMIX_RENAME@PMIx_Data_unpack
-#define pmix_debug_threads                                      @PMIX_RENAME@pmix_debug_threads
-#define PMIx_Deregister_event_handler                           @PMIX_RENAME@PMIx_Deregister_event_handler
-#define pmix_deregister_params                                  @PMIX_RENAME@pmix_deregister_params
-#define pmix_dirname                                            @PMIX_RENAME@pmix_dirname
-#define PMIx_Disconnect                                         @PMIX_RENAME@PMIx_Disconnect
-#define PMIx_Disconnect_nb                                      @PMIX_RENAME@PMIx_Disconnect_nb
-#define pmix_environ_merge                                      @PMIX_RENAME@pmix_environ_merge
-#define PMIx_Error_string                                       @PMIX_RENAME@PMIx_Error_string
-#define pmix_fd_read                                            @PMIX_RENAME@pmix_fd_read
-#define pmix_fd_set_cloexec                                     @PMIX_RENAME@pmix_fd_set_cloexec
-#define pmix_fd_write                                           @PMIX_RENAME@pmix_fd_write
-#define PMIx_Fence                                              @PMIX_RENAME@PMIx_Fence
-#define PMIx_Fence_nb                                           @PMIX_RENAME@PMIx_Fence_nb
-#define PMIx_Finalize                                           @PMIX_RENAME@PMIx_Finalize
-#define pmix_find_absolute_path                                 @PMIX_RENAME@pmix_find_absolute_path
-#define pmix_gds_base_assign_module                             @PMIX_RENAME@pmix_gds_base_assign_module
-#define pmix_gds_base_framework                                 @PMIX_RENAME@pmix_gds_base_framework
-#define pmix_gds_base_get_available_modules                     @PMIX_RENAME@pmix_gds_base_get_available_modules
-#define pmix_gds_base_output                                    @PMIX_RENAME@pmix_gds_base_output
-#define pmix_gds_base_select                                    @PMIX_RENAME@pmix_gds_base_select
-#define pmix_gds_base_setup_fork                                @PMIX_RENAME@pmix_gds_base_setup_fork
-#define pmix_gds_globals                                        @PMIX_RENAME@pmix_gds_globals
-#define PMIx_generate_ppn                                       @PMIX_RENAME@PMIx_generate_ppn
-#define PMIx_generate_regex                                     @PMIX_RENAME@PMIx_generate_regex
-#define PMIx_Get                                                @PMIX_RENAME@PMIx_Get
-#define PMIx_Get_nb                                             @PMIX_RENAME@PMIx_Get_nb
-#define PMIx_Get_version                                        @PMIX_RENAME@PMIx_Get_version
-#define pmix_global_lock                                        @PMIX_RENAME@pmix_global_lock
-#define pmix_globals                                            @PMIX_RENAME@pmix_globals
-#define pmix_hash_fetch                                         @PMIX_RENAME@pmix_hash_fetch
-#define pmix_hash_fetch_by_key                                  @PMIX_RENAME@pmix_hash_fetch_by_key
-#define pmix_hash_remove_data                                   @PMIX_RENAME@pmix_hash_remove_data
-#define pmix_hash_store                                         @PMIX_RENAME@pmix_hash_store
-#define pmix_hash_table_get_first_key_ptr                       @PMIX_RENAME@pmix_hash_table_get_first_key_ptr
-#define pmix_hash_table_get_first_key_uint32                    @PMIX_RENAME@pmix_hash_table_get_first_key_uint32
-#define pmix_hash_table_get_first_key_uint64                    @PMIX_RENAME@pmix_hash_table_get_first_key_uint64
-#define pmix_hash_table_get_next_key_ptr                        @PMIX_RENAME@pmix_hash_table_get_next_key_ptr
-#define pmix_hash_table_get_next_key_uint32                     @PMIX_RENAME@pmix_hash_table_get_next_key_uint32
-#define pmix_hash_table_get_next_key_uint64                     @PMIX_RENAME@pmix_hash_table_get_next_key_uint64
-#define pmix_hash_table_get_value_ptr                           @PMIX_RENAME@pmix_hash_table_get_value_ptr
-#define pmix_hash_table_get_value_uint32                        @PMIX_RENAME@pmix_hash_table_get_value_uint32
-#define pmix_hash_table_get_value_uint64                        @PMIX_RENAME@pmix_hash_table_get_value_uint64
-#define pmix_hash_table_init                                    @PMIX_RENAME@pmix_hash_table_init
-#define pmix_hash_table_init2                                   @PMIX_RENAME@pmix_hash_table_init2
-#define pmix_hash_table_remove_all                              @PMIX_RENAME@pmix_hash_table_remove_all
-#define pmix_hash_table_remove_value_ptr                        @PMIX_RENAME@pmix_hash_table_remove_value_ptr
-#define pmix_hash_table_remove_value_uint32                     @PMIX_RENAME@pmix_hash_table_remove_value_uint32
-#define pmix_hash_table_remove_value_uint64                     @PMIX_RENAME@pmix_hash_table_remove_value_uint64
-#define pmix_hash_table_set_value_ptr                           @PMIX_RENAME@pmix_hash_table_set_value_ptr
-#define pmix_hash_table_set_value_uint32                        @PMIX_RENAME@pmix_hash_table_set_value_uint32
-#define pmix_hash_table_set_value_uint64                        @PMIX_RENAME@pmix_hash_table_set_value_uint64
-#define pmix_hash_table_t_class                                 @PMIX_RENAME@pmix_hash_table_t_class
-#define pmix_home_directory                                     @PMIX_RENAME@pmix_home_directory
-#define pmix_host_server                                        @PMIX_RENAME@pmix_host_server
-#define pmix_hotel_init                                         @PMIX_RENAME@pmix_hotel_init
-#define pmix_ifaddrtokindex                                     @PMIX_RENAME@pmix_ifaddrtokindex
-#define pmix_ifaddrtoname                                       @PMIX_RENAME@pmix_ifaddrtoname
-#define pmix_ifbegin                                            @PMIX_RENAME@pmix_ifbegin
-#define pmix_ifcount                                            @PMIX_RENAME@pmix_ifcount
-#define pmix_ifgetaliases                                       @PMIX_RENAME@pmix_ifgetaliases
-#define pmix_ifindextoaddr                                      @PMIX_RENAME@pmix_ifindextoaddr
-#define pmix_ifindextoflags                                     @PMIX_RENAME@pmix_ifindextoflags
-#define pmix_ifindextokindex                                    @PMIX_RENAME@pmix_ifindextokindex
-#define pmix_ifindextomac                                       @PMIX_RENAME@pmix_ifindextomac
-#define pmix_ifindextomask                                      @PMIX_RENAME@pmix_ifindextomask
-#define pmix_ifindextomtu                                       @PMIX_RENAME@pmix_ifindextomtu
-#define pmix_ifindextoname                                      @PMIX_RENAME@pmix_ifindextoname
-#define pmix_ifislocal                                          @PMIX_RENAME@pmix_ifislocal
-#define pmix_ifisloopback                                       @PMIX_RENAME@pmix_ifisloopback
-#define pmix_ifkindextoaddr                                     @PMIX_RENAME@pmix_ifkindextoaddr
-#define pmix_ifkindextoname                                     @PMIX_RENAME@pmix_ifkindextoname
-#define pmix_ifmatches                                          @PMIX_RENAME@pmix_ifmatches
-#define pmix_ifnametoaddr                                       @PMIX_RENAME@pmix_ifnametoaddr
-#define pmix_ifnametoindex                                      @PMIX_RENAME@pmix_ifnametoindex
-#define pmix_ifnametokindex                                     @PMIX_RENAME@pmix_ifnametokindex
-#define pmix_ifnext                                             @PMIX_RENAME@pmix_ifnext
-#define pmix_iftupletoaddr                                      @PMIX_RENAME@pmix_iftupletoaddr
-#define pmix_info_caddy_t_class                                 @PMIX_RENAME@pmix_info_caddy_t_class
-#define PMIx_Info_directives_string                             @PMIX_RENAME@PMIx_Info_directives_string
-#define PMIx_Init                                               @PMIX_RENAME@PMIx_Init
-#define pmix_init_called                                        @PMIX_RENAME@pmix_init_called
-#define pmix_initialized                                        @PMIX_RENAME@pmix_initialized
-#define PMIx_Initialized                                        @PMIX_RENAME@PMIx_Initialized
-#define PMIx_Job_control_nb                                     @PMIX_RENAME@PMIx_Job_control_nb
-#define pmix_kval_t_class                                       @PMIX_RENAME@pmix_kval_t_class
-#define pmix_listener_t_class                                   @PMIX_RENAME@pmix_listener_t_class
-#define pmix_list_item_t_class                                  @PMIX_RENAME@pmix_list_item_t_class
-#define pmix_list_t_class                                       @PMIX_RENAME@pmix_list_t_class
-#define PMIx_Log_nb                                             @PMIX_RENAME@PMIx_Log_nb
-#define PMIx_Lookup                                             @PMIX_RENAME@PMIx_Lookup
-#define PMIx_Lookup_nb                                          @PMIX_RENAME@PMIx_Lookup_nb
-#define pmix_mca_base_close                                     @PMIX_RENAME@pmix_mca_base_close
-#define pmix_mca_base_cmd_line_process_args                     @PMIX_RENAME@pmix_mca_base_cmd_line_process_args
-#define pmix_mca_base_cmd_line_setup                            @PMIX_RENAME@pmix_mca_base_cmd_line_setup
-#define pmix_mca_base_cmd_line_wrap_args                        @PMIX_RENAME@pmix_mca_base_cmd_line_wrap_args
-#define pmix_mca_base_component_close                           @PMIX_RENAME@pmix_mca_base_component_close
-#define pmix_mca_base_component_compare                         @PMIX_RENAME@pmix_mca_base_component_compare
-#define pmix_mca_base_component_compare_priority                @PMIX_RENAME@pmix_mca_base_component_compare_priority
-#define pmix_mca_base_component_compatible                      @PMIX_RENAME@pmix_mca_base_component_compatible
-#define pmix_mca_base_component_disable_dlopen                  @PMIX_RENAME@pmix_mca_base_component_disable_dlopen
-#define pmix_mca_base_component_find                            @PMIX_RENAME@pmix_mca_base_component_find
-#define pmix_mca_base_component_find_finalize                   @PMIX_RENAME@pmix_mca_base_component_find_finalize
-#define pmix_mca_base_component_list_item_t_class               @PMIX_RENAME@pmix_mca_base_component_list_item_t_class
-#define pmix_mca_base_component_parse_requested                 @PMIX_RENAME@pmix_mca_base_component_parse_requested
-#define pmix_mca_base_component_path                            @PMIX_RENAME@pmix_mca_base_component_path
-#define pmix_mca_base_component_priority_list_item_t_class      @PMIX_RENAME@pmix_mca_base_component_priority_list_item_t_class
-#define pmix_mca_base_component_repository_add                  @PMIX_RENAME@pmix_mca_base_component_repository_add
-#define pmix_mca_base_component_repository_finalize             @PMIX_RENAME@pmix_mca_base_component_repository_finalize
-#define pmix_mca_base_component_repository_get_components       @PMIX_RENAME@pmix_mca_base_component_repository_get_components
-#define pmix_mca_base_component_repository_init                 @PMIX_RENAME@pmix_mca_base_component_repository_init
-#define pmix_mca_base_component_repository_item_t_class         @PMIX_RENAME@pmix_mca_base_component_repository_item_t_class
-#define pmix_mca_base_component_repository_open                 @PMIX_RENAME@pmix_mca_base_component_repository_open
-#define pmix_mca_base_component_repository_release              @PMIX_RENAME@pmix_mca_base_component_repository_release
-#define pmix_mca_base_component_repository_retain_component     @PMIX_RENAME@pmix_mca_base_component_repository_retain_component
-#define pmix_mca_base_components_close                          @PMIX_RENAME@pmix_mca_base_components_close
-#define pmix_mca_base_components_filter                         @PMIX_RENAME@pmix_mca_base_components_filter
-#define pmix_mca_base_component_show_load_errors                @PMIX_RENAME@pmix_mca_base_component_show_load_errors
-#define pmix_mca_base_component_to_string                       @PMIX_RENAME@pmix_mca_base_component_to_string
-#define pmix_mca_base_component_track_load_errors               @PMIX_RENAME@pmix_mca_base_component_track_load_errors
-#define pmix_mca_base_component_unload                          @PMIX_RENAME@pmix_mca_base_component_unload
-#define pmix_mca_base_component_var_register                    @PMIX_RENAME@pmix_mca_base_component_var_register
-#define pmix_mca_base_failed_component_t_class                  @PMIX_RENAME@pmix_mca_base_failed_component_t_class
-#define pmix_mca_base_framework_close                           @PMIX_RENAME@pmix_mca_base_framework_close
-#define pmix_mca_base_framework_components_close                @PMIX_RENAME@pmix_mca_base_framework_components_close
-#define pmix_mca_base_framework_components_open                 @PMIX_RENAME@pmix_mca_base_framework_components_open
-#define pmix_mca_base_framework_components_register             @PMIX_RENAME@pmix_mca_base_framework_components_register
-#define pmix_mca_base_framework_is_open                         @PMIX_RENAME@pmix_mca_base_framework_is_open
-#define pmix_mca_base_framework_is_registered                   @PMIX_RENAME@pmix_mca_base_framework_is_registered
-#define pmix_mca_base_framework_open                            @PMIX_RENAME@pmix_mca_base_framework_open
-#define pmix_mca_base_framework_register                        @PMIX_RENAME@pmix_mca_base_framework_register
-#define pmix_mca_base_framework_var_register                    @PMIX_RENAME@pmix_mca_base_framework_var_register
-#define pmix_mca_base_open                                      @PMIX_RENAME@pmix_mca_base_open
-#define pmix_mca_base_select                                    @PMIX_RENAME@pmix_mca_base_select
-#define pmix_mca_base_system_default_path                       @PMIX_RENAME@pmix_mca_base_system_default_path
-#define pmix_mca_base_user_default_path                         @PMIX_RENAME@pmix_mca_base_user_default_path
-#define pmix_mca_base_var_build_env                             @PMIX_RENAME@pmix_mca_base_var_build_env
-#define pmix_mca_base_var_cache_files                           @PMIX_RENAME@pmix_mca_base_var_cache_files
-#define pmix_mca_base_var_check_exclusive                       @PMIX_RENAME@pmix_mca_base_var_check_exclusive
-#define pmix_mca_base_var_deregister                            @PMIX_RENAME@pmix_mca_base_var_deregister
-#define pmix_mca_base_var_dump                                  @PMIX_RENAME@pmix_mca_base_var_dump
-#define pmix_mca_base_var_env_name                              @PMIX_RENAME@pmix_mca_base_var_env_name
-#define pmix_mca_base_var_finalize                              @PMIX_RENAME@pmix_mca_base_var_finalize
-#define pmix_mca_base_var_find                                  @PMIX_RENAME@pmix_mca_base_var_find
-#define pmix_mca_base_var_find_by_name                          @PMIX_RENAME@pmix_mca_base_var_find_by_name
-#define pmix_mca_base_var_get                                   @PMIX_RENAME@pmix_mca_base_var_get
-#define pmix_mca_base_var_get_count                             @PMIX_RENAME@pmix_mca_base_var_get_count
-#define pmix_mca_base_var_get_value                             @PMIX_RENAME@pmix_mca_base_var_get_value
-#define pmix_mca_base_var_group_component_register              @PMIX_RENAME@pmix_mca_base_var_group_component_register
-#define pmix_mca_base_var_group_deregister                      @PMIX_RENAME@pmix_mca_base_var_group_deregister
-#define pmix_mca_base_var_group_find                            @PMIX_RENAME@pmix_mca_base_var_group_find
-#define pmix_mca_base_var_group_find_by_name                    @PMIX_RENAME@pmix_mca_base_var_group_find_by_name
-#define pmix_mca_base_var_group_get                             @PMIX_RENAME@pmix_mca_base_var_group_get
-#define pmix_mca_base_var_group_get_count                       @PMIX_RENAME@pmix_mca_base_var_group_get_count
-#define pmix_mca_base_var_group_get_stamp                       @PMIX_RENAME@pmix_mca_base_var_group_get_stamp
-#define pmix_mca_base_var_group_register                        @PMIX_RENAME@pmix_mca_base_var_group_register
-#define pmix_mca_base_var_group_set_var_flag                    @PMIX_RENAME@pmix_mca_base_var_group_set_var_flag
-#define pmix_mca_base_var_group_t_class                         @PMIX_RENAME@pmix_mca_base_var_group_t_class
-#define pmix_mca_base_var_init                                  @PMIX_RENAME@pmix_mca_base_var_init
-#define pmix_mca_base_var_process_env_list                      @PMIX_RENAME@pmix_mca_base_var_process_env_list
-#define pmix_mca_base_var_process_env_list_from_file            @PMIX_RENAME@pmix_mca_base_var_process_env_list_from_file
-#define pmix_mca_base_var_register                              @PMIX_RENAME@pmix_mca_base_var_register
-#define pmix_mca_base_var_register_synonym                      @PMIX_RENAME@pmix_mca_base_var_register_synonym
-#define pmix_mca_base_var_set_flag                              @PMIX_RENAME@pmix_mca_base_var_set_flag
-#define pmix_mca_base_var_set_value                             @PMIX_RENAME@pmix_mca_base_var_set_value
-#define pmix_mca_base_var_t_class                               @PMIX_RENAME@pmix_mca_base_var_t_class
-#define pmix_mutex_t_class                                      @PMIX_RENAME@pmix_mutex_t_class
-#define pmix_namelist_t_class                                   @PMIX_RENAME@pmix_namelist_t_class
-#define pmix_net_addr_isipv4public                              @PMIX_RENAME@pmix_net_addr_isipv4public
-#define pmix_net_finalize                                       @PMIX_RENAME@pmix_net_finalize
-#define pmix_net_get_hostname                                   @PMIX_RENAME@pmix_net_get_hostname
-#define pmix_net_get_port                                       @PMIX_RENAME@pmix_net_get_port
-#define pmix_net_init                                           @PMIX_RENAME@pmix_net_init
-#define pmix_net_isaddr                                         @PMIX_RENAME@pmix_net_isaddr
-#define pmix_net_islocalhost                                    @PMIX_RENAME@pmix_net_islocalhost
-#define pmix_net_prefix2netmask                                 @PMIX_RENAME@pmix_net_prefix2netmask
-#define pmix_net_samenetwork                                    @PMIX_RENAME@pmix_net_samenetwork
-#define PMIx_Notify_event                                       @PMIX_RENAME@PMIx_Notify_event
-#define pmix_nspace_caddy_t_class                               @PMIX_RENAME@pmix_nspace_caddy_t_class
-#define pmix_nspace_t_class                                     @PMIX_RENAME@pmix_nspace_t_class
-#define pmix_object_t_class                                     @PMIX_RENAME@pmix_object_t_class
-#define pmix_os_path                                            @PMIX_RENAME@pmix_os_path
-#define pmix_output                                             @PMIX_RENAME@pmix_output
-#define pmix_output_close                                       @PMIX_RENAME@pmix_output_close
-#define pmix_output_finalize                                    @PMIX_RENAME@pmix_output_finalize
-#define pmix_output_get_verbosity                               @PMIX_RENAME@pmix_output_get_verbosity
-#define pmix_output_check_verbosity                             @PMIX_RENAME@pmix_output_check_verbosity
-#define pmix_output_hexdump                                     @PMIX_RENAME@pmix_output_hexdump
-#define pmix_output_init                                        @PMIX_RENAME@pmix_output_init
-#define pmix_output_open                                        @PMIX_RENAME@pmix_output_open
-#define pmix_output_reopen                                      @PMIX_RENAME@pmix_output_reopen
-#define pmix_output_reopen_all                                  @PMIX_RENAME@pmix_output_reopen_all
-#define pmix_output_set_output_file_info                        @PMIX_RENAME@pmix_output_set_output_file_info
-#define pmix_output_set_verbosity                               @PMIX_RENAME@pmix_output_set_verbosity
-#define pmix_output_switch                                      @PMIX_RENAME@pmix_output_switch
-#define pmix_output_vverbose                                    @PMIX_RENAME@pmix_output_vverbose
-#define pmix_path_access                                        @PMIX_RENAME@pmix_path_access
-#define pmix_path_df                                            @PMIX_RENAME@pmix_path_df
-#define pmix_path_find                                          @PMIX_RENAME@pmix_path_find
-#define pmix_path_findv                                         @PMIX_RENAME@pmix_path_findv
-#define pmix_path_is_absolute                                   @PMIX_RENAME@pmix_path_is_absolute
-#define pmix_path_nfs                                           @PMIX_RENAME@pmix_path_nfs
-#define pmix_pdl_base_framework                                 @PMIX_RENAME@pmix_pdl_base_framework
-#define pmix_peer_t_class                                       @PMIX_RENAME@pmix_peer_t_class
-#define pmix_pending_connection_t_class                         @PMIX_RENAME@pmix_pending_connection_t_class
-#define PMIx_Persistence_string                                 @PMIX_RENAME@PMIx_Persistence_string
-#define pmix_pif_base_framework                                 @PMIX_RENAME@pmix_pif_base_framework
-#define pmix_pinstall_dirs                                      @PMIX_RENAME@pmix_pinstall_dirs
-#define pmix_pinstalldirs_base_framework                        @PMIX_RENAME@pmix_pinstalldirs_base_framework
-#define pmix_pnet                                               @PMIX_RENAME@pmix_pnet
-#define pmix_pnet_base_child_finalized                          @PMIX_RENAME@pmix_pnet_base_child_finalized
-#define pmix_pnet_base_framework                                @PMIX_RENAME@pmix_pnet_base_framework
-#define pmix_pnet_base_local_app_finalized                      @PMIX_RENAME@pmix_pnet_base_local_app_finalized
-#define pmix_pnet_base_select                                   @PMIX_RENAME@pmix_pnet_base_select
-#define pmix_pnet_base_setup_app                                @PMIX_RENAME@pmix_pnet_base_setup_app
-#define pmix_pnet_base_setup_fork                               @PMIX_RENAME@pmix_pnet_base_setup_fork
-#define pmix_pnet_base_setup_local_network                      @PMIX_RENAME@pmix_pnet_base_setup_local_network
-#define pmix_pnet_globals                                       @PMIX_RENAME@pmix_pnet_globals
-#define pmix_pointer_array_add                                  @PMIX_RENAME@pmix_pointer_array_add
-#define pmix_pointer_array_init                                 @PMIX_RENAME@pmix_pointer_array_init
-#define pmix_pointer_array_set_item                             @PMIX_RENAME@pmix_pointer_array_set_item
-#define pmix_pointer_array_set_size                             @PMIX_RENAME@pmix_pointer_array_set_size
-#define pmix_pointer_array_t_class                              @PMIX_RENAME@pmix_pointer_array_t_class
-#define pmix_pointer_array_test_and_set_item                    @PMIX_RENAME@pmix_pointer_array_test_and_set_item
-#define pmix_preg                                               @PMIX_RENAME@pmix_preg
-#define pmix_preg_base_framework                                @PMIX_RENAME@pmix_preg_base_framework
-#define pmix_preg_base_generate_node_regex                      @PMIX_RENAME@pmix_preg_base_generate_node_regex
-#define pmix_preg_base_generate_ppn                             @PMIX_RENAME@pmix_preg_base_generate_ppn
-#define pmix_preg_base_parse_nodes                              @PMIX_RENAME@pmix_preg_base_parse_nodes
-#define pmix_preg_base_parse_procs                              @PMIX_RENAME@pmix_preg_base_parse_procs
-#define pmix_preg_base_resolve_nodes                            @PMIX_RENAME@pmix_preg_base_resolve_nodes
-#define pmix_preg_base_resolve_peers                            @PMIX_RENAME@pmix_preg_base_resolve_peers
-#define pmix_preg_base_select                                   @PMIX_RENAME@pmix_preg_base_select
-#define pmix_preg_globals                                       @PMIX_RENAME@pmix_preg_globals
-#define PMIx_Process_monitor_nb                                 @PMIX_RENAME@PMIx_Process_monitor_nb
-#define PMIx_Proc_state_string                                  @PMIX_RENAME@PMIx_Proc_state_string
-#define pmix_psec_base_assign_module                            @PMIX_RENAME@pmix_psec_base_assign_module
-#define pmix_psec_base_framework                                @PMIX_RENAME@pmix_psec_base_framework
-#define pmix_psec_base_get_available_modules                    @PMIX_RENAME@pmix_psec_base_get_available_modules
-#define pmix_psec_base_select                                   @PMIX_RENAME@pmix_psec_base_select
-#define pmix_psensor                                            @PMIX_RENAME@pmix_psensor
-#define pmix_psensor_base                                       @PMIX_RENAME@pmix_psensor_base
-#define pmix_psensor_base_framework                             @PMIX_RENAME@pmix_psensor_base_framework
-#define pmix_psensor_base_select                                @PMIX_RENAME@pmix_psensor_base_select
-#define pmix_psensor_base_start                                 @PMIX_RENAME@pmix_psensor_base_start
-#define pmix_psensor_base_stop                                  @PMIX_RENAME@pmix_psensor_base_stop
-#define pmix_pshmem                                             @PMIX_RENAME@pmix_pshmem
-#define pmix_pshmem_base_framework                              @PMIX_RENAME@pmix_pshmem_base_framework
-#define pmix_pshmem_base_select                                 @PMIX_RENAME@pmix_pshmem_base_select
-#define pmix_ptl_base_assign_module                             @PMIX_RENAME@pmix_ptl_base_assign_module
-#define pmix_ptl_base_cancel_recv                               @PMIX_RENAME@pmix_ptl_base_cancel_recv
-#define pmix_ptl_base_connect                                   @PMIX_RENAME@pmix_ptl_base_connect
-#define pmix_ptl_base_connect_to_peer                           @PMIX_RENAME@pmix_ptl_base_connect_to_peer
-#define pmix_ptl_base_framework                                 @PMIX_RENAME@pmix_ptl_base_framework
-#define pmix_ptl_base_get_available_modules                     @PMIX_RENAME@pmix_ptl_base_get_available_modules
-#define pmix_ptl_base_lost_connection                           @PMIX_RENAME@pmix_ptl_base_lost_connection
-#define pmix_ptl_base_output                                    @PMIX_RENAME@pmix_ptl_base_output
-#define pmix_ptl_base_process_msg                               @PMIX_RENAME@pmix_ptl_base_process_msg
-#define pmix_ptl_base_recv_blocking                             @PMIX_RENAME@pmix_ptl_base_recv_blocking
-#define pmix_ptl_base_recv_handler                              @PMIX_RENAME@pmix_ptl_base_recv_handler
-#define pmix_ptl_base_register_recv                             @PMIX_RENAME@pmix_ptl_base_register_recv
-#define pmix_ptl_base_select                                    @PMIX_RENAME@pmix_ptl_base_select
-#define pmix_ptl_base_send                                      @PMIX_RENAME@pmix_ptl_base_send
-#define pmix_ptl_base_send_blocking                             @PMIX_RENAME@pmix_ptl_base_send_blocking
-#define pmix_ptl_base_send_handler                              @PMIX_RENAME@pmix_ptl_base_send_handler
-#define pmix_ptl_base_send_recv                                 @PMIX_RENAME@pmix_ptl_base_send_recv
-#define pmix_ptl_base_set_blocking                              @PMIX_RENAME@pmix_ptl_base_set_blocking
-#define pmix_ptl_base_set_nonblocking                           @PMIX_RENAME@pmix_ptl_base_set_nonblocking
-#define pmix_ptl_base_set_notification_cbfunc                   @PMIX_RENAME@pmix_ptl_base_set_notification_cbfunc
-#define pmix_ptl_base_start_listening                           @PMIX_RENAME@pmix_ptl_base_start_listening
-#define pmix_ptl_base_stop_listening                            @PMIX_RENAME@pmix_ptl_base_stop_listening
-#define pmix_ptl_globals                                        @PMIX_RENAME@pmix_ptl_globals
-#define pmix_ptl_posted_recv_t_class                            @PMIX_RENAME@pmix_ptl_posted_recv_t_class
-#define pmix_ptl_queue_t_class                                  @PMIX_RENAME@pmix_ptl_queue_t_class
-#define pmix_ptl_recv_t_class                                   @PMIX_RENAME@pmix_ptl_recv_t_class
-#define pmix_ptl_send_t_class                                   @PMIX_RENAME@pmix_ptl_send_t_class
-#define pmix_ptl_sr_t_class                                     @PMIX_RENAME@pmix_ptl_sr_t_class
-#define PMIx_Publish                                            @PMIX_RENAME@PMIx_Publish
-#define PMIx_Publish_nb                                         @PMIX_RENAME@PMIx_Publish_nb
-#define PMIx_Put                                                @PMIX_RENAME@PMIx_Put
-#define pmix_query_caddy_t_class                                @PMIX_RENAME@pmix_query_caddy_t_class
-#define PMIx_Query_info_nb                                      @PMIX_RENAME@PMIx_Query_info_nb
-#define pmix_rand                                               @PMIX_RENAME@pmix_rand
-#define pmix_random                                             @PMIX_RENAME@pmix_random
-#define pmix_rank_info_t_class                                  @PMIX_RENAME@pmix_rank_info_t_class
-#define pmix_recursive_mutex_t_class                            @PMIX_RENAME@pmix_recursive_mutex_t_class
-#define pmix_regex_range_t_class                                @PMIX_RENAME@pmix_regex_range_t_class
-#define pmix_regex_value_t_class                                @PMIX_RENAME@pmix_regex_value_t_class
-#define PMIx_Register_event_handler                             @PMIX_RENAME@PMIx_Register_event_handler
-#define pmix_register_params                                    @PMIX_RENAME@pmix_register_params
-#define PMIx_Resolve_nodes                                      @PMIX_RENAME@PMIx_Resolve_nodes
-#define PMIx_Resolve_peers                                      @PMIX_RENAME@PMIx_Resolve_peers
-#define pmix_ring_buffer_init                                   @PMIX_RENAME@pmix_ring_buffer_init
-#define pmix_ring_buffer_poke                                   @PMIX_RENAME@pmix_ring_buffer_poke
-#define pmix_ring_buffer_pop                                    @PMIX_RENAME@pmix_ring_buffer_pop
-#define pmix_ring_buffer_push                                   @PMIX_RENAME@pmix_ring_buffer_push
-#define pmix_rte_finalize                                       @PMIX_RENAME@pmix_rte_finalize
-#define pmix_rte_init                                           @PMIX_RENAME@pmix_rte_init
-#define PMIx_Scope_string                                       @PMIX_RENAME@PMIx_Scope_string
-#define PMIx_server_deregister_client                           @PMIX_RENAME@PMIx_server_deregister_client
-#define PMIx_server_deregister_nspace                           @PMIX_RENAME@PMIx_server_deregister_nspace
-#define PMIx_server_dmodex_request                              @PMIX_RENAME@PMIx_server_dmodex_request
-#define PMIx_server_finalize                                    @PMIX_RENAME@PMIx_server_finalize
-#define pmix_server_globals                                     @PMIX_RENAME@pmix_server_globals
-#define PMIx_server_init                                        @PMIX_RENAME@PMIx_server_init
-#define PMIx_server_register_client                             @PMIX_RENAME@PMIx_server_register_client
-#define PMIx_server_register_nspace                             @PMIX_RENAME@PMIx_server_register_nspace
-#define PMIx_server_setup_application                           @PMIX_RENAME@PMIx_server_setup_application
-#define PMIx_server_setup_fork                                  @PMIX_RENAME@PMIx_server_setup_fork
-#define PMIx_server_setup_local_support                         @PMIX_RENAME@PMIx_server_setup_local_support
-#define pmix_setenv                                             @PMIX_RENAME@pmix_setenv
-#define pmix_setup_caddy_t_class                                @PMIX_RENAME@pmix_setup_caddy_t_class
-#define pmix_shift_caddy_t_class                                @PMIX_RENAME@pmix_shift_caddy_t_class
-#define pmix_show_help                                          @PMIX_RENAME@pmix_show_help
-#define pmix_show_help_add_dir                                  @PMIX_RENAME@pmix_show_help_add_dir
-#define pmix_show_help_finalize                                 @PMIX_RENAME@pmix_show_help_finalize
-#define pmix_show_help_init                                     @PMIX_RENAME@pmix_show_help_init
-#define pmix_show_help_string                                   @PMIX_RENAME@pmix_show_help_string
-#define pmix_show_help_vstring                                  @PMIX_RENAME@pmix_show_help_vstring
-#define pmix_show_vhelp                                         @PMIX_RENAME@pmix_show_vhelp
-#define pmix_snprintf                                           @PMIX_RENAME@pmix_snprintf
-#define PMIx_Spawn                                              @PMIX_RENAME@PMIx_Spawn
-#define PMIx_Spawn_nb                                           @PMIX_RENAME@PMIx_Spawn_nb
-#define pmix_srand                                              @PMIX_RENAME@pmix_srand
-#define PMIx_Store_internal                                     @PMIX_RENAME@PMIx_Store_internal
-#define pmix_sync_wait_mt                                       @PMIX_RENAME@pmix_sync_wait_mt
-#define pmix_thread_get_self                                    @PMIX_RENAME@pmix_thread_get_self
-#define pmix_thread_join                                        @PMIX_RENAME@pmix_thread_join
-#define pmix_thread_kill                                        @PMIX_RENAME@pmix_thread_kill
-#define pmix_thread_self_compare                                @PMIX_RENAME@pmix_thread_self_compare
-#define pmix_thread_set_main                                    @PMIX_RENAME@pmix_thread_set_main
-#define pmix_thread_start                                       @PMIX_RENAME@pmix_thread_start
-#define pmix_thread_t_class                                     @PMIX_RENAME@pmix_thread_t_class
-#define pmix_tmp_directory                                      @PMIX_RENAME@pmix_tmp_directory
-#define PMIx_tool_finalize                                      @PMIX_RENAME@PMIx_tool_finalize
-#define PMIx_tool_init                                          @PMIX_RENAME@PMIx_tool_init
-#define pmix_tsd_key_create                                     @PMIX_RENAME@pmix_tsd_key_create
-#define pmix_tsd_keys_destruct                                  @PMIX_RENAME@pmix_tsd_keys_destruct
-#define PMIx_Unpublish                                          @PMIX_RENAME@PMIx_Unpublish
-#define PMIx_Unpublish_nb                                       @PMIX_RENAME@PMIx_Unpublish_nb
-#define pmix_unsetenv                                           @PMIX_RENAME@pmix_unsetenv
-#define pmix_util_compress_string                               @PMIX_RENAME@pmix_util_compress_string
-#define pmix_util_getid                                         @PMIX_RENAME@pmix_util_getid
-#define pmix_util_get_ranges                                    @PMIX_RENAME@pmix_util_get_ranges
-#define pmix_util_keyval_parse                                  @PMIX_RENAME@pmix_util_keyval_parse
-#define pmix_util_keyval_parse_finalize                         @PMIX_RENAME@pmix_util_keyval_parse_finalize
-#define pmix_util_keyval_parse_init                             @PMIX_RENAME@pmix_util_keyval_parse_init
-#define pmix_util_keyval_parse_lineno                           @PMIX_RENAME@pmix_util_keyval_parse_lineno
-#define pmix_util_keyval_save_internal_envars                   @PMIX_RENAME@pmix_util_keyval_save_internal_envars
-#define pmix_util_parse_range_options                           @PMIX_RENAME@pmix_util_parse_range_options
-#define pmix_util_uncompress_string                             @PMIX_RENAME@pmix_util_uncompress_string
-#define pmix_value_array_set_size                               @PMIX_RENAME@pmix_value_array_set_size
-#define pmix_value_array_t_class                                @PMIX_RENAME@pmix_value_array_t_class
-#define pmix_value_load                                         @PMIX_RENAME@pmix_value_load
-#define pmix_value_xfer                                         @PMIX_RENAME@pmix_value_xfer
-#define pmix_var_type_names                                     @PMIX_RENAME@pmix_var_type_names
-#define pmix_vasprintf                                          @PMIX_RENAME@pmix_vasprintf
-#define pmix_vsnprintf                                          @PMIX_RENAME@pmix_vsnprintf
diff -Nru pmix-3.2.2~rc1/include/pmix_server.h pmix-4.0.0/include/pmix_server.h
--- pmix-3.2.2~rc1/include/pmix_server.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmix_server.h	2021-01-02 08:56:17.000000000 +0000
@@ -85,6 +85,12 @@
 typedef pmix_status_t (*pmix_server_client_connected_fn_t)(const pmix_proc_t *proc, void* server_object,
                                                            pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+/* REPLACES ABOVE FUNCTION TO ALLOW PASSING ADDITIONAL INFO */
+typedef pmix_status_t (*pmix_server_client_connected2_fn_t)(const pmix_proc_t *proc, void* server_object,
+                                                            pmix_info_t info[], size_t ninfo,
+                                                            pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+
 /* Notify the host server that a client called PMIx_Finalize - note
  * that the client will be in a blocked state until the host server
  * executes the callback function, thus allowing the PMIx server support
@@ -492,7 +498,44 @@
                                                 pmix_op_cbfunc_t cbfunc, void *cbdata);
 
 
-typedef struct pmix_server_module_2_0_0_t {
+/* Perform a "fence" operation across the specified procs, plus any special
+ * actions included in the directives. Return the result of any special action
+ * requests in the info cbfunc when the fence is completed. Actions may include:
+ *
+ * PMIX_GROUP_ASSIGN_CONTEXT_ID - request that the RM assign a unique
+ *                                numerical (size_t) ID to this group
+ *
+ * grp - user-assigned string ID of this group
+ *
+ * op - pmix_group_operation_t value indicating the operation to perform
+ *      Current values support construct and destruct of the group
+ *
+ * procs - pointer to array of pmix_proc_t ID's of group members
+ *
+ * nprocs - number of group members
+ *
+ * directives - array of key-value attributes specifying special actions.
+ *
+ * ndirs - size of the directives array
+ *
+ * cbfunc - callback function when the operation is completed
+ *
+ * cbdata - object to be returned in cbfunc
+ */
+typedef pmix_status_t (*pmix_server_grp_fn_t)(pmix_group_operation_t op, char grp[],
+                                              const pmix_proc_t procs[], size_t nprocs,
+                                              const pmix_info_t directives[], size_t ndirs,
+                                              pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+/* Retrieve fabric-related information from the server supporting
+ * the system scheduler.
+ */
+typedef pmix_status_t (*pmix_server_fabric_fn_t)(const pmix_proc_t *requestor,
+                                                 pmix_fabric_operation_t op,
+                                                 const pmix_info_t directives[], size_t ndirs,
+                                                 pmix_info_cbfunc_t cbfunc, void *cbdata);
+
+typedef struct pmix_server_module_4_0_0_t {
     /* v1x interfaces */
     pmix_server_client_connected_fn_t   client_connected;
     pmix_server_client_finalized_fn_t   client_finalized;
@@ -521,6 +564,10 @@
     pmix_server_validate_cred_fn_t      validate_credential;
     pmix_server_iof_fn_t                iof_pull;
     pmix_server_stdin_fn_t              push_stdin;
+    /* v4x interfaces */
+    pmix_server_grp_fn_t                group;
+    pmix_server_fabric_fn_t             fabric;
+    pmix_server_client_connected2_fn_t  client_connected2;
 } pmix_server_module_t;
 
 /****    HOST RM FUNCTIONS FOR INTERFACE TO PMIX SERVER    ****/
@@ -758,6 +805,68 @@
                                                         pmix_info_t directives[], size_t ndirs,
                                                         pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+\
+/******      ATTRIBUTE REGISTRATION      ******/
+/**
+ * This function is used by the host environment to register with its
+ * server library the attributes it supports for each pmix_server_module_t
+ * function.
+ *
+ * Parameters include:
+ *
+ * function - the string name of the server module function
+ *            (e.g., "register_events", "validate_credential",
+ *            or "allocate") whose attributes are being registered.
+ *
+ * attrs - NULL-terminated argv array of attributes supported
+ *         by the host environment for the specified function
+ *
+ */
+PMIX_EXPORT pmix_status_t PMIx_Register_attributes(char *function, char *attrs[]);
+
+/* Generate a PMIx locality string from a given cpuset.
+ * Provide a function by which the host environment can generate a PMIx locality
+ * string for inclusion in the call to PMIx_server_register_nspace . This function
+ * shall only be called for local client processes, with the returned locality
+ * included in the job-level information (via the PMIX_LOCALITY_STRING attribute)
+ * provided to local clients.
+ */
+PMIX_EXPORT pmix_status_t PMIx_server_generate_locality_string(const pmix_cpuset_t *cpuset,
+                                                               char **locality);
+
+/* Generate a PMIx string representation of the provided cpuset.
+ * Provide a function by which the host environment can generate a string
+ * representation of the cpuset bitmap for inclusion in the call to
+ * PMIx_server_register_nspace . This function shall only be called for local
+ * client processes, with the returned string included in the job-level information
+ * (via the PMIX_CPUSET attribute) provided to local clients.
+ */
+PMIX_EXPORT pmix_status_t PMIx_server_generate_cpuset_string(const pmix_cpuset_t *cpuset,
+                                                             char **cpuset_string);
+
+/* Define a process set
+ * Provide a function by which the host environment can define a new process set.
+ */
+PMIX_EXPORT pmix_status_t PMIx_server_define_process_set(const pmix_proc_t *members,
+                                                         size_t nmembers, char *pset_name);
+
+/* Delete a process set
+ * Provide a function by which the host environment can delete a new process set.
+ */
+PMIX_EXPORT pmix_status_t PMIx_server_delete_process_set(char *pset_name);
+
+
+/* Register non-namespace related information with the local PMIx server library.
+ */
+PMIX_EXPORT pmix_status_t PMIx_server_register_resources(pmix_info_t info[], size_t ninfo,
+                                                         pmix_op_cbfunc_t cbfunc,
+                                                         void *cbdata);
+
+/* Remove specified non-namespace related information from the local PMIx server library.
+ */
+PMIX_EXPORT pmix_status_t PMIx_server_deregister_resources(pmix_info_t info[], size_t ninfo,
+                                                           pmix_op_cbfunc_t cbfunc,
+                                                           void *cbdata);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
diff -Nru pmix-3.2.2~rc1/include/pmix_tool.h pmix-4.0.0/include/pmix_tool.h
--- pmix-3.2.2~rc1/include/pmix_tool.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/include/pmix_tool.h	2021-01-02 08:56:17.000000000 +0000
@@ -98,8 +98,7 @@
  * operation. */
 PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void);
 
-/* Switch server connection. Closes the connection, if existing, to a server
- * and establishes a connection to the specified server. The target server can
+/* Establish a connection to a PMIx server. The target server can
  * be given as:
  *
  * - PMIX_CONNECT_TO_SYSTEM: connect solely to the system server
@@ -126,8 +125,171 @@
  * proc parameter will be filled with the tool's existing nspace/rank, and
  * the caller is welcome to pass _NULL_ in that location
  */
-PMIX_EXPORT pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc,
-                                                      pmix_info_t info[], size_t ninfo);
+/* REPLACES CONNECT_TO_SERVER, ADDING ABILITY TO RETURN
+ * IDENTIFIER OF SERVER TO WHICH TOOL ATTACHED
+ */
+PMIX_EXPORT pmix_status_t PMIx_tool_attach_to_server(pmix_proc_t *myproc, pmix_proc_t *server,
+                                                     pmix_info_t info[], size_t ninfo);
+
+
+/* Disconnect the PMIx tool from the specified server connection while
+ * leaving the tool library initialized.
+ *
+ * server - Process identifier of the server from which the tool is
+ *          to be disconnected
+ *
+ * Returns PMIX_SUCCESS or a PMIx error constant
+ */
+PMIX_EXPORT pmix_status_t PMIx_tool_disconnect(const pmix_proc_t *server);
+
+
+/* Get an array containing the pmix_proc_t process identifiers of all
+ * servers to which the tool is currently connected.
+ *
+ * servers - Address where the pointer to an array of pmix_proc_t
+ *           structures shall be returned
+ *
+ * nservers - Address where the number of elements in servers
+ *            shall be returned
+ *
+ * Returns PMIX_SUCCESS or a PMIx error constant
+ */
+PMIX_EXPORT pmix_status_t PMIx_tool_get_servers(pmix_proc_t *servers[], size_t *nservers);
+
+
+/* Designate a server as the tool’s primary server.
+ *
+ * server - Process identifier of the target server
+ *
+ * Returns PMIX_SUCCESS or a PMIx error constant
+ */
+PMIX_EXPORT pmix_status_t PMIx_tool_set_server(const pmix_proc_t *server,
+                                               pmix_info_t info[], size_t ninfo);
+
+
+/* Define a callback function for delivering forwarded IO to a process
+ * This function will be called whenever data becomes available, or a
+ * specified buffering size and/or time has been met. The function
+ * will be passed the following values:
+ *
+ * iofhdlr - the returned registration number of the handler being invoked.
+ *           This is required when deregistering the handler.
+ *
+ * channel - a bitmask identifying the channel the data arrived on
+ *
+ * source - the nspace/rank of the process that generated the data
+ *
+ * payload - pointer to a PMIx byte object containing the data. Note that
+ *           multiple strings may be included, and that the data may
+ *           _not_ be NULL terminated
+ *
+ * info - an optional array of info provided by the source containing
+ *        metadata about the payload. This could include PMIX_IOF_COMPLETE
+ *
+ * ninfo - number of elements in the optional info array
+ */
+ typedef void (*pmix_iof_cbfunc_t)(size_t iofhdlr, pmix_iof_channel_t channel,
+                                   pmix_proc_t *source, pmix_byte_object_t *payload,
+                                   pmix_info_t info[], size_t ninfo);
+
+
+/* Register to receive output forwarded from a remote process.
+ *
+ * procs - array of identifiers for sources whose IO is being
+ *         requested. Wildcard rank indicates that all procs
+ *         in the specified nspace are included in the request
+ *
+ * nprocs - number of identifiers in the procs array
+ *
+ * directives - optional array of attributes to control the
+ *              behavior of the request. For example, this
+ *              might include directives on buffering IO
+ *              before delivery, and/or directives to include
+ *              or exclude any backlogged data
+ *
+ * ndirs - number of elements in the directives array
+ *
+ * channel - bitmask of IO channels included in the request.
+ *           NOTE: STDIN is not supported as it will always
+ *           be delivered to the stdin file descriptor
+ *
+ * cbfunc - function to be called when relevant IO is received. A
+ *          NULL indicates that the IO is to be written to stdout
+ *          or stderr as per the originating channel
+ *
+ * regcbfunc - since registration is async, this is the
+ *             function to be called when registration is
+ *             completed. The function itself will return
+ *             a non-success error if the registration cannot
+ *             be submitted - in this case, the regcbfunc
+ *             will _not_ be called.
+ *             If regcbfunc is NULL, then this will be treated
+ *             as a BLOCKING call - a positive return value
+ *             represents the reference ID for the request,
+ *             while negative values indicate the corresponding
+ *             error
+ *
+ * cbdata - pointer to object to be returned in regcbfunc
+ */
+PMIX_EXPORT pmix_status_t PMIx_IOF_pull(const pmix_proc_t procs[], size_t nprocs,
+                                        const pmix_info_t directives[], size_t ndirs,
+                                        pmix_iof_channel_t channel, pmix_iof_cbfunc_t cbfunc,
+                                        pmix_hdlr_reg_cbfunc_t regcbfunc, void *regcbdata);
+
+/* Deregister from output forwarded from a remote process.
+ *
+ * iofhdlr - the registration number returned from the
+ *           call to PMIx_IOF_pull
+ *
+ * directives - optional array of attributes to control the
+ *              behavior of the request. For example, this
+ *              might include directives regarding what to
+ *              do with any data currently in the IO buffer
+ *              for this process
+ *
+ * cbfunc - function to be called when deregistration has
+ *          been completed. Note that any IO to be flushed
+ *          may continue to be received after deregistration
+ *          has completed. If cbfunc is NULL, then this is
+ *          treated as a BLOCKING call and the result of
+ *          the operation will be provided in the returned status
+ *
+ * cbdata - pointer to object to be returned in cbfunc
+ */
+PMIX_EXPORT pmix_status_t PMIx_IOF_deregister(size_t iofhdlr,
+                                              const pmix_info_t directives[], size_t ndirs,
+                                              pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/* Push data collected locally (typically from stdin) to
+ * stdin of target recipients.
+ *
+ * targets - array of process identifiers to which the data is to be delivered. Note
+ *           that a WILDCARD rank indicates that all procs in the given nspace are
+ *           to receive a copy of the data
+ *
+ * ntargets - number of procs in the targets array
+ *
+ * directives - optional array of attributes to control the
+ *              behavior of the request. For example, this
+ *              might include directives on buffering IO
+ *              before delivery, and/or directives to include
+ *              or exclude any backlogged data
+ *
+ * ndirs - number of elements in the directives array
+ *
+ * bo - pointer to a byte object containing the stdin data
+ *
+ * cbfunc - callback function when the data has been forwarded. If
+ *          cbfunc is NULL, then this is treated as a BLOCKING call
+ *          and the result of the operation will be provided in the
+ *          returned status
+ *
+ * cbdata - object to be returned in cbfunc
+ */
+PMIX_EXPORT pmix_status_t PMIx_IOF_push(const pmix_proc_t targets[], size_t ntargets,
+                                        pmix_byte_object_t *bo,
+                                        const pmix_info_t directives[], size_t ndirs,
+                                        pmix_op_cbfunc_t cbfunc, void *cbdata);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
diff -Nru pmix-3.2.2~rc1/Makefile.am pmix-4.0.0/Makefile.am
--- pmix-3.2.2~rc1/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
 # Copyright (c) 2019      Amazon.com, Inc. or its affiliates.  All Rights
 #                         reserved.
 # $COPYRIGHT$
@@ -25,7 +25,7 @@
 # via AC_CONFIG_MACRO_DIR in configure.ac.
 ACLOCAL_AMFLAGS = -I ./config
 
-SUBDIRS = config contrib include src etc
+SUBDIRS = config contrib include src etc bindings
 
 AM_DISTCHECK_CONFIGURE_FLAGS = --disable-dlopen
 
diff -Nru pmix-3.2.2~rc1/NEWS pmix-4.0.0/NEWS
--- pmix-3.2.2~rc1/NEWS	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/NEWS	2021-01-02 08:56:17.000000000 +0000
@@ -23,8 +23,41 @@
 Master (not on release branches yet)
 ------------------------------------
 
+4.0.0 -- 30 Dec 2020
+----------------------
+**** NOTE: This release implements the complete PMIx v4.0 Standard
+**** and therefore includes a number of new APIs and features. These
+**** are fully documented in the official document - the details of
+**** the revisions included in v4.0 are summarized here:
+**** https://pmix.github.io/uploads/2020/12/pmix-standard-4.0.pdf#page=549
+**** Note that this version of OpenPMIx includes a first-cut at the
+**** Python bindings described in Appendix A of the v4.0 Standard.
+
+Beyond the v4.0 modifications and additions to APIs, datatypes, attributes,
+and macros, changes to the library include:
+
+ - Removal of the usock messaging component - only TCP is now supported
+ - Removal of the PMI-1 and PMI-2 backward compatibility libraries into
+   a new separate repository
+ - Packaging changes to push the headers into *-devel packages
+ - libtool patch for Mac BigSur OS
+ - Fixed dependency issue with HWLOC to protect against stone-age versions
+ - Changed man page format to Markdown, requires pandoc to generate from
+   Git repository (but not from tarball)
+ - Enable local fork/exec by tools when not connected to a server - this
+   is done transparently
+ - Support reproducible builds
+ - Multiple bug fixes and memory leak repairs
+ - Add support for network interface and GPU device distances
+ - Allow retrieval of the caller's own rank and process ID via PMIx_Get
+ - Provide full delineation of client, server, and tool attribute support
+ - Add support for libev in lieu of libevent
+ - Detect/avoid conflict with LSF version of "libevent"
+ - Auto-detect and forward envars from various identified programming models
+   (e.g., OpenMPI, OpenSHMEM) and fabrics
+
 
-3.2.2 -- TBD
+3.2.2 -- 7 Dec 2020
 ----------------------
  - PR #1930: Remove man page setup as there are no manpages in v3.2
  - PR #1933: Remove stale config command
@@ -66,14 +99,6 @@
  - PR #1885: Fix immediate flag behavior at the server
 
 
-3.1.6 -- 20 Aug 2020
-----------------------
- - PR #1669: Silence unnecessary error log message
- - PR #1776: Fixed crash of dstore locks destructor
- - PR #1825: Add option to bypass local-only fence optimization
- - PR #1830: Increase timeout in test case
-
-
 3.1.5 -- 14 Feb 2020
 ----------------------
 NOTE: The signature of the PMIx_Allocation_request has changed
diff -Nru pmix-3.2.2~rc1/README pmix-4.0.0/README
--- pmix-3.2.2~rc1/README	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/README	2021-01-02 08:56:17.000000000 +0000
@@ -15,7 +15,7 @@
 Copyright (c) 2008      IBM Corporation.  All rights reserved.
 Copyright (c) 2010      Oak Ridge National Labs.  All rights reserved.
 Copyright (c) 2011      University of Houston. All rights reserved.
-Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
 $COPYRIGHT$
 
 Additional copyrights may follow
@@ -23,18 +23,14 @@
 $HEADER$
 
 ===========================================================================
-
-When submitting questions and problems, be sure to include as much
-extra information as possible.  This web page details all the
-information that we request in order to provide assistance:
-
-     https://pmix.org/support/
-
 The best way to report bugs, send comments, or ask questions is to
 post them on the OpenPMIx GitHub issue tracker:
 
      https://github.com/openpmix/openpmix/issues
 
+When submitting questions and problems, be sure to include as much
+extra information as possible.
+
 Alternatively, you can sign up on the PMIx mailing list, which is
 hosted by Google Groups:
 
@@ -53,7 +49,7 @@
 
 More information is available in the PMIx FAQ:
 
-    https://pmix.org/support/faq/
+    https://openpmix.org/support/faq/
 
 We are in early days, so please be patient - info will grow as questions
 are addressed.
@@ -68,7 +64,7 @@
 
 - The majority of PMIx's documentation is here in this file, the
   included man pages, and on the web site FAQ
-  (https://pmix.org/support/faq/).  This will eventually be
+  (https://openpmix.org/support/faq/).  This will eventually be
   supplemented with cohesive installation and user documentation files.
 
 - Systems that have been tested are:
@@ -77,6 +73,20 @@
     and Portland (*)
   - OS X (10.7 and above), 32 and 64 bit (x86_64), with gcc (*)
 
+- OpenPMIx has taken some steps towards Reproducible Builds
+  (https://reproducible-builds.org/).  Specifically, OpenPMIx's
+  "configure" and "make" process, by default, records the build date
+  and some system-specific information such as the hostname where OpenPMIx
+  was built and the username who built it.  If you desire a
+  Reproducible Build, set the $SOURCE_DATE_EPOCH, $USER and $HOSTNAME
+  environment variables before invoking "configure" and "make", and
+  OpenPMIx will use those values instead of invoking "whoami" and/or
+  "hostname", respectively.  See
+  https://reproducible-builds.org/docs/source-date-epoch/ for
+  information on the expected format and content of the
+  $SOURCE_DATE_EPOCH variable.
+
+
 (*) Compiler Notes
 --------------
 
@@ -100,10 +110,10 @@
 
 ===========================================================================
 
-Building PMIx
+Building OpenPMIx
 -----------------
 
-PMIx uses a traditional configure script paired with "make" to
+OpenPMIx uses a traditional configure script paired with "make" to
 build.  Typical installs can be of the pattern:
 
 ---------------------------------------------------------------------------
@@ -118,7 +128,7 @@
 
 --prefix=<directory>
   Install PMIx into the base directory named <directory>.  Hence,
-  PMIx will place its executables in <directory>/bin, its header
+  OpenPMIx will place its executables in <directory>/bin, its header
   files in <directory>/include, its libraries in <directory>/lib, etc.
 
 --disable-shared
@@ -141,7 +151,7 @@
    at run time via the usual MCA-variable-setting mechanisms; this
    configure option simply sets the default value.
 
-   The --disable form of this option is intended for PMIx packagers
+   The --disable form of this option is intended for OpenPMIx packagers
    who tend to enable support for many different types of networks and
    systems in their packages.  For example, consider a packager who
    includes support for both the FOO and BAR networks in their PMIx
@@ -152,7 +162,7 @@
    potentially confusing warnings about the FOO components failing to
    load because libFOO.so is not available on their systems.
 
-   Conversely, system administrators tend to build a PMIx that is
+   Conversely, system administrators tend to build an OpenPMIx that is
    targeted at their specific environment, and contains few (if any)
    components that are not needed.  In such cases, they might want
    their users to be warned that the FOO network components failed to
@@ -164,23 +174,23 @@
   command line that are not in FILE are also used.  Options on the
   command line and in FILE are replaced by what is in FILE.
 
-Once PMIx has been built and installed, it is safe to run "make
+Once OpenPMIx has been built and installed, it is safe to run "make
 clean" and/or remove the entire build tree.
 
 VPATH and parallel builds are fully supported.
 
-Generally speaking, the only thing that users need to do to use PMIx
+Generally speaking, the only thing that users need to do to use OpenPMIx
 is ensure that <prefix>/lib is in their LD_LIBRARY_PATH.  Users may
 need to ensure to set LD_LIBRARY_PATH in their shell setup files (e.g.,
 .bashrc, .cshrc) so that non-interactive rsh/ssh-based logins will
-be able to find the PMIx library.
+be able to find the OpenPMIx library.
 
 ===========================================================================
 
-PMIx Version Numbers and Binary Compatibility
+OpenPMIx Version Numbers and Binary Compatibility
 -------------------------------------------------
 
-PMIx has two sets of version numbers that are likely of interest
+OpenPMIx has two sets of version numbers that are likely of interest
 to end users / system administrator:
 
     * Software version number
@@ -192,7 +202,7 @@
 Software Version Number
 -----------------------
 
-PMIx's version numbers are the union of several different values:
+OpenPMIx's version numbers are the union of several different values:
 major, minor, release, and an optional quantifier.
 
   * Major: The major number is the first integer in the version string
@@ -212,7 +222,7 @@
     indicate a bug fix in the code base and/or end-user
     functionality.
 
-  * Quantifier: PMIx version numbers sometimes have an arbitrary
+  * Quantifier: OpenPMIx version numbers sometimes have an arbitrary
     string affixed to the end of the version number. Common strings
     include:
 
@@ -227,7 +237,7 @@
       indicates the 4th release candidate of version 1.2.3).
 
 Although the major, minor, and release values (and optional
-quantifiers) are reported in PMIx nightly snapshot tarballs, the
+quantifiers) are reported in OpenPMIx nightly snapshot tarballs, the
 filenames of these snapshot tarballs follow a slightly different
 convention.
 
@@ -247,7 +257,7 @@
 created from the v1.0 branch, 57 Git commits after the "v1.0.2" tag,
 specifically at Git hash gb9f1fd9.
 
-PMIx's Git master branch contains a single "dev" tag.  For example,
+OpenPMIx's Git master branch contains a single "dev" tag.  For example,
 "pmix-dev-8-gf21c349.tar.bz2" represents a snapshot tarball created
 from the master branch, 8 Git commits after the "dev" tag,
 specifically at Git hash gf21c349.
@@ -259,9 +269,9 @@
 Shared Library Version Number
 -----------------------------
 
-PMIx uses the GNU Libtool shared library versioning scheme.
+OpenPMIx uses the GNU Libtool shared library versioning scheme.
 
-NOTE: Only official releases of PMIx adhere to this versioning
+NOTE: Only official releases of OpenPMIx adhere to this versioning
       scheme. "Beta" releases, release candidates, and nightly
       tarballs, developer snapshots, and Git snapshot tarballs likely
       will all have arbitrary/meaningless shared library version
@@ -296,16 +306,16 @@
 Application Binary Interface (ABI) Compatibility
 ------------------------------------------------
 
-PMIx provides forward ABI compatibility in all versions of a given
+OpenPMIx provides forward ABI compatibility in all versions of a given
 feature release series and its corresponding
-super stable series.  For example, on a single platform, an pmix
-application linked against PMIx v1.3.2 shared libraries can be
+super stable series.  For example, on a single platform, a PMIx
+application linked against OpenPMIx v1.3.2 shared libraries can be
 updated to point to the shared libraries in any successive v1.3.x or
 v1.4 release and still work properly (e.g., via the LD_LIBRARY_PATH
 environment variable or other operating system mechanism).
 
-PMIx reserves the right to break ABI compatibility at new feature
-release series.  For example, the same pmix application from above
+OpenPMIx reserves the right to break ABI compatibility at new feature
+release series.  For example, the same PMIx application from above
 (linked against PMIx v1.3.2 shared libraries) will *not* work with
 PMIx v1.5 shared libraries.
 
@@ -314,10 +324,10 @@
 Common Questions
 ----------------
 
-Many common questions about building and using PMIx are answered
+Many common questions about building and using OpenPMIx are answered
 on the FAQ:
 
-    https://pmix.org/support/faq/
+    https://openpmix.org/support/faq/
 
 ===========================================================================
 
@@ -325,20 +335,20 @@
 -------------------
 
 Found a bug?  Got a question?  Want to make a suggestion?  Want to
-contribute to PMIx?  Please let us know!
+contribute to OpenPMIx?  Please let us know!
 
 When submitting questions and problems, be sure to include as much
 extra information as possible.  This web page details all the
 information that we request in order to provide assistance:
 
-     https://pmix.org/support/
+     https://openpmix.org/support/
 
 Questions and comments should generally be posted to the OpenPMIx
 GitHub issue tracker:
 
     https://github.com/openpmix/openpmix/issues
 
-Alternatively, question can also be sent to the PMIx mailing list
+Alternatively, question can also be sent to the OpenPMIx mailing list
 (pmix@googlegroups.com).  Because of spam, only subscribers are
 allowed to post to this list (ensure that you subscribe with and post
 from *exactly* the same e-mail address -- joe@example.com is
@@ -347,4 +357,4 @@
 
      https://groups.google.com/d/forum/pmix
 
-Make today an PMIx day!
+Make today a PMIx day!
diff -Nru pmix-3.2.2~rc1/src/client/Makefile.include pmix-4.0.0/src/client/Makefile.include
--- pmix-3.2.2~rc1/src/client/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/client/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -20,11 +20,7 @@
         client/pmix_client_get.c \
         client/pmix_client_pub.c \
         client/pmix_client_spawn.c \
-        client/pmix_client_connect.c
-
-if WANT_PMI_BACKWARD
-pmi1_sources += \
-        client/pmi1.c
-pmi2_sources += \
-        client/pmi2.c
-endif
+        client/pmix_client_connect.c \
+        client/pmix_client_group.c \
+        client/pmix_client_fabric.c \
+        client/pmix_client_topology.c
diff -Nru pmix-3.2.2~rc1/src/client/pmi1.c pmix-4.0.0/src/client/pmi1.c
--- pmix-3.2.2~rc1/src/client/pmi1.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/client/pmi1.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,928 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "src/include/pmix_config.h"
-
-#include "include/pmix.h"
-#include "include/pmi.h"
-
-#include "src/include/pmix_globals.h"
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif
-#include PMIX_EVENT_HEADER
-
-#define ANL_MAPPING "PMI_process_mapping"
-
-#include "src/mca/bfrops/bfrops.h"
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/util/output.h"
-
-#define PMI_MAX_ID_LEN       PMIX_MAX_NSLEN  /* Maximim size of PMI process group ID */
-#define PMI_MAX_KEY_LEN      PMIX_MAX_KEYLEN /* Maximum size of a PMI key */
-#define PMI_MAX_KVSNAME_LEN  PMIX_MAX_NSLEN  /* Maximum size of KVS name */
-#define PMI_MAX_VAL_LEN      4096            /* Maximum size of a PMI value */
-
-
-#define PMI_CHECK()             \
-    do {                        \
-        if (!pmi_init) {        \
-            return PMI_FAIL;    \
-        }                       \
-    } while (0)
-
-/* local functions */
-static pmix_status_t convert_int(int *value, pmix_value_t *kv);
-static int convert_err(pmix_status_t rc);
-static pmix_proc_t myproc;
-static int pmi_init = 0;
-static bool pmi_singleton = false;
-
-PMIX_EXPORT int PMI_Init(int *spawned)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_proc_t proc;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-
-    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
-        /* if we didn't see a PMIx server (e.g., missing envar),
-         * then allow us to run as a singleton */
-        if (PMIX_ERR_INVALID_NAMESPACE == rc) {
-            if (NULL != spawned) {
-                *spawned = 0;
-            }
-            pmi_singleton = true;
-            pmix_strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN);
-            myproc.rank = 0;
-            pmi_init = 1;
-            return PMI_SUCCESS;
-        }
-        return PMI_ERR_INIT;
-    }
-
-    /* getting internal key requires special rank value */
-    memcpy(&proc, &myproc, sizeof(myproc));
-    proc.rank = PMIX_RANK_WILDCARD;
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    if (NULL != spawned) {
-        /* get the spawned flag */
-        if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_SPAWNED, info, 1, &val)) {
-            rc = convert_int(spawned, val);
-            PMIX_VALUE_RELEASE(val);
-            if (PMIX_SUCCESS != rc) {
-                goto error;
-            }
-        } else {
-            /* if not found, default to not spawned */
-            *spawned = 0;
-        }
-    }
-    pmi_init = 1;
-
-    rc = PMIX_SUCCESS;
-
-error:
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Initialized(PMI_BOOL *initialized)
-{
-    if (NULL == initialized) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        *initialized = PMI_TRUE;
-    } else {
-        *initialized = (PMIx_Initialized() ? PMI_TRUE : PMI_FALSE);
-    }
-
-    return PMI_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_Finalize(void)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-
-    PMI_CHECK();
-
-    if (pmi_singleton) {
-        return PMI_SUCCESS;
-    }
-
-    pmi_init = 0;
-    rc = PMIx_Finalize(NULL, 0);
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Abort(int flag, const char msg[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-
-    PMI_CHECK();
-
-    if (pmi_singleton) {
-        return PMI_SUCCESS;
-    }
-
-    rc = PMIx_Abort(flag, msg, NULL, 0);
-    return convert_err(rc);
-}
-
-/* KVS_Put - we default to PMIX_GLOBAL scope and ignore the
- * provided kvsname as we only put into our own nspace */
-PMIX_EXPORT int PMI_KVS_Put(const char kvsname[], const char key[], const char value[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t val;
-
-    PMI_CHECK();
-
-    if ((kvsname == NULL) || (strlen(kvsname) > PMI_MAX_KVSNAME_LEN)) {
-        return PMI_ERR_INVALID_KVS;
-    }
-    if ((key == NULL) || (strlen(key) >PMI_MAX_KEY_LEN)) {
-        return PMI_ERR_INVALID_KEY;
-    }
-    if ((value == NULL) || (strlen(value) > PMI_MAX_VAL_LEN)) {
-        return PMI_ERR_INVALID_VAL;
-    }
-    if (pmi_singleton) {
-        return PMI_SUCCESS;
-    }
-
-    pmix_output_verbose(2, pmix_globals.debug_output,
-            "PMI_KVS_Put: KVS=%s, key=%s value=%s", kvsname, key, value);
-
-    val.type = PMIX_STRING;
-    val.data.string = (char*)value;
-    rc = PMIx_Put(PMIX_GLOBAL, key, &val);
-    return convert_err(rc);
-}
-
-/* KVS_Commit */
-PMIX_EXPORT int PMI_KVS_Commit(const char kvsname[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-
-    PMI_CHECK();
-
-    if ((kvsname == NULL) || (strlen(kvsname) > PMI_MAX_KVSNAME_LEN)) {
-        return PMI_ERR_INVALID_KVS;
-    }
-    if (pmi_singleton) {
-        return PMI_SUCCESS;
-    }
-
-    pmix_output_verbose(2, pmix_globals.debug_output, "PMI_KVS_Commit: KVS=%s",
-            kvsname);
-
-    rc = PMIx_Commit();
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[], int length)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_proc_t proc;
-
-    PMI_CHECK();
-
-    if ((kvsname == NULL) || (strlen(kvsname) > PMI_MAX_KVSNAME_LEN)) {
-        return PMI_ERR_INVALID_KVS;
-    }
-    if ((key == NULL) || (strlen(key) > PMI_MAX_KEY_LEN)) {
-        return PMI_ERR_INVALID_KEY;
-    }
-    if (value == NULL) {
-        return PMI_ERR_INVALID_VAL;
-    }
-
-    pmix_output_verbose(2, pmix_globals.debug_output,
-            "PMI_KVS_Get: KVS=%s, key=%s value=%s", kvsname, key, value);
-
-    /* PMI-1 expects resource manager to set
-     * process mapping in ANL notation. */
-    if (!strcmp(key, ANL_MAPPING)) {
-        /* we are looking in the job-data. If there is nothing there
-         * we don't want to look in rank's data, thus set rank to widcard */
-        proc = myproc;
-        proc.rank = PMIX_RANK_WILDCARD;
-        if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) &&
-               (NULL != val) && (PMIX_STRING == val->type)) {
-            pmix_strncpy(value, val->data.string, length-1);
-            PMIX_VALUE_FREE(val, 1);
-            return PMI_SUCCESS;
-        } else {
-            /* artpol:
-             * Some RM's (i.e. SLURM) already have ANL precomputed. The export it
-             * through PMIX_ANL_MAP variable.
-             * If we haven't found it we want to have our own packing functionality
-             * since it's common.
-             * Somebody else has to write it since I've already done that for
-             * GPL'ed SLURM :) */
-            return PMI_FAIL;
-        }
-    }
-
-    /* retrieve the data from PMIx - since we don't have a rank,
-     * we indicate that by passing the UNDEF value */
-    pmix_strncpy(proc.nspace, kvsname, PMIX_MAX_NSLEN);
-    proc.rank = PMIX_RANK_UNDEF;
-
-    rc = PMIx_Get(&proc, key, NULL, 0, &val);
-    if (PMIX_SUCCESS == rc && NULL != val) {
-        if (PMIX_STRING != val->type) {
-            rc = PMIX_ERROR;
-        } else if (NULL != val->data.string) {
-            pmix_strncpy(value, val->data.string, length-1);
-        }
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    return convert_err(rc);
-}
-
-/* Barrier only applies to our own nspace, and we want all
- * data to be collected upon completion */
-PMIX_EXPORT int PMI_Barrier(void)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_info_t buf;
-    int ninfo = 0;
-    pmix_info_t *info = NULL;
-    bool val = 1;
-
-    PMI_CHECK();
-
-    if (pmi_singleton) {
-        return PMI_SUCCESS;
-    }
-
-    info = &buf;
-    PMIX_INFO_CONSTRUCT(info);
-    PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &val, PMIX_BOOL);
-    ninfo = 1;
-    rc = PMIx_Fence(NULL, 0, info, ninfo);
-
-    PMIX_INFO_DESTRUCT(info);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Get_size(int *size)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_WILDCARD;
-
-    PMI_CHECK();
-
-    if (NULL == size) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        *size = 1;
-        return PMI_SUCCESS;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    rc = PMIx_Get(&proc, PMIX_JOB_SIZE, info, 1, &val);
-    if (PMIX_SUCCESS == rc) {
-        rc = convert_int(size, val);
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Get_rank(int *rk)
-{
-    PMI_CHECK();
-
-    if (NULL == rk) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    *rk = myproc.rank;
-    return PMI_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_Get_universe_size(int *size)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_WILDCARD;
-
-    PMI_CHECK();
-
-    if (NULL == size) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        *size = 1;
-        return PMI_SUCCESS;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, info, 1, &val);
-    if (PMIX_SUCCESS == rc) {
-        rc = convert_int(size, val);
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Get_appnum(int *appnum)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-
-    PMI_CHECK();
-
-    if (NULL == appnum) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        *appnum = 0;
-        return PMI_SUCCESS;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    rc = PMIx_Get(&myproc, PMIX_APPNUM, info, 1, &val);
-    if (PMIX_SUCCESS == rc) {
-        rc = convert_int(appnum, val);
-        PMIX_VALUE_RELEASE(val);
-    } else {
-        /* this is optional value, set to 0 */
-        *appnum = 0;
-        rc = PMIX_SUCCESS;
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Publish_name(const char service_name[], const char port[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_info_t info;
-
-    PMI_CHECK();
-
-    if (NULL == service_name || NULL == port) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        return PMI_FAIL;
-    }
-
-    /* pass the service/port */
-    pmix_strncpy(info.key, service_name, PMIX_MAX_KEYLEN);
-    info.value.type = PMIX_STRING;
-    info.value.data.string = (char*) port;
-
-    /* publish the info - PMI-1 doesn't support
-     * any scope other than inside our own nspace */
-    rc = PMIx_Publish(&info, 1);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Unpublish_name(const char service_name[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    char *keys[2];
-
-    PMI_CHECK();
-
-    if (NULL == service_name) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        return PMI_FAIL;
-    }
-
-    /* pass the service */
-    keys[0] = (char*) service_name;
-    keys[1] = NULL;
-
-    rc = PMIx_Unpublish(keys, NULL, 0);
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Lookup_name(const char service_name[], char port[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_pdata_t pdata;
-
-    PMI_CHECK();
-
-    if (NULL == service_name || NULL == port) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        return PMI_FAIL;
-    }
-
-    PMIX_PDATA_CONSTRUCT(&pdata);
-
-    /* pass the service */
-    pmix_strncpy(pdata.key, service_name, PMIX_MAX_KEYLEN);
-
-    /* PMI-1 doesn't want the nspace back */
-    if (PMIX_SUCCESS != (rc = PMIx_Lookup(&pdata, 1, NULL, 0))) {
-        return convert_err(rc);
-    }
-
-    /* should have received a string back */
-    if (PMIX_STRING != pdata.value.type || NULL == pdata.value.data.string) {
-        return convert_err(PMIX_ERR_NOT_FOUND);
-    }
-
-    /* return the port - sadly, this API doesn't tell us
-     * the size of the port array, and so there is a
-     * potential we could overrun it. As this feature
-     * isn't widely supported in PMI-1, try being
-     * conservative */
-    pmix_strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN);
-    PMIX_PDATA_DESTRUCT(&pdata);
-
-    return PMIX_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_Get_id(char id_str[], int length)
-{
-    /* we already obtained our nspace during PMI_Init,
-     * so all we have to do here is return it */
-
-    PMI_CHECK();
-
-    /* bozo check */
-    if (NULL == id_str) {
-        return PMI_ERR_INVALID_ARGS;
-    }
-    if (length < PMI_MAX_ID_LEN) {
-        return PMI_ERR_INVALID_LENGTH;
-    }
-
-    pmix_strncpy(id_str, myproc.nspace, length-1);
-    return PMI_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_Get_kvs_domain_id(char id_str[], int length)
-{
-    PMI_CHECK();
-
-    /* same as PMI_Get_id */
-    return PMI_Get_id(id_str, length);
-}
-
-PMIX_EXPORT int PMI_Get_id_length_max(int *length)
-{
-    PMI_CHECK();
-
-    if (NULL == length) {
-        return PMI_ERR_INVALID_VAL_LENGTH;
-    }
-
-    *length = PMI_MAX_ID_LEN;
-    return PMI_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_Get_clique_size(int *size)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_WILDCARD;
-
-    PMI_CHECK();
-
-    if (NULL == size) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        *size = 1;
-        return PMI_SUCCESS;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    rc = PMIx_Get(&proc, PMIX_LOCAL_SIZE, info, 1, &val);
-    if (PMIX_SUCCESS == rc) {
-        rc = convert_int(size, val);
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_Get_clique_ranks(int ranks[], int length)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    char **rks;
-    int i;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_WILDCARD;
-
-    PMI_CHECK();
-
-    if (NULL == ranks) {
-        return PMI_ERR_INVALID_ARGS;
-    }
-
-    if (pmi_singleton) {
-        ranks[0] = 0;
-        return PMI_SUCCESS;
-    }
-
-    rc = PMIx_Get(&proc, PMIX_LOCAL_PEERS, NULL, 0, &val);
-    if (PMIX_SUCCESS == rc) {
-        /* kv will contain a string of comma-separated
-         * ranks on my node */
-        rks = pmix_argv_split(val->data.string, ',');
-        for (i = 0; NULL != rks[i] && i < length; i++) {
-            ranks[i] = strtol(rks[i], NULL, 10);
-        }
-        pmix_argv_free(rks);
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI_KVS_Get_my_name(char kvsname[], int length)
-{
-    PMI_CHECK();
-
-    /* same as PMI_Get_id */
-    return PMI_Get_id(kvsname, length);
-}
-
-PMIX_EXPORT int PMI_KVS_Get_name_length_max(int *length)
-{
-    PMI_CHECK();
-
-    if (NULL == length) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    *length = PMI_MAX_KVSNAME_LEN;
-    return PMI_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_KVS_Get_key_length_max(int *length)
-{
-    PMI_CHECK();
-
-    if (NULL == length) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    *length = PMI_MAX_KEY_LEN;
-    return PMI_SUCCESS;
-}
-
-PMIX_EXPORT int PMI_KVS_Get_value_length_max(int *length)
-{
-    PMI_CHECK();
-
-    if (NULL == length) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    /* don't give them an enormous size of some implementations
-     * immediately malloc a data block for their use */
-    *length = PMI_MAX_VAL_LEN;
-    return PMI_SUCCESS;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_KVS_Create(char kvsname[], int length)
-{
-    return PMI_FAIL;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_KVS_Destroy(const char kvsname[])
-{
-    return PMI_FAIL;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_KVS_Iter_first(const char kvsname[], char key[], int key_len, char val[], int val_len)
-{
-    return PMI_FAIL;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_KVS_Iter_next(const char kvsname[], char key[], int key_len, char val[], int val_len)
-{
-    return PMI_FAIL;
-}
-
-PMIX_EXPORT int PMI_Spawn_multiple(int count,
-                       const char * cmds[],
-                       const char ** argvs[],
-                       const int maxprocs[],
-                       const int info_keyval_sizesp[],
-                       const PMI_keyval_t * info_keyval_vectors[],
-                       int preput_keyval_size,
-                       const PMI_keyval_t preput_keyval_vector[],
-                       int errors[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_app_t *apps;
-    int i, k;
-    size_t j;
-    char *evar;
-
-    PMI_CHECK();
-
-    if (NULL == cmds) {
-        return PMI_ERR_INVALID_ARG;
-    }
-
-    if (pmi_singleton) {
-        return PMI_FAIL;
-    }
-
-    /* setup the apps */
-    PMIX_APP_CREATE(apps, count);
-    for (i = 0; i < count; i++) {
-        apps[i].cmd = strdup(cmds[i]);
-        apps[i].maxprocs = maxprocs[i];
-        apps[i].argv = pmix_argv_copy((char**) argvs[i]);
-        apps[i].ninfo = info_keyval_sizesp[i];
-        if (0 < apps[i].ninfo) {
-            apps[i].info = (pmix_info_t*)malloc(apps[i].ninfo * sizeof(pmix_info_t));
-            /* copy the info objects */
-            for (j = 0; j < apps[i].ninfo; j++) {
-                pmix_strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN);
-                apps[i].info[j].value.type = PMIX_STRING;
-                apps[i].info[j].value.data.string = strdup(info_keyval_vectors[i][j].val);
-            }
-        }
-        /* push the preput values into the apps environ */
-        for (k = 0; k < preput_keyval_size; k++) {
-            if (0 > asprintf(&evar, "%s=%s", preput_keyval_vector[k].key, preput_keyval_vector[k].val)) {
-                for (i = 0; i < count; i++) {
-                    PMIX_APP_DESTRUCT(&apps[i]);
-                }
-                free(apps);
-                return PMIX_ERR_NOMEM;
-            }
-            pmix_argv_append_nosize(&apps[i].env, evar);
-            free(evar);
-        }
-    }
-
-    rc = PMIx_Spawn(NULL, 0, apps, count, NULL);
-    /* tear down the apps array */
-    for (i = 0; i < count; i++) {
-        PMIX_APP_DESTRUCT(&apps[i]);
-    }
-    free(apps);
-    if (NULL != errors) {
-        for (i = 0; i < count; i++) {
-            errors[i] = convert_err(rc);
-        }
-    }
-    return convert_err(rc);
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_Parse_option(int num_args, char *args[], int *num_parsed, PMI_keyval_t **keyvalp, int *size)
-{
-    return PMI_FAIL;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_Args_to_keyval(int *argcp, char *((*argvp)[]), PMI_keyval_t **keyvalp, int *size)
-{
-    return PMI_FAIL;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_Free_keyvals(PMI_keyval_t keyvalp[], int size)
-{
-    return PMI_FAIL;
-}
-
-/* nobody supports this call, which is why it was
- * dropped for PMI-2 */
-PMIX_EXPORT int PMI_Get_options(char *str, int *length)
-{
-    return PMI_FAIL;
-}
-
-/***   UTILITY FUNCTIONS   ***/
-/* internal function */
-static pmix_status_t convert_int(int *value, pmix_value_t *kv)
-{
-    switch (kv->type) {
-    case PMIX_INT:
-        *value = kv->data.integer;
-        break;
-    case PMIX_INT8:
-        *value = kv->data.int8;
-        break;
-    case PMIX_INT16:
-        *value = kv->data.int16;
-        break;
-    case PMIX_INT32:
-        *value = kv->data.int32;
-        break;
-    case PMIX_INT64:
-        *value = kv->data.int64;
-        break;
-    case PMIX_UINT:
-        *value = kv->data.uint;
-        break;
-    case PMIX_UINT8:
-        *value = kv->data.uint8;
-        break;
-    case PMIX_UINT16:
-        *value = kv->data.uint16;
-        break;
-    case PMIX_UINT32:
-        *value = kv->data.uint32;
-        break;
-    case PMIX_UINT64:
-        *value = kv->data.uint64;
-        break;
-    case PMIX_BYTE:
-        *value = kv->data.byte;
-        break;
-    case PMIX_SIZE:
-        *value = kv->data.size;
-        break;
-    case PMIX_BOOL:
-        *value = kv->data.flag;
-        break;
-    default:
-        /* not an integer type */
-        return PMIX_ERR_BAD_PARAM;
-    }
-    return PMIX_SUCCESS;
-}
-
-static int convert_err(pmix_status_t rc)
-{
-    switch (rc) {
-    case PMIX_ERR_INVALID_SIZE:
-        return PMI_ERR_INVALID_SIZE;
-
-    case PMIX_ERR_INVALID_KEYVALP:
-        return PMI_ERR_INVALID_KEYVALP;
-
-    case PMIX_ERR_INVALID_NUM_PARSED:
-        return PMI_ERR_INVALID_NUM_PARSED;
-
-    case PMIX_ERR_INVALID_ARGS:
-        return PMI_ERR_INVALID_ARGS;
-
-    case PMIX_ERR_INVALID_NUM_ARGS:
-        return PMI_ERR_INVALID_NUM_ARGS;
-
-    case PMIX_ERR_INVALID_LENGTH:
-        return PMI_ERR_INVALID_LENGTH;
-
-    case PMIX_ERR_INVALID_VAL_LENGTH:
-        return PMI_ERR_INVALID_VAL_LENGTH;
-
-    case PMIX_ERR_INVALID_VAL:
-        return PMI_ERR_INVALID_VAL;
-
-    case PMIX_ERR_INVALID_KEY_LENGTH:
-        return PMI_ERR_INVALID_KEY_LENGTH;
-
-    case PMIX_ERR_INVALID_KEY:
-        return PMI_ERR_INVALID_KEY;
-
-    case PMIX_ERR_INVALID_ARG:
-        return PMI_ERR_INVALID_ARG;
-
-    case PMIX_ERR_NOMEM:
-        return PMI_ERR_NOMEM;
-
-    case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER:
-    case PMIX_ERR_LOST_CONNECTION_TO_SERVER:
-    case PMIX_ERR_LOST_PEER_CONNECTION:
-    case PMIX_ERR_LOST_CONNECTION_TO_CLIENT:
-    case PMIX_ERR_NOT_SUPPORTED:
-    case PMIX_ERR_NOT_FOUND:
-    case PMIX_ERR_SERVER_NOT_AVAIL:
-    case PMIX_ERR_INVALID_NAMESPACE:
-    case PMIX_ERR_DATA_VALUE_NOT_FOUND:
-    case PMIX_ERR_OUT_OF_RESOURCE:
-    case PMIX_ERR_RESOURCE_BUSY:
-    case PMIX_ERR_BAD_PARAM:
-    case PMIX_ERR_IN_ERRNO:
-    case PMIX_ERR_UNREACH:
-    case PMIX_ERR_TIMEOUT:
-    case PMIX_ERR_NO_PERMISSIONS:
-    case PMIX_ERR_PACK_MISMATCH:
-    case PMIX_ERR_PACK_FAILURE:
-    case PMIX_ERR_UNPACK_FAILURE:
-    case PMIX_ERR_UNPACK_INADEQUATE_SPACE:
-    case PMIX_ERR_TYPE_MISMATCH:
-    case PMIX_ERR_PROC_ENTRY_NOT_FOUND:
-    case PMIX_ERR_UNKNOWN_DATA_TYPE:
-    case PMIX_ERR_WOULD_BLOCK:
-    case PMIX_EXISTS:
-    case PMIX_ERROR:
-        return PMI_FAIL;
-
-    case PMIX_ERR_INIT:
-        return PMI_ERR_INIT;
-
-    case PMIX_SUCCESS:
-        return PMI_SUCCESS;
-    default:
-        return PMI_FAIL;
-    }
-}
diff -Nru pmix-3.2.2~rc1/src/client/pmi2.c pmix-4.0.0/src/client/pmi2.c
--- pmix-3.2.2~rc1/src/client/pmi2.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/client/pmi2.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,882 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015-2019 Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2016      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "src/include/pmix_config.h"
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_STDLIB_H
-#include <stdlib.h>
-#endif
-#include PMIX_EVENT_HEADER
-
-#include "include/pmi2.h"
-#include "include/pmix.h"
-
-#include "src/mca/bfrops/bfrops.h"
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/util/output.h"
-#include "src/include/pmix_globals.h"
-
-#define ANL_MAPPING "PMI_process_mapping"
-
-#define PMI2_CHECK()                \
-    do {                            \
-        if (!pmi2_init) {           \
-            return PMI2_FAIL;       \
-        }                           \
-    } while (0)
-
-/* local functions */
-static pmix_status_t convert_int(int *value, pmix_value_t *kv);
-static int convert_err(pmix_status_t rc);
-static pmix_proc_t myproc;
-static int pmi2_init = 0;
-static bool commit_reqd = false;
-static bool pmi2_singleton = false;
-
-PMIX_EXPORT int PMI2_Init(int *spawned, int *size, int *rank, int *appnum)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_WILDCARD;
-
-    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
-        /* if we didn't see a PMIx server (e.g., missing envar),
-         * then allow us to run as a singleton */
-        if (PMIX_ERR_INVALID_NAMESPACE == rc) {
-            if (NULL != spawned) {
-                *spawned = 0;
-            }
-            if (NULL != size) {
-                *size = 1;
-            }
-            if (NULL != rank) {
-                *rank = 0;
-            }
-            if (NULL != appnum) {
-                *appnum = 0;
-            }
-            pmi2_singleton = true;
-            pmix_strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN);
-            myproc.rank = 0;
-            pmi2_init = 1;
-            return PMI2_SUCCESS;
-        }
-        return PMI2_ERR_INIT;
-    }
-
-    /* get the rank */
-    *rank = myproc.rank;
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    if (NULL != size) {
-        /* get the universe size - this will likely pull
-         * down all attributes assigned to the job, thus
-         * making all subsequent "get" operations purely
-         * local */
-        if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_UNIV_SIZE, info, 1, &val)) {
-            rc = convert_int(size, val);
-            PMIX_VALUE_RELEASE(val);
-            if (PMIX_SUCCESS != rc) {
-                goto error;
-            }
-        } else {
-            /* cannot continue without this info */
-            rc = PMIX_ERR_INIT;
-            goto error;
-        }
-    }
-
-    if (NULL != spawned) {
-        /* get the spawned flag */
-        if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_SPAWNED, info, 1, &val)) {
-            rc = convert_int(spawned, val);
-            PMIX_VALUE_RELEASE(val);
-            if (PMIX_SUCCESS != rc) {
-                goto error;
-            }
-        } else {
-            /* if not found, default to not spawned */
-            *spawned = 0;
-        }
-    }
-
-    if (NULL != appnum) {
-        /* get our appnum */
-        if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_APPNUM, info, 1, &val)) {
-            rc = convert_int(appnum, val);
-            PMIX_VALUE_RELEASE(val);
-            if (PMIX_SUCCESS != rc) {
-                goto error;
-            }
-        } else {
-            /* if not found, default to 0 */
-            *appnum = 0;
-        }
-    }
-    pmi2_init = 1;
-
-    rc = PMIX_SUCCESS;
-
-error:
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Initialized(void)
-{
-    int initialized;
-    if (pmi2_singleton) {
-        return 1;
-    }
-
-    initialized = (int)PMIx_Initialized();
-    return initialized;
-}
-
-PMIX_EXPORT int PMI2_Finalize(void)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-
-    PMI2_CHECK();
-
-    pmi2_init = 0;
-    if (pmi2_singleton) {
-        return PMI2_SUCCESS;
-    }
-
-    rc = PMIx_Finalize(NULL, 0);
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Abort(int flag, const char msg[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-
-    PMI2_CHECK();
-
-    if (pmi2_singleton) {
-        return PMI2_SUCCESS;
-    }
-
-    rc = PMIx_Abort(flag, msg, NULL, 0);
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Job_Spawn(int count, const char * cmds[],
-                   int argcs[], const char ** argvs[],
-                   const int maxprocs[],
-                   const int info_keyval_sizes[],
-                   const PMI_keyval_t *info_keyval_vectors[],
-                   int preput_keyval_size,
-                   const PMI_keyval_t *preput_keyval_vector[],
-                   char jobId[], int jobIdSize,
-                   int errors[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_app_t *apps;
-    int i, k;
-    size_t j;
-    char *evar;
-
-    PMI2_CHECK();
-
-    if (NULL == cmds) {
-        return PMI2_ERR_INVALID_ARGS;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    /* setup the apps */
-    PMIX_APP_CREATE(apps, count);
-    for (i=0; i < count; i++) {
-        apps[i].cmd = strdup(cmds[i]);
-        apps[i].maxprocs = maxprocs[i];
-        apps[i].argv = pmix_argv_copy((char**)argvs[i]);
-        apps[i].ninfo = info_keyval_sizes[i];
-        apps[i].info = (pmix_info_t*)malloc(apps[i].ninfo * sizeof(pmix_info_t));
-        /* copy the info objects */
-        for (j=0; j < apps[i].ninfo; j++) {
-            pmix_strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN);
-            apps[i].info[j].value.type = PMIX_STRING;
-            apps[i].info[j].value.data.string = strdup(info_keyval_vectors[i][j].val);
-        }
-        /* push the preput values into the apps environ */
-        for (k=0; k < preput_keyval_size; k++) {
-            if (0 > asprintf(&evar, "%s=%s", preput_keyval_vector[j]->key, preput_keyval_vector[j]->val)) {
-                for (i = 0; i < count; i++) {
-                    PMIX_APP_DESTRUCT(&apps[i]);
-                }
-                free(apps);
-                return PMIX_ERR_NOMEM;
-            }
-            pmix_argv_append_nosize(&apps[i].env, evar);
-            free(evar);
-        }
-    }
-
-    rc = PMIx_Spawn(NULL, 0, apps, count, NULL);
-    /* tear down the apps array */
-    for (i=0; i < count; i++) {
-        PMIX_APP_DESTRUCT(&apps[i]);
-    }
-    free(apps);
-    if (NULL != errors) {
-        for (i=0; i < count; i++) {
-            errors[i] = convert_err(rc);
-        }
-    }
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Job_GetId(char jobid[], int jobid_size)
-{
-    /* we already obtained our nspace during pmi2_init,
-     * so all we have to do here is return it */
-
-    PMI2_CHECK();
-
-    /* bozo check */
-    if (NULL == jobid) {
-        return PMI2_ERR_INVALID_ARGS;
-    }
-    pmix_strncpy(jobid, myproc.nspace, jobid_size-1);
-    return PMI2_SUCCESS;
-}
-
-PMIX_EXPORT int PMI2_Job_GetRank(int *rank)
-{
-    PMI2_CHECK();
-
-    if (NULL == rank) {
-        return PMI2_ERR_INVALID_ARGS;
-    }
-    *rank = myproc.rank;
-    return PMI2_SUCCESS;
-}
-
-PMIX_EXPORT int PMI2_Info_GetSize(int *size)
-{
-    pmix_status_t rc = PMIX_ERROR;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_WILDCARD;
-
-
-    PMI2_CHECK();
-
-    if (NULL == size) {
-        return PMI2_ERR_INVALID_ARGS;
-    }
-
-    if (pmi2_singleton) {
-        *size = 1;
-        return PMI2_SUCCESS;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_LOCAL_SIZE, info, 1, &val)) {
-        rc = convert_int(size, val);
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_proc_t proc;
-
-    PMI2_CHECK();
-
-    if (NULL == conn) {
-        return PMI2_ERR_INVALID_ARGS;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    memset(proc.nspace, 0, sizeof(proc.nspace));
-    pmix_strncpy(proc.nspace, (jobid ? jobid : proc.nspace), PMIX_MAX_NSLEN);
-    proc.rank = PMIX_RANK_WILDCARD;
-    rc = PMIx_Connect(&proc, 1, NULL, 0);
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_proc_t proc;
-
-    PMI2_CHECK();
-
-    if (pmi2_singleton) {
-        return PMI2_SUCCESS;
-    }
-
-    memset(proc.nspace, 0, sizeof(proc.nspace));
-    pmix_strncpy(proc.nspace, (jobid ? jobid : proc.nspace), PMIX_MAX_NSLEN);
-    proc.rank = PMIX_RANK_WILDCARD;
-    rc = PMIx_Disconnect(&proc, 1, NULL, 0);
-    return convert_err(rc);
-}
-
-/* KVS_Put - we default to PMIX_GLOBAL scope */
-PMIX_EXPORT int PMI2_KVS_Put(const char key[], const char value[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t val;
-
-    PMI2_CHECK();
-
-    if ((NULL == key) || (NULL == value)) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_SUCCESS;
-    }
-
-    pmix_output_verbose(3, pmix_globals.debug_output,
-            "PMI2_KVS_Put: key=%s value=%s", key, value);
-
-    val.type = PMIX_STRING;
-    val.data.string = (char*)value;
-    if (PMIX_SUCCESS == (rc = PMIx_Put(PMIX_GLOBAL, key, &val))) {
-        commit_reqd = true;
-    }
-    return convert_err(rc);
-}
-
-/* KVS_Fence */
-PMIX_EXPORT int PMI2_KVS_Fence(void)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-
-    PMI2_CHECK();
-
-    pmix_output_verbose(3, pmix_globals.debug_output, "PMI2_KVS_Fence");
-
-    if (pmi2_singleton) {
-        return PMI2_SUCCESS;
-    }
-
-    if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
-        return convert_err(rc);
-    }
-    commit_reqd = false;
-
-    /* we want all data to be collected upon completion */
-    {
-        pmix_info_t info[1];
-        bool  val_data = 1;
-
-        /* set controlling parameters
-         * PMIX_COLLECT_DATA - meet legacy PMI2 requirement
-         */
-        PMIX_INFO_CONSTRUCT(&info[0]);
-        PMIX_INFO_LOAD(&info[0], PMIX_COLLECT_DATA, &val_data, PMIX_BOOL);
-
-        rc = PMIx_Fence(NULL, 0, &info[0], 1);
-        PMIX_INFO_DESTRUCT(&info[0]);
-    }
-
-    return convert_err(rc);
-}
-
-/* the jobid is equated to the nspace in PMIx, and the
- * src_pmi_id equates to the rank. If jobid=NULL, then PMIx
- * will use the local nspace, which matches the PMI2 spec.
- * The only type of value supported by PMI2 is a string, so
- * the return of anything else is an error */
-PMIX_EXPORT int PMI2_KVS_Get(const char *jobid, int src_pmi_id,
-                             const char key[], char value [],
-                             int maxvalue, int *vallen)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_proc_t proc;
-
-    PMI2_CHECK();
-
-    if (commit_reqd) {
-        /* they didn't commit after a put */
-        return PMI2_FAIL;
-    }
-    /* set default */
-    *vallen = 0;
-
-    if ((NULL == key) || (NULL == value)) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    pmix_output_verbose(3, pmix_globals.debug_output,
-            "PMI2_KVS_Get: key=%s jobid=%s src_pmi_id=%d", key, (jobid ? jobid : "null"), src_pmi_id);
-
-    pmix_strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), PMIX_MAX_NSLEN);
-    if (src_pmi_id == PMI2_ID_NULL) {
-        /* the rank is UNDEF */
-        proc.rank = PMIX_RANK_UNDEF;
-    } else {
-        proc.rank = src_pmi_id;
-    }
-
-    rc = PMIx_Get(&proc, key, NULL, 0, &val);
-    if (PMIX_SUCCESS == rc && NULL != val) {
-        if (PMIX_STRING != val->type) {
-            rc = PMIX_ERROR;
-        } else if (NULL != val->data.string) {
-            pmix_strncpy(value, val->data.string, maxvalue-1);
-            *vallen = strlen(val->data.string);
-        }
-        PMIX_VALUE_RELEASE(val);
-    }
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Info_GetNodeAttr(const char name[],
-                                      char value[], int valuelen,
-                                      int *found, int waitfor)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_UNDEF;
-
-    PMI2_CHECK();
-
-    if ((NULL == name) || (NULL == value) || (NULL == found)) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    *found = 0;
-    /* TODO: does PMI2's "name" makes sense to PMIx? */
-    rc = PMIx_Get(&proc, name, info, 1, &val);
-    if (PMIX_SUCCESS == rc && NULL != val) {
-        if (PMIX_STRING != val->type) {
-            rc = PMIX_ERROR;
-        } else if (NULL != val->data.string) {
-            pmix_strncpy(value, val->data.string, valuelen-1);
-            *found = 1;
-        }
-        PMIX_VALUE_RELEASE(val);
-    } else if (PMIX_ERR_NOT_FOUND == rc) {
-        rc = PMIX_SUCCESS;
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Info_GetNodeAttrIntArray(const char name[], int array[],
-                                  int arraylen, int *outlen, int *found)
-{
-    return PMI2_FAIL;
-}
-
-/* push info at the PMIX_LOCAL scope */
-PMIX_EXPORT int PMI2_Info_PutNodeAttr(const char name[], const char value[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t val;
-
-    PMI2_CHECK();
-
-    if ((NULL == name) || (NULL == value)) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_SUCCESS;
-    }
-
-    val.type = PMIX_STRING;
-    val.data.string = (char*)value;
-    rc = PMIx_Put(PMIX_LOCAL, name, &val);
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Info_GetJobAttr(const char name[], char value[], int valuelen, int *found)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    pmix_value_t *val;
-    pmix_info_t info[1];
-    bool  val_optinal = 1;
-    pmix_proc_t proc = myproc;
-    proc.rank = PMIX_RANK_UNDEF;
-
-    PMI2_CHECK();
-
-    if ((NULL == name) || (NULL == value) || (NULL == found)) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    /* set controlling parameters
-     * PMIX_OPTIONAL - expect that these keys should be available on startup
-     */
-    PMIX_INFO_CONSTRUCT(&info[0]);
-    PMIX_INFO_LOAD(&info[0], PMIX_OPTIONAL, &val_optinal, PMIX_BOOL);
-
-    /* PMI-2 expects resource manager to set
-     * process mapping in ANL notation. */
-    if (!strcmp(name, ANL_MAPPING)) {
-        /* we are looking in the job-data. If there is nothing there
-         * we don't want to look in rank's data, thus set rank to widcard */
-        proc = myproc;
-        proc.rank = PMIX_RANK_WILDCARD;
-        if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) &&
-               (NULL != val) && (PMIX_STRING == val->type)) {
-            pmix_strncpy(value, val->data.string, valuelen);
-            PMIX_VALUE_FREE(val, 1);
-            *found = 1;
-            return PMI2_SUCCESS;
-        } else {
-            /* artpol:
-             * Some RM's (i.e. SLURM) already have ANL precomputed. The export it
-             * through PMIX_ANL_MAP variable.
-             * If we haven't found it we want to have our own packing functionality
-             * since it's common.
-             * Somebody else has to write it since I've already done that for
-             * GPL'ed SLURM :) */
-            *found = 1;
-            return PMI2_FAIL;
-        }
-    }
-
-
-    *found = 0;
-    rc = PMIx_Get(&proc, name, info, 1, &val);
-    if (PMIX_SUCCESS == rc && NULL != val) {
-        if (PMIX_STRING != val->type) {
-            rc = PMIX_ERROR;
-        } else if (NULL != val->data.string) {
-            pmix_strncpy(value, val->data.string, valuelen-1);
-            *found = 1;
-        }
-        PMIX_VALUE_RELEASE(val);
-    } else if (PMIX_ERR_NOT_FOUND == rc) {
-        rc = PMIX_SUCCESS;
-    }
-
-    PMIX_INFO_DESTRUCT(&info[0]);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Info_GetJobAttrIntArray(const char name[],
-                                             int array[], int arraylen,
-                                             int *outlen, int *found)
-{
-    return PMI2_FAIL;
-}
-
-PMIX_EXPORT int PMI2_Nameserv_publish(const char service_name[],
-                                      const PMI_keyval_t *info_ptr, const char port[])
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    int nvals;
-    pmix_info_t info[2];
-
-    PMI2_CHECK();
-
-    if (NULL == service_name || NULL == port) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    /* pass the service/port */
-    pmix_strncpy(info[0].key, service_name, PMIX_MAX_KEYLEN);
-    info[0].value.type = PMIX_STRING;
-    info[0].value.data.string = (char*)port;
-    nvals = 1;
-
-    /* if provided, add any other value */
-    if (NULL != info_ptr) {
-        pmix_strncpy(info[1].key, info_ptr->key, PMIX_MAX_KEYLEN);
-        info[1].value.type = PMIX_STRING;
-        info[1].value.data.string = (char*)info_ptr->val;
-        nvals = 2;
-    }
-    /* publish the info - PMI-2 doesn't support
-     * any scope other than inside our own nspace */
-    rc = PMIx_Publish(info, nvals);
-
-    return convert_err(rc);
-}
-
-PMIX_EXPORT int PMI2_Nameserv_lookup(const char service_name[],
-                                     const PMI_keyval_t *info_ptr,
-                                     char port[], int portLen)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    int nvals;
-    pmix_pdata_t pdata[2];
-
-    PMI2_CHECK();
-
-    if (NULL == service_name || NULL == info_ptr || NULL == port) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    PMIX_PDATA_CONSTRUCT(&pdata[0]);
-    PMIX_PDATA_CONSTRUCT(&pdata[1]);
-
-    /* pass the service */
-    pmix_strncpy(pdata[0].key, service_name, PMIX_MAX_KEYLEN);
-    nvals = 1;
-
-    /* if provided, add any other value */
-    if (NULL != info_ptr) {
-        pmix_strncpy(pdata[1].key, info_ptr->key, PMIX_MAX_KEYLEN);
-        pdata[1].value.type = PMIX_STRING;
-        pdata[1].value.data.string = info_ptr->val;
-        nvals = 2;
-    }
-
-    /* lookup the info */
-    if (PMIX_SUCCESS != (rc = PMIx_Lookup(pdata, nvals, NULL, 0))) {
-        PMIX_PDATA_DESTRUCT(&pdata[0]);
-        PMIX_PDATA_DESTRUCT(&pdata[1]);
-        return convert_err(rc);
-    }
-
-    /* should have received a string back */
-    if (PMIX_STRING != pdata[0].value.type ||
-        NULL == pdata[0].value.data.string) {
-        PMIX_PDATA_DESTRUCT(&pdata[0]);
-        PMIX_PDATA_DESTRUCT(&pdata[1]);
-        return PMI2_FAIL;
-    }
-
-    /* return the port */
-    pmix_strncpy(port, pdata[0].value.data.string, portLen-1);
-    PMIX_PDATA_DESTRUCT(&pdata[0]);
-
-    if (NULL != info_ptr) {
-    }
-    PMIX_PDATA_DESTRUCT(&pdata[1]);
-
-    return PMI2_SUCCESS;
-}
-
-PMIX_EXPORT int PMI2_Nameserv_unpublish(const char service_name[],
-                           const PMI_keyval_t *info_ptr)
-{
-    pmix_status_t rc = PMIX_SUCCESS;
-    char *keys[3];
-
-    PMI2_CHECK();
-
-    if (NULL == service_name || NULL == info_ptr) {
-        return PMI2_ERR_INVALID_ARG;
-    }
-
-    if (pmi2_singleton) {
-        return PMI2_FAIL;
-    }
-
-    /* pass the service */
-    keys[0] = (char*)service_name;
-    keys[1] = NULL;
-    keys[2] = NULL;
-
-    /* if provided, add any other value */
-    if (NULL != info_ptr) {
-        keys[1] = info_ptr->key;
-    }
-
-    rc = PMIx_Unpublish(keys, NULL, 0);
-    return convert_err(rc);
-}
-
-/****    CONVERSION ROUTINES    ****/
-static pmix_status_t convert_int(int *value, pmix_value_t *kv)
-{
-    switch(kv->type) {
-    case PMIX_INT:
-        *value = kv->data.integer;
-        break;
-    case PMIX_INT8:
-        *value = kv->data.int8;
-        break;
-    case PMIX_INT16:
-        *value = kv->data.int16;
-        break;
-    case PMIX_INT32:
-        *value = kv->data.int32;
-        break;
-    case PMIX_INT64:
-        *value = kv->data.int64;
-        break;
-    case PMIX_UINT:
-        *value = kv->data.uint;
-        break;
-    case PMIX_UINT8:
-        *value = kv->data.uint8;
-        break;
-    case PMIX_UINT16:
-        *value = kv->data.uint16;
-        break;
-    case PMIX_UINT32:
-        *value = kv->data.uint32;
-        break;
-    case PMIX_UINT64:
-        *value = kv->data.uint64;
-        break;
-    case PMIX_BYTE:
-        *value = kv->data.byte;
-        break;
-    case PMIX_SIZE:
-        *value = kv->data.size;
-        break;
-    case PMIX_BOOL:
-        *value = kv->data.flag;
-        break;
-    default:
-        /* not an integer type */
-        return PMIX_ERR_BAD_PARAM;
-    }
-    return PMIX_SUCCESS;
-}
-
-static int convert_err(pmix_status_t rc)
-{
-    switch(rc) {
-    case PMIX_ERR_INVALID_SIZE:
-        return PMI2_ERR_INVALID_SIZE;
-
-    case PMIX_ERR_INVALID_KEYVALP:
-        return PMI2_ERR_INVALID_KEYVALP;
-
-    case PMIX_ERR_INVALID_NUM_PARSED:
-        return PMI2_ERR_INVALID_NUM_PARSED;
-
-    case PMIX_ERR_INVALID_ARGS:
-        return PMI2_ERR_INVALID_ARGS;
-
-    case PMIX_ERR_INVALID_NUM_ARGS:
-        return PMI2_ERR_INVALID_NUM_ARGS;
-
-    case PMIX_ERR_INVALID_LENGTH:
-        return PMI2_ERR_INVALID_LENGTH;
-
-    case PMIX_ERR_INVALID_VAL_LENGTH:
-        return PMI2_ERR_INVALID_VAL_LENGTH;
-
-    case PMIX_ERR_INVALID_VAL:
-        return PMI2_ERR_INVALID_VAL;
-
-    case PMIX_ERR_INVALID_KEY_LENGTH:
-        return PMI2_ERR_INVALID_KEY_LENGTH;
-
-    case PMIX_ERR_INVALID_KEY:
-        return PMI2_ERR_INVALID_KEY;
-
-    case PMIX_ERR_INVALID_ARG:
-        return PMI2_ERR_INVALID_ARG;
-
-    case PMIX_ERR_NOMEM:
-        return PMI2_ERR_NOMEM;
-
-    case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER:
-    case PMIX_ERR_LOST_CONNECTION_TO_SERVER:
-    case PMIX_ERR_LOST_PEER_CONNECTION:
-    case PMIX_ERR_LOST_CONNECTION_TO_CLIENT:
-    case PMIX_ERR_NOT_SUPPORTED:
-    case PMIX_ERR_NOT_FOUND:
-    case PMIX_ERR_SERVER_NOT_AVAIL:
-    case PMIX_ERR_INVALID_NAMESPACE:
-    case PMIX_ERR_DATA_VALUE_NOT_FOUND:
-    case PMIX_ERR_OUT_OF_RESOURCE:
-    case PMIX_ERR_RESOURCE_BUSY:
-    case PMIX_ERR_BAD_PARAM:
-    case PMIX_ERR_IN_ERRNO:
-    case PMIX_ERR_UNREACH:
-    case PMIX_ERR_TIMEOUT:
-    case PMIX_ERR_NO_PERMISSIONS:
-    case PMIX_ERR_PACK_MISMATCH:
-    case PMIX_ERR_PACK_FAILURE:
-    case PMIX_ERR_UNPACK_FAILURE:
-    case PMIX_ERR_UNPACK_INADEQUATE_SPACE:
-    case PMIX_ERR_TYPE_MISMATCH:
-    case PMIX_ERR_PROC_ENTRY_NOT_FOUND:
-    case PMIX_ERR_UNKNOWN_DATA_TYPE:
-    case PMIX_ERR_WOULD_BLOCK:
-    case PMIX_EXISTS:
-    case PMIX_ERROR:
-        return PMI2_FAIL;
-
-    case PMIX_ERR_INIT:
-        return PMI2_ERR_INIT;
-
-    case PMIX_SUCCESS:
-        return PMI2_SUCCESS;
-    default:
-        return PMI2_FAIL;
-    }
-}
diff -Nru pmix-3.2.2~rc1/src/client/pmix_client.c pmix-4.0.0/src/client/pmix_client.c
--- pmix-3.2.2~rc1/src/client/pmix_client.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/client/pmix_client.c	2021-01-02 08:56:17.000000000 +0000
@@ -68,6 +68,7 @@
 #include "src/mca/preg/preg.h"
 #include "src/mca/ptl/base/base.h"
 #include "src/include/pmix_globals.h"
+#include "src/common/pmix_attributes.h"
 #include "src/common/pmix_iof.h"
 
 #include "pmix_client_ops.h"
@@ -521,7 +522,7 @@
     pmix_cb_t cb;
     pmix_buffer_t *req;
     pmix_cmd_t cmd = PMIX_REQ_CMD;
-    pmix_status_t code = PMIX_ERR_DEBUGGER_RELEASE;
+    pmix_status_t code;
     pmix_proc_t wildcard;
     pmix_info_t ginfo, evinfo[2];
     pmix_value_t *val = NULL;
@@ -555,6 +556,23 @@
     }
     ++pmix_globals.init_cntr;
 
+    /* backward compatibility fix - remove any directive to use
+     * the old usock component so we avoid a warning message */
+    if (NULL != (evar = getenv("PMIX_MCA_ptl"))) {
+        if (0 == strcmp(evar, "usock")) {
+            /* we cannot support a usock-only environment */
+            PMIX_RELEASE_THREAD(&pmix_global_lock);
+            fprintf(stderr, "-------------------------------------------------------------------\n");
+            fprintf(stderr, "PMIx no longer supports the \"usock\" transport for client-server\n");
+            fprintf(stderr, "communication. A directive was detected that only allows that mode.\n");
+            fprintf(stderr, "We cannot continue - please remove that constraint and try again.\n");
+            fprintf(stderr, "-------------------------------------------------------------------\n");
+            return PMIX_ERR_INIT;
+        }
+        /* anything else should just be cleared */
+        pmix_unsetenv("PMIX_MCA_ptl", &environ);
+    }
+
     /* setup the runtime - this init's the globals,
      * opens and initializes the required frameworks */
     if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_CLIENT, info, ninfo,
@@ -569,7 +587,7 @@
     rcv->tag = PMIX_PTL_TAG_IOF;
     rcv->cbfunc = client_iof_handler;
     /* add it to the end of the list of recvs */
-    pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super);
+    pmix_list_append(&pmix_ptl_base.posted_recvs, &rcv->super);
 
 
     /* setup the globals */
@@ -724,7 +742,6 @@
     PMIX_INFO_DESTRUCT(&ginfo);
 
     if (pmix_client_globals.singleton) {
-        pmix_globals.mypeer->nptr->compat.ptl = pmix_ptl_base_assign_module();
         pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL);
         pmix_client_globals.myserver->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL);
         /* initialize our data values */
@@ -737,14 +754,12 @@
         rc = PMIX_ERR_UNREACH;
     } else {
         /* connect to the server */
-        rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo);
+        rc = pmix_ptl.connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo);
         if (PMIX_SUCCESS != rc) {
             pmix_init_result = rc;
             PMIX_RELEASE_THREAD(&pmix_global_lock);
             return rc;
         }
-        /* mark that we are using the same module as used for the server */
-        pmix_globals.mypeer->nptr->compat.ptl = pmix_client_globals.myserver->nptr->compat.ptl;
 
         /* send a request for our job info - we do this as a non-blocking
          * transaction because some systems cannot handle very large
@@ -798,6 +813,7 @@
         pmix_output_verbose(2, pmix_client_globals.event_output,
                             "[%s:%d] WAITING IN INIT FOR DEBUGGER",
                             pmix_globals.myid.nspace, pmix_globals.myid.rank);
+        code = PMIX_ERR_DEBUGGER_RELEASE;
         PMIx_Register_event_handler(&code, 1, evinfo, 2,
                                     notification_fn, evhandler_reg_callbk, (void*)&reglock);
         /* wait for registration to complete */
@@ -847,6 +863,8 @@
         PMIX_RELEASE(kptr); // maintain accounting
     }
 
+    /* register the client supported attrs */
+    rc = pmix_register_client_attrs();
     return rc;
 }
 
@@ -986,13 +1004,11 @@
                              pmix_globals.myid.nspace, pmix_globals.myid.rank);
     }
 
-    if (!pmix_globals.external_evbase) {
-        /* stop the progress thread, but leave the event base
-         * still constructed. This will allow us to safely
-         * tear down the infrastructure, including removal
-         * of any events objects may be holding */
-        (void)pmix_progress_thread_pause(NULL);
-    }
+    /* stop the progress thread, but leave the event base
+     * still constructed. This will allow us to safely
+     * tear down the infrastructure, including removal
+     * of any events objects may be holding */
+    (void)pmix_progress_thread_pause(NULL);
 
     PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests);
     for (i=0; i < pmix_client_globals.peers.size; i++) {
@@ -1406,11 +1422,11 @@
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    /* if I am a client and my server is earlier than v3.1.5, then
+    /* if I am a client and my server is earlier than v3.2.x, then
      * I need to look for this data under rank=PMIX_RANK_WILDCARD
      * with a key equal to the nodename */
     if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer) &&
-        PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 5)) {
+        PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 100)) {
         proc.rank = PMIX_RANK_WILDCARD;
         iptr = NULL;
         ninfo = 0;
@@ -1602,7 +1618,7 @@
             /* add to our list of results, ensuring uniqueness */
             p = pmix_argv_split(val->data.string, ',');
             for (n=0; NULL != p[n]; n++) {
-                pmix_argv_append_unique_nosize(&tmp, p[n], true);
+                pmix_argv_append_unique_nosize(&tmp, p[n]);
             }
             pmix_argv_free(p);
             PMIX_VALUE_FREE(val, 1);
diff -Nru pmix-3.2.2~rc1/src/client/pmix_client_fabric.c pmix-4.0.0/src/client/pmix_client_fabric.c
--- pmix-3.2.2~rc1/src/client/pmix_client_fabric.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/client/pmix_client_fabric.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,481 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include "src/include/types.h"
+#include "src/include/pmix_stdint.h"
+#include "src/include/pmix_socket_errno.h"
+
+#include "src/include/pmix_globals.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#include <fcntl.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_UN_H
+#include <sys/un.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#include PMIX_EVENT_HEADER
+
+#include "src/class/pmix_list.h"
+#include "src/mca/ploc/ploc.h"
+#include "src/mca/pnet/base/base.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/util/pmix_environ.h"
+#include "src/client/pmix_client_ops.h"
+
+static void fcb(pmix_status_t status,
+                pmix_info_t *info, size_t ninfo,
+                void *cbdata,
+                pmix_release_cbfunc_t release_fn,
+                void *release_cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    size_t n;
+
+    cb->status = status;
+    if (PMIX_SUCCESS == status && 0 < ninfo) {
+        PMIX_INFO_CREATE(cb->fabric->info, ninfo);
+        cb->fabric->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&cb->fabric->info[n], &info[n]);
+        }
+    }
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    /* release the caller */
+    if (NULL != cb->cbfunc.opfn) {
+        cb->cbfunc.opfn(status, cb->cbdata);
+        PMIX_RELEASE(cb);
+    } else {
+        PMIX_WAKEUP_THREAD(&cb->lock);
+    }
+}
+
+static void frecv(struct pmix_peer_t *peer,
+                  pmix_ptl_hdr_t *hdr,
+                  pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    pmix_status_t rc;
+    int cnt;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric recv from server with %d bytes",
+                        (int)buf->bytes_used);
+
+    /* a zero-byte buffer indicates that this recv is being
+     * completed due to a lost connection */
+    if (PMIX_BUFFER_IS_EMPTY(buf)) {
+        rc = PMIX_ERR_UNREACH;
+        goto complete;
+    }
+
+    /* unpack the status */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cb->status, &cnt, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (PMIX_SUCCESS != cb->status) {
+        goto complete;
+    }
+
+    /* unpack any returned data */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cb->fabric->ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc && PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < cb->fabric->ninfo) {
+        PMIX_INFO_CREATE(cb->fabric->info, cb->fabric->ninfo);
+        cnt = cb->fabric->ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, cb->fabric->info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric recv from server releasing");
+    /* release the caller */
+    if (NULL != cb->cbfunc.opfn) {
+        cb->cbfunc.opfn(rc, cb->cbdata);
+        PMIX_RELEASE(cb);
+    } else {
+        PMIX_WAKEUP_THREAD(&cb->lock);
+    }
+}
+
+
+static void mycbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+    cb->status = status;
+    PMIX_POST_OBJECT(cb);
+    PMIX_WAKEUP_THREAD(&cb->lock);
+    return;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_register(pmix_fabric_t *fabric,
+                                               const pmix_info_t directives[],
+                                               size_t ndirs)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric register");
+
+    /* create a callback object so we can be notified when
+     * the non-blocking operation is complete */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.fabric = fabric;
+    rc = PMIx_Fabric_register_nb(fabric, directives, ndirs, mycbfunc, &cb);
+    if (PMIX_OPERATION_SUCCEEDED == rc) {
+        PMIX_DESTRUCT(&cb);
+        return PMIX_SUCCESS;
+    } else if (PMIX_SUCCESS != rc) {
+        /* got an error */
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the data to return */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    /* the fabric info was directly filled into the fabric object */
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric register completed");
+
+    return rc;
+}
+
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_register_nb(pmix_fabric_t *fabric,
+                                                  const pmix_info_t directives[],
+                                                  size_t ndirs,
+                                                  pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_cb_t *cb;
+    pmix_status_t rc;
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_FABRIC_REGISTER_CMD;
+
+    /* if I am a scheduler server, then I should be able
+     * to support this myself */
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+        /* see if we can do it ourselves */
+        rc = pmix_pnet.register_fabric(fabric, directives, ndirs, cbfunc, cbdata);
+        if (PMIX_OPERATION_SUCCEEDED == rc) {
+            /* did it atomically */
+            return rc;
+        } else if (PMIX_SUCCESS == rc) {
+            /* in progress - the pnet component will callback
+             * when it is complete */
+            return rc;
+        }
+        /* otherwise, see if we can pass
+         * it up to our host so they can send it to the scheduler */
+        if (NULL == pmix_host_server.fabric) {
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+        if (NULL != cbfunc) {
+            cb = PMIX_NEW(pmix_cb_t);
+            cb->fabric = fabric;
+            cb->cbfunc.opfn = cbfunc;
+            cb->cbdata = cbdata;
+        } else {
+            cb = (pmix_cb_t*)cbdata;
+        }
+        rc = pmix_host_server.fabric(&pmix_globals.myid, PMIX_FABRIC_REQUEST_INFO,
+                                     directives, ndirs,
+                                     fcb, (void*)cb);
+        if (PMIX_SUCCESS != rc && NULL != cbfunc) {
+            PMIX_RELEASE(cb);
+        }
+        return rc;
+    }
+
+    /* finally, if I am a tool or client, then I need to send it to
+     * a daemon for processing */
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if we are a client, then relay this request to the server */
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* pack the directives */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ndirs, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    if (NULL != directives && 0 < ndirs) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, directives, ndirs, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            return rc;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    if (NULL != cbfunc) {
+        cb = PMIX_NEW(pmix_cb_t);
+        cb->fabric = fabric;
+        cb->cbfunc.opfn = cbfunc;
+        cb->cbdata = cbdata;
+    } else {
+        cb = (pmix_cb_t*)cbdata;
+    }
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, frecv, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        if (NULL != cbfunc) {
+            PMIX_RELEASE(cb);
+        }
+    }
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_update(pmix_fabric_t *fabric)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric update");
+
+    /* create a callback object so we can be notified when
+     * the non-blocking operation is complete */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.fabric = fabric;
+    if (PMIX_SUCCESS != (rc = PMIx_Fabric_update_nb(fabric, NULL, &cb))) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the data to return */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    /* the fabric info was directly filled into the fabric object */
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric update completed");
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_update_nb(pmix_fabric_t *fabric,
+                                                pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_cb_t *cb;
+    pmix_status_t rc;
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_FABRIC_UPDATE_CMD;
+    pmix_info_t info;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    /* if I am a scheduler server, then I should be able
+     * to support this myself */
+    if (PMIX_PEER_IS_SCHEDULER(pmix_globals.mypeer)) {
+        rc = pmix_pnet.update_fabric(fabric);
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
+    }
+
+    /* otherwise, if we are a server, then see if we can pass
+     * it up to our host so they can send it to the scheduler */
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        if (NULL == pmix_host_server.fabric) {
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+        if (NULL != cbfunc) {
+            cb = PMIX_NEW(pmix_cb_t);
+            cb->fabric = fabric;
+            cb->cbfunc.opfn = cbfunc;
+            cb->cbdata = cbdata;
+        } else {
+            cb = (pmix_cb_t*)cbdata;
+        }
+        cb->infocopy = true;
+        PMIX_INFO_CREATE(cb->info, 1);
+        cb->ninfo = 1;
+        PMIX_INFO_LOAD(&cb->info[0], PMIX_FABRIC_INDEX, &fabric->index, PMIX_SIZE);
+        rc = pmix_host_server.fabric(&pmix_globals.myid, PMIX_FABRIC_UPDATE_INFO,
+                                     cb->info, 1, fcb, (void*)cb);
+        if (PMIX_SUCCESS != rc && NULL != cbfunc) {
+            PMIX_INFO_DESTRUCT(&info);
+            PMIX_RELEASE(cb);
+        }
+        return rc;
+    }
+
+    /* finally, if I am a tool or client, then I need to send it to
+     * a daemon for processing */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if we are a client, then relay this request to the server */
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+    /* pack the fabric index */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &fabric->index, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        return rc;
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    if (NULL != cbfunc) {
+        cb = PMIX_NEW(pmix_cb_t);
+        cb->fabric = fabric;
+        cb->cbfunc.opfn = cbfunc;
+        cb->cbdata = cbdata;
+    } else {
+        cb = (pmix_cb_t*)cbdata;
+    }
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, frecv, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        if (NULL != cbfunc) {
+            PMIX_RELEASE(cb);
+        }
+    }
+    return rc;
+}
+
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_deregister(pmix_fabric_t *fabric)
+{
+    pmix_status_t rc;
+
+    rc = PMIx_Fabric_deregister_nb(fabric, NULL, NULL);
+    if (PMIX_OPERATION_SUCCEEDED == rc) {
+        rc = PMIX_SUCCESS;
+    }
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Fabric_deregister_nb(pmix_fabric_t *fabric,
+                                                    pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    /* if I am a scheduler server, then I should be able
+     * to support this myself */
+    if (PMIX_PEER_IS_SCHEDULER(pmix_globals.mypeer)) {
+        rc = pmix_pnet.deregister_fabric(fabric);
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        if (PMIX_SUCCESS == rc) {
+            rc = PMIX_OPERATION_SUCCEEDED;
+        }
+        return rc;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* otherwise, just remove any storage in it */
+    if (NULL != fabric->info) {
+        PMIX_INFO_FREE(fabric->info, fabric->ninfo);
+    }
+
+    return PMIX_OPERATION_SUCCEEDED;
+}
diff -Nru pmix-3.2.2~rc1/src/client/pmix_client_get.c pmix-4.0.0/src/client/pmix_client_get.c
--- pmix-3.2.2~rc1/src/client/pmix_client_get.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/client/pmix_client_get.c	2021-01-02 08:56:17.000000000 +0000
@@ -123,6 +123,14 @@
     return rc;
 }
 
+static void gcbfn(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+
+    cb->cbfunc.valuefn(cb->status, cb->value, cb->cbdata);
+    PMIX_RELEASE(cb);
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const pmix_key_t key,
                                       const pmix_info_t info[], size_t ninfo,
                                       pmix_value_cbfunc_t cbfunc, void *cbdata)
@@ -170,6 +178,21 @@
         return PMIX_ERR_BAD_PARAM;
     }
 
+    /* see if they just want their own process ID */
+    if (NULL == proc && 0 == strncmp(key, PMIX_PROCID, PMIX_MAX_KEYLEN)) {
+        PMIX_VALUE_CREATE(ival, 1);
+        ival->type = PMIX_PROC;
+        ival->data.proc = (pmix_proc_t*)malloc(sizeof(pmix_proc_t));
+        PMIX_LOAD_PROCID(ival->data.proc, pmix_globals.myid.nspace, pmix_globals.myid.rank);
+        cb = PMIX_NEW(pmix_cb_t);
+        cb->status = PMIX_SUCCESS;
+        cb->value = ival;
+        cb->cbfunc.valuefn = cbfunc;
+        cb->cbdata = cbdata;
+        PMIX_THREADSHIFT(cb, gcbfn);
+        return PMIX_SUCCESS;
+    }
+
     /* if the key is NULL, the rank cannot be WILDCARD as
      * we cannot return all info from every rank */
     if (NULL != proc && PMIX_RANK_WILDCARD == proc->rank && NULL == key) {
@@ -201,7 +224,7 @@
                         "pmix: get_nb value for proc %s key %s",
                         PMIX_NAME_PRINT(&p), (NULL == key) ? "NULL" : key);
 
-    if (!PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 5)) {
+    if (!PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 100)) {
         /* don't consider the fastpath option
          * for undefined rank or NULL keys */
         if (PMIX_RANK_UNDEF == p.rank || NULL == key) {
@@ -211,20 +234,30 @@
          * for PMIX_RANK, then they are asking for our process rank */
         if (PMIX_RANK_INVALID == p.rank &&
             PMIX_CHECK_NSPACE(p.nspace, pmix_globals.myid.nspace) &&
-            NULL != key && 0 == strcmp(key, PMIX_RANK)) {
+            NULL != key && 0 == strncmp(key, PMIX_RANK, PMIX_MAX_KEYLEN)) {
             PMIX_VALUE_CREATE(ival, 1);
             if (NULL == ival) {
                 return PMIX_ERR_NOMEM;
             }
             ival->type = PMIX_PROC_RANK;
             ival->data.rank = pmix_globals.myid.rank;
-            cbfunc(PMIX_SUCCESS, ival, cbdata);
-            /* ownership of the memory in ival is passed to the
-             * user in the cbfunc, so don't release it here */
+            /* threadshift to return the result - cannot call the
+             * cbfunc from within the API */
+            cb = PMIX_NEW(pmix_cb_t);
+            cb->status = PMIX_SUCCESS;
+            cb->value = ival;
+            cb->cbfunc.valuefn = cbfunc;
+            cb->cbdata = cbdata;
+            PMIX_THREADSHIFT(cb, gcbfn);
             return PMIX_SUCCESS;
         }
         /* see if they are asking about a node-level piece of info */
         if (pmix_check_node_info(key)) {
+
+            pmix_output_verbose(2, pmix_client_globals.get_output,
+                                "pmix: get_nb value requesting node-level info for proc %s key %s",
+                                PMIX_NAME_PRINT(&p), (NULL == key) ? "NULL" : key);
+
             /* the key is node-related - see if the target node is in the
              * info array and if they tagged the request accordingly */
             if (NULL != info) {
@@ -242,24 +275,22 @@
                     }
                 }
             }
-            if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 2, PMIX_RELEASE_WILDCARD)) {
-                p.rank = PMIX_RANK_UNDEF;
-               /* see if they told us to get node info */
-                if (!wantinfo) {
-                    /* guess not - better do it */
-                    nfo = ninfo + 1;
-                    PMIX_INFO_CREATE(iptr, nfo);
-                    for (n=0; n < ninfo; n++) {
-                        PMIX_INFO_XFER(&iptr[n], &info[n]);
-                    }
-                    PMIX_INFO_LOAD(&iptr[ninfo], PMIX_NODE_INFO, NULL, PMIX_BOOL);
-                    copy = true;
-                    p.rank = PMIX_RANK_UNDEF;
-                    goto doget;
+           /* see if they told us to get node info */
+            if (!wantinfo) {
+                pmix_output_verbose(2, pmix_client_globals.get_output,
+                                    "pmix: get_nb value did not specify node info for proc %s key %s",
+                                    PMIX_NAME_PRINT(&p), (NULL == key) ? "NULL" : key);
+                /* guess not - better do it */
+                nfo = ninfo + 1;
+                PMIX_INFO_CREATE(iptr, nfo);
+                for (n=0; n < ninfo; n++) {
+                    PMIX_INFO_XFER(&iptr[n], &info[n]);
                 }
+                PMIX_INFO_LOAD(&iptr[ninfo], PMIX_NODE_INFO, NULL, PMIX_BOOL);
+                copy = true;
                 goto doget;
             }
-            if (wantinfo && (NULL != hostname || UINT32_MAX != nodeid)) {
+            if (NULL != hostname || UINT32_MAX != nodeid) {
                 /* they provided the "node-info" attribute. if they also
                  * specified the target node and it is NOT us, then dstore cannot
                  * resolve it and we need the rank to be undefined */
@@ -267,12 +298,7 @@
                     nodeid == pmix_globals.nodeid) {
                     goto fastpath;
                 }
-                p.rank = PMIX_RANK_UNDEF;
                 goto doget;
-            } else if (wantinfo) {
-                /* they provided "node-info" but are missing the nodeid/hostname - assume
-                 * they are asking for info about our node. The dstore would have that */
-                goto fastpath;
             } else if (NULL != hostname) {
                 /* they did not provide the "node-info" attribute but did specify
                  * a hostname - if the ID is other than us, then we just need to
@@ -288,7 +314,6 @@
                 }
                 PMIX_INFO_LOAD(&iptr[ninfo], PMIX_NODE_INFO, NULL, PMIX_BOOL);
                 copy = true;
-                p.rank = PMIX_RANK_UNDEF;
                 goto doget;
             } else if (UINT32_MAX != nodeid) {
                 /* they did not provide the "node-info" attribute but did specify
@@ -305,7 +330,6 @@
                 }
                 PMIX_INFO_LOAD(&iptr[ninfo], PMIX_NODE_INFO, NULL, PMIX_BOOL);
                 copy = true;
-                p.rank = PMIX_RANK_UNDEF;
                 goto doget;
             } else {
                 /* nothing was given, so assume this is about our node and
@@ -333,8 +357,7 @@
                     }
                 }
             }
-            if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 2, PMIX_RELEASE_WILDCARD)) {
-                p.rank = PMIX_RANK_UNDEF;
+            if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 2, 100)) {
                /* see if they told us to get app info */
                 if (!wantinfo) {
                     /* guess not - better do it */
@@ -345,7 +368,6 @@
                     }
                     PMIX_INFO_LOAD(&iptr[ninfo], PMIX_APP_INFO, NULL, PMIX_BOOL);
                     copy = true;
-                    p.rank = PMIX_RANK_UNDEF;
                     goto doget;
                 }
                 goto doget;
@@ -365,7 +387,6 @@
                         goto fastpath;
                     }
                 }
-                p.rank = PMIX_RANK_UNDEF;
                 goto doget;
             } else if (wantinfo) {
                 /* missing the appnum - assume it is ours */
@@ -394,7 +415,6 @@
                 }
                 PMIX_INFO_LOAD(&iptr[ninfo], PMIX_APP_INFO, NULL, PMIX_BOOL);
                 copy = true;
-                p.rank = PMIX_RANK_UNDEF;
                 goto doget;
             } else {
                 /* missing both - all we can do is assume they want our info */
@@ -413,15 +433,26 @@
 
   fastpath:
     /* try to get data directly, without threadshift */
+    pmix_output_verbose(2, pmix_client_globals.get_output,
+                        "pmix: get_nb value trying fastpath for proc %s key %s",
+                        PMIX_NAME_PRINT(&p), (NULL == key) ? "NULL" : key);
     if (PMIX_SUCCESS == (rc = _getfn_fastpath(&p, key, iptr, nfo, &ival))) {
-        cbfunc(rc, ival, cbdata);
-        /* ownership of the memory in ival is passed to the
-         * user in the cbfunc, so don't release it here */
-        return rc;
+        /* threadshift to return the result - cannot execute the cbfunc
+         * from within the API */
+        cb = PMIX_NEW(pmix_cb_t);
+        cb->status = rc;
+        cb->value = ival;
+        cb->cbfunc.valuefn = cbfunc;
+        cb->cbdata = cbdata;
+        PMIX_THREADSHIFT(cb, gcbfn);
+        return PMIX_SUCCESS;
     }
 
   doget:
     /* threadshift this request so we can access global structures */
+    pmix_output_verbose(2, pmix_client_globals.get_output,
+                        "pmix: get_nb value trying slowpath for proc %s key %s",
+                        PMIX_NAME_PRINT(&p), (NULL == key) ? "NULL" : key);
     cb = PMIX_NEW(pmix_cb_t);
     cb->pname.nspace = strdup(p.nspace);
     cb->pname.rank = p.rank;
@@ -549,7 +580,8 @@
     /* a zero-byte buffer indicates that this recv is being
      * completed due to a lost connection */
     if (PMIX_BUFFER_IS_EMPTY(buf)) {
-        ret = PMIX_ERR_UNREACH;
+        pmix_output_verbose(2, pmix_client_globals.get_output,
+                            "pmix: get_nb server lost connection");
         goto done;
     }
 
@@ -565,22 +597,19 @@
     }
 
     if (PMIX_SUCCESS != ret) {
+        pmix_output_verbose(2, pmix_client_globals.get_output,
+                            "pmix: get_nb server returned %s", PMIx_Error_string(ret));
         goto done;
     }
-    if (PMIX_RANK_UNDEF == proc.rank || diffnspace) {
-        PMIX_GDS_ACCEPT_KVS_RESP(rc, pmix_globals.mypeer, buf);
-    } else {
-        PMIX_GDS_ACCEPT_KVS_RESP(rc, pmix_client_globals.myserver, buf);
-    }
-    if (PMIX_SUCCESS != rc) {
-        goto done;
-    }
+    PMIX_GDS_ACCEPT_KVS_RESP(rc, pmix_globals.mypeer, buf);
 
   done:
     /* now search any pending requests (including the one this was in
      * response to) to see if they can be met. Note that this function
      * will only be called if the user requested a specific key - we
      * don't support calls to "get" for a NULL key */
+    pmix_output_verbose(2, pmix_client_globals.get_output,
+                        "pmix: get_nb looking for requested key");
     PMIX_LIST_FOREACH_SAFE(cb, cb2, &pmix_client_globals.pending_requests, pmix_cb_t) {
         if (0 == strncmp(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN) &&
             cb->pname.rank == proc.rank) {
@@ -591,19 +620,14 @@
              * it back to the user, we need a copy of it */
             cb->copy = true;
             if (PMIX_RANK_UNDEF == proc.rank || diffnspace) {
-                if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 5)) {
-                    /* everything is under rank=wildcard */
-                    proc.rank = PMIX_RANK_WILDCARD;
-                }
-                PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb);
-            } else {
-                if (PMIX_RANK_UNDEF == proc.rank &&
-                    PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 5)) {
+                if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 100)) {
                     /* everything is under rank=wildcard */
                     proc.rank = PMIX_RANK_WILDCARD;
                 }
-                PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb);
             }
+            pmix_output_verbose(2, pmix_client_globals.get_output,
+                                "pmix: get_nb searching for key %s for rank %s", cb->key, PMIX_RANK_PRINT(proc.rank));
+            PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb);
             if (PMIX_SUCCESS == rc) {
                 if (1 != pmix_list_get_size(&cb->kvs)) {
                     rc = PMIX_ERR_INVALID_VAL;
@@ -768,6 +792,8 @@
     /* if the key is NULL or starts with "pmix", then they are looking
      * for data that was provided by the server at startup */
     if (!internal_only && (NULL == cb->key || 0 == strncmp(cb->key, "pmix", 4))) {
+        pmix_output_verbose(2, pmix_client_globals.get_output,
+                            "pmix: get_nb looking for job info");
         cb->proc = &proc;
         /* fetch the data from my server's module - since we are passing
          * it back to the user, we need a copy of it */
@@ -792,7 +818,15 @@
                 goto request;
             } else if (NULL != cb->key) {
                 /* => cb->key starts with pmix
-                 * we should have had this info, so respond with the error - if
+                 * if the server is pre-v3.2, then we have to go up and ask
+                 * for the info */
+                if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 100)) {
+                    pmix_output_verbose(5, pmix_client_globals.get_output,
+                                        "pmix:client old server - fetching data");
+                    proc.rank = PMIX_RANK_WILDCARD;
+                    goto request;
+                }
+                /* we should have had this info, so respond with the error - if
                  * they want us to check with the server, they should ask us to
                  * refresh the cache */
                 pmix_output_verbose(5, pmix_client_globals.get_output,
@@ -811,6 +845,8 @@
     } else if (PMIX_RANK_UNDEF == proc.rank) {
         /* the data would have to be stored on our own peer, so
          * we need to go request it */
+        pmix_output_verbose(2, pmix_client_globals.get_output,
+                            "pmix: get_nb UNDEF rank");
         goto request;
     } else {
         /* if the peer and server GDS component are the same, then no
@@ -819,6 +855,8 @@
             val = NULL;
             goto request;
         }
+        pmix_output_verbose(2, pmix_client_globals.get_output,
+                            "pmix: get_nb checking myserver");
         cb->proc = &proc;
         cb->copy = true;
         PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb);
@@ -834,6 +872,8 @@
     }
 
   respond:
+    pmix_output_verbose(2, pmix_client_globals.get_output,
+                        "pmix: get_nb responding with answer %s", PMIx_Error_string(rc));
     /* if a callback was provided, execute it */
     if (NULL != cb->cbfunc.valuefn) {
         if (NULL != val)  {
diff -Nru pmix-3.2.2~rc1/src/client/pmix_client_group.c pmix-4.0.0/src/client/pmix_client_group.c
--- pmix-3.2.2~rc1/src/client/pmix_client_group.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/client/pmix_client_group.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1128 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include "src/include/pmix_stdint.h"
+
+#include "include/pmix.h"
+
+#include "src/include/pmix_globals.h"
+#include "src/mca/gds/base/base.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#include <fcntl.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_UN_H
+#include <sys/un.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#include PMIX_EVENT_HEADER
+
+#include "src/class/pmix_list.h"
+#include "src/mca/bfrops/bfrops.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/threads/threads.h"
+#include "src/mca/gds/gds.h"
+#include "src/mca/ptl/ptl.h"
+
+#include "pmix_client_ops.h"
+
+/* define a tracking object for group operations */
+typedef struct {
+    pmix_object_t super;
+    pmix_lock_t lock;
+    pmix_status_t status;
+    size_t ref;
+    size_t accepted;
+    pmix_proc_t *members;
+    size_t nmembers;
+    pmix_info_t *info;
+    size_t ninfo;
+    pmix_info_t *results;
+    size_t nresults;
+    pmix_op_cbfunc_t opcbfunc;
+    pmix_info_cbfunc_t cbfunc;
+    void *cbdata;
+} pmix_group_tracker_t;
+
+static void gtcon(pmix_group_tracker_t *p)
+{
+    PMIX_CONSTRUCT_LOCK(&p->lock);
+    p->status = PMIX_SUCCESS;
+    p->ref = 0;
+    p->accepted = 0;
+    p->members = NULL;
+    p->nmembers = 0;
+    p->info = NULL;
+    p->ninfo = 0;
+    p->results = NULL;
+    p->nresults = 0;
+    p->cbfunc = NULL;
+    p->opcbfunc = NULL;
+    p->cbdata = NULL;
+}
+static void gtdes(pmix_group_tracker_t *p)
+{
+    PMIX_DESTRUCT_LOCK(&p->lock);
+    if (NULL != p->members) {
+        PMIX_PROC_FREE(p->members, p->nmembers);
+    }
+    if (NULL != p->info) {
+        PMIX_INFO_FREE(p->info, p->ninfo);
+    }
+}
+PMIX_CLASS_INSTANCE(pmix_group_tracker_t,
+                    pmix_object_t,
+                    gtcon, gtdes);
+
+/* callback for wait completion */
+static void grp_cbfunc(struct pmix_peer_t *pr,
+                        pmix_ptl_hdr_t *hdr,
+                        pmix_buffer_t *buf, void *cbdata);
+static void op_cbfunc(pmix_status_t status, void *cbdata);
+
+static void info_cbfunc(pmix_status_t status,
+                        pmix_info_t *info, size_t ninfo,
+                        void *cbdata,
+                        pmix_release_cbfunc_t release_fn,
+                        void *release_cbdata);
+
+
+PMIX_EXPORT pmix_status_t PMIx_Group_construct(const char grp[],
+                                               const pmix_proc_t procs[], size_t nprocs,
+                                               const pmix_info_t info[], size_t ninfo,
+                                               pmix_info_t **results, size_t *nresults)
+{
+    pmix_status_t rc;
+    pmix_group_tracker_t *cb;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix: group_construct called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_group_tracker_t);
+
+    /* push the message into our event base to send to the server */
+    if (PMIX_SUCCESS != (rc = PMIx_Group_construct_nb(grp, procs, nprocs, info, ninfo, info_cbfunc, cb))) {
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+
+    /* wait for the connect to complete */
+    PMIX_WAIT_THREAD(&cb->lock);
+    rc = cb->status;
+    /* user takes responsibility for releasing any results */
+    *results = cb->results;
+    *nresults =  cb->nresults;
+    PMIX_RELEASE(cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix: group construct completed");
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_construct_nb(const char grp[],
+                                                  const pmix_proc_t procs[], size_t nprocs,
+                                                  const pmix_info_t info[], size_t ninfo,
+                                                  pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg = NULL;
+    pmix_cmd_t cmd = PMIX_GROUP_CONSTRUCT_CMD;
+    pmix_status_t rc;
+    pmix_group_tracker_t *cb = NULL;
+    size_t n, num;
+    bool embed = true;
+    pmix_info_t *iptr = NULL;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix:group_construct_nb called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* check for bozo input */
+    if (NULL == procs || 0 >= nprocs) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* need to add the fence request to the provided info
+     * structs as this is a blocking operation - only add
+     * it if the user didn't specify this themselves */
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_CHECK_KEY(&info[n], PMIX_EMBED_BARRIER)) {
+            embed = PMIX_INFO_TRUE(&info[n]);
+            break;
+        }
+    }
+    if (embed) {
+        PMIX_INFO_CREATE(iptr, ninfo + 1);
+        num = ninfo + 1;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&iptr[n], &info[n]);
+        }
+        PMIX_INFO_LOAD(&iptr[ninfo], PMIX_EMBED_BARRIER, &embed, PMIX_BOOL);
+    } else {
+        iptr = (pmix_info_t*)info;
+        num = ninfo;
+    }
+
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the group ID */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &grp, 1, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the number of procs */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &nprocs, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, procs, nprocs, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the info structs */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &num, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        goto done;
+    }
+    if (0 < num) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, iptr, num, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            goto done;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_group_tracker_t);
+    cb->cbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, grp_cbfunc, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cb);
+    }
+
+  done:
+    if (embed && NULL != iptr) {
+        PMIX_INFO_FREE(iptr, num);
+    }
+    if (PMIX_SUCCESS != rc && NULL != msg) {
+        PMIX_RELEASE(msg);
+    }
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_destruct(const char grp[],
+                                              const pmix_info_t info[], size_t ninfo)
+{
+    pmix_status_t rc;
+    pmix_group_tracker_t cb;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix: group_destruct called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_group_tracker_t);
+
+    /* push the message into our event base to send to the server */
+    if (PMIX_SUCCESS != (rc = PMIx_Group_destruct_nb(grp, info, ninfo, op_cbfunc, (void*)&cb))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the connect to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix: group destruct completed");
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_destruct_nb(const char grp[],
+                                                 const pmix_info_t info[], size_t ninfo,
+                                                 pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg = NULL;
+    pmix_cmd_t cmd = PMIX_GROUP_DESTRUCT_CMD;
+    pmix_status_t rc;
+    pmix_group_tracker_t *cb = NULL;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix:group_destruct_nb called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* check for bozo input */
+    if (NULL == grp) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the group ID */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &grp, 1, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the info structs */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        goto done;
+    }
+    if (0 < ninfo) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, info, ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            goto done;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_group_tracker_t);
+    cb->opcbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, grp_cbfunc, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cb);
+    }
+
+  done:
+    if (PMIX_SUCCESS != rc && NULL != msg) {
+        PMIX_RELEASE(msg);
+    }
+    return rc;
+}
+
+static void chaincbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_group_tracker_t *cb = (pmix_group_tracker_t*)cbdata;
+
+    if (NULL != cb) {
+        PMIX_RELEASE(cb);
+    }
+}
+
+static void relcbfunc(void *cbdata)
+{
+    pmix_group_tracker_t *cb = (pmix_group_tracker_t*)cbdata;
+
+    PMIX_RELEASE(cb);
+}
+
+static void invite_handler(size_t evhdlr_registration_id,
+                           pmix_status_t status,
+                           const pmix_proc_t *source,
+                           pmix_info_t info[], size_t ninfo,
+                           pmix_info_t *results, size_t nresults,
+                           pmix_event_notification_cbfunc_fn_t cbfunc,
+                           void *cbdata)
+{
+    pmix_group_tracker_t *cb = NULL;
+    pmix_proc_t *affected = NULL;
+    size_t n;
+    pmix_status_t rc = PMIX_GROUP_INVITE_DECLINED;
+    size_t contextid = SIZE_MAX;
+
+    /* find the object we asked to be returned with event */
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_RETURN_OBJECT)) {
+            if (PMIX_POINTER != info[n].value.type) {
+                /* this is an unrecoverable error - need to abort */
+            }
+            cb = (pmix_group_tracker_t*)info[n].value.data.ptr;
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_AFFECTED_PROC)) {
+            if (PMIX_PROC != info[n].value.type) {
+                /* this is an unrecoverable error - need to abort */
+            }
+            affected = info[n].value.data.proc;
+        }  else if (PMIX_CHECK_KEY(&info[n], PMIX_GROUP_CONTEXT_ID)) {
+            PMIX_VALUE_GET_NUMBER(rc, &info[n].value, contextid, size_t);
+        }
+    }
+    if (NULL == cb) {
+        pmix_output(0, "[%s:%d] INVITE HANDLER NULL OBJECT", pmix_globals.myid.nspace, pmix_globals.myid.rank);
+        /* this is an unrecoverable error - need to abort */
+        /* always must continue the chain */
+        cbfunc(rc, NULL, 0, chaincbfunc, NULL, cbdata);
+        return;
+    }
+
+    /* if the status is accepted, then record it */
+    if (PMIX_GROUP_INVITE_ACCEPTED == status) {
+        cb->accepted++;
+        /* allow the chain to continue */
+        rc = PMIX_SUCCESS;
+        goto complete;
+    }
+
+    /* if the reporting process terminated, then issue the corresponding
+     * group event - it only goes to this process */
+    if (PMIX_PROC_TERMINATED == status) {
+        cb->ninfo = 2;
+        PMIX_INFO_CREATE(cb->info, cb->ninfo);
+        PMIX_INFO_LOAD(&cb->info[0], PMIX_EVENT_AFFECTED_PROC, affected, PMIX_PROC);
+        PMIX_INFO_LOAD(&cb->info[1], PMIX_GROUP_CONTEXT_ID, &contextid, PMIX_SIZE);
+        rc = PMIx_Notify_event(PMIX_GROUP_INVITE_FAILED,
+                               &pmix_globals.myid,
+                               PMIX_RANGE_PROC_LOCAL,
+                               cb->info, cb->ninfo,
+                               chaincbfunc, (void*)cb);
+        if (PMIX_SUCCESS != rc) {
+            /* this is an unrecoverable error - need to abort */
+            pmix_output(0, "[%s:%d] INVITE HANDLER ERROR", pmix_globals.myid.nspace, pmix_globals.myid.rank);
+        }
+        PMIX_INFO_FREE(cb->info, cb->ninfo);
+        goto complete;
+    }
+
+    /* otherwise, we consider the process as having "declined" and
+     * ignore it here - if the user registered a handler for that
+     * case, then the chain will eventually call it
+     */
+
+  complete:
+    /* check for complete */
+    if (cb->accepted == cb->nmembers) {
+        /* execute the completion callback */
+        if (NULL != cb->cbfunc) {
+            cb->cbfunc(PMIX_SUCCESS, NULL, 0, cb->cbdata, relcbfunc, cb);
+            rc = PMIX_EVENT_ACTION_COMPLETE;
+        }
+    }
+
+    /* always must continue the chain */
+    cbfunc(rc, cb->results, cb->nresults, chaincbfunc, cb, cbdata);
+    return;
+}
+
+static void regcbfunc(pmix_status_t status,
+                      size_t refid,
+                      void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+
+    cb->status = status;
+    cb->errhandler_ref = refid;
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_invite(const char grp[],
+                                            const pmix_proc_t procs[], size_t nprocs,
+                                            const pmix_info_t info[], size_t ninfo,
+                                            pmix_info_t **results, size_t *nresults)
+{
+    pmix_group_tracker_t cb;
+    pmix_status_t rc;
+    size_t n;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, then we cannot notify */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* check for bozo input */
+    if (NULL == grp || NULL == procs) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_CONSTRUCT(&cb, pmix_group_tracker_t);
+
+    rc = PMIx_Group_invite_nb(grp, procs, nprocs, info, ninfo, info_cbfunc, (void*)&cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the group construction to complete or fail */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    *results = cb.results;
+    *nresults = cb.nresults;
+    PMIX_DESTRUCT(&cb);
+
+    /* announce group completion, including list of final
+     * members - only sent to members. Servers will intercept
+     * and update their membership lists */
+    PMIX_CONSTRUCT(&cb, pmix_group_tracker_t);
+    PMIX_INFO_CREATE(cb.info, 3);
+    if (NULL == cb.info) {
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_NOMEM;
+    }
+    cb.ninfo = 3;
+    n = 0;
+    (void)strncpy(cb.info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN);
+    cb.info[n].value.type = PMIX_DATA_ARRAY;
+    PMIX_DATA_ARRAY_CREATE(cb.info[n].value.data.darray, nprocs, PMIX_PROC);
+    if (NULL == cb.info[n].value.data.darray ||
+        NULL == cb.info[n].value.data.darray->array) {
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_NOMEM;
+    }
+    memcpy(cb.info[n].value.data.darray->array, procs, nprocs * sizeof(pmix_proc_t));
+    ++n;
+    /* mark that this only goes to non-default handlers */
+    PMIX_INFO_LOAD(&cb.info[n], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    ++n;
+    /* provide the group ID */
+    PMIX_INFO_LOAD(&cb.info[n], PMIX_GROUP_ID, grp, PMIX_STRING);
+
+    /* notify members */
+    rc = PMIx_Notify_event(PMIX_GROUP_CONSTRUCT_COMPLETE,
+                           &pmix_globals.myid,
+                           PMIX_RANGE_CUSTOM,
+                           cb.info, cb.ninfo,
+                           op_cbfunc, (void*)&cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the notify to get out */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_invite_nb(const char grp[],
+                                               const pmix_proc_t procs[], size_t nprocs,
+                                               const pmix_info_t info[], size_t ninfo,
+                                               pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_group_tracker_t *cb;
+    pmix_cb_t lock;
+    pmix_status_t codes[] = {
+        PMIX_GROUP_INVITE_ACCEPTED,
+        PMIX_GROUP_INVITE_DECLINED,
+        PMIX_PROC_TERMINATED
+    };
+    size_t ncodes, n;
+    pmix_info_t myinfo[2];
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, then we cannot notify */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* check for bozo input */
+    if (NULL == grp || NULL == procs) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    cb = PMIX_NEW(pmix_group_tracker_t);
+    if (NULL == cb) {
+        return PMIX_ERR_NOMEM;
+    }
+    cb->cbfunc = cbfunc;
+    cb->cbdata = cbdata;
+    cb->accepted = 1;  // obviously, we accept
+
+    /* compute the number of proposed members */
+    for (n=0; n < nprocs; n++) {
+        if (PMIX_RANK_WILDCARD == procs[n].rank) {
+            /* must get the number of procs in this nspace */
+        } else {
+            cb->nmembers++;
+        }
+    }
+
+    /* register an event handler specifically to respond
+     * to accept responses */
+    PMIX_INFO_LOAD(&myinfo[0], PMIX_EVENT_RETURN_OBJECT, cb, PMIX_POINTER);
+    PMIX_INFO_LOAD(&myinfo[1], PMIX_EVENT_HDLR_PREPEND, NULL, PMIX_BOOL);
+    ncodes = sizeof(codes)/sizeof(pmix_status_t);
+    PMIX_CONSTRUCT(&lock, pmix_cb_t);
+    PMIx_Register_event_handler(codes, ncodes, myinfo, 2,
+                                invite_handler, regcbfunc, &lock);
+    PMIX_WAIT_THREAD(&lock.lock);
+    rc = lock.status;
+    cb->ref = lock.errhandler_ref;
+    PMIX_DESTRUCT(&lock);
+    PMIX_INFO_DESTRUCT(&myinfo[0]);
+    PMIX_INFO_DESTRUCT(&myinfo[1]);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+
+    /* check for directives */
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+                /* setup a timer */
+                break;
+            }
+        }
+    }
+
+    /* limit the range to just the procs we are inviting */
+    PMIX_INFO_CREATE(cb->info, 3);
+    if (NULL == cb->info) {
+        PMIX_CONSTRUCT(&lock, pmix_cb_t);
+        PMIx_Deregister_event_handler(cb->ref,
+                                      op_cbfunc, &lock);
+        PMIX_WAIT_THREAD(&lock.lock);
+        PMIX_DESTRUCT(&lock);
+        PMIX_RELEASE(cb);
+        return PMIX_ERR_NOMEM;
+    }
+    cb->ninfo = 3;
+    n = 0;
+    (void)strncpy(cb->info[n].key, PMIX_EVENT_CUSTOM_RANGE, PMIX_MAX_KEYLEN);
+    cb->info[n].value.type = PMIX_DATA_ARRAY;
+    PMIX_DATA_ARRAY_CREATE(cb->info[n].value.data.darray, nprocs, PMIX_PROC);
+    if (NULL == cb->info[n].value.data.darray ||
+        NULL == cb->info[n].value.data.darray->array) {
+        PMIX_CONSTRUCT(&lock, pmix_cb_t);
+        PMIx_Deregister_event_handler(cb->ref,
+                                      op_cbfunc, &lock);
+        PMIX_WAIT_THREAD(&lock.lock);
+        PMIX_DESTRUCT(&lock);
+        PMIX_RELEASE(cb);
+        return PMIX_ERR_NOMEM;
+    }
+    memcpy(cb->info[n].value.data.darray->array, procs, nprocs * sizeof(pmix_proc_t));
+    ++n;
+    /* mark that this only goes to non-default handlers */
+    PMIX_INFO_LOAD(&cb->info[n], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    ++n;
+    /* provide the group ID */
+    PMIX_INFO_LOAD(&cb->info[n], PMIX_GROUP_ID, grp, PMIX_STRING);
+
+    /* notify everyone of the invitation */
+    PMIX_CONSTRUCT(&lock, pmix_cb_t);
+    rc = PMIx_Notify_event(PMIX_GROUP_INVITED,
+                           &pmix_globals.myid,
+                           PMIX_RANGE_CUSTOM,
+                           cb->info, cb->ninfo,
+                           op_cbfunc, (void*)&lock);
+    PMIX_WAIT_THREAD(&lock.lock);
+    rc = lock.status;
+    PMIX_DESTRUCT(&lock);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_CONSTRUCT(&lock, pmix_cb_t);
+        PMIx_Deregister_event_handler(cb->ref,
+                                      op_cbfunc, &lock);
+        PMIX_WAIT_THREAD(&lock.lock);
+        PMIX_DESTRUCT(&lock);
+        PMIX_RELEASE(cb);
+    }
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_join(const char grp[],
+                                          const pmix_proc_t *leader,
+                                          pmix_group_opt_t opt,
+                                          const pmix_info_t info[], size_t ninfo,
+                                          pmix_info_t **results, size_t *nresults)
+{
+    pmix_status_t rc;
+    pmix_group_tracker_t *cb;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which lock to release when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_group_tracker_t);
+
+    if (PMIX_SUCCESS != (rc = PMIx_Group_join_nb(grp, leader, opt, info, ninfo, info_cbfunc, cb))) {
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+
+    /* wait for the group construction to complete */
+    PMIX_WAIT_THREAD(&cb->lock);
+    rc = cb->status;
+    PMIX_RELEASE(cb);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix: group construction completed");
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_join_nb(const char grp[],
+                                             const pmix_proc_t *leader,
+                                             pmix_group_opt_t opt,
+                                             const pmix_info_t info[], size_t ninfo,
+                                             pmix_info_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_status_t rc;
+    pmix_group_tracker_t *cb;
+    pmix_status_t code;
+    size_t n;
+    pmix_data_range_t range;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "[%s:%d] pmix: join nb called",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, then we cannot notify */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which lock to release when
+     * the notification is done */
+    cb = PMIX_NEW(pmix_group_tracker_t);
+    cb->cbdata = cbdata;
+
+    /* check for directives */
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+                /* setup a timer */
+                break;
+            }
+        }
+    }
+
+    /* set the code according to their request */
+    if (PMIX_GROUP_ACCEPT == opt) {
+        code = PMIX_GROUP_INVITE_ACCEPTED;
+    } else {
+        code = PMIX_GROUP_INVITE_DECLINED;
+    }
+
+    /* only notify the leader so we don't hit all procs */
+    if (NULL != leader) {
+        range = PMIX_RANGE_CUSTOM;
+        PMIX_INFO_CREATE(cb->info, 1);
+        if (NULL == cb->info) {
+            PMIX_RELEASE(cb);
+            return PMIX_ERR_NOMEM;
+        }
+        PMIX_INFO_LOAD(&cb->info[0], PMIX_EVENT_CUSTOM_RANGE, leader, PMIX_PROC);
+        cb->ninfo = 1;
+    } else {
+        range = PMIX_RANGE_SESSION;
+    }
+
+    rc = PMIx_Notify_event(code,
+                           &pmix_globals.myid,
+                           range,
+                           cb->info, cb->ninfo,
+                           op_cbfunc, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cb);
+    }
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "[%s:%d] pmix: group invite %s",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                        (PMIX_GROUP_INVITE_ACCEPTED == code) ? "ACCEPTED" : "DECLINED");
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_leave(const char grp[],
+                                           const pmix_info_t info[], size_t ninfo)
+{
+    pmix_status_t rc;
+    pmix_group_tracker_t cb;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix: group_leave called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_group_tracker_t);
+
+    /* push the message into our event base to send to the server */
+    if (PMIX_SUCCESS != (rc = PMIx_Group_leave_nb(grp, info, ninfo, op_cbfunc, (void*)&cb))) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the connect to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix: group leave completed");
+
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Group_leave_nb(const char grp[],
+                                              const pmix_info_t info[], size_t ninfo,
+                                              pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_buffer_t *msg = NULL;
+    pmix_cmd_t cmd = PMIX_GROUP_LEAVE_CMD;
+    pmix_status_t rc;
+    pmix_group_tracker_t *cb = NULL;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix:group_leave_nb called");
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* check for bozo input */
+    if (NULL == grp) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the group ID */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &grp, 1, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto done;
+    }
+
+    /* pack the info structs */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        goto done;
+    }
+    if (0 < ninfo) {
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, info, ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            goto done;
+        }
+    }
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    cb = PMIX_NEW(pmix_group_tracker_t);
+    cb->opcbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, grp_cbfunc, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(cb);
+    }
+
+  done:
+    if (PMIX_SUCCESS != rc && NULL != msg) {
+        PMIX_RELEASE(msg);
+    }
+    return rc;
+}
+
+static void op_cbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_group_tracker_t *cb = (pmix_group_tracker_t*)cbdata;
+
+    cb->status = status;
+    if (NULL != cb->opcbfunc) {
+        cb->opcbfunc(status, cb->cbdata);
+    }
+    PMIX_POST_OBJECT(cb);
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
+static void relfn(void *cbdata)
+{
+    pmix_group_tracker_t *cb = (pmix_group_tracker_t*)cbdata;
+    PMIX_RELEASE(cb);
+}
+static void grp_cbfunc(struct pmix_peer_t *pr,
+                        pmix_ptl_hdr_t *hdr,
+                        pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_group_tracker_t *cb = (pmix_group_tracker_t*)cbdata;
+    pmix_status_t rc;
+    pmix_status_t ret;
+    int32_t cnt;
+    size_t ctxid, ninfo=0;
+    pmix_info_t info, *iptr=NULL;
+
+    pmix_output_verbose(2, pmix_client_globals.connect_output,
+                        "pmix:client recv callback activated with %d bytes",
+                        (NULL == buf) ? -1 : (int)buf->bytes_used);
+
+    if (NULL == buf) {
+        ret = PMIX_ERR_BAD_PARAM;
+        goto report;
+    }
+
+    /* a zero-byte buffer indicates that this recv is being
+     * completed due to a lost connection */
+    if (PMIX_BUFFER_IS_EMPTY(buf)) {
+        ret = PMIX_ERR_UNREACH;
+        goto report;
+    }
+
+    /* unpack the returned status */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
+                       buf, &ret, &cnt, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        ret = rc;
+    }
+    /* unpack any ctxid that was provided - it is okay if
+     * this attempts to unpack past end of buffer */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
+                       buf, &ctxid, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc && PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
+        PMIX_ERROR_LOG(rc);
+        ret = rc;
+    } else {
+        PMIX_INFO_LOAD(&info, PMIX_GROUP_CONTEXT_ID, &ctxid, PMIX_SIZE);
+        iptr = &info;
+        ninfo = 1;
+    }
+
+  report:
+    if (NULL != cb->cbfunc) {
+        cb->cbfunc(ret, iptr, ninfo, cb->cbdata, relfn, cb);
+        return;
+    } else if (NULL != cb->opcbfunc) {
+        cb->opcbfunc(ret, cb->cbdata);
+        return;
+    }
+    PMIX_RELEASE(cb);
+}
+
+static void info_cbfunc(pmix_status_t status,
+                        pmix_info_t *info, size_t ninfo,
+                        void *cbdata,
+                        pmix_release_cbfunc_t release_fn,
+                        void *release_cbdata)
+{
+    pmix_group_tracker_t *cb = (pmix_group_tracker_t*)cbdata;
+    size_t n;
+
+    /* see if anything was returned - e.g., a context id */
+    cb->status = status;
+    /* copy/save any returned info */
+    if (NULL != info) {
+        cb->nresults = ninfo;
+        PMIX_INFO_CREATE(cb->results, cb->nresults);
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&cb->results[n], &info[n]);
+        }
+    }
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    PMIX_POST_OBJECT(cb);
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
diff -Nru pmix-3.2.2~rc1/src/client/pmix_client_spawn.c pmix-4.0.0/src/client/pmix_client_spawn.c
--- pmix-3.2.2~rc1/src/client/pmix_client_spawn.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/client/pmix_client_spawn.c	2021-01-02 08:56:17.000000000 +0000
@@ -54,6 +54,8 @@
 #include "src/util/output.h"
 #include "src/util/pmix_environ.h"
 #include "src/mca/gds/gds.h"
+#include "src/mca/pfexec/pfexec.h"
+#include "src/mca/pmdl/pmdl.h"
 #include "src/mca/ptl/ptl.h"
 
 #include "pmix_client_ops.h"
@@ -82,7 +84,7 @@
     }
 
     /* if we aren't connected, don't attempt to send */
-    if (!pmix_globals.connected) {
+    if (!pmix_globals.connected && !PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_UNREACH;
     }
@@ -129,6 +131,10 @@
     pmix_cb_t *cb;
     size_t n, m;
     pmix_app_t *aptr;
+    bool jobenvars = false;
+    bool forkexec = false;
+    pmix_kval_t *kv;
+    pmix_list_t ilist;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
@@ -143,11 +149,42 @@
 
     /* if we aren't connected, don't attempt to send */
     if (!pmix_globals.connected) {
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_UNREACH;
+        /* if I am a launcher, we default to local fork/exec */
+        if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+            forkexec = true;
+        } else {
+            PMIX_RELEASE_THREAD(&pmix_global_lock);
+            return PMIX_ERR_UNREACH;
+        }
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
+    /* check job info for directives */
+    if (NULL != job_info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&job_info[n], PMIX_SETUP_APP_ENVARS)) {
+                PMIX_CONSTRUCT(&ilist, pmix_list_t);
+                rc = pmix_pmdl.harvest_envars(NULL, job_info, ninfo, &ilist);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_LIST_DESTRUCT(&ilist);
+                    return rc;
+                }
+                PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) {
+                    /* cycle across all the apps and set this envar */
+                    for (m=0; m < napps; m++) {
+                        aptr = (pmix_app_t*)&apps[m];
+                        pmix_setenv(kv->value->data.envar.envar,
+                                    kv->value->data.envar.value,
+                                    true, &aptr->env);
+                    }
+                }
+                jobenvars = true;
+                PMIX_LIST_DESTRUCT(&ilist);
+                break;
+            }
+        }
+    }
+
     for (n=0; n < napps; n++) {
         /* do a quick check of the apps directive array to ensure
          * the ninfo field has been set */
@@ -164,6 +201,35 @@
             }
             aptr->ninfo = m;
         }
+        if (!jobenvars) {
+            for (m=0; m < aptr->ninfo; m++) {
+                if (PMIX_CHECK_KEY(&aptr->info[m], PMIX_SETUP_APP_ENVARS)) {
+                    PMIX_CONSTRUCT(&ilist, pmix_list_t);
+                    rc = pmix_pmdl.harvest_envars(NULL, aptr->info, aptr->ninfo, &ilist);
+                    if (PMIX_SUCCESS != rc) {
+                        PMIX_LIST_DESTRUCT(&ilist);
+                        return rc;
+                    }
+                    PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) {
+                        pmix_setenv(kv->value->data.envar.envar,
+                                    kv->value->data.envar.value,
+                                    true, &aptr->env);
+                    }
+                    jobenvars = true;
+                    PMIX_LIST_DESTRUCT(&ilist);
+                    break;
+                }
+            }
+        }
+    }
+
+    /* if I am a tool and not connected, then just fork/exec
+     * the specified application */
+    if (forkexec) {
+        rc = pmix_pfexec.spawn_job(job_info, ninfo,
+                                   apps, napps,
+                                   cbfunc, cbdata);
+        return rc;
     }
 
     msg = PMIX_NEW(pmix_buffer_t);
diff -Nru pmix-3.2.2~rc1/src/client/pmix_client_topology.c pmix-4.0.0/src/client/pmix_client_topology.c
--- pmix-3.2.2~rc1/src/client/pmix_client_topology.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/client/pmix_client_topology.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,344 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2015 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include "src/include/pmix_globals.h"
+#include "src/util/error.h"
+#include "src/client/pmix_client_ops.h"
+#include "src/mca/ploc/ploc.h"
+
+PMIX_EXPORT pmix_status_t PMIx_Load_topology(pmix_topology_t *topo)
+{
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    rc = pmix_ploc.load_topology(topo);
+    return rc;
+}
+
+
+PMIX_EXPORT pmix_status_t PMIx_Parse_cpuset_string(const char *cpuset_string,
+                                                   pmix_cpuset_t *cpuset)
+{
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    rc = pmix_ploc.parse_cpuset_string(cpuset_string, cpuset);
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Get_cpuset(pmix_cpuset_t *cpuset, pmix_bind_envelope_t ref)
+{
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    rc = pmix_ploc.get_cpuset(cpuset, ref);
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Get_relative_locality(const char *locality1,
+                                                     const char *locality2,
+                                                     pmix_locality_t *locality)
+{
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    rc = pmix_ploc.get_relative_locality(locality1, locality2, locality);
+    return rc;
+}
+
+static void icbrelfn(void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    PMIX_RELEASE(cb);
+}
+
+static void distcb(pmix_status_t status,
+                   pmix_device_distance_t *dist, size_t ndist,
+                   void *cbdata,
+                   pmix_release_cbfunc_t release_fn,
+                   void *release_cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    size_t n;
+
+    cb->status = status;
+    cb->nvals = ndist;
+
+    if (PMIX_SUCCESS == status && 0 < ndist) {
+        PMIX_DEVICE_DIST_CREATE(cb->dist, cb->nvals);
+        for (n=0; n < cb->nvals; n++) {
+            if (NULL != dist[n].uuid) {
+                cb->dist[n].uuid = strdup(dist[n].uuid);
+            }
+            if (NULL != dist[n].osname) {
+                cb->dist[n].osname = strdup(dist[n].osname);
+            }
+            cb->dist[n].type = dist[n].type;
+            cb->dist[n].mindist = dist[n].mindist;
+            cb->dist[n].maxdist = dist[n].maxdist;
+        }
+    }
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    /* release the caller */
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
+pmix_status_t PMIx_Compute_distances(pmix_topology_t *topo,
+                                     pmix_cpuset_t *cpuset,
+                                     pmix_info_t info[], size_t ninfo,
+                                     pmix_device_distance_t *distances[],
+                                     size_t *ndist)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric update_distances");
+
+    *distances = NULL;
+    *ndist = 0;
+
+    /* create a callback object so we can be notified when
+     * the non-blocking operation is complete */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    rc = PMIx_Compute_distances_nb(topo, cpuset, info, ninfo, distcb, &cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* wait for the data to return */
+    PMIX_WAIT_THREAD(&cb.lock);
+
+    rc = cb.status;
+    /* xfer the results */
+    if (NULL != cb.dist) {
+        *distances = cb.dist;
+        *ndist = cb.nvals;
+        cb.dist = NULL;
+        cb.nvals = 0;
+    }
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric update_distances completed");
+
+    return rc;
+}
+
+static void dcbfunc(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+
+    if (NULL != cb->cbfunc.distfn) {
+        cb->cbfunc.distfn(cb->status, cb->dist, cb->nvals, cb->cbdata,
+                          icbrelfn, cb);
+        return;
+    }
+
+    PMIX_RELEASE(cb);
+}
+
+static void direcv(struct pmix_peer_t *peer,
+                   pmix_ptl_hdr_t *hdr,
+                   pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    pmix_status_t rc;
+    int cnt;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric direcv from server with %d bytes",
+                        (int)buf->bytes_used);
+
+    /* a zero-byte buffer indicates that this recv is being
+     * completed due to a lost connection */
+    if (PMIX_BUFFER_IS_EMPTY(buf)) {
+        rc = PMIX_ERR_UNREACH;
+        goto complete;
+    }
+
+    /* unpack the status */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cb->status, &cnt, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (PMIX_SUCCESS != cb->status) {
+        rc = cb->status;
+        goto complete;
+    }
+
+    /* unpack any returned data */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &cb->nvals, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc && PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < cb->nvals) {
+        PMIX_DEVICE_DIST_CREATE(cb->dist, cb->nvals);
+        cnt = cb->nvals;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, cb->dist, &cnt, PMIX_DEVICE_DIST);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto complete;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:fabric ifrecv from server releasing");
+    /* release the caller */
+    cb->cbfunc.distfn(rc, cb->dist, cb->nvals, cb->cbdata, icbrelfn, (void*)cb);
+}
+
+pmix_status_t PMIx_Compute_distances_nb(pmix_topology_t *topo,
+                                        pmix_cpuset_t *cpuset,
+                                        pmix_info_t info[], size_t ninfo,
+                                        pmix_device_dist_cbfunc_t cbfunc,
+                                        void *cbdata)
+{
+    pmix_cb_t *cb;
+    pmix_status_t rc;
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_FABRIC_COMPUTE_DISTANCES_CMD;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    cb = PMIX_NEW(pmix_cb_t);
+    cb->cbfunc.distfn = cbfunc;
+    cb->cbdata = cbdata;
+
+    /* see if I can support this myself */
+    cb->status = pmix_ploc.compute_distances(topo, cpuset, info, ninfo, &cb->dist, &cb->nvals);
+    if (PMIX_SUCCESS == cb->status || PMIX_ERR_NOT_AVAILABLE == cb->status) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        /* threadshift to return the result */
+        PMIX_THREADSHIFT(cb, dcbfunc);
+        return PMIX_SUCCESS;
+    }
+
+    /* if I am a server, there is nothing more I can do */
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) || !pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        PMIX_RELEASE(cb);
+        return PMIX_ERR_UNREACH;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* if we are a tool or client, then relay this request to the server */
+    msg = PMIX_NEW(pmix_buffer_t);
+    /* pack the cmd */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+
+    /* pack the topology we want them to use */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, topo, 1, PMIX_TOPO);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+    /* pack the cpuset */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, cpuset, 1, PMIX_PROC_CPUSET);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+
+    /* pack the directives */
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+        return rc;
+    }
+    if (0 < ninfo) {
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         msg, info, ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(msg);
+            PMIX_RELEASE(cb);
+            return rc;
+        }
+    }
+
+    /* push the message into our event base to send to the server */
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, direcv, (void*)cb);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        PMIX_RELEASE(cb);
+    }
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/src/common/Makefile.include pmix-4.0.0/src/common/Makefile.include
--- pmix-3.2.2~rc1/src/common/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/common/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -16,7 +16,9 @@
         common/pmix_control.c \
         common/pmix_data.c \
         common/pmix_security.c \
-        common/pmix_iof.c
+        common/pmix_iof.c \
+        common/pmix_attributes.c
 
 headers += \
-        common/pmix_iof.h
+        common/pmix_iof.h \
+        common/pmix_attributes.h
diff -Nru pmix-3.2.2~rc1/src/common/pmix_attributes.c pmix-4.0.0/src/common/pmix_attributes.c
--- pmix-3.2.2~rc1/src/common/pmix_attributes.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_attributes.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,952 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "src/include/pmix_config.h"
+
+#include "include/pmix.h"
+#include "include/pmix_common.h"
+#include "include/pmix_server.h"
+
+#include "src/mca/bfrops/bfrops.h"
+#include "src/mca/gds/base/base.h"
+#include "src/include/pmix_globals.h"
+#include "src/threads/threads.h"
+#include "src/client/pmix_client_ops.h"
+
+#include "src/common/pmix_attributes.h"
+#include "src/include/dictionary.h"
+
+static bool initialized = false;
+static pmix_list_t client_attrs;
+static pmix_list_t server_attrs;
+static pmix_list_t host_attrs;
+static pmix_list_t tool_attrs;
+
+typedef struct {
+    char *function;
+    char **attrs;
+} pmix_attr_init_t;
+
+typedef struct {
+    pmix_list_item_t super;
+    char *function;
+    char **attrs;
+} pmix_attribute_trk_t;
+
+static void atrkcon(pmix_attribute_trk_t *p)
+{
+    p->function = NULL;
+    p->attrs = NULL;
+}
+static void atrkdes(pmix_attribute_trk_t *p)
+{
+    if (NULL != p->function) {
+        free(p->function);
+    }
+    pmix_argv_free(p->attrs);
+}
+static PMIX_CLASS_INSTANCE(pmix_attribute_trk_t,
+                           pmix_list_item_t,
+                           atrkcon, atrkdes);
+
+PMIX_EXPORT void pmix_init_registered_attrs(void)
+{
+    if (!initialized) {
+        PMIX_CONSTRUCT(&client_attrs, pmix_list_t);
+        PMIX_CONSTRUCT(&server_attrs, pmix_list_t);
+        PMIX_CONSTRUCT(&host_attrs, pmix_list_t);
+        PMIX_CONSTRUCT(&tool_attrs, pmix_list_t);
+        initialized = true;
+    }
+}
+
+static pmix_status_t process_reg(char *level, char *function,
+                                 char **attrs)
+{
+    pmix_attribute_trk_t *fnptr;
+    pmix_list_t *lst;
+
+    /* select the list this will appear on */
+    if (0 == strcmp(level, PMIX_CLIENT_ATTRIBUTES)) {
+        lst = &client_attrs;
+    } else if (0 == strcmp(level, PMIX_SERVER_ATTRIBUTES)) {
+        lst = &server_attrs;
+    } else if (0 == strcmp(level, PMIX_HOST_ATTRIBUTES)) {
+        lst = &host_attrs;
+    } else if (0 == strcmp(level, PMIX_TOOL_ATTRIBUTES)) {
+        lst = &tool_attrs;
+    } else {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* see if we already have this function */
+    PMIX_LIST_FOREACH(fnptr, lst, pmix_attribute_trk_t) {
+        if (0 == strcmp(function, fnptr->function)) {
+            /* we already have this function at this level
+             * so we must return an error */
+            return PMIX_ERR_REPEAT_ATTR_REGISTRATION;
+        }
+    }
+
+    fnptr = PMIX_NEW(pmix_attribute_trk_t);
+    pmix_list_append(lst, &fnptr->super);
+    fnptr->function = strdup(function);
+    if (NULL != attrs) {
+        fnptr->attrs = pmix_argv_copy(attrs);
+    }
+    return PMIX_SUCCESS;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Register_attributes(char *function, char *attrs[])
+{
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+
+    rc = process_reg(PMIX_HOST_ATTRIBUTES, function, attrs);
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+    return rc;
+}
+
+PMIX_EXPORT void pmix_release_registered_attrs(void)
+{
+    if (initialized) {
+        PMIX_LIST_DESTRUCT(&client_attrs);
+        PMIX_LIST_DESTRUCT(&server_attrs);
+        PMIX_LIST_DESTRUCT(&host_attrs);
+        PMIX_LIST_DESTRUCT(&tool_attrs);
+    }
+    initialized = false;
+}
+
+/* sadly, we cannot dynamically register our supported attributes
+ * as that would require the user to first call the function whose
+ * attributes they want to know about - which somewhat defeats the
+ * purpose. Until someone comes up with a better solution, we will
+ * manually maintain the list */
+static pmix_attr_init_t client_fns[] = {
+    {.function = "PMIx_Init", .attrs = (char *[]){"PMIX_GDS_MODULE", "PMIX_EVENT_BASE", "PMIX_HOSTNAME",
+                                                  "PMIX_NODEID", "PMIX_DEBUG_STOP_IN_INIT", NULL}},
+    {.function = "PMIx_Finalize", .attrs = (char *[]){"PMIX_EMBED_BARRIER", NULL}},
+    {.function = "PMIx_Initialized", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Abort", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Store_internal", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Put", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Commit", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Fence", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Fence_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Get", .attrs = (char *[]){"PMIX_NODE_INFO", "PMIX_HOSTNAME", "PMIX_NODEID",
+                                                 "PMIX_APP_INFO", "PMIX_APPNUM", "PMIX_SESSION_INFO",
+                                                 "PMIX_GET_REFRESH_CACHE", NULL}},
+    {.function = "PMIx_Get_nb", .attrs = (char *[]){"PMIX_NODE_INFO", "PMIX_HOSTNAME", "PMIX_NODEID",
+                                                    "PMIX_APP_INFO", "PMIX_APPNUM", "PMIX_SESSION_INFO",
+                                                    "PMIX_GET_REFRESH_CACHE", NULL}},
+    {.function = "PMIx_Publish", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Publish_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Lookup", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Lookup_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Unpublish", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Unpublish_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Spawn", .attrs = (char *[]){"PMIX_SETUP_APP_ENVARS", NULL}},
+    {.function = "PMIx_Spawn_nb", .attrs = (char *[]){"PMIX_SETUP_APP_ENVARS", NULL}},
+    {.function = "PMIx_Connect", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Connect_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Disconnect", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Disconnect_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Resolve_peers", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Resolve_nodes", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Query_info", .attrs = (char *[]){"PMIX_QUERY_SUPPORTED_KEYS", "PMIX_QUERY_NAMESPACES",
+                                                        "PMIX_QUERY_ATTRIBUTE_SUPPORT", "PMIX_QUERY_AVAIL_SERVERS",
+                                                        "PMIX_QUERY_REFRESH_CACHE", "PMIX_QUERY_SUPPORTED_KEYS",
+                                                        "PMIX_QUERY_SUPPORTED_QUALIFIERS", "PMIX_TIME_REMAINING", NULL}},
+    {.function = "PMIx_Query_info_nb", .attrs = (char *[]){"PMIX_QUERY_SUPPORTED_KEYS", "PMIX_QUERY_NAMESPACES",
+                                                           "PMIX_QUERY_ATTRIBUTE_SUPPORT", "PMIX_QUERY_AVAIL_SERVERS",
+                                                           "PMIX_QUERY_REFRESH_CACHE", "PMIX_QUERY_SUPPORTED_KEYS",
+                                                           "PMIX_QUERY_SUPPORTED_QUALIFIERS", "PMIX_TIME_REMAINING", NULL}},
+    {.function = "PMIx_Log", .attrs = (char *[]){"PMIX_LOG_GENERATE_TIMESTAMP", "PMIX_LOG_SOURCE", NULL}},
+    {.function = "PMIx_Log_nb", .attrs = (char *[]){"PMIX_LOG_GENERATE_TIMESTAMP", "PMIX_LOG_SOURCE", NULL}},
+    {.function = "PMIx_Allocation_request", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Allocation_request_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Job_control", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Job_control_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Process_monitor", .attrs = (char *[]){"PMIX_SEND_HEARTBEAT", NULL}},
+    {.function = "PMIx_Process_monitor_nb", .attrs = (char *[]){"PMIX_SEND_HEARTBEAT", NULL}},
+    {.function = "PMIx_Get_credential", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
+    {.function = "PMIx_Get_credential_nb", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
+    {.function = "PMIx_Validate_credential", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
+    {.function = "PMIx_Validate_credential_nb", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
+    {.function = "PMIx_Group_construct", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_construct_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_invite", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_invite_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_join", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_join_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_leave", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_leave_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_destruct", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Group_destruct_nb", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Register_event_handler", .attrs = (char *[]){"PMIX_EVENT_HDLR_FIRST", "PMIX_EVENT_HDLR_LAST",
+                                                                    "PMIX_EVENT_HDLR_PREPEND", "PMIX_EVENT_HDLR_APPEND",
+                                                                    "PMIX_EVENT_HDLR_NAME", "PMIX_EVENT_RETURN_OBJECT",
+                                                                    "PMIX_EVENT_HDLR_FIRST_IN_CATEGORY",
+                                                                    "PMIX_EVENT_HDLR_LAST_IN_CATEGORY",
+                                                                    "PMIX_EVENT_HDLR_BEFORE", "PMIX_EVENT_HDLR_AFTER",
+                                                                    "PMIX_RANGE", "PMIX_EVENT_CUSTOM_RANGE",
+                                                                    "PMIX_EVENT_AFFECTED_PROC", "PMIX_EVENT_AFFECTED_PROCS",
+                                                                     NULL}},
+    {.function = "PMIx_Deregister_event_handler", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Notify_event", .attrs = (char *[]){"PMIX_EVENT_NON_DEFAULT", "PMIX_EVENT_CUSTOM_RANGE",
+                                                          "PMIX_EVENT_AFFECTED_PROC", "PMIX_EVENT_AFFECTED_PROCS", NULL}},
+    {.function = "PMIx_Error_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Proc_state_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Scope_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Persistence_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_range_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Info_directives_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_type_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Alloc_directive_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_IOF_channel_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Job_state_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Get_attribute_string", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Get_attribute_name", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Get_version", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_pack", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_unpack", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_copy", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_print", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_Data_copy_payload", .attrs = (char *[]){"N/A", NULL}},
+    {.function = ""}
+};
+
+/*****    REGISTER CLIENT ATTRS    *****/
+static bool client_attrs_regd = false;
+
+PMIX_EXPORT pmix_status_t pmix_register_client_attrs(void)
+{
+    size_t n;
+    pmix_status_t rc = PMIX_SUCCESS;
+
+    if (client_attrs_regd) {
+        return PMIX_SUCCESS;
+    }
+    client_attrs_regd = true;
+
+    for (n=0; 0 != strlen(client_fns[n].function); n++) {
+        rc = process_reg(PMIX_CLIENT_ATTRIBUTES,
+                         client_fns[n].function, client_fns[n].attrs);
+        if (PMIX_SUCCESS != rc) {
+            break;
+        }
+    }
+
+    return rc;
+}
+
+static pmix_attr_init_t server_fns[] = {
+    {.function = "PMIx_server_init", .attrs = (char *[]){"PMIX_SERVER_GATEWAY", "PMIX_SERVER_SCHEDULER",
+                                                         "PMIX_SERVER_TMPDIR", "PMIX_SYSTEM_TMPDIR",
+                                                         "PMIX_SERVER_NSPACE", "PMIX_SERVER_RANK",
+                                                         "PMIX_GDS_MODULE", "PMIX_EVENT_BASE",
+                                                         "PMIX_HOSTNAME", "PMIX_NODEID", "PMIX_TCP_IF_INCLUDE",
+                                                         "PMIX_TCP_IPV4_PORT", "PMIX_TCP_IPV6_PORT",
+                                                         "PMIX_TCP_DISABLE_IPV4", "PMIX_TCP_DISABLE_IPV6",
+                                                         "PMIX_SERVER_REMOTE_CONNECTIONS", "PMIX_TCP_REPORT_URI",
+                                                         "PMIX_SERVER_SESSION_SUPPORT", "PMIX_SERVER_SYSTEM_SUPPORT",
+                                                         "PMIX_SERVER_TOOL_SUPPORT", "PMIX_LAUNCHER_RENDEZVOUS_FILE",
+                                                         "PMIX_SERVER_ENABLE_MONITORING", NULL}},
+    {.function = "PMIx_server_finalize", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_generate_regex", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_generate_ppn", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_register_nspace", .attrs = (char *[]){"PMIX_REGISTER_NODATA", NULL}},
+    {.function = "PMIx_server_deregister_nspace", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_register_client", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_deregister_client", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_setup_fork", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_dmodex_request", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_setup_application", .attrs = (char *[]){"PMIX_SETUP_APP_ENVARS", "PMIX_SETUP_APP_ALL",
+                                                                      "PMIX_SETUP_APP_NONENVARS", "PMIX_ALLOC_FABRIC",
+                                                                      "PMIX_ALLOC_FABRIC_SEC_KEY", "PMIX_ALLOC_FABRIC_ID",
+                                                                      "PMIX_ALLOC_FABRIC_TYPE", "PMIX_ALLOC_FABRIC_PLANE",
+                                                                      "PMIX_ALLOC_FABRIC_ENDPTS", NULL}},
+    {.function = "PMIx_server_setup_local_support", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_server_IOF_deliver", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_server_collect_inventory", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_server_deliver_inventory", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_Register_attributes", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_register_fabric", .attrs = (char *[]){"PMIX_FABRIC_PLANE", "PMIX_FABRIC_IDENTIFIER",
+                                                                    "PMIX_FABRIC_VENDOR", NULL}},
+    {.function = "PMIx_server_update_fabric", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_deregister_fabric", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_get_vertex_info", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_server_get_index", .attrs = (char *[]){"PMIX_HOSTNAME", "PMIX_NODEID", "PMIX_FABRIC_DEVICE_NAME",
+                                                              "PMIX_FABRIC_DEVICE_VENDOR", "PMIX_FABRIC_DEVICE_BUS_TYPE",
+                                                              "PMIX_FABRIC_DEVICE_PCI_DEVID", NULL}},
+    {.function = ""}
+};
+
+/*****    REGISTER SERVER ATTRS    *****/
+static bool server_attrs_regd = false;
+
+PMIX_EXPORT pmix_status_t pmix_register_server_attrs(void)
+{
+    pmix_status_t rc = PMIX_SUCCESS;
+    size_t n;
+
+    if (server_attrs_regd) {
+        return PMIX_SUCCESS;
+    }
+    server_attrs_regd = true;
+
+    for (n=0; 0 != strlen(server_fns[n].function); n++) {
+        rc = process_reg(PMIX_SERVER_ATTRIBUTES,
+                         server_fns[n].function, server_fns[n].attrs);
+        if (PMIX_SUCCESS != rc) {
+            break;
+        }
+    }
+
+    return rc;
+}
+
+static pmix_attr_init_t tool_fns[] = {
+    {.function = "PMIx_tool_init", .attrs = (char *[]){"PMIX_GDS_MODULE", "PMIX_EVENT_BASE", "PMIX_HOSTNAME",
+                                                       "PMIX_NODEID", "PMIX_TOOL_DO_NOT_CONNECT", "PMIX_TOOL_NSPACE",
+                                                       "PMIX_TOOL_RANK", "PMIX_FWD_STDIN", "PMIX_LAUNCHER",
+                                                       "PMIX_SERVER_TMPDIR", "PMIX_SYSTEM_TMPDIR",
+                                                       "PMIX_TOOL_CONNECT_OPTIONAL", "PMIX_RECONNECT_SERVER",
+                                                       "PMIX_TOOL_ATTACHMENT_FILE", "PMIX_CONNECT_MAX_RETRIES",
+                                                       "PMIX_CONNECT_RETRY_DELAY", "PMIX_CONNECT_TO_SYSTEM",
+                                                       "PMIX_CONNECT_SYSTEM_FIRST", "PMIX_SERVER_PIDINFO",
+                                                       "PMIX_SERVER_NSPACE", "PMIX_SERVER_URI", NULL}},
+    {.function = "PMIx_tool_finalize", .attrs = (char *[]){"N/A", NULL}},
+    {.function = "PMIx_tool_connect_to_server", .attrs = (char *[]){"PMIX_CONNECT_TO_SYSTEM", "PMIX_CONNECT_SYSTEM_FIRST",
+                                                                    "PMIX_SERVER_PIDINFO", "PMIX_SERVER_NSPACE",
+                                                                    "PMIX_SERVER_URI", "PMIX_CONNECT_RETRY_DELAY",
+                                                                    "PMIX_CONNECT_MAX_RETRIES", "PMIX_RECONNECT_SERVER",
+                                                                    "PMIX_TOOL_ATTACHMENT_FILE", NULL}},
+    {.function = "PMIx_IOF_pull", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_IOF_deregister", .attrs = (char *[]){"NONE", NULL}},
+    {.function = "PMIx_IOF_push", .attrs = (char *[]){"PMIX_IOF_PUSH_STDIN", "PMIX_IOF_COMPLETE", NULL}},
+    {.function = ""}
+};
+
+/*****    REGISTER TOOL ATTRS    *****/
+static bool tool_attrs_regd = false;
+
+PMIX_EXPORT pmix_status_t pmix_register_tool_attrs(void)
+{
+    pmix_status_t rc = PMIX_SUCCESS;
+    size_t n;
+
+    if (tool_attrs_regd) {
+        return PMIX_SUCCESS;
+    }
+    tool_attrs_regd = true;
+
+    for (n=0; 0 != strlen(tool_fns[n].function); n++) {
+        rc = process_reg(PMIX_TOOL_ATTRIBUTES,
+                         tool_fns[n].function, tool_fns[n].attrs);
+        if (PMIX_SUCCESS != rc) {
+            break;
+        }
+    }
+
+    return rc;
+}
+
+/*****   PROCESS QUERY ATTRS    *****/
+static void _get_attrs(pmix_list_t *lst,
+                       pmix_info_t *info,
+                       pmix_list_t *attrs)
+{
+    pmix_attribute_trk_t *trk, *tptr;
+    pmix_infolist_t *ip;
+    pmix_data_array_t *darray;
+    pmix_regattr_t *regarray;
+    size_t m, nattr;
+    char **fns;
+    const pmix_regattr_input_t *dptr;
+
+    /* the value in the info is a comma-delimited list of
+     * functions whose attributes are being requested */
+    fns = pmix_argv_split(info->value.data.string, ',');
+
+    /* search the list for these functions */
+    PMIX_LIST_FOREACH(tptr, attrs, pmix_attribute_trk_t) {
+        trk = NULL;
+        for (m=0; NULL != fns[m] && NULL == trk; m++) {
+            if (0 == strcmp(fns[m], tptr->function) ||
+                0 == strcmp(fns[m], "all")) {
+                trk = tptr;
+                break;
+            }
+        }
+        if (NULL == trk || NULL == trk->attrs) {
+            /* function wasn't found - no attrs
+             * registered for it */
+            continue;
+        }
+        /* add the found attrs to the results */
+        ip = PMIX_NEW(pmix_infolist_t);
+        PMIX_LOAD_KEY(ip->info.key, tptr->function);
+        /* create the data array to hold the results */
+        nattr = pmix_argv_count(tptr->attrs);
+        PMIX_DATA_ARRAY_CREATE(darray, nattr, PMIX_REGATTR);
+        ip->info.value.type = PMIX_DATA_ARRAY;
+        ip->info.value.data.darray = darray;
+        regarray = (pmix_regattr_t*)darray->array;
+        for (m=0; m < nattr; m++) {
+            regarray[m].name = strdup(tptr->attrs[m]);
+            PMIX_LOAD_KEY(regarray[m].string, pmix_attributes_lookup(tptr->attrs[m]));
+            dptr = pmix_attributes_lookup_term(tptr->attrs[m]);
+            if (NULL == dptr) {
+                PMIX_RELEASE(ip);
+                return;
+            }
+            regarray[m].type = dptr->type;
+            regarray[m].description = pmix_argv_copy(dptr->description);
+        }
+        pmix_list_append(lst, &ip->super);
+    }
+    pmix_argv_free(fns);
+}
+
+static void _get_fns(pmix_list_t *lst,
+                     pmix_info_t *info,
+                     pmix_list_t *attrs)
+{
+    pmix_attribute_trk_t *tptr;
+    pmix_infolist_t *ip;
+    char **fns = NULL, *tmp;
+
+    /* search the list for these functions */
+    PMIX_LIST_FOREACH(tptr, attrs, pmix_attribute_trk_t) {
+        pmix_argv_append_nosize(&fns, tptr->function);
+    }
+    if (0 < pmix_argv_count(fns)) {
+        ip = PMIX_NEW(pmix_infolist_t);
+        tmp = pmix_argv_join(fns, ',');
+        PMIX_INFO_LOAD(&ip->info, info->key, tmp, PMIX_STRING);
+        pmix_list_append(lst, &ip->super);
+        pmix_argv_free(fns);
+    }
+}
+
+static void _local_relcb(void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    PMIX_RELEASE(cd);
+}
+
+static void relcbfunc(void *cbdata)
+{
+    pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query release callback");
+
+    if (NULL != cd->info) {
+        PMIX_INFO_FREE(cd->info, cd->ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+static void query_cbfunc(struct pmix_peer_t *peer,
+                         pmix_ptl_hdr_t *hdr,
+                         pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    pmix_status_t rc;
+    pmix_shift_caddy_t *results;
+    int cnt;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:attrs:query cback from server");
+
+    results = PMIX_NEW(pmix_shift_caddy_t);
+
+    /* unpack the status */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &results->status, &cnt, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        results->status = rc;
+        goto complete;
+    }
+    if (PMIX_SUCCESS != results->status) {
+        goto complete;
+    }
+
+    /* unpack any returned data */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &results->ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        results->status = rc;
+        goto complete;
+    }
+    if (0 < results->ninfo) {
+        PMIX_INFO_CREATE(results->info, results->ninfo);
+        cnt = results->ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, results->info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            results->status = rc;
+            goto complete;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query cback from server releasing");
+    /* release the caller */
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(results->status, results->info, results->ninfo, cd->cbdata, relcbfunc, results);
+    }
+    PMIX_RELEASE(cd);
+}
+
+PMIX_EXPORT void pmix_attrs_query_support(int sd, short args, void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    pmix_infolist_t *info, *head;
+    pmix_list_t kyresults;
+    size_t n, m, p;
+    pmix_info_t *iptr;
+    pmix_data_array_t *darray;
+    pmix_buffer_t *msg;
+    pmix_cmd_t cmd = PMIX_QUERY_CMD;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    for (n=0; n < cd->nqueries; n++) {
+        if (0 != strcmp(cd->queries[n].keys[0], PMIX_QUERY_ATTRIBUTE_SUPPORT)) {
+            /* skip this one */
+            continue;
+        }
+        head = NULL;
+        for (m=0; m < cd->queries[n].nqual; m++) {
+            PMIX_CONSTRUCT(&kyresults, pmix_list_t);
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_CLIENT_ATTRIBUTES)) {
+                /* everyone has access to the client attrs */
+                _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &client_attrs);
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_CLIENT_FUNCTIONS)) {
+                /* everyone has access to the client functions */
+                _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &client_attrs);
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_SERVER_ATTRIBUTES)) {
+                /* if I am a server, add in my attrs */
+                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+                    _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &server_attrs);
+                 } else {
+                    /* we need to ask our server for them */
+                    PMIX_LIST_DESTRUCT(&kyresults);
+                    goto query;
+                }
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_SERVER_FUNCTIONS)) {
+                /* if I am a server, add in my fns */
+                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+                    _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &server_attrs);
+                } else {
+                    /* we need to ask our server for them */
+                    PMIX_LIST_DESTRUCT(&kyresults);
+                    goto query;
+                }
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_TOOL_ATTRIBUTES)) {
+                if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+                    _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &tool_attrs);
+                }
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_TOOL_FUNCTIONS)) {
+                if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+                    _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &tool_attrs);
+                }
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_HOST_ATTRIBUTES)) {
+                /* if I am a server, add in the host's */
+                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+                    _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &host_attrs);
+                } else {
+                    /* we need to ask our server for them */
+                    PMIX_LIST_DESTRUCT(&kyresults);
+                    goto query;
+                }
+            }
+            if (NULL == cd->queries[n].qualifiers ||
+                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_HOST_FUNCTIONS)) {
+                /* if I am a server, add in the host's */
+                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+                    _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &host_attrs);
+                } else {
+                    /* we need to ask our server for them */
+                    PMIX_LIST_DESTRUCT(&kyresults);
+                    goto query;
+                }
+            }
+            if (0 < (p = pmix_list_get_size(&kyresults))) {
+                head = PMIX_NEW(pmix_infolist_t);
+                PMIX_LOAD_KEY(head->info.key, cd->queries[n].keys[m]);
+                head->info.value.type = PMIX_DATA_ARRAY;
+                /* create the data array to hold the results */
+                PMIX_DATA_ARRAY_CREATE(darray, p, PMIX_INFO);
+                head->info.value.data.darray = darray;
+                iptr = (pmix_info_t*)darray->array;
+                p = 0;
+                PMIX_LIST_FOREACH(info, &kyresults, pmix_infolist_t) {
+                    PMIX_INFO_XFER(&iptr[p], &info->info);
+                    ++p;
+                }
+                pmix_list_append(&cd->results, &head->super);
+            }
+            PMIX_LIST_DESTRUCT(&kyresults);
+        }
+    }
+    /* prep the response by converting the list
+     * of results into an array */
+    if (0 < (cd->ninfo = pmix_list_get_size(&cd->results))) {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        n = 0;
+        PMIX_LIST_FOREACH(info, &cd->results, pmix_infolist_t) {
+            PMIX_INFO_XFER(&cd->info[n], &info->info);
+            ++n;
+        }
+        cd->status = PMIX_SUCCESS;
+    } else {
+        cd->status = PMIX_ERR_NOT_FOUND;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+    goto release;
+
+  query:
+    /* if we aren't connected, don't attempt to send */
+    if (!pmix_globals.connected) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        cd->status = PMIX_ERR_NOT_FOUND;
+        goto release;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* relay this request to the server */
+    msg = PMIX_NEW(pmix_buffer_t);
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cmd, 1, PMIX_COMMAND);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        cd->status = rc;
+        goto release;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, &cd->nqueries, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        cd->status = rc;
+        goto release;
+    }
+    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                     msg, cd->queries, cd->nqueries, PMIX_QUERY);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(msg);
+        cd->status = rc;
+        goto release;
+    }
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:query sending to server");
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                       msg, query_cbfunc, (void*)cd);
+    if (PMIX_SUCCESS != rc) {
+        cd->status = rc;
+        goto release;
+    }
+    return;
+
+  release:
+    if (NULL != cd->cbfunc) {
+        cd->cbfunc(cd->status, cd->info, cd->ninfo, cd, _local_relcb, cd);
+        return;
+    }
+
+    PMIX_RELEASE(cd);
+}
+
+/*****   LOCATE A GIVEN ATTRIBUTE    *****/
+PMIX_EXPORT const char* pmix_attributes_lookup(char *attr)
+{
+    size_t n;
+
+    for (n=0; 0 != strlen(dictionary[n].name); n++) {
+        if (0 == strcasecmp(dictionary[n].name, attr)) {
+            return dictionary[n].string;
+        }
+    }
+    return NULL;
+}
+
+PMIX_EXPORT const char* pmix_attributes_reverse_lookup(char *attrstring)
+{
+    size_t n;
+
+    for (n=0; 0 != strlen(dictionary[n].name); n++) {
+        if (0 == strcasecmp(dictionary[n].string, attrstring)) {
+            return dictionary[n].name;
+        }
+    }
+    return NULL;
+}
+
+PMIX_EXPORT const pmix_regattr_input_t* pmix_attributes_lookup_term(char *attr)
+{
+    size_t n;
+
+    for (n=0; 0 != strlen(dictionary[n].name); n++) {
+        if (0 == strcmp(dictionary[n].name, attr)) {
+            return &dictionary[n];
+        }
+    }
+    return NULL;
+}
+
+/*****   PRINT QUERY FUNCTIONS RESULTS   *****/
+PMIX_EXPORT char** pmix_attributes_print_functions(char *level)
+{
+    char *title1 = "CLIENT SUPPORTED FUNCTIONS: ";
+    char *title2 = "SERVER SUPPORTED FUNCTIONS: ";
+    char *title3 = "HOST SUPPORTED FUNCTIONS: ";
+    char *title4 = "TOOL SUPPORTED FUNCTIONS: ";
+    char **ans = NULL;
+    pmix_list_t *lst;
+    pmix_attribute_trk_t *fnptr;
+
+    /* select title */
+    if (0 == strcmp(level, PMIX_CLIENT_FUNCTIONS)) {
+        pmix_argv_append_nosize(&ans, title1);
+        lst = &client_attrs;
+    } else if (0 == strcmp(level, PMIX_SERVER_FUNCTIONS)) {
+        pmix_argv_append_nosize(&ans, title2);
+        lst = &server_attrs;
+    } else if (0 == strcmp(level, PMIX_HOST_FUNCTIONS)) {
+        pmix_argv_append_nosize(&ans, title3);
+        lst = &host_attrs;
+    } else if (0 == strcmp(level, PMIX_TOOL_FUNCTIONS)) {
+        pmix_argv_append_nosize(&ans, title4);
+        lst = &tool_attrs;
+    } else {
+        return NULL;
+    }
+
+    PMIX_LIST_FOREACH(fnptr, lst, pmix_attribute_trk_t) {
+        pmix_argv_append_nosize(&ans, fnptr->function);
+    }
+    return ans;
+}
+
+/*****   PRINT QUERY ATTRS RESULTS   *****/
+
+#define PMIX_PRINT_NAME_COLUMN_WIDTH      35
+#define PMIX_PRINT_STRING_COLUMN_WIDTH    25
+#define PMIX_PRINT_TYPE_COLUMN_WIDTH      20
+#define PMIX_PRINT_ATTR_COLUMN_WIDTH     141
+
+void pmix_attributes_print_attrs(char ***ans, char *function,
+                                 pmix_regattr_t *attrs,
+                                 size_t nattrs)
+{
+    char line[PMIX_PRINT_ATTR_COLUMN_WIDTH], *tmp;
+    size_t n, m, len;
+
+    /* print the function */
+    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
+    m = 0;
+    for (n=0; n < strlen(function); n++) {
+        line[m] = function[n];
+        ++m;
+    }
+    line[m++] = ':';
+    line[m] = '\0';
+    pmix_argv_append_nosize(ans, line);
+
+    for (n=0; n < nattrs; n++) {
+        memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
+        line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
+        len = strlen(attrs[n].name);
+        if (PMIX_PRINT_NAME_COLUMN_WIDTH < len) {
+            len = PMIX_PRINT_NAME_COLUMN_WIDTH;
+        }
+        memcpy(line, attrs[n].name, len);
+
+        len = strlen(attrs[n].string);
+        if (PMIX_PRINT_STRING_COLUMN_WIDTH < len) {
+            len = PMIX_PRINT_STRING_COLUMN_WIDTH;
+        }
+        memcpy(&line[PMIX_PRINT_NAME_COLUMN_WIDTH+2], attrs[n].string, len);
+
+        tmp = (char*)PMIx_Data_type_string(attrs[n].type);
+        len = strlen(tmp);
+        if (PMIX_PRINT_STRING_COLUMN_WIDTH < len) {
+            len = PMIX_PRINT_STRING_COLUMN_WIDTH;
+        }
+        memcpy(&line[PMIX_PRINT_NAME_COLUMN_WIDTH+PMIX_PRINT_STRING_COLUMN_WIDTH+4], tmp, len);
+
+        for (m=0; NULL != attrs[n].description[m]; m++) {
+            len = strlen(attrs[n].description[m]);
+            memcpy(&line[PMIX_PRINT_NAME_COLUMN_WIDTH+PMIX_PRINT_STRING_COLUMN_WIDTH+PMIX_PRINT_TYPE_COLUMN_WIDTH+6], attrs[n].description[m], len);
+            line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';  // ensure NULL termination
+            pmix_argv_append_nosize(ans, line);
+            memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
+            line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
+        }
+    }
+}
+
+void pmix_attributes_print_headers(char ***ans, char *level)
+{
+    size_t n, m, left;
+    char *title1 = "CLIENT SUPPORTED ATTRIBUTES: ";
+    char *title2 = "SERVER SUPPORTED ATTRIBUTES: ";
+    char *title3 = "HOST SUPPORTED ATTRIBUTES: ";
+    char *title4 = "TOOL SUPPORTED ATTRIBUTES: ";
+    char line[PMIX_PRINT_ATTR_COLUMN_WIDTH];
+
+    /* select title */
+    if (0 == strcmp(level, PMIX_CLIENT_ATTRIBUTES)) {
+        pmix_argv_append_nosize(ans, title1);
+    } else if (0 == strcmp(level, PMIX_SERVER_ATTRIBUTES)) {
+        pmix_argv_append_nosize(ans, title2);
+    } else if (0 == strcmp(level, PMIX_HOST_ATTRIBUTES)) {
+        pmix_argv_append_nosize(ans, title3);
+    } else if (0 == strcmp(level, PMIX_TOOL_ATTRIBUTES)) {
+        pmix_argv_append_nosize(ans, title4);
+    } else {
+        return;
+    }
+
+    /* print the column headers */
+    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
+    line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
+    left = PMIX_PRINT_NAME_COLUMN_WIDTH/2 - 1;
+    memcpy(&line[left], "NAME", 4);
+
+    left = 3 + PMIX_PRINT_NAME_COLUMN_WIDTH + (PMIX_PRINT_STRING_COLUMN_WIDTH/2) - 2;
+    memcpy(&line[left], "STRING", 6);
+
+    left = 3 + PMIX_PRINT_NAME_COLUMN_WIDTH + PMIX_PRINT_STRING_COLUMN_WIDTH + (PMIX_PRINT_TYPE_COLUMN_WIDTH/2) - 2;
+    memcpy(&line[left], "TYPE", 4);
+
+    left = PMIX_PRINT_NAME_COLUMN_WIDTH + PMIX_PRINT_STRING_COLUMN_WIDTH + PMIX_PRINT_TYPE_COLUMN_WIDTH +
+           ((PMIX_PRINT_ATTR_COLUMN_WIDTH-PMIX_PRINT_NAME_COLUMN_WIDTH-PMIX_PRINT_STRING_COLUMN_WIDTH-PMIX_PRINT_TYPE_COLUMN_WIDTH)/2) - 3 - strlen("DESCRIPTION")/2;
+    memcpy(&line[left], "DESCRIPTION", strlen("DESCRIPTION"));
+    left += strlen("DESCRIPTION") + 1;
+    line[left] = '\0';
+    pmix_argv_append_nosize(ans, line);
+
+    /* print the dashes under the column headers */
+    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
+    line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
+    m=0;
+    for (n=0; n < PMIX_PRINT_NAME_COLUMN_WIDTH; n++) {
+        line[m] = '-';
+        ++m;
+    }
+    m += 2; // leave gap
+    for (n=0; n < PMIX_PRINT_STRING_COLUMN_WIDTH; n++) {
+        line[m] = '-';
+        ++m;
+    }
+    m += 2; // leave gap
+    for (n=0; n < PMIX_PRINT_TYPE_COLUMN_WIDTH; n++) {
+        line[m] = '-';
+        ++m;
+    }
+    m += 2; // leave gap
+    while (m < PMIX_PRINT_ATTR_COLUMN_WIDTH-1) {
+        line[m] = '-';
+        ++m;
+    }
+    pmix_argv_append_nosize(ans, line);
+}
+
+
+PMIX_EXPORT char** pmix_attributes_print_attr(char *level, char *function)
+{
+    size_t n, m, nattr;
+    char **tmp, **ans=NULL;
+    pmix_list_t *lst;
+    pmix_attribute_trk_t *fnptr;
+    char line[PMIX_PRINT_ATTR_COLUMN_WIDTH];
+    pmix_regattr_t *rptr;
+    const pmix_regattr_input_t *dptr;
+
+    /* select title */
+    if (0 == strcmp(level, PMIX_CLIENT_ATTRIBUTES)) {
+        lst = &client_attrs;
+    } else if (0 == strcmp(level, PMIX_SERVER_ATTRIBUTES)) {
+        lst = &server_attrs;
+    } else if (0 == strcmp(level, PMIX_HOST_ATTRIBUTES)) {
+        lst = &host_attrs;
+    } else if (0 == strcmp(level, PMIX_TOOL_ATTRIBUTES)) {
+        lst = &tool_attrs;
+    } else {
+        return NULL;
+    }
+
+    /* print the column headers */
+    pmix_attributes_print_headers(&ans, level);
+    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
+    line[1] = '\0';
+
+    /* can be comma-delimited list of functions */
+    tmp = pmix_argv_split(function, ',');
+    for (n=0; NULL != tmp[n]; n++) {
+        PMIX_LIST_FOREACH(fnptr, lst, pmix_attribute_trk_t) {
+            if (0 == strcmp(tmp[n], "all") ||
+                0 == strcmp(tmp[n], fnptr->function)) {
+                /* create an array of pmix_regattr_t for this function's attributes */
+                nattr = pmix_argv_count(fnptr->attrs);
+                PMIX_REGATTR_CREATE(rptr, nattr);
+                for (m=0; m < nattr; m++) {
+                    rptr[m].name = strdup(fnptr->attrs[m]);
+                    PMIX_LOAD_KEY(rptr[m].string, pmix_attributes_lookup(fnptr->attrs[m]));
+                    dptr = pmix_attributes_lookup_term(fnptr->attrs[m]);
+                    if (NULL == dptr) {
+                        pmix_argv_free(tmp);
+                        pmix_argv_free(ans);
+                        PMIX_REGATTR_FREE(rptr, nattr);
+                        return NULL;
+                    }
+                    rptr[m].type = dptr->type;
+                    rptr[m].description = pmix_argv_copy(dptr->description);
+                }
+                pmix_attributes_print_attrs(&ans, fnptr->function, rptr, nattr);
+                PMIX_REGATTR_FREE(rptr, nattr);
+                pmix_argv_append_nosize(&ans, line);
+                if (0 == strcmp(tmp[n], fnptr->function)) {
+                    break;
+                }
+            }
+        }
+    }
+    pmix_argv_free(tmp);
+
+
+    return ans;
+}
diff -Nru pmix-3.2.2~rc1/src/common/pmix_attributes.h pmix-4.0.0/src/common/pmix_attributes.h
--- pmix-3.2.2~rc1/src/common/pmix_attributes.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_attributes.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017      Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * I/O Forwarding Service
+ */
+
+#ifndef PMIX_ATTRIBUTES_H
+#define PMIX_ATTRIBUTES_H
+
+#include "src/include/pmix_config.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+#ifdef HAVE_NET_UIO_H
+#include <net/uio.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/include/pmix_globals.h"
+
+BEGIN_C_DECLS
+
+PMIX_EXPORT void pmix_init_registered_attrs(void);
+PMIX_EXPORT void pmix_release_registered_attrs(void);
+PMIX_EXPORT pmix_status_t pmix_register_tool_attrs(void);
+
+PMIX_EXPORT pmix_status_t pmix_register_client_attrs(void);
+PMIX_EXPORT pmix_status_t pmix_register_server_attrs(void);
+
+PMIX_EXPORT char** pmix_attributes_print_functions(char *level);
+PMIX_EXPORT char** pmix_attributes_print_attr(char *level, char *function);
+PMIX_EXPORT void pmix_attributes_print_attrs(char ***ans, char *function,
+                                 pmix_regattr_t *attrs,
+                                 size_t nattrs);
+PMIX_EXPORT void pmix_attributes_print_headers(char ***ans, char *level);
+
+PMIX_EXPORT void pmix_attrs_query_support(int sd, short args, void *cbdata);
+PMIX_EXPORT const char* pmix_attributes_lookup(char *name);
+PMIX_EXPORT const char* pmix_attributes_reverse_lookup(char *name);
+PMIX_EXPORT const pmix_regattr_input_t* pmix_attributes_lookup_term(char *attr);
+
+END_C_DECLS
+
+#endif /* PMIX_IOF_H */
diff -Nru pmix-3.2.2~rc1/src/common/pmix_control.c pmix-4.0.0/src/common/pmix_control.c
--- pmix-3.2.2~rc1/src/common/pmix_control.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_control.c	2021-01-02 08:56:17.000000000 +0000
@@ -135,7 +135,8 @@
 }
 
 PMIX_EXPORT pmix_status_t PMIx_Job_control(const pmix_proc_t targets[], size_t ntargets,
-                                           const pmix_info_t directives[], size_t ndirs)
+                                           const pmix_info_t directives[], size_t ndirs,
+                                           pmix_info_t **results, size_t *nresults)
 {
     pmix_cb_t cb;
     pmix_status_t rc;
@@ -165,6 +166,14 @@
     /* wait for the operation to complete */
     PMIX_WAIT_THREAD(&cb.lock);
     rc = cb.status;
+    if (0 < cb.ninfo) {
+        if (NULL != results && NULL != nresults) {
+            *results = cb.info;
+            *nresults = cb.ninfo;
+            cb.info = NULL;
+            cb.ninfo = 0;
+        }
+    }
     PMIX_DESTRUCT(&cb);
 
     pmix_output_verbose(2, pmix_globals.debug_output,
@@ -286,7 +295,8 @@
 }
 
 PMIX_EXPORT pmix_status_t PMIx_Process_monitor(const pmix_info_t *monitor, pmix_status_t error,
-                                               const pmix_info_t directives[], size_t ndirs)
+                                               const pmix_info_t directives[], size_t ndirs,
+                                               pmix_info_t **results, size_t *nresults)
 {
     pmix_cb_t cb;
     pmix_status_t rc;
@@ -316,6 +326,12 @@
     /* wait for the operation to complete */
     PMIX_WAIT_THREAD(&cb.lock);
     rc = cb.status;
+    if (0 < cb.ninfo) {
+        *results = cb.info;
+        *nresults = cb.ninfo;
+        cb.info = NULL;
+        cb.ninfo = 0;
+    }
     PMIX_DESTRUCT(&cb);
 
     pmix_output_verbose(2, pmix_globals.debug_output,
diff -Nru pmix-3.2.2~rc1/src/common/pmix_iof.c pmix-4.0.0/src/common/pmix_iof.c
--- pmix-3.2.2~rc1/src/common/pmix_iof.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_iof.c	2021-01-02 08:56:17.000000000 +0000
@@ -35,6 +35,7 @@
 #include "src/util/name_fns.h"
 #include "src/util/output.h"
 #include "src/mca/bfrops/bfrops.h"
+#include "src/mca/pfexec/base/base.h"
 #include "src/mca/ptl/ptl.h"
 
 #include "src/client/pmix_client_ops.h"
@@ -1126,6 +1127,12 @@
         pmix_event_del(&stdinev->ev);
         stdinev->active = false;
         PMIX_POST_OBJECT(stdinev);
+        if (PMIX_ERR_IOF_COMPLETE != ret) {
+            /* generate an IOF-failed event so the tool knows */
+            PMIx_Notify_event(PMIX_ERR_IOF_FAILURE,
+                              &pmix_globals.myid, PMIX_RANGE_PROC_LOCAL,
+                              NULL, 0, NULL, NULL);
+        }
         return;
     }
 
@@ -1143,6 +1150,7 @@
     pmix_cmd_t cmd = PMIX_IOF_PUSH_CMD;
     pmix_byte_object_t bo;
     int fd;
+    pmix_pfexec_child_t *child = (pmix_pfexec_child_t*)rev->childproc;
 
     PMIX_ACQUIRE_OBJECT(rev);
 
@@ -1178,6 +1186,21 @@
        re-add it */
     rev->active = false;
 
+    /* if this is from our own child proc, then
+     * just push it to the corresponding sink */
+    if (NULL != child) {
+        bo.bytes = (char*)data;
+        bo.size = numbytes;
+        pmix_iof_write_output(&rev->name, rev->channel, &bo, NULL);
+        if (0 == numbytes && child->completed &&
+            (NULL == child->stdoutev || !child->stdoutev->active) &&
+            (NULL == child->stderrev || !child->stderrev->active)) {
+            PMIX_PFEXEC_CHK_COMPLETE(child);
+            return;
+        }
+        goto reactivate;
+    }
+
     /* pass the data to our PMIx server so it can relay it
      * to the host RM for distribution */
     msg = PMIX_NEW(pmix_buffer_t);
@@ -1248,6 +1271,7 @@
         PMIX_RELEASE(msg);
     }
 
+  reactivate:
     if (0 < numbytes) {
         PMIX_IOF_READ_ACTIVATE(rev);
     }
diff -Nru pmix-3.2.2~rc1/src/common/pmix_query.c pmix-4.0.0/src/common/pmix_query.c
--- pmix-3.2.2~rc1/src/common/pmix_query.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_query.c	2021-01-02 08:56:17.000000000 +0000
@@ -28,6 +28,7 @@
 #include "src/util/output.h"
 #include "src/mca/bfrops/bfrops.h"
 #include "src/mca/ptl/base/base.h"
+#include "src/common/pmix_attributes.h"
 
 #include "src/client/pmix_client_ops.h"
 #include "src/server/pmix_server_ops.h"
@@ -115,6 +116,26 @@
     PMIX_RELEASE(cd);
 }
 
+static void qinfocb(pmix_status_t status, pmix_info_t info[], size_t ninfo,
+                    void *cbdata, pmix_release_cbfunc_t release_fn, void *release_cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    size_t n;
+
+    cb->status = status;
+    if (NULL != info) {
+        cb->ninfo = ninfo;
+        PMIX_INFO_CREATE(cb->info, cb->ninfo);
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&cb->info[n], &info[n]);
+        }
+    }
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
 static pmix_status_t request_help(pmix_query_t queries[], size_t nqueries,
                                   pmix_info_cbfunc_t cbfunc, void *cbdata)
 {
@@ -188,16 +209,6 @@
 
 }
 
-static void _local_relcb(void *cbdata)
-{
-    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
-
-    if (NULL != cd->info) {
-        PMIX_INFO_FREE(cd->info, cd->ninfo);
-    }
-    PMIX_RELEASE(cd);
-}
-
 static void localquery(int sd, short args, void *cbdata)
 {
     pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
@@ -276,49 +287,61 @@
     }
 
   nextstep:
-    if (PMIX_OPERATION_SUCCEEDED == rc) {
-        /* if we get here, then all queries were locally
-         * resolved, so construct the results for return */
-        cd->status = PMIX_SUCCESS;
-        cd->ninfo = pmix_list_get_size(&results);
-        if (0 < cd->ninfo) {
-            PMIX_INFO_CREATE(cd->info, cd->ninfo);
-            n = 0;
-            PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &results, pmix_kval_t) {
-                PMIX_LOAD_KEY(cd->info[n].key, kv->key);
-                rc = pmix_value_xfer(&cd->info[n].value, kv->value);
-                if (PMIX_SUCCESS != rc) {
-                    cd->status = rc;
-                    PMIX_INFO_FREE(cd->info, cd->ninfo);
-                    break;
-                }
-                ++n;
-            }
-        }
-        /* done with the list of results */
-        PMIX_LIST_DESTRUCT(&results);
-
+    /* need to ask our host */
+    rc = request_help(cd->queries, cd->nqueries, cd->cbfunc, cd->cbdata);
+    if (PMIX_SUCCESS != rc) {
+        /* we have to return the error to the caller */
         if (NULL != cd->cbfunc) {
-            cd->cbfunc(cd->status, cd->info, cd->ninfo, cd->cbdata, _local_relcb, cd);
-        }
-    } else if (PMIX_SUCCESS != rc) {
-        /* need to ask our host */
-        rc = request_help(cd->queries, cd->nqueries, cd->cbfunc, cd->cbdata);
-        if (PMIX_SUCCESS != rc) {
-            /* we have to return the error to the caller */
-            if (NULL != cd->cbfunc) {
-                cd->cbfunc(rc, NULL, 0, cd->cbdata, NULL, NULL);
-            }
+            cd->cbfunc(rc, NULL, 0, cd->cbdata, NULL, NULL);
         }
-        PMIX_RELEASE(cd);
-        return;
     }
-
-    /* get here if the query returned PMIX_SUCCESS, which means
-     * that the query is being processed and will call the cbfunc
-     * when complete */
+    PMIX_RELEASE(cd);
+    return;
 }
 
+PMIX_EXPORT pmix_status_t PMIx_Query_info(pmix_query_t queries[], size_t nqueries,
+                                          pmix_info_t **results, size_t *nresults)
+{
+    pmix_cb_t cb;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "%s pmix:query", PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    /* create a callback object as we need to pass it to the
+     * recv routine so we know which callback to use when
+     * the return message is recvd */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(queries, nqueries,
+                                                 qinfocb, &cb))) {
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+    /* wait for the operation to complete */
+    PMIX_WAIT_THREAD(&cb.lock);
+    rc = cb.status;
+    if (NULL != cb.info) {
+        *results = cb.info;
+        *nresults = cb.ninfo;
+        cb.info = NULL;
+        cb.ninfo = 0;
+    }
+    PMIX_DESTRUCT(&cb);
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "pmix:job_ctrl completed");
+
+    return rc;
+}
 PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nqueries,
                                              pmix_info_cbfunc_t cbfunc, void *cbdata)
 
@@ -367,6 +390,33 @@
      * assume that any requirement to refresh will force all to
      * do so */
     for (n=0; n < nqueries; n++) {
+        /* check for requests to report supported attributes */
+        if (0 == strcmp(queries[n].keys[0], PMIX_QUERY_ATTRIBUTE_SUPPORT)) {
+            cd = PMIX_NEW(pmix_query_caddy_t);
+            cd->queries = queries;
+            cd->nqueries = nqueries;
+            cd->cbfunc = cbfunc;
+            cd->cbdata = cbdata;
+            PMIX_THREADSHIFT(cd, pmix_attrs_query_support);
+            /* regardless of the result of the query, we return
+             * PMIX_SUCCESS here to indicate that the operation
+             * was accepted for processing */
+            return PMIX_SUCCESS;
+        }
+        /* check for request to scan the local node for available
+         * servers the caller could connect to */
+        if (0 == strcmp(queries[n].keys[0], PMIX_QUERY_AVAIL_SERVERS)) {
+            cd = PMIX_NEW(pmix_query_caddy_t);
+            cd->queries = queries;
+            cd->nqueries = nqueries;
+            cd->cbfunc = cbfunc;
+            cd->cbdata = cbdata;
+            PMIX_THREADSHIFT(cd, pmix_ptl_base_query_servers);
+            /* regardless of the result of the query, we return
+             * PMIX_SUCCESS here to indicate that the operation
+             * was accepted for processing */
+            return PMIX_SUCCESS;
+        }
         for (p=0; p < queries[n].nqual; p++) {
             if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_QUERY_REFRESH_CACHE)) {
                 if (PMIX_INFO_TRUE(&queries[n].qualifiers[p])) {
diff -Nru pmix-3.2.2~rc1/src/common/pmix_security.c pmix-4.0.0/src/common/pmix_security.c
--- pmix-3.2.2~rc1/src/common/pmix_security.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_security.c	2021-01-02 08:56:17.000000000 +0000
@@ -109,8 +109,46 @@
     PMIX_RELEASE(cd);
 }
 
+static void mycdcb(pmix_status_t status,
+                   pmix_byte_object_t *credential,
+                   pmix_info_t info[], size_t ninfo,
+                   void *cbdata)
+{
+    pmix_query_caddy_t *cb = (pmix_query_caddy_t*)cbdata;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+    cb->status = status;
+    if (PMIX_SUCCESS == status && NULL != credential) {
+        cb->bo.bytes = malloc(credential->size);
+        memcpy(cb->bo.bytes, credential->bytes, credential->size);
+        cb->bo.size = credential->size;
+    }
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Get_credential(const pmix_info_t info[], size_t ninfo,
-                                              pmix_credential_cbfunc_t cbfunc, void *cbdata)
+                                              pmix_byte_object_t *credential)
+{
+    pmix_query_caddy_t cb;
+    pmix_status_t rc;
+
+    PMIX_CONSTRUCT(&cb, pmix_query_caddy_t);
+    rc = PMIx_Get_credential_nb(info, ninfo, mycdcb, &cb);
+    if (PMIX_SUCCESS == rc) {
+        PMIX_WAIT_THREAD(&cb.lock);
+        rc = cb.status;
+        if (NULL != cb.bo.bytes) {
+            credential->bytes = malloc(cb.bo.size);
+            memcpy(credential->bytes, cb.bo.bytes, cb.bo.size);
+            credential->size = cb.bo.size;
+        }
+    }
+    PMIX_DESTRUCT(&cb);
+    return rc;
+}
+
+PMIX_EXPORT pmix_status_t PMIx_Get_credential_nb(const pmix_info_t info[], size_t ninfo,
+                                                 pmix_credential_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_buffer_t *msg;
     pmix_cmd_t cmd = PMIX_GET_CREDENTIAL_CMD;
@@ -295,9 +333,52 @@
     PMIX_RELEASE(cd);
 }
 
+static void myvalcb(pmix_status_t status,
+                    pmix_info_t info[], size_t ninfo,
+                    void *cbdata)
+{
+    pmix_query_caddy_t *cb = (pmix_query_caddy_t*)cbdata;
+    size_t n;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+    cb->status = status;
+    if (PMIX_SUCCESS == status && NULL != info) {
+        cb->ninfo = ninfo;
+        PMIX_INFO_CREATE(cb->info, cb->ninfo);
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&cb->info[n], &info[n]);
+        }
+    }
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
 PMIX_EXPORT pmix_status_t PMIx_Validate_credential(const pmix_byte_object_t *cred,
                                                    const pmix_info_t directives[], size_t ndirs,
-                                                   pmix_validation_cbfunc_t cbfunc, void *cbdata)
+                                                   pmix_info_t *results[], size_t *nresults)
+{
+    pmix_query_caddy_t cb;
+    pmix_status_t rc;
+
+    PMIX_CONSTRUCT(&cb, pmix_query_caddy_t);
+    rc = PMIx_Validate_credential_nb(cred, directives, ndirs, myvalcb, &cb);
+    if (PMIX_SUCCESS == rc) {
+        PMIX_WAIT_THREAD(&cb.lock);
+        rc = cb.status;
+        if (NULL != cb.info) {
+            *results = cb.info;
+            *nresults = cb.ninfo;
+            cb.info = NULL;
+            cb.ninfo = 0;
+        }
+    }
+    PMIX_DESTRUCT(&cb);
+    return rc;
+}
+
+
+PMIX_EXPORT pmix_status_t PMIx_Validate_credential_nb(const pmix_byte_object_t *cred,
+                                                      const pmix_info_t directives[], size_t ndirs,
+                                                      pmix_validation_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_buffer_t *msg;
     pmix_cmd_t cmd = PMIX_VALIDATE_CRED_CMD;
diff -Nru pmix-3.2.2~rc1/src/common/pmix_strings.c pmix-4.0.0/src/common/pmix_strings.c
--- pmix-3.2.2~rc1/src/common/pmix_strings.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/common/pmix_strings.c	2021-01-02 08:56:17.000000000 +0000
@@ -36,6 +36,7 @@
 #include "include/pmix_common.h"
 
 #include "src/include/pmix_globals.h"
+#include "src/common/pmix_attributes.h"
 
 PMIX_EXPORT const char* PMIx_Proc_state_string(pmix_proc_state_t state)
 {
@@ -273,3 +274,75 @@
     }
     return answer;
 }
+
+PMIX_EXPORT const char* PMIx_Job_state_string(pmix_job_state_t state)
+{
+    switch(state) {
+        case PMIX_JOB_STATE_UNDEF:
+            return "UNDEFINED";
+        case PMIX_JOB_STATE_PREPPED:
+            return "PREPPED FOR LAUNCH";
+        case PMIX_JOB_STATE_LAUNCH_UNDERWAY:
+            return "LAUNCHING";
+        case PMIX_JOB_STATE_RUNNING:
+            return "RUNNING";
+        case PMIX_JOB_STATE_SUSPENDED:
+            return "SUSPENDED";
+        case PMIX_JOB_STATE_CONNECTED:
+            return "CONNECTED";
+        case PMIX_JOB_STATE_UNTERMINATED:
+            return "UNTERMINATED";
+        case PMIX_JOB_STATE_TERMINATED:
+            return "TERMINATED";
+        case PMIX_JOB_STATE_TERMINATED_WITH_ERROR:
+            return "TERMINATED WITH ERROR";
+      default:
+            return "UNKNOWN";
+    }
+}
+
+PMIX_EXPORT const char* PMIx_Get_attribute_string(char *attribute)
+{
+    return pmix_attributes_lookup(attribute);
+}
+
+PMIX_EXPORT const char* PMIx_Get_attribute_name(char *attrstring)
+{
+    return pmix_attributes_reverse_lookup(attrstring);
+}
+
+PMIX_EXPORT const char* PMIx_Link_state_string(pmix_link_state_t state)
+{
+    switch(state) {
+        case PMIX_LINK_STATE_UNKNOWN:
+            return "UNKNOWN";
+        case PMIX_LINK_DOWN:
+            return "INACTIVE";
+        case PMIX_LINK_UP:
+            return "ACTIVE";
+        default:
+            return "UNKNOWN";
+    }
+}
+
+const char* PMIx_Device_type_string(pmix_device_type_t type)
+{
+    switch(type) {
+        case PMIX_DEVTYPE_UNKNOWN:
+            return "UNKNOWN";
+        case PMIX_DEVTYPE_BLOCK:
+            return "BLOCK";
+        case PMIX_DEVTYPE_GPU:
+            return "GPU";
+        case PMIX_DEVTYPE_NETWORK:
+            return "NETWORK";
+        case PMIX_DEVTYPE_OPENFABRICS:
+            return "OPENFABRICS";
+        case PMIX_DEVTYPE_DMA:
+            return "DMA";
+        case PMIX_DEVTYPE_COPROC:
+            return "COPROCESSOR";
+        default:
+            return "UNKNOWN";
+    }
+}
diff -Nru pmix-3.2.2~rc1/src/event/pmix_event.h pmix-4.0.0/src/event/pmix_event.h
--- pmix-3.2.2~rc1/src/event/pmix_event.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/event/pmix_event.h	2021-01-02 08:56:17.000000000 +0000
@@ -140,7 +140,11 @@
     size_t nallocated;
     pmix_info_t *results;
     size_t nresults;
+    pmix_info_t *interim;
+    size_t ninterim;
     pmix_event_hdlr_t *evhdlr;
+    pmix_op_cbfunc_t opcbfunc;
+    void *cbdata;
     pmix_op_cbfunc_t final_cbfunc;
     void *final_cbdata;
 } pmix_event_chain_t;
@@ -211,7 +215,7 @@
             }                                                                           \
             /* if this is lost-connection-to-server, then we let it go to */            \
             /* the default event handler - otherwise, we don't */                       \
-            if (PMIX_ERR_LOST_CONNECTION_TO_SERVER != (e) &&                            \
+            if (PMIX_ERR_LOST_CONNECTION != (e) &&                            \
                 PMIX_ERR_UNREACH != (e)) {                                              \
                 ch->ninfo = 1;                                                          \
                 ch->nallocated = 3;                                                     \
diff -Nru pmix-3.2.2~rc1/src/event/pmix_event_notification.c pmix-4.0.0/src/event/pmix_event_notification.c
--- pmix-3.2.2~rc1/src/event/pmix_event_notification.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/event/pmix_event_notification.c	2021-01-02 08:56:17.000000000 +0000
@@ -32,6 +32,10 @@
                                             pmix_data_range_t range,
                                             const pmix_info_t info[], size_t ninfo,
                                             pmix_op_cbfunc_t cbfunc, void *cbdata);
+static void progress_local_event_hdlr(pmix_status_t status,
+                                      pmix_info_t *results, size_t nresults,
+                                      pmix_op_cbfunc_t cbfunc, void *thiscbdata,
+                                      void *notification_cbdata);
 
 /* if we are a client, we call this function to notify the server of
  * an event. If we are a server, our host RM will call this function
@@ -51,8 +55,8 @@
         return PMIX_ERR_INIT;
     }
 
-    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) &&
-        !PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
+        PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
 
         pmix_output_verbose(2, pmix_server_globals.event_output,
@@ -67,7 +71,11 @@
         if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
             PMIX_ERROR_LOG(rc);
         }
-        return rc;
+        if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) &&
+            !PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+            return rc;
+        }
+        PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     }
 
     /* if we aren't connected, don't attempt to send */
@@ -313,19 +321,13 @@
 }
 
 
-static void progress_local_event_hdlr(pmix_status_t status,
-                                      pmix_info_t *results, size_t nresults,
-                                      pmix_op_cbfunc_t cbfunc, void *thiscbdata,
-                                      void *notification_cbdata)
+static void cycle_events(int sd, short args, void *cbdata)
 {
-    /* this may be in the host's thread, so we need to threadshift it
-     * before accessing our internal data */
-
-    pmix_event_chain_t *chain = (pmix_event_chain_t*)notification_cbdata;
+    pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata;
     size_t n, nsave, cnt;
-    pmix_info_t *newinfo;
     pmix_list_item_t *item;
     pmix_event_hdlr_t *nxt;
+    pmix_info_t *newinfo;
 
     pmix_output_verbose(2, pmix_client_globals.event_output,
                         "%s progressing local event",
@@ -345,7 +347,7 @@
      * the array to make space */
 
     /* add in any new results plus space for the returned status */
-    nsave += nresults + 1;
+    nsave += chain->ninterim + 1;
     /* create the new space */
     PMIX_INFO_CREATE(newinfo, nsave);
     /* transfer over the prior data */
@@ -364,11 +366,11 @@
         pmix_strncpy(newinfo[cnt].key, "UNKNOWN", PMIX_MAX_KEYLEN);
     }
     newinfo[cnt].value.type = PMIX_STATUS;
-    newinfo[cnt].value.data.status = status;
+    newinfo[cnt].value.data.status = chain->status;
     ++cnt;
     /* transfer across the new results */
-    for (n=0; n < nresults; n++) {
-        PMIX_INFO_XFER(&newinfo[cnt], &results[n]);
+    for (n=0; n < chain->ninterim; n++) {
+        PMIX_INFO_XFER(&newinfo[cnt], &chain->interim[n]);
         ++cnt;
     }
     /* release the prior results */
@@ -382,14 +384,27 @@
     chain->ninfo = chain->nallocated - 2;
     PMIX_INFO_DESTRUCT(&chain->info[chain->nallocated-2]);
     PMIX_INFO_DESTRUCT(&chain->info[chain->nallocated-1]);
-
+    // call their interim cbfunc
+    if (NULL != chain->opcbfunc) {
+        chain->opcbfunc(PMIX_SUCCESS, chain->cbdata);
+    }
+    
     /* if the caller indicates that the chain is completed,
      * or we completed the "last" event */
-    if (PMIX_EVENT_ACTION_COMPLETE == status || chain->endchain) {
-        goto complete;
+    if (PMIX_EVENT_ACTION_COMPLETE == chain->status || chain->endchain) {
+        if (PMIX_EVENT_ACTION_COMPLETE == chain->status) {
+            chain->status = PMIX_SUCCESS;
+        }
+        /* we still have to call their final callback */
+        if (NULL != chain->final_cbfunc) {
+            chain->final_cbfunc(chain->status, chain->final_cbdata);
+        }
+        /* maintain acctng */
+        PMIX_RELEASE(chain);
+        return;
     }
-    item = NULL;
 
+    item = NULL;
     /* see if we need to continue, starting with the single code events */
     if (1 == chain->evhdlr->ncodes) {
         /* the last handler was for a single code - see if there are
@@ -591,18 +606,31 @@
         }
     }
 
-  complete:
-    /* we still have to call their final callback */
+    /* if we get here, there was nothing more to do, but
+     * we still have to call their final callback */
     if (NULL != chain->final_cbfunc) {
-        chain->final_cbfunc(PMIX_SUCCESS, chain->final_cbdata);
+        chain->final_cbfunc(chain->status, chain->final_cbdata);
         return;
     }
     /* maintain acctng */
     PMIX_RELEASE(chain);
-    /* let the caller know that we are done with their callback */
-    if (NULL != cbfunc) {
-        cbfunc(PMIX_SUCCESS, thiscbdata);
-    }
+}
+
+static void progress_local_event_hdlr(pmix_status_t status,
+                                      pmix_info_t *results, size_t nresults,
+                                      pmix_op_cbfunc_t cbfunc, void *thiscbdata,
+                                      void *notification_cbdata)
+{
+    /* this may be in the host's thread, so we need to threadshift it
+     * before accessing our internal data */
+
+    pmix_event_chain_t *chain = (pmix_event_chain_t*)notification_cbdata;
+    
+    chain->interim = results;
+    chain->ninterim = nresults;
+    chain->opcbfunc = cbfunc;
+    chain->cbdata = thiscbdata;
+    PMIX_THREADSHIFT(chain, cycle_events);
 }
 
 /* given notification of an event, cycle thru our list of
@@ -929,6 +957,20 @@
     PMIX_INFO_CREATE(chain->info, chain->nallocated);
     /* prep the chain for processing */
     pmix_prep_event_chain(chain, cd->info, cd->ninfo, true);
+    /* if we are a tool, we send it to our clients regardless
+     * of the range - we assume that if we are in range, then
+     * so are any clients attached to us
+     *
+     * if we are a server but not a tool, then check the range
+     * before sending it to our client */
+    if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        /* remove the target designations */
+        if (NULL != chain->targets) {
+            free(chain->targets);
+            chain->targets = NULL;
+            chain->ntargets = 0;
+        }
+    }
 
     /* copy setup to the cd object */
     cd->nondefault = chain->nondefault;
@@ -992,6 +1034,49 @@
         return;
     }
 
+    /* check to see if this is a group_complete notification
+     * indicating that a group has asynchronously been formed.
+     * If it is, then we need to track the group */
+    if (PMIX_GROUP_CONSTRUCT_COMPLETE == cd->status) {
+        char *grpid = NULL;
+        pmix_group_t *grp;
+        /* must include the group id */
+        for (n=0; n < cd->ninfo; n++) {
+            if (PMIX_CHECK_KEY(&cd->info[n], PMIX_GROUP_ID)) {
+                grpid = cd->info[n].value.data.string;
+                break;
+            }
+        }
+        if (NULL == grpid) {
+            /* failed to provide the ID */
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            /* notify the caller */
+            if (NULL != cd->cbfunc) {
+                cd->cbfunc(PMIX_ERR_BAD_PARAM, cd->cbdata);
+            }
+            PMIX_RELEASE(cd);
+            PMIX_RELEASE(chain);
+            return;
+        }
+        /* must include members */
+        if (NULL == cd->targets || 0 == cd->ntargets) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            /* notify the caller */
+            if (NULL != cd->cbfunc) {
+                cd->cbfunc(PMIX_ERR_BAD_PARAM, cd->cbdata);
+            }
+            PMIX_RELEASE(cd);
+            PMIX_RELEASE(chain);
+            return;
+        }
+        grp = PMIX_NEW(pmix_group_t);
+        grp->grpid = strdup(grpid);
+        grp->nmbrs = cd->ntargets;
+        PMIX_PROC_CREATE(grp->members, grp->nmbrs);
+        memcpy(grp->members, cd->targets, cd->ntargets * sizeof(pmix_proc_t));
+        pmix_list_append(&pmix_server_globals.groups, &grp->super);
+    }
+
     holdcd = false;
     if (PMIX_RANGE_PROC_LOCAL != cd->range) {
         PMIX_CONSTRUCT(&trk, pmix_list_t);
@@ -1026,18 +1111,20 @@
                                                     pr->affected, pr->naffected)) {
                         continue;
                     }
-                    /* check the range */
-                    if (NULL == cd->targets) {
-                        rngtrk.procs = &cd->source;
-                        rngtrk.nprocs = 1;
-                    } else {
-                        rngtrk.procs = cd->targets;
-                        rngtrk.nprocs = cd->ntargets;
-                    }
-                    rngtrk.range = cd->range;
-                    PMIX_LOAD_PROCID(&proc, pr->peer->info->pname.nspace, pr->peer->info->pname.rank);
-                    if (!pmix_notify_check_range(&rngtrk, &proc)) {
-                        continue;
+                    if (!PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+                        /* check the range, if given */
+                        if (NULL == cd->targets) {
+                            rngtrk.procs = &cd->source;
+                            rngtrk.nprocs = 1;
+                        } else {
+                            rngtrk.procs = cd->targets;
+                            rngtrk.nprocs = cd->ntargets;
+                        }
+                        rngtrk.range = cd->range;
+                        PMIX_LOAD_PROCID(&proc, pr->peer->info->pname.nspace, pr->peer->info->pname.rank);
+                        if (!pmix_notify_check_range(&rngtrk, &proc)) {
+                            continue;
+                        }
                     }
                     pmix_output_verbose(2, pmix_server_globals.event_output,
                                         "pmix_server: notifying client %s:%u on status %s",
@@ -1154,17 +1241,15 @@
     size_t n;
 
     pmix_output_verbose(2, pmix_server_globals.event_output,
-                        "pmix_server: notify client of event %s",
-                        PMIx_Error_string(status));
+                        "pmix_server: notify client of event %s range %s",
+                        PMIx_Error_string(status), PMIx_Data_range_string(range));
 
     cd = PMIX_NEW(pmix_notify_caddy_t);
     cd->status = status;
     if (NULL == source) {
-        pmix_strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN);
-        cd->source.rank = PMIX_RANK_UNDEF;
+        PMIX_LOAD_PROCID(&cd->source, "UNDEF", PMIX_RANK_UNDEF);
     } else {
-        pmix_strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN);
-        cd->source.rank = source->rank;
+        PMIX_LOAD_PROCID(&cd->source, source->nspace, source->rank);
     }
     cd->range = range;
     /* have to copy the info to preserve it for future when cached */
@@ -1277,6 +1362,9 @@
     /* remove it from the list */
     pmix_list_remove_item(&pmix_globals.cached_events, &ch->super);
 
+    /* protect the chain */
+    PMIX_RETAIN(ch);
+
     /* process this event thru the regular channels */
     if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) &&
         !PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
@@ -1440,7 +1528,11 @@
     p->nallocated = 0;
     p->results = NULL;
     p->nresults = 0;
+    p->interim = NULL;
+    p->ninterim = 0;
     p->evhdlr = NULL;
+    p->opcbfunc = NULL;
+    p->cbdata = NULL;
     p->final_cbfunc = NULL;
     p->final_cbdata = NULL;
 }
diff -Nru pmix-3.2.2~rc1/src/event/pmix_event_registration.c pmix-4.0.0/src/event/pmix_event_registration.c
--- pmix-3.2.2~rc1/src/event/pmix_event_registration.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/event/pmix_event_registration.c	2021-01-02 08:56:17.000000000 +0000
@@ -413,7 +413,7 @@
         /* we always leave space for event hdlr name and a callback object */
         chain->nallocated = ncd->ninfo + 2;
         PMIX_INFO_CREATE(chain->info, chain->nallocated);
-        if (0 < cd->ninfo) {
+        if (0 < ncd->ninfo) {
             chain->ninfo = ncd->ninfo;
             /* need to copy the info */
             for (n=0; n < ncd->ninfo; n++) {
@@ -864,23 +864,20 @@
     PMIX_WAKEUP_THREAD(&cd->lock);
 }
 
-PMIX_EXPORT void PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes,
-                                             pmix_info_t info[], size_t ninfo,
-                                             pmix_notification_fn_t event_hdlr,
-                                             pmix_hdlr_reg_cbfunc_t cbfunc,
-                                             void *cbdata)
+PMIX_EXPORT pmix_status_t PMIx_Register_event_handler(pmix_status_t codes[], size_t ncodes,
+                                                      pmix_info_t info[], size_t ninfo,
+                                                      pmix_notification_fn_t event_hdlr,
+                                                      pmix_hdlr_reg_cbfunc_t cbfunc,
+                                                      void *cbdata)
 {
     pmix_rshift_caddy_t *cd;
-    size_t n;
+    pmix_status_t rc = PMIX_SUCCESS;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
     if (pmix_globals.init_cntr <= 0) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
-        if (NULL != cbfunc) {
-            cbfunc(PMIX_ERR_INIT, SIZE_MAX, cbdata);
-        }
-        return;
+        return PMIX_ERR_INIT;
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
@@ -895,14 +892,9 @@
         if (NULL == cd->codes) {
             /* immediately return error */
             PMIX_RELEASE(cd);
-            if (NULL != cbfunc) {
-                cbfunc(PMIX_ERR_NOMEM, SIZE_MAX, cbdata);
-            }
-            return;
-        }
-        for (n=0; n < ncodes; n++) {
-            cd->codes[n] = codes[n];
+            return PMIX_ERR_NOMEM;
         }
+        memcpy(cd->codes, codes, ncodes * sizeof(pmix_status_t));
     }
     cd->ncodes = ncodes;
     cd->info = info;
@@ -922,16 +914,10 @@
     	PMIX_RETAIN(cd);
         reg_event_hdlr(0, 0, (void*)cd);
         PMIX_WAIT_THREAD(&cd->lock);
-        if (NULL != cbfunc) {
-            if (0 > cd->status) {
-                cbfunc(cd->status, SIZE_MAX, cbdata);
-            } else {
-                cbfunc(PMIX_SUCCESS, cd->status, cbdata);
-            }
-        }
+        rc = cd->status;
         PMIX_RELEASE(cd);
     }
-    return;
+    return rc;
 }
 
 static void dereg_event_hdlr(int sd, short args, void *cbdata)
@@ -1131,19 +1117,17 @@
     PMIX_WAKEUP_THREAD(&cd->lock);
 }
 
-PMIX_EXPORT void PMIx_Deregister_event_handler(size_t event_hdlr_ref,
-                                               pmix_op_cbfunc_t cbfunc,
-                                               void *cbdata)
+PMIX_EXPORT pmix_status_t PMIx_Deregister_event_handler(size_t event_hdlr_ref,
+                                                        pmix_op_cbfunc_t cbfunc,
+                                                        void *cbdata)
 {
     pmix_shift_caddy_t *cd;
+    pmix_status_t rc = PMIX_SUCCESS;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
-        if (NULL != cbfunc) {
-            cbfunc(PMIX_ERR_INIT, cbdata);
-        }
-        return;
+        return PMIX_ERR_INIT;
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
@@ -1163,5 +1147,10 @@
                         "pmix_deregister_event_hdlr shifting to progress thread");
     PMIX_THREADSHIFT(cd, dereg_event_hdlr);
 
-    return;
+    if (NULL == cbfunc) {
+        PMIX_WAIT_THREAD(&cd->lock);
+        rc = cd->status;
+        PMIX_RELEASE(cd);
+    }
+    return rc;
 }
diff -Nru pmix-3.2.2~rc1/src/hwloc/hwloc.c pmix-4.0.0/src/hwloc/hwloc.c
--- pmix-3.2.2~rc1/src/hwloc/hwloc.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/hwloc/hwloc.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,754 +0,0 @@
-/*
- * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2017      Inria.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "src/include/pmix_config.h"
-#include "include/pmix_common.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <assert.h>
-#include <errno.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
-#include <string.h>
-#include <sys/mman.h>
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#if HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-
-#include "src/util/error.h"
-#include "src/util/fd.h"
-#include "src/util/path.h"
-#include "src/mca/bfrops/bfrops_types.h"
-#include "src/server/pmix_server_ops.h"
-#include "hwloc-internal.h"
-
-#if PMIX_HAVE_HWLOC
-
-#if HWLOC_API_VERSION >= 0x20000
-#include <hwloc/shmem.h>
-#endif
-
-
-PMIX_EXPORT hwloc_topology_t pmix_hwloc_topology = NULL;
-static bool external_topology = false;
-
-#if HWLOC_API_VERSION >= 0x20000
-static size_t shmemsize = 0;
-static size_t shmemaddr;
-static char *shmemfile = NULL;
-static int shmemfd = -1;
-
-static int parse_map_line(const char *line,
-                          unsigned long *beginp,
-                          unsigned long *endp,
-                          pmix_hwloc_vm_map_kind_t *kindp);
-static int use_hole(unsigned long holebegin,
-                    unsigned long holesize,
-                    unsigned long *addrp,
-                    unsigned long size);
-static int find_hole(pmix_hwloc_vm_hole_kind_t hkind,
-                     size_t *addrp,
-                     size_t size);
-static int enough_space(const char *filename,
-                        size_t space_req,
-                        uint64_t *space_avail,
-                        bool *result);
-#endif
-
-static int set_flags(hwloc_topology_t topo, unsigned int flags)
-{
-    #if HWLOC_API_VERSION < 0x20000
-            flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
-    #else
-            int ret = hwloc_topology_set_io_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-            if (0 != ret) return ret;
-    #endif
-    if (0 != hwloc_topology_set_flags(topo, flags)) {
-        return PMIX_ERR_INIT;
-    }
-    return PMIX_SUCCESS;
-}
-#endif // have_hwloc
-
-pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo)
-{
-#if PMIX_HAVE_HWLOC
-    size_t n;
-    bool save_xml_v1 = false;
-    bool save_xml_v2 = false;
-#if HWLOC_API_VERSION < 0x20000
-    bool save_xml_v2_reqd = false;
-#endif
-    bool share_topo = false;
-    bool share_reqd = false;
-    pmix_kval_t *kp2;
-    char *xml;
-    int sz;
-    pmix_status_t rc;
-#if HWLOC_API_VERSION >= 0x20000
-    pmix_hwloc_vm_hole_kind_t hole = VM_HOLE_BIGGEST;
-#endif
-
-    if (NULL == info || 0 == ninfo) {
-        if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
-            return PMIX_ERR_INIT;
-        }
-
-        if (0 != set_flags(pmix_hwloc_topology, 0)) {
-            hwloc_topology_destroy(pmix_hwloc_topology);
-            return PMIX_ERR_INIT;
-        }
-
-        if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-            hwloc_topology_destroy(pmix_hwloc_topology);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-        return PMIX_SUCCESS;
-    }
-
-    /* check for directives */
-    for (n=0; n < ninfo; n++) {
-        if (0 == strncmp(info[n].key, PMIX_TOPOLOGY, PMIX_MAX_KEYLEN)) {
-            /* if the pointer is NULL, then they want us to
-             * get the topology - it not NULL, then they
-             * are giving us the topology */
-            if (NULL != pmix_hwloc_topology) {
-                /* cannot have two topologies */
-                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-                return PMIX_ERR_BAD_PARAM;
-            }
-            if (NULL != info[n].value.data.ptr) {
-                pmix_hwloc_topology = (hwloc_topology_t)info[n].value.data.ptr;
-                external_topology = true;
-            } else {
-                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
-                    return PMIX_ERR_INIT;
-                }
-
-                if (0 != set_flags(pmix_hwloc_topology, 0)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERR_INIT;
-                }
-
-                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
-                    PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERR_NOT_SUPPORTED;
-                }
-            }
-        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_XML_V1, PMIX_MAX_KEYLEN)) {
-            /* if the string pointer is NULL or empty, then they
-             * want us to create it and store it for later sharing.
-             * if non-NULL, then this is the topology we are to use */
-            if (NULL == info[n].value.data.string) {
-                save_xml_v1 = true;
-            } else if (NULL != pmix_hwloc_topology) {
-                /* cannot have two topologies */
-                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-                return PMIX_ERR_BAD_PARAM;
-            } else {
-                /* load the topology */
-                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
-                    return PMIX_ERROR;
-                }
-                if (0 != hwloc_topology_set_xmlbuffer(pmix_hwloc_topology,
-                                                      info[n].value.data.string,
-                                                      strlen(info[n].value.data.string))) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERROR;
-                }
-                /* since we are loading this from an external source, we have to
-                 * explicitly set a flag so hwloc sets things up correctly
-                 */
-                if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERROR;
-                }
-                /* now load the topology */
-                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERROR;
-                }
-                /* store the string */
-                kp2 = PMIX_NEW(pmix_kval_t);
-                if (NULL == kp2) {
-                    return PMIX_ERR_NOMEM;
-                }
-                kp2->key = strdup(info[n].key);
-                PMIX_VALUE_XFER(rc, kp2->value, &info[n].value);
-                if (PMIX_SUCCESS != rc) {
-                    PMIX_ERROR_LOG(rc);
-                    PMIX_RELEASE(kp2);
-                    return rc;
-                }
-                pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-            }
-        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_XML_V2, PMIX_MAX_KEYLEN)) {
-            /* if the string pointer is NULL or empty, then they
-             * want us to create it and store it for later sharing.
-             * if non-NULL, then this is the topology we are to use */
-            if (NULL == info[n].value.data.string) {
-                save_xml_v2 = true;
-#if HWLOC_API_VERSION < 0x20000
-                save_xml_v2_reqd = PMIX_INFO_REQUIRED(&info[n]);
-#endif
-            } else if (NULL != pmix_hwloc_topology) {
-                /* cannot have two topologies */
-                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-                return PMIX_ERR_BAD_PARAM;
-            } else {
-                /* load the topology */
-                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
-                    return PMIX_ERROR;
-                }
-                if (0 != hwloc_topology_set_xmlbuffer(pmix_hwloc_topology,
-                                                      info[n].value.data.string,
-                                                      strlen(info[n].value.data.string))) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERROR;
-                }
-                /* since we are loading this from an external source, we have to
-                 * explicitly set a flag so hwloc sets things up correctly
-                 */
-                if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERROR;
-                }
-                /* now load the topology */
-                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERROR;
-                }
-                /* store the string */
-                kp2 = PMIX_NEW(pmix_kval_t);
-                if (NULL == kp2) {
-                    return PMIX_ERR_NOMEM;
-                }
-                kp2->key = strdup(info[n].key);
-                PMIX_VALUE_XFER(rc, kp2->value, &info[n].value);
-                if (PMIX_SUCCESS != rc) {
-                    PMIX_ERROR_LOG(rc);
-                    PMIX_RELEASE(kp2);
-                    return rc;
-                }
-                pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-            }
-        } else if (0 == strncmp(info[n].key, PMIX_TOPOLOGY_FILE, PMIX_MAX_KEYLEN)) {
-            if (NULL == info[n].value.data.string) {
-                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-                return PMIX_ERR_BAD_PARAM;
-            } else if (NULL != pmix_hwloc_topology) {
-                /* cannot have two topologies */
-                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-                return PMIX_ERR_BAD_PARAM;
-            } else {
-                if (0 != hwloc_topology_init(&pmix_hwloc_topology)) {
-                    return PMIX_ERR_NOT_SUPPORTED;
-                }
-                if (0 != hwloc_topology_set_xml(pmix_hwloc_topology, info[n].value.data.string)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERR_NOT_SUPPORTED;
-                }
-                /* since we are loading this from an external source, we have to
-                 * explicitly set a flag so hwloc sets things up correctly
-                 */
-                if (0 != set_flags(pmix_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                    return PMIX_ERR_NOT_SUPPORTED;
-                }
-                if (0 != hwloc_topology_load(pmix_hwloc_topology)) {
-                    hwloc_topology_destroy(pmix_hwloc_topology);
-                     return PMIX_ERR_NOT_SUPPORTED;
-                }
-                /* store the filename */
-                kp2 = PMIX_NEW(pmix_kval_t);
-                if (NULL == kp2) {
-                    return PMIX_ERR_NOMEM;
-                }
-                kp2->key = strdup(info[n].key);
-                PMIX_VALUE_XFER(rc, kp2->value, &info[n].value);
-                if (PMIX_SUCCESS != rc) {
-                    PMIX_ERROR_LOG(rc);
-                    PMIX_RELEASE(kp2);
-                    return rc;
-                }
-                pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-            }
-        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_SHARE_TOPO, PMIX_MAX_KEYLEN)) {
-            share_topo = PMIX_INFO_TRUE(&info[n]);
-            share_reqd = PMIX_INFO_IS_REQUIRED(&info[n]);
-        } else if (0 == strncmp(info[n].key, PMIX_HWLOC_HOLE_KIND, PMIX_MAX_KEYLEN)) {
-#if HWLOC_API_VERSION >= 0x20000
-            if (0 == strcasecmp(info[n].value.data.string, "none")) {
-                hole = VM_HOLE_NONE;
-            } else if (0 == strcasecmp(info[n].value.data.string, "begin")) {
-                hole = VM_HOLE_BEGIN;
-            } else if (0 == strcasecmp(info[n].value.data.string, "biggest")) {
-                hole = VM_HOLE_BIGGEST;
-            } else if (0 == strcasecmp(info[n].value.data.string, "libs")) {
-                hole = VM_HOLE_IN_LIBS;
-            } else if (0 == strcasecmp(info[n].value.data.string, "heap")) {
-                hole = VM_HOLE_AFTER_HEAP;
-            } else if (0 == strcasecmp(info[n].value.data.string, "stack")) {
-                hole = VM_HOLE_BEFORE_STACK;
-            } else {
-                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-                return PMIX_ERR_BAD_PARAM;
-            }
-#endif
-        }
-    }
-
-    if (save_xml_v1) {
-        /* create the XML string */
-#if HWLOC_API_VERSION >= 0x20000
-        if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz, HWLOC_TOPOLOGY_EXPORT_XML_FLAG_V1)) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-#else
-        if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz)) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-#endif
-        /* store it */
-        kp2 = PMIX_NEW(pmix_kval_t);
-        if (NULL == kp2) {
-            return PMIX_ERR_NOMEM;
-        }
-        kp2->key = strdup(PMIX_HWLOC_XML_V1);
-        PMIX_VALUE_LOAD(kp2->value, xml, PMIX_STRING);
-        hwloc_free_xmlbuffer(pmix_hwloc_topology, xml);
-        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-    }
-    if (save_xml_v2) {
-        /* create the XML string */
-#if HWLOC_API_VERSION >= 0x20000
-        if (0 != hwloc_topology_export_xmlbuffer(pmix_hwloc_topology, &xml, &sz, 0)) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-        /* store it */
-        kp2 = PMIX_NEW(pmix_kval_t);
-        if (NULL == kp2) {
-            return PMIX_ERR_NOMEM;
-        }
-        kp2->key = strdup(PMIX_HWLOC_XML_V1);
-        PMIX_VALUE_LOAD(kp2->value, xml, PMIX_STRING);
-        hwloc_free_xmlbuffer(pmix_hwloc_topology, xml);
-        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-#else
-        if (save_xml_v2_reqd) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-#endif
-    }
-
-    if (share_topo) {
-#if HWLOC_API_VERSION < 0x20000
-        if (share_reqd) {
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-#else
-        pmix_status_t rc;
-        bool space_available = false;
-        uint64_t amount_space_avail = 0;
-
-        if (VM_HOLE_NONE == hole) {
-            return PMIX_SUCCESS;
-        }
-
-        /* get the size of the topology shared memory segment */
-        if (0 != hwloc_shmem_topology_get_length(pmix_hwloc_topology, &shmemsize, 0)) {
-            if (share_reqd) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;
-            }
-            return PMIX_SUCCESS;
-        }
-
-        if (PMIX_SUCCESS != (rc = find_hole(hole, &shmemaddr, shmemsize))) {
-            /* we couldn't find a hole, so don't use the shmem support */
-            if (share_reqd) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;
-            }
-            return PMIX_SUCCESS;
-        }
-        /* create the shmem file in our session dir so it
-         * will automatically get cleaned up */
-        asprintf(&shmemfile, "%s/hwloc.sm", pmix_server_globals.tmpdir);
-        /* let's make sure we have enough space for the backing file */
-        if (PMIX_SUCCESS != (rc = enough_space(shmemfile, shmemsize,
-                                               &amount_space_avail,
-                                               &space_available))) {
-            free(shmemfile);
-            shmemfile = NULL;
-            if (share_reqd) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;
-            } else {
-                return PMIX_SUCCESS;
-            }
-        }
-        if (!space_available) {
-            free(shmemfile);
-            shmemfile = NULL;
-            if (share_reqd) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;
-            } else {
-                return PMIX_SUCCESS;
-            }
-        }
-        /* enough space is available, so create the segment */
-        if (-1 == (shmemfd = open(shmemfile, O_CREAT | O_RDWR, 0600))) {
-            free(shmemfile);
-            shmemfile = NULL;
-            if (share_reqd) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;
-            } else {
-                return PMIX_SUCCESS;
-            }
-        }
-        /* ensure nobody inherits this fd */
-        pmix_fd_set_cloexec(shmemfd);
-        /* populate the shmem segment with the topology */
-        if (0 != (rc = hwloc_shmem_topology_write(pmix_hwloc_topology, shmemfd, 0,
-                                                  (void*)shmemaddr, shmemsize, 0))) {
-            unlink(shmemfile);
-            free(shmemfile);
-            shmemfile = NULL;
-            close(shmemfd);
-            shmemfd = -1;
-            if (share_reqd) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-                return PMIX_ERR_NOT_SUPPORTED;
-            } else {
-                return PMIX_SUCCESS;
-            }
-        }
-        /* store the rendezvous info */
-        kp2 = PMIX_NEW(pmix_kval_t);
-        if (NULL == kp2) {
-            return PMIX_ERR_NOMEM;
-        }
-        kp2->key = strdup(PMIX_HWLOC_SHMEM_FILE);
-        PMIX_VALUE_CREATE(kp2->value, 1);
-        PMIX_VALUE_LOAD(kp2->value, shmemfile, PMIX_STRING);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(kp2);
-            return rc;
-        }
-        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-        kp2 = PMIX_NEW(pmix_kval_t);
-        if (NULL == kp2) {
-            return PMIX_ERR_NOMEM;
-        }
-        kp2->key = strdup(PMIX_HWLOC_SHMEM_ADDR);
-        PMIX_VALUE_CREATE(kp2->value, 1);
-        PMIX_VALUE_LOAD(kp2->value, &shmemaddr, PMIX_SIZE);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(kp2);
-            return rc;
-        }
-        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-        kp2 = PMIX_NEW(pmix_kval_t);
-        if (NULL == kp2) {
-            return PMIX_ERR_NOMEM;
-        }
-        kp2->key = strdup(PMIX_HWLOC_SHMEM_SIZE);
-        PMIX_VALUE_CREATE(kp2->value, 1);
-        PMIX_VALUE_LOAD(kp2->value, &shmemsize, PMIX_SIZE);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(kp2);
-            return rc;
-        }
-        pmix_list_append(&pmix_server_globals.gdata, &kp2->super);
-
-
-#endif
-    }
-
-    return PMIX_SUCCESS;
-#else  // PMIX_HAVE_HWLOC
-    return PMIX_SUCCESS;
-#endif
-}
-
-void pmix_hwloc_cleanup(void)
-{
-#if PMIX_HAVE_HWLOC
-#if HWLOC_API_VERSION >= 0x20000
-    if (NULL != shmemfile) {
-        unlink(shmemfile);
-        free(shmemfile);
-    }
-    if (0 <= shmemfd) {
-        close(shmemfd);
-    }
-#endif
-    if (NULL != pmix_hwloc_topology && !external_topology) {
-        hwloc_topology_destroy(pmix_hwloc_topology);
-    }
-#endif
-    return;
-}
-
-#if PMIX_HAVE_HWLOC
-#if HWLOC_API_VERSION >= 0x20000
-
-static int parse_map_line(const char *line,
-                          unsigned long *beginp,
-                          unsigned long *endp,
-                          pmix_hwloc_vm_map_kind_t *kindp)
-{
-    const char *tmp = line, *next;
-    unsigned long value;
-
-    /* "beginaddr-endaddr " */
-    value = strtoull(tmp, (char **) &next, 16);
-    if (next == tmp) {
-        return PMIX_ERROR;
-    }
-
-    *beginp = (unsigned long) value;
-
-    if (*next != '-') {
-        return PMIX_ERROR;
-    }
-
-     tmp = next + 1;
-
-    value = strtoull(tmp, (char **) &next, 16);
-    if (next == tmp) {
-        return PMIX_ERROR;
-    }
-    *endp = (unsigned long) value;
-    tmp = next;
-
-    if (*next != ' ') {
-        return PMIX_ERROR;
-    }
-    tmp = next + 1;
-
-    /* look for ending absolute path */
-    next = strchr(tmp, '/');
-    if (next) {
-        *kindp = VM_MAP_FILE;
-    } else {
-        /* look for ending special tag [foo] */
-        next = strchr(tmp, '[');
-        if (next) {
-            if (!strncmp(next, "[heap]", 6)) {
-                *kindp = VM_MAP_HEAP;
-            } else if (!strncmp(next, "[stack]", 7)) {
-                *kindp = VM_MAP_STACK;
-            } else {
-                char *end;
-                if ((end = strchr(next, '\n')) != NULL) {
-                    *end = '\0';
-                }
-                *kindp = VM_MAP_OTHER;
-            }
-        } else {
-            *kindp = VM_MAP_ANONYMOUS;
-        }
-    }
-
-    return PMIX_SUCCESS;
-}
-
-#define ALIGN2MB (2*1024*1024UL)
-
-static int use_hole(unsigned long holebegin,
-                    unsigned long holesize,
-                    unsigned long *addrp,
-                    unsigned long size)
-{
-    unsigned long aligned;
-    unsigned long middle = holebegin+holesize/2;
-
-    if (holesize < size) {
-        return PMIX_ERROR;
-    }
-
-    /* try to align the middle of the hole on 64MB for POWER's 64k-page PMD */
-    #define ALIGN64MB (64*1024*1024UL)
-    aligned = (middle + ALIGN64MB) & ~(ALIGN64MB-1);
-    if (aligned + size <= holebegin + holesize) {
-        *addrp = aligned;
-        return PMIX_SUCCESS;
-    }
-
-    /* try to align the middle of the hole on 2MB for x86 PMD */
-    aligned = (middle + ALIGN2MB) & ~(ALIGN2MB-1);
-    if (aligned + size <= holebegin + holesize) {
-        *addrp = aligned;
-        return PMIX_SUCCESS;
-    }
-
-    /* just use the end of the hole */
-    *addrp = holebegin + holesize - size;
-    return PMIX_SUCCESS;
-}
-
-static int find_hole(pmix_hwloc_vm_hole_kind_t hkind,
-                     size_t *addrp, size_t size)
-{
-    unsigned long biggestbegin = 0;
-    unsigned long biggestsize = 0;
-    unsigned long prevend = 0;
-    pmix_hwloc_vm_map_kind_t prevmkind = VM_MAP_OTHER;
-    int in_libs = 0;
-    FILE *file;
-    char line[96];
-
-    file = fopen("/proc/self/maps", "r");
-    if (!file) {
-        return PMIX_ERROR;
-    }
-
-    while (fgets(line, sizeof(line), file) != NULL) {
-        unsigned long begin=0, end=0;
-        pmix_hwloc_vm_map_kind_t mkind=VM_MAP_OTHER;
-
-        if (!parse_map_line(line, &begin, &end, &mkind)) {
-            switch (hkind) {
-                case VM_HOLE_BEGIN:
-                    fclose(file);
-                    return use_hole(0, begin, addrp, size);
-
-                case VM_HOLE_AFTER_HEAP:
-                    if (prevmkind == VM_MAP_HEAP && mkind != VM_MAP_HEAP) {
-                        /* only use HEAP when there's no other HEAP after it
-                         * (there can be several of them consecutively).
-                         */
-                        fclose(file);
-                        return use_hole(prevend, begin-prevend, addrp, size);
-                    }
-                    break;
-
-                case VM_HOLE_BEFORE_STACK:
-                    if (mkind == VM_MAP_STACK) {
-                        fclose(file);
-                        return use_hole(prevend, begin-prevend, addrp, size);
-                    }
-                    break;
-
-                case VM_HOLE_IN_LIBS:
-                    /* see if we are between heap and stack */
-                    if (prevmkind == VM_MAP_HEAP) {
-                        in_libs = 1;
-                    }
-                    if (mkind == VM_MAP_STACK) {
-                        in_libs = 0;
-                    }
-                    if (!in_libs) {
-                        /* we're not in libs, ignore this entry */
-                        break;
-                    }
-                    /* we're in libs, consider this entry for searching the biggest hole below */
-                    /* fallthrough */
-
-                case VM_HOLE_BIGGEST:
-                    if (begin-prevend > biggestsize) {
-                        biggestbegin = prevend;
-                        biggestsize = begin-prevend;
-                    }
-                    break;
-
-                    default:
-                        assert(0);
-            }
-        }
-
-        while (!strchr(line, '\n')) {
-            if (!fgets(line, sizeof(line), file)) {
-                goto done;
-            }
-        }
-
-        if (mkind == VM_MAP_STACK) {
-          /* Don't go beyond the stack. Other VMAs are special (vsyscall, vvar, vdso, etc),
-           * There's no spare room there. And vsyscall is even above the userspace limit.
-           */
-          break;
-        }
-
-        prevend = end;
-        prevmkind = mkind;
-
-    }
-
-  done:
-    fclose(file);
-    if (hkind == VM_HOLE_IN_LIBS || hkind == VM_HOLE_BIGGEST) {
-        return use_hole(biggestbegin, biggestsize, addrp, size);
-    }
-
-    return PMIX_ERROR;
-}
-
-static int enough_space(const char *filename,
-                        size_t space_req,
-                        uint64_t *space_avail,
-                        bool *result)
-{
-    uint64_t avail = 0;
-    size_t fluff = (size_t)(.05 * space_req);
-    bool enough = false;
-    char *last_sep = NULL;
-    /* the target file name is passed here, but we need to check the parent
-     * directory. store it so we can extract that info later. */
-    char *target_dir = strdup(filename);
-    int rc;
-
-    if (NULL == target_dir) {
-        rc = PMIX_ERR_OUT_OF_RESOURCE;
-        goto out;
-    }
-    /* get the parent directory */
-    last_sep = strrchr(target_dir, PMIX_PATH_SEP[0]);
-    *last_sep = '\0';
-    /* now check space availability */
-    if (PMIX_SUCCESS != (rc = pmix_path_df(target_dir, &avail))) {
-        goto out;
-    }
-    /* do we have enough space? */
-    if (avail >= space_req + fluff) {
-        enough = true;
-    }
-
-out:
-    if (NULL != target_dir) {
-        free(target_dir);
-    }
-    *result = enough;
-    *space_avail = avail;
-    return rc;
-}
-#endif
-
-#endif  // PMIX_HAVE_HWLOC
diff -Nru pmix-3.2.2~rc1/src/hwloc/hwloc-internal.h pmix-4.0.0/src/hwloc/hwloc-internal.h
--- pmix-3.2.2~rc1/src/hwloc/hwloc-internal.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/hwloc/hwloc-internal.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2016      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- *
- * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * this file is included in the rest of
- * the code base via pmix/hwloc/hwloc-internal.h.
- */
-
-#ifndef PMIX_HWLOC_INTERNAL_H
-#define PMIX_HWLOC_INTERNAL_H
-
-
-#include "src/include/pmix_config.h"
-#include "include/pmix_common.h"
-
-#if PMIX_HAVE_HWLOC
-#include PMIX_HWLOC_HEADER
-
-#if HWLOC_API_VERSION < 0x00010b00
-#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
-#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
-#endif
-
-PMIX_EXPORT extern hwloc_topology_t pmix_hwloc_topology;
-#endif
-
-BEGIN_C_DECLS
-
-typedef enum {
-    VM_HOLE_NONE = -1,
-    VM_HOLE_BEGIN = 0,        /* use hole at the very beginning */
-    VM_HOLE_AFTER_HEAP = 1,   /* use hole right after heap */
-    VM_HOLE_BEFORE_STACK = 2, /* use hole right before stack */
-    VM_HOLE_BIGGEST = 3,      /* use biggest hole */
-    VM_HOLE_IN_LIBS = 4,      /* use biggest hole between heap and stack */
-    VM_HOLE_CUSTOM = 5,       /* use given address if available */
-} pmix_hwloc_vm_hole_kind_t;
-
-typedef enum {
-    VM_MAP_FILE = 0,
-    VM_MAP_ANONYMOUS = 1,
-    VM_MAP_HEAP = 2,
-    VM_MAP_STACK = 3,
-    VM_MAP_OTHER = 4 /* vsyscall/vdso/vvar shouldn't occur since we stop after stack */
-} pmix_hwloc_vm_map_kind_t;
-
-PMIX_EXPORT pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo);
-PMIX_EXPORT void pmix_hwloc_cleanup(void);
-
-END_C_DECLS
-
-#endif /* PMIX_HWLOC_INTERNAL_H */
diff -Nru pmix-3.2.2~rc1/src/hwloc/Makefile.include pmix-4.0.0/src/hwloc/Makefile.include
--- pmix-3.2.2~rc1/src/hwloc/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/hwloc/Makefile.include	1970-01-01 00:00:00.000000000 +0000
@@ -1,15 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2018      Intel, Inc. All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-sources += \
-        hwloc/hwloc.c
-
-headers += \
-        hwloc/hwloc-internal.h
diff -Nru pmix-3.2.2~rc1/src/include/Makefile.am pmix-4.0.0/src/include/Makefile.am
--- pmix-3.2.2~rc1/src/include/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/include/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,66 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2007-2020 Cisco Systems, Inc.  All rights reserved
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+pmixdir = $(pmixincludedir)/$(subdir)
+
+noinst_LTLIBRARIES = libpmixglobal.la
+
+libpmixglobal_la_SOURCES = \
+	pmix_globals.h \
+        pmix_globals.c
+
+headers = pmix_globals.h
+
+nodist_headers = \
+    pmix_config.h
+
+if ! PMIX_EMBEDDED_MODE
+headers += \
+        align.h \
+        hash_string.h \
+        pmix_socket_errno.h \
+        pmix_stdint.h \
+        prefetch.h \
+        types.h \
+        pmix_config_top.h \
+        pmix_config_bottom.h \
+        pmix_portable_platform.h \
+        frameworks.h \
+        pmix_stdatomic.h \
+        dictionary.h
+
+endif ! PMIX_EMBEDDED_MODE
+
+if WANT_INSTALL_HEADERS
+nobase_pmix_HEADERS = $(headers)
+nobase_nodist_pmix_HEADERS = $(nodist_headers)
+endif
+
+# Need to ensure that dictionary.h is built first -- before any other
+# targets.
+BUILT_SOURCES = dictionary.h
+
+dictionary.h: $(top_srcdir)/include/pmix_common.h.in
+	$(PYTHON) $(top_srcdir)/contrib/construct_dictionary.py
+
+MAINTAINERCLEANFILES = dictionary.h
diff -Nru pmix-3.2.2~rc1/src/include/Makefile.include pmix-4.0.0/src/include/Makefile.include
--- pmix-3.2.2~rc1/src/include/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/include/Makefile.include	1970-01-01 00:00:00.000000000 +0000
@@ -1,55 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
-# Copyright (c) 2007-2016 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017      Research Organization for Information Science
-#                         and Technology (RIST). All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-# This makefile.am does not stand on its own - it is included from
-# Makefile.am
-
-headers += \
-        include/pmix_globals.h
-
-sources += \
-        include/pmix_globals.c
-
-nodist_headers += \
-    include/pmix_config.h
-
-if ! PMIX_EMBEDDED_MODE
-headers += \
-        include/align.h \
-        include/hash_string.h \
-        include/pmix_socket_errno.h \
-        include/pmix_stdint.h \
-        include/prefetch.h \
-        include/types.h \
-        include/pmix_config_top.h \
-        include/pmix_config_bottom.h \
-        include/pmix_portable_platform.h \
-        include/frameworks.h \
-        include/pmix_stdatomic.h
-
-endif ! PMIX_EMBEDDED_MODE
-
-if WANT_INSTALL_HEADERS
-nobase_nodist_pmix_HEADERS = \
-    include/pmix_config.h
-endif
diff -Nru pmix-3.2.2~rc1/src/include/pmix_globals.c pmix-4.0.0/src/include/pmix_globals.c
--- pmix-3.2.2~rc1/src/include/pmix_globals.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/include/pmix_globals.c	2021-01-02 08:56:17.000000000 +0000
@@ -68,6 +68,14 @@
                                 pmix_list_item_t,
                                 NULL, NULL);
 
+PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_proclist_t,
+                                pmix_list_item_t,
+                                NULL, NULL);
+
+PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_topo_obj_t,
+                                pmix_object_t,
+                                NULL, NULL);
+
 static void cfcon(pmix_cleanup_file_t *p)
 {
     p->path = NULL;
@@ -176,7 +184,7 @@
     p->proc_type.major = PMIX_MAJOR_WILDCARD;
     p->proc_type.minor = PMIX_MINOR_WILDCARD;
     p->proc_type.release = PMIX_RELEASE_WILDCARD;
-    p->proc_type.padding = 0;
+    p->proc_type.flag = 0;
     p->protocol = PMIX_PROTOCOL_UNDEF;
     p->finalized = false;
     p->info = NULL;
@@ -261,6 +269,7 @@
     PMIX_CONSTRUCT_LOCK(&p->lock);
     p->codes = NULL;
     p->ncodes = 0;
+    p->peer = NULL;
     p->pname.nspace = NULL;
     p->pname.rank = PMIX_RANK_UNDEF;
     p->data = NULL;
@@ -284,6 +293,9 @@
 static void scdes(pmix_shift_caddy_t *p)
 {
     PMIX_DESTRUCT_LOCK(&p->lock);
+    if (NULL != p->peer) {
+        PMIX_RELEASE(p->peer);
+    }
     if (NULL != p->pname.nspace) {
         free(p->pname.nspace);
     }
@@ -311,11 +323,13 @@
     p->nprocs = 0;
     p->info = NULL;
     p->ninfo = 0;
+    p->dist = NULL;
     p->infocopy = false;
     p->nvals = 0;
     PMIX_CONSTRUCT(&p->kvs, pmix_list_t);
     p->copy = false;
     p->timer_running = false;
+    p->fabric = NULL;
 }
 static void cbdes(pmix_cb_t *p)
 {
@@ -329,6 +343,9 @@
     if (p->infocopy) {
         PMIX_INFO_FREE(p->info, p->ninfo);
     }
+    if (NULL != p->dist) {
+        PMIX_DEVICE_DIST_FREE(p->dist, p->nvals);
+    }
     PMIX_LIST_DESTRUCT(&p->kvs);
 }
 PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_cb_t,
@@ -378,6 +395,7 @@
     p->relcbfunc = NULL;
     p->credcbfunc = NULL;
     p->validcbfunc = NULL;
+    p->stqcbfunc = NULL;
 }
 static void qdes(pmix_query_caddy_t *p)
 {
diff -Nru pmix-3.2.2~rc1/src/include/pmix_globals.h pmix-4.0.0/src/include/pmix_globals.h
--- pmix-3.2.2~rc1/src/include/pmix_globals.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/include/pmix_globals.h	2021-01-02 08:56:17.000000000 +0000
@@ -15,7 +15,6 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2019      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2020      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -82,6 +81,23 @@
 } pmix_namelist_t;
 PMIX_CLASS_DECLARATION(pmix_namelist_t);
 
+/* define a struct for holding entries in the
+ * dictionary of attributes */
+typedef struct {
+    char *name;
+    char *string;
+    pmix_data_type_t type;
+    char **description;
+} pmix_regattr_input_t;
+
+/* define a struct for passing topology objects */
+typedef struct {
+    pmix_object_t super;
+    char *source;
+    void *object;
+} pmix_topo_obj_t;
+PMIX_CLASS_DECLARATION(pmix_topo_obj_t);
+
 /* define a command type for communicating to the
  * pmix server */
 typedef uint8_t pmix_cmd_t;
@@ -117,6 +133,10 @@
 #define PMIX_GROUP_LEAVE_CMD                27
 #define PMIX_GROUP_DESTRUCT_CMD             28
 #define PMIX_IOF_DEREG_CMD                  29
+#define PMIX_FABRIC_REGISTER_CMD            30
+#define PMIX_FABRIC_UPDATE_CMD              31
+#define PMIX_FABRIC_COMPUTE_DISTANCES_CMD   32
+
 
 /* provide a "pretty-print" function for cmds */
 const char* pmix_command_string(pmix_cmd_t cmd);
@@ -144,7 +164,6 @@
     pmix_bfrop_buffer_type_t type;
     pmix_bfrops_module_t *bfrops;
     pmix_psec_module_t *psec;
-    pmix_ptl_module_t *ptl;
     pmix_gds_base_module_t *gds;
 } pmix_personality_t;
 
@@ -237,6 +256,11 @@
 } pmix_querylist_t;
 PMIX_CLASS_DECLARATION(pmix_querylist_t);
 
+typedef struct {
+    pmix_list_item_t super;
+    pmix_proc_t proc;
+} pmix_proclist_t;
+PMIX_CLASS_DECLARATION(pmix_proclist_t);
 
 /* object for tracking peers - each peer can have multiple
  * connections. This can occur if the initial app executes
@@ -280,6 +304,10 @@
 } pmix_iof_req_t;
 PMIX_CLASS_DECLARATION(pmix_iof_req_t);
 
+typedef void (*pmix_pstrg_query_cbfunc_t)(pmix_status_t status,
+                                          pmix_list_t *results,
+                                          void *cbdata);
+
 
 /* caddy for query requests */
 typedef struct {
@@ -302,6 +330,7 @@
     pmix_release_cbfunc_t relcbfunc;
     pmix_credential_cbfunc_t credcbfunc;
     pmix_validation_cbfunc_t validcbfunc;
+    pmix_pstrg_query_cbfunc_t stqcbfunc;
     void *cbdata;
 } pmix_query_caddy_t;
 PMIX_CLASS_DECLARATION(pmix_query_caddy_t);
@@ -363,6 +392,7 @@
     pmix_status_t *codes;
     size_t ncodes;
     pmix_name_t pname;
+    pmix_peer_t *peer;
     const char *data;
     size_t ndata;
     const char *key;
@@ -405,6 +435,8 @@
         pmix_lookup_cbfunc_t lookupfn;
         pmix_spawn_cbfunc_t spawnfn;
         pmix_hdlr_reg_cbfunc_t hdlrregfn;
+        pmix_info_cbfunc_t infofn;
+        pmix_device_dist_cbfunc_t distfn;
     } cbfunc;
     size_t errhandler_ref;
     void *cbdata;
@@ -416,11 +448,13 @@
     size_t nprocs;
     pmix_info_t *info;
     size_t ninfo;
+    pmix_device_distance_t *dist;
     bool infocopy;
     size_t nvals;
     pmix_list_t kvs;
     bool copy;
     bool timer_running;
+    pmix_fabric_t *fabric;
 } pmix_cb_t;
 PMIX_CLASS_DECLARATION(pmix_cb_t);
 
@@ -432,16 +466,16 @@
     pmix_event_active(&((r)->ev), EV_WRITE, 1);             \
 } while (0)
 
-#define PMIX_TIMED_THREADSHIFT(r, c, t)                         \
-    do {                                                        \
-        struct timeval _tv = {0, 0};                            \
-        _tv.tv_sec = (t);                                       \
-        pmix_event_evtimer_set(pmix_globals.evbase,             \
-                               &((r)->ev),                      \
-                               (c), (r));                       \
-        PMIX_POST_OBJECT((r));                                  \
-        pmix_event_evtimer_add(&((r)->ev), &_tv);               \
-    } while (0)
+#define PMIX_THREADSHIFT_DELAY(r, c, t)                     \
+ do {                                                       \
+    struct timeval _tv = {0, 0};                            \
+    pmix_event_evtimer_set(pmix_globals.evbase, &(r)->ev,   \
+                               (c), (r));                   \
+    _tv.tv_sec = (int)(t);                                  \
+    _tv.tv_usec = ((t) - _tv.tv_sec) * 1000000.0;           \
+    PMIX_POST_OBJECT((r));                                  \
+    pmix_event_evtimer_add(&(r)->ev, &_tv);                 \
+} while (0)
 
 
 typedef struct {
@@ -506,7 +540,6 @@
     uint32_t nodeid;                    // my nodeid, if given
     int pindex;
     pmix_event_base_t *evbase;
-    bool external_evbase;
     int debug_output;
     pmix_events_t events;               // my event handler registrations.
     bool connected;
@@ -532,6 +565,9 @@
     bool timestamp_output;
     size_t output_limit;
     pmix_list_t nspaces;
+    pmix_topology_t topology;
+    bool external_topology;
+    bool external_progress;
 } pmix_globals_t;
 
 /* provide access to a function to cleanup epilogs */
@@ -545,7 +581,13 @@
 static inline bool pmix_check_node_info(const char* key)
 {
     char *keys[] = {
+    	PMIX_HOSTNAME,
+    	PMIX_HOSTNAME_ALIASES,
+    	PMIX_NODEID,
+    	PMIX_AVAIL_PHYS_MEMORY,
         PMIX_LOCAL_PEERS,
+        PMIX_LOCAL_PROCS,
+        PMIX_LOCAL_CPUSETS,
         PMIX_LOCAL_SIZE,
         PMIX_NODE_SIZE,
         PMIX_LOCALLDR,
@@ -566,7 +608,34 @@
     char *keys[] = {
         PMIX_APP_SIZE,
         PMIX_APPLDR,
+        PMIX_APP_ARGV,
         PMIX_WDIR,
+        PMIX_PSET_NAME,
+        PMIX_APP_MAP_TYPE,
+        PMIX_APP_MAP_REGEX,
+        NULL
+    };
+    size_t n;
+
+    for (n=0; NULL != keys[n]; n++) {
+        if (0 == strncmp(key, keys[n], PMIX_MAX_KEYLEN)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static inline bool pmix_check_session_info(const char* key)
+{
+    char *keys[] = {
+        PMIX_SESSION_ID,
+        PMIX_CLUSTER_ID,
+        PMIX_UNIV_SIZE,
+        PMIX_TMPDIR,
+        PMIX_TDIR_RMCLEAN,
+        PMIX_HOSTNAME_KEEP_FQDN,
+        PMIX_RM_NAME,
+        PMIX_RM_VERSION,
         NULL
     };
     size_t n;
@@ -579,6 +648,7 @@
     return false;
 }
 
+
 END_C_DECLS
 
 #endif /* PMIX_GLOBALS_H */
diff -Nru pmix-3.2.2~rc1/src/Makefile.am pmix-4.0.0/src/Makefile.am
--- pmix-3.2.2~rc1/src/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -9,9 +9,9 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2006-2016 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,12 +19,9 @@
 # $HEADER$
 #
 
-# Note that the -I directory must *exactly* match what was specified
-# via AC_CONFIG_MACRO_DIR in configure.ac.
-ACLOCAL_AMFLAGS = -I ./config
-
 SUBDIRS = \
-	util/keyval \
+    include \
+    util/keyval \
     mca/base \
     $(MCA_pmix_FRAMEWORKS_SUBDIRS) \
     $(MCA_pmix_FRAMEWORK_COMPONENT_STATIC_SUBDIRS) \
@@ -32,7 +29,8 @@
     $(MCA_pmix_FRAMEWORK_COMPONENT_DSO_SUBDIRS)
 
 DIST_SUBDIRS = \
-	util/keyval \
+    include \
+    util/keyval \
     mca/base \
     $(MCA_pmix_FRAMEWORKS_SUBDIRS) \
     $(MCA_pmix_FRAMEWORK_COMPONENT_ALL_SUBDIRS)
@@ -76,19 +74,10 @@
 lib_LTLIBRARIES = libpmix.la
 libpmix_la_SOURCES = $(headers) $(sources)
 libpmix_la_LDFLAGS = -version-info $(libpmix_so_version)
-
-if WANT_PMI_BACKWARD
-lib_LTLIBRARIES += libpmi.la libpmi2.la
-libpmi_la_SOURCES = $(headers) $(sources) $(pmi1_sources)
-libpmi_la_LDFLAGS = -version-info $(libpmi_so_version)
-libpmi_la_LIBADD = $(libpmix_la_LIBADD)
-libpmi_la_DEPENDENCIES = $(libpmi_la_LIBADD)
-
-libpmi2_la_SOURCES = $(headers) $(sources) $(pmi2_sources)
-libpmi2_la_LDFLAGS = -version-info $(libpmi2_so_version)
-libpmi2_la_LIBADD = $(libpmix_la_LIBADD)
-libpmi2_la_DEPENDENCIES = $(libpmi2_la_LIBADD)
-endif
+libpmix_la_LIBADD += \
+        include/libpmixglobal.la
+libpmix_la_DEPENDENCIES += \
+        include/libpmixglobal.la
 
 endif !PMIX_EMBEDDED_MODE
 
@@ -96,7 +85,6 @@
 include threads/Makefile.include
 include class/Makefile.include
 include event/Makefile.include
-include include/Makefile.include
 include mca/Makefile.include
 include util/Makefile.include
 include client/Makefile.include
@@ -105,13 +93,9 @@
 include tool/Makefile.include
 include tools/Makefile.include
 include common/Makefile.include
-include hwloc/Makefile.include
 
 if WANT_INSTALL_HEADERS
 nobase_pmix_HEADERS += $(headers)
 endif
 
-
-MAINTAINERCLEANFILES = Makefile.in config.h config.h.in
-DISTCLEANFILES = Makefile
 CLEANFILES = core.* *~
diff -Nru pmix-3.2.2~rc1/src/mca/base/help-pmix-mca-base.txt pmix-4.0.0/src/mca/base/help-pmix-mca-base.txt
--- pmix-3.2.2~rc1/src/mca/base/help-pmix-mca-base.txt	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/base/help-pmix-mca-base.txt	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2008-2019 Cisco Systems, Inc.  All rights reserved
-# Copyright (c) 2018      Intel, Inc. All rights reserved.
+# Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -25,7 +25,7 @@
 means that this component is either not installed or is unable to be
 used on your system (e.g., sometimes this means that shared libraries
 that the component requires are unable to be found/loaded).  Note that
-PMIX stopped checking at the first component that it did not find.
+PMIx stopped checking at the first component that it did not find.
 
 Host:      %s
 Framework: %s
@@ -35,7 +35,7 @@
 No components were able to be opened in the %s framework.
 
 This typically means that either no components of this type were
-installed, or none of the installed componnets can be loaded.
+installed, or none of the installed components can be loaded.
 Sometimes this means that shared libraries required by these
 components are unable to be found/loaded.
 
@@ -63,7 +63,7 @@
 #
 [failed to add component dir]
 The pmix_mca_base_component_path MCA variable was used to add paths to
-search for PMIX components.  At least one directory failed to add
+search for PMIx components.  At least one directory failed to add
 properly:
 
     %s
diff -Nru pmix-3.2.2~rc1/src/mca/base/pmix_mca_base_component_find.c pmix-4.0.0/src/mca/base/pmix_mca_base_component_find.c
--- pmix-3.2.2~rc1/src/mca/base/pmix_mca_base_component_find.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/base/pmix_mca_base_component_find.c	2021-01-02 08:56:17.000000000 +0000
@@ -338,13 +338,12 @@
             }
         }
 
-        if (!found) {
+        if (!found && pmix_mca_base_component_show_load_errors) {
             char h[PMIX_MAXHOSTNAMELEN] = {0};
             gethostname(h, sizeof(h)-1);
             pmix_show_help("help-pmix-mca-base.txt",
                            "find-available:not-valid", true,
                            h, framework->framework_name, requested_component_names[i]);
-            return PMIX_ERR_NOT_FOUND;
         }
     }
 
diff -Nru pmix-3.2.2~rc1/src/mca/base/pmix_mca_base_var.c pmix-4.0.0/src/mca/base/pmix_mca_base_var.c
--- pmix-3.2.2~rc1/src/mca/base/pmix_mca_base_var.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/base/pmix_mca_base_var.c	2021-01-02 08:56:17.000000000 +0000
@@ -221,7 +221,7 @@
 {
     int i, count;
 
-    (void) pmix_argv_append_unique_nosize(&pmix_mca_base_var_file_list, filename, false);
+    (void) pmix_argv_append_unique_nosize(&pmix_mca_base_var_file_list, filename);
 
     count = pmix_argv_count(pmix_mca_base_var_file_list);
 
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/base.h pmix-4.0.0/src/mca/bfrops/base/base.h
--- pmix-3.2.2~rc1/src/mca/bfrops/base/base.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -470,9 +470,43 @@
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_envar(pmix_pointer_array_t *regtypes,
                                                       pmix_buffer_t *buffer, const void *src,
                                                       int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_coord(pmix_pointer_array_t *regtypes,
+                                                      pmix_buffer_t *buffer, const void *src,
+                                                      int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_regattr(pmix_pointer_array_t *regtypes,
+                                                        pmix_buffer_t *buffer, const void *src,
+                                                        int32_t num_vals, pmix_data_type_t type);
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_regex(pmix_pointer_array_t *regtypes,
                                                       pmix_buffer_t *buffer, const void *src,
                                                       int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_jobstate(pmix_pointer_array_t *regtypes,
+                                                         pmix_buffer_t *buffer, const void *src,
+                                                         int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_linkstate(pmix_pointer_array_t *regtypes,
+                                                          pmix_buffer_t *buffer, const void *src,
+                                                          int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_cpuset(pmix_pointer_array_t *regtypes,
+                                                       pmix_buffer_t *buffer, const void *src,
+                                                       int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_geometry(pmix_pointer_array_t *regtypes,
+                                                         pmix_buffer_t *buffer, const void *src,
+                                                         int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_devdist(pmix_pointer_array_t *regtypes,
+                                                        pmix_buffer_t *buffer, const void *src,
+                                                        int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_endpoint(pmix_pointer_array_t *regtypes,
+                                                         pmix_buffer_t *buffer, const void *src,
+                                                         int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_topology(pmix_pointer_array_t *regtypes,
+                                                         pmix_buffer_t *buffer, const void *src,
+                                                         int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_devtype(pmix_pointer_array_t *regtypes,
+                                                        pmix_buffer_t *buffer, const void *src,
+                                                        int32_t num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_pack_locality(pmix_pointer_array_t *regtypes,
+                                                         pmix_buffer_t *buffer, const void *src,
+                                                         int32_t num_vals, pmix_data_type_t type);
+
 
 /*
 * "Standard" unpack functions
@@ -604,9 +638,42 @@
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_envar(pmix_pointer_array_t *regtypes,
                                                         pmix_buffer_t *buffer, void *dest,
                                                         int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_coord(pmix_pointer_array_t *regtypes,
+                                                        pmix_buffer_t *buffer, void *dest,
+                                                        int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_regattr(pmix_pointer_array_t *regtypes,
+                                                          pmix_buffer_t *buffer, void *dest,
+                                                          int32_t *num_vals, pmix_data_type_t type);
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_regex(pmix_pointer_array_t *regtypes,
                                                         pmix_buffer_t *buffer, void *dest,
                                                         int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_jobstate(pmix_pointer_array_t *regtypes,
+                                                           pmix_buffer_t *buffer, void *dest,
+                                                           int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_linkstate(pmix_pointer_array_t *regtypes,
+                                                            pmix_buffer_t *buffer, void *dest,
+                                                            int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_cpuset(pmix_pointer_array_t *regtypes,
+                                                         pmix_buffer_t *buffer, void *dest,
+                                                         int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_geometry(pmix_pointer_array_t *regtypes,
+                                                           pmix_buffer_t *buffer, void *dest,
+                                                           int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_devdist(pmix_pointer_array_t *regtypes,
+                                                          pmix_buffer_t *buffer, void *dest,
+                                                          int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_endpoint(pmix_pointer_array_t *regtypes,
+                                                           pmix_buffer_t *buffer, void *dest,
+                                                           int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_topology(pmix_pointer_array_t *regtypes,
+                                                           pmix_buffer_t *buffer, void *dest,
+                                                           int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_devtype(pmix_pointer_array_t *regtypes,
+                                                          pmix_buffer_t *buffer, void *dest,
+                                                          int32_t *num_vals, pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_locality(pmix_pointer_array_t *regtypes,
+                                                           pmix_buffer_t *buffer, void *dest,
+                                                           int32_t *num_vals, pmix_data_type_t type);
 
 /**** DEPRECATED ****/
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_unpack_array(pmix_pointer_array_t *regtypes,
@@ -667,9 +734,30 @@
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_envar(pmix_envar_t **dest,
                                                       pmix_envar_t *src,
                                                       pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_coord(pmix_coord_t **dest,
+                                                      pmix_coord_t *src,
+                                                      pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_regattr(pmix_regattr_t **dest,
+                                                        pmix_regattr_t *src,
+                                                        pmix_data_type_t type);
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_regex(char **dest,
                                                       char *src,
                                                       pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_cpuset(pmix_cpuset_t **dest,
+                                                       pmix_cpuset_t *src,
+                                                       pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_geometry(pmix_geometry_t **dest,
+                                                         pmix_geometry_t *src,
+                                                         pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_devdist(pmix_device_distance_t **dest,
+                                                        pmix_device_distance_t *src,
+                                                        pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_endpoint(pmix_endpoint_t **dest,
+                                                         pmix_endpoint_t *src,
+                                                         pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_topology(pmix_topology_t **dest,
+                                                         pmix_topology_t *src,
+                                                         pmix_data_type_t type);
 
 /*
 * "Standard" print functions
@@ -778,9 +866,43 @@
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_envar(char **output, char *prefix,
                                                        pmix_envar_t *src,
                                                        pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_coord(char **output, char *prefix,
+                                                       pmix_coord_t *src,
+                                                       pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_regattr(char **output, char *prefix,
+                                                         pmix_regattr_t *src,
+                                                         pmix_data_type_t type);
 PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_regex(char **output, char *prefix,
                                                        char *src,
                                                        pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_jobstate(char **output, char *prefix,
+                                                          pmix_job_state_t *src,
+                                                          pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_linkstate(char **output, char *prefix,
+                                                           pmix_link_state_t *src,
+                                                           pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_cpuset(char **output, char *prefix,
+                                                        pmix_cpuset_t *src,
+                                                        pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_geometry(char **output, char *prefix,
+                                                          pmix_geometry_t *src,
+                                                          pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_devdist(char **output, char *prefix,
+                                                         pmix_device_distance_t *src,
+                                                         pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_endpoint(char **output, char *prefix,
+                                                          pmix_endpoint_t *src,
+                                                          pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_topology(char **output, char *prefix,
+                                                          pmix_topology_t *src,
+                                                          pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_devtype(char **output, char *prefix,
+                                                         pmix_device_type_t *src,
+                                                         pmix_data_type_t type);
+PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_locality(char **output, char *prefix,
+                                                          pmix_locality_t *src,
+                                                          pmix_data_type_t type);
+
 
 
 /*
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_copy.c pmix-4.0.0/src/mca/bfrops/base/bfrop_base_copy.c
--- pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_copy.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/bfrop_base_copy.c	2021-01-02 08:56:17.000000000 +0000
@@ -27,6 +27,7 @@
 #include "src/util/output.h"
 #include "src/include/pmix_globals.h"
 #include "src/mca/preg/preg.h"
+#include "src/mca/ploc/ploc.h"
 
 #include "src/mca/bfrops/base/base.h"
 
@@ -109,12 +110,14 @@
     case PMIX_BYTE:
     case PMIX_INT8:
     case PMIX_UINT8:
+    case PMIX_LINK_STATE:
         datasize = 1;
         break;
 
     case PMIX_INT16:
     case PMIX_UINT16:
     case PMIX_IOF_CHANNEL:
+    case PMIX_LOCTYPE:
         datasize = 2;
         break;
 
@@ -125,6 +128,7 @@
 
     case PMIX_INT64:
     case PMIX_UINT64:
+    case PMIX_DEVTYPE:
         datasize = 8;
         break;
 
@@ -180,6 +184,10 @@
         datasize = sizeof(pmix_alloc_directive_t);
         break;
 
+    case PMIX_JOB_STATE:
+        datasize = sizeof(pmix_job_state_t);
+        break;
+
     default:
         return PMIX_ERR_UNKNOWN_DATA_TYPE;
     }
@@ -346,7 +354,8 @@
                                        pmix_byte_object_t *src,
                                        pmix_data_type_t type)
 {
-    if (PMIX_BYTE_OBJECT != type) {
+    if (PMIX_BYTE_OBJECT != type &&
+        PMIX_COMPRESSED_BYTE_OBJECT != type) {
         return PMIX_ERR_BAD_PARAM;
     }
     *dest = (pmix_byte_object_t*)malloc(sizeof(pmix_byte_object_t));
@@ -400,6 +409,22 @@
     return PMIX_SUCCESS;
 }
 
+static pmix_status_t fill_coord(pmix_coord_t *dst,
+                                pmix_coord_t *src)
+{
+    dst->view = src->view;
+    dst->dims = src->dims;
+    if (0 < dst->dims) {
+        dst->coord = (uint32_t*)malloc(dst->dims * sizeof(uint32_t));
+        if (NULL == dst->coord) {
+            return PMIX_ERR_NOMEM;
+        }
+        memcpy(dst->coord, src->coord, dst->dims * sizeof(uint32_t));
+    }
+    return PMIX_SUCCESS;
+}
+
+
 /* the pmix_data_array_t is a little different in that it
  * is an array of values, and so we cannot just copy one
  * value at a time. So handle all value types here */
@@ -421,6 +446,11 @@
     pmix_proc_info_t *pi, *si;
     pmix_query_t *pq, *sq;
     pmix_envar_t *pe, *se;
+    pmix_regattr_t *pr, *sr;
+    pmix_coord_t *pc, *sc;
+    pmix_cpuset_t *pcpuset, *scpuset;
+    pmix_geometry_t *pgeoset, *sgeoset;
+    pmix_device_distance_t *pdevdist, *sdevdist;
 
     if (PMIX_DATA_ARRAY != type) {
         return PMIX_ERR_BAD_PARAM;
@@ -838,6 +868,112 @@
                 pe[n].separator = se[n].separator;
             }
             break;
+        case PMIX_COORD:
+            p->array = malloc(src->size * sizeof(pmix_coord_t));
+            if (NULL == p->array) {
+                free(p);
+                return PMIX_ERR_NOMEM;
+            }
+            pc = (pmix_coord_t*)p->array;
+            sc = (pmix_coord_t*)src->array;
+            for (n=0; n < src->size; n++) {
+                rc = fill_coord(&pc[n], &sc[n]);
+                if (PMIX_SUCCESS != rc) {
+                    PMIX_COORD_FREE(pc, src->size);
+                    free(p);
+                    return rc;
+                }
+            }
+            break;
+        case PMIX_REGATTR:
+            PMIX_REGATTR_CREATE(p->array, src->size);
+            if (NULL == p->array) {
+                free(p);
+                return PMIX_ERR_NOMEM;
+            }
+            pr = (pmix_regattr_t*)p->array;
+            sr = (pmix_regattr_t*)src->array;
+            for (n=0; n < src->size; n++) {
+                if (NULL != sr[n].name) {
+                    pr[n].name = strdup(sr[n].name);
+                }
+                PMIX_LOAD_KEY(pr[n].string, sr[n].string);
+                pr[n].type = sr[n].type;
+                pr[n].description = pmix_argv_copy(sr[n].description);
+            }
+            break;
+        case PMIX_PROC_CPUSET:
+            PMIX_CPUSET_CREATE(p->array, src->size);
+            if (NULL == p->array) {
+                free(p);
+                return PMIX_ERR_NOMEM;
+            }
+            pcpuset = (pmix_cpuset_t*)p->array;
+            scpuset = (pmix_cpuset_t*)src->array;
+            for (n=0; n < src->size; n++) {
+                rc = pmix_ploc.copy_cpuset(&pcpuset[n], &scpuset[n]);
+                if (PMIX_SUCCESS != rc) {
+                    pmix_ploc.release_cpuset(pcpuset, src->size);
+                    free(p->array);
+                    free(p);
+                    return rc;
+                }
+            }
+            break;
+        case PMIX_GEOMETRY:
+            PMIX_GEOMETRY_CREATE(p->array, src->size);
+            if (NULL == p->array) {
+                free(p);
+                return PMIX_ERR_NOMEM;
+            }
+            pgeoset = (pmix_geometry_t*)p->array;
+            sgeoset = (pmix_geometry_t*)src->array;
+            for (n=0; n < src->size; n++) {
+                pgeoset[n].fabric = sgeoset[n].fabric;
+                if (NULL != sgeoset[n].uuid) {
+                    pgeoset[n].uuid = strdup(sgeoset[n].uuid);
+                }
+                if (NULL != sgeoset[n].osname) {
+                    pgeoset[n].osname = strdup(sgeoset[n].osname);
+                }
+                if (NULL != sgeoset[n].coordinates) {
+                    pgeoset[n].ncoords = sgeoset[n].ncoords;
+                    pgeoset[n].coordinates = (pmix_coord_t*)malloc(pgeoset[n].ncoords * sizeof(pmix_coord_t));
+                    if (NULL == pgeoset[n].coordinates) {
+                        free(p);
+                        return PMIX_ERR_NOMEM;
+                    }
+                    for (m=0; m < pgeoset[n].ncoords; m++) {
+                        rc = fill_coord(&pgeoset[n].coordinates[m], &sgeoset[n].coordinates[m]);
+                        if (PMIX_SUCCESS != rc) {
+                            PMIX_GEOMETRY_FREE(pgeoset, src->size);
+                            free(p);
+                            return rc;
+                        }
+                    }
+                }
+            }
+            break;
+        case PMIX_DEVICE_DIST:
+            PMIX_DEVICE_DIST_CREATE(p->array, src->size);
+            if (NULL == p->array) {
+                free(p);
+                return PMIX_ERR_NOMEM;
+            }
+            pdevdist = (pmix_device_distance_t*)p->array;
+            sdevdist = (pmix_device_distance_t*)src->array;
+            for (n=0; n < src->size; n++) {
+                if (NULL != sdevdist[n].uuid) {
+                    pdevdist[n].uuid = strdup(sdevdist[n].uuid);
+                }
+                if (NULL != sdevdist[n].osname) {
+                    pdevdist[n].osname = strdup(sdevdist[n].osname);
+                }
+                pdevdist[n].type = sdevdist[n].type;
+                pdevdist[n].mindist = sdevdist[n].mindist;
+                pdevdist[n].maxdist = sdevdist[n].maxdist;
+            }
+            break;
         default:
             free(p);
             return PMIX_ERR_UNKNOWN_DATA_TYPE;
@@ -891,6 +1027,51 @@
     return PMIX_SUCCESS;
 }
 
+pmix_status_t pmix_bfrops_base_copy_coord(pmix_coord_t **dest,
+                                          pmix_coord_t *src,
+                                          pmix_data_type_t type)
+{
+    pmix_coord_t *d;
+    pmix_status_t rc;
+
+    if (PMIX_COORD != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    d = (pmix_coord_t*)malloc(sizeof(pmix_coord_t));
+    if (NULL == d) {
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_COORD_CONSTRUCT(d);
+    rc = fill_coord(d, src);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_COORD_DESTRUCT(d);
+        free(d);
+    } else {
+        *dest = d;
+    }
+    return rc;
+}
+
+pmix_status_t pmix_bfrops_base_copy_regattr(pmix_regattr_t **dest,
+                                            pmix_regattr_t *src,
+                                            pmix_data_type_t type)
+{
+    if (PMIX_REGATTR != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_REGATTR_CREATE(*dest, 1);
+    if (NULL == (*dest)) {
+        return PMIX_ERR_NOMEM;
+    }
+    if (NULL != src->name) {
+        (*dest)->name = strdup(src->name);
+    }
+    PMIX_LOAD_KEY((*dest)->string, src->string);
+    (*dest)->type = src->type;
+    (*dest)->description = pmix_argv_copy(src->description);
+    return PMIX_SUCCESS;
+}
+
 pmix_status_t pmix_bfrops_base_copy_regex(char **dest,
                                           char *src,
                                           pmix_data_type_t type)
@@ -903,3 +1084,145 @@
 
     return pmix_preg.copy(dest, &len, src);
 }
+
+pmix_status_t pmix_bfrops_base_copy_cpuset(pmix_cpuset_t **dest,
+                                           pmix_cpuset_t *src,
+                                           pmix_data_type_t type)
+{
+    pmix_cpuset_t *dst;
+    pmix_status_t rc;
+
+    if (PMIX_PROC_CPUSET != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_CPUSET_CREATE(dst, 1);
+    if (NULL == dst) {
+        return PMIX_ERR_NOMEM;
+    }
+
+    rc = pmix_ploc.copy_cpuset(dst, src);
+    if (PMIX_SUCCESS == rc) {
+        *dest = dst;
+    } else {
+        free(dst);
+    }
+    return rc;
+}
+
+pmix_status_t pmix_bfrops_base_copy_geometry(pmix_geometry_t **dest,
+                                             pmix_geometry_t *src,
+                                             pmix_data_type_t type)
+{
+    pmix_geometry_t *dst;
+    pmix_status_t rc;
+    size_t n;
+
+    if (PMIX_GEOMETRY != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_GEOMETRY_CREATE(dst, 1);
+    if (NULL == dst) {
+        return PMIX_ERR_NOMEM;
+    }
+
+    dst->fabric = src->fabric;
+    if (NULL != src->uuid) {
+        dst->uuid = strdup(src->uuid);
+    }
+    if (NULL != src->osname) {
+        dst->osname = strdup(src->osname);
+    }
+    if (NULL != src->coordinates) {
+        dst->ncoords = src->ncoords;
+        dst->coordinates = (pmix_coord_t*)calloc(dst->ncoords, sizeof(pmix_coord_t));
+        for (n=0; n < dst->ncoords; n++) {
+            rc = fill_coord(&dst->coordinates[n], &src->coordinates[n]);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_GEOMETRY_FREE(dst, 1);
+                return rc;
+            }
+        }
+    }
+
+    *dest = dst;
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_copy_devdist(pmix_device_distance_t **dest,
+                                            pmix_device_distance_t *src,
+                                            pmix_data_type_t type)
+{
+    pmix_device_distance_t *dst;
+
+    if (PMIX_DEVICE_DIST != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_DEVICE_DIST_CREATE(dst, 1);
+    if (NULL == dst) {
+        return PMIX_ERR_NOMEM;
+    }
+
+    if (NULL != src->uuid) {
+        dst->uuid = strdup(src->uuid);
+    }
+    if (NULL != src->osname) {
+        dst->osname = strdup(src->osname);
+    }
+    dst->type = src->type;
+    dst->mindist = src->mindist;
+    dst->maxdist = src->maxdist;
+
+    *dest = dst;
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_copy_endpoint(pmix_endpoint_t **dest,
+                                             pmix_endpoint_t *src,
+                                             pmix_data_type_t type)
+{
+    pmix_endpoint_t *dst;
+
+    if (PMIX_ENDPOINT != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_ENDPOINT_CREATE(dst, 1);
+    if (NULL == dst) {
+        return PMIX_ERR_NOMEM;
+    }
+
+    if (NULL != src->uuid) {
+        dst->uuid = strdup(src->uuid);
+    }
+    if (NULL != src->endpt.bytes) {
+        dst->endpt.bytes = (char*)malloc(src->endpt.size);
+        memcpy(dst->endpt.bytes, src->endpt.bytes, src->endpt.size);
+        dst->endpt.size = src->endpt.size;
+    }
+
+    *dest = dst;
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_copy_topology(pmix_topology_t **dest,
+                                             pmix_topology_t *src,
+                                             pmix_data_type_t type)
+{
+    pmix_topology_t *dst;
+    pmix_status_t rc;
+
+    if (PMIX_TOPO != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_TOPOLOGY_CREATE(dst, 1);
+    if (NULL == dst) {
+        return PMIX_ERR_NOMEM;
+    }
+
+    rc = pmix_ploc.copy_topology(dst, src);
+    if (PMIX_SUCCESS == rc) {
+        *dest = dst;
+    } else {
+        free(dst);
+    }
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_fns.c pmix-4.0.0/src/mca/bfrops/base/bfrop_base_fns.c
--- pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_fns.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/bfrop_base_fns.c	2021-01-02 08:56:17.000000000 +0000
@@ -62,6 +62,13 @@
     pmix_envar_t *envar;
     pmix_data_array_t *darray;
     pmix_status_t rc;
+    pmix_coord_t *coord;
+    pmix_regattr_t *regattr;
+    pmix_topology_t *topo;
+    pmix_cpuset_t *cpuset;
+    pmix_geometry_t *geometry;
+    pmix_endpoint_t *endpoint;
+    pmix_device_distance_t *devdist;
 
     v->type = type;
     if (NULL == data) {
@@ -146,6 +153,8 @@
             memcpy(v->data.proc, data, sizeof(pmix_proc_t));
             break;
         case PMIX_BYTE_OBJECT:
+        case PMIX_COMPRESSED_STRING:
+        case PMIX_COMPRESSED_BYTE_OBJECT:
             bo = (pmix_byte_object_t*)data;
             v->data.bo.bytes = (char*)malloc(bo->size);
             if (NULL == v->data.bo.bytes) {
@@ -184,9 +193,19 @@
             memcpy(&(v->data.pinfo->pid), &pi->pid, sizeof(pid_t));
             memcpy(&(v->data.pinfo->exit_code), &pi->exit_code, sizeof(int));
             break;
+        case PMIX_DATA_ARRAY:
+            darray = (pmix_data_array_t*)data;
+            rc = pmix_bfrops_base_copy_darray(&v->data.darray, darray, PMIX_DATA_ARRAY);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            break;
         case PMIX_POINTER:
             v->data.ptr = (void*)data;
             break;
+        case PMIX_ALLOC_DIRECTIVE:
+            memcpy(&(v->data.adir), data, sizeof(pmix_alloc_directive_t));
+            break;
         case PMIX_ENVAR:
             envar = (pmix_envar_t*)data;
             if (NULL != envar->envar) {
@@ -197,9 +216,67 @@
             }
             v->data.envar.separator = envar->separator;
             break;
-        case PMIX_DATA_ARRAY:
-            darray = (pmix_data_array_t*)data;
-            rc = pmix_bfrops_base_copy_darray(&v->data.darray, darray, PMIX_DATA_ARRAY);
+        case PMIX_COORD:
+            coord = (pmix_coord_t*)data;
+            rc = pmix_bfrops_base_copy_coord(&v->data.coord, coord, PMIX_COORD);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            break;
+        case PMIX_LINK_STATE:
+            memcpy(&(v->data.linkstate), data, sizeof(pmix_link_state_t));
+            break;
+        case PMIX_JOB_STATE:
+            memcpy(&(v->data.jstate), data, sizeof(pmix_job_state_t));
+            break;
+        case PMIX_TOPO:
+            topo = (pmix_topology_t*)data;
+            rc = pmix_bfrops_base_copy_topology(&v->data.topo, topo, PMIX_TOPO);
+            if (PMIX_ERR_INIT == rc || PMIX_ERR_NOT_SUPPORTED == rc) {
+                /* we are not initialized yet, so just copy the pointer */
+                v->data.topo = topo;
+                rc = PMIX_SUCCESS;
+            }
+            break;
+        case PMIX_PROC_CPUSET:
+            cpuset = (pmix_cpuset_t*)data;
+            rc = pmix_bfrops_base_copy_cpuset(&v->data.cpuset, cpuset, PMIX_PROC_CPUSET);
+            if (PMIX_ERR_INIT == rc || PMIX_ERR_NOT_SUPPORTED == rc) {
+                /* we are not initialized yet, so just copy the pointer */
+                v->data.cpuset = cpuset;
+                rc = PMIX_SUCCESS;
+            }
+            break;
+        case PMIX_LOCTYPE:
+            memcpy(&(v->data.locality), data, sizeof(pmix_locality_t));
+            break;
+        case PMIX_GEOMETRY:
+            geometry = (pmix_geometry_t*)data;
+            rc = pmix_bfrops_base_copy_geometry(&v->data.geometry, geometry, PMIX_GEOMETRY);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            break;
+        case PMIX_DEVTYPE:
+            memcpy(&(v->data.devtype), data, sizeof(pmix_device_type_t));
+            break;
+        case PMIX_DEVICE_DIST:
+            devdist = (pmix_device_distance_t*)data;
+            rc = pmix_bfrops_base_copy_devdist(&v->data.devdist, devdist, PMIX_DEVICE_DIST);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            break;
+        case PMIX_ENDPOINT:
+            endpoint = (pmix_endpoint_t*)data;
+            rc = pmix_bfrops_base_copy_endpoint(&v->data.endpoint, endpoint, PMIX_ENDPOINT);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            break;
+        case PMIX_REGATTR:
+            regattr = (pmix_regattr_t*)data;
+            rc = pmix_bfrops_base_copy_regattr((pmix_regattr_t**)&v->data.ptr, regattr, PMIX_REGATTR);
             if (PMIX_SUCCESS != rc) {
                 PMIX_ERROR_LOG(rc);
             }
@@ -226,6 +303,7 @@
     pmix_status_t rc;
     pmix_envar_t *envar;
     pmix_data_array_t **darray;
+    pmix_regattr_t *regattr, *r;
 
     rc = PMIX_SUCCESS;
     if (NULL == data ||
@@ -314,7 +392,23 @@
             memcpy(*data, &(kv->data.time), sizeof(time_t));
             *sz = sizeof(time_t);
             break;
+        case PMIX_STATUS:
+            memcpy(*data, &(kv->data.status), sizeof(pmix_status_t));
+            *sz = sizeof(pmix_status_t);
+            break;
+        case PMIX_PROC_RANK:
+            memcpy(*data, &(kv->data.rank), sizeof(pmix_rank_t));
+            *sz = sizeof(pmix_rank_t);
+            break;
+        case PMIX_PROC:
+            rc = pmix_bfrops_base_copy_proc((pmix_proc_t**)data, kv->data.proc, PMIX_PROC);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_proc_t);
+            }
+            break;
         case PMIX_BYTE_OBJECT:
+        case PMIX_COMPRESSED_STRING:
+        case PMIX_COMPRESSED_BYTE_OBJECT:
             if (NULL != kv->data.bo.bytes && 0 < kv->data.bo.size) {
                 *data = kv->data.bo.bytes;
                 *sz = kv->data.bo.size;
@@ -339,14 +433,26 @@
             memcpy(*data, &(kv->data.state), sizeof(pmix_proc_state_t));
             *sz = sizeof(pmix_proc_state_t);
             break;
-        case PMIX_POINTER:
-            *data = (void*)kv->data.ptr;
-            *sz = sizeof(void*);
+        case PMIX_PROC_INFO:
+            rc = pmix_bfrops_base_copy_pinfo((pmix_proc_info_t**)data, kv->data.pinfo, PMIX_PROC_INFO);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_proc_info_t);
+            }
             break;
         case PMIX_DATA_ARRAY:
             darray = (pmix_data_array_t**)data;
             rc = pmix_bfrops_base_copy_darray(darray, kv->data.darray, PMIX_DATA_ARRAY);
-            *sz = sizeof(pmix_data_array_t);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_data_array_t);
+            }
+            break;
+        case PMIX_POINTER:
+            *data = (void*)kv->data.ptr;
+            *sz = sizeof(void*);
+            break;
+        case PMIX_ALLOC_DIRECTIVE:
+            memcpy(*data, &(kv->data.adir), sizeof(pmix_alloc_directive_t));
+            *sz = sizeof(pmix_alloc_directive_t);
             break;
         case PMIX_ENVAR:
             PMIX_ENVAR_CREATE(envar, 1);
@@ -363,6 +469,83 @@
             *data = envar;
             *sz = sizeof(pmix_envar_t);
             break;
+        case PMIX_COORD:
+            rc = pmix_bfrops_base_copy_coord((pmix_coord_t**)data, kv->data.coord, PMIX_COORD);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_coord_t);
+            }
+            break;
+        case PMIX_LINK_STATE:
+            memcpy(*data, &(kv->data.linkstate), sizeof(pmix_link_state_t));
+            *sz = sizeof(pmix_link_state_t);
+            break;
+        case PMIX_JOB_STATE:
+            memcpy(*data, &(kv->data.jstate), sizeof(pmix_job_state_t));
+            *sz = sizeof(pmix_job_state_t);
+            break;
+        case PMIX_TOPO:
+            rc = pmix_bfrops_base_copy_topology((pmix_topology_t**)data, kv->data.topo, PMIX_TOPO);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_topology_t);
+            } else if (PMIX_ERR_INIT == rc || PMIX_ERR_NOT_SUPPORTED == rc) {
+                *data = malloc(sizeof(pmix_topology_t));
+                memcpy(*data, kv->data.topo, sizeof(pmix_topology_t));
+                *sz = sizeof(pmix_topology_t);
+                rc = PMIX_SUCCESS;
+            }
+            break;
+        case PMIX_PROC_CPUSET:
+            rc = pmix_bfrops_base_copy_cpuset((pmix_cpuset_t**)data, kv->data.cpuset, PMIX_PROC_CPUSET);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_cpuset_t);
+            } else if (PMIX_ERR_INIT == rc || PMIX_ERR_NOT_SUPPORTED == rc) {
+                *data = malloc(sizeof(pmix_cpuset_t));
+                memcpy(*data, kv->data.cpuset, sizeof(pmix_cpuset_t));
+                *sz = sizeof(pmix_cpuset_t);
+                rc = PMIX_SUCCESS;
+            }
+            break;
+        case PMIX_LOCTYPE:
+            memcpy(*data, &(kv->data.locality), sizeof(pmix_locality_t));
+            *sz = sizeof(pmix_locality_t);
+            break;
+        case PMIX_GEOMETRY:
+            rc = pmix_bfrops_base_copy_geometry((pmix_geometry_t**)data, kv->data.geometry, PMIX_GEOMETRY);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_geometry_t);
+            }
+            break;
+        case PMIX_DEVTYPE:
+            memcpy(*data, &(kv->data.devtype), sizeof(pmix_device_type_t));
+            *sz = sizeof(pmix_device_type_t);
+            break;
+        case PMIX_DEVICE_DIST:
+            rc = pmix_bfrops_base_copy_devdist((pmix_device_distance_t**)data, kv->data.devdist, PMIX_DEVICE_DIST);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_device_distance_t);
+            }
+            break;
+        case PMIX_ENDPOINT:
+            rc = pmix_bfrops_base_copy_endpoint((pmix_endpoint_t**)data, kv->data.endpoint, PMIX_ENDPOINT);
+            if (PMIX_SUCCESS == rc) {
+                *sz = sizeof(pmix_endpoint_t);
+            }
+            break;
+        case PMIX_REGATTR:
+            PMIX_REGATTR_CREATE(regattr, 1);
+            if (NULL == regattr) {
+                return PMIX_ERR_NOMEM;
+            }
+            r = (pmix_regattr_t*)kv->data.ptr;
+            if (NULL != r->name) {
+                regattr->name = strdup(r->name);
+            }
+            PMIX_LOAD_KEY(regattr->string, r->string);
+            regattr->type = r->type;
+            regattr->description = pmix_argv_copy(r->description);
+            *data = regattr;
+            *sz = sizeof(pmix_regattr_t);
+            break;
         case PMIX_REGEX:
             if (NULL != kv->data.bo.bytes && 0 < kv->data.bo.size) {
                 *data = kv->data.bo.bytes;
@@ -519,6 +702,26 @@
             }
             rc = PMIX_EQUAL;
             break;
+        case PMIX_COORD:
+            ret = memcmp(p->data.coord, p1->data.coord, sizeof(pmix_coord_t));
+            if (0 > ret) {
+                return PMIX_VALUE2_GREATER;
+            } else if (0 < ret) {
+                return PMIX_VALUE1_GREATER;
+            } else {
+                return PMIX_EQUAL;
+            }
+            break;
+        case PMIX_REGATTR:
+            ret = memcmp(p->data.ptr, p1->data.ptr, sizeof(pmix_regattr_t));
+            if (0 > ret) {
+                return PMIX_VALUE2_GREATER;
+            } else if (0 < ret) {
+                return PMIX_VALUE1_GREATER;
+            } else {
+                return PMIX_EQUAL;
+            }
+            break;
 
         default:
             pmix_output(0, "COMPARE-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)p->type);
@@ -530,6 +733,8 @@
 pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p,
                                           const pmix_value_t *src)
 {
+    pmix_status_t rc;
+
     /* copy the right field */
     p->type = src->type;
     switch (src->type) {
@@ -607,6 +812,9 @@
     case PMIX_STATUS:
         memcpy(&p->data.status, &src->data.status, sizeof(pmix_status_t));
         break;
+    case PMIX_PROC_RANK:
+        memcpy(&p->data.rank, &src->data.rank, sizeof(pmix_rank_t));
+        break;
     case PMIX_PROC:
         PMIX_PROC_CREATE(p->data.proc, 1);
         if (NULL == p->data.proc) {
@@ -614,12 +822,10 @@
         }
         memcpy(p->data.proc, src->data.proc, sizeof(pmix_proc_t));
         break;
-    case PMIX_PROC_RANK:
-        memcpy(&p->data.rank, &src->data.rank, sizeof(pmix_rank_t));
-        break;
     case PMIX_BYTE_OBJECT:
     case PMIX_COMPRESSED_STRING:
     case PMIX_REGEX:
+    case PMIX_COMPRESSED_BYTE_OBJECT:
         memset(&p->data.bo, 0, sizeof(pmix_byte_object_t));
         if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) {
             p->data.bo.bytes = malloc(src->data.bo.size);
@@ -649,6 +855,9 @@
     case PMIX_POINTER:
         p->data.ptr = src->data.ptr;
         break;
+    case PMIX_ALLOC_DIRECTIVE:
+        memcpy(&p->data.adir, &src->data.adir, sizeof(pmix_alloc_directive_t));
+        break;
     case PMIX_ENVAR:
         PMIX_ENVAR_CONSTRUCT(&p->data.envar);
         if (NULL != src->data.envar.envar) {
@@ -659,7 +868,44 @@
         }
         p->data.envar.separator = src->data.envar.separator;
         break;
-
+    case PMIX_COORD:
+        return pmix_bfrops_base_copy_coord(&p->data.coord, src->data.coord, PMIX_COORD);
+    case PMIX_LINK_STATE:
+        memcpy(&p->data.linkstate, &src->data.linkstate, sizeof(pmix_link_state_t));
+        break;
+    case PMIX_JOB_STATE:
+        memcpy(&p->data.jstate, &src->data.jstate, sizeof(pmix_job_state_t));
+        break;
+    case PMIX_TOPO:
+        rc = pmix_bfrops_base_copy_topology(&p->data.topo, src->data.topo, PMIX_TOPO);
+        if (PMIX_ERR_INIT == rc || PMIX_ERR_NOT_SUPPORTED == rc) {
+            /* we are being asked to do this before init, so
+             * just copy the pointer across */
+            p->data.topo = src->data.topo;
+        }
+        break;
+    case PMIX_PROC_CPUSET:
+        rc = pmix_bfrops_base_copy_cpuset(&p->data.cpuset, src->data.cpuset, PMIX_PROC_CPUSET);
+        if (PMIX_ERR_INIT == rc || PMIX_ERR_NOT_SUPPORTED == rc) {
+            /* we are being asked to do this before init, so
+             * just copy the pointer across */
+            p->data.cpuset = src->data.cpuset;
+        }
+        break;
+    case PMIX_LOCTYPE:
+        memcpy(&p->data.locality, &src->data.locality, sizeof(pmix_locality_t));
+        break;
+    case PMIX_GEOMETRY:
+        return pmix_bfrops_base_copy_geometry(&p->data.geometry, src->data.geometry, PMIX_GEOMETRY);
+    case PMIX_DEVTYPE:
+        memcpy(&p->data.devtype, &src->data.devtype, sizeof(pmix_device_type_t));
+        break;
+    case PMIX_DEVICE_DIST:
+        return pmix_bfrops_base_copy_devdist(&p->data.devdist, src->data.devdist, PMIX_DEVICE_DIST);
+    case PMIX_ENDPOINT:
+        return pmix_bfrops_base_copy_endpoint(&p->data.endpoint, src->data.endpoint, PMIX_ENDPOINT);
+    case PMIX_REGATTR:
+        return pmix_bfrops_base_copy_regattr((pmix_regattr_t**)&p->data.ptr, src->data.ptr, PMIX_REGATTR);
     default:
         pmix_output(0, "PMIX-XFER-VALUE: UNSUPPORTED TYPE %d", (int)src->type);
         return PMIX_ERROR;
@@ -778,3 +1024,82 @@
 
     return info->odti_name;
 }
+
+PMIX_EXPORT void* pmix_info_list_start(void)
+{
+    pmix_list_t *p;
+
+    p = PMIX_NEW(pmix_list_t);
+    return p;
+}
+
+PMIX_EXPORT pmix_status_t pmix_info_list_add(void *ptr,
+                                             pmix_key_t key,
+                                             void *value,
+                                             pmix_data_type_t type)
+{
+    pmix_list_t *p = (pmix_list_t*)ptr;
+    pmix_infolist_t *iptr;
+
+    iptr = PMIX_NEW(pmix_infolist_t);
+    if (NULL == iptr) {
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_INFO_LOAD(&iptr->info, key, value, type);
+    pmix_list_append(p, &iptr->super);
+    return PMIX_SUCCESS;
+}
+
+PMIX_EXPORT pmix_status_t pmix_info_list_xfer(void *ptr,
+                                              const pmix_info_t *info)
+{
+    pmix_list_t *p = (pmix_list_t*)ptr;
+    pmix_infolist_t *iptr;
+
+    iptr = PMIX_NEW(pmix_infolist_t);
+    if (NULL == iptr) {
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_INFO_XFER(&iptr->info, info);
+    pmix_list_append(p, &iptr->super);
+    return PMIX_SUCCESS;
+}
+
+PMIX_EXPORT pmix_status_t pmix_info_list_convert(void *ptr, pmix_data_array_t *par)
+{
+    pmix_list_t *p = (pmix_list_t*)ptr;
+    size_t n;
+    pmix_infolist_t *iptr;
+    pmix_info_t *array;
+
+    if (NULL == par || NULL == ptr) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    n = pmix_list_get_size(p);
+    if (0 == n) {
+        return PMIX_ERR_EMPTY;
+    }
+    PMIX_INFO_CREATE(par->array, n);
+    if (NULL == par->array) {
+        return PMIX_ERR_NOMEM;
+    }
+    par->type = PMIX_INFO;
+    par->size = n;
+    array = (pmix_info_t*)par->array;
+
+    /* transfer the elements across */
+    n=0;
+    PMIX_LIST_FOREACH(iptr, p, pmix_infolist_t) {
+        PMIX_INFO_XFER(&array[n], &iptr->info);
+        ++n;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+PMIX_EXPORT void pmix_info_list_release(void *ptr)
+{
+    pmix_list_t *p = (pmix_list_t*)ptr;
+    PMIX_LIST_RELEASE(p);
+}
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_pack.c pmix-4.0.0/src/mca/bfrops/base/bfrop_base_pack.c
--- pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_pack.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/bfrop_base_pack.c	2021-01-02 08:56:17.000000000 +0000
@@ -33,6 +33,7 @@
 #include "src/util/output.h"
 #include "src/include/pmix_globals.h"
 #include "src/mca/preg/preg.h"
+#include "src/mca/ploc/ploc.h"
 
 #include "src/mca/bfrops/base/base.h"
 
@@ -575,7 +576,8 @@
     if (NULL == regtypes) {
         return PMIX_ERR_BAD_PARAM;
     }
-    if (PMIX_BYTE_OBJECT != type) {
+    if (PMIX_BYTE_OBJECT != type &&
+        PMIX_COMPRESSED_BYTE_OBJECT != type) {
         return PMIX_ERR_BAD_PARAM;
     }
     bo = (pmix_byte_object_t*)src;
@@ -1145,9 +1147,16 @@
     switch (p->type) {
         case PMIX_UNDEF:
             break;
+        case PMIX_PROC:
         case PMIX_PROC_INFO:
         case PMIX_DATA_ARRAY:
-        case PMIX_PROC:
+        case PMIX_COORD:
+        case PMIX_TOPO:
+        case PMIX_PROC_CPUSET:
+        case PMIX_GEOMETRY:
+        case PMIX_DEVICE_DIST:
+        case PMIX_ENDPOINT:
+        case PMIX_REGATTR:
             PMIX_BFROPS_PACK_TYPE(ret, buffer, p->data.ptr, 1, p->type, regtypes);
             if (PMIX_SUCCESS != ret) {
                 return ret;
@@ -1232,6 +1241,89 @@
     return PMIX_SUCCESS;
 }
 
+pmix_status_t pmix_bfrops_base_pack_coord(pmix_pointer_array_t *regtypes,
+                                          pmix_buffer_t *buffer, const void *src,
+                                          int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_coord_t *ptr = (pmix_coord_t*)src;
+    int32_t i;
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_COORD != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    for (i=0; i < num_vals; ++i) {
+        /* pack the view */
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].view, 1, PMIX_UINT8, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* pack the number of dimensions */
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].dims, 1, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* pack the array of coordinates */
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, ptr[i].coord, ptr[i].dims, PMIX_UINT32, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_pack_regattr(pmix_pointer_array_t *regtypes,
+                                            pmix_buffer_t *buffer, const void *src,
+                                            int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_regattr_t *ptr = (pmix_regattr_t*)src;
+    int32_t i, nd;
+    pmix_status_t ret;
+    char *foo;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_REGATTR != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    for (i=0; i < num_vals; ++i) {
+        /* pack the name */
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].name, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* pack the string */
+        foo = ptr[i].string;
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &foo, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* pack the type */
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].type, 1, PMIX_DATA_TYPE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        /* pack the description */
+        nd = pmix_argv_count(ptr[i].description);
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &nd, 1, PMIX_INT32, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, ptr[i].description, nd, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
 pmix_status_t pmix_bfrops_base_pack_regex(pmix_pointer_array_t *regtypes,
                                           pmix_buffer_t *buffer, const void *src,
                                           int32_t num_vals, pmix_data_type_t type)
@@ -1254,3 +1346,232 @@
     }
     return PMIX_SUCCESS;
 }
+
+pmix_status_t pmix_bfrops_base_pack_jobstate(pmix_pointer_array_t *regtypes,
+                                             pmix_buffer_t *buffer, const void *src,
+                                             int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_JOB_STATE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes);
+    return ret;
+}
+
+pmix_status_t pmix_bfrops_base_pack_linkstate(pmix_pointer_array_t *regtypes,
+                                              pmix_buffer_t *buffer, const void *src,
+                                              int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_LINK_STATE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes);
+    return ret;
+}
+
+pmix_status_t pmix_bfrops_base_pack_cpuset(pmix_pointer_array_t *regtypes,
+                                           pmix_buffer_t *buffer, const void *src,
+                                           int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_cpuset_t *ptr = (pmix_cpuset_t*)src;
+    int32_t i;
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_PROC_CPUSET != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    for (i=0; i < num_vals; ++i) {
+        ret = pmix_ploc.pack_cpuset(buffer, &ptr[i], regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_pack_geometry(pmix_pointer_array_t *regtypes,
+                                             pmix_buffer_t *buffer, const void *src,
+                                             int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_geometry_t *ptr = (pmix_geometry_t*)src;
+    int32_t i;
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_GEOMETRY != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    for (i=0; i < num_vals; ++i) {
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].fabric, 1, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].uuid, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].osname, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].ncoords, 1, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, ptr[i].coordinates, ptr[i].ncoords, PMIX_COORD, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_pack_devdist(pmix_pointer_array_t *regtypes,
+                                            pmix_buffer_t *buffer, const void *src,
+                                            int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_device_distance_t *ptr = (pmix_device_distance_t*)src;
+    int32_t i;
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_DEVICE_DIST != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    for (i=0; i < num_vals; ++i) {
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].uuid, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].osname, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].type, 1, PMIX_DEVTYPE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].mindist, 1, PMIX_UINT16, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].maxdist, 1, PMIX_UINT16, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_pack_endpoint(pmix_pointer_array_t *regtypes,
+                                             pmix_buffer_t *buffer, const void *src,
+                                             int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_endpoint_t *ptr = (pmix_endpoint_t*)src;
+    int32_t i;
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_ENDPOINT != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    for (i=0; i < num_vals; ++i) {
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].uuid, 1, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].endpt.size, 1, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        if (0 < ptr[i].endpt.size) {
+            PMIX_BFROPS_PACK_TYPE(ret, buffer, ptr[i].endpt.bytes, ptr[i].endpt.size, PMIX_BYTE, regtypes);
+            if (PMIX_SUCCESS != ret) {
+                return ret;
+            }
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_pack_topology(pmix_pointer_array_t *regtypes,
+                                             pmix_buffer_t *buffer, const void *src,
+                                             int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_topology_t *ptr = (pmix_topology_t*)src;
+    int32_t i;
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_TOPO != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    for (i=0; i < num_vals; ++i) {
+        /* call the framework to pack it */
+        ret = pmix_ploc.pack_topology(buffer, &ptr[i], regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_pack_devtype(pmix_pointer_array_t *regtypes,
+                                            pmix_buffer_t *buffer, const void *src,
+                                            int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_DEVTYPE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT64, regtypes);
+    return ret;
+}
+
+pmix_status_t pmix_bfrops_base_pack_locality(pmix_pointer_array_t *regtypes,
+                                             pmix_buffer_t *buffer, const void *src,
+                                             int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (NULL == regtypes) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    if (PMIX_LOCTYPE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT16, regtypes);
+    return ret;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_print.c pmix-4.0.0/src/mca/bfrops/base/bfrop_base_print.c
--- pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_print.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/bfrop_base_print.c	2021-01-02 08:56:17.000000000 +0000
@@ -31,7 +31,7 @@
 #endif
 
 #include "src/util/error.h"
-
+#include "src/mca/ploc/ploc.h"
 #include "src/include/pmix_globals.h"
 #include "src/mca/bfrops/base/base.h"
 
@@ -935,6 +935,8 @@
  {
     char *prefx;
     int rc;
+    pmix_regattr_t *r;
+    char *tp;
 
     if (PMIX_VALUE != type) {
         return PMIX_ERR_BAD_PARAM;
@@ -1077,13 +1079,61 @@
             rc = asprintf(output, "%sPMIX_VALUE: Data type: DATA_ARRAY\tARRAY SIZE: %ld",
                           prefx, (long)src->data.darray->size);
             break;
+        case PMIX_REGATTR:
+            r = (pmix_regattr_t*)src->data.ptr;
+            rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_REGATTR\tName: %s\tString: %s",
+                          prefx, (NULL == r->name) ? "NULL" : r->name,
+                          (0 == strlen(r->string)) ? "NULL" : r->string);
+            break;
+        case PMIX_ALLOC_DIRECTIVE:
+            rc = pmix_bfrops_base_print_alloc_directive(output, prefx, &src->data.adir, PMIX_ALLOC_DIRECTIVE);
+            break;
         case PMIX_ENVAR:
             rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_ENVAR\tName: %s\tValue: %s\tSeparator: %c",
                           prefx, (NULL == src->data.envar.envar) ? "NULL" : src->data.envar.envar,
                           (NULL == src->data.envar.value) ? "NULL" : src->data.envar.value,
                           src->data.envar.separator);
             break;
-
+        case PMIX_COORD:
+            if (PMIX_COORD_VIEW_UNDEF == src->data.coord->view) {
+                tp = "UNDEF";
+            } else if (PMIX_COORD_LOGICAL_VIEW == src->data.coord->view) {
+                tp = "LOGICAL";
+            } else if (PMIX_COORD_PHYSICAL_VIEW == src->data.coord->view) {
+                tp = "PHYSICAL";
+            } else {
+                tp = "UNRECOGNIZED";
+            }
+            rc = asprintf(output, "%sPMIX_VALUE: Data type: PMIX_COORD\tView: %s\tDims: %lu",
+                          prefx, tp, (unsigned long)src->data.coord->dims);
+            break;
+        case PMIX_LINK_STATE:
+            rc = pmix_bfrops_base_print_linkstate(output, prefx, &src->data.linkstate, PMIX_LINK_STATE);
+            break;
+        case PMIX_JOB_STATE:
+            rc = pmix_bfrops_base_print_jobstate(output, prefx, &src->data.jstate, PMIX_JOB_STATE);
+            break;
+        case PMIX_TOPO:
+            rc = pmix_bfrops_base_print_topology(output, prefx, src->data.topo, PMIX_TOPO);
+            break;
+        case PMIX_PROC_CPUSET:
+            rc = pmix_bfrops_base_print_cpuset(output, prefx, src->data.cpuset, PMIX_PROC_CPUSET);
+            break;
+        case PMIX_LOCTYPE:
+            rc = pmix_bfrops_base_print_locality(output, prefx, &src->data.locality, PMIX_LOCTYPE);
+            break;
+        case PMIX_GEOMETRY:
+            rc = pmix_bfrops_base_print_geometry(output, prefx, src->data.geometry, PMIX_GEOMETRY);
+            break;
+        case PMIX_DEVTYPE:
+            rc = pmix_bfrops_base_print_devtype(output, prefx, &src->data.devtype, PMIX_DEVTYPE);
+            break;
+        case PMIX_DEVICE_DIST:
+            rc = pmix_bfrops_base_print_devdist(output, prefx, src->data.devdist, PMIX_DEVICE_DIST);
+            break;
+        case PMIX_ENDPOINT:
+            rc = pmix_bfrops_base_print_endpoint(output, prefx, src->data.endpoint, PMIX_ENDPOINT);
+            break;
         default:
             rc = asprintf(output, "%sPMIX_VALUE: Data type: UNKNOWN\tValue: UNPRINTABLE", prefx);
             break;
@@ -1424,7 +1474,8 @@
     char *prefx;
     int ret;
 
-    if (PMIX_BYTE_OBJECT != type) {
+    if (PMIX_BYTE_OBJECT != type &&
+        PMIX_COMPRESSED_BYTE_OBJECT != type) {
         return PMIX_ERR_BAD_PARAM;
     }
     /* deal with NULL prefix */
@@ -1438,7 +1489,8 @@
 
     /* if src is NULL, just print data type and return */
     if (NULL == src) {
-        ret = asprintf(output, "%sData type: PMIX_BYTE_OBJECT\tValue: NULL pointer", prefx);
+        ret = asprintf(output, "%sData type: %s\tValue: NULL pointer", prefx,
+                       (PMIX_COMPRESSED_BYTE_OBJECT == type) ? "PMIX_COMPRESSED_BYTE_OBJECT" : "PMIX_BYTE_OBJECT");
         if (prefx != prefix) {
             free(prefx);
         }
@@ -1449,7 +1501,9 @@
         }
     }
 
-    ret = asprintf(output, "%sData type: PMIX_BYTE_OBJECT\tSize: %ld", prefx, (long)src->size);
+    ret = asprintf(output, "%sData type: %s\tSize: %ld", prefx,
+                   (PMIX_COMPRESSED_BYTE_OBJECT == type) ? "PMIX_COMPRESSED_BYTE_OBJECT" : "PMIX_BYTE_OBJECT",
+                   (long)src->size);
     if (prefx != prefix) {
         free(prefx);
     }
@@ -1823,6 +1877,82 @@
     }
 }
 
+pmix_status_t pmix_bfrops_base_print_coord(char **output, char *prefix,
+                                           pmix_coord_t *src,
+                                           pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+    char *tp;
+
+    if (PMIX_COORD != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    if (PMIX_COORD_VIEW_UNDEF == src->view) {
+        tp = "UNDEF";
+    } else if (PMIX_COORD_LOGICAL_VIEW == src->view) {
+        tp = "LOGICAL";
+    } else if (PMIX_COORD_PHYSICAL_VIEW == src->view) {
+        tp = "PHYSICAL";
+    } else {
+        tp = "UNRECOGNIZED";
+    }
+    ret = asprintf(output, "%sData type: PMIX_COORD\tView: %s\tDims: %lu",
+                   prefx, tp, (unsigned long)src->dims);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_regattr(char **output, char *prefix,
+                                             pmix_regattr_t *src,
+                                             pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    if (PMIX_REGATTR != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_REGATTR\tName: %s\tString: %s",
+                   prefx, (NULL == src->name) ? "NULL" : src->name,
+                   (0 == strlen(src->string)) ? "NULL" : src->string);
+
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
 pmix_status_t pmix_bfrops_base_print_regex(char **output, char *prefix,
                                            char *src,
                                            pmix_data_type_t type)
@@ -1850,6 +1980,383 @@
 
     if (0 > ret) {
         return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_jobstate(char **output, char *prefix,
+                                              pmix_job_state_t *src,
+                                              pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    if (PMIX_JOB_STATE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_JOB_STATE\tValue: %s",
+                   prefx, PMIx_Job_state_string(*src));
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_linkstate(char **output, char *prefix,
+                                               pmix_link_state_t *src,
+                                               pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    if (PMIX_LINK_STATE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_LINK_STATE\tValue: %s",
+                   prefx, PMIx_Link_state_string(*src));
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_cpuset(char **output, char *prefix,
+                                            pmix_cpuset_t *src,
+                                            pmix_data_type_t type)
+{
+    char *prefx, *string;
+    int ret;
+
+    if (PMIX_PROC_CPUSET != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    string = pmix_ploc.print_cpuset(src);
+    if (NULL == string) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            free(string);
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_CPUSET\tValue: %s",
+                   prefx, string);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+    free(string);
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_geometry(char **output, char *prefix,
+                                              pmix_geometry_t *src,
+                                              pmix_data_type_t type)
+{
+    char *prefx, *tmp, *pfx2, **result = NULL;
+    int ret;
+    size_t n;
+
+    if (PMIX_GEOMETRY != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+    if (0 > asprintf(&pfx2, "%s\t", prefx)) {
+        if (prefx != prefix) {
+            free(prefx);
+        }
+        return PMIX_ERR_NOMEM;
+    }
+
+    ret = asprintf(&tmp, "%sData type: PMIX_GEOMETRY\tValue: Fabric: %"PRIsize_t" UUID: %s OSName: %s",
+                   prefx, src->fabric, (NULL == src->uuid) ? "NULL" : src->uuid,
+                   (NULL == src->osname) ? "NULL" : src->osname);
+    if (0 > ret) {
+        if (prefx != prefix) {
+            free(prefx);
+        }
+        free(pfx2);
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    }
+    pmix_argv_append_nosize(&result, tmp);
+    free(tmp);
+
+    for (n=0; n < src->ncoords; n++) {
+        ret = pmix_bfrops_base_print_coord(&tmp, pfx2, &src->coordinates[n], PMIX_COORD);
+        if (PMIX_SUCCESS != ret) {
+            if (prefx != prefix) {
+                free(prefx);
+            }
+            free(pfx2);
+            if (NULL != result) {
+                pmix_argv_free(result);
+            }
+            return ret;
+        }
+        pmix_argv_append_nosize(&result, tmp);
+        free(tmp);
+    }
+
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    *output = pmix_argv_join(result, '\n');
+    pmix_argv_free(result);
+
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_print_devdist(char **output, char *prefix,
+                                             pmix_device_distance_t *src,
+                                             pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    if (PMIX_DEVICE_DIST != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_DEVICE_DIST\tValue: UUID: %s OSName: %s Type: %s Min: %u Max: %u",
+                   prefx, (NULL == src->uuid) ? "NULL" : src->uuid,
+                   (NULL == src->osname) ? "NULL" : src->osname,
+                   PMIx_Device_type_string(src->type),
+                   (unsigned)src->mindist, (unsigned)src->maxdist);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_endpoint(char **output, char *prefix,
+                                              pmix_endpoint_t *src,
+                                              pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    if (PMIX_ENDPOINT != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_ENDPOINT\tValue: %s #bytes: %"PRIsize_t,
+                   prefx, (NULL == src->uuid) ? "NULL" : src->uuid, src->endpt.size);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_topology(char **output, char *prefix,
+                                              pmix_topology_t *src,
+                                              pmix_data_type_t type)
+{
+    char *prefx, *string;
+    int ret;
+
+    if (PMIX_TOPO != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    string = pmix_ploc.print_topology(src);
+    if (NULL == string) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            free(string);
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_TOPO\tValue: %s",
+                   prefx, string);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+    free(string);
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_devtype(char **output, char *prefix,
+                                             pmix_device_type_t *src,
+                                             pmix_data_type_t type)
+{
+    char *prefx;
+    int ret;
+
+    if (PMIX_DEVTYPE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_DEVICE_TYPE\tValue: 0x%"PRIx64,
+                   prefx, (uint64_t)src);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    } else {
+        return PMIX_SUCCESS;
+    }
+}
+
+pmix_status_t pmix_bfrops_base_print_locality(char **output, char *prefix,
+                                              pmix_locality_t *src,
+                                              pmix_data_type_t type)
+{
+    char *prefx, **tmp = NULL, *str;
+    int ret;
+
+    if (PMIX_LOCTYPE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) {
+        if (0 > asprintf(&prefx, " ")) {
+            return PMIX_ERR_NOMEM;
+        }
+    } else {
+        prefx = prefix;
+    }
+
+    if (PMIX_LOCALITY_UNKNOWN == *src) {
+        str = strdup("UNKNOWN");
+    } else if (PMIX_LOCALITY_NONLOCAL == *src) {
+        str = strdup("NONLOCAL");
+    } else {
+        if (PMIX_LOCALITY_SHARE_HWTHREAD & *src) {
+            pmix_argv_append_nosize(&tmp, "HWTHREAD");
+        }
+        if (PMIX_LOCALITY_SHARE_CORE & *src) {
+            pmix_argv_append_nosize(&tmp, "CORE");
+        }
+        if (PMIX_LOCALITY_SHARE_L1CACHE & *src) {
+            pmix_argv_append_nosize(&tmp, "L1");
+        }
+        if (PMIX_LOCALITY_SHARE_L2CACHE & *src) {
+            pmix_argv_append_nosize(&tmp, "L2");
+        }
+        if (PMIX_LOCALITY_SHARE_L3CACHE & *src) {
+            pmix_argv_append_nosize(&tmp, "L3");
+        }
+        if (PMIX_LOCALITY_SHARE_PACKAGE & *src) {
+            pmix_argv_append_nosize(&tmp, "CORE");
+        }
+        if (PMIX_LOCALITY_SHARE_NUMA & *src) {
+            pmix_argv_append_nosize(&tmp, "NUMA");
+        }
+        if (PMIX_LOCALITY_SHARE_NODE & *src) {
+            pmix_argv_append_nosize(&tmp, "NODE");
+        }
+        str = pmix_argv_join(tmp, ':');
+        pmix_argv_free(tmp);
+    }
+
+    ret = asprintf(output, "%sData type: PMIX_LOCALITY\tValue: %s",
+                   prefx, str);
+    if (prefx != prefix) {
+        free(prefx);
+    }
+    free(str);
+
+    if (0 > ret) {
+        return PMIX_ERR_OUT_OF_RESOURCE;
     } else {
         return PMIX_SUCCESS;
     }
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_stubs.c pmix-4.0.0/src/mca/bfrops/base/bfrop_base_stubs.c
--- pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_stubs.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/bfrop_base_stubs.c	2021-01-02 08:56:17.000000000 +0000
@@ -124,6 +124,26 @@
             return "PMIX_IOF_CHANNEL";
         case PMIX_ENVAR:
             return "PMIX_ENVAR";
+        case PMIX_COORD:
+            return "PMIX_COORD";
+        case PMIX_REGATTR:
+            return "PMIX_REGATTR";
+        case PMIX_JOB_STATE:
+            return "PMIX_JOB_STATE";
+        case PMIX_LINK_STATE:
+            return "PMIX_LINK_STATE";
+        case PMIX_PROC_CPUSET:
+            return "PMIX_PROC_CPUSET";
+        case PMIX_GEOMETRY:
+            return "PMIX_GEOMETRY";
+        case PMIX_DEVICE_DIST:
+            return "PMIX_DEVICE_DIST";
+        case PMIX_ENDPOINT:
+            return "PMIX_ENDPOINT";
+        case PMIX_TOPO:
+            return "PMIX_TOPO";
+        case PMIX_DEVTYPE:
+            return "PMIX_DEVTYPE";
         default:
             return "NOT INITIALIZED";
     }
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_unpack.c pmix-4.0.0/src/mca/bfrops/base/bfrop_base_unpack.c
--- pmix-3.2.2~rc1/src/mca/bfrops/base/bfrop_base_unpack.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/base/bfrop_base_unpack.c	2021-01-02 08:56:17.000000000 +0000
@@ -29,6 +29,7 @@
 #include "src/util/output.h"
 #include "src/include/pmix_globals.h"
 #include "src/mca/preg/preg.h"
+#include "src/mca/ploc/ploc.h"
 
 #include "src/mca/bfrops/bfrops_types.h"
 #include "src/mca/bfrops/base/base.h"
@@ -656,6 +657,52 @@
             }
             PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.darray, &m, PMIX_DATA_ARRAY, regtypes);
             break;
+        case PMIX_REGATTR:
+            val->data.ptr = (pmix_regattr_t*)calloc(1, sizeof(pmix_regattr_t));
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.ptr, &m, PMIX_REGATTR, regtypes);
+            return ret;
+        case PMIX_COORD:
+            val->data.coord = (pmix_coord_t*)calloc(1, sizeof(pmix_coord_t));
+            if (NULL == val->data.coord) {
+                return PMIX_ERR_NOMEM;
+            }
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.coord, &m, PMIX_COORD, regtypes);
+            return ret;
+        case PMIX_TOPO:
+            PMIX_TOPOLOGY_CREATE(val->data.topo, 1);
+            if (NULL == val->data.topo) {
+                return PMIX_ERR_NOMEM;
+            }
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.topo, &m, PMIX_TOPO, regtypes);
+            return ret;
+        case PMIX_PROC_CPUSET:
+            PMIX_CPUSET_CREATE(val->data.cpuset, 1);
+            if (NULL == val->data.cpuset) {
+                return PMIX_ERR_NOMEM;
+            }
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.cpuset, &m, PMIX_PROC_CPUSET, regtypes);
+            return ret;
+        case PMIX_GEOMETRY:
+            PMIX_GEOMETRY_CREATE(val->data.geometry, 1);
+            if (NULL == val->data.geometry) {
+                return PMIX_ERR_NOMEM;
+            }
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.geometry, &m, PMIX_GEOMETRY, regtypes);
+            return ret;
+        case PMIX_DEVICE_DIST:
+            PMIX_DEVICE_DIST_CREATE(val->data.devdist, 1);
+            if (NULL == val->data.devdist) {
+                return PMIX_ERR_NOMEM;
+            }
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.devdist, &m, PMIX_DEVICE_DIST, regtypes);
+            return ret;
+        case PMIX_ENDPOINT:
+            PMIX_ENDPOINT_CREATE(val->data.endpoint, 1);
+            if (NULL == val->data.endpoint) {
+                return PMIX_ERR_NOMEM;
+            }
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, val->data.endpoint, &m, PMIX_ENDPOINT, regtypes);
+            return ret;
         default:
             PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &val->data, &m, val->type, regtypes);
             if (PMIX_ERR_UNKNOWN_DATA_TYPE == ret) {
@@ -1076,7 +1123,8 @@
     pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
                         "pmix_bfrop_unpack: %d byte_object", *num_vals);
 
-    if (PMIX_BYTE_OBJECT != type) {
+    if (PMIX_BYTE_OBJECT != type &&
+        PMIX_COMPRESSED_BYTE_OBJECT != type) {
         return PMIX_ERR_BAD_PARAM;
     }
 
@@ -1446,6 +1494,121 @@
     return PMIX_SUCCESS;
 }
 
+pmix_status_t pmix_bfrops_base_unpack_coord(pmix_pointer_array_t *regtypes,
+                                            pmix_buffer_t *buffer, void *dest,
+                                            int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_coord_t *ptr;
+    int32_t i, n, m;
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d coordinates", *num_vals);
+
+    if (PMIX_COORD != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_coord_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        PMIX_COORD_CONSTRUCT(&ptr[i]);
+        /* unpack the view */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].view, &m, PMIX_UINT8, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* unpack the dims */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].dims, &m, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        if (0 < ptr[i].dims) {
+            ptr[i].coord = (uint32_t*)malloc(ptr[i].dims * sizeof(uint32_t));
+            /* unpack the coords */
+            m=ptr[i].dims;
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, ptr[i].coord, &m, PMIX_UINT32, regtypes);
+            if (PMIX_SUCCESS != ret) {
+                return ret;
+            }
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_regattr(pmix_pointer_array_t *regtypes,
+                                              pmix_buffer_t *buffer, void *dest,
+                                              int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_regattr_t *ptr;
+    int32_t i, n, m, nd;
+    pmix_status_t ret;
+    char *tmp;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d regattrs", *num_vals);
+
+    if (PMIX_REGATTR != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_regattr_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        PMIX_REGATTR_CONSTRUCT(&ptr[i]);
+        /* unpack the name */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].name, &m, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* unpack the string */
+        m=1;
+        tmp = NULL;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &tmp, &m, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        if (NULL == tmp) {
+            return PMIX_ERROR;
+        }
+        pmix_strncpy(ptr[i].string, tmp, PMIX_MAX_KEYLEN);
+        free(tmp);
+        /* unpack the type */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].type, &m, PMIX_DATA_TYPE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        /* unpack the description */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &nd, &m, PMIX_INT32, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        if (0 < nd) {
+            /* unpack the description */
+            if (NULL == (ptr[i].description = (char**)calloc(nd+1, sizeof(char*)))) {
+                return PMIX_ERR_NOMEM;
+            }
+            m=nd;
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, ptr[i].description, &m, PMIX_STRING, regtypes);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+                return ret;
+            }
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
 pmix_status_t pmix_bfrops_base_unpack_regex(pmix_pointer_array_t *regtypes,
                                             pmix_buffer_t *buffer, void *dest,
                                             int32_t *num_vals, pmix_data_type_t type)
@@ -1465,11 +1628,294 @@
     n = *num_vals;
 
     for (i = 0; i < n; ++i) {
-        ret = pmix_preg.unpack(buffer, &ptr[n]);
+        ret = pmix_preg.unpack(buffer, &ptr[i]);
+        if (PMIX_SUCCESS != ret) {
+            *num_vals = 0;
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_jobstate(pmix_pointer_array_t *regtypes,
+                                               pmix_buffer_t *buffer, void *dest,
+                                               int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (PMIX_JOB_STATE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes);
+    return ret;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_linkstate(pmix_pointer_array_t *regtypes,
+                                                pmix_buffer_t *buffer, void *dest,
+                                                int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (PMIX_LINK_STATE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes);
+    return ret;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_cpuset(pmix_pointer_array_t *regtypes,
+                                            pmix_buffer_t *buffer, void *dest,
+                                            int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_cpuset_t *ptr;
+    int32_t i, n;
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d cpuset", *num_vals);
+
+    if (PMIX_PROC_CPUSET != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_cpuset_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        ret = pmix_ploc.unpack_cpuset(buffer, &ptr[i], regtypes);
+        if (PMIX_SUCCESS != ret) {
+            *num_vals = 0;
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_geometry(pmix_pointer_array_t *regtypes,
+                                               pmix_buffer_t *buffer, void *dest,
+                                               int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_geometry_t *ptr;
+    int32_t i, n, m;
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d geometry", *num_vals);
+
+    if (PMIX_GEOMETRY != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_geometry_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        PMIX_GEOMETRY_CONSTRUCT(&ptr[i]);
+        /* unpack the fabric id */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].fabric, &m, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            return ret;
+        }
+        /* unpack the uuid */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].uuid, &m, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        /* unpack the osname */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].osname, &m, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        /* get the number of coords */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].ncoords, &m, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        if (0 < ptr[i].ncoords) {
+            /* allocate the coords */
+            ptr[i].coordinates = (pmix_coord_t*)calloc(ptr[i].ncoords, sizeof(pmix_coord_t));
+            /* unpack them */
+            m=ptr[i].ncoords;
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].coordinates, &m, PMIX_COORD, regtypes);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+                return ret;
+            }
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_devdist(pmix_pointer_array_t *regtypes,
+                                              pmix_buffer_t *buffer, void *dest,
+                                              int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_device_distance_t *ptr;
+    int32_t i, n, m;
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d device distances", *num_vals);
+
+    if (PMIX_DEVICE_DIST != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_device_distance_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        PMIX_DEVICE_DIST_CONSTRUCT(&ptr[i]);
+        /* unpack the uuid */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].uuid, &m, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].osname, &m, PMIX_STRING, regtypes);
         if (PMIX_SUCCESS != ret) {
-            *num_vals = n;
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].type, &m, PMIX_DEVTYPE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].mindist, &m, PMIX_UINT16, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].maxdist, &m, PMIX_UINT16, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
             return ret;
         }
     }
     return PMIX_SUCCESS;
 }
+
+pmix_status_t pmix_bfrops_base_unpack_endpoint(pmix_pointer_array_t *regtypes,
+                                               pmix_buffer_t *buffer, void *dest,
+                                               int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_endpoint_t *ptr;
+    int32_t i, n, m;
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d endpts", *num_vals);
+
+    if (PMIX_ENDPOINT != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_endpoint_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        PMIX_ENDPOINT_CONSTRUCT(&ptr[i]);
+        /* unpack the uuid */
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].uuid, &m, PMIX_STRING, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        m=1;
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &ptr[i].endpt.size, &m, PMIX_SIZE, regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        if (0 < ptr[i].endpt.size) {
+            ptr[i].endpt.bytes = (char*)malloc(ptr[i].endpt.size);
+            m = ptr[i].endpt.size;
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, ptr[i].endpt.bytes, &m, PMIX_BYTE, regtypes);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+                return ret;
+            }
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_topology(pmix_pointer_array_t *regtypes,
+                                               pmix_buffer_t *buffer, void *dest,
+                                               int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_topology_t *ptr;
+    int32_t i, n;
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d topology", *num_vals);
+
+    if (PMIX_TOPO != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    ptr = (pmix_topology_t *) dest;
+    n = *num_vals;
+
+    for (i = 0; i < n; ++i) {
+        ret = pmix_ploc.unpack_topology(buffer, &ptr[i], regtypes);
+        if (PMIX_SUCCESS != ret) {
+            *num_vals = 0;
+            return ret;
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_devtype(pmix_pointer_array_t *regtypes,
+                                              pmix_buffer_t *buffer, void *dest,
+                                              int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d device types", *num_vals);
+
+    if (PMIX_DEVTYPE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_BFROPS_UNPACK_TYPE(ret, buffer,dest, num_vals, PMIX_UINT64, regtypes);
+
+    return ret;
+}
+
+pmix_status_t pmix_bfrops_base_unpack_locality(pmix_pointer_array_t *regtypes,
+                                               pmix_buffer_t *buffer, void *dest,
+                                               int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrop_unpack: %d locality", *num_vals);
+
+    if (PMIX_LOCTYPE != type) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    PMIX_BFROPS_UNPACK_TYPE(ret, buffer,dest, num_vals, PMIX_UINT16, regtypes);
+
+    return ret;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/bfrops_types.h pmix-4.0.0/src/mca/bfrops/bfrops_types.h
--- pmix-3.2.2~rc1/src/mca/bfrops/bfrops_types.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/bfrops_types.h	2021-01-02 08:56:17.000000000 +0000
@@ -45,6 +45,19 @@
     PMIX_VALUE2_GREATER
 } pmix_value_cmp_t;
 
+/* set the bfrops module */
+#define PMIX_BFROPS_SET_MODULE(r, mp, p, v)                                 \
+    do {                                                                    \
+        (mp)->nptr->compat.bfrops = pmix_bfrops_base_assign_module((v));    \
+        if (NULL == (mp)->nptr->compat.bfrops) {                            \
+            (r) = PMIX_ERR_INIT;                                            \
+        } else {                                                            \
+            (p)->nptr->compat.bfrops = (mp)->nptr->compat.bfrops;           \
+            (mp)->protocol = PMIX_PROTOCOL_V2;                              \
+            (r) = PMIX_SUCCESS;                                             \
+        }                                                                   \
+    } while(0)
+
 /**
  * buffer type
  */
@@ -66,6 +79,13 @@
 } pmix_kval_t;
 PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_kval_t);
 
+/* helpful macro extension of the usual PMIX_NEW */
+#define PMIX_KVAL_NEW(k, s)                                         \
+    do {                                                            \
+        (k) = PMIX_NEW(pmix_kval_t);                                \
+        (k)->key = strdup((s));                                     \
+        (k)->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));   \
+    } while(0)
 
 /**
  * Structure for holding a buffer */
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/v4/bfrop_pmix4.c pmix-4.0.0/src/mca/bfrops/v4/bfrop_pmix4.c
--- pmix-3.2.2~rc1/src/mca/bfrops/v4/bfrop_pmix4.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/v4/bfrop_pmix4.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,803 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2014 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+
+#include "src/mca/bfrops/base/base.h"
+#include "bfrop_pmix4.h"
+
+#include "src/mca/psquash/psquash.h"
+#include "src/mca/psquash/base/base.h"
+#include "src/util/error.h"
+
+static pmix_status_t init(void);
+static void finalize(void);
+static pmix_status_t pmix4_pack(pmix_buffer_t *buffer,
+                                const void *src, int num_vals,
+                                pmix_data_type_t type);
+static pmix_status_t pmix4_unpack(pmix_buffer_t *buffer, void *dest,
+                                  int32_t *num_vals, pmix_data_type_t type);
+static pmix_status_t pmix4_copy(void **dest, void *src,
+                                pmix_data_type_t type);
+static pmix_status_t pmix4_print(char **output, char *prefix,
+                                 void *src, pmix_data_type_t type);
+static pmix_status_t register_type(const char *name,
+                                   pmix_data_type_t type,
+                                   pmix_bfrop_pack_fn_t pack,
+                                   pmix_bfrop_unpack_fn_t unpack,
+                                   pmix_bfrop_copy_fn_t copy,
+                                   pmix_bfrop_print_fn_t print);
+static const char* data_type_string(pmix_data_type_t type);
+
+static pmix_status_t
+pmix4_bfrops_base_pack_general_int(pmix_pointer_array_t *regtypes,
+                                   pmix_buffer_t *buffer, const void *src,
+                                   int32_t num_vals, pmix_data_type_t type);
+static pmix_status_t
+pmix4_bfrops_base_pack_int(pmix_pointer_array_t *regtypes,
+                           pmix_buffer_t *buffer, const void *src,
+                           int32_t num_vals, pmix_data_type_t type);
+static pmix_status_t
+pmix4_bfrops_base_pack_sizet(pmix_pointer_array_t *regtypes,
+                             pmix_buffer_t *buffer, const void *src,
+                             int32_t num_vals, pmix_data_type_t type);
+static pmix_status_t
+pmix4_bfrops_base_unpack_general_int(pmix_pointer_array_t *regtypes,
+                                     pmix_buffer_t *buffer, void *dest,
+                                     int32_t *num_vals, pmix_data_type_t type);
+static pmix_status_t
+pmix4_bfrops_base_unpack_int(pmix_pointer_array_t *regtypes,
+                             pmix_buffer_t *buffer, void *dest,
+                             int32_t *num_vals, pmix_data_type_t type);
+static pmix_status_t
+pmix4_bfrops_base_unpack_sizet(pmix_pointer_array_t *regtypes,
+                               pmix_buffer_t *buffer, void *dest,
+                               int32_t *num_vals, pmix_data_type_t type);
+
+pmix_bfrops_module_t pmix_bfrops_pmix4_module = {
+    .version = "v4",
+    .init = init,
+    .finalize = finalize,
+    .pack = pmix4_pack,
+    .unpack = pmix4_unpack,
+    .copy = pmix4_copy,
+    .print = pmix4_print,
+    .copy_payload = pmix_bfrops_base_copy_payload,
+    .value_xfer = pmix_bfrops_base_value_xfer,
+    .value_load = pmix_bfrops_base_value_load,
+    .value_unload = pmix_bfrops_base_value_unload,
+    .value_cmp = pmix_bfrops_base_value_cmp,
+    .register_type = register_type,
+    .data_type_string = data_type_string
+};
+
+static pmix_status_t init(void)
+{
+    /* some standard types don't require anything special */
+    PMIX_REGISTER_TYPE("PMIX_BOOL", PMIX_BOOL,
+                       pmix_bfrops_base_pack_bool,
+                       pmix_bfrops_base_unpack_bool,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_bool,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_BYTE", PMIX_BYTE,
+                       pmix_bfrops_base_pack_byte,
+                       pmix_bfrops_base_unpack_byte,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_byte,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_STRING", PMIX_STRING,
+                       pmix_bfrops_base_pack_string,
+                       pmix_bfrops_base_unpack_string,
+                       pmix_bfrops_base_copy_string,
+                       pmix_bfrops_base_print_string,
+                       &mca_bfrops_v4_component.types);
+
+    /* Register the rest of the standard generic types to point to internal functions */
+    PMIX_REGISTER_TYPE("PMIX_SIZE", PMIX_SIZE,
+                       pmix4_bfrops_base_pack_sizet,
+                       pmix4_bfrops_base_unpack_sizet,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_size,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PID", PMIX_PID,
+                       pmix_bfrops_base_pack_pid,
+                       pmix_bfrops_base_unpack_pid,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_pid,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT", PMIX_INT,
+                       pmix4_bfrops_base_pack_int,
+                       pmix4_bfrops_base_unpack_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int,
+                       &mca_bfrops_v4_component.types);
+
+    /* Register all the standard fixed types to point to base functions */
+    PMIX_REGISTER_TYPE("PMIX_INT8", PMIX_INT8,
+                       pmix_bfrops_base_pack_byte,
+                       pmix_bfrops_base_unpack_byte,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int8,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT16", PMIX_INT16,
+                       pmix4_bfrops_base_pack_general_int,
+                       pmix4_bfrops_base_unpack_general_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int16,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT32", PMIX_INT32,
+                       pmix4_bfrops_base_pack_general_int,
+                       pmix4_bfrops_base_unpack_general_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int32,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INT64", PMIX_INT64,
+                       pmix4_bfrops_base_pack_general_int,
+                       pmix4_bfrops_base_unpack_general_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_int64,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT", PMIX_UINT,
+                       pmix4_bfrops_base_pack_int,
+                       pmix4_bfrops_base_unpack_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT8", PMIX_UINT8,
+                       pmix_bfrops_base_pack_byte,
+                       pmix_bfrops_base_unpack_byte,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint8,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT16", PMIX_UINT16,
+                       pmix4_bfrops_base_pack_general_int,
+                       pmix4_bfrops_base_unpack_general_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint16,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT32", PMIX_UINT32,
+                       pmix4_bfrops_base_pack_general_int,
+                       pmix4_bfrops_base_unpack_general_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint32,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_UINT64", PMIX_UINT64,
+                       pmix4_bfrops_base_pack_general_int,
+                       pmix4_bfrops_base_unpack_general_int,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_uint64,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_FLOAT", PMIX_FLOAT,
+                       pmix_bfrops_base_pack_float,
+                       pmix_bfrops_base_unpack_float,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_float,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DOUBLE", PMIX_DOUBLE,
+                       pmix_bfrops_base_pack_double,
+                       pmix_bfrops_base_unpack_double,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_double,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_TIMEVAL", PMIX_TIMEVAL,
+                       pmix_bfrops_base_pack_timeval,
+                       pmix_bfrops_base_unpack_timeval,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_timeval,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_TIME", PMIX_TIME,
+                       pmix_bfrops_base_pack_time,
+                       pmix_bfrops_base_unpack_time,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_time,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_STATUS", PMIX_STATUS,
+                       pmix_bfrops_base_pack_status,
+                       pmix_bfrops_base_unpack_status,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_status,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_VALUE", PMIX_VALUE,
+                       pmix_bfrops_base_pack_value,
+                       pmix_bfrops_base_unpack_value,
+                       pmix_bfrops_base_copy_value,
+                       pmix_bfrops_base_print_value,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC", PMIX_PROC,
+                       pmix_bfrops_base_pack_proc,
+                       pmix_bfrops_base_unpack_proc,
+                       pmix_bfrops_base_copy_proc,
+                       pmix_bfrops_base_print_proc,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_APP", PMIX_APP,
+                       pmix_bfrops_base_pack_app,
+                       pmix_bfrops_base_unpack_app,
+                       pmix_bfrops_base_copy_app,
+                       pmix_bfrops_base_print_app,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INFO", PMIX_INFO,
+                       pmix_bfrops_base_pack_info,
+                       pmix_bfrops_base_unpack_info,
+                       pmix_bfrops_base_copy_info,
+                       pmix_bfrops_base_print_info,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PDATA", PMIX_PDATA,
+                       pmix_bfrops_base_pack_pdata,
+                       pmix_bfrops_base_unpack_pdata,
+                       pmix_bfrops_base_copy_pdata,
+                       pmix_bfrops_base_print_pdata,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_BUFFER", PMIX_BUFFER,
+                       pmix_bfrops_base_pack_buf,
+                       pmix_bfrops_base_unpack_buf,
+                       pmix_bfrops_base_copy_buf,
+                       pmix_bfrops_base_print_buf,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_BYTE_OBJECT", PMIX_BYTE_OBJECT,
+                       pmix_bfrops_base_pack_bo,
+                       pmix_bfrops_base_unpack_bo,
+                       pmix_bfrops_base_copy_bo,
+                       pmix_bfrops_base_print_bo,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_KVAL", PMIX_KVAL,
+                       pmix_bfrops_base_pack_kval,
+                       pmix_bfrops_base_unpack_kval,
+                       pmix_bfrops_base_copy_kval,
+                       pmix_bfrops_base_print_kval,
+                       &mca_bfrops_v4_component.types);
+
+    /* these are fixed-sized values and can be done by base */
+    PMIX_REGISTER_TYPE("PMIX_PERSIST", PMIX_PERSIST,
+                       pmix_bfrops_base_pack_persist,
+                       pmix_bfrops_base_unpack_persist,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_persist,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_POINTER", PMIX_POINTER,
+                       pmix_bfrops_base_pack_ptr,
+                       pmix_bfrops_base_unpack_ptr,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_ptr,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_SCOPE", PMIX_SCOPE,
+                       pmix_bfrops_base_pack_scope,
+                       pmix_bfrops_base_unpack_scope,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_std_copy,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DATA_RANGE", PMIX_DATA_RANGE,
+                       pmix_bfrops_base_pack_range,
+                       pmix_bfrops_base_unpack_range,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_ptr,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_COMMAND", PMIX_COMMAND,
+                       pmix_bfrops_base_pack_cmd,
+                       pmix_bfrops_base_unpack_cmd,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_cmd,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_INFO_DIRECTIVES", PMIX_INFO_DIRECTIVES,
+                       pmix_bfrops_base_pack_info_directives,
+                       pmix_bfrops_base_unpack_info_directives,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_info_directives,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DATA_TYPE", PMIX_DATA_TYPE,
+                       pmix_bfrops_base_pack_datatype,
+                       pmix_bfrops_base_unpack_datatype,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_datatype,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_STATE", PMIX_PROC_STATE,
+                       pmix_bfrops_base_pack_pstate,
+                       pmix_bfrops_base_unpack_pstate,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_pstate,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_INFO", PMIX_PROC_INFO,
+                       pmix_bfrops_base_pack_pinfo,
+                       pmix_bfrops_base_unpack_pinfo,
+                       pmix_bfrops_base_copy_pinfo,
+                       pmix_bfrops_base_print_pinfo,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DATA_ARRAY", PMIX_DATA_ARRAY,
+                       pmix_bfrops_base_pack_darray,
+                       pmix_bfrops_base_unpack_darray,
+                       pmix_bfrops_base_copy_darray,
+                       pmix_bfrops_base_print_darray,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_RANK", PMIX_PROC_RANK,
+                       pmix_bfrops_base_pack_rank,
+                       pmix_bfrops_base_unpack_rank,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_rank,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_QUERY", PMIX_QUERY,
+                       pmix_bfrops_base_pack_query,
+                       pmix_bfrops_base_unpack_query,
+                       pmix_bfrops_base_copy_query,
+                       pmix_bfrops_base_print_query,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_COMPRESSED_STRING",
+                       PMIX_COMPRESSED_STRING,
+                       pmix_bfrops_base_pack_bo,
+                       pmix_bfrops_base_unpack_bo,
+                       pmix_bfrops_base_copy_bo,
+                       pmix_bfrops_base_print_bo,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_ALLOC_DIRECTIVE",
+                       PMIX_ALLOC_DIRECTIVE,
+                       pmix_bfrops_base_pack_alloc_directive,
+                       pmix_bfrops_base_unpack_alloc_directive,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_alloc_directive,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_IOF_CHANNEL",
+                       PMIX_IOF_CHANNEL,
+                       pmix_bfrops_base_pack_iof_channel,
+                       pmix_bfrops_base_unpack_iof_channel,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_iof_channel,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_ENVAR",
+                       PMIX_ENVAR,
+                       pmix_bfrops_base_pack_envar,
+                       pmix_bfrops_base_unpack_envar,
+                       pmix_bfrops_base_copy_envar,
+                       pmix_bfrops_base_print_envar,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_COORD",
+                       PMIX_COORD,
+                       pmix_bfrops_base_pack_coord,
+                       pmix_bfrops_base_unpack_coord,
+                       pmix_bfrops_base_copy_coord,
+                       pmix_bfrops_base_print_coord,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_REGATTR",
+                       PMIX_REGATTR,
+                       pmix_bfrops_base_pack_regattr,
+                       pmix_bfrops_base_unpack_regattr,
+                       pmix_bfrops_base_copy_regattr,
+                       pmix_bfrops_base_print_regattr,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_REGEX",
+                       PMIX_REGEX,
+                       pmix_bfrops_base_pack_regex,
+                       pmix_bfrops_base_unpack_regex,
+                       pmix_bfrops_base_copy_regex,
+                       pmix_bfrops_base_print_regex,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_JOB_STATE",
+                       PMIX_JOB_STATE,
+                       pmix_bfrops_base_pack_jobstate,
+                       pmix_bfrops_base_unpack_jobstate,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_jobstate,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_LINK_STATE",
+                       PMIX_LINK_STATE,
+                       pmix_bfrops_base_pack_linkstate,
+                       pmix_bfrops_base_unpack_linkstate,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_linkstate,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_PROC_CPUSET",
+                       PMIX_PROC_CPUSET,
+                       pmix_bfrops_base_pack_cpuset,
+                       pmix_bfrops_base_unpack_cpuset,
+                       pmix_bfrops_base_copy_cpuset,
+                       pmix_bfrops_base_print_cpuset,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_GEOMETRY",
+                       PMIX_GEOMETRY,
+                       pmix_bfrops_base_pack_geometry,
+                       pmix_bfrops_base_unpack_geometry,
+                       pmix_bfrops_base_copy_geometry,
+                       pmix_bfrops_base_print_geometry,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DEVICE_DIST",
+                       PMIX_DEVICE_DIST,
+                       pmix_bfrops_base_pack_devdist,
+                       pmix_bfrops_base_unpack_devdist,
+                       pmix_bfrops_base_copy_devdist,
+                       pmix_bfrops_base_print_devdist,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_ENDPOINT",
+                       PMIX_ENDPOINT,
+                       pmix_bfrops_base_pack_endpoint,
+                       pmix_bfrops_base_unpack_endpoint,
+                       pmix_bfrops_base_copy_endpoint,
+                       pmix_bfrops_base_print_endpoint,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_TOPO",
+                       PMIX_TOPO,
+                       pmix_bfrops_base_pack_topology,
+                       pmix_bfrops_base_unpack_topology,
+                       pmix_bfrops_base_copy_topology,
+                       pmix_bfrops_base_print_topology,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_DEVTYPE",
+                       PMIX_DEVTYPE,
+                       pmix_bfrops_base_pack_devtype,
+                       pmix_bfrops_base_unpack_devtype,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_devtype,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_LOCTYPE",
+                       PMIX_LOCTYPE,
+                       pmix_bfrops_base_pack_locality,
+                       pmix_bfrops_base_unpack_locality,
+                       pmix_bfrops_base_std_copy,
+                       pmix_bfrops_base_print_locality,
+                       &mca_bfrops_v4_component.types);
+
+    PMIX_REGISTER_TYPE("PMIX_COMPRESSED_BYTE_OBJECT",
+    				   PMIX_COMPRESSED_BYTE_OBJECT,
+                       pmix_bfrops_base_pack_bo,
+                       pmix_bfrops_base_unpack_bo,
+                       pmix_bfrops_base_copy_bo,
+                       pmix_bfrops_base_print_bo,
+                       &mca_bfrops_v4_component.types);
+
+    return PMIX_SUCCESS;
+}
+
+static void finalize(void)
+{
+    int n;
+    pmix_bfrop_type_info_t *info;
+
+    for (n=0; n < mca_bfrops_v4_component.types.size; n++) {
+        if (NULL != (info = (pmix_bfrop_type_info_t*)pmix_pointer_array_get_item(&mca_bfrops_v4_component.types, n))) {
+            PMIX_RELEASE(info);
+            pmix_pointer_array_set_item(&mca_bfrops_v4_component.types, n, NULL);
+        }
+    }
+}
+
+static pmix_status_t pmix4_pack(pmix_buffer_t *buffer,
+                                const void *src, int num_vals,
+                                pmix_data_type_t type)
+{
+    /* kick the process off by passing this in to the base */
+    return pmix_bfrops_base_pack(&mca_bfrops_v4_component.types,
+                                 buffer, src, num_vals, type);
+}
+
+static pmix_status_t pmix4_unpack(pmix_buffer_t *buffer, void *dest,
+                                  int32_t *num_vals, pmix_data_type_t type)
+{
+     /* kick the process off by passing this in to the base */
+    return pmix_bfrops_base_unpack(&mca_bfrops_v4_component.types,
+                                   buffer, dest, num_vals, type);
+}
+
+static pmix_status_t pmix4_copy(void **dest, void *src,
+                                pmix_data_type_t type)
+{
+    return pmix_bfrops_base_copy(&mca_bfrops_v4_component.types,
+                                 dest, src, type);
+}
+
+static pmix_status_t pmix4_print(char **output, char *prefix,
+                                 void *src, pmix_data_type_t type)
+{
+    return pmix_bfrops_base_print(&mca_bfrops_v4_component.types,
+                                  output, prefix, src, type);
+}
+
+static pmix_status_t register_type(const char *name, pmix_data_type_t type,
+                                   pmix_bfrop_pack_fn_t pack,
+                                   pmix_bfrop_unpack_fn_t unpack,
+                                   pmix_bfrop_copy_fn_t copy,
+                                   pmix_bfrop_print_fn_t print)
+{
+    PMIX_REGISTER_TYPE(name, type,
+                       pack, unpack,
+                       copy, print,
+                       &mca_bfrops_v4_component.types);
+    return PMIX_SUCCESS;
+}
+
+static const char* data_type_string(pmix_data_type_t type)
+{
+    return pmix_bfrops_base_data_type_string(&mca_bfrops_v4_component.types, type);
+}
+
+/*
+ * INT16, INT32, INT64
+ */
+static pmix_status_t
+pmix4_bfrops_base_pack_general_int(pmix_pointer_array_t *regtypes,
+                                   pmix_buffer_t *buffer, const void *src,
+                                   int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_status_t rc;
+    int32_t i;
+    char *dst;
+    size_t val_size, max_size, pkg_size;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrops_base_pack_integer * %d\n", num_vals);
+
+    PMIX_SQUASH_TYPE_SIZEOF(rc, type, val_size);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
+    rc = pmix_psquash.get_max_size(type, &max_size);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
+    /* check to see if buffer needs extending */
+    if (NULL == (dst = pmix_bfrop_buffer_extend(buffer, num_vals*max_size))) {
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
+    for (i = 0; i < num_vals; ++i) {
+        rc = (pmix_psquash.encode_int)(type, (uint8_t*)src+i*val_size,
+                                       dst, &pkg_size);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            return rc;
+        }
+        dst += pkg_size;
+        buffer->pack_ptr += pkg_size;
+        buffer->bytes_used += pkg_size;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+/*
+ * INT
+ */
+static pmix_status_t
+pmix4_bfrops_base_pack_int(pmix_pointer_array_t *regtypes,
+                           pmix_buffer_t *buffer, const void *src,
+                           int32_t num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+
+    if (false == pmix_psquash.int_type_is_encoded) {
+        /* System types need to always be described so we can properly
+           unpack them */
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_store_data_type(regtypes, buffer, BFROP_TYPE_INT))) {
+            return ret;
+        }
+    }
+
+    /* Turn around and pack the real type */
+    PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, BFROP_TYPE_INT, regtypes);
+    return ret;
+}
+
+/*
+ * SIZE_T
+ */
+static pmix_status_t
+pmix4_bfrops_base_pack_sizet(pmix_pointer_array_t *regtypes,
+                             pmix_buffer_t *buffer, const void *src,
+                             int32_t num_vals, pmix_data_type_t type)
+{
+    int ret;
+
+    if (false == pmix_psquash.int_type_is_encoded) {
+        /* System types need to always be described so we can properly
+           unpack them. */
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_store_data_type(regtypes, buffer, BFROP_TYPE_SIZE_T))) {
+            return ret;
+        }
+    }
+
+    PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, BFROP_TYPE_SIZE_T, regtypes);
+    return ret;
+}
+
+/*
+ * INT16, INT32, INT64
+ */
+static pmix_status_t
+pmix4_bfrops_base_unpack_general_int(pmix_pointer_array_t *regtypes,
+                                     pmix_buffer_t *buffer, void *dest,
+                                     int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t rc;
+    size_t val_size, avail_size, unpack_size, max_size;
+    int32_t i;
+
+    pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output,
+                        "pmix_bfrops_base_unpack_integer * %d\n", (int)*num_vals);
+
+    /* check to see if there's enough data in buffer */
+    if (buffer->pack_ptr == buffer->unpack_ptr) {
+        return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER;
+    }
+
+    PMIX_SQUASH_TYPE_SIZEOF(rc, type, val_size);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
+    rc = pmix_psquash.get_max_size(type, &max_size);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
+    /* unpack the data */
+    for (i = 0; i < (*num_vals); ++i) {
+        avail_size = buffer->pack_ptr - buffer->unpack_ptr;
+        rc = (pmix_psquash.decode_int)(type, buffer->unpack_ptr, avail_size,
+                                       (uint8_t*)dest+i*val_size, &unpack_size);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            return rc;
+        }
+        /* sanity checks */
+        if (unpack_size > max_size) {
+            rc = PMIX_ERR_UNPACK_FAILURE;
+            PMIX_ERROR_LOG(rc);
+            return rc;
+        }
+        if (unpack_size > avail_size) {
+            rc = PMIX_ERR_FATAL;
+            PMIX_ERROR_LOG(rc);
+            return rc;
+        }
+        buffer->unpack_ptr += unpack_size;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+/*
+ * INT
+ */
+static pmix_status_t
+pmix4_bfrops_base_unpack_int(pmix_pointer_array_t *regtypes,
+                             pmix_buffer_t *buffer, void *dest,
+                             int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+    pmix_data_type_t remote_type;
+
+    if (false == pmix_psquash.int_type_is_encoded) {
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_get_data_type(regtypes, buffer, &remote_type))) {
+            return ret;
+        }
+        if (remote_type == BFROP_TYPE_INT) {
+            /* fast path it if the sizes are the same */
+            /* Turn around and unpack the real type */
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, BFROP_TYPE_INT, regtypes);
+        } else {
+            /* slow path - types are different sizes */
+            PMIX_BFROP_UNPACK_SIZE_MISMATCH(regtypes, int, remote_type, ret);
+        }
+    } else {
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, BFROP_TYPE_INT, regtypes);
+    }
+
+    return ret;
+}
+
+/*
+ * SIZE_T
+ */
+static pmix_status_t
+pmix4_bfrops_base_unpack_sizet(pmix_pointer_array_t *regtypes,
+                               pmix_buffer_t *buffer, void *dest,
+                               int32_t *num_vals, pmix_data_type_t type)
+{
+    pmix_status_t ret;
+    pmix_data_type_t remote_type;
+
+    if (false == pmix_psquash.int_type_is_encoded) {
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_get_data_type(regtypes, buffer,
+                                                            &remote_type))) {
+            PMIX_ERROR_LOG(ret);
+            return ret;
+        }
+        if (remote_type == BFROP_TYPE_SIZE_T) {
+            /* fast path it if the sizes are the same */
+            /* Turn around and unpack the real type */
+            PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, BFROP_TYPE_SIZE_T,
+                                    regtypes);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+            }
+        } else {
+            /* slow path - types are different sizes */
+            PMIX_BFROP_UNPACK_SIZE_MISMATCH(regtypes, size_t, remote_type, ret);
+        }
+    } else {
+        PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, BFROP_TYPE_SIZE_T,
+                                regtypes);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+        }
+    }
+    return ret;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/v4/bfrop_pmix4_component.c pmix-4.0.0/src/mca/bfrops/v4/bfrop_pmix4_component.c
--- pmix-3.2.2~rc1/src/mca/bfrops/v4/bfrop_pmix4_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/v4/bfrop_pmix4_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,99 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennbfropsee and The University
+ *                         of Tennbfropsee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/util/error.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/mca/bfrops/base/base.h"
+#include "bfrop_pmix4.h"
+
+extern pmix_bfrops_module_t pmix_bfrops_pmix4_module;
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+static pmix_status_t component_close(void);
+static pmix_bfrops_module_t* assign_module(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_bfrops_base_component_t mca_bfrops_v4_component = {
+    .base = {
+        PMIX_BFROPS_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "v4",
+        PMIX_MCA_BASE_MAKE_VERSION(component, PMIX_MAJOR_VERSION, PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_open_component = component_open,
+        .pmix_mca_close_component = component_close,
+        .pmix_mca_query_component = component_query,
+    },
+    .priority = 50,
+    .assign_module = assign_module
+};
+
+
+pmix_status_t component_open(void)
+{
+    /* setup the types array */
+    PMIX_CONSTRUCT(&mca_bfrops_v4_component.types, pmix_pointer_array_t);
+    pmix_pointer_array_init(&mca_bfrops_v4_component.types, 42, INT_MAX, 16);
+
+    return PMIX_SUCCESS;
+}
+
+
+pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+
+    *priority = mca_bfrops_v4_component.priority;
+    *module = (pmix_mca_base_module_t *)&pmix_bfrops_pmix4_module;
+    return PMIX_SUCCESS;
+}
+
+
+pmix_status_t component_close(void)
+{
+    PMIX_DESTRUCT(&mca_bfrops_v4_component.types);
+    return PMIX_SUCCESS;
+}
+
+static pmix_bfrops_module_t* assign_module(void)
+{
+    pmix_output_verbose(10, pmix_bfrops_base_framework.framework_output,
+                        "bfrops:pmix4x assigning module");
+    return &pmix_bfrops_pmix4_module;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/v4/bfrop_pmix4.h pmix-4.0.0/src/mca/bfrops/v4/bfrop_pmix4.h
--- pmix-3.2.2~rc1/src/mca/bfrops/v4/bfrop_pmix4.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/v4/bfrop_pmix4.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2019 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_BFROPS_PMIX4_H
+#define PMIX_BFROPS_PMIX4_H
+
+#include "src/mca/bfrops/bfrops.h"
+
+BEGIN_C_DECLS
+
+/* the component must be visible data for the linker to find it */
+ PMIX_EXPORT extern pmix_bfrops_base_component_t mca_bfrops_v4_component;
+
+extern pmix_bfrops_module_t pmix_bfrops_pmix4_module;
+
+END_C_DECLS
+
+#endif /* PMIX_BFROPS_PMIX4_H */
diff -Nru pmix-3.2.2~rc1/src/mca/bfrops/v4/Makefile.am pmix-4.0.0/src/mca/bfrops/v4/Makefile.am
--- pmix-3.2.2~rc1/src/mca/bfrops/v4/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/bfrops/v4/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = bfrop_pmix4.h
+sources = \
+        bfrop_pmix4_component.c \
+        bfrop_pmix4.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_bfrops_v4_DSO
+lib =
+lib_sources =
+component = mca_bfrops_v4.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_bfrops_v4.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_bfrops_v4_la_SOURCES = $(component_sources)
+mca_bfrops_v4_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_bfrops_v4_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_bfrops_v4_la_SOURCES = $(lib_sources)
+libmca_bfrops_v4_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/common/dstore/dstore_base.c pmix-4.0.0/src/mca/common/dstore/dstore_base.c
--- pmix-3.2.2~rc1/src/mca/common/dstore/dstore_base.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/common/dstore/dstore_base.c	2021-01-02 08:56:17.000000000 +0000
@@ -2790,9 +2790,9 @@
                     }
                 }
     		}
-            /* if the client is earlier than v3.1.5, we also need to store the
+            /* if the client is earlier than v3.2.x, we also need to store the
              * array using the hostname as key */
-            if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 5) &&
+            if (PMIX_PEER_IS_EARLIER(pmix_client_globals.myserver, 3, 1, 100) &&
                 NULL != hostname) {
                 kv2.key = hostname;
                 kv2.value = kv->value;
diff -Nru pmix-3.2.2~rc1/src/mca/common/sse/configure.m4 pmix-4.0.0/src/mca/common/sse/configure.m4
--- pmix-3.2.2~rc1/src/mca/common/sse/configure.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/common/sse/configure.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,28 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_pmix_common_sse_CONFIG([action-if-can-compile],
+#                            [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_common_sse_CONFIG], [
+    AC_CONFIG_FILES([src/mca/common/sse/Makefile])
+
+    AC_REQUIRE([PMIX_CHECK_CURL])
+    AC_REQUIRE([PMIX_CHECK_JANSSON])
+
+    AS_IF([test "$pmix_check_jansson_happy" = "yes" && test "$pmix_check_curl_happy" = "yes"],
+          [$1
+           pmix_common_sse_happy=yes],
+          [$2
+           pmix_common_sse_happy=no])
+
+    PMIX_SUMMARY_ADD([[Optional Support]],[[SSE]], [common_sse], [$pmix_check_curl_happy])
+
+])dnl
diff -Nru pmix-3.2.2~rc1/src/mca/common/sse/Makefile.am pmix-4.0.0/src/mca/common/sse/Makefile.am
--- pmix-3.2.2~rc1/src/mca/common/sse/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/common/sse/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2020      Intel, Inc. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Header files
+
+AM_CPPFLAGS = $(LTDLINCL) -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix $(pmix_check_curl_CPPFLAGS)
+AM_LFLAGS = -Ppmix_util_sse_yy
+LEX_OUTPUT_ROOT = lex.pmix_util_sse_yy
+
+headers = \
+	sse.h \
+	sse_internal.h
+
+# Source files
+
+sources = \
+	sse.c \
+    sse_lex.l
+
+lib_LTLIBRARIES =
+noinst_LTLIBRARIES =
+comp_inst = libmca_common_sse.la
+comp_noinst = libmca_common_sse_noinst.la
+
+if MCA_BUILD_pmix_common_sse_DSO
+lib_LTLIBRARIES += $(comp_inst)
+else
+noinst_LTLIBRARIES += $(comp_noinst)
+endif
+
+libmca_common_sse_la_SOURCES = $(headers) $(sources)
+libmca_common_sse_la_LDFLAGS = -version-info $(libmca_common_sse_so_version) $(pmix_check_curl_LDFLAGS)
+libmca_common_sse_la_LIBADD = $(pmix_check_curl_LIBS)
+libmca_common_sse_noinst_la_SOURCES = $(headers) $(sources)
+
+# Conditionally install the header files
+
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+pmix_HEADERS = $(headers)
+endif
+
+all-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+		rm -f "$(comp_inst)"; \
+		$(LN_S) "$(comp_noinst)" "$(comp_inst)"; \
+	fi
+
+clean-local:
+	if test -z "$(lib_LTLIBRARIES)"; then \
+		rm -f "$(comp_inst)"; \
+	fi
+
+MAINTAINERCLEANFILES = sse_lex.c
diff -Nru pmix-3.2.2~rc1/src/mca/common/sse/sse.c pmix-4.0.0/src/mca/common/sse/sse.c
--- pmix-3.2.2~rc1/src/mca/common/sse/sse.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/common/sse/sse.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,372 @@
+/*
+ * This file is derived from the sse package - per the BSD license, the following
+ * info is copied here:
+ ***********************
+ * This file is part of the sse package, copyright (c) 2011, 2012, @radiospiel.
+ * It is copyrighted under the terms of the modified BSD license, see LICENSE.BSD.
+ *
+ * For more information see https://https://github.com/radiospiel/sse.
+ ***********************
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <curl/curl.h>
+
+#include "src/class/pmix_list.h"
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+#include "sse.h"
+#include "sse_internal.h"
+
+pmix_common_sse_globals_t pmix_common_sse_globals = {0};
+
+#define PMIX_SSE_CLIENT_VERSION       "0.2"
+#define PMIX_SSE_CLIENT_USERAGENT     "sse/" PMIX_SSE_CLIENT_VERSION
+
+static const char* headers[] = {
+    "Accept: text/event-stream",
+    NULL
+};
+
+typedef struct {
+    pmix_list_item_t super;
+    pmix_event_t  ev;
+    bool event_active;
+    int max_retries;                            // max number of times to try to connect
+    CURL *curl;
+    struct curl_slist *myheaders;
+    char curl_error_buf[CURL_ERROR_SIZE];
+    char *expected_content_type;
+    bool allow_insecure;
+    char *ssl_cert;
+    char *ca_info;
+    pmix_sse_register_cbfunc_fn_t regcbfunc;    // fn to call upon request completion
+    void *regcbdata;                            // handle to pass back to regcbfunc
+    pmix_sse_on_data_cbfunc_fn_t ondata;        // fn to call when data arrives
+    void *datcbdata;                            // handle to pass back to ondata
+} pmix_sse_curl_handle_t;
+
+static void clcons(pmix_sse_curl_handle_t *p)
+{
+    p->curl = curl_easy_init();
+    memset(p->curl_error_buf, 0, CURL_ERROR_SIZE);
+    /* === set defaults ============================================== */
+    curl_easy_setopt(p->curl, CURLOPT_NOPROGRESS, 1);
+    curl_easy_setopt(p->curl, CURLOPT_USERAGENT, PMIX_SSE_CLIENT_USERAGENT);
+    curl_easy_setopt(p->curl, CURLOPT_FOLLOWLOCATION, 1);
+    curl_easy_setopt(p->curl, CURLOPT_MAXREDIRS, 10);
+    curl_easy_setopt(p->curl, CURLOPT_ERRORBUFFER, p->curl_error_buf);
+    p->myheaders = NULL;
+    // -- enable all supported built-in compressions  -------------------
+    curl_easy_setopt(p->curl, CURLOPT_ACCEPT_ENCODING, "");
+    p->event_active = false;
+    p->allow_insecure = false;
+    p->ssl_cert = NULL;
+    p->ca_info = NULL;
+    p->regcbfunc = NULL;
+    p->regcbdata = NULL;
+    p->ondata = NULL;
+    p->datcbdata = NULL;
+}
+static void cldes(pmix_sse_curl_handle_t *p)
+{
+    if (p->event_active) {
+        pmix_event_evtimer_del(&p->ev);
+        p->event_active = false;
+    }
+    curl_slist_free_all(p->myheaders);
+    curl_easy_cleanup(p->curl);
+    if (NULL != p->ondata) {
+        /* return a NULL stream so they know this has closed */
+        p->ondata(NULL, NULL, p->datcbdata);
+    }
+    if (NULL != p->expected_content_type) {
+        free(p->expected_content_type);
+    }
+    if (NULL != p->ssl_cert) {
+        free(p->ssl_cert);
+    }
+    if (NULL != p->ca_info) {
+        free(p->ca_info);
+    }
+}
+static PMIX_CLASS_INSTANCE(pmix_sse_curl_handle_t,
+                           pmix_list_item_t,
+                           clcons, cldes);
+
+
+static void curl_perform(int sd, short args, void *cbdata)
+{
+    pmix_sse_curl_handle_t *pcl = (pmix_sse_curl_handle_t*)cbdata;
+    int retries = 0;
+    long response_code;
+    const char* effective_url = NULL;
+    char* content_type;
+
+    while (1) {
+        CURLcode res = curl_easy_perform(pcl->curl);
+
+        switch(res) {
+            case CURLE_OK:
+                /* get the effective URL */
+                curl_easy_getinfo(pcl->curl, CURLINFO_EFFECTIVE_URL, &effective_url);
+                /* get the response code */
+                curl_easy_getinfo(pcl->curl, CURLINFO_RESPONSE_CODE, &response_code);
+                if (response_code < 200 || response_code >= 300) {
+                    pcl->regcbfunc(response_code, effective_url, pcl->regcbdata);
+                    PMIX_RELEASE(pcl);
+                    return;
+                }
+                if (NULL != pcl->expected_content_type) {
+                    /* check the expected content type */
+                    curl_easy_getinfo(pcl->curl, CURLINFO_CONTENT_TYPE, &content_type);
+                    if (NULL == content_type ||
+                        0 != strncmp(content_type, pcl->expected_content_type, strlen(pcl->expected_content_type))) {
+                        pcl->regcbfunc(PMIX_ERR_TYPE_MISMATCH, content_type, pcl->regcbdata);
+                        PMIX_RELEASE(pcl);
+                        return;
+                    }
+                }
+                pcl->regcbfunc(PMIX_SUCCESS, effective_url, pcl->regcbdata);
+                return;
+
+            case CURLE_COULDNT_RESOLVE_PROXY:
+            case CURLE_COULDNT_RESOLVE_HOST:
+            case CURLE_COULDNT_CONNECT:
+                pmix_output_verbose(1, pmix_common_sse_globals.output,
+                                    "curl: %s\n", pcl->curl_error_buf);
+                retries++;
+                if (pcl->max_retries <= retries) {
+                    pcl->regcbfunc(PMIX_ERR_UNREACH, pcl->curl_error_buf, pcl->regcbdata);
+                    return;
+                }
+                /* delay and retry */
+                PMIX_THREADSHIFT_DELAY(pcl, curl_perform, 3);
+                pcl->event_active = true;
+                return;
+
+            default:
+                pmix_output_verbose(1, pmix_common_sse_globals.output,
+                                  "curl: %s\n", pcl->curl_error_buf);
+                pcl->regcbfunc(PMIX_ERR_UNREACH, pcl->curl_error_buf, pcl->regcbdata);
+                return;
+        }
+    }
+}
+
+static void ondataprocessing(char *ptr, size_t size, size_t nmemb, void *cbdata)
+{
+    /* have lex parse the input data - it will callback to
+     * pmix_common_on_sse_event when done */
+      pmix_common_sse_parse_sse(ptr, size * nmemb, cbdata);
+}
+
+/* provide a hook for the curl reply if it is just to be ignored */
+static size_t http_ignore_data(char *ptr, size_t size, size_t nmemb, void *userdata)
+{
+    return size * nmemb;
+}
+
+void pmix_common_on_sse_event(char** headers, const char* data, const char* reply_url, void *cbdata)
+{
+    pmix_sse_curl_handle_t *pcl = (pmix_sse_curl_handle_t*)cbdata;
+    pmix_sse_curl_handle_t *reply;
+    char *result = NULL;
+    int n;
+
+    /* pass to the user for processing */
+    if (NULL != pcl->ondata) {
+        pcl->ondata(data, &result, pcl->datcbdata);
+    }
+
+    if (NULL != reply_url) {
+        printf("REPLY URL\n");
+        char* body = result ? result : "";
+        const char* reply_headers[] = {
+            "Content-Type:",
+            NULL
+        };
+
+        fprintf(stderr, "REPLY %s (%d byte)\n", reply_url, (int) strlen(body));
+        /* create a CURL handle for this request */
+        reply = PMIX_NEW(pmix_sse_curl_handle_t);
+        reply->max_retries = 0;
+        curl_easy_setopt(reply->curl, CURLOPT_URL, reply_url);
+        curl_easy_setopt(reply->curl, CURLOPT_WRITEFUNCTION, http_ignore_data);
+        for (n=0; NULL != reply_headers[n]; n++) {
+            reply->myheaders = curl_slist_append(reply->myheaders, reply_headers[n]);
+        }
+        reply->myheaders = curl_slist_append(pcl->myheaders, "Expect:");
+        curl_easy_setopt(reply->curl, CURLOPT_HTTPHEADER, reply->myheaders);
+        curl_easy_setopt(reply->curl, CURLOPT_POST, 1L);                     /* set POST */
+        curl_easy_setopt(reply->curl, CURLOPT_POSTFIELDS, body);        /* set POST data */
+        curl_easy_setopt(reply->curl, CURLOPT_POSTFIELDSIZE, strlen(body));
+        curl_easy_setopt(reply->curl, CURLOPT_SSL_VERIFYPEER, pcl->allow_insecure ? 0L : 1L);
+        curl_easy_setopt(reply->curl, CURLOPT_SSL_VERIFYHOST, pcl->allow_insecure ? 0L : 1L);
+        if (NULL != pcl->ssl_cert) {
+            curl_easy_setopt(reply->curl, CURLOPT_SSLCERT, pcl->ssl_cert);
+        }
+        if (NULL != pcl->ca_info) {
+            curl_easy_setopt(reply->curl, CURLOPT_CAINFO, pcl->ca_info);
+        }
+        PMIX_THREADSHIFT(pcl, curl_perform);
+    }
+
+    free(result);
+}
+
+static bool curl_initialised = false;
+
+void pmix_sse_common_init(void)
+{
+    if (!curl_initialised) {
+        curl_initialised = true;
+        curl_global_init(CURL_GLOBAL_ALL);
+#ifdef HAVE_ATEXIT
+        atexit(curl_global_cleanup);
+#endif
+    }
+}
+
+pmix_status_t pmix_sse_register_request(pmix_sse_request_t *req,
+                                        pmix_sse_register_cbfunc_fn_t regcbfunc,
+                                        void *regcbdata,
+                                        pmix_sse_on_data_cbfunc_fn_t ondata,
+                                        void *datcbdata)
+{
+    pmix_sse_curl_handle_t *pcl;
+    int n;
+
+    if (!curl_initialised) {
+        return PMIX_ERR_INIT;
+    }
+    if (PMIX_HTTP_UNDEF == req->verb) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* create a CURL handle for this request */
+    pcl = PMIX_NEW(pmix_sse_curl_handle_t);
+    pcl->max_retries = req->max_retries;
+    if (NULL != req->expected_content_type) {
+        pcl->expected_content_type = strdup(req->expected_content_type);
+    }
+    pcl->ondata = ondata;
+    pcl->datcbdata = datcbdata;
+    pcl->regcbfunc = regcbfunc;
+    pcl->regcbdata = regcbdata;
+    pcl->allow_insecure = req->allow_insecure;
+    if (NULL != req->ssl_cert) {
+        pcl->ssl_cert = strdup(req->ssl_cert);
+    }
+    if (NULL != req->ca_info) {
+        pcl->ca_info = strdup(req->ca_info);
+    }
+
+    // -- set URL -------------------------------------------------------
+    curl_easy_setopt(pcl->curl, CURLOPT_URL, req->url);
+    // -- set write function --------------------------------------------
+    curl_easy_setopt(pcl->curl, CURLOPT_WRITEFUNCTION, ondataprocessing);
+    curl_easy_setopt(pcl->curl, CURLOPT_WRITEDATA, pcl);
+
+    /* setup headers */
+    for (n=0; NULL != headers[n]; n++) {
+        pcl->myheaders = curl_slist_append(pcl->myheaders, headers[n]);
+    }
+
+    if (PMIX_HTTP_POST == req->verb) {
+        /*
+         * HTTP 1.1 specifies that a POST should use a "Expect: 100-continue"
+         * header. This should prepare the server for a potentially large
+         * upload.
+         *
+         * Our bodies are not that big anyways, and some servers - notably thin
+         * running standalone - do not send these, leading to a one or two-seconds
+         * delay.
+         *
+         * Therefore we disable this behavior.
+         */
+        pcl->myheaders = curl_slist_append(pcl->myheaders, "Expect:");
+    }
+    curl_easy_setopt(pcl->curl, CURLOPT_HTTPHEADER, pcl->myheaders);
+
+    // -- set body ------------------------------------------------------
+
+    if (PMIX_HTTP_POST == req->verb) {
+        curl_easy_setopt(pcl->curl, CURLOPT_POST, 1L);                     /* set POST */
+        curl_easy_setopt(pcl->curl, CURLOPT_POSTFIELDS, req->body);        /* set POST data */
+        curl_easy_setopt(pcl->curl, CURLOPT_POSTFIELDSIZE, req->len);
+    }
+
+    /*
+     * If you want to connect to a site who isn't using a certificate
+     * that is signed by one of the certs in the CA bundle you have, you
+     * can skip the verification of the server's certificate. This makes
+     * the connection A LOT LESS SECURE.
+     *
+     * If you have a CA cert for the server stored someplace else than
+     * in the default bundle, then the CURLOPT_CAPATH option might come
+     * handy for you.
+     */
+    curl_easy_setopt(pcl->curl, CURLOPT_SSL_VERIFYPEER, req->allow_insecure ? 0L : 1L);
+
+    /*
+     * If the site you're connecting to uses a different host name than
+     * what they have mentioned in their server certificate's commonName
+     * (or subjectAltName) fields, libcurl will refuse to connect. You can
+     * skip this check, but this will make the connection less secure.
+     */
+    curl_easy_setopt(pcl->curl, CURLOPT_SSL_VERIFYHOST, req->allow_insecure ? 0L : 1L);
+
+    /*
+     * Did the user request a specific set of certifications?
+     */
+    if (NULL != req->ssl_cert) {
+        curl_easy_setopt(pcl->curl, CURLOPT_SSLCERT, req->ssl_cert);
+    }
+    if (NULL != req->ca_info) {
+        curl_easy_setopt(pcl->curl, CURLOPT_CAINFO, req->ca_info);
+    }
+
+    // -- queue the request ---------------------------------------------
+    PMIX_THREADSHIFT(pcl, curl_perform);
+
+    return PMIX_SUCCESS;
+}
+
+/****  CLASS INSTANTIATION  ****/
+static void rqcon(pmix_sse_request_t *p)
+{
+    PMIX_CONSTRUCT_LOCK(&p->lock);
+    p->status = PMIX_ERR_NOT_SUPPORTED;
+    p->verb = PMIX_HTTP_UNDEF;
+    p->url = NULL;
+    p->max_retries = 5;
+    p->debug = false;
+    p->allow_insecure = false;
+    p->ssl_cert = NULL;
+    p->ca_info = NULL;
+    p->expected_content_type = NULL;
+    p->body = NULL;
+    p->len = 0;
+}
+static void rqdes(pmix_sse_request_t *p)
+{
+    PMIX_DESTRUCT_LOCK(&p->lock);
+}
+PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_sse_request_t,
+                                pmix_object_t,
+                                rqcon, rqdes);
diff -Nru pmix-3.2.2~rc1/src/mca/common/sse/sse.h pmix-4.0.0/src/mca/common/sse/sse.h
--- pmix-3.2.2~rc1/src/mca/common/sse/sse.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/common/sse/sse.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,91 @@
+/*
+ * This file is derived from the sse package - per the BSD license, the following
+ * info is copied here:
+ ***********************
+ * This file is part of the sse package, copyright (c) 2011, 2012, @radiospiel.
+ * It is copyrighted under the terms of the modified BSD license, see LICENSE.BSD.
+ *
+ * For more information see https://https://github.com/radiospiel/sse.
+ ***********************
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_SSE_H
+#define PMIX_SSE_H
+
+#include "src/include/pmix_config.h"
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <stdio.h>
+
+#include "src/class/pmix_object.h"
+#include "src/threads/threads.h"
+
+typedef enum {
+    PMIX_HTTP_UNDEF = -1,
+    PMIX_HTTP_GET   =  0,
+    PMIX_HTTP_POST  =  1
+} pmix_sse_verb_t;
+
+
+/* define a function to be called to verify the connection */
+typedef void (*pmix_sse_register_cbfunc_fn_t)(long response_code,
+                                              const char *effective_url,
+                                              void *cbdata);
+
+/* define a function to be called when data becomes available
+ * from the request */
+typedef void (*pmix_sse_on_data_cbfunc_fn_t)(const char *ptr, char **result, void *userdata);
+
+/*
+ * Application options
+ */
+typedef struct {
+    pmix_object_t                   super;
+    pmix_lock_t                     lock;                               // thread lock so requesters can wait for completion
+    pmix_status_t                   status;                             // return status of operation
+    pmix_sse_verb_t                 verb;                               // operation to perform
+    const char                      *url;                               // URL to get
+    int                             max_retries;                        // max number of times to try to connect
+    bool                            debug;                              // output debug messages
+    bool                            allow_insecure;                     // allow insecure connections
+    const char                      *ssl_cert;                          // SSL cert file
+    const char                      *ca_info;                           // CA cert file
+    const char                      *expected_content_type;             // curl-response string for expected content type
+    const char                      *body;                              // payload to be posted
+    unsigned                        len;                                // number of bytes in body
+} pmix_sse_request_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_sse_request_t);
+
+typedef struct {
+    int output;         // verbose output channel for debugging
+} pmix_common_sse_globals_t;
+PMIX_EXPORT extern pmix_common_sse_globals_t pmix_common_sse_globals;
+
+struct MemoryStruct {
+  char *memory;
+  size_t size;
+};
+
+/* initialize the sse system */
+PMIX_EXPORT void pmix_sse_common_init(void);
+
+/* register for an sse-based data stream, specifying the details of the
+ * request and providing a callback function to be executed whenever data
+ * becomes available on the specified channel */
+PMIX_EXPORT pmix_status_t pmix_sse_register_request(pmix_sse_request_t *req,
+                                                    pmix_sse_register_cbfunc_fn_t regcbfunc,
+                                                    void *reqcbdata,
+                                                    pmix_sse_on_data_cbfunc_fn_t ondata,
+                                                    void *datcbdata);
+
+#endif /* PMIX_SSE_H */
diff -Nru pmix-3.2.2~rc1/src/mca/common/sse/sse_internal.h pmix-4.0.0/src/mca/common/sse/sse_internal.h
--- pmix-3.2.2~rc1/src/mca/common/sse/sse_internal.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/common/sse/sse_internal.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,39 @@
+/*
+ * This file is derived from the sse package - per the BSD license, the following
+ * info is copied here:
+ ***********************
+ * This file is part of the sse package, copyright (c) 2011, 2012, @radiospiel.
+ * It is copyrighted under the terms of the modified BSD license, see LICENSE.BSD.
+ *
+ * For more information see https://https://github.com/radiospiel/sse.
+ ***********************
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_SSE_INTERNAL_H
+#define PMIX_SSE_INTERNAL_H
+
+#include "src/include/pmix_config.h"
+
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <stdio.h>
+
+#define PMIX_COMMON_SSE_MAX_HEADERS 100
+
+/* response limit: 128 kByte */
+#define PMIX_COMMON_SSE_RESPONSE_LIMIT  128 * 1024
+
+PMIX_EXPORT void pmix_common_sse_parse_sse(char *ptr, size_t size, void *cbdata);
+
+PMIX_EXPORT void pmix_common_on_sse_event(char** headers, const char* data, const char* reply_url, void *cbdata);
+
+#endif /* PMIX_SSE_INTERNAL_H */
diff -Nru pmix-3.2.2~rc1/src/mca/common/sse/sse_lex.l pmix-4.0.0/src/mca/common/sse/sse_lex.l
--- pmix-3.2.2~rc1/src/mca/common/sse/sse_lex.l	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/common/sse/sse_lex.l	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,180 @@
+/*
+ * This file is derived from the sse package - per the BSD license, the following
+ * info is copied here:
+ ***********************
+ * This file is part of the sse package, copyright (c) 2011, 2012, @radiospiel.
+ * It is copyrighted under the terms of the modified BSD license, see LICENSE.BSD.
+ *
+ * For more information see https://https://github.com/radiospiel/sse.
+ ***********************
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+%option always-interactive
+
+%{
+
+#include "sse.h"
+#include "sse_internal.h"
+
+#ifndef NDEBUG
+#define YYDEBUG 1
+#endif
+
+/* === data ======================================================== */
+
+static char* data_buf = 0;
+static const char* data = 0;
+static char* reply_url = 0;
+static void *usercbdata = NULL;
+
+static void set_reply_url(const char* url) {
+  free(reply_url);
+  reply_url = url ? strdup(url) : 0;
+}
+
+/*
+ * skip the first character in string, if there is one;
+ * then replace the string in \a *pTarget with this string;
+ * return a pointer to the string.
+ */
+static void data_add(const char* string)
+{
+  size_t new_len = (data_buf ? strlen(data_buf) : 0) + strlen(string) + 1;
+
+  data_buf = realloc(data_buf, new_len + 1);
+  strcat(data_buf, "\n");
+  strcat(data_buf, string);
+
+  data = data_buf;
+}
+
+static void data_reset(void)
+{
+  if(data_buf)
+    *data_buf = 0;
+
+  data = NULL;
+}
+
+/* === headers ===================================================== */
+
+static char* headers[PMIX_COMMON_SSE_MAX_HEADERS] = { 0 };
+static char** header_ptr = headers;
+
+static void headers_reset(void) {
+  char** ph = headers;
+  while(*ph) {
+    free(*ph);
+    *ph++ = 0;
+  }
+
+  header_ptr = headers;
+}
+
+static void header_add(const char* s) {
+  if(header_ptr - headers < PMIX_COMMON_SSE_MAX_HEADERS - 1) {
+    *header_ptr++ = strdup(s);
+    *header_ptr = NULL;
+  }
+}
+
+static void header_add_from_line(char* line)
+{
+  char* colon = strchr(line, ':');
+  if(!colon || colon == line) return;
+
+  *colon = '=';
+
+  /*
+   * upcase all chars until colon
+   */
+  {
+    char* s;
+    for(s = line; s < colon; ++s) {
+      *s = toupper(*s);
+    }
+  }
+
+  /*
+   * if the value starts with a space we'll have to skip that. We do
+   * this by memmoving the name one character to the right.
+   */
+
+  if(isspace(colon[1])) {
+    memmove(line+1, line, colon - line);
+    colon[1] = '=';
+    line += 1;
+  }
+
+  header_add(line);
+}
+
+/* === flush the event ============================================= */
+
+static void flush(void)
+{
+  /*
+   * If neither headers nor data are set, then we flush after some
+   * keep-alive traffic (or some other traffic that does not conform
+   * to SSE)
+   */
+  if(*headers || (data && *data)) {
+    pmix_common_on_sse_event(headers, data ? data : "", reply_url, usercbdata);
+  }
+
+  set_reply_url(0);
+  data_reset();
+  headers_reset();
+}
+
+%}
+
+%%
+
+                  /* the data attribute is special: multiple lines are concatenated */
+data:\ .*         data_add(yytext + 6);
+data:.*           data_add(yytext + 5);
+
+                  /* the data attribute is special: multiple lines are concatenated */
+reply:\ .*        set_reply_url(yytext + 7);
+reply:.*          set_reply_url(yytext + 6);
+
+                  /* all other attributes should appear only one */
+
+[-_0-9a-z]+:\ .*  header_add_from_line(yytext);
+[-_0-9a-z]+:.*    header_add_from_line(yytext);
+
+                  /* ignore lines starting with a colon. */
+:.*               ;
+
+                  /* ignore lines without colon IN VIOLATION WITH THE SPECS. */
+.+                ;
+
+\n\n              flush();
+\n                /* no yet flushing - this is within an event */
+
+%%
+
+/*
+ * curl callback to feed some data into the lexer.
+ */
+void pmix_common_sse_parse_sse(char *ptr, size_t size, void *cbdata) {
+  (void)yyunput;
+  usercbdata = cbdata;
+
+  YY_BUFFER_STATE buffer = yy_scan_bytes(ptr, size);
+  while(yylex());
+  yy_delete_buffer(buffer);
+}
+
+int yywrap() {
+  return 1;
+}
+
diff -Nru pmix-3.2.2~rc1/src/mca/gds/hash/gds_hash.c pmix-4.0.0/src/mca/gds/hash/gds_hash.c
--- pmix-3.2.2~rc1/src/mca/gds/hash/gds_hash.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/gds/hash/gds_hash.c	2021-01-02 08:56:17.000000000 +0000
@@ -37,6 +37,7 @@
 #include "src/client/pmix_client_ops.h"
 #include "src/server/pmix_server_ops.h"
 #include "src/mca/pcompress/base/base.h"
+#include "src/mca/pmdl/pmdl.h"
 #include "src/mca/preg/preg.h"
 #include "src/mca/ptl/base/base.h"
 #include "src/util/argv.h"
@@ -659,6 +660,13 @@
                 }
             }
         }
+        if (PMIX_CHECK_KEY(kp2, PMIX_MODEL_LIBRARY_NAME) ||
+            PMIX_CHECK_KEY(kp2, PMIX_PROGRAMMING_MODEL) ||
+            PMIX_CHECK_KEY(kp2, PMIX_MODEL_LIBRARY_VERSION) ||
+            PMIX_CHECK_KEY(kp2, PMIX_PERSONALITY)) {
+            // pass this info to the pmdl framework
+            pmix_pmdl.setup_nspace_kv(trk->nptr, kp2);
+        }
         pmix_list_append(&app->appinfo, &kp2->super);
         kp2 = (pmix_kval_t*)pmix_list_remove_first(&cache);
     }
@@ -736,6 +744,12 @@
             }
             /* mark that we got the map */
             *flags |= PMIX_HASH_NODE_MAP;
+        } else if (PMIX_CHECK_KEY(&iptr[j], PMIX_MODEL_LIBRARY_NAME) ||
+                   PMIX_CHECK_KEY(&iptr[j], PMIX_PROGRAMMING_MODEL) ||
+                   PMIX_CHECK_KEY(&iptr[j], PMIX_MODEL_LIBRARY_VERSION)  ||
+                   PMIX_CHECK_KEY(&iptr[j], PMIX_PERSONALITY)) {
+            // pass this info to the pmdl framework
+            pmix_pmdl.setup_nspace(trk->nptr, &iptr[j]);
         } else {
             kp2 = PMIX_NEW(pmix_kval_t);
             kp2->key = strdup(iptr[j].key);
@@ -1386,6 +1400,7 @@
                 }
                 /* if this is the appnum, pass it to the pmdl framework */
                 if (PMIX_CHECK_KEY(kp2, PMIX_APPNUM)) {
+                    pmix_pmdl.setup_client(trk->nptr, rank, kp2->value->data.uint32);
                     found = true;
                     if (rank == pmix_globals.myid.rank) {
                         pmix_globals.appnum = kp2->value->data.uint32;
@@ -1410,8 +1425,15 @@
                     PMIX_RELEASE(kp2);
                     goto release;
                 }
+                pmix_pmdl.setup_client(trk->nptr, rank, kp2->value->data.uint32);
                 PMIX_RELEASE(kp2);  // maintain acctg
             }
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_MODEL_LIBRARY_NAME) ||
+                   PMIX_CHECK_KEY(&info[n], PMIX_PROGRAMMING_MODEL) ||
+                   PMIX_CHECK_KEY(&info[n], PMIX_MODEL_LIBRARY_VERSION) ||
+                   PMIX_CHECK_KEY(&info[n], PMIX_PERSONALITY)) {
+            // pass this info to the pmdl framework
+            pmix_pmdl.setup_nspace(trk->nptr, &info[n]);
         } else if (pmix_check_node_info(info[n].key)) {
             /* they are passing us the node-level info for just this
              * node - start by seeing if our node is on the list */
@@ -1623,10 +1645,10 @@
     rc = fetch_nodeinfo(NULL, &trk->nodeinfo, NULL, 0, &results);
     if (PMIX_SUCCESS == rc) {
         PMIX_LIST_FOREACH(kvptr, &results, pmix_kval_t) {
-            /* if the peer is earlier than v3.1.5, it is expecting
+            /* if the peer is earlier than v3.2.x, it is expecting
              * node info to be in the form of an array, but with the
              * hostname as the key. Detect and convert that here */
-            if (PMIX_PEER_IS_EARLIER(peer, 3, 1, 5)) {
+            if (PMIX_PEER_IS_EARLIER(peer, 3, 1, 100)) {
                 info = (pmix_info_t*)kvptr->value->data.darray->array;
                 ninfo = kvptr->value->data.darray->size;
                 hname = NULL;
@@ -1895,6 +1917,9 @@
                 }
                 /* this is data provided by a job-level exchange, so store it
                  * in the job-level data hash_table */
+                pmix_output_verbose(2, pmix_gds_base_framework.framework_output,
+                                    "[%s:%u] pmix:gds:hash store proc info for rank %u working key %s",
+                                    pmix_globals.myid.nspace, pmix_globals.myid.rank, rank, kp2->key);
                 if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) {
                     PMIX_ERROR_LOG(rc);
                     PMIX_RELEASE(kp2);
@@ -1993,6 +2018,9 @@
                     kp2->value->type = PMIX_STRING;
                     kp2->value->data.string = strdup(kv.key);
                     rank = strtol(procs[j], NULL, 10);
+                    pmix_output_verbose(2, pmix_gds_base_framework.framework_output,
+                                        "[%s:%u] pmix:gds:hash store map info for rank %u working key %s",
+                                        pmix_globals.myid.nspace, pmix_globals.myid.rank, rank, kp2->key);
                     if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) {
                         PMIX_ERROR_LOG(rc);
                         PMIX_RELEASE(kp2);
@@ -2116,6 +2144,20 @@
         return PMIX_ERR_NOMEM;
     }
 
+    /* if this is node/app data, then process it accordingly */
+    if (PMIX_CHECK_KEY(kv, PMIX_NODE_INFO_ARRAY)) {
+        rc = process_node_array(kv->value, &trk->nodeinfo);
+        return rc;
+    } else if (PMIX_CHECK_KEY(kv, PMIX_APP_INFO_ARRAY)) {
+        rc = process_app_array(kv->value, trk);
+        return rc;
+    } else if (PMIX_CHECK_KEY(kv, PMIX_SESSION_INFO_ARRAY)) {
+        rc = process_session_array(kv->value, trk);
+        return rc;
+    } else if (PMIX_CHECK_KEY(kv, PMIX_JOB_INFO_ARRAY)) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
     /* see if the proc is me - cannot use CHECK_PROCID as
      * we don't want rank=wildcard to match */
     if (proc->rank == pmix_globals.myid.rank &&
@@ -2757,6 +2799,8 @@
     pmix_list_t rkvs;
     bool nodeinfo = false;
     bool appinfo = false;
+    bool nigiven = false;
+    bool apigiven = false;
 
     pmix_output_verbose(2, pmix_gds_base_framework.framework_output,
                         "%s pmix:gds:hash fetch %s for proc %s on scope %s",
@@ -2885,13 +2929,15 @@
             return PMIX_ERR_NOT_FOUND;
         } else if (PMIX_CHECK_KEY(&qualifiers[n], PMIX_NODE_INFO)) {
             nodeinfo = PMIX_INFO_TRUE(&qualifiers[n]);
+            nigiven = true;
         } else if (PMIX_CHECK_KEY(&qualifiers[n], PMIX_APP_INFO)) {
             appinfo = PMIX_INFO_TRUE(&qualifiers[n]);
+            apigiven = true;
         }
     }
 
     /* check for node/app keys in the absence of corresponding qualifier */
-    if (NULL != key) {
+    if (NULL != key && !nigiven && !apigiven) {
         if (pmix_check_node_info(key)) {
             nodeinfo = true;
         } else if (pmix_check_app_info(key)) {
@@ -2905,22 +2951,24 @@
         return PMIX_ERR_INVALID_NAMESPACE;
     }
 
-    if (nodeinfo) {
-        rc = fetch_nodeinfo(key, &trk->nodeinfo, qualifiers, nqual, kvs);
-        if (PMIX_SUCCESS != rc && PMIX_RANK_WILDCARD == proc->rank) {
-            /* need to check internal as we might have an older peer */
-            ht = &trk->internal;
-            goto doover;
-        }
-        return rc;
-    } else if (appinfo) {
-        rc = fetch_appinfo(key, &trk->apps, qualifiers, nqual, kvs);
-        if (PMIX_SUCCESS != rc && PMIX_RANK_WILDCARD == proc->rank) {
-            /* need to check internal as we might have an older peer */
-            ht = &trk->internal;
-            goto doover;
+    if (!PMIX_RANK_IS_VALID(proc->rank)) {
+        if (nodeinfo) {
+            rc = fetch_nodeinfo(key, &trk->nodeinfo, qualifiers, nqual, kvs);
+            if (PMIX_SUCCESS != rc && PMIX_RANK_WILDCARD == proc->rank) {
+                /* need to check internal as we might have an older peer */
+                ht = &trk->internal;
+                goto doover;
+            }
+            return rc;
+        } else if (appinfo) {
+            rc = fetch_appinfo(key, &trk->apps, qualifiers, nqual, kvs);
+            if (PMIX_SUCCESS != rc && PMIX_RANK_WILDCARD == proc->rank) {
+                /* need to check internal as we might have an older peer */
+                ht = &trk->internal;
+                goto doover;
+            }
+            return rc;
         }
-        return rc;
     }
 
     /* fetch from the corresponding hash table - note that
@@ -3009,7 +3057,38 @@
         }
     }
     if (0 == pmix_list_get_size(kvs)) {
-        rc = PMIX_ERR_NOT_FOUND;
+        /* if we didn't find it and the rank was valid, then
+         * check to see if the data exists in a different scope.
+         * This is done to avoid having the process go into a
+         * timeout wait when the data will never appear within
+         * the specified scope */
+        if (PMIX_RANK_IS_VALID(proc->rank)) {
+            if (PMIX_LOCAL == scope) {
+                /* check the remote scope */
+                rc = dohash(&trk->remote, key, proc->rank, false, kvs);
+                if (PMIX_SUCCESS == rc || 0 < pmix_list_get_size(kvs)) {
+                    while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(kvs))) {
+                        PMIX_RELEASE(kv);
+                    }
+                    rc = PMIX_ERR_EXISTS_OUTSIDE_SCOPE;
+                } else {
+                    rc = PMIX_ERR_NOT_FOUND;
+                }
+            } else if (PMIX_REMOTE == scope) {
+                /* check the local scope */
+                rc = dohash(&trk->local, key, proc->rank, false, kvs);
+                if (PMIX_SUCCESS == rc || 0 < pmix_list_get_size(kvs)) {
+                    while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(kvs))) {
+                        PMIX_RELEASE(kv);
+                    }
+                    rc = PMIX_ERR_EXISTS_OUTSIDE_SCOPE;
+                } else {
+                    rc = PMIX_ERR_NOT_FOUND;
+                }
+            }
+        } else {
+            rc = PMIX_ERR_NOT_FOUND;
+        }
     }
 
     return rc;
diff -Nru pmix-3.2.2~rc1/src/mca/pcompress/base/pcompress_base_frame.c pmix-4.0.0/src/mca/pcompress/base/pcompress_base_frame.c
--- pmix-3.2.2~rc1/src/mca/pcompress/base/pcompress_base_frame.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pcompress/base/pcompress_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -6,7 +6,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -24,19 +24,40 @@
 /*
  * Globals
  */
-static bool compress_block(char *instring,
+static bool compress_block(uint8_t *inblock, size_t size,
                            uint8_t **outbytes,
                            size_t *nbytes)
 {
-    (void)instring;
+    (void)inblock;
+    (void)size;
     (void)outbytes;
     (void)nbytes;
     return false;
 }
 
-static bool decompress_block(char **outstring,
+static bool decompress_block(uint8_t **outbytes, size_t *outlen,
                              uint8_t *inbytes, size_t len)
 {
+    (void)outbytes;
+    (void)outlen;
+    (void)inbytes;
+    (void)len;
+    return false;
+}
+
+static bool compress_string(char *instring,
+                            uint8_t **outbytes,
+                            size_t *nbytes)
+{
+    (void)instring;
+    (void)outbytes;
+    (void)nbytes;
+    return false;
+}
+
+static bool decompress_string(char **outstring,
+                              uint8_t *inbytes, size_t len)
+{
     (void)outstring;
     (void)inbytes;
     (void)len;
@@ -44,14 +65,10 @@
 }
 
 pmix_compress_base_module_t pmix_compress = {
-    NULL, /* init             */
-    NULL, /* finalize         */
-    NULL, /* compress         */
-    NULL, /* compress_nb      */
-    NULL, /* decompress       */
-    NULL,  /* decompress_nb    */
-    compress_block,
-    decompress_block
+    .compress = compress_block,
+    .decompress = decompress_block,
+    .compress_string = compress_string,
+    .decompress_string = decompress_string
 };
 pmix_compress_base_t pmix_compress_base = {0};
 
diff -Nru pmix-3.2.2~rc1/src/mca/pcompress/base/pcompress_base_select.c pmix-4.0.0/src/mca/pcompress/base/pcompress_base_select.c
--- pmix-3.2.2~rc1/src/mca/pcompress/base/pcompress_base_select.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pcompress/base/pcompress_base_select.c	2021-01-02 08:56:17.000000000 +0000
@@ -52,8 +52,10 @@
 
     /* Initialize the winner */
     if (NULL != best_module) {
-        if (PMIX_SUCCESS != (ret = best_module->init()) ) {
-            goto cleanup;
+        if (NULL != best_module->init) {
+            if (PMIX_SUCCESS != (ret = best_module->init()) ) {
+                goto cleanup;
+            }
         }
         pmix_compress = *best_module;
     }
diff -Nru pmix-3.2.2~rc1/src/mca/pcompress/pcompress.h pmix-4.0.0/src/mca/pcompress/pcompress.h
--- pmix-3.2.2~rc1/src/mca/pcompress/pcompress.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pcompress/pcompress.h	2021-01-02 08:56:17.000000000 +0000
@@ -6,7 +6,7 @@
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  *
- * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,36 +54,6 @@
      (void);
 
 /**
- * Compress the file provided
- *
- * Arguments:
- *   fname   = Filename to compress
- *   cname   = Compressed filename
- *   postfix = postfix added to filename to create compressed filename
- * Returns:
- *   PMIX_SUCCESS on success, ow PMIX_ERROR
- */
-typedef int (*pmix_compress_base_module_compress_fn_t)
-    (char * fname, char **cname, char **postfix);
-
-typedef int (*pmix_compress_base_module_compress_nb_fn_t)
-    (char * fname, char **cname, char **postfix, pid_t *child_pid);
-
-/**
- * Decompress the file provided
- *
- * Arguments:
- *   fname = Filename to compress
- *   cname = Compressed filename
- * Returns:
- *   PMIX_SUCCESS on success, ow PMIX_ERROR
- */
-typedef int (*pmix_compress_base_module_decompress_fn_t)
-    (char * cname, char **fname);
-typedef int (*pmix_compress_base_module_decompress_nb_fn_t)
-    (char * cname, char **fname, pid_t *child_pid);
-
-/**
  * Compress a string
  *
  * Arguments:
@@ -95,6 +65,20 @@
 typedef bool (*pmix_compress_base_module_decompress_string_fn_t)(char **outstring,
                                                                  uint8_t *inbytes, size_t len);
 
+/**
+ * Compress a block
+ *
+ * Arguments:
+ *
+ */
+typedef bool (*pmix_compress_base_module_compress_fn_t)(uint8_t *inbytes,
+                                                        size_t size,
+                                                        uint8_t **outbytes,
+                                                        size_t *nbytes);
+
+typedef bool (*pmix_compress_base_module_decompress_fn_t)(uint8_t **outbytes, size_t *outlen,
+                                                          uint8_t *inbytes, size_t len);
+
 
 /**
  * Structure for COMPRESS components.
@@ -126,11 +110,9 @@
 
     /** Compress interface */
     pmix_compress_base_module_compress_fn_t       compress;
-    pmix_compress_base_module_compress_nb_fn_t    compress_nb;
 
     /** Decompress Interface */
     pmix_compress_base_module_decompress_fn_t     decompress;
-    pmix_compress_base_module_decompress_nb_fn_t  decompress_nb;
 
     /* COMPRESS STRING */
     pmix_compress_base_module_compress_string_fn_t      compress_string;
diff -Nru pmix-3.2.2~rc1/src/mca/pcompress/zlib/compress_zlib.c pmix-4.0.0/src/mca/pcompress/zlib/compress_zlib.c
--- pmix-3.2.2~rc1/src/mca/pcompress/zlib/compress_zlib.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pcompress/zlib/compress_zlib.c	2021-01-02 08:56:17.000000000 +0000
@@ -26,6 +26,7 @@
 #endif  /* HAVE_UNISTD_H */
 #include <zlib.h>
 
+#include "src/include/pmix_stdint.h"
 #include "src/util/pmix_environ.h"
 #include "src/util/output.h"
 #include "src/util/argv.h"
@@ -39,31 +40,44 @@
 
 #include "compress_zlib.h"
 
-int pmix_compress_zlib_module_init(void)
-{
-    return PMIX_SUCCESS;
-}
-
-int pmix_compress_zlib_module_finalize(void)
-{
-    return PMIX_SUCCESS;
-}
-
-bool pmix_compress_zlib_compress_block(char *instring,
-                                       uint8_t **outbytes,
-                                       size_t *nbytes)
+static bool zlib_compress(uint8_t *inbytes,
+                          size_t inlen,
+                          uint8_t **outbytes,
+                          size_t *outlen);
+
+static bool zlib_decompress(uint8_t **outbytes, size_t *outlen,
+                            uint8_t *inbytes, size_t inlen);
+
+static bool compress_string(char *instring,
+                            uint8_t **outbytes,
+                            size_t *nbytes);
+
+static bool decompress_string(char **outstring,
+                              uint8_t *inbytes, size_t len);
+
+pmix_compress_base_module_t pmix_pcompress_zlib_module = {
+    .compress = zlib_compress,
+    .decompress = zlib_decompress,
+    .compress_string = compress_string,
+    .decompress_string = decompress_string,
+};
+
+
+static bool zlib_compress(uint8_t *inbytes,
+                          size_t inlen,
+                          uint8_t **outbytes,
+                          size_t *outlen)
 {
     z_stream strm;
-    size_t len, outlen;
+    size_t len, len2;
     uint8_t *tmp, *ptr;
-    uint32_t inlen;
     int rc;
 
     /* set default output */
     *outbytes = NULL;
+    *outlen = 0;
 
     /* setup the stream */
-    inlen = strlen(instring);
     memset (&strm, 0, sizeof (strm));
     deflateInit (&strm, 9);
 
@@ -80,8 +94,8 @@
         (void)deflateEnd(&strm);
         return false;
     }
-    strm.next_in = (uint8_t*)instring;
-    strm.avail_in = strlen(instring);
+    strm.next_in = inbytes;
+    strm.avail_in = inlen;
 
     /* allocating the upper bound guarantees zlib will
      * always successfully compress into the available space */
@@ -96,70 +110,127 @@
     }
 
     /* allocate 4 bytes beyond the size reqd by zlib so we
-     * can pass the size of the uncompressed string to the
+     * can pass the size of the uncompressed block to the
      * decompress side */
-    outlen = len - strm.avail_out + sizeof(uint32_t);
-    ptr = (uint8_t*)malloc(outlen);
+    len2 = len - strm.avail_out + sizeof(uint32_t);
+    ptr = (uint8_t*)malloc(len2);
     if (NULL == ptr) {
         free(tmp);
         return false;
     }
     *outbytes = ptr;
-    *nbytes = outlen;
+    *outlen = len2;
 
     /* fold the uncompressed length into the buffer */
     memcpy(ptr, &inlen, sizeof(uint32_t));
     ptr += sizeof(uint32_t);
     /* bring over the compressed data */
-    memcpy(ptr, tmp, outlen-sizeof(uint32_t));
+    memcpy(ptr, tmp, len2-sizeof(uint32_t));
     free(tmp);
     pmix_output_verbose(2, pmix_pcompress_base_framework.framework_output,
-                        "COMPRESS INPUT STRING OF LEN %d OUTPUT SIZE %lu",
-                        inlen, outlen-sizeof(uint32_t));
+                        "COMPRESS INPUT BLOCK OF LEN %"PRIsize_t" OUTPUT SIZE %"PRIsize_t"",
+                        inlen, len2-sizeof(uint32_t));
     return true;  // we did the compression
 }
 
-bool pmix_compress_zlib_uncompress_block(char **outstring,
-                                         uint8_t *inbytes, size_t len)
+static bool compress_string(char *instring,
+                            uint8_t **outbytes,
+                            size_t *nbytes)
+{
+    uint32_t inlen;
+
+    /* setup the stream */
+    inlen = strlen(instring);
+
+    /* compress the string */
+    return zlib_compress((uint8_t*)instring, inlen, outbytes, nbytes);
+}
+
+static bool doit(uint8_t **outbytes, size_t len2,
+                 uint8_t *inbytes, size_t inlen)
 {
     uint8_t *dest;
-    int32_t len2;
     z_stream strm;
     int rc;
 
     /* set the default error answer */
-    *outstring = NULL;
-
-    /* the first 4 bytes contains the uncompressed size */
-    memcpy(&len2, inbytes, sizeof(uint32_t));
-
-    pmix_output_verbose(2, pmix_pcompress_base_framework.framework_output,
-                        "DECOMPRESSING INPUT OF LEN %lu OUTPUT %d", len, len2);
+    *outbytes = NULL;
 
-    /* setting destination to the fully decompressed size, +1 to
-     * hold the NULL terminator */
-    dest = (uint8_t*)malloc(len2+1);
+    /* setting destination to the fully decompressed size */
+    dest = (uint8_t*)malloc(len2);
     if (NULL == dest) {
         return false;
     }
-    memset(dest, 0, len2+1);
+    memset(dest, 0, len2);
 
     memset (&strm, 0, sizeof (strm));
     if (Z_OK != inflateInit(&strm)) {
         free(dest);
         return false;
     }
-    strm.avail_in = len;
-    strm.next_in = (uint8_t*)(inbytes + sizeof(uint32_t));
+    strm.avail_in = inlen;
+    strm.next_in = inbytes;
     strm.avail_out = len2;
-    strm.next_out = (uint8_t*)dest;
+    strm.next_out = dest;
 
     rc = inflate (&strm, Z_FINISH);
     inflateEnd (&strm);
-    /* ensure this is NULL terminated! */
-    dest[len2] = '\0';
-    *outstring = (char*)dest;
+    if (Z_OK == rc) {
+        *outbytes = dest;
+        return true;
+    }
+    free(dest);
+    return false;
+
+}
+static bool zlib_decompress(uint8_t **outbytes, size_t *outlen,
+                            uint8_t *inbytes, size_t inlen)
+{
+    int32_t len2;
+    bool rc;
+    uint8_t *input;
+
+    /* set the default error answer */
+    *outlen = 0;
+
+    /* the first 4 bytes contains the uncompressed size */
+    memcpy(&len2, inbytes, sizeof(uint32_t));
+
     pmix_output_verbose(2, pmix_pcompress_base_framework.framework_output,
-                        "\tFINAL LEN: %lu CODE: %d", strlen(*outstring), rc);
-    return true;
+                        "DECOMPRESSING INPUT OF LEN %"PRIsize_t" OUTPUT %d", inlen, len2);
+
+    input = (uint8_t*)(inbytes + sizeof(uint32_t));  // step over the size
+    rc = doit(outbytes, len2, input, inlen);
+    if (rc) {
+        *outlen = len2;
+        return true;
+    }
+    return false;
+}
+
+static bool decompress_string(char **outstring,
+                              uint8_t *inbytes, size_t len)
+{
+    int32_t len2;
+    bool rc;
+    uint8_t *input;
+
+    /* the first 4 bytes contains the uncompressed size */
+    memcpy(&len2, inbytes, sizeof(uint32_t));
+    /* add one to hold the NULL terminator */
+    ++len2;
+
+    /* decompress the bytes */
+    input = (uint8_t*)(inbytes + sizeof(uint32_t));  // step over the size
+    rc = doit((uint8_t**)outstring, len2, input, len);
+
+    if (rc) {
+        /* ensure this is NULL terminated! */
+        outstring[len2-1] = NULL;
+        return true;
+    }
+
+    /* set the default error answer */
+    *outstring = NULL;
+    return false;
 }
diff -Nru pmix-3.2.2~rc1/src/mca/pcompress/zlib/compress_zlib_component.c pmix-4.0.0/src/mca/pcompress/zlib/compress_zlib_component.c
--- pmix-3.2.2~rc1/src/mca/pcompress/zlib/compress_zlib_component.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pcompress/zlib/compress_zlib_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -27,8 +27,6 @@
 /*
  * Local functionality
  */
-static int compress_zlib_open(void);
-static int compress_zlib_close(void);
 static int compress_zlib_query(pmix_mca_base_module_t **module, int *priority);
 
 /*
@@ -47,40 +45,12 @@
                                PMIX_RELEASE_VERSION),
 
     /* Component open and close functions */
-    .pmix_mca_open_component = compress_zlib_open,
-    .pmix_mca_close_component = compress_zlib_close,
     .pmix_mca_query_component = compress_zlib_query
 };
 
-/*
- * Zlib module
- */
-static pmix_compress_base_module_t loc_module = {
-    /** Initialization Function */
-    .init = pmix_compress_zlib_module_init,
-    /** Finalization Function */
-    .finalize = pmix_compress_zlib_module_finalize,
-
-    /** Compress Function */
-    .compress_string = pmix_compress_zlib_compress_block,
-
-    /** Decompress Function */
-    .decompress_string = pmix_compress_zlib_uncompress_block,
-};
-
-static int compress_zlib_open(void)
-{
-    return PMIX_SUCCESS;
-}
-
-static int compress_zlib_close(void)
-{
-    return PMIX_SUCCESS;
-}
-
 static int compress_zlib_query(pmix_mca_base_module_t **module, int *priority)
 {
-    *module   = (pmix_mca_base_module_t *)&loc_module;
+    *module   = (pmix_mca_base_module_t *)&pmix_pcompress_zlib_module;
     *priority = 50;
 
     return PMIX_SUCCESS;
diff -Nru pmix-3.2.2~rc1/src/mca/pcompress/zlib/compress_zlib.h pmix-4.0.0/src/mca/pcompress/zlib/compress_zlib.h
--- pmix-3.2.2~rc1/src/mca/pcompress/zlib/compress_zlib.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pcompress/zlib/compress_zlib.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2004-2010 The Trustees of Indiana University.
  *                         All rights reserved.
- * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,22 +31,9 @@
 extern "C" {
 #endif
 
-    extern pmix_mca_base_component_t mca_pcompress_zlib_component;
-
-    /*
-     * Module functions
-     */
-    int pmix_compress_zlib_module_init(void);
-    int pmix_compress_zlib_module_finalize(void);
-
-    /*
-     * Actual funcationality
-     */
-    bool pmix_compress_zlib_compress_block(char *instring,
-                                           uint8_t **outbytes,
-                                           size_t *nbytes);
-    bool pmix_compress_zlib_uncompress_block(char **outstring,
-                                             uint8_t *inbytes, size_t len);
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_mca_base_component_t mca_pcompress_zlib_component;
+extern pmix_compress_base_module_t pmix_pcompress_zlib_module;
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/base/base.h pmix-4.0.0/src/mca/pfexec/base/base.h
--- pmix-3.2.2~rc1/src/mca/pfexec/base/base.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+ * Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ */
+
+#ifndef MCA_PFEXEC_BASE_H
+#define MCA_PFEXEC_BASE_H
+
+/*
+ * includes
+ */
+#include "pmix_config.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/common/pmix_iof.h"
+#include "src/mca/pfexec/pfexec.h"
+
+
+BEGIN_C_DECLS
+
+/*
+ * MCA framework
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_pfexec_base_framework;
+/*
+ * Select an available component.
+ */
+PMIX_EXPORT pmix_status_t pmix_pfexec_base_select(void);
+
+typedef struct {
+    int usepty;
+    bool connect_stdin;
+
+    /* private - callers should not modify these fields */
+    int p_stdin[2];
+    int p_stdout[2];
+    int p_stderr[2];
+} pmix_pfexec_base_io_conf_t;
+
+typedef struct {
+    pmix_list_item_t super;
+    pmix_event_t ev;
+    pmix_proc_t proc;
+    pid_t pid;
+    bool completed;
+    int exitcode;
+    int keepalive[2];
+    pmix_pfexec_base_io_conf_t opts;
+    pmix_iof_read_event_t *stdoutev;
+    pmix_iof_read_event_t *stderrev;
+} pmix_pfexec_child_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_child_t);
+
+typedef struct {
+    pmix_event_t *handler;
+    bool active;
+    pmix_list_t children;
+    int timeout_before_sigkill;
+    size_t nextid;
+    bool selected;
+} pmix_pfexec_globals_t;
+
+PMIX_EXPORT extern pmix_pfexec_globals_t pmix_pfexec_globals;
+
+/* define a function that will fork/exec a local proc */
+typedef pmix_status_t (*pmix_pfexec_base_fork_proc_fn_t)(pmix_app_t *app,
+                                                         pmix_pfexec_child_t *child,
+                                                         char **env);
+
+/* define a function type for signaling a local proc */
+typedef pmix_status_t (*pmix_pfexec_base_signal_local_fn_t)(pid_t pd, int signum);
+
+typedef struct {
+    pmix_object_t super;
+    pmix_event_t ev;
+    const pmix_info_t *jobinfo;
+    size_t njinfo;
+    const pmix_app_t *apps;
+    size_t napps;
+    pmix_pfexec_base_fork_proc_fn_t frkfn;
+    pmix_spawn_cbfunc_t cbfunc;
+    void *cbdata;
+} pmix_pfexec_fork_caddy_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_fork_caddy_t);
+
+typedef struct {
+    pmix_object_t super;
+    pmix_event_t ev;
+    pmix_proc_t *proc;
+    int signal;
+    pmix_pfexec_base_signal_local_fn_t sigfn;
+    pmix_lock_t *lock;
+} pmix_pfexec_signal_caddy_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_signal_caddy_t);
+
+
+PMIX_EXPORT void pmix_pfexec_base_spawn_proc(int sd, short args, void *cbdata);
+
+PMIX_EXPORT void pmix_pfexec_base_kill_proc(int sd, short args, void *cbdata);
+
+PMIX_EXPORT void pmix_pfexec_base_signal_proc(int sd, short args, void *cbdata);
+
+PMIX_EXPORT void pmix_pfexec_check_complete(int sd, short args, void *cbdata);
+
+#define PMIX_PFEXEC_SPAWN(j, nj, a, na, fn, cbf, cbd)                       \
+    do {                                                                    \
+        pmix_pfexec_fork_caddy_t *fcd;                                      \
+        fcd = PMIX_NEW(pmix_pfexec_fork_caddy_t);                           \
+        fcd->jobinfo = (j);                                                 \
+        fcd->njinfo = (nj);                                                 \
+        fcd->apps = (a);                                                    \
+        fcd->napps = (na);                                                  \
+        fcd->frkfn = (fn);                                                  \
+        fcd->cbfunc = (cbf);                                                \
+        fcd->cbdata = (cbd);                                                \
+        pmix_event_assign(&(fcd->ev), pmix_globals.evbase, -1,              \
+                          EV_WRITE, pmix_pfexec_base_spawn_proc, fcd);      \
+        PMIX_POST_OBJECT((fcd));                                            \
+        pmix_event_active(&((fcd)->ev), EV_WRITE, 1);                       \
+    } while(0)
+
+#define PMIX_PFEXEC_KILL(scd, r, fn, lk)                                    \
+    do {                                                                    \
+        (scd) = PMIX_NEW(pmix_pfexec_signal_caddy_t);                       \
+        (scd)->proc = (r);                                                  \
+        (scd)->sigfn = (fn);                                                \
+        (scd)->lock = (lk);                                                 \
+        pmix_event_assign(&((scd)->ev), pmix_globals.evbase, -1,            \
+                          EV_WRITE, pmix_pfexec_base_kill_proc, (scd));     \
+        PMIX_POST_OBJECT((scd));                                            \
+        pmix_event_active(&((scd)->ev), EV_WRITE, 1);                       \
+    } while(0)
+
+#define PMIX_PFEXEC_SIGNAL(scd, r, nm, fn, lk)                              \
+    do {                                                                    \
+        (scd) = PMIX_NEW(pmix_pfexec_signal_caddy_t);                       \
+        (scd)->proc = (r);                                                  \
+        (scd)->signal = (nm);                                               \
+        (scd)->sigfn = (fn);                                                \
+        (scd)->lock = (lk);                                                 \
+        pmix_event_assign(&((scd)->ev), pmix_globals.evbase, -1,            \
+                          EV_WRITE, pmix_pfexec_base_signal_proc, (scd));   \
+        PMIX_POST_OBJECT((scd));                                            \
+        pmix_event_active(&((scd)->ev), EV_WRITE, 1);                       \
+    } while(0)
+
+typedef struct {
+    pmix_object_t super;
+    pmix_event_t ev;
+    pmix_pfexec_child_t *child;
+} pmix_pfexec_cmpl_caddy_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_cmpl_caddy_t);
+
+#define PMIX_PFEXEC_CHK_COMPLETE(c)                                         \
+    do {                                                                    \
+        pmix_pfexec_cmpl_caddy_t *pc = PMIX_NEW(pmix_pfexec_cmpl_caddy_t);  \
+        pc->child = (c);                                                    \
+        pmix_event_assign(&((pc)->ev), pmix_globals.evbase, -1,             \
+                          EV_WRITE, pmix_pfexec_check_complete, (pc));      \
+        PMIX_POST_OBJECT((pc));                                             \
+        pmix_event_active(&((pc)->ev), EV_WRITE, 1);                        \
+    } while(0)
+
+/*
+ * Struct written up the pipe from the child to the parent.
+ */
+typedef struct {
+    /* True if the child has died; false if this is just a warning to
+       be printed. */
+    bool fatal;
+    /* Relevant only if fatal==true */
+    int exit_status;
+
+    /* Length of the strings that are written up the pipe after this
+       struct */
+    int file_str_len;
+    int topic_str_len;
+    int msg_str_len;
+} pmix_pfexec_pipe_err_msg_t;
+
+PMIX_EXPORT pmix_status_t pmix_pfexec_base_setup_child(pmix_pfexec_child_t *child);
+
+/*
+ * Max length of strings from the pmix_pfexec_pipe_err_msg_t
+ */
+#define PMIX_PFEXEC_MAX_FILE_LEN 511
+#define PMIX_PFEXEC_MAX_TOPIC_LEN PMIX_PFEXEC_MAX_FILE_LEN
+
+
+END_C_DECLS
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/base/help-pfexec-base.txt pmix-4.0.0/src/mca/pfexec/base/help-pfexec-base.txt
--- pmix-3.2.2~rc1/src/mca/pfexec/base/help-pfexec-base.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/base/help-pfexec-base.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,103 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2014      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2017-2019 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English general help file for Open RTE's ODLS Framework
+#
+[orte-odls-base:could-not-kill]
+WARNING: A process refused to die despite all the efforts!
+This process may still be running and/or consuming resources.
+
+Host: %s
+PID:  %d
+
+[orte-odls-base:could-not-preload-binary]
+WARNING: Could not preload the binary file.
+
+Binary: %s
+
+Will continue attempting to launch the process.
+[orte-odls-base:could-not-preload-files]
+WARNING: Could not preload the files specified.
+
+Fileset: %s
+
+Will continue attempting to launch the process.
+[orte-odls-base:could-not-preload]
+WARNING: Could not preload the requested files and directories.
+
+Binary : %s
+Fileset: %s
+
+Will continue attempting to launch the process.
+
+#
+[orte-odls-base:xterm-rank-out-of-bounds]
+The xterm option was asked to display a rank that is larger
+than the number of procs in the job:
+
+Node:      %s
+Rank:      %d
+Num procs: %d
+
+Note that ranks start with 0, not 1, and must be specified
+accordingly.
+#
+[orte-odls-base:xterm-neg-rank]
+The xterm option was asked to display a rank that is negative:
+
+Rank:      %d
+Num procs: %d
+
+Note that ranks start with 0, not 1, and must be specified
+accordingly.
+#
+[orte-odls-base:show-bindings]
+System has detected external process binding to cores %04lx.
+#
+[warn not bound]
+A request to bind the processes to a %s was made, but the operation
+resulted in the processes being unbound. This was most likely caused
+by the following:
+
+  %s
+
+This is only a warning that can be suppressed in the future by
+setting the odls_warn_if_not_bound MCA parameter to 0. Execution
+will continue.
+
+  Local host:        %s
+  Application name:  %s
+  Action requested:  %s %s
+#
+[error not bound]
+A request to bind the processes to a %s was made, but the operation
+resulted in the processes being unbound. This was most likely caused
+by the following:
+
+  %s
+
+This is an error; your job will now abort.
+
+  Local host:        %s
+  Application name:  %s
+  Action requested:  %s %s
+#
+[orte-odls-base:fork-agent-not-found]
+The specified fork agent was not found:
+
+  Node:        %s
+  Fork agent:  %s
+
+The application cannot be launched.
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/base/Makefile.am pmix-4.0.0/src/mca/pfexec/base/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pfexec/base/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/base/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012-2013 Los Alamos National Security, LLC.
+#                         All rights reserved
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers += \
+        base/base.h
+
+libmca_pfexec_la_SOURCES += \
+        base/pfexec_base_frame.c \
+        base/pfexec_base_select.c \
+        base/pfexec_base_default_fns.c
+
+dist_pmixdata_DATA += base/help-pfexec-base.txt
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/base/pfexec_base_default_fns.c pmix-4.0.0/src/mca/pfexec/base/pfexec_base_default_fns.c
--- pmix-3.2.2~rc1/src/mca/pfexec/base/pfexec_base_default_fns.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/base/pfexec_base_default_fns.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,785 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2011 Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017      Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "pmix_config.h"
+
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#include <sys/types.h>
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+#include <signal.h>
+#ifdef HAVE_UTIL_H
+#include <util.h>
+#endif
+#ifdef HAVE_PTY_H
+#include <pty.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_TERMIOS_H
+#include <termios.h>
+# ifdef HAVE_TERMIO_H
+#  include <termio.h>
+# endif
+#endif
+#ifdef HAVE_LIBUTIL_H
+#include <libutil.h>
+#endif
+
+#include "include/pmix.h"
+#include "include/pmix_server.h"
+#include "include/pmix_common.h"
+
+#include "src/include/pmix_stdint.h"
+#include "src/include/pmix_globals.h"
+#include "src/threads/threads.h"
+#include "src/util/argv.h"
+#include "src/util/context_fns.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/os_dirpath.h"
+#include "src/util/os_path.h"
+#include "src/util/path.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/pmix_pty.h"
+#include "src/util/printf.h"
+#include "src/util/show_help.h"
+#include "src/mca/gds/base/base.h"
+#include "src/mca/ptl/base/base.h"
+
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/mca/pfexec/base/base.h"
+
+static pmix_status_t setup_prefork(pmix_pfexec_child_t *child);
+static pmix_status_t register_nspace(char *nspace,
+                                     pmix_pfexec_fork_caddy_t *fcd);
+
+
+static pmix_status_t setup_path(pmix_app_t *app)
+{
+    pmix_status_t rc;
+    char dir[MAXPATHLEN];
+
+    /* see if the app specifies a working dir */
+    if (NULL != app->cwd) {
+        /* Try to change to the app's cwd and check that the app
+           exists and is executable The function will
+           take care of outputting a pretty error message, if required
+        */
+        if (PMIX_SUCCESS != (rc = pmix_util_check_context_cwd(app))) {
+            /* do not ERROR_LOG - it will be reported elsewhere */
+            return rc;
+        }
+
+        /* The prior function will have done a chdir() to jump us to
+         * wherever the app is to be executed. It seems that chdir doesn't
+         * adjust the $PWD enviro variable when it changes the directory. This
+         * can cause a user to get a different response when doing getcwd vs
+         * looking at the enviro variable. To keep this consistent, we explicitly
+         * ensure that the PWD enviro variable matches the CWD we moved to.
+         *
+         * NOTE: if a user's program does a chdir(), then $PWD will once
+         * again not match getcwd! This is beyond our control - we are only
+         * ensuring they start out matching.
+         */
+        if (NULL == getcwd(dir, sizeof(dir))) {
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        pmix_setenv("PWD", dir, true, &app->env);
+    }
+
+    /* ensure the app is pointing to a full path */
+    rc = pmix_util_check_context_app(app, app->env);
+
+    return rc;
+}
+
+
+void pmix_pfexec_base_spawn_proc(int sd, short args, void *cbdata)
+{
+    (void)sd;
+    (void)args;
+    pmix_pfexec_fork_caddy_t *fcd = (pmix_pfexec_fork_caddy_t*)cbdata;
+    pmix_app_t *app;
+    int i, n;
+    size_t m, k;
+    pmix_status_t rc;
+    char **argv = NULL, **env = NULL;
+    pmix_nspace_t nspace;
+    char basedir[MAXPATHLEN], sock[10];
+    pmix_pfexec_child_t *child;
+    pmix_rank_info_t *info;
+    pmix_namespace_t *nptr;
+    pmix_rank_t rank=0;
+    char tmp[2048], *ptr;
+    bool nohup = false;
+
+    pmix_output_verbose(5, pmix_pfexec_base_framework.framework_output,
+                        "%s pfexec:base spawn proc",
+                        PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    /* establish our baseline working directory - we will be potentially
+     * bouncing around as we execute various apps, but we will always return
+     * to this place as our default directory
+     */
+    if (NULL == getcwd(basedir, sizeof(basedir))) {
+        rc = PMIX_ERROR;
+        goto complete;
+    }
+
+    /* create a namespace for the new job */
+    memset(tmp, 0, 2048);
+    (void)snprintf(tmp, 2047, "%s:%lu", pmix_globals.myid.nspace, (unsigned long)pmix_pfexec_globals.nextid);
+    PMIX_LOAD_NSPACE(nspace, tmp);
+    ++pmix_pfexec_globals.nextid;
+
+    /* add the nspace to the server global list */
+    nptr = PMIX_NEW(pmix_namespace_t);
+    nptr->nspace = strdup(nspace);
+    pmix_list_append(&pmix_globals.nspaces, &nptr->super);
+
+    /* locally cache any job info that will later need to
+     * be communicated to the spawned job */
+    rc = register_nspace(nspace, fcd);
+    if (PMIX_SUCCESS != rc) {
+        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+        PMIX_RELEASE(nptr);
+        goto complete;
+    }
+
+    for (k=0; k < fcd->njinfo; k++) {
+        if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_SET_ENVAR)) {
+
+        } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_ADD_ENVAR)) {
+
+        } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_UNSET_ENVAR)) {
+        } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_PREPEND_ENVAR)) {
+        } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_APPEND_ENVAR)) {
+        } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_NOHUP)) {
+            nohup = PMIX_INFO_TRUE(&fcd->jobinfo[k]);
+        }
+    }
+
+    /* cycle across the apps to prep their environment and
+     * launch their procs */
+    for (m=0; m < fcd->napps; m++) {
+        app = (pmix_app_t*)&fcd->apps[m];
+
+        /* merge our launch environment into the app's */
+        for (i=0; NULL != environ[i]; i++) {
+            ptr = strchr(environ[i], '=');
+            if (NULL == ptr) {
+                continue;
+            }
+            *ptr = '\0';
+            ++ptr;
+            pmix_setenv(environ[i], ptr, false, &app->env);  // do not overwrite a given value
+            --ptr;
+            *ptr = '=';
+        }
+
+        /* process any job-info */
+        if (NULL != fcd->jobinfo) {
+            for (k=0; k < fcd->njinfo; k++) {
+                if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_SET_ENVAR)) {
+
+                } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_ADD_ENVAR)) {
+
+                } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_UNSET_ENVAR)) {
+                } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_PREPEND_ENVAR)) {
+                } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_APPEND_ENVAR)) {
+                } else if (PMIX_CHECK_KEY(&fcd->jobinfo[k], PMIX_NOHUP)) {
+                    nohup = PMIX_INFO_TRUE(&fcd->jobinfo[k]);
+                }
+            }
+        }
+
+        /* check for a fork/exec agent we should use */
+        if (NULL != app->info) {
+            for (k=0; k < app->ninfo; k++) {
+                if (PMIX_CHECK_KEY(&app->info[k], PMIX_FORKEXEC_AGENT)) {
+                    /* we were given a fork agent - use it. We have to put its
+                     * argv at the beginning of the app argv array */
+                    argv = pmix_argv_split(app->info[k].value.data.string, ' ');
+                    /* add in the argv from the app */
+                    for (i=0; NULL != argv[i]; i++) {
+                        pmix_argv_prepend_nosize(&app->argv, argv[i]);
+                    }
+                    if (NULL != app->cmd) {
+                        free(app->cmd);
+                    }
+                    app->cmd = pmix_path_findv(argv[0], X_OK, app->env, NULL);
+                    if (NULL == app->cmd) {
+                        pmix_show_help("help-pfexec-base.txt",
+                                       "fork-agent-not-found",
+                                       true, pmix_globals.hostname, argv[0]);
+                        rc = PMIX_ERR_NOT_FOUND;
+                        pmix_argv_free(argv);
+                        goto complete;
+                    }
+                    pmix_argv_free(argv);
+                }
+            }
+        }
+
+        /* setup the path */
+        if (PMIX_SUCCESS != (rc = setup_path(app))) {
+            goto complete;
+        }
+
+        for (n=0; n < app->maxprocs; n++) {
+            /* create a tracker for this child */
+            child = PMIX_NEW(pmix_pfexec_child_t);
+            PMIX_LOAD_PROCID(&child->proc, nspace, rank);
+            ++rank;
+            pmix_list_append(&pmix_pfexec_globals.children, &child->super);
+
+            /* setup any IOF */
+            child->opts.usepty = PMIX_ENABLE_PTY_SUPPORT;
+            if (PMIX_SUCCESS != (rc = setup_prefork(child))) {
+                PMIX_ERROR_LOG(rc);
+                pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super);
+                PMIX_RELEASE(child);
+                goto complete;
+            }
+
+            /* track some details about the child from our
+             * perspective - note that the child may determine
+             * its own nspace/rank, but that is irrelevant here
+             * as we just need an ID for our own internal tracking */
+            info = PMIX_NEW(pmix_rank_info_t);
+            if (NULL == info) {
+                rc = PMIX_ERR_NOMEM;
+                pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super);
+                PMIX_RELEASE(child);
+                goto complete;
+            }
+            info->pname.nspace = strdup(child->proc.nspace);
+            info->pname.rank = child->proc.rank;
+            info->uid = pmix_globals.uid;
+            info->gid = pmix_globals.gid;
+            pmix_list_append(&nptr->ranks, &info->super);
+
+            /* setup the environment */
+            env = pmix_argv_copy(app->env);
+
+            /* we only support the start of tools and servers, not apps,
+             * so we don't register this nspace or child. However, we do
+             * still want to pass along our connection info and active
+             * modules so the forked process can connect back to us */
+
+            /* pass the nspace */
+            pmix_setenv("PMIX_NAMESPACE", child->proc.nspace, true, &env);
+            pmix_setenv("PMIX_SERVER_NSPACE", child->proc.nspace, true, &env);
+
+            /* pass the rank */
+            memset(tmp, 0, 2048);
+            (void)snprintf(tmp, 2047, "%u", child->proc.rank);
+            pmix_setenv("PMIX_RANK", tmp, true, &env);
+            pmix_setenv("PMIX_SERVER_RANK", tmp, true, &env);
+
+            /* get any PTL contribution such as tmpdir settings for session files */
+            if (PMIX_SUCCESS != (rc = pmix_ptl.setup_fork(&child->proc, &env))) {
+                PMIX_ERROR_LOG(rc);
+                pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super);
+                PMIX_RELEASE(child);
+                goto complete;
+            }
+
+            /* ensure we agree on our hostname */
+            pmix_setenv("PMIX_HOSTNAME", pmix_globals.hostname, true, &env);
+
+            /* communicate our version */
+            pmix_setenv("PMIX_VERSION", PMIX_VERSION, true, &env);
+
+            /* setup a keepalive pipe unless "nohup" was given */
+            if (!nohup) {
+                pipe(child->keepalive);
+                snprintf(sock, 10, "%d", child->keepalive[1]);
+                pmix_setenv("PMIX_KEEPALIVE_PIPE", sock, true, &env);
+            }
+
+            pmix_output_verbose(5, pmix_pfexec_base_framework.framework_output,
+                                "%s pfexec:base spawning child %s",
+                                PMIX_NAME_PRINT(&pmix_globals.myid), app->cmd);
+
+            rc = fcd->frkfn(app, child, env);
+            pmix_argv_free(env);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+                pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super);
+                PMIX_RELEASE(child);
+                goto complete;
+            }
+            PMIX_IOF_READ_ACTIVATE(child->stdoutev);
+            PMIX_IOF_READ_ACTIVATE(child->stderrev);
+        }
+    }
+    rc = PMIX_SUCCESS;
+
+  complete:
+    /* ensure we reset our working directory back to our default location  */
+    if (0 != chdir(basedir)) {
+        PMIX_ERROR_LOG(PMIX_ERROR);
+    }
+
+    /* execute the callback */
+    fcd->cbfunc(rc, nspace, fcd->cbdata);
+    PMIX_RELEASE(fcd);
+    return;
+}
+
+void pmix_pfexec_base_kill_proc(int sd, short args, void *cbdata)
+{
+    (void)sd;
+    (void)args;
+    pmix_pfexec_signal_caddy_t *scd = (pmix_pfexec_signal_caddy_t*)cbdata;
+    pmix_pfexec_child_t *child, *cd;
+
+    /* find the process */
+    child = NULL;
+    PMIX_LIST_FOREACH(cd, &pmix_pfexec_globals.children, pmix_pfexec_child_t) {
+        if (PMIX_CHECK_PROCID(scd->proc, &cd->proc)) {
+            child = cd;
+            break;
+        }
+    }
+    if (NULL == child) {
+        scd->lock->status = PMIX_SUCCESS;
+        PMIX_WAKEUP_THREAD(scd->lock);
+        return;
+    }
+
+#if 0
+    /* if we opened the stdin IOF channel, be sure
+     * we close it */
+    if (NULL != orte_iof.close) {
+        orte_iof.close(&child->name, ORTE_IOF_STDIN);
+    }
+#endif
+
+    /* remove the child from the list so waitpid callback won't
+     * find it as this induces unmanageable race
+     * conditions when we are deliberately killing the process
+     */
+    pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super);
+
+    /* First send a SIGCONT in case the process is in stopped state.
+       If it is in a stopped state and we do not first change it to
+       running, then SIGTERM will not get delivered.  Ignore return
+       value. */
+    PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output,
+                         "%s SENDING SIGCONT",
+                         PMIX_NAME_PRINT(&pmix_globals.myid)));
+    scd->sigfn(child->pid, SIGCONT);
+
+    /* wait a little to give the proc a chance to wakeup */
+    sleep(pmix_pfexec_globals.timeout_before_sigkill);
+    /* issue a SIGTERM */
+    PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output,
+                         "%s SENDING SIGTERM",
+                         PMIX_NAME_PRINT(&pmix_globals.myid)));
+    scd->lock->status = scd->sigfn(child->pid, SIGTERM);
+
+    if (0 != scd->lock->status) {
+        /* wait a little again */
+        sleep(pmix_pfexec_globals.timeout_before_sigkill);
+        /* issue a SIGKILL */
+        PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output,
+                             "%s SENDING SIGKILL",
+                             PMIX_NAME_PRINT(&pmix_globals.myid)));
+        scd->lock->status = scd->sigfn(child->pid, SIGKILL);
+    }
+
+    /* cleanup */
+    PMIX_RELEASE(child);
+    PMIX_WAKEUP_THREAD(scd->lock);
+
+#if 0
+    /* ensure the child's session directory is cleaned up */
+    orte_session_dir_finalize(&child->name);
+#endif
+
+    return;
+}
+
+void pmix_pfexec_base_signal_proc(int sd, short args, void *cbdata)
+{
+    (void)sd;
+    (void)args;
+    pmix_pfexec_signal_caddy_t *scd = (pmix_pfexec_signal_caddy_t*)cbdata;
+    pmix_pfexec_child_t *child, *cd;
+
+    /* find the process */
+    child = NULL;
+    PMIX_LIST_FOREACH(cd, &pmix_pfexec_globals.children, pmix_pfexec_child_t) {
+        if (PMIX_CHECK_PROCID(scd->proc, &cd->proc)) {
+            child = cd;
+            break;
+        }
+    }
+    if (NULL == child) {
+        scd->lock->status = PMIX_SUCCESS;
+        PMIX_WAKEUP_THREAD(scd->lock);
+        return;
+    }
+
+    PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output,
+                         "%s SIGNALING %d",
+                         PMIX_NAME_PRINT(&pmix_globals.myid), scd->signal));
+    scd->lock->status = scd->sigfn(child->pid, scd->signal);
+
+    PMIX_WAKEUP_THREAD(scd->lock);
+}
+
+static pmix_status_t setup_prefork(pmix_pfexec_child_t *child)
+{
+    int ret = -1;
+    pmix_pfexec_base_io_conf_t *opts = &child->opts;
+    pmix_proc_t *targets = NULL;
+    pmix_info_t *directives = NULL;
+
+    fflush(stdout);
+
+    /* first check to make sure we can do ptys */
+#if PMIX_ENABLE_PTY_SUPPORT
+    if (opts->usepty) {
+        ret = pmix_openpty(&(opts->p_stdout[0]), &(opts->p_stdout[1]),
+                           (char*)NULL, (struct termios*)NULL, (struct winsize*)NULL);
+    }
+#else
+    opts->usepty = 0;
+#endif
+
+    if (ret < 0) {
+        opts->usepty = 0;
+        if (pipe(opts->p_stdout) < 0) {
+            PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER);
+            return PMIX_ERR_SYS_OTHER;
+        }
+    }
+    if (opts->connect_stdin) {
+        if (pipe(opts->p_stdin) < 0) {
+            PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER);
+            return PMIX_ERR_SYS_OTHER;
+        }
+    }
+    if (pipe(opts->p_stderr) < 0) {
+        PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER);
+        return PMIX_ERR_SYS_OTHER;
+    }
+
+#if 0
+    /* connect stdin endpoint */
+    if (opts->connect_stdin) {
+        /* and connect the pty to stdin */
+        ret = orte_iof.pull(name, ORTE_IOF_STDIN, opts->p_stdin[1]);
+        if(ORTE_SUCCESS != ret) {
+            ORTE_ERROR_LOG(ret);
+            return ret;
+        }
+    }
+#endif
+    /* connect read ends to IOF */
+    PMIX_IOF_READ_EVENT(&child->stdoutev,
+                        targets, 0, directives, 0, opts->p_stdout[0],
+                        pmix_iof_read_local_handler, false);
+    PMIX_LOAD_PROCID(&child->stdoutev->name, child->proc.nspace, child->proc.rank);
+    child->stdoutev->childproc = (void*)child;
+    child->stdoutev->channel = PMIX_FWD_STDOUT_CHANNEL;
+    PMIX_IOF_READ_EVENT(&child->stderrev,
+                        targets, 0, directives, 0, opts->p_stderr[0],
+                        pmix_iof_read_local_handler, false);
+    PMIX_LOAD_PROCID(&child->stderrev->name, child->proc.nspace, child->proc.rank);
+    child->stderrev->childproc = (void*)child;
+    child->stderrev->channel = PMIX_FWD_STDERR_CHANNEL;
+
+    return PMIX_SUCCESS;
+}
+
+
+pmix_status_t pmix_pfexec_base_setup_child(pmix_pfexec_child_t *child)
+{
+    int ret;
+    pmix_pfexec_base_io_conf_t *opts = &child->opts;
+
+    if (opts->connect_stdin && 0 <= opts->p_stdin[1]) {
+        close(opts->p_stdin[1]);
+        opts->p_stdin[1] = -1;
+    }
+    if (0 <= opts->p_stdout[0]) {
+        close(opts->p_stdout[0]);
+        opts->p_stdout[0] = -1;
+    }
+    if (0 <= opts->p_stderr[0]) {
+        close(opts->p_stderr[0]);
+        opts->p_stderr[0] = -1;
+    }
+
+    if (opts->usepty) {
+        /* disable echo */
+        struct termios term_attrs;
+        if (tcgetattr(opts->p_stdout[1], &term_attrs) < 0) {
+            return PMIX_ERR_SYS_OTHER;
+        }
+        term_attrs.c_lflag &= ~ (ECHO | ECHOE | ECHOK |
+                                 ECHOCTL | ECHOKE | ECHONL);
+        term_attrs.c_iflag &= ~ (ICRNL | INLCR | ISTRIP | INPCK | IXON);
+        term_attrs.c_oflag &= ~ (OCRNL | ONLCR);
+        if (tcsetattr(opts->p_stdout[1], TCSANOW, &term_attrs) == -1) {
+            return PMIX_ERR_SYS_OTHER;
+        }
+        ret = dup2(opts->p_stdout[1], fileno(stdout));
+        if (ret < 0) {
+            return PMIX_ERR_SYS_OTHER;
+        }
+        if (0 <= opts->p_stdout[1]) {
+            close(opts->p_stdout[1]);
+            opts->p_stdout[1] = -1;
+        }
+    } else {
+        if(opts->p_stdout[1] != fileno(stdout)) {
+            ret = dup2(opts->p_stdout[1], fileno(stdout));
+            if (ret < 0) {
+                return PMIX_ERR_SYS_OTHER;
+            }
+            if (0 <= opts->p_stdout[1]) {
+                close(opts->p_stdout[1]);
+                opts->p_stdout[1] = -1;
+            }
+        }
+    }
+    if (opts->connect_stdin) {
+        if(opts->p_stdin[0] != fileno(stdin)) {
+            ret = dup2(opts->p_stdin[0], fileno(stdin));
+            if (ret < 0) {
+                return PMIX_ERR_SYS_OTHER;
+            }
+            if (0 <= opts->p_stdin[0]) {
+                close(opts->p_stdin[0]);
+                opts->p_stdin[0] = -1;
+            }
+        }
+    } else {
+        int fd;
+
+        /* connect input to /dev/null */
+        fd = open("/dev/null", O_RDONLY, 0);
+        if (0 > fd) {
+            return PMIX_ERROR;
+        }
+        if (fd != fileno(stdin)) {
+            ret = dup2(fd, fileno(stdin));
+            if (ret < 0) {
+                close(fd);
+                return PMIX_ERR_SYS_OTHER;
+            }
+        }
+        close(fd);
+    }
+
+    if (opts->p_stderr[1] != fileno(stderr)) {
+        ret = dup2(opts->p_stderr[1], fileno(stderr));
+        if (ret < 0) {
+            return PMIX_ERR_SYS_OTHER;
+        }
+        if (0 <= opts->p_stderr[1]) {
+            close(opts->p_stderr[1]);
+            opts->p_stderr[1] = -1;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t register_nspace(char *nspace,
+                                     pmix_pfexec_fork_caddy_t *fcd)
+{
+    pmix_status_t rc;
+    size_t n, ninfo;
+    int m;
+    uint32_t nprocs, u32;
+    uint16_t u16;
+    pmix_proc_t proc;
+    pmix_rank_t zero=0, rk;
+    pmix_info_t *info = NULL;
+    pmix_namespace_t *nptr, *tmp;
+    void *jinfo, *tmpinfo, *pinfo;
+    pmix_data_array_t darray;
+    char *str;
+
+    /* quick compute the number of procs to be started */
+    nprocs = 0;
+    for (n=0; n < fcd->napps; n++) {
+        nprocs += fcd->apps[n].maxprocs;
+    }
+    if (0 == nprocs) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* see if we already have this nspace */
+    nptr = NULL;
+    PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_namespace_t) {
+        if (0 == strcmp(tmp->nspace, nspace)) {
+            nptr = tmp;
+            break;
+        }
+    }
+    if (NULL == nptr) {
+        nptr = PMIX_NEW(pmix_namespace_t);
+        if (NULL == nptr) {
+            return PMIX_ERR_NOMEM;
+        }
+        nptr->nspace = strdup(nspace);
+        pmix_list_append(&pmix_globals.nspaces, &nptr->super);
+    }
+    nptr->nlocalprocs = nprocs;
+
+    /* start assembling the list */
+    PMIX_INFO_LIST_START(jinfo);
+
+    /* jobid */
+    PMIX_LOAD_PROCID(&proc, nspace, PMIX_RANK_UNDEF);
+    PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_JOBID, &proc, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_INFO_LIST_RELEASE(jinfo);
+        return rc;
+    }
+
+    /* hostname */
+    PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_HOSTNAME, pmix_globals.hostname, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_INFO_LIST_RELEASE(jinfo);
+        return rc;
+    }
+
+    /* node size */
+    PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_NODE_SIZE, &nprocs, PMIX_UINT32);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_INFO_LIST_RELEASE(jinfo);
+        return rc;
+    }
+
+    /* local size */
+    PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_LOCAL_SIZE, &nprocs, PMIX_UINT32);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_INFO_LIST_RELEASE(jinfo);
+        return rc;
+    }
+
+    /* local leader */
+    PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_LOCALLDR, &zero, PMIX_PROC_RANK);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_INFO_LIST_RELEASE(jinfo);
+        return rc;
+    }
+
+    /* add in any info provided by the caller */
+    for (n=0; n < fcd->njinfo; n++) {
+        if (PMIX_ENVAR == fcd->jobinfo[n].value.type) {
+            continue;  // take care of these elsewhere
+        }
+        PMIX_INFO_LIST_XFER(rc, jinfo, &fcd->jobinfo[n]);
+    }
+
+    /* for each app in the job, create an app-array */
+    proc.rank = 0;
+    rk = 0;
+    for (n=0; n < fcd->napps; n++) {
+        PMIX_INFO_LIST_START(tmpinfo);
+        u32 = n;
+        PMIX_INFO_LIST_ADD(rc, tmpinfo, PMIX_APPNUM, &u32, PMIX_UINT32);
+        u32 = fcd->apps[n].maxprocs;
+        PMIX_INFO_LIST_ADD(rc, tmpinfo, PMIX_APP_SIZE, &u32, PMIX_UINT32);
+        PMIX_INFO_LIST_ADD(rc, tmpinfo, PMIX_APPLDR, &proc.rank, PMIX_PROC_RANK);
+        proc.rank += fcd->apps[n].maxprocs;
+        if (NULL != fcd->apps[n].cwd) {
+            PMIX_INFO_LIST_ADD(rc, tmpinfo, PMIX_WDIR, fcd->apps[n].cwd, PMIX_STRING);
+        }
+        str = pmix_argv_join(fcd->apps[n].argv, ' ');
+        PMIX_INFO_LIST_ADD(rc, tmpinfo, PMIX_APP_ARGV, str, PMIX_STRING);
+        /* convert the list into an array */
+        PMIX_INFO_LIST_CONVERT(rc, tmpinfo, &darray);
+        /* release the list */
+        PMIX_INFO_LIST_RELEASE(tmpinfo);
+        /* add it to the job info array */
+        PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_APP_INFO_ARRAY, &darray, PMIX_DATA_ARRAY);
+        /* release the memory - the array was copied */
+        PMIX_DATA_ARRAY_DESTRUCT(&darray);
+        /* for each proc in this app, create a proc-array */
+        for (m=0; m < fcd->apps[n].maxprocs; m++) {
+            PMIX_INFO_LIST_START(pinfo);
+            /* must start with the rank */
+            PMIX_INFO_LIST_ADD(rc, pinfo, PMIX_RANK, &rk, PMIX_PROC_RANK);
+            ++rk;
+            /* app number for this proc */
+            u32 = n;
+            PMIX_INFO_LIST_ADD(rc, pinfo, PMIX_APPNUM, &u32, PMIX_UINT32);
+            /* local rank is the same as rank since we only fork locally */
+            u16 = rk;
+            PMIX_INFO_LIST_ADD(rc, pinfo, PMIX_LOCAL_RANK, &u16, PMIX_UINT16);
+            /* convert the list into an array */
+            PMIX_INFO_LIST_CONVERT(rc, pinfo, &darray);
+            /* release the list */
+            PMIX_INFO_LIST_RELEASE(pinfo);
+            /* add it to the job info array */
+            PMIX_INFO_LIST_ADD(rc, jinfo, PMIX_PROC_DATA, &darray, PMIX_DATA_ARRAY);
+            /* release the memory - the array was copied */
+            PMIX_DATA_ARRAY_DESTRUCT(&darray);
+        }
+    }
+
+    /* convert the full list of job info into an array */
+    PMIX_INFO_LIST_CONVERT(rc, jinfo, &darray);
+    PMIX_INFO_LIST_RELEASE(jinfo);
+
+    info = (pmix_info_t*)darray.array;
+    ninfo = darray.size;
+
+    /* register nspace for each activate components */
+    PMIX_GDS_ADD_NSPACE(rc, nptr->nspace, nprocs, info, ninfo);
+    if (PMIX_SUCCESS == rc) {
+        /* store this data in our own GDS module - we will retrieve
+         * it later so it can be passed down to the launched procs
+         * once they connect to us and we know what GDS module they
+         * are using */
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr,
+                                info, ninfo);
+    }
+
+    PMIX_DATA_ARRAY_DESTRUCT(&darray);
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/base/pfexec_base_frame.c pmix-4.0.0/src/mca/pfexec/base/pfexec_base_frame.c
--- pmix-3.2.2~rc1/src/mca/pfexec/base/pfexec_base_frame.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/base/pfexec_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,285 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2011 Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2011-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2014-2018 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "pmix_config.h"
+#include "include/pmix_common.h"
+#include "src/include/types.h"
+
+#include <string.h>
+#include <signal.h>
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+#include "src/threads/threads.h"
+#include "src/include/pmix_globals.h"
+#include "src/common/pmix_iof.h"
+#include "src/client/pmix_client_ops.h"
+#include "src/util/error.h"
+
+#include "src/mca/pfexec/base/base.h"
+
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/pfexec/base/static-components.h"
+
+/*
+ * Instantiate globals
+ */
+pmix_pfexec_base_module_t pmix_pfexec = {0};
+
+/*
+ * Framework global variables
+ */
+pmix_pfexec_globals_t pmix_pfexec_globals = {0};
+
+static int pmix_pfexec_base_close(void)
+{
+    if (pmix_pfexec_globals.active) {
+        pmix_event_del(pmix_pfexec_globals.handler);
+        pmix_pfexec_globals.active = false;
+    }
+    PMIX_LIST_DESTRUCT(&pmix_pfexec_globals.children);
+    free(pmix_pfexec_globals.handler);
+    pmix_pfexec_globals.selected = false;
+
+    return pmix_mca_base_framework_components_close(&pmix_pfexec_base_framework, NULL);
+}
+
+/* callback from the event library whenever a SIGCHLD is received */
+static void wait_signal_callback(int fd, short event, void *arg)
+{
+    (void)fd;
+    (void)event;
+    pmix_event_t *signal = (pmix_event_t*) arg;
+    int status;
+    pid_t pid;
+    pmix_pfexec_child_t *child;
+
+    PMIX_ACQUIRE_OBJECT(signal);
+
+    if (SIGCHLD != PMIX_EVENT_SIGNAL(signal)) {
+        return;
+    }
+    /* if we haven't spawned anyone, then ignore this */
+    if (0 == pmix_list_get_size(&pmix_pfexec_globals.children)) {
+        return;
+    }
+
+    /* reap all queued waitpids until we
+     * don't get anything valid back */
+    while (1) {
+        pid = waitpid(-1, &status, WNOHANG);
+        if (-1 == pid && EINTR == errno) {
+            /* try it again */
+            continue;
+        }
+        /* if we got garbage, then nothing we can do */
+        if (pid <= 0) {
+            return;
+        }
+
+        /* we are already in an event, so it is safe to access globals */
+        PMIX_LIST_FOREACH(child, &pmix_pfexec_globals.children, pmix_pfexec_child_t) {
+            if (pid == child->pid) {
+                /* record the exit status */
+                if (WIFEXITED(status)) {
+                    child->exitcode = WEXITSTATUS(status);
+                } else {
+                    if (WIFSIGNALED(status)) {
+                        child->exitcode = WTERMSIG(status) + 128;
+                    }
+                }
+                /* mark the child as complete */
+                child->completed = true;
+                if ((NULL == child->stdoutev || !child->stdoutev->active) &&
+                    (NULL == child->stderrev || !child->stderrev->active)) {
+                    PMIX_PFEXEC_CHK_COMPLETE(child);
+                }
+                break;
+            }
+        }
+    }
+}
+
+void pmix_pfexec_check_complete(int sd, short args, void *cbdata)
+{
+    (void)sd;
+    (void)args;
+    pmix_pfexec_cmpl_caddy_t *cd = (pmix_pfexec_cmpl_caddy_t*)cbdata;
+    pmix_info_t info[2];
+    pmix_status_t rc;
+    pmix_pfexec_child_t *child;
+    bool stillalive = false;
+    pmix_proc_t wildcard;
+
+    pmix_list_remove_item(&pmix_pfexec_globals.children, &cd->child->super);
+    /* see if any more children from this nspace are alive */
+    PMIX_LIST_FOREACH(child, &pmix_pfexec_globals.children, pmix_pfexec_child_t) {
+        if (PMIX_CHECK_NSPACE(child->proc.nspace, cd->child->proc.nspace)) {
+            stillalive = true;
+        }
+    }
+    if (!stillalive) {
+        /* generate a local event indicating job terminated */
+        PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+        PMIX_LOAD_NSPACE(wildcard.nspace, cd->child->proc.nspace);
+        PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &wildcard, PMIX_PROC);
+        rc = PMIx_Notify_event(PMIX_ERR_JOB_TERMINATED,
+                               &pmix_globals.myid, PMIX_RANGE_PROC_LOCAL,
+                               info, 2, NULL, NULL);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+    }
+    PMIX_RELEASE(cd->child);
+    PMIX_RELEASE(cd);
+}
+
+static int pmix_pfexec_register(pmix_mca_base_register_flag_t flags)
+{
+    (void)flags;
+    pmix_pfexec_globals.timeout_before_sigkill = 1;
+    pmix_mca_base_var_register("pmix", "pfexec", "base", "sigkill_timeout",
+                                 "Time to wait for a process to die after issuing a kill signal to it",
+                               PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                               PMIX_INFO_LVL_2,
+                               PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                               &pmix_pfexec_globals.timeout_before_sigkill);
+    return PMIX_SUCCESS;
+}
+
+/**
+ * Function for finding and opening either all MCA components, or the one
+ * that was specifically requested via a MCA parameter.
+ */
+static int pmix_pfexec_base_open(pmix_mca_base_open_flag_t flags)
+{
+    sigset_t unblock;
+
+    memset(&pmix_pfexec_globals, 0, sizeof(pmix_pfexec_globals_t));
+
+    /* setup the list of children */
+    PMIX_CONSTRUCT(&pmix_pfexec_globals.children, pmix_list_t);
+    pmix_pfexec_globals.nextid = 1;
+
+    /* ensure that SIGCHLD is unblocked as we need to capture it */
+    if (0 != sigemptyset(&unblock)) {
+        return PMIX_ERROR;
+    }
+    if (0 != sigaddset(&unblock, SIGCHLD)) {
+        return PMIX_ERROR;
+    }
+    if (0 != sigprocmask(SIG_UNBLOCK, &unblock, NULL)) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* set to catch SIGCHLD events */
+    pmix_pfexec_globals.handler = (pmix_event_t*)malloc(sizeof(pmix_event_t));
+    pmix_event_set(pmix_globals.evbase,
+                   pmix_pfexec_globals.handler,
+                   SIGCHLD, PMIX_EV_SIGNAL|PMIX_EV_PERSIST,
+                   wait_signal_callback,
+                   pmix_pfexec_globals.handler);
+    pmix_pfexec_globals.active = true;
+    pmix_event_add(pmix_pfexec_globals.handler, NULL);
+
+     /* Open up all available components */
+    return pmix_mca_base_framework_components_open(&pmix_pfexec_base_framework, flags);
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, pfexec, "PMIx fork/exec Subsystem",
+                                pmix_pfexec_register, pmix_pfexec_base_open, pmix_pfexec_base_close,
+                                mca_pfexec_base_static_components, 0);
+
+
+/**** FRAMEWORK CLASS INSTANTIATIONS ****/
+
+static void chcon(pmix_pfexec_child_t *p)
+{
+    memset(&p->ev, 0, sizeof(pmix_event_t));
+    PMIX_LOAD_PROCID(&p->proc, NULL, PMIX_RANK_UNDEF);
+    p->pid = 0;
+    p->completed = false;
+    p->keepalive[0] = -1;
+    p->keepalive[1] = -1;
+    memset(&p->opts, 0, sizeof(pmix_pfexec_base_io_conf_t));
+    p->opts.p_stdin[0] = -1;
+    p->opts.p_stdin[1] = -1;
+    p->opts.p_stdout[0] = -1;
+    p->opts.p_stdout[1] = -1;
+    p->opts.p_stderr[0] = -1;
+    p->opts.p_stderr[1] = -1;
+    p->stdoutev = NULL;
+    p->stderrev = NULL;
+}
+static void chdes(pmix_pfexec_child_t *p)
+{
+    if (NULL != p->stdoutev) {
+        PMIX_RELEASE(p->stdoutev);
+    }
+    if (NULL != p->stderrev) {
+        PMIX_RELEASE(p->stderrev);
+    }
+    if (0 <= p->keepalive[0]) {
+        close(p->keepalive[0]);
+    }
+    if (0 <= p->keepalive[1]) {
+        close(p->keepalive[1]);
+    }
+}
+PMIX_CLASS_INSTANCE(pmix_pfexec_child_t,
+                    pmix_list_item_t,
+                    chcon, chdes);
+
+static void fccon(pmix_pfexec_fork_caddy_t *p)
+{
+    p->jobinfo = NULL;
+    p->njinfo = 0;
+    p->apps = NULL;
+    p->napps = 0;
+    p->frkfn = NULL;
+    p->cbfunc = NULL;
+    p->cbdata = NULL;
+}
+PMIX_CLASS_INSTANCE(pmix_pfexec_fork_caddy_t,
+                    pmix_object_t,
+                    fccon, NULL);
+
+PMIX_CLASS_INSTANCE(pmix_pfexec_signal_caddy_t,
+                    pmix_object_t,
+                    NULL, NULL);
+
+PMIX_CLASS_INSTANCE(pmix_pfexec_cmpl_caddy_t,
+                    pmix_object_t,
+                    NULL, NULL);
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/base/pfexec_base_select.c pmix-4.0.0/src/mca/pfexec/base/pfexec_base_select.c
--- pmix-3.2.2~rc1/src/mca/pfexec/base/pfexec_base_select.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/base/pfexec_base_select.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,63 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+
+#include "pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/pfexec/base/base.h"
+
+
+/**
+ * Function for selecting one component from all those that are
+ * available.
+ */
+int pmix_pfexec_base_select(void)
+{
+    pmix_pfexec_base_component_t *best_component = NULL;
+    pmix_pfexec_base_module_t *best_module = NULL;
+
+    if (pmix_pfexec_globals.selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    pmix_pfexec_globals.selected = true;
+
+    /*
+     * Select the best component
+     */
+    if (PMIX_SUCCESS != pmix_mca_base_select("pfexec", pmix_pfexec_base_framework.framework_output,
+                                             &pmix_pfexec_base_framework.framework_components,
+                                             (pmix_mca_base_module_t **) &best_module,
+                                             (pmix_mca_base_component_t **) &best_component, NULL) ) {
+        /* This will only happen if no component was selected */
+        return PMIX_ERR_NOT_FOUND;
+    }
+
+    /* Save the winner */
+    pmix_pfexec = *best_module;
+
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/linux/configure.m4 pmix-4.0.0/src/mca/pfexec/linux/configure.m4
--- pmix-3.2.2~rc1/src/mca/pfexec/linux/configure.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/linux/configure.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2011      Los Alamos National Security, LLC.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_pfexec_linux_CONFIG([action-if-found], [action-if-not-found])
+# -----------------------------------------------------------
+AC_DEFUN([MCA_pmix_pfexec_linux_CONFIG],[
+    AC_CONFIG_FILES([src/mca/pfexec/linux/Makefile])
+
+    AC_CHECK_FUNC([fork], [pfexec_linux_happy="yes"], [pfexec_linux_happy="no"])
+
+    AS_IF([test "$pfexec_linux_happy" = "yes"], [$1], [$2])
+
+])dnl
+
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/linux/help-pfexec-linux.txt pmix-4.0.0/src/mca/pfexec/linux/help-pfexec-linux.txt
--- pmix-3.2.2~rc1/src/mca/pfexec/linux/help-pfexec-linux.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/linux/help-pfexec-linux.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,147 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
+# Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017-2019 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is a US/English help file.
+#
+[execve error]
+PMIx tried to fork a new process via the "execve" system call but
+failed.  PMIx checks many things before attempting to launch a
+child process, but nothing is perfect. This error may be indicative
+of another problem on the target host, or even something as silly as
+having specified a directory for your application. Your job will now
+abort.
+
+  Local host:        %s
+  Working dir:       %s
+  Application name:  %s
+  Error:             %s
+#
+[binding not supported]
+PMIx tried to bind a new process, but process binding is not
+supported on the host where it was launched.  The process was killed
+without launching the target application.  Your job will now abort.
+
+  Local host:        %s
+  Application name:  %s
+#
+[binding generic error]
+PMIx tried to bind a new process, but something went wrong.  The
+process was killed without launching the target application.  Your job
+will now abort.
+
+  Local host:        %s
+  Application name:  %s
+  Error message:     %s
+  Location:          %s:%d
+#
+[bound to everything]
+PMIx tried to bind a new process to a specific set of processors,
+but ended up binding it to *all* processors.  This means that the new
+process is effectively unbound.
+
+This is only a warning -- your job will continue.  You can suppress
+this warning in the future by setting the odls_warn_if_not_bound MCA
+parameter to 0.
+
+  Local host:        %s
+  Application name:  %s
+  Location:          %s:%d
+#
+[slot list and paffinity_alone]
+PMIx detected that both a slot list was specified and the MCA
+parameter "paffinity_alone" was set to true.  Only one of these can be
+used at a time.  Your job will now abort.
+
+  Local host:        %s
+  Application name:  %s
+#
+[iof setup failed]
+PMIx tried to launch a child process but the "IOF child setup"
+failed.  This should not happen.  Your job will now abort.
+
+  Local host:        %s
+  Application name:  %s
+#
+[not bound]
+WARNING: PMIx tried to bind a process but failed.  This is a
+warning only; your job will continue.
+
+  Local host:        %s
+  Application name:  %s
+  Error message:     %s
+  Location:          %s:%d
+#
+[syscall fail]
+A system call failed that should not have.  In this particular case,
+a warning or error message was not displayed that should have been.
+Your job may behave unpredictably after this, or abort.
+
+  Local host:        %s
+  Application name:  %s
+  Function:          %s
+  Location:          %s:%d
+#
+[memory not bound]
+WARNING: PMIx tried to bind a process but failed.  This is a
+warning only; your job will continue, though performance may
+be degraded.
+
+  Local host:        %s
+  Application name:  %s
+  Error message:     %s
+  Location:          %s:%d
+
+#
+[memory binding error]
+PMIx tried to bind memory for a new process but something went
+wrong. The process was killed without launching the target
+application. Your job will now abort.
+
+  Local host:        %s
+  Application name:  %s
+  Error message: %s
+  Location:  %s:%d
+#
+[set limit]
+Error message received from:
+
+  Local host:        %s
+  Application name:  %s
+  Location:  %s:%d
+
+Message:
+
+%s
+#
+[incorrectly-bound]
+WARNING: PMIx incorrectly bound a process to the daemon's cores.
+This is a warning only; your job will continue.
+
+  Local host:        %s
+  Application name:  %s
+  Location:          %s:%d
+#
+[wdir-not-found]
+PMIx was unable to launch the specified application as it could not
+change to the specified working directory:
+
+Working directory: %s
+Node: %s
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/linux/Makefile.am pmix-4.0.0/src/mca/pfexec/linux/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pfexec/linux/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/linux/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2017-2019 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+dist_pmixdata_DATA = help-pfexec-linux.txt
+
+sources = \
+        pfexec_linux.h \
+        pfexec_linux_component.c \
+        pfexec_linux.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_pfexec_linux_DSO
+component_noinst =
+component_install = mca_pfexec_linux.la
+else
+component_noinst = libmca_pfexec_linux.la
+component_install =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_pfexec_linux_la_SOURCES = $(sources)
+mca_pfexec_linux_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_pfexec_linux_la_SOURCES =$(sources)
+libmca_pfexec_linux_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/linux/pfexec_linux.c pmix-4.0.0/src/mca/pfexec/linux/pfexec_linux.c
--- pmix-3.2.2~rc1/src/mca/pfexec/linux/pfexec_linux.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/linux/pfexec_linux.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,609 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2008 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2010 Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2007      Evergrid, Inc. All rights reserved.
+ * Copyright (c) 2008-2017 Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2010      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017      Rutgers, The State University of New Jersey.
+ *                         All rights reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/*
+ * There is a complicated sequence of events that occurs when the
+ * parent forks a child process that is intended to launch the target
+ * executable.
+ *
+ * Before the child process exec's the target executable, it might tri
+ * to set the affinity of that new child process according to a
+ * complex series of rules.  This binding may fail in a myriad of
+ * different ways.  A lot of this code deals with reporting that error
+ * occurately to the end user.  This is a complex task in itself
+ * because the child process is not "really" an PMIX process -- all
+ * error reporting must be proxied up to the parent who can use normal
+ * PMIX error reporting mechanisms.
+ *
+ * Here's a high-level description of what is occurring in this file:
+ *
+ * - parent opens a pipe
+ * - parent forks a child
+ * - parent blocks reading on the pipe: the pipe will either close
+ *   (indicating that the child successfully exec'ed) or the child will
+ *   write some proxied error data up the pipe
+ *
+ * - the child tries to set affinity and do other housekeeping in
+ *   preparation of exec'ing the target executable
+ * - if the child fails anywhere along the way, it sends a message up
+ *   the pipe to the parent indicating what happened -- including a
+ *   rendered error message detailing the problem (i.e., human-readable).
+ * - it is important that the child renders the error message: there
+ *   are so many errors that are possible that the child is really the
+ *   only entity that has enough information to make an accuate error string
+ *   to report back to the user.
+ * - the parent reads this message + rendered string in and uses PMIX
+ *   reporting mechanisms to display it to the user
+ * - if the problem was only a warning, the child continues processing
+ *   (potentially eventually exec'ing the target executable).
+ * - if the problem was an error, the child exits and the parent
+ *   handles the death of the child as appropriate (i.e., this PFEXEC
+ *   simply reports the error -- other things decide what to do).
+ */
+
+#include "pmix_config.h"
+#include "pmix.h"
+#include "src/include/types.h"
+
+#include <string.h>
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+#include <signal.h>
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+#include <stdlib.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif  /* HAVE_SYS_STAT_H */
+#include <stdarg.h>
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+#include <ctype.h>
+
+#include "src/class/pmix_pointer_array.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/show_help.h"
+#include "src/util/fd.h"
+#include "src/util/error.h"
+
+#include "src/include/pmix_globals.h"
+#include "src/util/name_fns.h"
+#include "src/threads/threads.h"
+
+#include "src/mca/pfexec/base/base.h"
+#include "src/mca/pfexec/linux/pfexec_linux.h"
+
+/*
+ * Module functions (function pointers used in a struct)
+ */
+static pmix_status_t spawn_job(const pmix_info_t job_info[], size_t ninfo,
+                               const pmix_app_t apps[], size_t napps,
+                               pmix_spawn_cbfunc_t cbfunc, void *cbdata);
+static pmix_status_t kill_proc(pmix_proc_t *proc);
+static pmix_status_t signal_proc(pmix_proc_t *proc, int32_t signal);
+
+/*
+ * Explicitly declared functions so that we can get the noreturn
+ * attribute registered with the compiler.
+ */
+static void send_error_show_help(int fd, int exit_status,
+                                 const char *file, const char *topic, ...)
+    __pmix_attribute_noreturn__;
+
+static void do_child(pmix_app_t *cd, char **env, pmix_pfexec_child_t *child, int write_fd)
+    __pmix_attribute_noreturn__;
+
+
+/*
+ * Module
+ */
+pmix_pfexec_base_module_t pmix_pfexec_linux_module = {
+    .spawn_job = spawn_job,
+    .kill_proc = kill_proc,
+    .signal_proc = signal_proc,
+};
+
+
+/* deliver a signal to a specified pid. */
+static pmix_status_t sigproc(pid_t pd, int signum)
+{
+    pid_t pgrp;
+    pid_t pid;
+
+    pid = pd;
+
+#if HAVE_SETPGID
+    pgrp = getpgid(pd);
+    if (-1 != pgrp) {
+        /* target the lead process of the process
+         * group so we ensure that the signal is
+         * seen by all members of that group. This
+         * ensures that the signal is seen by any
+         * child processes our child may have
+         * started
+         */
+        pid = -pgrp;
+    }
+#endif
+
+    if (0 != kill(pid, signum)) {
+        if (ESRCH != errno) {
+            PMIX_OUTPUT_VERBOSE((2, pmix_pfexec_base_framework.framework_output,
+                                 "%s pfexec:linux:SENT SIGNAL %d TO PID %d GOT ERRNO %d",
+                                 PMIX_NAME_PRINT(&pmix_globals.myid), signum, (int)pid, errno));
+            return errno;
+        }
+    }
+    PMIX_OUTPUT_VERBOSE((2, pmix_pfexec_base_framework.framework_output,
+                         "%s pfexec:linux:SENT SIGNAL %d TO PID %d SUCCESS",
+                         PMIX_NAME_PRINT(&pmix_globals.myid), signum, (int)pid));
+    return 0;
+}
+
+static pmix_status_t kill_proc(pmix_proc_t *proc)
+{
+    pmix_status_t rc;
+    pmix_lock_t mylock;
+    pmix_pfexec_signal_caddy_t *kcd;
+
+    PMIX_CONSTRUCT_LOCK(&mylock);
+    PMIX_PFEXEC_KILL(kcd, proc, sigproc, &mylock);
+    PMIX_WAIT_THREAD(&mylock);
+    rc = mylock.status;
+    PMIX_DESTRUCT_LOCK(&mylock);
+    PMIX_RELEASE(kcd);
+
+    return rc;
+}
+
+static pmix_status_t signal_proc(pmix_proc_t *proc, int32_t signal)
+{
+    pmix_status_t rc;
+    pmix_lock_t mylock;
+    pmix_pfexec_signal_caddy_t *scd;
+
+    PMIX_CONSTRUCT_LOCK(&mylock);
+    PMIX_PFEXEC_SIGNAL(scd, proc, signal, sigproc, &mylock);
+    PMIX_WAIT_THREAD(&mylock);
+    rc = mylock.status;
+    PMIX_DESTRUCT_LOCK(&mylock);
+    PMIX_RELEASE(scd);
+
+    return rc;
+}
+
+static void set_handler_linux(int sig)
+{
+    struct sigaction act;
+
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    sigemptyset(&act.sa_mask);
+
+    sigaction(sig, &act, (struct sigaction *)0);
+}
+
+/*
+ * Internal function to write a rendered show_help message back up the
+ * pipe to the waiting parent.
+ */
+static int write_help_msg(int fd, pmix_pfexec_pipe_err_msg_t *msg, const char *file,
+                          const char *topic, va_list ap)
+{
+    int ret;
+    char *str;
+
+    if (NULL == file || NULL == topic) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    str = pmix_show_help_vstring(file, topic, true, ap);
+
+    msg->file_str_len = (int) strlen(file);
+    if (msg->file_str_len > PMIX_PFEXEC_MAX_FILE_LEN) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    msg->topic_str_len = (int) strlen(topic);
+    if (msg->topic_str_len > PMIX_PFEXEC_MAX_TOPIC_LEN) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    msg->msg_str_len = (int) strlen(str);
+
+    /* Only keep writing if each write() succeeds */
+    if (PMIX_SUCCESS != (ret = pmix_fd_write(fd, sizeof(*msg), msg))) {
+        goto out;
+    }
+    if (msg->file_str_len > 0 &&
+        PMIX_SUCCESS != (ret = pmix_fd_write(fd, msg->file_str_len, file))) {
+        goto out;
+    }
+    if (msg->topic_str_len > 0 &&
+        PMIX_SUCCESS != (ret = pmix_fd_write(fd, msg->topic_str_len, topic))) {
+        goto out;
+    }
+    if (msg->msg_str_len > 0 &&
+        PMIX_SUCCESS != (ret = pmix_fd_write(fd, msg->msg_str_len, str))) {
+        goto out;
+    }
+
+ out:
+    free(str);
+    return ret;
+}
+
+
+/* Called from the child to send an error message up the pipe to the
+   waiting parent. */
+static void send_error_show_help(int fd, int exit_status,
+                                 const char *file, const char *topic, ...)
+{
+    va_list ap;
+    pmix_pfexec_pipe_err_msg_t msg;
+
+    msg.fatal = true;
+    msg.exit_status = exit_status;
+
+    /* Send it */
+    va_start(ap, topic);
+    write_help_msg(fd, &msg, file, topic, ap);
+    va_end(ap);
+
+    exit(exit_status);
+}
+
+/* close all open file descriptors w/ exception of stdin/stdout/stderr
+   the pipe up to the parent, and the keepalive pipe. */
+static int close_open_file_descriptors(int write_fd, int keepalive) {
+    DIR *dir = opendir("/proc/self/fd");
+    if (NULL == dir) {
+        return PMIX_ERR_FILE_OPEN_FAILURE;
+    }
+    struct dirent *files;
+
+    /* grab the fd of the opendir above so we don't close in the
+     * middle of the scan. */
+    int dir_scan_fd = dirfd(dir);
+    if(dir_scan_fd < 0 ) {
+        return PMIX_ERR_FILE_OPEN_FAILURE;
+    }
+
+
+    while (NULL != (files = readdir(dir))) {
+        if (!isdigit(files->d_name[0])) {
+            continue;
+        }
+        int fd = strtol(files->d_name, NULL, 10);
+        if (errno == EINVAL || errno == ERANGE) {
+            closedir(dir);
+            return PMIX_ERR_TYPE_MISMATCH;
+        }
+        if (fd >=3 &&
+            fd != write_fd &&
+	        fd != dir_scan_fd &&
+            fd != keepalive) {
+            close(fd);
+        }
+    }
+    closedir(dir);
+    return PMIX_SUCCESS;
+}
+
+static void do_child(pmix_app_t *app, char **env,
+                     pmix_pfexec_child_t *child, int write_fd)
+{
+    int i, errval;
+    sigset_t sigs;
+    long fd, fdmax = sysconf(_SC_OPEN_MAX);
+    char dir[MAXPATHLEN];
+
+#if HAVE_SETPGID
+    /* Set a new process group for this child, so that any
+     * signals we send to it will reach any children it spawns */
+    setpgid(0, 0);
+#endif
+
+    /* Setup the pipe to be close-on-exec */
+    pmix_fd_set_cloexec(write_fd);
+
+    /* setup stdout/stderr so that any error messages that we
+       may print out will get displayed back at pmixrun.
+
+       NOTE: Definitely do this AFTER we check contexts so
+       that any error message from those two functions doesn't
+       come out to the user. IF we didn't do it in this order,
+       THEN a user who gives us a bad executable name or
+       working directory would get N error messages, where
+       N=num_procs. This would be very annoying for large
+       jobs, so instead we set things up so that pmixrun
+       always outputs a nice, single message indicating what
+       happened
+    */
+    if (PMIX_SUCCESS != (i = pmix_pfexec_base_setup_child(child))) {
+        PMIX_ERROR_LOG(i);
+        send_error_show_help(write_fd, 1,
+                             "help-pfexec-linux.txt",
+                             "iof setup failed",
+                             pmix_globals.hostname, app->cmd);
+        /* Does not return */
+    }
+
+    /* close all open file descriptors w/ exception of stdin/stdout/stderr,
+       the pipe used for the IOF INTERNAL messages, and the pipe up to
+       the parent. */
+    if (PMIX_SUCCESS != close_open_file_descriptors(write_fd, child->keepalive[1])) {
+        // close *all* file descriptors -- slow
+        for(fd=3; fd<fdmax; fd++) {
+            if (fd != write_fd && fd != child->keepalive[1]) {
+                close(fd);
+            }
+        }
+    }
+
+    /* Set signal handlers back to the default.  Do this close to
+       the exev() because the event library may (and likely will)
+       reset them.  If we don't do this, the event library may
+       have left some set that, at least on some OS's, don't get
+       reset via fork() or exec().  Hence, the launched process
+       could be unkillable (for example). */
+
+    set_handler_linux(SIGTERM);
+    set_handler_linux(SIGINT);
+    set_handler_linux(SIGHUP);
+    set_handler_linux(SIGPIPE);
+    set_handler_linux(SIGCHLD);
+
+    /* Unblock all signals, for many of the same reasons that we
+       set the default handlers, above.  This is noticable on
+       Linux where the event library blocks SIGTERM, but we don't
+       want that blocked by the launched process. */
+    sigprocmask(0, 0, &sigs);
+    sigprocmask(SIG_UNBLOCK, &sigs, 0);
+
+    /* take us to the correct wdir */
+    if (NULL != app->cwd) {
+        if (0 != chdir(app->cwd)) {
+            send_error_show_help(write_fd, 1,
+                                 "help-pfexec-linux.txt",
+                                 "wdir-not-found",
+                                 "pmixd",
+                                 app->cwd,
+                                 pmix_globals.hostname);
+            /* Does not return */
+        }
+    }
+
+    /* Exec the new executable */
+    execve(app->cmd, app->argv, env);
+    errval = errno;
+    getcwd(dir, sizeof(dir));
+    send_error_show_help(write_fd, 1,
+                         "help-pfexec-linux.txt", "execve error",
+                         pmix_globals.hostname, dir, app->cmd, strerror(errval));
+    /* Does not return */
+}
+
+
+static pmix_status_t do_parent(pmix_app_t *app, pmix_pfexec_child_t *child, int read_fd)
+{
+    pmix_status_t rc;
+    pmix_pfexec_pipe_err_msg_t msg;
+    char file[PMIX_PFEXEC_MAX_FILE_LEN + 1], topic[PMIX_PFEXEC_MAX_TOPIC_LEN + 1], *str = NULL;
+
+    if (child->opts.connect_stdin && 0 <= child->opts.p_stdin[0]) {
+        close(child->opts.p_stdin[0]);
+    }
+    if (0 <= child->opts.p_stdout[1]) {
+        close(child->opts.p_stdout[1]);
+    }
+    if (0 <= child->opts.p_stderr[1])
+        close(child->opts.p_stderr[1]);
+    if (0 <= child->keepalive[1]) {
+        close(child->keepalive[1]);
+    }
+
+    /* Block reading a message from the pipe */
+    while (1) {
+        rc = pmix_fd_read(read_fd, sizeof(msg), &msg);
+
+        /* If the pipe closed, then the child successfully launched */
+        if (PMIX_ERR_TIMEOUT == rc) {
+            break;
+        }
+
+        /* If Something Bad happened in the read, error out */
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            close(read_fd);
+            return rc;
+        }
+
+        /* Read in the strings; ensure to terminate them with \0 */
+        if (msg.file_str_len > 0) {
+            rc = pmix_fd_read(read_fd, msg.file_str_len, file);
+            if (PMIX_SUCCESS != rc) {
+                pmix_show_help("help-pfexec-linux.txt", "syscall fail",
+                               true,
+                               pmix_globals.hostname, app->cmd,
+                               "pmix_fd_read", __FILE__, __LINE__);
+                return rc;
+            }
+            file[msg.file_str_len] = '\0';
+        }
+        if (msg.topic_str_len > 0) {
+            rc = pmix_fd_read(read_fd, msg.topic_str_len, topic);
+            if (PMIX_SUCCESS != rc) {
+                pmix_show_help("help-pfexec-linux.txt", "syscall fail",
+                               true,
+                               pmix_globals.hostname, app->cmd,
+                               "pmix_fd_read", __FILE__, __LINE__);
+                return rc;
+            }
+            topic[msg.topic_str_len] = '\0';
+        }
+        if (msg.msg_str_len > 0) {
+            str = calloc(1, msg.msg_str_len + 1);
+            if (NULL == str) {
+                pmix_show_help("help-pfexec-linux.txt", "syscall fail",
+                               true,
+                               pmix_globals.hostname, app->cmd,
+                               "calloc", __FILE__, __LINE__);
+                return PMIX_ERR_NOMEM;
+            }
+            rc = pmix_fd_read(read_fd, msg.msg_str_len, str);
+            if (PMIX_SUCCESS != rc) {
+                pmix_show_help("help-pfexec-linux.txt", "syscall fail",
+                               true,
+                               pmix_globals.hostname, app->cmd,
+                               "pmix_fd_read", __FILE__, __LINE__);
+                free(str);
+                return rc;
+            }
+        }
+
+        /* Print out what we got.  We already have a rendered string,
+           so use pmix_show_help_norender(). */
+        if (msg.msg_str_len > 0) {
+            fprintf(stderr, "%s\n", str);
+            free(str);
+            str = NULL;
+        }
+
+        /* If msg.fatal is true, then the child exited with an error.
+           Otherwise, whatever we just printed was a warning, so loop
+           around and see what else is on the pipe (or if the pipe
+           closed, indicating that the child launched
+           successfully). */
+        if (msg.fatal) {
+            close(read_fd);
+            if (NULL != str) {
+                free(str);
+            }
+            return PMIX_ERR_SYS_OTHER;
+        }
+        if (NULL != str) {
+            free(str);
+            str = NULL;
+        }
+    }
+
+    /* If we got here, it means that the pipe closed without
+       indication of a fatal error, meaning that the child process
+       launched successfully. */
+    close(read_fd);
+    return PMIX_SUCCESS;
+}
+
+
+/**
+ *  Fork/exec the specified processes
+ */
+static int fork_proc(pmix_app_t *app, pmix_pfexec_child_t *child, char **env)
+{
+    int p[2];
+
+    /* A pipe is used to communicate between the parent and child to
+       indicate whether the exec ultimately succeeded or failed.  The
+       child sets the pipe to be close-on-exec; the child only ever
+       writes anything to the pipe if there is an error (e.g.,
+       executable not found, exec() fails, etc.).  The parent does a
+       blocking read on the pipe; if the pipe closed with no data,
+       then the exec() succeeded.  If the parent reads something from
+       the pipe, then the child was letting us know why it failed. */
+    if (pipe(p) < 0) {
+        PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER);
+        return PMIX_ERR_SYS_OTHER;
+    }
+
+    /* Fork off the child */
+    child->pid = fork();
+
+    if (child->pid < 0) {
+        PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER);
+        return PMIX_ERR_SYS_OTHER;
+    }
+
+    if (child->pid == 0) {
+        if (0 <= p[0]) {
+            close(p[0]);
+        }
+        if (0 <= child->keepalive[0]) {
+            close(child->keepalive[0]);
+            child->keepalive[0] = -1;
+        }
+        do_child(app, env, child, p[1]);
+        /* Does not return */
+    }
+
+    close(p[1]);
+    return do_parent(app, child, p[0]);
+}
+
+
+/**
+ * Launch all processes allocated to the current node.
+ */
+
+static pmix_status_t spawn_job(const pmix_info_t job_info[], size_t ninfo,
+                               const pmix_app_t apps[], size_t napps,
+                               pmix_spawn_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_output_verbose(5, pmix_pfexec_base_framework.framework_output,
+                        "%s pfexec:linux spawning child job",
+                        PMIX_NAME_PRINT(&pmix_globals.myid));
+
+    PMIX_PFEXEC_SPAWN(job_info, ninfo, apps, napps, fork_proc, cbfunc, cbdata);
+
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/linux/pfexec_linux_component.c pmix-4.0.0/src/mca/pfexec/linux/pfexec_linux_component.c
--- pmix-3.2.2~rc1/src/mca/pfexec/linux/pfexec_linux_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/linux/pfexec_linux_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,98 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <stdlib.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <ctype.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/pfexec/pfexec.h"
+#include "src/mca/pfexec/linux/pfexec_linux.h"
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_close(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+pmix_pfexec_base_component_t mca_pfexec_linux_component = {
+    /* First, the mca_component_t struct containing meta information
+    about the component itself */
+    .version = {
+        PMIX_PFEXEC_BASE_VERSION_1_0_0,
+        /* Component name and version */
+        .pmix_mca_component_name = "linux",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_open_component = component_open,
+        .pmix_mca_close_component = component_close,
+        .pmix_mca_query_component = component_query,
+    },
+};
+
+
+
+static pmix_status_t component_open(void)
+{
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_close(void)
+{
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    /* the base open/select logic protects us against operation when
+     * we are NOT in a daemon, so we don't have to check that here
+     */
+
+    /* we have built some logic into the configure.m4 file that checks
+     * to see if we have "fork" support and only builds this component
+     * if we do. Hence, we only get here if we CAN build - in which
+     * case, we definitely should be considered for selection
+     */
+    *priority = 10; /* let others override us - we are the linux */
+    *module = (pmix_mca_base_module_t *) &pmix_pfexec_linux_module;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/linux/pfexec_linux.h pmix-4.0.0/src/mca/pfexec/linux/pfexec_linux.h
--- pmix-3.2.2~rc1/src/mca/pfexec/linux/pfexec_linux.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/linux/pfexec_linux.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2019      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file:
+ */
+
+#ifndef PMIX_PFEXEC_LINUX_H
+#define PMIX_PFEXEC_LINUX_H
+
+#include "pmix_config.h"
+
+#include "src/mca/mca.h"
+
+#include "src/mca/pfexec/pfexec.h"
+
+BEGIN_C_DECLS
+
+/*
+ * PFEXEC Linux module
+ */
+PMIX_EXPORT extern pmix_pfexec_base_module_t pmix_pfexec_linux_module;
+PMIX_EXPORT extern pmix_pfexec_base_component_t mca_pfexec_linux_component;
+
+END_C_DECLS
+
+#endif /* PMIX_PFEXEC_H */
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/Makefile.am pmix-4.0.0/src/mca/pfexec/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pfexec/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# main library setup
+noinst_LTLIBRARIES = libmca_pfexec.la
+libmca_pfexec_la_SOURCES =
+
+# pkgdata setup
+dist_pmixdata_DATA =
+
+# local files
+headers = pfexec.h
+libmca_pfexec_la_SOURCES += $(headers)
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.am
+
+distclean-local:
+	rm -f base/static-components.h
diff -Nru pmix-3.2.2~rc1/src/mca/pfexec/pfexec.h pmix-4.0.0/src/mca/pfexec/pfexec.h
--- pmix-3.2.2~rc1/src/mca/pfexec/pfexec.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pfexec/pfexec.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,93 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * The PMIx Fork/Exec Subsystem
+ *
+ */
+
+#ifndef PMIX_MCA_PFEXEC_H
+#define PMIX_MCA_PFEXEC_H
+
+#include "pmix_config.h"
+#include "include/pmix_common.h"
+#include "src/include/types.h"
+
+#include "src/mca/mca.h"
+
+BEGIN_C_DECLS
+
+/*
+ * pfexec module functions
+ */
+
+/**
+ * Locally fork/exec the provided job
+ */
+typedef pmix_status_t (*pmix_pfexec_base_module_spawn_job_fn_t)(const pmix_info_t job_info[], size_t ninfo,
+                                                                const pmix_app_t apps[], size_t napps,
+                                                                pmix_spawn_cbfunc_t cbfunc, void *cbdata);
+
+/**
+ * Kill the local process we started
+ */
+typedef pmix_status_t (*pmix_pfexec_base_module_kill_process_fn_t)(pmix_proc_t *proc);
+
+/**
+ * Signal local process we started
+ */
+typedef pmix_status_t (*pmix_pfexec_base_module_signal_process_fn_t)(pmix_proc_t *proc, int signum);
+
+/**
+ * pfexec module version
+ */
+typedef struct {
+    pmix_pfexec_base_module_spawn_job_fn_t           spawn_job;
+    pmix_pfexec_base_module_kill_process_fn_t        kill_proc;
+    pmix_pfexec_base_module_signal_process_fn_t      signal_proc;
+} pmix_pfexec_base_module_t;
+
+/**
+ * pfexec component
+ */
+typedef struct {
+    /** component version */
+    pmix_mca_base_component_t version;
+    /** component data */
+    pmix_mca_base_component_data_t base_data;
+} pmix_pfexec_base_component_t;
+
+
+/**
+ * Macro for use in modules that are of type pfexec
+ */
+#define PMIX_PFEXEC_BASE_VERSION_1_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("pfexec", 1, 0, 0)
+
+/* Global structure for accessing PFEXEC functions
+*/
+PMIX_EXPORT extern pmix_pfexec_base_module_t pmix_pfexec;  /* holds selected module's function pointers */
+
+END_C_DECLS
+
+#endif /* MCA_PFEXEC_H */
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/base/base.h pmix-4.0.0/src/mca/ploc/base/base.h
--- pmix-3.2.2~rc1/src/mca/ploc/base/base.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,120 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PMIX_PLOC_BASE_H_
+#define PMIX_PLOC_BASE_H_
+
+#include "src/include/pmix_config.h"
+
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h> /* for struct timeval */
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/ploc/ploc.h"
+
+
+BEGIN_C_DECLS
+
+/*
+ * MCA Framework
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_ploc_base_framework;
+/**
+ * PLOC select function
+ *
+ * Cycle across available components and construct the list
+ * of active modules
+ */
+PMIX_EXPORT pmix_status_t pmix_ploc_base_select(void);
+
+/**
+ * Track an active component / module
+ */
+struct pmix_ploc_base_active_module_t {
+    pmix_list_item_t super;
+    int pri;
+    pmix_ploc_module_t *module;
+    pmix_ploc_base_component_t *component;
+};
+typedef struct pmix_ploc_base_active_module_t pmix_ploc_base_active_module_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_ploc_base_active_module_t);
+
+/* framework globals */
+struct pmix_ploc_globals_t {
+    pmix_lock_t lock;
+    pmix_list_t actives;
+    bool initialized;
+    bool selected;
+};
+typedef struct pmix_ploc_globals_t pmix_ploc_globals_t;
+
+PMIX_EXPORT extern pmix_ploc_globals_t pmix_ploc_globals;
+
+PMIX_EXPORT pmix_status_t pmix_ploc_base_setup_topology(pmix_info_t *info, size_t ninfo);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_load_topology(pmix_topology_t *topo);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_generate_cpuset_string(const pmix_cpuset_t *cpuset,
+                                                                char **cpuset_string);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_parse_cpuset_string(const char *cpuset_string,
+                                                             pmix_cpuset_t *cpuset);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_generate_locality_string(const pmix_cpuset_t *cpuset,
+                                                                  char **locality);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_get_relative_locality(const char *loc1,
+                                                               const char *loc2,
+                                                               pmix_locality_t *locality);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_get_cpuset(pmix_cpuset_t *cpuset,
+                                                    pmix_bind_envelope_t ref);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_compute_distances(pmix_topology_t *topo,
+                                                           pmix_cpuset_t *cpuset,
+                                                           pmix_info_t info[], size_t ninfo,
+                                                           pmix_device_distance_t **dist,
+                                                           size_t *ndist);
+
+PMIX_EXPORT pmix_status_t pmix_ploc_base_pack_cpuset(pmix_buffer_t *buf, pmix_cpuset_t *src,
+                                                     pmix_pointer_array_t *regtypes);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_unpack_cpuset(pmix_buffer_t *buf, pmix_cpuset_t *dest,
+                                                       pmix_pointer_array_t *regtypes);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_copy_cpuset(pmix_cpuset_t *dest, pmix_cpuset_t *src);
+PMIX_EXPORT char* pmix_ploc_base_print_cpuset(pmix_cpuset_t *src);
+PMIX_EXPORT void pmix_ploc_base_destruct_cpuset(pmix_cpuset_t *cpuset);
+PMIX_EXPORT void pmix_ploc_base_release_cpuset(pmix_cpuset_t *ptr, size_t sz);
+
+PMIX_EXPORT pmix_status_t pmix_ploc_base_pack_topology(pmix_buffer_t *buf, pmix_topology_t *src,
+                                                       pmix_pointer_array_t *regtypes);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_unpack_topology(pmix_buffer_t *buf, pmix_topology_t *dest,
+                                                         pmix_pointer_array_t *regtypes);
+PMIX_EXPORT pmix_status_t pmix_ploc_base_copy_topology(pmix_topology_t *dest, pmix_topology_t *src);
+PMIX_EXPORT char* pmix_ploc_base_print_topology(pmix_topology_t *src);
+PMIX_EXPORT void pmix_ploc_base_destruct_topology(pmix_topology_t *ptr);
+PMIX_EXPORT void pmix_ploc_base_release_topology(pmix_topology_t *ptr, size_t sz);
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/base/help-ploc.txt pmix-4.0.0/src/mca/ploc/base/help-ploc.txt
--- pmix-3.2.2~rc1/src/mca/ploc/base/help-ploc.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/base/help-ploc.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,56 @@
+# -*- text -*-
+#
+# Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is a US/English help file
+#
+[reqd-not-found]
+The plog_base_order MCA parameter included a required logging
+channel that is not available:
+
+  Channel:  %s
+
+Please update the parameter and try again.
+#
+[syslog:unrec-level]
+An unrecognized syslog level was given:
+
+  Level:  %s
+
+Please see "man syslog" for a list of defined levels. Input
+parameter strings and their corresponding syslog levels
+recognized by PMIx include:
+
+  Parameter       Level
+   err           LOG_ERR  (default)
+   alert         LOG_ALERT
+   crit          LOG_CRIT
+   emerg         LOG_EMERG
+   warn          LOG_WARNING
+   not           LOG_NOTICE
+   info          LOG_INFO
+   debug         LOG_DEBUG
+
+Please redefine the MCA parameter and try again.
+#
+[syslog:unrec-facility]
+An unsupported or unrecognized value was given for the
+syslog facility (i.e., the type of program calling syslog):
+
+  Value:  %s
+
+Please see "man syslog" for a list of defined facility values.
+PMIx currently supports only the following designations:
+
+   Parameter       Level
+    auth          LOG_AUTH
+    priv          LOG_AUTHPRIV
+    daemon        LOG_DAEMON
+    user          LOG_USER  (default)
+
+Please redefine the MCA parameter and try again.
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/base/Makefile.include pmix-4.0.0/src/mca/ploc/base/Makefile.include
--- pmix-3.2.2~rc1/src/mca/ploc/base/Makefile.include	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/base/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from
+# src/Makefile.am
+
+headers += \
+         base/base.h
+
+sources += \
+        base/ploc_base_frame.c \
+        base/ploc_base_select.c \
+        base/ploc_base_stubs.c
+
+dist_pmixdata_DATA = base/help-ploc.txt
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/base/ploc_base_frame.c pmix-4.0.0/src/mca/ploc/base/ploc_base_frame.c
--- pmix-3.2.2~rc1/src/mca/ploc/base/ploc_base_frame.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/base/ploc_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,112 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ *
+ */
+#include "src/include/pmix_config.h"
+
+#include "include/pmix_common.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/mca/base/base.h"
+#include "src/mca/ploc/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/ploc/base/static-components.h"
+
+/* Instantiate the global vars */
+pmix_ploc_globals_t pmix_ploc_globals = {{0}};
+pmix_ploc_API_module_t pmix_ploc = {
+    .setup_topology = pmix_ploc_base_setup_topology,
+    .load_topology = pmix_ploc_base_load_topology,
+    .generate_cpuset_string = pmix_ploc_base_generate_cpuset_string,
+    .parse_cpuset_string = pmix_ploc_base_parse_cpuset_string,
+    .generate_locality_string = pmix_ploc_base_generate_locality_string,
+    .get_relative_locality = pmix_ploc_base_get_relative_locality,
+    .get_cpuset = pmix_ploc_base_get_cpuset,
+    .compute_distances = pmix_ploc_base_compute_distances,
+    .pack_cpuset = pmix_ploc_base_pack_cpuset,
+    .unpack_cpuset = pmix_ploc_base_unpack_cpuset,
+    .copy_cpuset = pmix_ploc_base_copy_cpuset,
+    .print_cpuset = pmix_ploc_base_print_cpuset,
+    .destruct_cpuset = pmix_ploc_base_destruct_cpuset,
+    .release_cpuset = pmix_ploc_base_release_cpuset,
+    .pack_topology = pmix_ploc_base_pack_topology,
+    .unpack_topology = pmix_ploc_base_unpack_topology,
+    .copy_topology = pmix_ploc_base_copy_topology,
+    .print_topology = pmix_ploc_base_print_topology,
+    .destruct_topology = pmix_ploc_base_destruct_topology,
+    .release_topology = pmix_ploc_base_release_topology
+};
+
+static pmix_status_t pmix_ploc_close(void)
+{
+  pmix_ploc_base_active_module_t *active, *prev;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_SUCCESS;
+    }
+    pmix_ploc_globals.initialized = false;
+    pmix_ploc_globals.selected = false;
+
+    PMIX_LIST_FOREACH_SAFE(active, prev, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+      pmix_list_remove_item(&pmix_ploc_globals.actives, &active->super);
+      if (NULL != active->module->finalize) {
+        active->module->finalize();
+      }
+      PMIX_RELEASE(active);
+    }
+    PMIX_DESTRUCT(&pmix_ploc_globals.actives);
+
+    PMIX_DESTRUCT_LOCK(&pmix_ploc_globals.lock);
+    return pmix_mca_base_framework_components_close(&pmix_ploc_base_framework, NULL);
+}
+
+static pmix_status_t pmix_ploc_open(pmix_mca_base_open_flag_t flags)
+{
+    /* initialize globals */
+    pmix_ploc_globals.initialized = true;
+    PMIX_CONSTRUCT_LOCK(&pmix_ploc_globals.lock);
+    pmix_ploc_globals.lock.active = false;
+    PMIX_CONSTRUCT(&pmix_ploc_globals.actives, pmix_list_t);
+
+    /* Open up all available components */
+    return pmix_mca_base_framework_components_open(&pmix_ploc_base_framework, flags);
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, ploc, "PMIx Network Operations",
+                                NULL, pmix_ploc_open, pmix_ploc_close,
+                                mca_ploc_base_static_components, 0);
+
+PMIX_CLASS_INSTANCE(pmix_ploc_base_active_module_t,
+                    pmix_list_item_t,
+                    NULL, NULL);
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/base/ploc_base_select.c pmix-4.0.0/src/mca/ploc/base/ploc_base_select.c
--- pmix-3.2.2~rc1/src/mca/ploc/base/ploc_base_select.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/base/ploc_base_select.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <string.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/ploc/base/base.h"
+
+/* Function for selecting a prioritized list of components
+ * from all those that are available. */
+int pmix_ploc_base_select(void)
+{
+    pmix_mca_base_component_list_item_t *cli = NULL;
+    pmix_mca_base_component_t *component = NULL;
+    pmix_mca_base_module_t *module = NULL;
+    pmix_ploc_module_t *nmodule;
+    pmix_ploc_base_active_module_t *newmodule, *mod;
+    int rc, priority;
+    bool inserted;
+
+    if (pmix_ploc_globals.selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    pmix_ploc_globals.selected = true;
+
+    /* Query all available components and ask if they have a module */
+    PMIX_LIST_FOREACH(cli, &pmix_ploc_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
+        component = (pmix_mca_base_component_t *) cli->cli_component;
+
+        pmix_output_verbose(5, pmix_ploc_base_framework.framework_output,
+                            "mca:ploc:select: checking available component %s", component->pmix_mca_component_name);
+
+        /* If there's no query function, skip it */
+        if (NULL == component->pmix_mca_query_component) {
+            pmix_output_verbose(5, pmix_ploc_base_framework.framework_output,
+                                "mca:ploc:select: Skipping component [%s]. It does not implement a query function",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* Query the component */
+        pmix_output_verbose(5, pmix_ploc_base_framework.framework_output,
+                            "mca:ploc:select: Querying component [%s]",
+                            component->pmix_mca_component_name);
+        rc = component->pmix_mca_query_component(&module, &priority);
+
+        /* If no module was returned, then skip component */
+        if (PMIX_SUCCESS != rc || NULL == module) {
+            pmix_output_verbose(5, pmix_ploc_base_framework.framework_output,
+                                "mca:ploc:select: Skipping component [%s]. Query failed to return a module",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* If we got a module, keep it */
+        nmodule = (pmix_ploc_module_t*) module;
+        /* let it initialize */
+        if (NULL != nmodule->init && PMIX_SUCCESS != nmodule->init()) {
+            continue;
+        }
+        /* add to the list of selected modules */
+        newmodule = PMIX_NEW(pmix_ploc_base_active_module_t);
+        newmodule->pri = priority;
+        newmodule->module = nmodule;
+        newmodule->component = (pmix_ploc_base_component_t*)cli->cli_component;
+
+        /* maintain priority order */
+        inserted = false;
+        PMIX_LIST_FOREACH(mod, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+            if (priority > mod->pri) {
+                pmix_list_insert_pos(&pmix_ploc_globals.actives,
+                                     (pmix_list_item_t*)mod, &newmodule->super);
+                inserted = true;
+                break;
+            }
+        }
+        if (!inserted) {
+            /* must be lowest priority - add to end */
+            pmix_list_append(&pmix_ploc_globals.actives, &newmodule->super);
+        }
+    }
+
+    if (4 < pmix_output_get_verbosity(pmix_ploc_base_framework.framework_output)) {
+        pmix_output(0, "Final ploc priorities");
+        /* show the prioritized list */
+        PMIX_LIST_FOREACH(mod, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+            pmix_output(0, "\tploc: %s Priority: %d", mod->component->base.pmix_mca_component_name, mod->pri);
+        }
+    }
+
+    return PMIX_SUCCESS;;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/base/ploc_base_stubs.c pmix-4.0.0/src/mca/ploc/base/ploc_base_stubs.c
--- pmix-3.2.2~rc1/src/mca/ploc/base/ploc_base_stubs.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/base/ploc_base_stubs.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,624 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_UTSNAME_H
+#include <sys/utsname.h>
+#endif
+#include <time.h>
+
+#include "include/pmix_common.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/class/pmix_list.h"
+#include "src/util/error.h"
+#include "src/mca/ploc/base/base.h"
+
+
+pmix_status_t pmix_ploc_base_setup_topology(pmix_info_t *info, size_t ninfo)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:setup_topology called");
+
+    /* if we are not a server, then this is an error */
+    if (!PMIX_PROC_IS_SERVER(&pmix_globals.mypeer->proc_type) &&
+        !PMIX_PROC_IS_LAUNCHER(&pmix_globals.mypeer->proc_type)) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->setup_topology) {
+            rc = active->module->setup_topology(info, ninfo);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_load_topology(pmix_topology_t *topo)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:load_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->load_topology) {
+            rc = active->module->load_topology(topo);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_generate_cpuset_string(const pmix_cpuset_t *cpuset,
+                                                    char **cpuset_string)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:generate_cpuset_string called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->get_cpuset) {
+            rc = active->module->generate_cpuset_string(cpuset, cpuset_string);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_parse_cpuset_string(const char *cpuset_string,
+                                                 pmix_cpuset_t *cpuset)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:get_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->parse_cpuset_string) {
+            rc = active->module->parse_cpuset_string(cpuset_string, cpuset);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_generate_locality_string(const pmix_cpuset_t *cpuset,
+                                                      char **locality)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:generate_locality_string called");
+
+    /* if we are not a server, then this is an error */
+    if (!PMIX_PROC_IS_SERVER(&pmix_globals.mypeer->proc_type) &&
+        !PMIX_PROC_IS_LAUNCHER(&pmix_globals.mypeer->proc_type)) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->generate_locality_string) {
+            rc = active->module->generate_locality_string(cpuset, locality);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_get_relative_locality(const char *loc1,
+                                                   const char *loc2,
+                                                   pmix_locality_t *locality)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:get_relative_locality called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->get_relative_locality) {
+            rc = active->module->get_relative_locality(loc1, loc2, locality);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_get_cpuset(pmix_cpuset_t *cpuset,
+                                        pmix_bind_envelope_t ref)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:get_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->get_cpuset) {
+            rc = active->module->get_cpuset(cpuset, ref);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_compute_distances(pmix_topology_t *topo,
+                                               pmix_cpuset_t *cpuset,
+                                               pmix_info_t info[], size_t ninfo,
+                                               pmix_device_distance_t **dist,
+                                               size_t *ndist)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:compute_distances called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->compute_distances) {
+            rc = active->module->compute_distances(topo, cpuset, info, ninfo, dist, ndist);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* indicate that we were able to process the
+                 * request, but didn't get a successful answer.
+                 * We need to return a PMIX_ERR_NOT_AVAILABLE
+                 * response so the caller knows not to raise
+                 * the request to our host */
+                return PMIX_ERR_NOT_AVAILABLE;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_pack_cpuset(pmix_buffer_t *buf,
+                                         pmix_cpuset_t *src,
+                                         pmix_pointer_array_t *regtypes)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:pack_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->pack_cpuset) {
+            rc = active->module->pack_cpuset(buf, src, regtypes);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_unpack_cpuset(pmix_buffer_t *buf,
+                                           pmix_cpuset_t *dest,
+                                           pmix_pointer_array_t *regtypes)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:unpack_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->unpack_cpuset) {
+            rc = active->module->unpack_cpuset(buf, dest, regtypes);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_copy_cpuset(pmix_cpuset_t *dest, pmix_cpuset_t *src)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:copy_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->copy_cpuset) {
+            rc = active->module->copy_cpuset(dest, src);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+char* pmix_ploc_base_print_cpuset(pmix_cpuset_t *src)
+{
+    pmix_ploc_base_active_module_t *active;
+    char *string;
+
+    if (!pmix_ploc_globals.initialized) {
+        return NULL;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:print_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->print_cpuset) {
+            string = active->module->print_cpuset(src);
+            if (NULL != string) {
+                return string;
+            }
+        }
+    }
+
+    return NULL;
+}
+
+void pmix_ploc_base_destruct_cpuset(pmix_cpuset_t *cpuset)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:destruct_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->destruct_cpuset) {
+            rc = active->module->destruct_cpuset(cpuset);
+            if (PMIX_SUCCESS == rc) {
+                return;
+            }
+        }
+    }
+}
+
+void pmix_ploc_base_release_cpuset(pmix_cpuset_t *ptr, size_t sz)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:release_cpuset called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->release_cpuset) {
+            rc = active->module->release_cpuset(ptr, sz);
+            if (PMIX_SUCCESS == rc) {
+                return;
+            }
+        }
+    }
+}
+
+pmix_status_t pmix_ploc_base_pack_topology(pmix_buffer_t *buf,
+                                           pmix_topology_t *src,
+                                           pmix_pointer_array_t *regtypes)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:pack_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->pack_topology) {
+            rc = active->module->pack_topology(buf, src, regtypes);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_unpack_topology(pmix_buffer_t *buf,
+                                             pmix_topology_t *dest,
+                                             pmix_pointer_array_t *regtypes)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:unpack_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->unpack_topology) {
+            rc = active->module->unpack_topology(buf, dest, regtypes);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+pmix_status_t pmix_ploc_base_copy_topology(pmix_topology_t *dest, pmix_topology_t *src)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:copy_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->copy_topology) {
+            rc = active->module->copy_topology(dest, src);
+            if (PMIX_SUCCESS == rc) {
+                return rc;
+            }
+            if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_ERR_NOT_SUPPORTED;
+}
+
+char* pmix_ploc_base_print_topology(pmix_topology_t *src)
+{
+    pmix_ploc_base_active_module_t *active;
+    char *string;
+
+    if (!pmix_ploc_globals.initialized) {
+        return NULL;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:print_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->print_topology) {
+            string = active->module->print_topology(src);
+            if (NULL != string) {
+                return string;
+            }
+        }
+    }
+
+    return NULL;
+}
+
+void pmix_ploc_base_destruct_topology(pmix_topology_t *ptr)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:destruct_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->destruct_topology) {
+            rc = active->module->destruct_topology(ptr);
+            if (PMIX_SUCCESS == rc) {
+                return;
+            }
+        }
+    }
+}
+
+void pmix_ploc_base_release_topology(pmix_topology_t *ptr, size_t sz)
+{
+    pmix_ploc_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_ploc_globals.initialized) {
+        return;
+    }
+
+    pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                        "ploc:release_topology called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_ploc_globals.actives, pmix_ploc_base_active_module_t) {
+        if (NULL != active->module->release_topology) {
+            rc = active->module->release_topology(ptr, sz);
+            if (PMIX_SUCCESS == rc) {
+                return;
+            }
+        }
+    }
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/hwloc/configure.m4 pmix-4.0.0/src/mca/ploc/hwloc/configure.m4
--- pmix-3.2.2~rc1/src/mca/ploc/hwloc/configure.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/hwloc/configure.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,239 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2013      Los Alamos National Security, LLC.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#
+# We have two modes for building hwloc.
+#
+# First is as a co-built hwloc.  In this case, PMIx's CPPFLAGS
+# will be set before configure to include the right -Is to pick up
+# hwloc headers and LIBS will point to where the .la file for
+# hwloc will exist.  When co-building, hwloc's configure will be
+# run already, but the library will not yet be built.  It is ok to run
+# any compile-time (not link-time) tests in this mode.  This mode is
+# used when the --with-hwloc=cobuild option is specified.
+#
+# Second is an external package.  In this case, all compile and link
+# time tests can be run.  This macro must do any CPPFLAGS/LDFLAGS/LIBS
+# modifications it desires in order to compile and link against
+# hwloc.  This mode is used whenever the other modes are not used.
+#
+# MCA_ploc_hwloc_CONFIG([action-if-found], [action-if-not-found])
+# --------------------------------------------------------------------
+AC_DEFUN([MCA_pmix_ploc_hwloc_CONFIG],[
+    AC_CONFIG_FILES([src/mca/ploc/hwloc/Makefile])
+
+    PMIX_VAR_SCOPE_PUSH([hwloc_build_mode])
+
+    AC_ARG_WITH([hwloc],
+                [AC_HELP_STRING([--with-hwloc=DIR],
+                                [Search for hwloc headers and libraries in DIR ])])
+
+    AC_ARG_WITH([hwloc-libdir],
+                [AC_HELP_STRING([--with-hwloc-libdir=DIR],
+                                [Search for hwloc libraries in DIR ])])
+
+    pmix_hwloc_support=0
+    pmix_hwloc_source=""
+    pmix_hwloc_support_will_build=no
+    pmix_have_topology_dup=0
+
+    # figure out our mode...
+    AS_IF([test "$with_hwloc" = "cobuild"],
+          [_PMIX_HWLOC_EMBEDDED_MODE],
+          [_PMIX_HWLOC_EXTERNAL])
+
+    AS_IF([test $pmix_hwloc_support -eq 1],
+          [AC_MSG_CHECKING([hwloc header])
+           AC_MSG_RESULT([$PMIX_HWLOC_HEADER])])
+
+    AC_DEFINE_UNQUOTED([PMIX_HWLOC_HEADER], [$PMIX_HWLOC_HEADER],
+                   [Location of hwloc.h])
+    AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC], [$pmix_hwloc_support],
+                   [Whether or not we have hwloc support])
+    AM_CONDITIONAL([PMIX_HAVE_HWLOC], [test $pmix_hwloc_support -eq 1])
+    AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC_TOPOLOGY_DUP], [$pmix_have_topology_dup],
+                   [Whether or not we have hwloc_topology_dup support])
+
+    PMIX_SUMMARY_ADD([[External Packages]],[[HWLOC]], [pmix_hwloc], [$pmix_hwloc_support_will_build ($pmix_hwloc_source)])
+
+    AS_IF([test $pmix_hwloc_support -eq 1],
+          [$1], [$2])
+])
+
+AC_DEFUN([_PMIX_HWLOC_EMBEDDED_MODE],[
+    AC_MSG_CHECKING([for hwloc])
+    AC_MSG_RESULT([$1])
+
+    AS_IF([test -z "$with_hwloc_header" || test "$with_hwloc_header" = "yes"],
+          [PMIX_HWLOC_HEADER="<hwloc.h>"],
+          [PMIX_HWLOC_HEADER="$with_hwloc_header"])
+
+    AC_MSG_CHECKING([if external hwloc version is 1.5 or greater])
+    AC_COMPILE_IFELSE(
+              [AC_LANG_PROGRAM([[#include <hwloc.h>]],
+              [[
+    #if HWLOC_API_VERSION < 0x00010500
+    #error "hwloc API version is less than 0x00010500"
+    #endif
+              ]])],
+              [AC_MSG_RESULT([yes])],
+              [AC_MSG_RESULT([no])
+               AC_MSG_ERROR([Cannot continue])])
+
+    AC_MSG_CHECKING([if external hwloc version is 1.8 or greater])
+    AC_COMPILE_IFELSE(
+          [AC_LANG_PROGRAM([[#include <hwloc.h>]],
+          [[
+    #if HWLOC_API_VERSION < 0x00010800
+    #error "hwloc API version is less than 0x00010800"
+    #endif
+          ]])],
+          [AC_MSG_RESULT([yes])
+           pmix_have_topology_dup=1],
+          [AC_MSG_RESULT([no])])
+
+    pmix_hwloc_support=1
+    pmix_hwloc_source=cobuild
+    pmix_hwloc_support_will_build=yes
+])
+
+AC_DEFUN([_PMIX_HWLOC_EXTERNAL],[
+    PMIX_VAR_SCOPE_PUSH([pmix_hwloc_dir pmix_hwloc_libdir pmix_hwloc_standard_lib_location pmix_hwloc_standard_header_location pmix_check_hwloc_save_CPPFLAGS pmix_check_hwloc_save_LDFLAGS pmix_check_hwloc_save_LIBS])
+
+    pmix_hwloc_support=0
+    pmix_hwloc_have_dup=0
+    pmix_check_hwloc_save_CPPFLAGS="$CPPFLAGS"
+    pmix_check_hwloc_save_LDFLAGS="$LDFLAGS"
+    pmix_check_hwloc_save_LIBS="$LIBS"
+    pmix_hwloc_standard_header_location=yes
+    pmix_hwloc_standard_lib_location=yes
+
+    AS_IF([test "$with_hwloc" = "internal" || test "$with_hwloc" = "external"],
+          [with_hwloc=])
+
+    if test "$with_hwloc" != "no"; then
+        AC_MSG_CHECKING([for hwloc in])
+        if test ! -z "$with_hwloc" && test "$with_hwloc" != "yes"; then
+            pmix_hwloc_dir=$with_hwloc/include
+            pmix_hwloc_standard_header_location=no
+            pmix_hwloc_standard_lib_location=no
+            AS_IF([test -z "$with_hwloc_libdir" || test "$with_hwloc_libdir" = "yes"],
+                  [if test -d $with_hwloc/lib; then
+                       pmix_hwloc_libdir=$with_hwloc/lib
+                   elif test -d $with_hwloc/lib64; then
+                       pmix_hwloc_libdir=$with_hwloc/lib64
+                   else
+                       AC_MSG_RESULT([Could not find $with_hwloc/lib or $with_hwloc/lib64])
+                       AC_MSG_ERROR([Can not continue])
+                   fi
+                   AC_MSG_RESULT([$pmix_hwloc_dir and $pmix_hwloc_libdir])],
+                  [AC_MSG_RESULT([$with_hwloc_libdir])])
+        else
+            pmix_hwloc_dir=/usr/include
+            if test -d /usr/lib64; then
+                pmix_hwloc_libdir=/usr/lib64
+            elif test -d /usr/lib; then
+                pmix_hwloc_libdir=/usr/lib
+            else
+                AC_MSG_RESULT([not found])
+                AC_MSG_WARN([Could not find /usr/lib or /usr/lib64 - you may])
+                AC_MSG_WARN([need to specify --with-hwloc_libdir=<path>])
+                AC_MSG_ERROR([Can not continue])
+            fi
+            AC_MSG_RESULT([(default search paths)])
+            pmix_hwloc_standard_header_location=yes
+            pmix_hwloc_standard_lib_location=yes
+        fi
+        AS_IF([test ! -z "$with_hwloc_libdir" && test "$with_hwloc_libdir" != "yes"],
+              [pmix_hwloc_libdir="$with_hwloc_libdir"
+               pmix_hwloc_standard_lib_location=no])
+
+        PMIX_CHECK_PACKAGE([pmix_hwloc],
+                           [hwloc.h],
+                           [hwloc],
+                           [hwloc_topology_init],
+                           [-lhwloc],
+                           [$pmix_hwloc_dir],
+                           [$pmix_hwloc_libdir],
+                           [pmix_hwloc_support=1],
+                           [pmix_hwloc_support=0])
+
+        AS_IF([test "$pmix_hwloc_standard_header_location" != "yes"],
+              [PMIX_FLAGS_APPEND_UNIQ(CPPFLAGS, $pmix_hwloc_CPPFLAGS)])
+
+        AS_IF([test "$pmix_hwloc_standard_lib_location" != "yes"],
+              [PMIX_FLAGS_APPEND_UNIQ(LDFLAGS, $pmix_hwloc_LDFLAGS)])
+        PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_hwloc_LIBS)
+    fi
+
+    if test ! -z "$with_hwloc" && test "$with_hwloc" != "no" && test "$pmix_hwloc_support" != "1"; then
+        AC_MSG_WARN([HWLOC SUPPORT REQUESTED AND NOT FOUND])
+        AC_MSG_ERROR([CANNOT CONTINUE])
+    fi
+
+    if test $pmix_hwloc_support = "1"; then
+        AC_MSG_CHECKING([if external hwloc version is 1.5 or greater])
+        AC_COMPILE_IFELSE(
+              [AC_LANG_PROGRAM([[#include <hwloc.h>]],
+              [[
+    #if HWLOC_API_VERSION < 0x00010500
+    #error "hwloc API version is less than 0x00010500"
+    #endif
+              ]])],
+              [AC_MSG_RESULT([yes])],
+              [AC_MSG_RESULT([no])
+               AC_MSG_ERROR([Cannot continue])])
+
+        AC_MSG_CHECKING([if external hwloc version is 1.8 or greater])
+        AC_COMPILE_IFELSE(
+              [AC_LANG_PROGRAM([[#include <hwloc.h>]],
+              [[
+    #if HWLOC_API_VERSION < 0x00010800
+    #error "hwloc API version is less than 0x00010800"
+    #endif
+              ]])],
+              [AC_MSG_RESULT([yes])
+               pmix_have_topology_dup=1],
+              [AC_MSG_RESULT([no])])
+    fi
+
+    CPPFLAGS=$pmix_check_hwloc_save_CPPFLAGS
+    LDFLAGS=$pmix_check_hwloc_save_LDFLAGS
+    LIBS=$pmix_check_hwloc_save_LIBS
+
+    AC_MSG_CHECKING([will hwloc support be built])
+    if test "$pmix_hwloc_support" != "1"; then
+        AC_MSG_RESULT([no])
+        pmix_hwloc_source=none
+        pmix_hwloc_support_will_build=no
+    else
+        AC_MSG_RESULT([yes])
+        pmix_hwloc_source=$pmix_hwloc_dir
+        pmix_hwloc_support_will_build=yes
+        AS_IF([test "$pmix_hwloc_standard_header_location" != "yes"],
+              [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_CPPFLAGS, $pmix_hwloc_CPPFLAGS)
+               PMIX_WRAPPER_FLAGS_ADD(CPPFLAGS, $pmix_hwloc_CPPFLAGS)])
+
+        AS_IF([test "$pmix_hwloc_standard_lib_location" != "yes"],
+              [PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LDFLAGS, $pmix_hwloc_LDFLAGS)
+               PMIX_WRAPPER_FLAGS_ADD(LDFLAGS, $pmix_hwloc_LDFLAGS)])
+        PMIX_FLAGS_APPEND_UNIQ(PMIX_FINAL_LIBS, $pmix_hwloc_LIBS)
+        PMIX_WRAPPER_FLAGS_ADD(LIBS, $pmix_hwloc_LIBS)
+    fi
+
+    # Set output variables
+    PMIX_HWLOC_HEADER="<hwloc.h>"
+
+    PMIX_VAR_SCOPE_POP
+])dnl
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/hwloc/Makefile.am pmix-4.0.0/src/mca/ploc/hwloc/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ploc/hwloc/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/hwloc/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,59 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(ploc_hwloc_CPPFLAGS)
+
+headers = ploc_hwloc.h
+sources = \
+        ploc_hwloc_component.c \
+        ploc_hwloc.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_ploc_hwloc_DSO
+lib =
+lib_sources =
+component = mca_ploc_hwloc.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_ploc_hwloc.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_ploc_hwloc_la_SOURCES = $(component_sources)
+mca_ploc_hwloc_la_LIBADD = $(ploc_hwloc_LIBS)
+mca_ploc_hwloc_la_LDFLAGS = -module -avoid-version $(ploc_hwloc_LDFLAGS)
+if NEED_LIBPMIX
+mca_ploc_hwloc_la_LIBADD += $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_ploc_hwloc_la_SOURCES = $(lib_sources)
+libmca_ploc_hwloc_la_LIBADD = $(ploc_hwloc_LIBS)
+libmca_ploc_hwloc_la_LDFLAGS = -module -avoid-version $(ploc_hwloc_LDFLAGS)
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/hwloc/ploc_hwloc.c pmix-4.0.0/src/mca/ploc/hwloc/ploc_hwloc.c
--- pmix-3.2.2~rc1/src/mca/ploc/hwloc/ploc_hwloc.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/hwloc/ploc_hwloc.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1982 @@
+/*
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017      Cisco Systems, Inc.  All rights reserved
+ * Copyright (c) 2017      Inria.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNISTD_H */
+#include <string.h>
+#include <sys/mman.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#if HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include PMIX_HWLOC_HEADER
+
+#include "src/util/error.h"
+#include "src/util/fd.h"
+#include "src/util/name_fns.h"
+#include "src/util/path.h"
+#include "src/util/printf.h"
+#include "src/util/show_help.h"
+#include "src/mca/bfrops/base/base.h"
+#include "src/mca/pnet/pnet.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/client/pmix_client_ops.h"
+
+#include "src/mca/ploc/base/base.h"
+#include "ploc_hwloc.h"
+
+#if HWLOC_API_VERSION >= 0x20000
+#include <hwloc/shmem.h>
+#endif
+
+static pmix_status_t setup_topology(pmix_info_t *info, size_t ninfo);
+static pmix_status_t load_topology(pmix_topology_t *topo);
+static pmix_status_t generate_cpuset_string(const pmix_cpuset_t *cpuset,
+                                            char **cpuset_string);
+static pmix_status_t parse_cpuset_string(const char *cpuset_string,
+                                         pmix_cpuset_t *cpuset);
+static pmix_status_t generate_locality_string(const pmix_cpuset_t *cpuset,
+                                              char **locality);
+static pmix_status_t get_relative_locality(const char *loc1,
+                                           const char *loc2,
+                                           pmix_locality_t *locality);
+static pmix_status_t get_cpuset(pmix_cpuset_t *cpuset,
+                                pmix_bind_envelope_t ref);
+static pmix_status_t compute_distances(pmix_topology_t *topo,
+                                       pmix_cpuset_t *cpuset,
+                                       pmix_info_t info[], size_t ninfo,
+                                       pmix_device_distance_t **dist,
+                                       size_t *ndist);
+static pmix_status_t pack_cpuset(pmix_buffer_t *buf, pmix_cpuset_t *src,
+                                 pmix_pointer_array_t *regtypes);
+static pmix_status_t unpack_cpuset(pmix_buffer_t *buf, pmix_cpuset_t *dest,
+                                   pmix_pointer_array_t *regtypes);
+static pmix_status_t copy_cpuset(pmix_cpuset_t *dest, pmix_cpuset_t *src);
+static char* print_cpuset(pmix_cpuset_t *src);
+static pmix_status_t destruct_cpuset(pmix_cpuset_t *src);
+static pmix_status_t release_cpuset(pmix_cpuset_t *src, size_t ncpu);
+static pmix_status_t pack_topology(pmix_buffer_t *buf, pmix_topology_t *src,
+                                   pmix_pointer_array_t *regtypes);
+static pmix_status_t unpack_topology(pmix_buffer_t *buf, pmix_topology_t *dest,
+                                     pmix_pointer_array_t *regtypes);
+static pmix_status_t copy_topology(pmix_topology_t *dest, pmix_topology_t *src);
+static char* print_topology(pmix_topology_t *src);
+static pmix_status_t destruct_topology(pmix_topology_t *src);
+static pmix_status_t release_topology(pmix_topology_t *src, size_t ncpu);
+
+static void finalize(void);
+
+pmix_ploc_module_t pmix_ploc_hwloc_module = {
+    .finalize = finalize,
+    .setup_topology = setup_topology,
+    .load_topology = load_topology,
+    .generate_cpuset_string = generate_cpuset_string,
+    .parse_cpuset_string = parse_cpuset_string,
+    .generate_locality_string = generate_locality_string,
+    .get_relative_locality = get_relative_locality,
+    .get_cpuset = get_cpuset,
+    .compute_distances = compute_distances,
+    .pack_cpuset = pack_cpuset,
+    .unpack_cpuset = unpack_cpuset,
+    .copy_cpuset = copy_cpuset,
+    .print_cpuset = print_cpuset,
+    .destruct_cpuset = destruct_cpuset,
+    .release_cpuset = release_cpuset,
+    .pack_topology = pack_topology,
+    .unpack_topology = unpack_topology,
+    .copy_topology = copy_topology,
+    .print_topology = print_topology,
+    .destruct_topology = destruct_topology,
+    .release_topology = release_topology,
+};
+
+static bool topo_in_shmem = false;
+
+#if HWLOC_API_VERSION >= 0x20000
+static bool shmem_available = false;
+static size_t shmemsize = 0;
+static size_t shmemaddr;
+static char *shmemfile = NULL;
+static int shmemfd = -1;
+static bool space_available = false;
+static uint64_t amount_space_avail = 0;
+
+static int parse_map_line(const char *line,
+                          unsigned long *beginp,
+                          unsigned long *endp,
+                          pmix_hwloc_vm_map_kind_t *kindp);
+static int use_hole(unsigned long holebegin,
+                    unsigned long holesize,
+                    unsigned long *addrp,
+                    unsigned long size);
+static int find_hole(pmix_hwloc_vm_hole_kind_t hkind,
+                     size_t *addrp,
+                     size_t size);
+static int enough_space(const char *filename,
+                        size_t space_req,
+                        uint64_t *space_avail,
+                        bool *result);
+#endif
+static pmix_status_t load_xml(char *xml);
+
+static int set_flags(hwloc_topology_t topo, unsigned int flags)
+{
+    #if HWLOC_API_VERSION < 0x20000
+            flags = HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
+    #else
+            int ret = hwloc_topology_set_io_types_filter(topo, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+            if (0 != ret) return ret;
+    #endif
+    if (0 != hwloc_topology_set_flags(topo, flags)) {
+        return PMIX_ERR_INIT;
+    }
+    return PMIX_SUCCESS;
+}
+
+static void finalize(void)
+{
+#if HWLOC_API_VERSION >= 0x20000
+    if (NULL != shmemfile) {
+        unlink(shmemfile);
+        free(shmemfile);
+    }
+    if (0 <= shmemfd) {
+        close(shmemfd);
+    }
+#endif
+    if (NULL != pmix_globals.topology.topology && !pmix_globals.external_topology && !topo_in_shmem) {
+        hwloc_topology_destroy(pmix_globals.topology.topology);
+    }
+    return;
+}
+
+
+pmix_status_t setup_topology(pmix_info_t *info, size_t ninfo)
+{
+    char *xmlbuffer=NULL;
+    int len;
+    size_t n;
+    pmix_kval_t *kv;
+    bool share = false;
+#if HWLOC_API_VERSION >= 0x20000
+    pmix_status_t rc;
+#endif
+
+    /* see if they want us to share the topology with our clients */
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SHARE_TOPOLOGY)) {
+            share = PMIX_INFO_TRUE(&info[n]);
+            break;
+        }
+    }
+
+    if (NULL != pmix_globals.topology.topology) {
+        /* if we need to share it, go do that */
+        if (share) {
+            goto sharetopo;
+        }
+        /* otherwise, we are done */
+        return PMIX_SUCCESS;
+    }
+
+    /* see if they stipulated the type of topology they want */
+    if (NULL != pmix_globals.topology.source) {
+        if (0 != strcasecmp(pmix_globals.topology.source, "hwloc")) {
+            /* they want somebody else */
+            return PMIX_ERR_TAKE_NEXT_OPTION;
+        }
+    }
+
+    /* did they give us one to use? */
+    if (NULL != mca_ploc_hwloc_component.topo_file) {
+        if (0 != hwloc_topology_init((hwloc_topology_t*)&pmix_globals.topology.topology)) {
+            return PMIX_ERR_TAKE_NEXT_OPTION;
+        }
+        if (0 != hwloc_topology_set_xml((hwloc_topology_t)pmix_globals.topology.topology,
+                                        mca_ploc_hwloc_component.topo_file)) {
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+        /* since we are loading this from an external source, we have to
+         * explicitly set a flag so hwloc sets things up correctly
+         */
+        if (0 != set_flags(pmix_globals.topology.topology,
+                           HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
+            hwloc_topology_destroy(pmix_globals.topology.topology);
+            return PMIX_ERROR;
+        }
+        /* now load the topology */
+        if (0 != hwloc_topology_load(pmix_globals.topology.topology)) {
+            hwloc_topology_destroy(pmix_globals.topology.topology);
+            return PMIX_ERROR;
+        }
+    } else {
+        /* we weren't given a topology, so get it for ourselves */
+        if (0 != hwloc_topology_init((hwloc_topology_t*)&pmix_globals.topology.topology)) {
+            return PMIX_ERR_TAKE_NEXT_OPTION;
+        }
+
+        if (0 != set_flags(pmix_globals.topology.topology, 0)) {
+            hwloc_topology_destroy(pmix_globals.topology.topology);
+            return PMIX_ERR_INIT;
+        }
+
+        if (0 != hwloc_topology_load(pmix_globals.topology.topology)) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            hwloc_topology_destroy(pmix_globals.topology.topology);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+    }
+    pmix_globals.topology.source = strdup("hwloc");
+
+    /* if we don't need to share it, then we are done */
+    if (!share) {
+        return PMIX_SUCCESS;
+    }
+
+  sharetopo:
+    /* setup the XML representation(s) */
+
+#if HWLOC_API_VERSION < 0x20000
+     /* pass the topology string as we don't
+      * have HWLOC shared memory available - we do
+      * this so the procs won't read the topology
+      * themselves as this could overwhelm the local
+      * system on large-scale SMPs */
+    if (0 == hwloc_topology_export_xmlbuffer(pmix_globals.topology.topology, &xmlbuffer, &len)) {
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup(PMIX_HWLOC_XML_V1);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        PMIX_VALUE_LOAD(kv->value, xmlbuffer, PMIX_STRING);
+        pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+        /* save it with the deprecated key for older RMs */
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup(PMIX_LOCAL_TOPO);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        PMIX_VALUE_LOAD(kv->value, xmlbuffer, PMIX_STRING);
+        pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+        /* done with the buffer */
+        hwloc_free_xmlbuffer(pmix_globals.topology.topology, xmlbuffer);
+    }
+    /* we don't have the ability to do shared memory, so we are done */
+    return PMIX_SUCCESS;
+#else
+    /* pass the topology as a v2 xml string */
+    if (0 == hwloc_topology_export_xmlbuffer(pmix_globals.topology.topology, &xmlbuffer, &len, 0)) {
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup(PMIX_HWLOC_XML_V2);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        PMIX_VALUE_LOAD(kv->value, xmlbuffer, PMIX_STRING);
+        pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+        /* save it with the deprecated key for older RMs */
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup(PMIX_LOCAL_TOPO);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        PMIX_VALUE_LOAD(kv->value, xmlbuffer, PMIX_STRING);
+        pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+        hwloc_free_xmlbuffer(pmix_globals.topology.topology, xmlbuffer);
+    }
+    /* and as a v1 xml string, should an older client attach */
+    if (0 == hwloc_topology_export_xmlbuffer(pmix_globals.topology.topology, &xmlbuffer, &len, HWLOC_TOPOLOGY_EXPORT_XML_FLAG_V1)) {
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup(PMIX_HWLOC_XML_V1);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        PMIX_VALUE_LOAD(kv->value, xmlbuffer, PMIX_STRING);
+        hwloc_free_xmlbuffer(pmix_globals.topology.topology, xmlbuffer);
+        pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+        /* cannot support the deprecated key here as it would overwrite the HWLOC v2 string */
+    }
+
+    /* if they specified no shared memory, then we are done */
+    if (VM_HOLE_NONE == mca_ploc_hwloc_component.hole_kind) {
+        return PMIX_SUCCESS;
+    }
+
+    /* try and find a hole */
+    if (PMIX_SUCCESS != (rc = find_hole(mca_ploc_hwloc_component.hole_kind,
+                                        &shmemaddr, shmemsize))) {
+        /* we couldn't find a hole, so don't use the shmem support */
+        if (4 < pmix_output_get_verbosity(pmix_ploc_base_framework.framework_output)) {
+            FILE *file = fopen("/proc/self/maps", "r");
+            if (file) {
+                char line[256];
+                pmix_output(0, "%s Dumping /proc/self/maps",
+                            PMIX_NAME_PRINT(&pmix_globals.myid));
+                while (fgets(line, sizeof(line), file) != NULL) {
+                    char *end = strchr(line, '\n');
+                    if (end) {
+                       *end = '\0';
+                    }
+                    pmix_output(0, "%s", line);
+                }
+                fclose(file);
+            }
+        }
+        return PMIX_SUCCESS;
+    }
+    /* create the shmem file in our session dir so it
+     * will automatically get cleaned up */
+    pmix_asprintf(&shmemfile, "%s/hwloc.sm", pmix_server_globals.tmpdir);
+    /* let's make sure we have enough space for the backing file */
+    if (PMIX_SUCCESS != (rc = enough_space(shmemfile, shmemsize,
+                                           &amount_space_avail,
+                                           &space_available))) {
+        pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                            "%s an error occurred while determining "
+                            "whether or not %s could be created for topo shmem.",
+                            PMIX_NAME_PRINT(&pmix_globals.myid), shmemfile);
+        free(shmemfile);
+        shmemfile = NULL;
+        return PMIX_SUCCESS;
+    }
+    if (!space_available) {
+        if (1 < pmix_output_get_verbosity(pmix_ploc_base_framework.framework_output)) {
+            pmix_show_help("help-pmix-ploc-hwloc.txt", "target full", true,
+                           shmemfile, pmix_globals.hostname,
+                           (unsigned long)shmemsize,
+                           (unsigned long long)amount_space_avail);
+        }
+        free(shmemfile);
+        shmemfile = NULL;
+        return PMIX_SUCCESS;
+    }
+    /* enough space is available, so create the segment */
+    if (-1 == (shmemfd = open(shmemfile, O_CREAT | O_RDWR, 0600))) {
+        int err = errno;
+        if (1 < pmix_output_get_verbosity(pmix_ploc_base_framework.framework_output)) {
+            pmix_show_help("help-pmix-ploc-hwloc-hwloc.txt", "sys call fail", true,
+                           pmix_globals.hostname,
+                           "open(2)", "", strerror(err), err);
+        }
+        free(shmemfile);
+        shmemfile = NULL;
+        return PMIX_SUCCESS;
+    }
+    /* ensure nobody inherits this fd */
+    pmix_fd_set_cloexec(shmemfd);
+    /* populate the shmem segment with the topology */
+    if (0 != (rc = hwloc_shmem_topology_write(pmix_globals.topology.topology, shmemfd, 0,
+                                              (void*)shmemaddr, shmemsize, 0))) {
+        pmix_output_verbose(2, pmix_ploc_base_framework.framework_output,
+                            "%s an error occurred while writing topology to %s",
+                            PMIX_NAME_PRINT(&pmix_globals.myid), shmemfile);
+        unlink(shmemfile);
+        free(shmemfile);
+        shmemfile = NULL;
+        close(shmemfd);
+        shmemfd = -1;
+        return PMIX_SUCCESS;
+    }
+    /* record that we did this so we know to clean it up */
+    shmem_available = true;
+
+    /* add the requisite key-values to the global data to be
+     * give to each client */
+    kv = PMIX_NEW(pmix_kval_t);
+    kv->key = strdup(PMIX_HWLOC_SHMEM_FILE);
+    kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+    PMIX_VALUE_LOAD(kv->value, shmemfile, PMIX_STRING);
+    pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+
+    kv = PMIX_NEW(pmix_kval_t);
+    kv->key = strdup(PMIX_HWLOC_SHMEM_ADDR);
+    kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+    PMIX_VALUE_LOAD(kv->value, &shmemaddr, PMIX_SIZE);
+    pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+
+    kv = PMIX_NEW(pmix_kval_t);
+    kv->key = strdup(PMIX_HWLOC_SHMEM_SIZE);
+    kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+    PMIX_VALUE_LOAD(kv->value, &shmemsize, PMIX_SIZE);
+    pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+
+#endif
+
+    return PMIX_SUCCESS;
+}
+
+static char* popstr(pmix_cb_t *cb)
+{
+    pmix_list_t *kvs = &cb->kvs;
+    pmix_kval_t *kv;
+    char *str;
+
+    if (1 != pmix_list_get_size(kvs)) {
+        return NULL;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(kvs);
+    if (PMIX_STRING != kv->value->type) {
+        return NULL;
+    }
+    str = kv->value->data.string;
+    kv->value->data.string = NULL;
+    kv = (pmix_kval_t*)pmix_list_remove_first(kvs);
+    while (NULL != kv) {
+        PMIX_RELEASE(kv);
+        kv = (pmix_kval_t*)pmix_list_remove_first(kvs);
+    }
+    return str;
+}
+
+#if HWLOC_API_VERSION >= 0x20000
+static size_t popsize(pmix_cb_t *cb)
+{
+    pmix_list_t *kvs = &cb->kvs;
+    pmix_kval_t *kv;
+    size_t sz;
+
+    if (1 != pmix_list_get_size(kvs)) {
+        return UINT64_MAX;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(kvs);
+    if (PMIX_SIZE != kv->value->type) {
+        return UINT64_MAX;
+    }
+    sz = kv->value->data.size;
+    kv = (pmix_kval_t*)pmix_list_remove_first(kvs);
+    while (NULL != kv) {
+        PMIX_RELEASE(kv);
+        kv = (pmix_kval_t*)pmix_list_remove_first(kvs);
+    }
+    return sz;
+}
+#endif
+
+static int topology_set_flags (hwloc_topology_t topology, unsigned long flags) {
+#if HWLOC_API_VERSION < 0x20000
+    flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
+#else
+    int ret = hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+    if (0 != ret) {
+        return ret;
+    }
+#endif
+    return hwloc_topology_set_flags(topology, flags);
+}
+
+static pmix_status_t load_xml(char *xml)
+{
+    /* load the topology */
+    if (0 != hwloc_topology_init((hwloc_topology_t*)&pmix_globals.topology.topology)) {
+        return PMIX_ERROR;
+    }
+    if (0 != hwloc_topology_set_xmlbuffer(pmix_globals.topology.topology, xml, strlen(xml)+1)) {
+        hwloc_topology_destroy(pmix_globals.topology.topology);
+        return PMIX_ERROR;
+    }
+    /* since we are loading this from an external source, we have to
+     * explicitly set a flag so hwloc sets things up correctly
+     */
+    if (0 != topology_set_flags(pmix_globals.topology.topology,
+                                HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM)) {
+        hwloc_topology_destroy(pmix_globals.topology.topology);
+        return PMIX_ERROR;
+    }
+    /* now load the topology */
+    if (0 != hwloc_topology_load(pmix_globals.topology.topology)) {
+        hwloc_topology_destroy(pmix_globals.topology.topology);
+        return PMIX_ERROR;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t load_topology(pmix_topology_t *topo)
+{
+    pmix_cb_t cb;
+    pmix_proc_t wildcard;
+    pmix_status_t rc;
+    char *file;
+
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    PMIX_LOAD_PROCID(&wildcard, pmix_globals.myid.nspace, PMIX_RANK_WILDCARD);
+    cb.proc = &wildcard;
+    cb.copy = true;
+
+#if HWLOC_API_VERSION >= 0x20000
+    int fd;
+    uint64_t addr, size;
+
+    /* first try to get the shmem link, if available */
+    cb.key = PMIX_HWLOC_SHMEM_FILE;
+    PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb);
+    if (PMIX_SUCCESS != rc) {
+        cb.key = NULL;
+        PMIX_DESTRUCT(&cb);
+        goto tryxml;
+    }
+    file = popstr(&cb);
+
+    cb.key = PMIX_HWLOC_SHMEM_ADDR;
+    PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb);
+    if (PMIX_SUCCESS != rc) {
+        cb.key = NULL;
+        PMIX_DESTRUCT(&cb);
+        free(file);
+        goto tryxml;
+    }
+    addr = popsize(&cb);
+
+    cb.key = PMIX_HWLOC_SHMEM_SIZE;
+    PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb);
+    if (PMIX_SUCCESS != rc) {
+        cb.key = NULL;
+        PMIX_DESTRUCT(&cb);
+        free(file);
+        goto tryxml;
+    }
+    size = popsize(&cb);
+    cb.key = NULL;
+    PMIX_DESTRUCT(&cb);
+
+    if (0 > (fd = open(file, O_RDONLY))) {
+        free(file);
+        PMIX_ERROR_LOG(PMIX_ERROR);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERROR;
+    }
+    free(file);
+    rc = hwloc_shmem_topology_adopt((hwloc_topology_t*)&pmix_globals.topology.topology,
+                                    fd, 0, (void*)addr, size, 0);
+    if (0 == rc) {
+        /* got it - we are done */
+        pmix_globals.topology.source = strdup("hwloc");
+        topo_in_shmem = true;
+        return PMIX_SUCCESS;
+    }
+
+    /* failed to adopt from shmem, so provide some feedback and
+     * then fallback to other ways to get the topology */
+    if (4 < pmix_output_get_verbosity(pmix_ploc_base_framework.framework_output)) {
+        FILE *file = fopen("/proc/self/maps", "r");
+        if (file) {
+            char line[256];
+            pmix_output(0, "Dumping /proc/self/maps");
+
+            while (fgets(line, sizeof(line), file) != NULL) {
+                char *end = strchr(line, '\n');
+                if (end) {
+                    *end = '\0';
+                }
+                pmix_output(0, "%s", line);
+            }
+            fclose(file);
+        }
+    }
+
+  tryxml:
+    /* try to get the v2 XML string */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.key = PMIX_HWLOC_XML_V2;
+    PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb);
+    if (PMIX_SUCCESS == rc) {
+        file = popstr(&cb);
+        rc = load_xml(file);
+        free(file);
+        cb.key = NULL;
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+
+#endif
+
+    /* try to get the v1 XML string */
+    cb.key = PMIX_HWLOC_XML_V1;
+    PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, &cb);
+    if (PMIX_SUCCESS == rc) {
+        file = popstr(&cb);
+        rc = load_xml(file);
+        free(file);
+        cb.key = NULL;
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* if we got here, then we couldn't find anything */
+    cb.key = NULL;
+    PMIX_DESTRUCT(&cb);
+
+    /* try discovering it for ourselves */
+    if (0 != hwloc_topology_init((hwloc_topology_t*)&pmix_globals.topology.topology) ||
+        0 != topology_set_flags(pmix_globals.topology.topology, 0) ||
+        0 != hwloc_topology_load(pmix_globals.topology.topology)) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t generate_cpuset_string(const pmix_cpuset_t *cpuset,
+                                            char **cpuset_string)
+{
+    char *tmp;
+
+    if (NULL == cpuset || NULL == cpuset->bitmap) {
+        *cpuset_string = NULL;
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* if we aren't the source, then nothing we can do */
+    if (0 != strcasecmp(cpuset->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    hwloc_bitmap_list_asprintf(&tmp, cpuset->bitmap);
+    pmix_asprintf(cpuset_string, "hwloc:%s", tmp);
+    free(tmp);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t parse_cpuset_string(const char *cpuset_string,
+                                         pmix_cpuset_t *cpuset)
+{
+    char *src;
+
+    /* if we aren't the source, then pass */
+    src = strchr(cpuset_string, ':');
+    if (NULL == src) {
+        /* bad string */
+        return PMIX_ERR_BAD_PARAM;
+    }
+    *src = '\0';
+    if (0 != strcasecmp(cpuset_string, "hwloc")) {
+        *src = ':';
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+    *src = ':';
+    ++src;
+
+    cpuset->source = strdup("hwloc");
+    cpuset->bitmap = hwloc_bitmap_alloc();
+    hwloc_bitmap_list_sscanf(cpuset->bitmap, src);
+
+    return PMIX_SUCCESS;
+}
+
+static int get_locality_string_by_depth(int d,
+                                        hwloc_cpuset_t cpuset,
+                                        hwloc_cpuset_t result)
+{
+    hwloc_obj_t obj;
+    unsigned width, w;
+
+    /* get the width of the topology at this depth */
+    width = hwloc_get_nbobjs_by_depth(pmix_globals.topology.topology, d);
+    if (0 == width) {
+        return -1;
+    }
+
+    /* scan all objects at this depth to see if
+     * the location overlaps with them
+     */
+    for (w=0; w < width; w++) {
+        /* get the object at this depth/index */
+        obj = hwloc_get_obj_by_depth(pmix_globals.topology.topology, d, w);
+        /* see if the location intersects with it */
+        if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) {
+            hwloc_bitmap_set(result, w);
+        }
+    }
+
+    return 0;
+}
+
+static pmix_status_t generate_locality_string(const pmix_cpuset_t *cpuset,
+                                              char **loc)
+{
+    char *locality=NULL, *tmp, *t2;
+    unsigned depth, d;
+    hwloc_cpuset_t result;
+    hwloc_obj_type_t type;
+
+    /* if we aren't the source, then pass */
+    if (0 != strcasecmp(cpuset->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* if this proc is not bound, then there is no locality. We
+     * know it isn't bound if the cpuset is NULL, or if it is
+     * all 1's */
+    if (NULL == cpuset->bitmap || hwloc_bitmap_isfull(cpuset->bitmap)) {
+        *loc = NULL;
+        return PMIX_SUCCESS;
+    }
+
+    /* we are going to use a bitmap to save the results so
+     * that we can use a hwloc utility to print them */
+    result = hwloc_bitmap_alloc();
+
+    /* get the max depth of the topology */
+    depth = hwloc_topology_get_depth(pmix_globals.topology.topology);
+
+    /* start at the first depth below the top machine level */
+    for (d=1; d < depth; d++) {
+        /* get the object type at this depth */
+        type = hwloc_get_depth_type(pmix_globals.topology.topology, d);
+        /* if it isn't one of interest, then ignore it */
+        if (HWLOC_OBJ_NODE != type &&
+            HWLOC_OBJ_PACKAGE != type &&
+#if HWLOC_API_VERSION < 0x20000
+            HWLOC_OBJ_CACHE != type &&
+#else
+            HWLOC_OBJ_L1CACHE != type &&
+            HWLOC_OBJ_L2CACHE != type &&
+            HWLOC_OBJ_L3CACHE != type &&
+#endif
+            HWLOC_OBJ_CORE != type &&
+            HWLOC_OBJ_PU != type) {
+            continue;
+        }
+
+        if (get_locality_string_by_depth(d, cpuset->bitmap, result) < 0) {
+            continue;
+        }
+
+        /* it should be impossible, but allow for the possibility
+         * that we came up empty at this depth */
+        if (!hwloc_bitmap_iszero(result)) {
+            hwloc_bitmap_list_asprintf(&tmp, result);
+            switch(type) {
+                case HWLOC_OBJ_NODE:
+                    pmix_asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+                case HWLOC_OBJ_PACKAGE:
+                    pmix_asprintf(&t2, "%sSK%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+#if HWLOC_API_VERSION < 0x20000
+                case HWLOC_OBJ_CACHE:
+                {
+                    unsigned cachedepth = hwloc_get_obj_by_depth(pmix_globals.topology.topology, d, 0)->attr->cache.depth;
+                    if (3 == cachedepth) {
+                        pmix_asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
+                        if (NULL != locality) {
+                            free(locality);
+                        }
+                        locality = t2;
+                        break;
+                    } else if (2 == cachedepth) {
+                        pmix_asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp);
+                        if (NULL != locality) {
+                            free(locality);
+                        }
+                        locality = t2;
+                        break;
+                    } else {
+                        pmix_asprintf(&t2, "%sL1%s:", (NULL == locality) ? "" : locality, tmp);
+                        if (NULL != locality) {
+                            free(locality);
+                        }
+                        locality = t2;
+                        break;
+                    }
+                }
+                    break;
+#else
+                case HWLOC_OBJ_L3CACHE:
+                    pmix_asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+                case HWLOC_OBJ_L2CACHE:
+                    pmix_asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+                case HWLOC_OBJ_L1CACHE:
+                    pmix_asprintf(&t2, "%sL1%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+#endif
+                case HWLOC_OBJ_CORE:
+                    pmix_asprintf(&t2, "%sCR%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+                case HWLOC_OBJ_PU:
+                    pmix_asprintf(&t2, "%sHT%s:", (NULL == locality) ? "" : locality, tmp);
+                    if (NULL != locality) {
+                        free(locality);
+                    }
+                    locality = t2;
+                    break;
+                default:
+                    /* just ignore it */
+                    break;
+            }
+            free(tmp);
+        }
+        hwloc_bitmap_zero(result);
+    }
+
+#if HWLOC_API_VERSION >= 0x20000
+    if (get_locality_string_by_depth(HWLOC_TYPE_DEPTH_NUMANODE, cpuset->bitmap, result) == 0) {
+        /* it should be impossible, but allow for the possibility
+         * that we came up empty at this depth */
+        if (!hwloc_bitmap_iszero(result)) {
+            hwloc_bitmap_list_asprintf(&tmp, result);
+            pmix_asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp);
+            if (NULL != locality) {
+                free(locality);
+            }
+            locality = t2;
+            free(tmp);
+        }
+        hwloc_bitmap_zero(result);
+    }
+#endif
+
+    hwloc_bitmap_free(result);
+
+    /* remove the trailing colon */
+    if (NULL != locality) {
+        locality[strlen(locality)-1] = '\0';
+    }
+    *loc = locality;
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t get_relative_locality(const char *locality1,
+                                           const char *locality2,
+                                           pmix_locality_t *loc)
+{
+    pmix_locality_t locality;
+    char *loc1, *loc2, **set1, **set2;
+    hwloc_bitmap_t bit1, bit2;
+    size_t n1, n2;
+    pmix_status_t rc = PMIX_ERR_TAKE_NEXT_OPTION;
+
+    /* check that locality was generated by us */
+    if (0 != strncasecmp(locality1, "hwloc:", strlen("hwloc:")) ||
+        0 != strncasecmp(locality2, "hwloc:", strlen("hwloc:"))) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+    /* point to the first character past the ':' delimiter */
+    loc1 = (char*)&locality1[strlen("hwloc:")];
+    loc2 = (char*)&locality2[strlen("hwloc:")];
+
+    /* start with what we know - they share a node */
+    locality = PMIX_LOCALITY_SHARE_NODE;
+
+    set1 = pmix_argv_split(loc1, ':');
+    set2 = pmix_argv_split(loc2, ':');
+    bit1 = hwloc_bitmap_alloc();
+    bit2 = hwloc_bitmap_alloc();
+
+    /* check each matching type */
+    for (n1=0; NULL != set1[n1]; n1++) {
+        /* convert the location into bitmap */
+        hwloc_bitmap_list_sscanf(bit1, &set1[n1][2]);
+        /* find the matching type in set2 */
+        for (n2=0; NULL != set2[n2]; n2++) {
+            if (0 == strncmp(set1[n1], set2[n2], 2)) {
+                /* convert the location into bitmap */
+                hwloc_bitmap_list_sscanf(bit2, &set2[n2][2]);
+                /* see if they intersect */
+                if (hwloc_bitmap_intersects(bit1, bit2)) {
+                    /* set the corresponding locality bit */
+                    if (0 == strncmp(set1[n1], "NM", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_NUMA;
+                    } else if (0 == strncmp(set1[n1], "SK", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_PACKAGE;
+                    } else if (0 == strncmp(set1[n1], "L3", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_L3CACHE;
+                    } else if (0 == strncmp(set1[n1], "L2", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_L2CACHE;
+                    } else if (0 == strncmp(set1[n1], "L1", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_L1CACHE;
+                    } else if (0 == strncmp(set1[n1], "CR", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_CORE;
+                    } else if (0 == strncmp(set1[n1], "HT", 2)) {
+                        locality |= PMIX_LOCALITY_SHARE_HWTHREAD;
+                    } else {
+                        /* should never happen */
+                        pmix_output(0, "UNRECOGNIZED LOCALITY %s", set1[n1]);
+                        rc = PMIX_ERROR;
+                    }
+                }
+                break;
+            }
+        }
+    }
+    pmix_argv_free(set1);
+    pmix_argv_free(set2);
+    hwloc_bitmap_free(bit1);
+    hwloc_bitmap_free(bit2);
+    *loc = locality;
+    return rc;
+}
+
+static pmix_status_t get_cpuset(pmix_cpuset_t *cpuset,
+                                pmix_bind_envelope_t ref)
+{
+    int rc, flag;
+
+    if (NULL != cpuset->source && 0 != strcasecmp(cpuset->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    if (PMIX_CPUBIND_PROCESS == ref) {
+        flag = HWLOC_CPUBIND_PROCESS;
+    } else if (PMIX_CPUBIND_THREAD == ref) {
+        flag = HWLOC_CPUBIND_THREAD;
+    } else {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    cpuset->bitmap = hwloc_bitmap_alloc();
+    if (NULL != mca_ploc_hwloc_component.testcpuset) {
+        rc = hwloc_bitmap_sscanf(cpuset->bitmap, mca_ploc_hwloc_component.testcpuset);
+    } else {
+        rc = hwloc_get_cpubind(pmix_globals.topology.topology, cpuset->bitmap, flag);
+    }
+    if (0 != rc) {
+        hwloc_bitmap_free(cpuset->bitmap);
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+    if (NULL == cpuset->source) {
+        cpuset->source = strdup("hwloc");
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static hwloc_obj_t dsearch(hwloc_topology_t t, int depth,
+                           hwloc_cpuset_t cpuset)
+{
+    hwloc_obj_t obj;
+    unsigned width, w;
+
+    /* get the width of the topology at this depth */
+    width = hwloc_get_nbobjs_by_depth(t, depth);
+    if (0 == width) {
+        return NULL;
+    }
+    /* scan all objects at this depth to see if
+     * the location is under one of them
+     */
+    for (w=0; w < width; w++) {
+        /* get the object at this depth/index */
+        obj = hwloc_get_obj_by_depth(t, depth, w);
+        /* if this object doesn't have a cpuset, then ignore it */
+        if (NULL == obj->cpuset) {
+            continue;
+        }
+        /* see if the provided cpuset is completely included in this object */
+        if (hwloc_bitmap_isincluded(cpuset, obj->cpuset)) {
+            return obj;
+        }
+    }
+
+    return NULL;
+}
+
+typedef struct {
+    pmix_list_item_t super;
+    pmix_device_distance_t dist;
+} pmix_devdist_item_t;
+static void dvcon(pmix_devdist_item_t *p)
+{
+    PMIX_DEVICE_DIST_CONSTRUCT(&p->dist);
+}
+static void dvdes(pmix_devdist_item_t *p)
+{
+    PMIX_DEVICE_DIST_DESTRUCT(&p->dist);
+}
+static PMIX_CLASS_INSTANCE(pmix_devdist_item_t,
+                           pmix_list_item_t,
+                           dvcon, dvdes);
+
+typedef struct {
+    hwloc_obj_osdev_type_t hwtype;
+    pmix_device_type_t pxtype;
+    char *name;
+} pmix_type_conversion_t;
+
+static pmix_type_conversion_t table[] =
+{
+    {.hwtype = HWLOC_OBJ_OSDEV_BLOCK, .pxtype = PMIX_DEVTYPE_BLOCK, .name = "BLOCK"},
+    {.hwtype = HWLOC_OBJ_OSDEV_GPU, .pxtype = PMIX_DEVTYPE_GPU, .name = "GPU"},
+    {.hwtype = HWLOC_OBJ_OSDEV_NETWORK, .pxtype = PMIX_DEVTYPE_NETWORK, .name = "NETWORK"},
+    {.hwtype = HWLOC_OBJ_OSDEV_OPENFABRICS, .pxtype = PMIX_DEVTYPE_OPENFABRICS, .name = "OPENFABRICS"},
+    {.hwtype = HWLOC_OBJ_OSDEV_DMA, .pxtype = PMIX_DEVTYPE_DMA, .name = "DMA"},
+#if HWLOC_API_VERSION >= 0x00010800
+    {.hwtype = HWLOC_OBJ_OSDEV_COPROC, .pxtype = PMIX_DEVTYPE_COPROC, .name = "COPROCESSOR"},
+#endif
+};
+
+static int countcolons(char *str)
+{
+    int cnt=0;
+    char *p;
+
+    p = strchr(str, ':');
+    while (NULL != p) {
+        ++cnt;
+        ++p;
+        p = strchr(p, ':');
+    }
+
+    return cnt;
+}
+
+static pmix_status_t compute_distances(pmix_topology_t *topo,
+                                       pmix_cpuset_t *cpuset,
+                                       pmix_info_t info[], size_t ninfo,
+                                       pmix_device_distance_t **dist,
+                                       size_t *ndist)
+{
+    hwloc_obj_t obj = NULL;
+    hwloc_obj_t tgt;
+    hwloc_obj_t device;
+    hwloc_obj_t ancestor;
+    hwloc_obj_t pu;
+    unsigned dp, depth;
+    unsigned maxdist = 0;
+    unsigned mindist = UINT_MAX;
+    unsigned i;
+    pmix_list_t dists;
+    pmix_devdist_item_t *d;
+    pmix_device_distance_t *array;
+    size_t n, ntypes, dn;
+    int cnt;
+    unsigned w, width, pudepth;
+    pmix_device_type_t type=0;
+    char **devids = NULL;
+    bool found;
+
+    if (NULL == topo->source || NULL == cpuset->source) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    if (0 != strcasecmp(topo->source, "hwloc") || 0 != strcasecmp(cpuset->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* set default returns */
+    *dist = NULL;
+    *ndist = 0;
+
+    /* determine number of types we support */
+    ntypes = sizeof(table) / sizeof(pmix_type_conversion_t);
+
+    /* determine what they want us to look at */
+    if (NULL == info) {
+        /* find everything */
+        for (n=0; n < ntypes; n++) {
+            type |= table[n].pxtype;
+        }
+    } else {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_DEVICE_TYPE)) {
+                type |= info[n].value.data.devtype;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_DEVICE_ID)) {
+                pmix_argv_append_nosize(&devids, info[n].value.data.string);
+            }
+        }
+    }
+
+    /* find the max depth of this topology */
+    depth = hwloc_topology_get_depth(topo->topology);
+    /* get the lowest object that completely covers the cpuset */
+    for (dp=1; dp < depth; dp++) {
+        tgt = dsearch(topo->topology, dp, cpuset->bitmap);
+        if (NULL == tgt) {
+            /* nothing found at that depth, so we are done */
+            break;
+        }
+        obj = tgt;
+    }
+    if (NULL == obj) {
+        /* only the entire machine covers this cpuset - typically,
+         * this means we are in some odd container where every
+         * PU is in its own package. There is nothing useful
+         * that can be done here */
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+    /* get the PU depth */
+    pudepth = (unsigned)hwloc_get_type_depth(topo->topology, HWLOC_OBJ_PU);
+    width = hwloc_get_nbobjs_by_depth(topo->topology, pudepth);
+
+    PMIX_CONSTRUCT(&dists, pmix_list_t);
+
+    /* loop over the specified devices in the topology */
+    for (n=0; n < ntypes; n++) {
+        if (!(type & table[n].pxtype)) {
+            continue;
+        }
+        if (HWLOC_OBJ_OSDEV_BLOCK == table[n].hwtype ||
+            HWLOC_OBJ_OSDEV_DMA == table[n].hwtype
+#if HWLOC_API_VERSION >= 0x00010800
+             ||
+            HWLOC_OBJ_OSDEV_COPROC == table[n].hwtype
+#endif
+                                                     ) {
+            continue;
+        }
+        device = hwloc_get_obj_by_type(topo->topology, HWLOC_OBJ_OS_DEVICE, 0);
+        while (NULL != device) {
+            if (device->attr->osdev.type == table[n].hwtype) {
+
+                d = PMIX_NEW(pmix_devdist_item_t);
+                pmix_list_append(&dists, &d->super);
+
+                d->dist.type = table[n].pxtype;
+
+                /* Construct a UUID for this device */
+                if (HWLOC_OBJ_OSDEV_NETWORK == table[n].hwtype) {
+                    char *addr = NULL;
+                    /* find the address */
+                    for (i=0; i < device->infos_count; i++) {
+                        if (0 == strcasecmp(device->infos[i].name, "Address")) {
+                            addr = device->infos[i].value;
+                            break;
+                        }
+                    }
+                    if (NULL == addr) {
+                        /* couldn't find an address - report it as an error */
+                        PMIX_LIST_DESTRUCT(&dists);
+                        return PMIX_ERROR;
+                    }
+                    /* could be IPv4 or IPv6 */
+                    cnt = countcolons(addr);
+                    if (5 == cnt) {
+                        pmix_asprintf(&d->dist.uuid, "ipv4://%s", addr);
+                    } else if (19 == cnt) {
+                        pmix_asprintf(&d->dist.uuid, "ipv6://%s", addr);
+                    } else {
+                        /* unknown address type */
+                        PMIX_LIST_DESTRUCT(&dists);
+                        return PMIX_ERROR;
+                    }
+                } else if (HWLOC_OBJ_OSDEV_OPENFABRICS == table[n].hwtype) {
+                    char *ngid = NULL;
+                    char *sgid = NULL;
+                    /* find the UIDs */
+                    for (i=0; i < device->infos_count; i++) {
+                        if (0 == strcasecmp(device->infos[i].name, "NodeGUID")) {
+                            ngid = device->infos[i].value;
+                        } else if (0 == strcasecmp(device->infos[i].name, "SysImageGUID")) {
+                            sgid = device->infos[i].value;
+                        }
+                    }
+                    if (NULL == ngid || NULL == sgid) {
+                        PMIX_LIST_DESTRUCT(&dists);
+                        return PMIX_ERROR;
+                    }
+                    pmix_asprintf(&d->dist.uuid, "fab://%s::%s", ngid, sgid);
+                } else if (HWLOC_OBJ_OSDEV_GPU == table[n].hwtype) {
+                    /* if the name starts with "card", then this is just the aux card of the GPU */
+                    if (0 == strncasecmp(device->name, "card", 4)) {
+                        pmix_list_remove_item(&dists, &d->super);
+                        PMIX_RELEASE(d);
+                        device = hwloc_get_next_osdev(topo->topology, device);
+                        continue;
+                    }
+                    pmix_asprintf(&d->dist.uuid, "gpu://%s::%s", pmix_globals.hostname, device->name);
+                } else {
+                    /* unknown type */
+                    pmix_list_remove_item(&dists, &d->super);
+                    PMIX_RELEASE(d);
+                    device = hwloc_get_next_osdev(topo->topology, device);
+                    continue;
+                }
+
+                /* if device id was given, then check if this one matches either
+                 * the UUID or osname */
+                if (NULL != devids) {
+                    found = false;
+                    for (dn=0; NULL != devids[dn]; dn++) {
+                        if (0 == strcasecmp(devids[dn], device->name) ||
+                            0 == strcasecmp(devids[dn], d->dist.uuid)) {
+                            found = true;
+                        }
+                    }
+                    if (!found) {
+                        /* skip this one */
+                        pmix_list_remove_item(&dists, &d->super);
+                        PMIX_RELEASE(d);
+                        device = hwloc_get_next_osdev(topo->topology, device);
+                        continue;
+                    }
+                }
+                /* save the osname */
+                d->dist.osname = strdup(device->name);
+                if (NULL == device->cpuset) {
+                    /* climb the topology until we find a non-NULL cpuset */
+                    tgt = device->parent;
+                    while (NULL != tgt && NULL == tgt->cpuset) {
+                        tgt = tgt->parent;
+                    }
+                    if (NULL == tgt) {
+                        PMIX_LIST_DESTRUCT(&dists);
+                        return PMIX_ERR_NOT_FOUND;
+                    }
+                } else {
+                    tgt = device;
+                }
+                /* loop over the PUs on this node */
+                maxdist = 0;
+                mindist = UINT_MAX;
+                for (w=0; w < width; w++) {
+                    /* get the pu at this index */
+                    pu = hwloc_get_obj_by_depth(topo->topology, pudepth, w);
+                    /* is this PU in our cpuset? */
+                    if (!hwloc_bitmap_intersects(pu->cpuset, cpuset->bitmap)) {
+                        continue;
+                    }
+                    /* find the common ancestor between the cpuset and NIC objects */
+                    ancestor = hwloc_get_common_ancestor_obj(topo->topology, obj, tgt);
+                    if (NULL != ancestor) {
+                        if (0 == ancestor->depth) {
+                            /* we only share the machine - need to do something more
+                             * to compute the distance. This can, however, get a little
+                             * hairy as there is no good measure of package-to-package
+                             * distance - it is all typically given in terms of NUMA
+                             * domains, which is no longer a valid way of looking at
+                             * locations due to overlapping domains. For now, we will
+                             * just take the depth of the device in its package and
+                             * add that to the depth of the object in its package
+                             * plus the depth of a package to ensure it is further away */
+                            dp = obj->depth + depth;
+                        } else {
+                            /* the depth value can be used as an indicator of relative
+                             * locality - the higher the value, the closer the device.
+                             * We invert the pyramid to set the dist to be closer for
+                             * smaller values */
+                            dp = depth - ancestor->depth;
+                        }
+                    } else {
+                        /* shouldn't happen - consider this an error condition */
+                        PMIX_LIST_DESTRUCT(&dists);
+                        return PMIX_ERROR;
+                    }
+                    if (mindist > dp) {
+                        mindist = dp;
+                    }
+                    if (maxdist < dp) {
+                        maxdist = dp;
+                    }
+                }
+                d->dist.mindist = mindist;
+                d->dist.maxdist = maxdist;
+            }
+            device = hwloc_get_next_osdev(topo->topology, device);
+        }
+    }
+
+    /* create the return array */
+    n = pmix_list_get_size(&dists);
+    if (0 == n) {
+        /* no devices found */
+        return PMIX_ERR_NOT_FOUND;
+    }
+    PMIX_DEVICE_DIST_CREATE(array, n);
+    *ndist = n;
+    n = 0;
+    PMIX_LIST_FOREACH(d, &dists, pmix_devdist_item_t) {
+        array[n].uuid = strdup(d->dist.uuid);
+        array[n].osname = strdup(d->dist.osname);
+        array[n].type = d->dist.type;
+        array[n].mindist = d->dist.mindist;
+        array[n].maxdist = d->dist.maxdist;
+        ++n;
+    }
+    PMIX_LIST_DESTRUCT(&dists);
+    *dist = array;
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t pack_cpuset(pmix_buffer_t *buf, pmix_cpuset_t *src,
+                                 pmix_pointer_array_t *regtypes)
+{
+    char *tmp;
+    pmix_status_t rc;
+
+    if (NULL == src->source || 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* express the cpuset as a string */
+    if (0 != hwloc_bitmap_list_asprintf(&tmp, src->bitmap)) {
+        return PMIX_ERROR;
+    }
+
+    /* pack the string */
+    PMIX_BFROPS_PACK_TYPE(rc, buf, &tmp, 1, PMIX_STRING, regtypes);
+    free(tmp);
+
+    return rc;
+}
+
+static pmix_status_t unpack_cpuset(pmix_buffer_t *buf, pmix_cpuset_t *dest,
+                                   pmix_pointer_array_t *regtypes)
+{
+    pmix_status_t rc;
+    int cnt;
+    char *tmp;
+
+    /* unpack the cpustring */
+    cnt=1;
+    PMIX_BFROPS_UNPACK_TYPE(rc, buf, &tmp, &cnt, PMIX_STRING, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* convert to a bitmap */
+    dest->source = strdup("hwloc");
+    dest->bitmap = hwloc_bitmap_alloc();
+    hwloc_bitmap_list_sscanf(dest->bitmap, tmp);
+    free(tmp);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t copy_cpuset(pmix_cpuset_t *dest, pmix_cpuset_t *src)
+{
+    if (NULL == src->source || 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+    if (NULL == src->bitmap) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    /* copy the src bitmap */
+    dest->bitmap = hwloc_bitmap_dup(src->bitmap);
+    dest->source = strdup("hwloc");
+
+    return PMIX_SUCCESS;
+}
+
+static char* print_cpuset(pmix_cpuset_t *src)
+{
+    char *tmp;
+
+    if (NULL == src->source || 0 != strcasecmp(src->source, "hwloc")) {
+        return NULL;
+    }
+    if (NULL == src->bitmap) {
+        return NULL;
+    }
+
+    /* express the cpuset as a string */
+    if (0 != hwloc_bitmap_list_asprintf(&tmp, src->bitmap)) {
+        return NULL;
+    }
+
+    return tmp;
+}
+
+static pmix_status_t destruct_cpuset(pmix_cpuset_t *src)
+{
+    if (NULL == src->source || 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+    if (NULL == src->bitmap) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    hwloc_bitmap_free(src->bitmap);
+    src->bitmap = NULL;
+    free(src->source);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t release_cpuset(pmix_cpuset_t *src, size_t ncpu)
+{
+    size_t n;
+
+    if (NULL == src->source || 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    for (n=0; n < ncpu; n++) {
+        destruct_cpuset(&src[n]);
+    }
+    free(src);
+
+    return PMIX_SUCCESS;
+}
+
+
+static pmix_status_t pack_topology(pmix_buffer_t *buf, pmix_topology_t *src,
+                                   pmix_pointer_array_t *regtypes)
+{
+    /* NOTE: hwloc defines topology_t as a pointer to a struct! */
+    pmix_status_t rc;
+    char *xmlbuffer=NULL;
+    int len;
+    struct hwloc_topology_support *support;
+
+    if (NULL == src->source || 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* extract an xml-buffer representation of the tree */
+#if HWLOC_API_VERSION < 0x20000
+    if (0 != hwloc_topology_export_xmlbuffer(src->topology, &xmlbuffer, &len)) {
+        return PMIX_ERROR;
+    }
+#else
+    if (0 != hwloc_topology_export_xmlbuffer(src->topology, &xmlbuffer, &len, 0)) {
+        return PMIX_ERROR;
+    }
+#endif
+
+    /* add to buffer */
+    PMIX_BFROPS_PACK_TYPE(rc, buf, &xmlbuffer, 1, PMIX_STRING, regtypes);
+    free(xmlbuffer);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* get the available support - hwloc unfortunately does
+     * not include this info in its xml export!
+     */
+    support = (struct hwloc_topology_support*)hwloc_topology_get_support(src->topology);
+    /* pack the discovery support */
+    PMIX_BFROPS_PACK_TYPE(rc, buf, support->discovery,
+                          sizeof(struct hwloc_topology_discovery_support),
+                          PMIX_BYTE, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+    /* pack the cpubind support */
+    PMIX_BFROPS_PACK_TYPE(rc, buf, support->cpubind,
+                          sizeof(struct hwloc_topology_cpubind_support),
+                          PMIX_BYTE, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* pack the membind support */
+    PMIX_BFROPS_PACK_TYPE(rc, buf, support->membind,
+                          sizeof(struct hwloc_topology_membind_support),
+                          PMIX_BYTE, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t unpack_topology(pmix_buffer_t *buf, pmix_topology_t *dest,
+                                     pmix_pointer_array_t *regtypes)
+{
+    /* NOTE: hwloc defines topology_t as a pointer to a struct! */
+    pmix_status_t rc;
+    char *xmlbuffer=NULL;
+    int cnt;
+    struct hwloc_topology_support *support;
+    hwloc_topology_t t;
+    unsigned long flags;
+
+    /* unpack the xml string */
+    cnt=1;
+    PMIX_BFROPS_UNPACK_TYPE(rc, buf, &xmlbuffer, &cnt, PMIX_STRING, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* convert the xml */
+    if (0 != hwloc_topology_init(&t)) {
+        rc = PMIX_ERROR;
+        free(xmlbuffer);
+        return rc;
+    }
+    if (0 != hwloc_topology_set_xmlbuffer(t, xmlbuffer, strlen(xmlbuffer))) {
+        rc = PMIX_ERROR;
+        free(xmlbuffer);
+        hwloc_topology_destroy(t);
+        return rc;
+    }
+    free(xmlbuffer);
+
+    /* since we are loading this from an external source, we have to
+     * explicitly set a flag so hwloc sets things up correctly
+     */
+    flags = HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM;
+#if HWLOC_API_VERSION < 0x00020000
+        flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
+        flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES;
+#else
+        if (0 != hwloc_topology_set_io_types_filter(t, HWLOC_TYPE_FILTER_KEEP_IMPORTANT)) {
+            hwloc_topology_destroy(t);
+            return PMIX_ERROR;
+        }
+#if HWLOC_API_VERSION < 0x00020100
+        flags |= HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM;
+#else
+        flags |= HWLOC_TOPOLOGY_FLAG_INCLUDE_DISALLOWED;
+#endif
+#endif
+    if (0 != hwloc_topology_set_flags(t, flags)) {
+        hwloc_topology_destroy(t);
+        return PMIX_ERROR;
+    }
+    /* now load the topology */
+    if (0 != hwloc_topology_load(t)) {
+        hwloc_topology_destroy(t);
+        return PMIX_ERROR;
+    }
+
+    /* get the available support - hwloc unfortunately does
+     * not include this info in its xml import!
+     */
+    support = (struct hwloc_topology_support*)hwloc_topology_get_support(t);
+    cnt = sizeof(struct hwloc_topology_discovery_support);
+    PMIX_BFROPS_UNPACK_TYPE(rc, buf, support->discovery, &cnt, PMIX_BYTE, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        hwloc_topology_destroy(t);
+        return PMIX_ERROR;
+    }
+    cnt = sizeof(struct hwloc_topology_cpubind_support);
+    PMIX_BFROPS_UNPACK_TYPE(rc, buf, support->cpubind, &cnt, PMIX_BYTE, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        hwloc_topology_destroy(t);
+        return PMIX_ERROR;
+    }
+    cnt = sizeof(struct hwloc_topology_membind_support);
+    PMIX_BFROPS_UNPACK_TYPE(rc, buf, support->membind, &cnt, PMIX_BYTE, regtypes);
+    if (PMIX_SUCCESS != rc) {
+        hwloc_topology_destroy(t);
+        return PMIX_ERROR;
+    }
+
+    dest->source = strdup("hwloc");
+    dest->topology = (void*)t;
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t copy_topology(pmix_topology_t *dest, pmix_topology_t *src)
+{
+    if (NULL == src->source|| 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+#if PMIX_HAVE_HWLOC_TOPOLOGY_DUP
+    /* use the hwloc dup function */
+    if (0 != hwloc_topology_dup((hwloc_topology_t*)&dest->topology, src->topology)) {
+        return PMIX_ERROR;
+    }
+    dest->source = strdup("hwloc");
+
+    return PMIX_SUCCESS;
+#else
+    /* we have to do this in a convoluted manner */
+    char *xmlbuffer=NULL;
+    int len;
+    struct hwloc_topology_support *srcsup, *destsup;
+    pmix_status_t rc;
+
+    /* extract an xml-buffer representation of the tree */
+#if HWLOC_API_VERSION < 0x20000
+    if (0 != hwloc_topology_export_xmlbuffer(src->topology, &xmlbuffer, &len)) {
+        return PMIX_ERROR;
+    }
+#else
+    if (0 != hwloc_topology_export_xmlbuffer(src->topology, &xmlbuffer, &len, 0)) {
+        return PMIX_ERROR;
+    }
+#endif
+
+    /* convert the xml back */
+    if (0 != hwloc_topology_init((hwloc_topology_t*)&dest->topology)) {
+        rc = PMIX_ERROR;
+        free(xmlbuffer);
+        return rc;
+    }
+    if (0 != hwloc_topology_set_xmlbuffer(dest->topology, xmlbuffer, strlen(xmlbuffer))) {
+        rc = PMIX_ERROR;
+        free(xmlbuffer);
+        hwloc_topology_destroy(dest->topology);
+        return rc;
+    }
+    free(xmlbuffer);
+
+    /* transfer the support struct */
+    srcsup = (struct hwloc_topology_support*)hwloc_topology_get_support((hwloc_topology_t)src->topology);
+    destsup = (struct hwloc_topology_support*)hwloc_topology_get_support(dest->topology);
+    memcpy(destsup, srcsup, sizeof(struct hwloc_topology_support));
+
+    return PMIX_SUCCESS;
+#endif
+}
+
+#define PMIX_HWLOC_MAX_STRING   2048
+
+static void print_hwloc_obj(char **output, char *prefix,
+                            hwloc_topology_t topo, hwloc_obj_t obj)
+{
+    hwloc_obj_t obj2;
+    char string[1024], *tmp, *tmp2, *pfx;
+    unsigned i;
+    struct hwloc_topology_support *support;
+
+    /* print the object type */
+    hwloc_obj_type_snprintf(string, 1024, obj, 1);
+    pmix_asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix);
+    pmix_asprintf(&tmp, "%sType: %s Number of child objects: %u%sName=%s",
+             (NULL == prefix) ? "" : prefix, string, obj->arity,
+             pfx, (NULL == obj->name) ? "NULL" : obj->name);
+    if (0 < hwloc_obj_attr_snprintf(string, 1024, obj, pfx, 1)) {
+        /* print the attributes */
+        pmix_asprintf(&tmp2, "%s%s%s", tmp, pfx, string);
+        free(tmp);
+        tmp = tmp2;
+    }
+    /* print the cpusets - apparently, some new HWLOC types don't
+     * have cpusets, so protect ourselves here
+     */
+    if (NULL != obj->cpuset) {
+        hwloc_bitmap_snprintf(string, PMIX_HWLOC_MAX_STRING, obj->cpuset);
+        pmix_asprintf(&tmp2, "%s%sCpuset:  %s", tmp, pfx, string);
+        free(tmp);
+        tmp = tmp2;
+    }
+    if (HWLOC_OBJ_MACHINE == obj->type) {
+        /* root level object - add support values */
+        support = (struct hwloc_topology_support*)hwloc_topology_get_support(topo);
+        pmix_asprintf(&tmp2, "%s%sBind CPU proc:   %s%sBind CPU thread: %s", tmp, pfx,
+                 (support->cpubind->set_thisproc_cpubind) ? "TRUE" : "FALSE", pfx,
+                 (support->cpubind->set_thisthread_cpubind) ? "TRUE" : "FALSE");
+        free(tmp);
+        tmp = tmp2;
+        pmix_asprintf(&tmp2, "%s%sBind MEM proc:   %s%sBind MEM thread: %s", tmp, pfx,
+                 (support->membind->set_thisproc_membind) ? "TRUE" : "FALSE", pfx,
+                 (support->membind->set_thisthread_membind) ? "TRUE" : "FALSE");
+        free(tmp);
+        tmp = tmp2;
+    }
+    pmix_asprintf(&tmp2, "%s%s\n", (NULL == *output) ? "" : *output, tmp);
+    free(tmp);
+    free(pfx);
+    pmix_asprintf(&pfx, "%s\t", (NULL == prefix) ? "" : prefix);
+    for (i=0; i < obj->arity; i++) {
+        obj2 = obj->children[i];
+        /* print the object */
+        print_hwloc_obj(&tmp2, pfx, topo, obj2);
+    }
+    free(pfx);
+    if (NULL != *output) {
+        free(*output);
+    }
+    *output = tmp2;
+}
+
+static char* print_topology(pmix_topology_t *src)
+{
+    hwloc_obj_t obj;
+    char *tmp=NULL;
+
+    if (NULL == src->source|| 0 != strcasecmp(src->source, "hwloc")) {
+        return NULL;
+    }
+
+    /* get root object */
+    obj = hwloc_get_root_obj(src->topology);
+    /* print it */
+    print_hwloc_obj(&tmp, NULL, src->topology, obj);
+    return tmp;
+}
+
+static pmix_status_t destruct_topology(pmix_topology_t *src)
+{
+    if (NULL == src->source|| 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+    if (NULL == src->topology) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    hwloc_topology_destroy(src->topology);
+    free(src->source);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t release_topology(pmix_topology_t *src, size_t ncpu)
+{
+    size_t n;
+
+    if (NULL == src->source|| 0 != strcasecmp(src->source, "hwloc")) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    for (n=0; n < ncpu; n++) {
+        destruct_topology(&src[n]);
+    }
+    free(src);
+
+    return PMIX_SUCCESS;
+}
+
+#if HWLOC_API_VERSION >= 0x20000
+
+static int parse_map_line(const char *line,
+                          unsigned long *beginp,
+                          unsigned long *endp,
+                          pmix_hwloc_vm_map_kind_t *kindp)
+{
+    const char *tmp = line, *next;
+    unsigned long value;
+
+    /* "beginaddr-endaddr " */
+    value = strtoull(tmp, (char **) &next, 16);
+    if (next == tmp) {
+        return PMIX_ERROR;
+    }
+
+    *beginp = (unsigned long) value;
+
+    if (*next != '-') {
+        return PMIX_ERROR;
+    }
+
+     tmp = next + 1;
+
+    value = strtoull(tmp, (char **) &next, 16);
+    if (next == tmp) {
+        return PMIX_ERROR;
+    }
+    *endp = (unsigned long) value;
+    tmp = next;
+
+    if (*next != ' ') {
+        return PMIX_ERROR;
+    }
+    tmp = next + 1;
+
+    /* look for ending absolute path */
+    next = strchr(tmp, '/');
+    if (next) {
+        *kindp = VM_MAP_FILE;
+    } else {
+        /* look for ending special tag [foo] */
+        next = strchr(tmp, '[');
+        if (next) {
+            if (!strncmp(next, "[heap]", 6)) {
+                *kindp = VM_MAP_HEAP;
+            } else if (!strncmp(next, "[stack]", 7)) {
+                *kindp = VM_MAP_STACK;
+            } else {
+                char *end;
+                if ((end = strchr(next, '\n')) != NULL) {
+                    *end = '\0';
+                }
+                *kindp = VM_MAP_OTHER;
+            }
+        } else {
+            *kindp = VM_MAP_ANONYMOUS;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+#define ALIGN2MB (2*1024*1024UL)
+
+static int use_hole(unsigned long holebegin,
+                    unsigned long holesize,
+                    unsigned long *addrp,
+                    unsigned long size)
+{
+    unsigned long aligned;
+    unsigned long middle = holebegin+holesize/2;
+
+    if (holesize < size) {
+        return PMIX_ERROR;
+    }
+
+    /* try to align the middle of the hole on 64MB for POWER's 64k-page PMD */
+    #define ALIGN64MB (64*1024*1024UL)
+    aligned = (middle + ALIGN64MB) & ~(ALIGN64MB-1);
+    if (aligned + size <= holebegin + holesize) {
+        *addrp = aligned;
+        return PMIX_SUCCESS;
+    }
+
+    /* try to align the middle of the hole on 2MB for x86 PMD */
+    aligned = (middle + ALIGN2MB) & ~(ALIGN2MB-1);
+    if (aligned + size <= holebegin + holesize) {
+        *addrp = aligned;
+        return PMIX_SUCCESS;
+    }
+
+    /* just use the end of the hole */
+    *addrp = holebegin + holesize - size;
+    return PMIX_SUCCESS;
+}
+
+static int find_hole(pmix_hwloc_vm_hole_kind_t hkind,
+                     size_t *addrp, size_t size)
+{
+    unsigned long biggestbegin = 0;
+    unsigned long biggestsize = 0;
+    unsigned long prevend = 0;
+    pmix_hwloc_vm_map_kind_t prevmkind = VM_MAP_OTHER;
+    int in_libs = 0;
+    FILE *file;
+    char line[96];
+
+    file = fopen("/proc/self/maps", "r");
+    if (!file) {
+        return PMIX_ERROR;
+    }
+
+    while (fgets(line, sizeof(line), file) != NULL) {
+        unsigned long begin=0, end=0;
+        pmix_hwloc_vm_map_kind_t mkind=VM_MAP_OTHER;
+
+        if (!parse_map_line(line, &begin, &end, &mkind)) {
+            switch (hkind) {
+                case VM_HOLE_BEGIN:
+                    fclose(file);
+                    return use_hole(0, begin, addrp, size);
+
+                case VM_HOLE_AFTER_HEAP:
+                    if (prevmkind == VM_MAP_HEAP && mkind != VM_MAP_HEAP) {
+                        /* only use HEAP when there's no other HEAP after it
+                         * (there can be several of them consecutively).
+                         */
+                        fclose(file);
+                        return use_hole(prevend, begin-prevend, addrp, size);
+                    }
+                    break;
+
+                case VM_HOLE_BEFORE_STACK:
+                    if (mkind == VM_MAP_STACK) {
+                        fclose(file);
+                        return use_hole(prevend, begin-prevend, addrp, size);
+                    }
+                    break;
+
+                case VM_HOLE_IN_LIBS:
+                    /* see if we are between heap and stack */
+                    if (prevmkind == VM_MAP_HEAP) {
+                        in_libs = 1;
+                    }
+                    if (mkind == VM_MAP_STACK) {
+                        in_libs = 0;
+                    }
+                    if (!in_libs) {
+                        /* we're not in libs, ignore this entry */
+                        break;
+                    }
+                    /* we're in libs, consider this entry for searching the biggest hole below */
+                    /* fallthrough */
+
+                case VM_HOLE_BIGGEST:
+                    if (begin-prevend > biggestsize) {
+                        biggestbegin = prevend;
+                        biggestsize = begin-prevend;
+                    }
+                    break;
+
+                    default:
+                        assert(0);
+            }
+        }
+
+        while (!strchr(line, '\n')) {
+            if (!fgets(line, sizeof(line), file)) {
+                goto done;
+            }
+        }
+
+        if (mkind == VM_MAP_STACK) {
+          /* Don't go beyond the stack. Other VMAs are special (vsyscall, vvar, vdso, etc),
+           * There's no spare room there. And vsyscall is even above the userspace limit.
+           */
+          break;
+        }
+
+        prevend = end;
+        prevmkind = mkind;
+
+    }
+
+  done:
+    fclose(file);
+    if (hkind == VM_HOLE_IN_LIBS || hkind == VM_HOLE_BIGGEST) {
+        return use_hole(biggestbegin, biggestsize, addrp, size);
+    }
+
+    return PMIX_ERROR;
+}
+
+static int enough_space(const char *filename,
+                        size_t space_req,
+                        uint64_t *space_avail,
+                        bool *result)
+{
+    uint64_t avail = 0;
+    size_t fluff = (size_t)(.05 * space_req);
+    bool enough = false;
+    char *last_sep = NULL;
+    /* the target file name is passed here, but we need to check the parent
+     * directory. store it so we can extract that info later. */
+    char *target_dir = strdup(filename);
+    int rc;
+
+    if (NULL == target_dir) {
+        rc = PMIX_ERR_OUT_OF_RESOURCE;
+        goto out;
+    }
+    /* get the parent directory */
+    last_sep = strrchr(target_dir, PMIX_PATH_SEP[0]);
+    *last_sep = '\0';
+    /* now check space availability */
+    if (PMIX_SUCCESS != (rc = pmix_path_df(target_dir, &avail))) {
+        goto out;
+    }
+    /* do we have enough space? */
+    if (avail >= space_req + fluff) {
+        enough = true;
+    }
+
+out:
+    if (NULL != target_dir) {
+        free(target_dir);
+    }
+    *result = enough;
+    *space_avail = avail;
+    return rc;
+}
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/hwloc/ploc_hwloc_component.c pmix-4.0.0/src/mca/ploc/hwloc/ploc_hwloc_component.c
--- pmix-3.2.2~rc1/src/mca/ploc/hwloc/ploc_hwloc_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/hwloc/ploc_hwloc_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,114 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/mca/ploc/ploc.h"
+#include "ploc_hwloc.h"
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+static int component_register(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_ploc_hwloc_component_t mca_ploc_hwloc_component = {
+    .super = {
+        .base = {
+            PMIX_PLOC_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "hwloc",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .pmix_mca_query_component = component_query,
+            .pmix_mca_register_component_params = component_register,
+        },
+        .data = {
+            /* The component is checkpoint ready */
+            PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+        }
+    },
+    .hole_kind = VM_HOLE_BIGGEST,
+    .topo_file = NULL,
+    .testcpuset = NULL
+};
+
+static char *vmhole = "biggest";
+
+static int component_register(void)
+{
+    pmix_mca_base_component_t *component = &mca_ploc_hwloc_component.super.base;
+
+    (void)pmix_mca_base_component_var_register(component, "hole_kind",
+                                           "Kind of VM hole to identify - none, begin, biggest, libs, heap, stack (default=biggest)",
+                                           PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                           PMIX_INFO_LVL_9,
+                                           PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                           &vmhole);
+    if (0 == strcasecmp(vmhole, "none")) {
+        mca_ploc_hwloc_component.hole_kind = VM_HOLE_NONE;
+    } else if (0 == strcasecmp(vmhole, "begin")) {
+        mca_ploc_hwloc_component.hole_kind = VM_HOLE_BEGIN;
+    } else if (0 == strcasecmp(vmhole, "biggest")) {
+        mca_ploc_hwloc_component.hole_kind = VM_HOLE_BIGGEST;
+    } else if (0 == strcasecmp(vmhole, "libs")) {
+        mca_ploc_hwloc_component.hole_kind = VM_HOLE_IN_LIBS;
+    } else if (0 == strcasecmp(vmhole, "heap")) {
+        mca_ploc_hwloc_component.hole_kind = VM_HOLE_AFTER_HEAP;
+    } else if (0 == strcasecmp(vmhole, "stack")) {
+        mca_ploc_hwloc_component.hole_kind = VM_HOLE_BEFORE_STACK;
+    } else {
+        pmix_output(0, "INVALID VM HOLE TYPE");
+        return PMIX_ERROR;
+    }
+
+    (void)pmix_mca_base_component_var_register(component, "topo_file",
+                                           "Topology file to use instead of discovering it (mostly for testing purposes)",
+                                           PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                           PMIX_INFO_LVL_9,
+                                           PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_ploc_hwloc_component.topo_file);
+
+    (void)pmix_mca_base_component_var_register(component, "test_cpuset",
+                                           "Cpuset for testing purposes",
+                                           PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                           PMIX_INFO_LVL_9,
+                                           PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_ploc_hwloc_component.testcpuset);
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 100;
+    *module = (pmix_mca_base_module_t *)&pmix_ploc_hwloc_module;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/hwloc/ploc_hwloc.h pmix-4.0.0/src/mca/ploc/hwloc/ploc_hwloc.h
--- pmix-3.2.2~rc1/src/mca/ploc/hwloc/ploc_hwloc.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/hwloc/ploc_hwloc.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PLOC_HWLOC_H
+#define PMIX_PLOC_HWLOC_H
+
+#include "src/include/pmix_config.h"
+
+#include PMIX_HWLOC_HEADER
+
+#include "src/mca/ploc/ploc.h"
+
+BEGIN_C_DECLS
+
+#if HWLOC_API_VERSION < 0x20000
+
+#ifndef HAVE_HWLOC_TOPOLOGY_DUP
+#define HAVE_HWLOC_TOPOLOGY_DUP 0
+#endif
+
+#define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
+#define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
+#define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
+
+#if HWLOC_API_VERSION < 0x10a00
+#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE
+#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET
+#endif
+
+#define HAVE_DECL_HWLOC_OBJ_OSDEV_COPROC 0
+
+#else
+
+#define HAVE_DECL_HWLOC_OBJ_OSDEV_COPROC 1
+
+#endif
+
+
+typedef enum {
+    VM_HOLE_NONE = -1,
+    VM_HOLE_BEGIN = 0,        /* use hole at the very beginning */
+    VM_HOLE_AFTER_HEAP = 1,   /* use hole right after heap */
+    VM_HOLE_BEFORE_STACK = 2, /* use hole right before stack */
+    VM_HOLE_BIGGEST = 3,      /* use biggest hole */
+    VM_HOLE_IN_LIBS = 4,      /* use biggest hole between heap and stack */
+    VM_HOLE_CUSTOM = 5,       /* use given address if available */
+} pmix_hwloc_vm_hole_kind_t;
+
+typedef enum {
+    VM_MAP_FILE = 0,
+    VM_MAP_ANONYMOUS = 1,
+    VM_MAP_HEAP = 2,
+    VM_MAP_STACK = 3,
+    VM_MAP_OTHER = 4 /* vsyscall/vdso/vvar shouldn't occur since we stop after stack */
+} pmix_hwloc_vm_map_kind_t;
+
+typedef struct {
+    pmix_ploc_base_component_t super;
+    int hole_kind;
+    char *topo_file;
+    char *testcpuset;
+} pmix_ploc_hwloc_component_t;
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_ploc_hwloc_component_t mca_ploc_hwloc_component;
+extern pmix_ploc_module_t pmix_ploc_hwloc_module;
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/Makefile.am pmix-4.0.0/src/mca/ploc/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ploc/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,44 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(LTDLINCL)
+
+# main library setup
+noinst_LTLIBRARIES = libmca_ploc.la
+libmca_ploc_la_SOURCES =
+
+# local files
+headers = ploc.h
+sources =
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.include
+
+libmca_ploc_la_SOURCES += $(headers) $(sources)
+
+distclean-local:
+	rm -f base/static-components.h
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/ploc.h pmix-4.0.0/src/mca/ploc/ploc.h
--- pmix-3.2.2~rc1/src/mca/ploc/ploc.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/ploc.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,201 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This interface is for use by PMIx servers to obtain topology-related info
+*
+ * Available plugins may be defined at runtime via the typical MCA parameter
+ * syntax.
+ */
+
+#ifndef PMIX_PLOC_H
+#define PMIX_PLOC_H
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+#include "src/include/pmix_globals.h"
+#include "src/server/pmix_server_ops.h"
+
+BEGIN_C_DECLS
+
+/******    MODULE DEFINITION    ******/
+
+/**
+ * Initialize the module. Returns an error if the module cannot
+ * run, success if it can and wants to be used.
+ */
+typedef pmix_status_t (*pmix_ploc_base_module_init_fn_t)(void);
+
+/**
+ * Finalize the module. Tear down any allocated storage, disconnect
+ * from any system support (e.g., LDAP server)
+ */
+typedef void (*pmix_ploc_base_module_fini_fn_t)(void);
+
+/* Setup the topology for delivery to clients */
+typedef pmix_status_t (*pmix_ploc_base_module_setup_topo_fn_t)(pmix_info_t *info, size_t ninfo);
+
+/* Load the topology */
+typedef pmix_status_t (*pmix_ploc_base_module_load_topo_fn_t)(pmix_topology_t *topo);
+
+/* Generate the string representation of a cpuset */
+typedef pmix_status_t (*pmix_ploc_base_module_generate_cpuset_string_fn_t)(const pmix_cpuset_t *cpuset,
+                                                                           char **cpuset_string);
+
+/* get cpuset from its string representation */
+typedef pmix_status_t (*pmix_ploc_base_module_parse_cpuset_string_fn_t)(const char *cpuset_string,
+                                                                        pmix_cpuset_t *cpuset);
+
+/* Get locality string */
+typedef pmix_status_t (*pmix_ploc_base_module_generate_loc_str_fn_t)(const pmix_cpuset_t *cpuset,
+                                                                     char **locality);
+
+
+/* Get relative locality */
+typedef pmix_status_t (*pmix_ploc_base_module_get_rel_loc_fn_t)(const char *loc1,
+                                                                const char *loc2,
+                                                                pmix_locality_t *locality);
+
+/* Get current bound location */
+typedef pmix_status_t (*pmix_ploc_base_module_get_cpuset_fn_t)(pmix_cpuset_t *cpuset,
+                                                               pmix_bind_envelope_t ref);
+
+/* Get distance array */
+typedef pmix_status_t (*pmix_ploc_base_module_compute_dist_fn_t)(pmix_topology_t *topo,
+                                                                 pmix_cpuset_t *cpuset,
+                                                                 pmix_info_t info[], size_t ninfo,
+                                                                 pmix_device_distance_t **dist,
+                                                                 size_t *ndist);
+
+/* cpuset pack/unpack/copy/print functions */
+typedef pmix_status_t (*pmix_ploc_base_module_pack_fn_t)(pmix_buffer_t *buf, pmix_cpuset_t *src,
+                                                         pmix_pointer_array_t *regtypes);
+
+typedef pmix_status_t (*pmix_ploc_base_module_unpack_fn_t)(pmix_buffer_t *buf, pmix_cpuset_t *dest,
+                                                           pmix_pointer_array_t *regtypes);
+
+typedef pmix_status_t (*pmix_ploc_base_module_copy_fn_t)(pmix_cpuset_t *dest,
+                                                         pmix_cpuset_t *src);
+
+typedef char* (*pmix_ploc_base_module_print_fn_t)(pmix_cpuset_t *src);
+
+typedef pmix_status_t (*pmix_ploc_base_module_destruct_fn_t)(pmix_cpuset_t *ptr);
+typedef void (*pmix_ploc_base_API_destruct_fn_t)(pmix_cpuset_t *ptr);
+
+typedef pmix_status_t (*pmix_ploc_base_module_release_fn_t)(pmix_cpuset_t *ptr, size_t sz);
+
+typedef void (*pmix_ploc_base_API_release_fn_t)(pmix_cpuset_t *ptr, size_t sz);
+
+/* topology pack/unpack/copy/print functions */
+typedef pmix_status_t (*pmix_ploc_base_module_pack_topo_fn_t)(pmix_buffer_t *buf, pmix_topology_t *src,
+                                                              pmix_pointer_array_t *regtypes);
+
+typedef pmix_status_t (*pmix_ploc_base_module_unpack_topo_fn_t)(pmix_buffer_t *buf, pmix_topology_t *dest,
+                                                                pmix_pointer_array_t *regtypes);
+
+typedef pmix_status_t (*pmix_ploc_base_module_copy_topo_fn_t)(pmix_topology_t *dest,
+                                                              pmix_topology_t *src);
+
+typedef char* (*pmix_ploc_base_module_print_topo_fn_t)(pmix_topology_t *src);
+
+typedef pmix_status_t (*pmix_ploc_base_module_destruct_topo_fn_t)(pmix_topology_t *ptr);
+typedef void (*pmix_ploc_base_API_destruct_topo_fn_t)(pmix_topology_t *ptr);
+
+typedef pmix_status_t (*pmix_ploc_base_module_release_topo_fn_t)(pmix_topology_t *ptr, size_t sz);
+
+typedef void (*pmix_ploc_base_API_release_topo_fn_t)(pmix_topology_t *ptr, size_t sz);
+
+/**
+ * Base structure for a PLOC module. Each component should malloc a
+ * copy of the module structure for each fabric plane they support.
+ */
+typedef struct {
+    char *name;
+    pmix_ploc_base_module_init_fn_t                     init;
+    pmix_ploc_base_module_fini_fn_t                     finalize;
+    pmix_ploc_base_module_setup_topo_fn_t               setup_topology;
+    pmix_ploc_base_module_load_topo_fn_t                load_topology;
+    pmix_ploc_base_module_generate_cpuset_string_fn_t   generate_cpuset_string;
+    pmix_ploc_base_module_parse_cpuset_string_fn_t      parse_cpuset_string;
+    pmix_ploc_base_module_generate_loc_str_fn_t         generate_locality_string;
+    pmix_ploc_base_module_get_rel_loc_fn_t              get_relative_locality;
+    pmix_ploc_base_module_get_cpuset_fn_t               get_cpuset;
+    pmix_ploc_base_module_compute_dist_fn_t             compute_distances;
+    pmix_ploc_base_module_pack_fn_t                     pack_cpuset;
+    pmix_ploc_base_module_unpack_fn_t                   unpack_cpuset;
+    pmix_ploc_base_module_copy_fn_t                     copy_cpuset;
+    pmix_ploc_base_module_print_fn_t                    print_cpuset;
+    pmix_ploc_base_module_destruct_fn_t                 destruct_cpuset;
+    pmix_ploc_base_module_release_fn_t                  release_cpuset;
+    pmix_ploc_base_module_pack_topo_fn_t                pack_topology;
+    pmix_ploc_base_module_unpack_topo_fn_t              unpack_topology;
+    pmix_ploc_base_module_copy_topo_fn_t                copy_topology;
+    pmix_ploc_base_module_print_topo_fn_t               print_topology;
+    pmix_ploc_base_module_destruct_topo_fn_t            destruct_topology;
+    pmix_ploc_base_module_release_topo_fn_t             release_topology;
+} pmix_ploc_module_t;
+
+/* define a public API */
+typedef struct {
+    pmix_ploc_base_module_setup_topo_fn_t               setup_topology;
+    pmix_ploc_base_module_load_topo_fn_t                load_topology;
+    pmix_ploc_base_module_generate_cpuset_string_fn_t   generate_cpuset_string;
+    pmix_ploc_base_module_parse_cpuset_string_fn_t      parse_cpuset_string;
+    pmix_ploc_base_module_generate_loc_str_fn_t         generate_locality_string;
+    pmix_ploc_base_module_get_rel_loc_fn_t              get_relative_locality;
+    pmix_ploc_base_module_get_cpuset_fn_t               get_cpuset;
+    pmix_ploc_base_module_compute_dist_fn_t             compute_distances;
+    pmix_ploc_base_module_pack_fn_t                     pack_cpuset;
+    pmix_ploc_base_module_unpack_fn_t                   unpack_cpuset;
+    pmix_ploc_base_module_copy_fn_t                     copy_cpuset;
+    pmix_ploc_base_module_print_fn_t                    print_cpuset;
+    pmix_ploc_base_API_destruct_fn_t                    destruct_cpuset;
+    pmix_ploc_base_API_release_fn_t                     release_cpuset;
+    pmix_ploc_base_module_pack_topo_fn_t                pack_topology;
+    pmix_ploc_base_module_unpack_topo_fn_t              unpack_topology;
+    pmix_ploc_base_module_copy_topo_fn_t                copy_topology;
+    pmix_ploc_base_module_print_topo_fn_t               print_topology;
+    pmix_ploc_base_API_destruct_topo_fn_t               destruct_topology;
+    pmix_ploc_base_API_release_topo_fn_t                release_topology;
+} pmix_ploc_API_module_t;
+
+
+/* declare the global APIs */
+PMIX_EXPORT extern pmix_ploc_API_module_t pmix_ploc;
+
+/*
+ * the standard component data structure
+ */
+struct pmix_ploc_base_component_t {
+    pmix_mca_base_component_t                        base;
+    pmix_mca_base_component_data_t                   data;
+};
+typedef struct pmix_ploc_base_component_t pmix_ploc_base_component_t;
+
+/*
+ * Macro for use in components that are of type ploc
+ */
+#define PMIX_PLOC_BASE_VERSION_1_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("ploc", 1, 0, 0)
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/ploc/ploc_types.h pmix-4.0.0/src/mca/ploc/ploc_types.h
--- pmix-3.2.2~rc1/src/mca/ploc/ploc_types.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ploc/ploc_types.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,40 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2011 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2019      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/**
+ * @file
+ *
+ * Buffer management types.
+ */
+
+#ifndef PMIX_MCA_PLOC_TYPES_H_
+#define PMIX_MCA_PLOC_TYPES_H_
+
+#include "src/include/pmix_config.h"
+
+
+BEGIN_C_DECLS
+
+
+END_C_DECLS
+
+#endif /* PMIX_PREG_TYPES_H */
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/base/base.h pmix-4.0.0/src/mca/pmdl/base/base.h
--- pmix-3.2.2~rc1/src/mca/pmdl/base/base.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,98 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PMIX_PMDL_BASE_H_
+#define PMIX_PMDL_BASE_H_
+
+#include "src/include/pmix_config.h"
+
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h> /* for struct timeval */
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/pmdl/pmdl.h"
+
+
+BEGIN_C_DECLS
+
+/*
+ * MCA Framework
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_pmdl_base_framework;
+/**
+ * PMDL select function
+ *
+ * Cycle across available components and construct the list
+ * of active modules
+ */
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_select(void);
+
+/**
+ * Track an active component / module
+ */
+struct pmix_pmdl_base_active_module_t {
+    pmix_list_item_t super;
+    int pri;
+    pmix_pmdl_module_t *module;
+    pmix_pmdl_base_component_t *component;
+};
+typedef struct pmix_pmdl_base_active_module_t pmix_pmdl_base_active_module_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pmdl_base_active_module_t);
+
+/* framework globals */
+struct pmix_pmdl_globals_t {
+    pmix_lock_t lock;
+    pmix_list_t actives;
+    bool initialized;
+    bool selected;
+};
+typedef struct pmix_pmdl_globals_t pmix_pmdl_globals_t;
+
+PMIX_EXPORT extern pmix_pmdl_globals_t pmix_pmdl_globals;
+
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_harvest_envars(char *nspace,
+                                                        const pmix_info_t info[], size_t ninfo,
+                                                        pmix_list_t *ilist);
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_setup_nspace(pmix_namespace_t *nptr,
+                                                      pmix_info_t *info);
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_setup_nspace_kv(pmix_namespace_t *nptr,
+                                                         pmix_kval_t *kv);
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_register_nspace(pmix_namespace_t *nptr);
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_setup_client(pmix_namespace_t *nptr,
+                                                      pmix_rank_t rank,
+                                                      uint32_t appnum);
+PMIX_EXPORT pmix_status_t pmix_pmdl_base_setup_fork(const pmix_proc_t *peer, char ***env);
+PMIX_EXPORT void pmix_pmdl_base_deregister_nspace(const char *nptr);
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/base/help-pmdl.txt pmix-4.0.0/src/mca/pmdl/base/help-pmdl.txt
--- pmix-3.2.2~rc1/src/mca/pmdl/base/help-pmdl.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/base/help-pmdl.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,56 @@
+# -*- text -*-
+#
+# Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is a US/English help file
+#
+[reqd-not-found]
+The plog_base_order MCA parameter included a required logging
+channel that is not available:
+
+  Channel:  %s
+
+Please update the parameter and try again.
+#
+[syslog:unrec-level]
+An unrecognized syslog level was given:
+
+  Level:  %s
+
+Please see "man syslog" for a list of defined levels. Input
+parameter strings and their corresponding syslog levels
+recognized by PMIx include:
+
+  Parameter       Level
+   err           LOG_ERR  (default)
+   alert         LOG_ALERT
+   crit          LOG_CRIT
+   emerg         LOG_EMERG
+   warn          LOG_WARNING
+   not           LOG_NOTICE
+   info          LOG_INFO
+   debug         LOG_DEBUG
+
+Please redefine the MCA parameter and try again.
+#
+[syslog:unrec-facility]
+An unsupported or unrecognized value was given for the
+syslog facility (i.e., the type of program calling syslog):
+
+  Value:  %s
+
+Please see "man syslog" for a list of defined facility values.
+PMIx currently supports only the following designations:
+
+   Parameter       Level
+    auth          LOG_AUTH
+    priv          LOG_AUTHPRIV
+    daemon        LOG_DAEMON
+    user          LOG_USER  (default)
+
+Please redefine the MCA parameter and try again.
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/base/Makefile.include pmix-4.0.0/src/mca/pmdl/base/Makefile.include
--- pmix-3.2.2~rc1/src/mca/pmdl/base/Makefile.include	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/base/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from
+# src/Makefile.am
+
+headers += \
+         base/base.h
+
+sources += \
+        base/pmdl_base_frame.c \
+        base/pmdl_base_select.c \
+        base/pmdl_base_stubs.c
+
+dist_pmixdata_DATA = base/help-pmdl.txt
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/base/pmdl_base_frame.c pmix-4.0.0/src/mca/pmdl/base/pmdl_base_frame.c
--- pmix-3.2.2~rc1/src/mca/pmdl/base/pmdl_base_frame.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/base/pmdl_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,99 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ *
+ */
+#include "src/include/pmix_config.h"
+
+#include "include/pmix_common.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/mca/base/base.h"
+#include "src/mca/pmdl/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/pmdl/base/static-components.h"
+
+/* Instantiate the global vars */
+pmix_pmdl_globals_t pmix_pmdl_globals = {{0}};
+pmix_pmdl_API_module_t pmix_pmdl = {
+    .harvest_envars = pmix_pmdl_base_harvest_envars,
+    .setup_nspace = pmix_pmdl_base_setup_nspace,
+    .setup_nspace_kv = pmix_pmdl_base_setup_nspace_kv,
+    .register_nspace = pmix_pmdl_base_register_nspace,
+    .setup_client = pmix_pmdl_base_setup_client,
+    .setup_fork = pmix_pmdl_base_setup_fork,
+    .deregister_nspace = pmix_pmdl_base_deregister_nspace
+};
+
+static pmix_status_t pmix_pmdl_close(void)
+{
+  pmix_pmdl_base_active_module_t *active, *prev;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_SUCCESS;
+    }
+    pmix_pmdl_globals.initialized = false;
+    pmix_pmdl_globals.selected = false;
+
+    PMIX_LIST_FOREACH_SAFE(active, prev, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+      pmix_list_remove_item(&pmix_pmdl_globals.actives, &active->super);
+      if (NULL != active->module->finalize) {
+        active->module->finalize();
+      }
+      PMIX_RELEASE(active);
+    }
+    PMIX_DESTRUCT(&pmix_pmdl_globals.actives);
+
+    PMIX_DESTRUCT_LOCK(&pmix_pmdl_globals.lock);
+    return pmix_mca_base_framework_components_close(&pmix_pmdl_base_framework, NULL);
+}
+
+static pmix_status_t pmix_pmdl_open(pmix_mca_base_open_flag_t flags)
+{
+    /* initialize globals */
+    pmix_pmdl_globals.initialized = true;
+    PMIX_CONSTRUCT_LOCK(&pmix_pmdl_globals.lock);
+    pmix_pmdl_globals.lock.active = false;
+    PMIX_CONSTRUCT(&pmix_pmdl_globals.actives, pmix_list_t);
+
+    /* Open up all available components */
+    return pmix_mca_base_framework_components_open(&pmix_pmdl_base_framework, flags);
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, pmdl, "PMIx Network Operations",
+                                NULL, pmix_pmdl_open, pmix_pmdl_close,
+                                mca_pmdl_base_static_components, 0);
+
+PMIX_CLASS_INSTANCE(pmix_pmdl_base_active_module_t,
+                    pmix_list_item_t,
+                    NULL, NULL);
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/base/pmdl_base_select.c pmix-4.0.0/src/mca/pmdl/base/pmdl_base_select.c
--- pmix-3.2.2~rc1/src/mca/pmdl/base/pmdl_base_select.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/base/pmdl_base_select.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <string.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/pmdl/base/base.h"
+
+/* Function for selecting a prioritized list of components
+ * from all those that are available. */
+int pmix_pmdl_base_select(void)
+{
+    pmix_mca_base_component_list_item_t *cli = NULL;
+    pmix_mca_base_component_t *component = NULL;
+    pmix_mca_base_module_t *module = NULL;
+    pmix_pmdl_module_t *nmodule;
+    pmix_pmdl_base_active_module_t *newmodule, *mod;
+    int rc, priority;
+    bool inserted;
+
+    if (pmix_pmdl_globals.selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    pmix_pmdl_globals.selected = true;
+
+    /* Query all available components and ask if they have a module */
+    PMIX_LIST_FOREACH(cli, &pmix_pmdl_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
+        component = (pmix_mca_base_component_t *) cli->cli_component;
+
+        pmix_output_verbose(5, pmix_pmdl_base_framework.framework_output,
+                            "mca:pmdl:select: checking available component %s", component->pmix_mca_component_name);
+
+        /* If there's no query function, skip it */
+        if (NULL == component->pmix_mca_query_component) {
+            pmix_output_verbose(5, pmix_pmdl_base_framework.framework_output,
+                                "mca:pmdl:select: Skipping component [%s]. It does not implement a query function",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* Query the component */
+        pmix_output_verbose(5, pmix_pmdl_base_framework.framework_output,
+                            "mca:pmdl:select: Querying component [%s]",
+                            component->pmix_mca_component_name);
+        rc = component->pmix_mca_query_component(&module, &priority);
+
+        /* If no module was returned, then skip component */
+        if (PMIX_SUCCESS != rc || NULL == module) {
+            pmix_output_verbose(5, pmix_pmdl_base_framework.framework_output,
+                                "mca:pmdl:select: Skipping component [%s]. Query failed to return a module",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* If we got a module, keep it */
+        nmodule = (pmix_pmdl_module_t*) module;
+        /* let it initialize */
+        if (NULL != nmodule->init && PMIX_SUCCESS != nmodule->init()) {
+            continue;
+        }
+        /* add to the list of selected modules */
+        newmodule = PMIX_NEW(pmix_pmdl_base_active_module_t);
+        newmodule->pri = priority;
+        newmodule->module = nmodule;
+        newmodule->component = (pmix_pmdl_base_component_t*)cli->cli_component;
+
+        /* maintain priority order */
+        inserted = false;
+        PMIX_LIST_FOREACH(mod, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+            if (priority > mod->pri) {
+                pmix_list_insert_pos(&pmix_pmdl_globals.actives,
+                                     (pmix_list_item_t*)mod, &newmodule->super);
+                inserted = true;
+                break;
+            }
+        }
+        if (!inserted) {
+            /* must be lowest priority - add to end */
+            pmix_list_append(&pmix_pmdl_globals.actives, &newmodule->super);
+        }
+    }
+
+    if (4 < pmix_output_get_verbosity(pmix_pmdl_base_framework.framework_output)) {
+        pmix_output(0, "Final pmdl priorities");
+        /* show the prioritized list */
+        PMIX_LIST_FOREACH(mod, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+            pmix_output(0, "\tpmdl: %s Priority: %d", mod->component->base.pmix_mca_component_name, mod->pri);
+        }
+    }
+
+    return PMIX_SUCCESS;;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/base/pmdl_base_stubs.c pmix-4.0.0/src/mca/pmdl/base/pmdl_base_stubs.c
--- pmix-3.2.2~rc1/src/mca/pmdl/base/pmdl_base_stubs.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/base/pmdl_base_stubs.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,270 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_UTSNAME_H
+#include <sys/utsname.h>
+#endif
+#include <time.h>
+
+#include "include/pmix_common.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/preg/preg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/pmix_environ.h"
+#include "src/server/pmix_server_ops.h"
+
+#include "src/mca/pmdl/base/base.h"
+
+
+pmix_status_t pmix_pmdl_base_harvest_envars(char *nspace,
+                                            const pmix_info_t info[], size_t ninfo,
+                                            pmix_list_t *ilist)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_status_t rc;
+    pmix_namespace_t *nptr = NULL, *ns;
+    char *params[2] = {"PMIX_MCA_", NULL};
+    char **priors = NULL;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:harvest envars called");
+
+    /* protect against bozo inputs */
+    if (NULL == ilist) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    if (NULL != nspace) {
+        nptr = NULL;
+        /* find this nspace - note that it may not have
+         * been registered yet */
+        PMIX_LIST_FOREACH(ns, &pmix_globals.nspaces, pmix_namespace_t) {
+            if (0 == strcmp(ns->nspace, nspace)) {
+                nptr = ns;
+                break;
+            }
+        }
+        if (NULL == nptr) {
+            /* add it */
+            nptr = PMIX_NEW(pmix_namespace_t);
+            if (NULL == nptr) {
+                return PMIX_ERR_NOMEM;
+            }
+            nptr->nspace = strdup(nspace);
+            pmix_list_append(&pmix_globals.nspaces, &nptr->super);
+        }
+    }
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->harvest_envars) {
+            rc = active->module->harvest_envars(nptr, info, ninfo, ilist, &priors);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                pmix_argv_free(priors);
+                return rc;
+            }
+        }
+    }
+    pmix_argv_free(priors);
+
+    /* add any local PMIx MCA params */
+    rc = pmix_util_harvest_envars(params, NULL, ilist);
+
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_pmdl_base_setup_nspace(pmix_namespace_t *nptr,
+                                          pmix_info_t *info)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:setup_nspace called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->setup_nspace) {
+            rc = active->module->setup_nspace(nptr, info);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_pmdl_base_setup_nspace_kv(pmix_namespace_t *nptr,
+                                            pmix_kval_t *kv)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:setup_nspace called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->setup_nspace_kv) {
+            rc = active->module->setup_nspace_kv(nptr, kv);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_pmdl_base_register_nspace(pmix_namespace_t *nptr)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:register_nspace called");
+
+    /* process the request */
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->register_nspace) {
+            rc = active->module->register_nspace(nptr);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+/* can only be called by a server */
+pmix_status_t pmix_pmdl_base_setup_client(pmix_namespace_t *nptr,
+                                          pmix_rank_t rank,
+                                          uint32_t appnum)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_status_t rc;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl: setup_client called");
+
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->setup_client) {
+            rc = active->module->setup_client(nptr, rank, appnum);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                return rc;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+/* can only be called by a server */
+pmix_status_t pmix_pmdl_base_setup_fork(const pmix_proc_t *proc, char ***env)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_status_t rc;
+    char **priors = NULL;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->setup_fork) {
+            rc = active->module->setup_fork(proc, env, &priors);
+            if (PMIX_SUCCESS != rc && PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* true error */
+                pmix_argv_free(priors);
+                return rc;
+            }
+        }
+    }
+    pmix_argv_free(priors);
+
+    return PMIX_SUCCESS;
+}
+
+void pmix_pmdl_base_deregister_nspace(const char *ns)
+{
+    pmix_pmdl_base_active_module_t *active;
+    pmix_namespace_t *nptr, *n2;
+
+    if (!pmix_pmdl_globals.initialized) {
+        return;
+    }
+
+    /* search for the namespace */
+    nptr = NULL;
+    PMIX_LIST_FOREACH(n2, &pmix_globals.nspaces, pmix_namespace_t) {
+        if (0 == strncmp(ns, n2->nspace, PMIX_MAX_NSLEN)) {
+            nptr = n2;
+            break;
+        }
+    }
+    if (NULL == nptr) {
+        return;
+    }
+
+    PMIX_LIST_FOREACH(active, &pmix_pmdl_globals.actives, pmix_pmdl_base_active_module_t) {
+        if (NULL != active->module->deregister_nspace) {
+            active->module->deregister_nspace(nptr);
+        }
+    }
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/Makefile.am pmix-4.0.0/src/mca/pmdl/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pmdl/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,44 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(LTDLINCL)
+
+# main library setup
+noinst_LTLIBRARIES = libmca_pmdl.la
+libmca_pmdl_la_SOURCES =
+
+# local files
+headers = pmdl.h
+sources =
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.include
+
+libmca_pmdl_la_SOURCES += $(headers) $(sources)
+
+distclean-local:
+	rm -f base/static-components.h
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi4/Makefile.am pmix-4.0.0/src/mca/pmdl/ompi4/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi4/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi4/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,55 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = pmdl_ompi4.h
+sources = \
+        pmdl_ompi4_component.c \
+        pmdl_ompi4.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_pmdl_ompi4_DSO
+lib =
+lib_sources =
+component = mca_pmdl_ompi4.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_pmdl_ompi4.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_pmdl_ompi4_la_SOURCES = $(component_sources)
+mca_pmdl_ompi4_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_pmdl_ompi4_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_pmdl_ompi4_la_SOURCES = $(lib_sources)
+libmca_pmdl_ompi4_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi4/pmdl_ompi4.c pmix-4.0.0/src/mca/pmdl/ompi4/pmdl_ompi4.c
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi4/pmdl_ompi4.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi4/pmdl_ompi4.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,844 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_UTSNAME_H
+#include <sys/utsname.h>
+#endif
+#include <time.h>
+
+#include "include/pmix.h"
+
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_vari.h"
+#include "src/include/pmix_socket_errno.h"
+#include "src/include/pmix_globals.h"
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/util/alfg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/output.h"
+#include "src/util/os_path.h"
+#include "src/util/printf.h"
+#include "src/util/pmix_environ.h"
+#include "src/mca/preg/preg.h"
+
+#include "src/mca/pmdl/pmdl.h"
+#include "src/mca/pmdl/base/base.h"
+#include "pmdl_ompi4.h"
+
+static pmix_status_t ompi4_init(void);
+static void ompi4_finalize(void);
+static pmix_status_t harvest_envars(pmix_namespace_t *nptr,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_list_t *ilist,
+                                    char ***priors);
+static pmix_status_t setup_nspace(pmix_namespace_t *nptr,
+                                  pmix_info_t *info);
+static pmix_status_t setup_nspace_kv(pmix_namespace_t *nptr,
+                                     pmix_kval_t *kv);
+static pmix_status_t register_nspace(pmix_namespace_t *nptr);
+static pmix_status_t setup_fork(const pmix_proc_t *proc,
+                                char ***env,
+                                char ***priors);
+static void deregister_nspace(pmix_namespace_t *nptr);
+pmix_pmdl_module_t pmix_pmdl_ompi4_module = {
+    .name = "ompi4",
+    .init = ompi4_init,
+    .finalize = ompi4_finalize,
+    .harvest_envars = harvest_envars,
+    .setup_nspace = setup_nspace,
+    .setup_nspace_kv = setup_nspace_kv,
+    .register_nspace = register_nspace,
+    .setup_fork = setup_fork,
+    .deregister_nspace = deregister_nspace
+};
+
+/* internal structures */
+typedef struct {
+	pmix_list_item_t super;
+	pmix_nspace_t nspace;
+	bool datacollected;
+	uint32_t univ_size;
+	uint32_t job_size;
+	uint32_t local_size;
+	uint32_t num_apps;
+} pmdl_nspace_t;
+static void nscon(pmdl_nspace_t *p)
+{
+	p->datacollected = false;
+	p->univ_size = 0;
+	p->job_size = 0;
+	p->local_size = 0;
+	p->num_apps = 0;
+}
+static PMIX_CLASS_INSTANCE(pmdl_nspace_t,
+						   pmix_list_item_t,
+						   nscon, NULL);
+
+/* internal variables */
+static pmix_list_t mynspaces;
+
+static pmix_status_t ompi4_init(void)
+{
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl: ompi4 init");
+
+    PMIX_CONSTRUCT(&mynspaces, pmix_list_t);
+
+    return PMIX_SUCCESS;
+}
+
+static void ompi4_finalize(void)
+{
+    PMIX_LIST_DESTRUCT(&mynspaces);
+}
+
+static bool checkus(const pmix_info_t info[], size_t ninfo)
+{
+    bool takeus = false;
+    char **tmp, *ptr;
+    size_t n, m;
+    uint vers;
+
+    if (NULL == info) {
+        return false;
+    }
+
+    /* check the directives */
+    for (n=0; n < ninfo && !takeus; n++) {
+        /* check the attribute */
+        if (PMIX_CHECK_KEY(&info[n], PMIX_PROGRAMMING_MODEL) ||
+            PMIX_CHECK_KEY(&info[n], PMIX_PERSONALITY)) {
+            tmp = pmix_argv_split(info[n].value.data.string, ',');
+            for (m=0; NULL != tmp[m]; m++) {
+                if (0 == strcmp(tmp[m], "ompi")) {
+                    /* they didn't specify a level, so we will service
+                     * them just in case */
+                    takeus = true;
+                    break;
+                }
+                if (0 == strncmp(tmp[m], "ompi", 4)) {
+                    /* if they specifically requested an ompi level less
+                     * than or equal to us, then we service it */
+                    ptr = &tmp[m][4];
+                    vers = strtoul(ptr, NULL, 10);
+                    if (4 >= vers) {
+                        takeus = true;
+                    }
+                    break;
+                }
+            }
+            pmix_argv_free(tmp);
+        }
+    }
+
+    return takeus;
+}
+
+static pmix_status_t harvest_envars(pmix_namespace_t *nptr,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_list_t *ilist,
+                                    char ***priors)
+{
+    pmdl_nspace_t *ns, *ns2;
+    pmix_status_t rc;
+    size_t n;
+    uint32_t uid = UINT32_MAX;
+    const char *home;
+    pmix_list_t params;
+    pmix_mca_base_var_file_value_t *fv;
+    pmix_kval_t *kv;
+    char *file, *tmp;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi4:harvest envars");
+
+    if (!checkus(info, ninfo)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* don't do OMPI again if already done - ompi5 has higher priority */
+    if (NULL != *priors) {
+        char **t2 = *priors;
+        for (n=0; NULL != t2[n]; n++) {
+            if (0 == strncmp(t2[n], "ompi", 4)) {
+                return PMIX_ERR_TAKE_NEXT_OPTION;
+            }
+        }
+    }
+    /* flag that we worked on this */
+    pmix_argv_append_nosize(priors, "ompi4");
+
+    if (NULL != nptr) {
+        /* see if we already have this nspace */
+        ns = NULL;
+        PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+            if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+                ns = ns2;
+                break;
+            }
+        }
+        if (NULL == ns) {
+            ns = PMIX_NEW(pmdl_nspace_t);
+            PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+            pmix_list_append(&mynspaces, &ns->super);
+        }
+    }
+
+    /* see if the user has a default MCA param file */
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_USERID)) {
+                PMIX_VALUE_GET_NUMBER(rc, &info[n].value, uid, uint32_t);
+                if (PMIX_SUCCESS != rc) {
+                    return rc;
+                }
+                break;
+            }
+        }
+    }
+    if (UINT32_MAX == uid) {
+        uid = geteuid();
+    }
+    /* try to get their home directory */
+    home = pmix_home_directory(uid);
+    if (NULL != home) {
+        file = pmix_os_path(false, home, ".openmpi", "mca-params.conf", NULL);
+        PMIX_CONSTRUCT(&params, pmix_list_t);
+        pmix_mca_base_parse_paramfile(file, &params);
+        free(file);
+        PMIX_LIST_FOREACH(fv, &params, pmix_mca_base_var_file_value_t) {
+            /* need to prefix the param name */
+            kv = PMIX_NEW(pmix_kval_t);
+            if (NULL == kv) {
+                PMIX_LIST_DESTRUCT(&params);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            kv->key = strdup(PMIX_SET_ENVAR);
+            kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+            if (NULL == kv->value) {
+                PMIX_RELEASE(kv);
+                PMIX_LIST_DESTRUCT(&params);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            kv->value->type = PMIX_ENVAR;
+            pmix_asprintf(&tmp, "OMPI_MCA_%s", fv->mbvfv_var);
+            PMIX_ENVAR_LOAD(&kv->value->data.envar, tmp, fv->mbvfv_value, ':');
+            free(tmp);
+            pmix_list_append(ilist, &kv->super);
+        }
+        PMIX_LIST_DESTRUCT(&params);
+        /* add an envar indicating that we did this so the OMPI
+         * processes won't duplicate it */
+        kv = PMIX_NEW(pmix_kval_t);
+        if (NULL == kv) {
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        kv->key = strdup(PMIX_SET_ENVAR);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        if (NULL == kv->value) {
+            PMIX_RELEASE(kv);
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        kv->value->type = PMIX_ENVAR;
+        PMIX_ENVAR_LOAD(&kv->value->data.envar, "OPAL_USER_PARAMS_GIVEN", "1", ':');
+        pmix_list_append(ilist, &kv->super);
+    }
+
+    /* harvest our local envars */
+    if (NULL != mca_pmdl_ompi4_component.include) {
+        pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                            "pmdl: ompi4 harvesting envars %s excluding %s",
+                            (NULL == mca_pmdl_ompi4_component.incparms) ? "NONE" : mca_pmdl_ompi4_component.incparms,
+                            (NULL == mca_pmdl_ompi4_component.excparms) ? "NONE" : mca_pmdl_ompi4_component.excparms);
+        rc = pmix_util_harvest_envars(mca_pmdl_ompi4_component.include,
+                                      mca_pmdl_ompi4_component.exclude,
+                                      ilist);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+
+static pmix_status_t setup_nspace(pmix_namespace_t *nptr,
+                                  pmix_info_t *info)
+{
+	pmdl_nspace_t *ns, *ns2;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi4: setup nspace for nspace %s with %s",
+                        nptr->nspace, info->value.data.string);
+
+    if (!checkus(info, 1)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		ns = PMIX_NEW(pmdl_nspace_t);
+		PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+		pmix_list_append(&mynspaces, &ns->super);
+	}
+
+	return PMIX_SUCCESS;
+}
+
+static pmix_status_t setup_nspace_kv(pmix_namespace_t *nptr,
+                                     pmix_kval_t *kv)
+{
+	pmdl_nspace_t *ns, *ns2;
+    char **tmp, *ptr;
+    size_t m;
+    uint vers;
+    bool takeus = false;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi4: setup nspace_kv for nspace %s with %s",
+                        nptr->nspace, kv->value->data.string);
+
+    /* check the attribute */
+    if (PMIX_CHECK_KEY(kv, PMIX_PROGRAMMING_MODEL) ||
+        PMIX_CHECK_KEY(kv, PMIX_PERSONALITY)) {
+        tmp = pmix_argv_split(kv->value->data.string, ',');
+        for (m=0; NULL != tmp[m]; m++) {
+            if (0 == strcmp(tmp[m], "ompi")) {
+                /* they didn't specify a level, so we will service
+                 * them just in case */
+                takeus = true;
+                break;
+            }
+            if (0 == strncmp(tmp[m], "ompi", 4)) {
+                /* if they specifically requested an ompi level less
+                 * than or equal to us, then we service it */
+                ptr = &tmp[m][4];
+                vers = strtoul(ptr, NULL, 10);
+                if (4 >= vers) {
+                    takeus = true;
+                }
+                break;
+            }
+        }
+        pmix_argv_free(tmp);
+    }
+    if (!takeus) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		ns = PMIX_NEW(pmdl_nspace_t);
+		PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+		pmix_list_append(&mynspaces, &ns->super);
+	}
+
+	return PMIX_SUCCESS;
+}
+
+static pmix_status_t register_nspace(pmix_namespace_t *nptr)
+{
+    pmdl_nspace_t *ns, *ns2;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi4: register_nspace for %s", nptr->nspace);
+
+    /* see if we already have this nspace */
+    ns = NULL;
+    PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+        if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+            ns = ns2;
+            break;
+        }
+    }
+    if (NULL == ns) {
+        /* we don't know anything about this one or
+         * it doesn't have any ompi-based apps */
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* just track the number of local procs so we don't have to
+     * retrieve it later */
+    ns->local_size = nptr->nlocalprocs;
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t setup_fork(const pmix_proc_t *proc,
+                                char ***env,
+                                char ***priors)
+{
+	pmdl_nspace_t *ns, *ns2;
+	char *param;
+    char *ev1, **tmp;
+	pmix_proc_t wildcard, undef;
+	pmix_status_t rc;
+	uint16_t u16;
+    pmix_kval_t *kv;
+    pmix_info_t info[2];
+    uint32_t n;
+    pmix_cb_t cb;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi4: setup fork for %s", PMIX_NAME_PRINT(proc));
+
+    /* don't do OMPI again if already done */
+    if (NULL != *priors) {
+        char **t2 = *priors;
+        for (n=0; NULL != t2[n]; n++) {
+            if (0 == strncmp(t2[n], "ompi", 4)) {
+                return PMIX_ERR_TAKE_NEXT_OPTION;
+            }
+        }
+    }
+    /* flag that we worked on this */
+    pmix_argv_append_nosize(priors, "ompi4");
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, proc->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		/* we don't know anything about this one or
+         * it doesn't have any ompi-based apps */
+		return PMIX_ERR_TAKE_NEXT_OPTION;
+	}
+
+    PMIX_LOAD_PROCID(&wildcard, proc->nspace, PMIX_RANK_WILDCARD);
+    PMIX_LOAD_PROCID(&undef, proc->nspace, PMIX_RANK_UNDEF);
+
+	/* do we already have the data we need here? */
+	if (!ns->datacollected) {
+
+		/* fetch the universe size */
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_UNIV_SIZE;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->univ_size = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+
+		/* fetch the job size */
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_JOB_SIZE;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->job_size = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+
+		/* fetch the number of apps */
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_JOB_NUM_APPS;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->num_apps = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+
+        ns->datacollected = true;
+    }
+
+    /* pass universe size */
+    if (0 > asprintf(&param, "%u", ns->univ_size)) {
+    	return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_UNIVERSE_SIZE", param, true, env);
+    free(param);
+
+    /* pass the comm_world size in various formats */
+    if (0 > asprintf(&param, "%u", ns->job_size)) {
+    	return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_SIZE", param, true, env);
+    pmix_setenv("OMPI_WORLD_SIZE", param, true, env);
+    pmix_setenv("OMPI_MCA_num_procs", param, true, env);
+    free(param);
+
+    /* pass the local size in various formats */
+    if (0 > asprintf(&param, "%u", ns->local_size)) {
+    	return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, env);
+    pmix_setenv("OMPI_WORLD_LOCAL_SIZE", param, true, env);
+    free(param);
+
+    /* pass the number of apps in the job */
+    if (0 > asprintf(&param, "%u", ns->num_apps)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_NUM_APP_CTX", param, true, env);
+    free(param);
+
+    /* pass an envar so the proc can find any files it had prepositioned */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_PROCDIR;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    pmix_setenv("OMPI_FILE_LOCATION", kv->value->data.string, true, env);
+    PMIX_DESTRUCT(&cb);
+
+    /* pass the cwd */
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = &undef;
+    cb.copy = true;
+    cb.info = info;
+    cb.ninfo = 2;
+    cb.key = PMIX_WDIR;
+    PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &pmix_globals.appnum, PMIX_UINT32);
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    PMIX_INFO_DESTRUCT(&info[1]);
+    cb.key = NULL;
+    cb.info = NULL;
+    cb.ninfo = 0;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    pmix_setenv("OMPI_MCA_initial_wdir", kv->value->data.string, true, env);
+    PMIX_DESTRUCT(&cb);
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    /* pass its command. */
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = &undef;
+    cb.copy = true;
+    cb.info = info;
+    cb.ninfo = 2;
+    cb.key = PMIX_APP_ARGV;
+    PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &pmix_globals.appnum, PMIX_UINT32);
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    PMIX_INFO_DESTRUCT(&info[1]);
+    cb.key = NULL;
+    cb.info = NULL;
+    cb.ninfo = 0;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    tmp = pmix_argv_split(kv->value->data.string, ' ');
+    PMIX_DESTRUCT(&cb);
+    PMIX_INFO_DESTRUCT(&info[0]);
+    pmix_setenv("OMPI_COMMAND", tmp[0], true, env);
+    ev1 = pmix_argv_join(&tmp[1], ' ');
+    pmix_setenv("OMPI_ARGV", ev1, true, env);
+    free(ev1);
+    pmix_argv_free(tmp);
+
+    /* pass the arch - if available */
+#ifdef HAVE_SYS_UTSNAME_H
+    struct utsname sysname;
+    memset(&sysname, 0, sizeof(sysname));
+    if (-1 < uname(&sysname)) {
+        if (sysname.machine[0] != '\0') {
+            pmix_setenv("OMPI_MCA_cpu_type", (const char *) &sysname.machine, true, env);
+        }
+    }
+#endif
+
+    /* pass the rank */
+    if (0 > asprintf(&param, "%lu", (unsigned long)proc->rank)) {
+    	return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_RANK", param, true, env);
+    free(param);  /* done with this now */
+
+    /* get the proc's local rank */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_LOCAL_RANK;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    u16 = kv->value->data.uint16;
+    PMIX_DESTRUCT(&cb);
+    if (0 > asprintf(&param, "%lu", (unsigned long)u16)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_LOCAL_RANK", param, true, env);
+    free(param);
+
+    /* get the proc's node rank */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_NODE_RANK;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    u16 = kv->value->data.uint16;
+    PMIX_DESTRUCT(&cb);
+    if (0 > asprintf(&param, "%lu", (unsigned long)u16)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_NODE_RANK", param, true, env);
+    free(param);
+
+    if (1 == ns->num_apps) {
+        return PMIX_SUCCESS;
+    }
+
+    PMIX_LOAD_PROCID(&undef, proc->nspace, PMIX_RANK_UNDEF);
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APP_SIZE;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        pmix_setenv("OMPI_APP_CTX_NUM_PROCS", ev1, true, env);
+        free(ev1);
+    }
+
+
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APPLDR;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        tmp = NULL;
+        pmix_setenv("OMPI_FIRST_RANKS", ev1, true, env);
+        free(ev1);
+    }
+
+    /* provide the reincarnation number */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_REINCARNATION;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+    pmix_setenv("OMPI_MCA_num_restarts", ev1, true, env);
+    free(ev1);
+    PMIX_DESTRUCT(&cb);
+
+    return PMIX_SUCCESS;
+}
+
+static void deregister_nspace(pmix_namespace_t *nptr)
+{
+	pmdl_nspace_t *ns;
+
+	/* find our tracker for this nspace */
+	PMIX_LIST_FOREACH(ns, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns->nspace, nptr->nspace)) {
+			pmix_list_remove_item(&mynspaces, &ns->super);
+			PMIX_RELEASE(ns);
+			return;
+		}
+	}
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi4/pmdl_ompi4_component.c pmix-4.0.0/src/mca/pmdl/ompi4/pmdl_ompi4_component.c
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi4/pmdl_ompi4_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi4/pmdl_ompi4_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,96 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/mca/pmdl/pmdl.h"
+#include "pmdl_ompi4.h"
+
+static pmix_status_t component_register(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_pmdl_ompi4_component_t mca_pmdl_ompi4_component = {
+    .super = {
+        .base = {
+            PMIX_PMDL_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "ompi4",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .pmix_mca_register_component_params = component_register,
+            .pmix_mca_query_component = component_query,
+        },
+        .data = {
+            /* The component is checkpoint ready */
+            PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+        }
+    },
+    .include = NULL,
+    .exclude = NULL
+};
+
+static pmix_status_t component_register(void)
+{
+    pmix_mca_base_component_t *component = &mca_pmdl_ompi4_component.super.base;
+
+    mca_pmdl_ompi4_component.incparms = "OMPI_*";
+    (void)pmix_mca_base_component_var_register(component, "include_envars",
+                                               "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                               &mca_pmdl_ompi4_component.incparms);
+    if (NULL != mca_pmdl_ompi4_component.incparms) {
+        mca_pmdl_ompi4_component.include = pmix_argv_split(mca_pmdl_ompi4_component.incparms, ',');
+    }
+
+    mca_pmdl_ompi4_component.excparms = NULL;
+    (void)pmix_mca_base_component_var_register(component, "exclude_envars",
+                                               "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                               &mca_pmdl_ompi4_component.excparms);
+    if (NULL != mca_pmdl_ompi4_component.excparms) {
+        mca_pmdl_ompi4_component.exclude = pmix_argv_split(mca_pmdl_ompi4_component.excparms, ',');
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 50;
+    *module = (pmix_mca_base_module_t *)&pmix_pmdl_ompi4_module;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi4/pmdl_ompi4.h pmix-4.0.0/src/mca/pmdl/ompi4/pmdl_ompi4.h
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi4/pmdl_ompi4.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi4/pmdl_ompi4.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PMDL_ompi4_H
+#define PMIX_PMDL_ompi4_H
+
+#include "src/include/pmix_config.h"
+
+
+#include "src/mca/pmdl/pmdl.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    pmix_pmdl_base_component_t super;
+    char *incparms;
+    char *excparms;
+    char **include;
+    char **exclude;
+} pmix_pmdl_ompi4_component_t;
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_pmdl_ompi4_component_t mca_pmdl_ompi4_component;
+extern pmix_pmdl_module_t pmix_pmdl_ompi4_module;
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi5/Makefile.am pmix-4.0.0/src/mca/pmdl/ompi5/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi5/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi5/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,55 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = pmdl_ompi5.h
+sources = \
+        pmdl_ompi5_component.c \
+        pmdl_ompi5.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_pmdl_ompi5_DSO
+lib =
+lib_sources =
+component = mca_pmdl_ompi5.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_pmdl_ompi5.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_pmdl_ompi5_la_SOURCES = $(component_sources)
+mca_pmdl_ompi5_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_pmdl_ompi5_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_pmdl_ompi5_la_SOURCES = $(lib_sources)
+libmca_pmdl_ompi5_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi5/pmdl_ompi5.c pmix-4.0.0/src/mca/pmdl/ompi5/pmdl_ompi5.c
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi5/pmdl_ompi5.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi5/pmdl_ompi5.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1014 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_UTSNAME_H
+#include <sys/utsname.h>
+#endif
+#include <time.h>
+
+#include "include/pmix.h"
+
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_vari.h"
+#include "src/include/pmix_socket_errno.h"
+#include "src/include/pmix_globals.h"
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/util/alfg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/output.h"
+#include "src/util/os_path.h"
+#include "src/util/printf.h"
+#include "src/util/pmix_environ.h"
+#include "src/mca/preg/preg.h"
+
+#include "src/mca/pmdl/pmdl.h"
+#include "src/mca/pmdl/base/base.h"
+#include "pmdl_ompi5.h"
+
+static pmix_status_t ompi5_init(void);
+static void ompi5_finalize(void);
+static pmix_status_t harvest_envars(pmix_namespace_t *nptr,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_list_t *ilist,
+                                    char ***priors);
+static pmix_status_t setup_nspace(pmix_namespace_t *nptr,
+                                  pmix_info_t *info);
+static pmix_status_t setup_nspace_kv(pmix_namespace_t *nptr,
+                                     pmix_kval_t *kv);
+static pmix_status_t register_nspace(pmix_namespace_t *nptr);
+static pmix_status_t setup_fork(const pmix_proc_t *proc,
+                                char ***env,
+                                char ***priors);
+static void deregister_nspace(pmix_namespace_t *nptr);
+static void deregister_nspace(pmix_namespace_t *nptr);
+pmix_pmdl_module_t pmix_pmdl_ompi5_module = {
+    .name = "ompi5",
+    .init = ompi5_init,
+    .finalize = ompi5_finalize,
+    .harvest_envars = harvest_envars,
+    .setup_nspace = setup_nspace,
+    .setup_nspace_kv = setup_nspace_kv,
+    .register_nspace = register_nspace,
+    .setup_fork = setup_fork,
+    .deregister_nspace = deregister_nspace
+};
+
+/* internal structures */
+typedef struct {
+	pmix_list_item_t super;
+	pmix_nspace_t nspace;
+	uint32_t univ_size;
+	uint32_t job_size;
+	uint32_t local_size;
+	uint32_t num_apps;
+} pmdl_nspace_t;
+static void nscon(pmdl_nspace_t *p)
+{
+	p->univ_size = UINT32_MAX;
+	p->job_size = UINT32_MAX;
+	p->local_size = UINT32_MAX;
+	p->num_apps = UINT32_MAX;
+}
+static PMIX_CLASS_INSTANCE(pmdl_nspace_t,
+						   pmix_list_item_t,
+						   nscon, NULL);
+
+/* internal variables */
+static pmix_list_t mynspaces;
+
+static pmix_status_t ompi5_init(void)
+{
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl: ompi5 init");
+
+    PMIX_CONSTRUCT(&mynspaces, pmix_list_t);
+
+    return PMIX_SUCCESS;
+}
+
+static void ompi5_finalize(void)
+{
+    PMIX_LIST_DESTRUCT(&mynspaces);
+}
+
+static bool checkus(const pmix_info_t info[], size_t ninfo)
+{
+    bool takeus = false;
+    char **tmp, *ptr;
+    size_t n, m;
+    uint vers;
+
+    if (NULL == info) {
+        return false;
+    }
+
+    /* check the directives */
+    for (n=0; n < ninfo && !takeus; n++) {
+        /* check the attribute */
+        if (PMIX_CHECK_KEY(&info[n], PMIX_PROGRAMMING_MODEL) ||
+            PMIX_CHECK_KEY(&info[n], PMIX_PERSONALITY)) {
+            tmp = pmix_argv_split(info[n].value.data.string, ',');
+            for (m=0; NULL != tmp[m]; m++) {
+                if (0 == strcmp(tmp[m], "ompi")) {
+                    /* they didn't specify a level, so we will service
+                     * them just in case */
+                    takeus = true;
+                    break;
+                }
+                if (0 == strncmp(tmp[m], "ompi", 4)) {
+                    /* if they specifically requested an ompi level greater
+                     * than or equal to us, then we service it */
+                    ptr = &tmp[m][4];
+                    vers = strtoul(ptr, NULL, 10);
+                    if (vers >= 5) {
+                        takeus = true;
+                    }
+                    break;
+                }
+            }
+            pmix_argv_free(tmp);
+        }
+    }
+
+    return takeus;
+}
+
+static pmix_status_t harvest_envars(pmix_namespace_t *nptr,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_list_t *ilist,
+                                    char ***priors)
+{
+    pmdl_nspace_t *ns, *ns2;
+    pmix_status_t rc;
+    uint32_t uid = UINT32_MAX;
+    const char *home;
+    pmix_list_t params;
+    pmix_mca_base_var_file_value_t *fv;
+    pmix_kval_t *kv;
+    size_t n;
+    char *file, *tmp, *evar;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi5:harvest envars");
+
+
+    if (!checkus(info, ninfo)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* don't do OMPI again if already done */
+    if (NULL != *priors) {
+        char **t2 = *priors;
+        for (n=0; NULL != t2[n]; n++) {
+            if (0 == strncmp(t2[n], "ompi", 4)) {
+                return PMIX_ERR_TAKE_NEXT_OPTION;
+            }
+        }
+    }
+    /* flag that we worked on this */
+    pmix_argv_append_nosize(priors, "ompi5");
+
+    if (NULL != nptr) {
+        /* see if we already have this nspace */
+        ns = NULL;
+        PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+            if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+                ns = ns2;
+                break;
+            }
+        }
+        if (NULL == ns) {
+            ns = PMIX_NEW(pmdl_nspace_t);
+            PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+            pmix_list_append(&mynspaces, &ns->super);
+        }
+    }
+
+    /* check if the user has set OMPIHOME in their environment */
+    if (NULL != (evar = getenv("OMPIHOME"))) {
+        /* look for the default MCA param file */
+        file = pmix_os_path(false, evar, "etc", "openmpi-mca-params.conf", NULL);
+        PMIX_CONSTRUCT(&params, pmix_list_t);
+        pmix_mca_base_parse_paramfile(file, &params);
+        free(file);
+        PMIX_LIST_FOREACH(fv, &params, pmix_mca_base_var_file_value_t) {
+            /* need to prefix the param name */
+            kv = PMIX_NEW(pmix_kval_t);
+            if (NULL == kv) {
+                PMIX_LIST_DESTRUCT(&params);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            kv->key = strdup(PMIX_SET_ENVAR);
+            kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+            if (NULL == kv->value) {
+                PMIX_RELEASE(kv);
+                PMIX_LIST_DESTRUCT(&params);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            kv->value->type = PMIX_ENVAR;
+            pmix_asprintf(&tmp, "OMPI_MCA_%s", fv->mbvfv_var);
+            PMIX_ENVAR_LOAD(&kv->value->data.envar, tmp, fv->mbvfv_value, ':');
+            free(tmp);
+            pmix_list_append(ilist, &kv->super);
+        }
+        PMIX_LIST_DESTRUCT(&params);
+        /* add an envar indicating that we did this so the OMPI
+         * processes won't duplicate it */
+        kv = PMIX_NEW(pmix_kval_t);
+        if (NULL == kv) {
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        kv->key = strdup(PMIX_SET_ENVAR);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        if (NULL == kv->value) {
+            PMIX_RELEASE(kv);
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        kv->value->type = PMIX_ENVAR;
+        PMIX_ENVAR_LOAD(&kv->value->data.envar, "OPAL_SYS_PARAMS_GIVEN", "1", ':');
+        pmix_list_append(ilist, &kv->super);
+    }
+
+    /* see if the user has a default MCA param file */
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_USERID)) {
+                PMIX_VALUE_GET_NUMBER(rc, &info[n].value, uid, uint32_t);
+                if (PMIX_SUCCESS != rc) {
+                    return rc;
+                }
+                break;
+            }
+        }
+    }
+    if (UINT32_MAX == uid) {
+        uid = geteuid();
+    }
+    /* try to get their home directory */
+    home = pmix_home_directory(uid);
+    if (NULL != home) {
+        file = pmix_os_path(false, home, ".openmpi", "mca-params.conf", NULL);
+        PMIX_CONSTRUCT(&params, pmix_list_t);
+        pmix_mca_base_parse_paramfile(file, &params);
+        free(file);
+        PMIX_LIST_FOREACH(fv, &params, pmix_mca_base_var_file_value_t) {
+            /* need to prefix the param name */
+            kv = PMIX_NEW(pmix_kval_t);
+            if (NULL == kv) {
+                PMIX_LIST_DESTRUCT(&params);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            kv->key = strdup(PMIX_SET_ENVAR);
+            kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+            if (NULL == kv->value) {
+                PMIX_RELEASE(kv);
+                PMIX_LIST_DESTRUCT(&params);
+                return PMIX_ERR_OUT_OF_RESOURCE;
+            }
+            kv->value->type = PMIX_ENVAR;
+            pmix_asprintf(&tmp, "OMPI_MCA_%s", fv->mbvfv_var);
+            PMIX_ENVAR_LOAD(&kv->value->data.envar, tmp, fv->mbvfv_value, ':');
+            free(tmp);
+            pmix_list_append(ilist, &kv->super);
+        }
+        PMIX_LIST_DESTRUCT(&params);
+        /* add an envar indicating that we did this so the OMPI
+         * processes won't duplicate it */
+        kv = PMIX_NEW(pmix_kval_t);
+        if (NULL == kv) {
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        kv->key = strdup(PMIX_SET_ENVAR);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        if (NULL == kv->value) {
+            PMIX_RELEASE(kv);
+            return PMIX_ERR_OUT_OF_RESOURCE;
+        }
+        kv->value->type = PMIX_ENVAR;
+        PMIX_ENVAR_LOAD(&kv->value->data.envar, "OPAL_USER_PARAMS_GIVEN", "1", ':');
+        pmix_list_append(ilist, &kv->super);
+    }
+
+    /* harvest our local envars */
+    if (NULL != mca_pmdl_ompi5_component.include) {
+        pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                            "pmdl: ompi5 harvesting envars %s excluding %s",
+                            (NULL == mca_pmdl_ompi5_component.incparms) ? "NONE" : mca_pmdl_ompi5_component.incparms,
+                            (NULL == mca_pmdl_ompi5_component.excparms) ? "NONE" : mca_pmdl_ompi5_component.excparms);
+        rc = pmix_util_harvest_envars(mca_pmdl_ompi5_component.include,
+                                      mca_pmdl_ompi5_component.exclude,
+                                      ilist);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+
+static pmix_status_t setup_nspace(pmix_namespace_t *nptr,
+                                  pmix_info_t *info)
+{
+	pmdl_nspace_t *ns, *ns2;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi5: setup nspace for nspace %s with %s",
+                        nptr->nspace, info->value.data.string);
+
+    if (!checkus(info, 1)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		ns = PMIX_NEW(pmdl_nspace_t);
+		PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+		pmix_list_append(&mynspaces, &ns->super);
+	}
+
+	return PMIX_SUCCESS;
+}
+
+static pmix_status_t setup_nspace_kv(pmix_namespace_t *nptr,
+                                     pmix_kval_t *kv)
+{
+	pmdl_nspace_t *ns, *ns2;
+    char **tmp, *ptr;
+    size_t m;
+    uint vers;
+    bool takeus = false;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi5: setup nspace_kv for nspace %s with %s",
+                        nptr->nspace, kv->value->data.string);
+
+    /* check the attribute */
+    if (PMIX_CHECK_KEY(kv, PMIX_PROGRAMMING_MODEL) ||
+        PMIX_CHECK_KEY(kv, PMIX_PERSONALITY)) {
+        tmp = pmix_argv_split(kv->value->data.string, ',');
+        for (m=0; NULL != tmp[m]; m++) {
+            if (0 == strcmp(tmp[m], "ompi")) {
+                /* they didn't specify a level, so we will service
+                 * them just in case */
+                takeus = true;
+                break;
+            }
+            if (0 == strncmp(tmp[m], "ompi", 4)) {
+                /* if they specifically requested an ompi level greater
+                 * than or equal to us, then we service it */
+                ptr = &tmp[m][4];
+                vers = strtoul(ptr, NULL, 10);
+                if (vers >= 5) {
+                    takeus = true;
+                }
+                break;
+            }
+        }
+        pmix_argv_free(tmp);
+    }
+    if (!takeus) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		ns = PMIX_NEW(pmdl_nspace_t);
+		PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+		pmix_list_append(&mynspaces, &ns->super);
+	}
+
+	return PMIX_SUCCESS;
+}
+
+static pmix_status_t register_nspace(pmix_namespace_t *nptr)
+{
+	pmdl_nspace_t *ns, *ns2;
+	char *ev1, **tmp;
+	pmix_proc_t wildcard, undef;
+	pmix_status_t rc;
+    pmix_kval_t *kv;
+    pmix_info_t info[2];
+    uint32_t n;
+    pmix_cb_t cb;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi5: register_nspace for %s", nptr->nspace);
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		/* we don't know anything about this one or
+         * it doesn't have any ompi-based apps */
+		return PMIX_ERR_TAKE_NEXT_OPTION;
+	}
+
+	/* do we already have the data we need here? Servers are
+     * allowed to call register_nspace multiple times with
+     * different info, so we really need to recheck those
+     * values that haven't already been filled */
+    PMIX_LOAD_PROCID(&wildcard, nptr->nspace, PMIX_RANK_WILDCARD);
+
+    /* fetch the universe size */
+    if (UINT32_MAX == ns->univ_size) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_UNIV_SIZE;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->univ_size = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+    }
+
+    /* fetch the job size */
+    if (UINT32_MAX == ns->job_size) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_JOB_SIZE;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->job_size = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+    }
+
+    /* fetch the number of apps */
+    if (UINT32_MAX == ns->num_apps) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_JOB_NUM_APPS;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->num_apps = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+    }
+
+    /* fetch the number of local peers */
+    if (UINT32_MAX == ns->local_size) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_LOCAL_SIZE;
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        /* it is okay if there are no local procs */
+        if (PMIX_SUCCESS == rc) {
+            /* the data is the first value on the cb.kvs list */
+            if (1 != pmix_list_get_size(&cb.kvs)) {
+                PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+                PMIX_DESTRUCT(&cb);
+                return PMIX_ERR_BAD_PARAM;
+            }
+            kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+            ns->local_size = kv->value->data.uint32;
+            PMIX_DESTRUCT(&cb);
+        }
+    }
+
+    if (1 == ns->num_apps) {
+        return PMIX_SUCCESS;
+    }
+
+    /* construct the list of app sizes */
+    PMIX_LOAD_PROCID(&undef, nptr->nspace, PMIX_RANK_UNDEF);
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APP_SIZE;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        PMIX_INFO_LOAD(&info[0], "OMPI_APP_SIZES", ev1, PMIX_STRING);
+        free(ev1);
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, info, 1);
+        PMIX_INFO_DESTRUCT(&info[0]);
+    }
+
+    /* construct the list of app leaders */
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APPLDR;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        tmp = NULL;
+        PMIX_INFO_LOAD(&info[0], "OMPI_FIRST_RANKS", ev1, PMIX_STRING);
+        free(ev1);
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, info, 1);
+        PMIX_INFO_DESTRUCT(&info[0]);
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t setup_fork(const pmix_proc_t *proc,
+                                char ***env,
+                                char ***priors)
+{
+    pmdl_nspace_t *ns, *ns2;
+    char *param;
+    char *ev1, **tmp;
+    pmix_proc_t wildcard, undef;
+    pmix_status_t rc;
+    uint16_t u16;
+    pmix_kval_t *kv;
+    pmix_info_t info[2];
+    uint32_t n;
+    pmix_cb_t cb;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:ompi5: setup fork for %s", PMIX_NAME_PRINT(proc));
+
+    /* don't do OMPI again if already done */
+    if (NULL != *priors) {
+        char **t2 = *priors;
+        for (n=0; NULL != t2[n]; n++) {
+            if (0 == strncmp(t2[n], "ompi", 4)) {
+                return PMIX_ERR_TAKE_NEXT_OPTION;
+            }
+        }
+    }
+    /* flag that we worked on this */
+    pmix_argv_append_nosize(priors, "ompi5");
+
+    /* see if we already have this nspace */
+    ns = NULL;
+    PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+        if (PMIX_CHECK_NSPACE(ns2->nspace, proc->nspace)) {
+            ns = ns2;
+            break;
+        }
+    }
+    if (NULL == ns) {
+        /* we don't know anything about this one or
+         * it doesn't have any ompi-based apps */
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    PMIX_LOAD_PROCID(&wildcard, proc->nspace, PMIX_RANK_WILDCARD);
+    PMIX_LOAD_PROCID(&undef, proc->nspace, PMIX_RANK_UNDEF);
+
+    /* pass universe size */
+    if (0 > asprintf(&param, "%u", ns->univ_size)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_UNIVERSE_SIZE", param, true, env);
+    free(param);
+
+    /* pass the comm_world size in various formats */
+    if (0 > asprintf(&param, "%u", ns->job_size)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_SIZE", param, true, env);
+    pmix_setenv("OMPI_WORLD_SIZE", param, true, env);
+    pmix_setenv("OMPI_MCA_num_procs", param, true, env);
+    free(param);
+
+    /* pass the local size in various formats */
+    if (0 > asprintf(&param, "%u", ns->local_size)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, env);
+    pmix_setenv("OMPI_WORLD_LOCAL_SIZE", param, true, env);
+    free(param);
+
+    /* pass the number of apps in the job */
+    if (0 > asprintf(&param, "%u", ns->num_apps)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_NUM_APP_CTX", param, true, env);
+    free(param);
+
+    /* pass an envar so the proc can find any files it had prepositioned */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_PROCDIR;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    pmix_setenv("OMPI_FILE_LOCATION", kv->value->data.string, true, env);
+    PMIX_DESTRUCT(&cb);
+
+    /* pass the cwd */
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = &undef;
+    cb.copy = true;
+    cb.info = info;
+    cb.ninfo = 2;
+    cb.key = PMIX_WDIR;
+    PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &pmix_globals.appnum, PMIX_UINT32);
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    PMIX_INFO_DESTRUCT(&info[1]);
+    cb.key = NULL;
+    cb.info = NULL;
+    cb.ninfo = 0;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    pmix_setenv("OMPI_MCA_initial_wdir", kv->value->data.string, true, env);
+    PMIX_DESTRUCT(&cb);
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    /* pass its command. */
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = &undef;
+    cb.copy = true;
+    cb.info = info;
+    cb.ninfo = 2;
+    cb.key = PMIX_APP_ARGV;
+    PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &pmix_globals.appnum, PMIX_UINT32);
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    PMIX_INFO_DESTRUCT(&info[1]);
+    cb.key = NULL;
+    cb.info = NULL;
+    cb.ninfo = 0;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    tmp = pmix_argv_split(kv->value->data.string, ' ');
+    PMIX_DESTRUCT(&cb);
+    PMIX_INFO_DESTRUCT(&info[0]);
+    pmix_setenv("OMPI_COMMAND", tmp[0], true, env);
+    ev1 = pmix_argv_join(&tmp[1], ' ');
+    pmix_setenv("OMPI_ARGV", ev1, true, env);
+    free(ev1);
+    pmix_argv_free(tmp);
+
+    /* pass the arch - if available */
+#ifdef HAVE_SYS_UTSNAME_H
+    struct utsname sysname;
+    memset(&sysname, 0, sizeof(sysname));
+    if (-1 < uname(&sysname)) {
+        if (sysname.machine[0] != '\0') {
+            pmix_setenv("OMPI_MCA_cpu_type", (const char *) &sysname.machine, true, env);
+        }
+    }
+#endif
+
+    /* pass the rank */
+    if (0 > asprintf(&param, "%lu", (unsigned long)proc->rank)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_RANK", param, true, env);
+    free(param);  /* done with this now */
+
+    /* get the proc's local rank */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_LOCAL_RANK;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    u16 = kv->value->data.uint16;
+    PMIX_DESTRUCT(&cb);
+    if (0 > asprintf(&param, "%lu", (unsigned long)u16)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_LOCAL_RANK", param, true, env);
+    free(param);
+
+    /* get the proc's node rank */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_NODE_RANK;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    u16 = kv->value->data.uint16;
+    PMIX_DESTRUCT(&cb);
+    if (0 > asprintf(&param, "%lu", (unsigned long)u16)) {
+        return PMIX_ERR_NOMEM;
+    }
+    pmix_setenv("OMPI_COMM_WORLD_NODE_RANK", param, true, env);
+    free(param);
+
+    if (1 == ns->num_apps) {
+        return PMIX_SUCCESS;
+    }
+
+    PMIX_LOAD_PROCID(&undef, proc->nspace, PMIX_RANK_UNDEF);
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APP_SIZE;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        pmix_setenv("OMPI_APP_CTX_NUM_PROCS", ev1, true, env);
+        free(ev1);
+    }
+
+
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APPLDR;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        tmp = NULL;
+        pmix_setenv("OMPI_FIRST_RANKS", ev1, true, env);
+        free(ev1);
+    }
+
+    /* provide the reincarnation number */
+    PMIX_CONSTRUCT(&cb, pmix_cb_t);
+    cb.proc = (pmix_proc_t*)proc;
+    cb.copy = true;
+    cb.key = PMIX_REINCARNATION;
+    PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+    cb.key = NULL;
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_DESTRUCT(&cb);
+        return rc;
+    }
+    /* the data is the first value on the cb.kvs list */
+    if (1 != pmix_list_get_size(&cb.kvs)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        PMIX_DESTRUCT(&cb);
+        return PMIX_ERR_BAD_PARAM;
+    }
+    kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+    pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+    pmix_setenv("OMPI_MCA_num_restarts", ev1, true, env);
+    free(ev1);
+    PMIX_DESTRUCT(&cb);
+
+    return PMIX_SUCCESS;
+}
+
+static void deregister_nspace(pmix_namespace_t *nptr)
+{
+	pmdl_nspace_t *ns;
+
+	/* find our tracker for this nspace */
+	PMIX_LIST_FOREACH(ns, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns->nspace, nptr->nspace)) {
+			pmix_list_remove_item(&mynspaces, &ns->super);
+			PMIX_RELEASE(ns);
+			return;
+		}
+	}
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi5/pmdl_ompi5_component.c pmix-4.0.0/src/mca/pmdl/ompi5/pmdl_ompi5_component.c
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi5/pmdl_ompi5_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi5/pmdl_ompi5_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,97 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/util/argv.h"
+#include "src/mca/pmdl/pmdl.h"
+#include "pmdl_ompi5.h"
+
+static pmix_status_t component_register(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_pmdl_ompi5_component_t mca_pmdl_ompi5_component = {
+    .super = {
+        .base = {
+            PMIX_PMDL_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "ompi5",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .pmix_mca_register_component_params = component_register,
+            .pmix_mca_query_component = component_query,
+        },
+        .data = {
+            /* The component is checkpoint ready */
+            PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+        }
+    },
+    .include = NULL,
+    .exclude = NULL
+};
+
+static pmix_status_t component_register(void)
+{
+    pmix_mca_base_component_t *component = &mca_pmdl_ompi5_component.super.base;
+
+    mca_pmdl_ompi5_component.incparms = "OMPI_*";
+    (void)pmix_mca_base_component_var_register(component, "include_envars",
+                                               "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                               &mca_pmdl_ompi5_component.incparms);
+    if (NULL != mca_pmdl_ompi5_component.incparms) {
+        mca_pmdl_ompi5_component.include = pmix_argv_split(mca_pmdl_ompi5_component.incparms, ',');
+    }
+
+    mca_pmdl_ompi5_component.excparms = NULL;
+    (void)pmix_mca_base_component_var_register(component, "exclude_envars",
+                                               "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                               &mca_pmdl_ompi5_component.excparms);
+    if (NULL != mca_pmdl_ompi5_component.excparms) {
+        mca_pmdl_ompi5_component.exclude = pmix_argv_split(mca_pmdl_ompi5_component.excparms, ',');
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 55;
+    *module = (pmix_mca_base_module_t *)&pmix_pmdl_ompi5_module;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/ompi5/pmdl_ompi5.h pmix-4.0.0/src/mca/pmdl/ompi5/pmdl_ompi5.h
--- pmix-3.2.2~rc1/src/mca/pmdl/ompi5/pmdl_ompi5.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/ompi5/pmdl_ompi5.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PMDL_ompi5_H
+#define PMIX_PMDL_ompi5_H
+
+#include "src/include/pmix_config.h"
+
+
+#include "src/mca/pmdl/pmdl.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    pmix_pmdl_base_component_t super;
+    char *incparms;
+    char *excparms;
+    char **include;
+    char **exclude;
+} pmix_pmdl_ompi5_component_t;
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_pmdl_ompi5_component_t mca_pmdl_ompi5_component;
+extern pmix_pmdl_module_t pmix_pmdl_ompi5_module;
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/oshmem/Makefile.am pmix-4.0.0/src/mca/pmdl/oshmem/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pmdl/oshmem/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/oshmem/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,55 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = pmdl_oshmem.h
+sources = \
+        pmdl_oshmem_component.c \
+        pmdl_oshmem.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_pmdl_oshmem_DSO
+lib =
+lib_sources =
+component = mca_pmdl_oshmem.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_pmdl_oshmem.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_pmdl_oshmem_la_SOURCES = $(component_sources)
+mca_pmdl_oshmem_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_pmdl_oshmem_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_pmdl_oshmem_la_SOURCES = $(lib_sources)
+libmca_pmdl_oshmem_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/oshmem/pmdl_oshmem.c pmix-4.0.0/src/mca/pmdl/oshmem/pmdl_oshmem.c
--- pmix-3.2.2~rc1/src/mca/pmdl/oshmem/pmdl_oshmem.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/oshmem/pmdl_oshmem.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,446 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include <time.h>
+
+#include "include/pmix.h"
+
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_vari.h"
+#include "src/include/pmix_socket_errno.h"
+#include "src/include/pmix_globals.h"
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/util/alfg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/output.h"
+#include "src/util/os_path.h"
+#include "src/util/printf.h"
+#include "src/util/pmix_environ.h"
+#include "src/mca/preg/preg.h"
+
+#include "src/mca/pmdl/pmdl.h"
+#include "src/mca/pmdl/base/base.h"
+#include "pmdl_oshmem.h"
+
+static pmix_status_t oshmem_init(void);
+static void oshmem_finalize(void);
+static pmix_status_t harvest_envars(pmix_namespace_t *nptr,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_list_t *ilist,
+                                    char ***priors);
+static pmix_status_t setup_nspace(pmix_namespace_t *nptr,
+                                  pmix_info_t *info);
+static pmix_status_t setup_nspace_kv(pmix_namespace_t *nptr,
+                                     pmix_kval_t *kv);
+static pmix_status_t register_nspace(pmix_namespace_t *nptr);
+static void deregister_nspace(pmix_namespace_t *nptr);
+pmix_pmdl_module_t pmix_pmdl_oshmem_module = {
+    .name = "oshmem",
+    .init = oshmem_init,
+    .finalize = oshmem_finalize,
+    .harvest_envars = harvest_envars,
+    .setup_nspace = setup_nspace,
+    .setup_nspace_kv = setup_nspace_kv,
+    .register_nspace = register_nspace,
+    .deregister_nspace = deregister_nspace
+};
+
+/* internal structures */
+typedef struct {
+	pmix_list_item_t super;
+	pmix_nspace_t nspace;
+	bool datacollected;
+	uint32_t univ_size;
+	uint32_t job_size;
+	uint32_t local_size;
+	uint32_t num_apps;
+} pmdl_nspace_t;
+static void nscon(pmdl_nspace_t *p)
+{
+	p->datacollected = false;
+	p->univ_size = 0;
+	p->job_size = 0;
+	p->local_size = 0;
+	p->num_apps = 0;
+}
+static PMIX_CLASS_INSTANCE(pmdl_nspace_t,
+						   pmix_list_item_t,
+						   nscon, NULL);
+
+/* internal variables */
+static pmix_list_t mynspaces;
+
+static pmix_status_t oshmem_init(void)
+{
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl: oshmem init");
+
+    PMIX_CONSTRUCT(&mynspaces, pmix_list_t);
+
+    return PMIX_SUCCESS;
+}
+
+static void oshmem_finalize(void)
+{
+    PMIX_LIST_DESTRUCT(&mynspaces);
+}
+
+static bool checkus(const pmix_info_t info[], size_t ninfo)
+{
+    bool takeus = false;
+    char **tmp;
+    size_t n, m;
+
+    if (NULL == info) {
+        return false;
+    }
+
+    /* check the directives */
+    for (n=0; n < ninfo && !takeus; n++) {
+        /* check the attribute */
+        if (PMIX_CHECK_KEY(&info[n], PMIX_PROGRAMMING_MODEL) ||
+            PMIX_CHECK_KEY(&info[n], PMIX_PERSONALITY)) {
+            tmp = pmix_argv_split(info[n].value.data.string, ',');
+            for (m=0; NULL != tmp[m]; m++) {
+                if (0 == strcmp(tmp[m], "oshmem")) {
+                    takeus = true;
+                    break;
+                }
+            }
+            pmix_argv_free(tmp);
+        }
+    }
+
+    return takeus;
+}
+
+static pmix_status_t harvest_envars(pmix_namespace_t *nptr,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_list_t *ilist,
+                                    char ***priors)
+{
+    pmdl_nspace_t *ns, *ns2;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:oshmem:harvest envars");
+
+
+    if (!checkus(info, ninfo)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    if (NULL != nptr) {
+        /* see if we already have this nspace */
+        ns = NULL;
+        PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+            if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+                ns = ns2;
+                break;
+            }
+        }
+        if (NULL == ns) {
+            ns = PMIX_NEW(pmdl_nspace_t);
+            PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+            pmix_list_append(&mynspaces, &ns->super);
+        }
+    }
+
+    /* harvest our local envars */
+    if (NULL != mca_pmdl_oshmem_component.include) {
+        pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                            "pmdl: oshmem harvesting envars %s excluding %s",
+                            (NULL == mca_pmdl_oshmem_component.incparms) ? "NONE" : mca_pmdl_oshmem_component.incparms,
+                            (NULL == mca_pmdl_oshmem_component.excparms) ? "NONE" : mca_pmdl_oshmem_component.excparms);
+        rc = pmix_util_harvest_envars(mca_pmdl_oshmem_component.include,
+                                      mca_pmdl_oshmem_component.exclude,
+                                      ilist);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+
+static pmix_status_t setup_nspace(pmix_namespace_t *nptr,
+                                  pmix_info_t *info)
+{
+	pmdl_nspace_t *ns, *ns2;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:oshmem: setup nspace for nspace %s with %s",
+                        nptr->nspace, info->value.data.string);
+
+    if (!checkus(info, 1)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		ns = PMIX_NEW(pmdl_nspace_t);
+		PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+		pmix_list_append(&mynspaces, &ns->super);
+	}
+
+	return PMIX_SUCCESS;
+}
+
+static pmix_status_t setup_nspace_kv(pmix_namespace_t *nptr,
+                                     pmix_kval_t *kv)
+{
+	pmdl_nspace_t *ns, *ns2;
+    char **tmp, *ptr;
+    size_t m;
+    uint vers;
+    bool takeus = false;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:oshmem: setup nspace_kv for nspace %s with %s",
+                        nptr->nspace, kv->value->data.string);
+
+    /* check the attribute */
+    if (PMIX_CHECK_KEY(kv, PMIX_PROGRAMMING_MODEL) ||
+        PMIX_CHECK_KEY(kv, PMIX_PERSONALITY)) {
+        tmp = pmix_argv_split(kv->value->data.string, ',');
+        for (m=0; NULL != tmp[m]; m++) {
+            if (0 == strcmp(tmp[m], "ompi")) {
+                /* they didn't specify a level, so we will service
+                 * them just in case */
+                takeus = true;
+                break;
+            }
+            if (0 == strncmp(tmp[m], "ompi", 4)) {
+                /* if they specifically requested an ompi level greater
+                 * than or equal to us, then we service it */
+                ptr = &tmp[m][4];
+                vers = strtoul(ptr, NULL, 10);
+                if (vers >= 5) {
+                    takeus = true;
+                }
+                break;
+            }
+        }
+        pmix_argv_free(tmp);
+    }
+    if (!takeus) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		ns = PMIX_NEW(pmdl_nspace_t);
+		PMIX_LOAD_NSPACE(ns->nspace, nptr->nspace);
+		pmix_list_append(&mynspaces, &ns->super);
+	}
+
+	return PMIX_SUCCESS;
+}
+
+static pmix_status_t register_nspace(pmix_namespace_t *nptr)
+{
+	pmdl_nspace_t *ns, *ns2;
+	char *ev1, **tmp;
+	pmix_proc_t wildcard, undef;
+	pmix_status_t rc;
+    pmix_kval_t *kv;
+    pmix_info_t info[2];
+    uint32_t n;
+    pmix_cb_t cb;
+
+    pmix_output_verbose(2, pmix_pmdl_base_framework.framework_output,
+                        "pmdl:oshmem: register_nspace for %s", nptr->nspace);
+
+	/* see if we already have this nspace */
+	ns = NULL;
+	PMIX_LIST_FOREACH(ns2, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns2->nspace, nptr->nspace)) {
+			ns = ns2;
+			break;
+		}
+	}
+	if (NULL == ns) {
+		/* we don't know anything about this one or
+         * it doesn't have any ompi-based apps */
+		return PMIX_ERR_TAKE_NEXT_OPTION;
+	}
+
+	/* do we already have the data we need here? */
+	if (!ns->datacollected) {
+        PMIX_LOAD_PROCID(&wildcard, nptr->nspace, PMIX_RANK_WILDCARD);
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &wildcard;
+        cb.copy = true;
+        cb.key = PMIX_JOB_NUM_APPS;
+		/* fetch the number of apps */
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        cb.key = NULL;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        ns->num_apps = kv->value->data.uint32;
+        PMIX_DESTRUCT(&cb);
+
+        ns->datacollected = true;
+    }
+
+    if (1 == ns->num_apps) {
+        return PMIX_SUCCESS;
+    }
+
+    /* construct the list of app sizes */
+    PMIX_LOAD_PROCID(&undef, nptr->nspace, PMIX_RANK_UNDEF);
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APP_SIZE;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        PMIX_INFO_LOAD(&info[0], "OMPI_APP_SIZES", ev1, PMIX_STRING);
+        free(ev1);
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, info, 1);
+        PMIX_INFO_DESTRUCT(&info[0]);
+    }
+
+    /* construct the list of app leaders */
+    PMIX_INFO_LOAD(&info[0], PMIX_APP_INFO, NULL, PMIX_BOOL);
+    tmp = NULL;
+    for (n=0; n < ns->num_apps; n++) {
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        cb.proc = &undef;
+        cb.copy = true;
+        cb.info = info;
+        cb.ninfo = 2;
+        cb.key = PMIX_APPLDR;
+        PMIX_INFO_LOAD(&info[1], PMIX_APPNUM, &n, PMIX_UINT32);
+        PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        PMIX_INFO_DESTRUCT(&info[1]);
+        cb.key = NULL;
+        cb.info = NULL;
+        cb.ninfo = 0;
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_DESTRUCT(&cb);
+            return rc;
+        }
+        /* the data is the first value on the cb.kvs list */
+        if (1 != pmix_list_get_size(&cb.kvs)) {
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs);
+        pmix_asprintf(&ev1, "%u", kv->value->data.uint32);
+        pmix_argv_append_nosize(&tmp, ev1);
+        free(ev1);
+        PMIX_DESTRUCT(&cb);
+    }
+    PMIX_INFO_DESTRUCT(&info[0]);
+
+    if (NULL != tmp) {
+        ev1 = pmix_argv_join(tmp, ' ');
+        pmix_argv_free(tmp);
+        tmp = NULL;
+        PMIX_INFO_LOAD(&info[0], "OMPI_FIRST_RANKS", ev1, PMIX_STRING);
+        free(ev1);
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, info, 1);
+        PMIX_INFO_DESTRUCT(&info[0]);
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static void deregister_nspace(pmix_namespace_t *nptr)
+{
+	pmdl_nspace_t *ns;
+
+	/* find our tracker for this nspace */
+	PMIX_LIST_FOREACH(ns, &mynspaces, pmdl_nspace_t) {
+		if (PMIX_CHECK_NSPACE(ns->nspace, nptr->nspace)) {
+			pmix_list_remove_item(&mynspaces, &ns->super);
+			PMIX_RELEASE(ns);
+			return;
+		}
+	}
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/oshmem/pmdl_oshmem_component.c pmix-4.0.0/src/mca/pmdl/oshmem/pmdl_oshmem_component.c
--- pmix-3.2.2~rc1/src/mca/pmdl/oshmem/pmdl_oshmem_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/oshmem/pmdl_oshmem_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,97 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/util/argv.h"
+#include "src/mca/pmdl/pmdl.h"
+#include "pmdl_oshmem.h"
+
+static pmix_status_t component_register(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_pmdl_oshmem_component_t mca_pmdl_oshmem_component = {
+    .super = {
+        .base = {
+            PMIX_PMDL_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "oshmem",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .pmix_mca_register_component_params = component_register,
+            .pmix_mca_query_component = component_query,
+        },
+        .data = {
+            /* The component is checkpoint ready */
+            PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+        }
+    },
+    .include = NULL,
+    .exclude = NULL
+};
+
+static pmix_status_t component_register(void)
+{
+    pmix_mca_base_component_t *component = &mca_pmdl_oshmem_component.super.base;
+
+    mca_pmdl_oshmem_component.incparms = "SHMEM_*,SMA_*";
+    (void)pmix_mca_base_component_var_register(component, "include_envars",
+                                               "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                               &mca_pmdl_oshmem_component.incparms);
+    if (NULL != mca_pmdl_oshmem_component.incparms) {
+        mca_pmdl_oshmem_component.include = pmix_argv_split(mca_pmdl_oshmem_component.incparms, ',');
+    }
+
+    mca_pmdl_oshmem_component.excparms = NULL;
+    (void)pmix_mca_base_component_var_register(component, "exclude_envars",
+                                               "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                               &mca_pmdl_oshmem_component.excparms);
+    if (NULL != mca_pmdl_oshmem_component.excparms) {
+        mca_pmdl_oshmem_component.exclude = pmix_argv_split(mca_pmdl_oshmem_component.excparms, ',');
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 30;
+    *module = (pmix_mca_base_module_t *)&pmix_pmdl_oshmem_module;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/oshmem/pmdl_oshmem.h pmix-4.0.0/src/mca/pmdl/oshmem/pmdl_oshmem.h
--- pmix-3.2.2~rc1/src/mca/pmdl/oshmem/pmdl_oshmem.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/oshmem/pmdl_oshmem.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PMDL_oshmem_H
+#define PMIX_PMDL_oshmem_H
+
+#include "src/include/pmix_config.h"
+
+
+#include "src/mca/pmdl/pmdl.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    pmix_pmdl_base_component_t super;
+    char *incparms;
+    char *excparms;
+    char **include;
+    char **exclude;
+} pmix_pmdl_oshmem_component_t;
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_pmdl_oshmem_component_t mca_pmdl_oshmem_component;
+extern pmix_pmdl_module_t pmix_pmdl_oshmem_module;
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/pmdl/pmdl.h pmix-4.0.0/src/mca/pmdl/pmdl.h
--- pmix-3.2.2~rc1/src/mca/pmdl/pmdl.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pmdl/pmdl.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,152 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This interface is for use by PMIx servers to obtain network-related info
+ * such as security keys that need to be shared across applications, and to
+ * setup network support for applications prior to launch
+ *
+ * Available plugins may be defined at runtime via the typical MCA parameter
+ * syntax.
+ */
+
+#ifndef PMIX_PMDL_H
+#define PMIX_PMDL_H
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+#include "src/include/pmix_globals.h"
+#include "src/server/pmix_server_ops.h"
+
+BEGIN_C_DECLS
+
+/******    MODULE DEFINITION    ******/
+
+/**
+ * Initialize the module. Returns an error if the module cannot
+ * run, success if it can and wants to be used.
+ */
+typedef pmix_status_t (*pmix_pmdl_base_module_init_fn_t)(void);
+
+/**
+ * Finalize the module. Tear down any allocated storage, disconnect
+ * from any system support (e.g., LDAP server)
+ */
+typedef void (*pmix_pmdl_base_module_fini_fn_t)(void);
+
+/* Harvest envars for this programming model so they can be forwarded
+ * to backend processes */
+typedef pmix_status_t (*pmix_pmdl_base_module_harvest_envars_fn_t)(pmix_namespace_t *nptr,
+                                                                   const pmix_info_t info[], size_t ninfo,
+                                                                   pmix_list_t *ilist,
+                                                                   char ***priors);
+/**
+ * Setup any programming model specific support for the given nspace
+ */
+typedef pmix_status_t (*pmix_pmdl_base_module_setup_ns_fn_t)(pmix_namespace_t *nptr,
+                                                             pmix_info_t *info);
+typedef pmix_status_t (*pmix_pmdl_base_module_setup_ns_kv_fn_t)(pmix_namespace_t *nptr,
+                                                                pmix_kval_t *kv);
+
+/* Allow programming models to add key-value pairs to the job-info - they
+ * can directly cache them in the GDS */
+typedef pmix_status_t (*pmix_pmdl_base_module_reg_nspace_fn_t)(pmix_namespace_t *nptr);
+
+/**
+ * Setup any programming model specific support for the given client */
+typedef pmix_status_t (*pmix_pmdl_base_module_setup_client_fn_t)(pmix_namespace_t *nptr,
+                                                                 pmix_rank_t rank,
+                                                                 uint32_t apppnum);
+
+/**
+ * Give the plugins an opportunity to add any envars to the
+ * environment of a local application process prior to fork/exec
+ */
+typedef pmix_status_t (*pmix_pmdl_base_module_setup_fork_fn_t)(const pmix_proc_t *peer,
+                                                               char ***env,
+                                                               char ***priors);
+
+/**
+ * Provide an opportunity for the fabric components to cleanup any
+ * resources they may have created to track the nspace
+ */
+typedef void (*pmix_pmdl_base_module_dregister_nspace_fn_t)(pmix_namespace_t *nptr);
+
+/**
+ * Base structure for a PMDL module. Each component should malloc a
+ * copy of the module structure for each fabric plane they support.
+ */
+typedef struct {
+    char *name;
+    pmix_pmdl_base_module_init_fn_t                 init;
+    pmix_pmdl_base_module_fini_fn_t                 finalize;
+    pmix_pmdl_base_module_harvest_envars_fn_t       harvest_envars;
+    pmix_pmdl_base_module_setup_ns_fn_t             setup_nspace;
+    pmix_pmdl_base_module_setup_ns_kv_fn_t          setup_nspace_kv;
+    pmix_pmdl_base_module_reg_nspace_fn_t           register_nspace;
+    pmix_pmdl_base_module_setup_client_fn_t         setup_client;
+    pmix_pmdl_base_module_setup_fork_fn_t           setup_fork;
+    pmix_pmdl_base_module_dregister_nspace_fn_t     deregister_nspace;
+} pmix_pmdl_module_t;
+
+/* define a public API */
+
+typedef pmix_status_t (*pmix_pmdl_base_API_harvest_envars_fn_t)(char *nspace,
+                                                                const pmix_info_t info[], size_t ninfo,
+                                                                pmix_list_t *ilist);
+typedef pmix_status_t (*pmix_pmdl_base_API_setup_fork_fn_t)(const pmix_proc_t *peer,
+                                                            char ***env);
+typedef void (*pmix_pmdl_base_API_dregister_nspace_fn_t)(const char *nptr);
+typedef struct {
+    char *name;
+    pmix_pmdl_base_module_init_fn_t                 init;
+    pmix_pmdl_base_module_fini_fn_t                 finalize;
+    pmix_pmdl_base_API_harvest_envars_fn_t          harvest_envars;
+    pmix_pmdl_base_module_setup_ns_fn_t             setup_nspace;
+    pmix_pmdl_base_module_setup_ns_kv_fn_t          setup_nspace_kv;
+    pmix_pmdl_base_module_reg_nspace_fn_t           register_nspace;
+    pmix_pmdl_base_module_setup_client_fn_t         setup_client;
+    pmix_pmdl_base_API_setup_fork_fn_t              setup_fork;
+    pmix_pmdl_base_API_dregister_nspace_fn_t        deregister_nspace;
+} pmix_pmdl_API_module_t;
+
+
+/* declare the global APIs */
+PMIX_EXPORT extern pmix_pmdl_API_module_t pmix_pmdl;
+
+/*
+ * the standard component data structure
+ */
+struct pmix_pmdl_base_component_t {
+    pmix_mca_base_component_t                        base;
+    pmix_mca_base_component_data_t                   data;
+};
+typedef struct pmix_pmdl_base_component_t pmix_pmdl_base_component_t;
+
+/*
+ * Macro for use in components that are of type pmdl
+ */
+#define PMIX_PMDL_BASE_VERSION_1_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("pmdl", 1, 0, 0)
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/base/base.h pmix-4.0.0/src/mca/pnet/base/base.h
--- pmix-3.2.2~rc1/src/mca/pnet/base/base.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -146,6 +146,13 @@
 PMIX_EXPORT pmix_status_t pmix_pnet_base_harvest_envars(char **incvars, char **excvars,
                                                         pmix_list_t *ilist);
 
+PMIX_EXPORT pmix_status_t pmix_pnet_base_register_fabric(pmix_fabric_t *fabric,
+                                                         const pmix_info_t directives[],
+                                                         size_t ndirs,
+                                                         pmix_op_cbfunc_t cbfunc, void *cbdata);
+PMIX_EXPORT pmix_status_t pmix_pnet_base_deregister_fabric(pmix_fabric_t *fabric);
+PMIX_EXPORT pmix_status_t pmix_pnet_base_update_fabric(pmix_fabric_t *fabric);
+
 END_C_DECLS
 
 #endif
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/base/pnet_base_fns.c pmix-4.0.0/src/mca/pnet/base/pnet_base_fns.c
--- pmix-3.2.2~rc1/src/mca/pnet/base/pnet_base_fns.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/base/pnet_base_fns.c	2021-01-02 08:56:17.000000000 +0000
@@ -55,7 +55,14 @@
     if (NULL == nspace || NULL == ilist) {
         return PMIX_ERR_BAD_PARAM;
     }
-    if (PMIX_PEER_IS_SCHEDULER(pmix_globals.mypeer)) {
+
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
+        PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+        if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+            PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+            return PMIX_SUCCESS;
+        }
+
         nptr = NULL;
         /* find this nspace - note that it may not have
          * been registered yet */
@@ -69,6 +76,7 @@
             /* add it */
             nptr = PMIX_NEW(pmix_namespace_t);
             if (NULL == nptr) {
+                PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
                 return PMIX_ERR_NOMEM;
             }
             nptr->nspace = strdup(nspace);
@@ -83,11 +91,13 @@
                 if (PMIX_CHECK_KEY(&info[n], PMIX_NODE_MAP)) {
                     rc = pmix_preg.parse_nodes(info[n].value.data.bo.bytes, &nodes);
                     if (PMIX_SUCCESS != rc) {
+                        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
                         return rc;
                     }
                 } else if (PMIX_CHECK_KEY(&info[n], PMIX_PROC_MAP)) {
                     rc = pmix_preg.parse_procs(info[n].value.data.bo.bytes, &procs);
                     if (PMIX_SUCCESS != rc) {
+                        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
                         return rc;
                     }
                 }
@@ -104,22 +114,25 @@
                 pmix_argv_free(nodes);
                 pmix_argv_free(procs);
                 if (PMIX_SUCCESS != rc) {
+                    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
                     return rc;
                 }
             }
-            /* process the allocation request */
-            PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) {
-                if (NULL != active->module->allocate) {
-                    if (PMIX_SUCCESS == (rc = active->module->allocate(nptr, info, ninfo, ilist))) {
-                        break;
-                    }
-                    if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
-                        /* true error */
-                        return rc;
-                    }
+        }
+        /* process the allocation request */
+        PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) {
+            if (NULL != active->module->allocate) {
+                if (PMIX_SUCCESS == (rc = active->module->allocate(nptr, info, ninfo, ilist))) {
+                    continue;
+                }
+                if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                    /* true error */
+                    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+                    return rc;
                 }
             }
         }
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
     }
 
     return PMIX_SUCCESS;
@@ -146,6 +159,12 @@
         return PMIX_ERR_BAD_PARAM;
     }
 
+    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+        return PMIX_SUCCESS;
+    }
+
     /* find this proc's nspace object */
     nptr = NULL;
     PMIX_LIST_FOREACH(ns, &pmix_globals.nspaces, pmix_namespace_t) {
@@ -158,6 +177,7 @@
         /* add it */
         nptr = PMIX_NEW(pmix_namespace_t);
         if (NULL == nptr) {
+            PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
             return PMIX_ERR_NOMEM;
         }
         nptr->nspace = strdup(nspace);
@@ -167,11 +187,13 @@
     PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) {
         if (NULL != active->module->setup_local_network) {
             if (PMIX_SUCCESS != (rc = active->module->setup_local_network(nptr, info, ninfo))) {
+                PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
                 return rc;
             }
         }
     }
 
+    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
     return PMIX_SUCCESS;
 }
 
@@ -186,11 +208,20 @@
         return PMIX_ERR_INIT;
     }
 
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet: setup_fork called");
+
     /* protect against bozo inputs */
     if (NULL == proc || NULL == env) {
         return PMIX_ERR_BAD_PARAM;
     }
 
+    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+        return PMIX_SUCCESS;
+    }
+
     /* find this proc's nspace object */
     nptr = NULL;
     PMIX_LIST_FOREACH(ns, &pmix_globals.nspaces, pmix_namespace_t) {
@@ -203,6 +234,7 @@
         /* add it */
         nptr = PMIX_NEW(pmix_namespace_t);
         if (NULL == nptr) {
+            PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
             return PMIX_ERR_NOMEM;
         }
         nptr->nspace = strdup(proc->nspace);
@@ -213,11 +245,13 @@
         if (NULL != active->module->setup_fork) {
             rc = active->module->setup_fork(nptr, proc, env);
             if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) {
+                PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
                 return rc;
             }
         }
     }
 
+    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
     return PMIX_SUCCESS;
 }
 
@@ -229,18 +263,28 @@
         return;
     }
 
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet: child_finalized called");
+
     /* protect against bozo inputs */
     if (NULL == peer) {
         PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
         return;
     }
 
+    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+        return;
+    }
+
     PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) {
         if (NULL != active->module->child_finalized) {
             active->module->child_finalized(peer);
         }
     }
 
+    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
     return;
 }
 
@@ -252,17 +296,27 @@
         return;
     }
 
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet: local_app_finalized called");
+
     /* protect against bozo inputs */
     if (NULL == nptr) {
         return;
     }
 
+    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+        return;
+    }
+
     PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) {
         if (NULL != active->module->local_app_finalized) {
             active->module->local_app_finalized(nptr);
         }
     }
 
+    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
     return;
 }
 
@@ -277,11 +331,20 @@
         return;
     }
 
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet: deregister_nspace called");
+
     /* protect against bozo inputs */
     if (NULL == nspace) {
         return;
     }
 
+    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+        return;
+    }
+
     /* find this nspace object */
     nptr = NULL;
     PMIX_LIST_FOREACH(ns, &pmix_globals.nspaces, pmix_namespace_t) {
@@ -292,6 +355,7 @@
     }
     if (NULL == nptr) {
         /* nothing we can do */
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
         return;
     }
 
@@ -319,6 +383,7 @@
             }
         }
     }
+    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
 }
 
 static void cicbfunc(pmix_status_t status,
@@ -378,6 +443,11 @@
         }
         return;
     }
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        cbfunc(PMIX_SUCCESS, NULL, cbdata);
+        return;
+    }
+
     /* create the rollup object */
     myrollup = PMIX_NEW(pmix_inventory_rollup_t);
     if (NULL == myrollup) {
@@ -481,6 +551,11 @@
         }
         return;
     }
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+        return;
+    }
+
     /* create the rollup object */
     myrollup = PMIX_NEW(pmix_inventory_rollup_t);
     if (NULL == myrollup) {
@@ -533,6 +608,126 @@
     return;
 }
 
+pmix_status_t pmix_pnet_base_register_fabric(pmix_fabric_t *fabric,
+                                             const pmix_info_t directives[],
+                                             size_t ndirs,
+                                             pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_pnet_base_active_module_t *active;
+    pmix_status_t rc;
+    pmix_pnet_fabric_t *ft;
+
+    /* ensure our fields of the fabric object are initialized */
+    fabric->info = NULL;
+    fabric->ninfo = 0;
+    fabric->module = NULL;
+
+    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
+
+    if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) {
+        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* scan across active modules until one returns success */
+    PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) {
+        if (NULL != active->module->register_fabric) {
+            rc = active->module->register_fabric(fabric, directives, ndirs, cbfunc, cbdata);
+            if (PMIX_OPERATION_SUCCEEDED == rc) {
+                /* track this fabric so we can respond to remote requests */
+                ft = PMIX_NEW(pmix_pnet_fabric_t);
+                ft->index = fabric->index;
+                if (NULL != fabric->name) {
+                    ft->name = strdup(fabric->name);
+                }
+                ft->module = active->module;
+                pmix_list_append(&pmix_pnet_globals.fabrics, &ft->super);
+            } else if (PMIX_ERR_TAKE_NEXT_OPTION != rc) {
+                /* just return the result */
+                PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+                return rc;
+            }
+        }
+    }
+
+    /* unlock prior to return */
+    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
+
+    return PMIX_ERR_NOT_FOUND;
+}
+
+pmix_status_t pmix_pnet_base_update_fabric(pmix_fabric_t *fabric)
+{
+    pmix_status_t rc = PMIX_SUCCESS;
+    pmix_pnet_fabric_t *active;
+    pmix_pnet_module_t *module = NULL;
+    pmix_pnet_fabric_t *ft;
+
+    /* protect against bozo input */
+    if (NULL == fabric) {
+        return PMIX_ERR_BAD_PARAM;
+    } else if (NULL == fabric->module) {
+        /* this might be a remote request, so look at the
+         * list of fabrics we have registered locally and
+         * see if we have one with the matching index */
+        PMIX_LIST_FOREACH(ft, &pmix_pnet_globals.fabrics, pmix_pnet_fabric_t) {
+            if (fabric->index == ft->index) {
+                module = ft->module;
+            } else if (NULL != fabric->name && NULL != ft->name
+                       && 0 == strcmp(ft->name, fabric->name)) {
+                module = ft->module;
+            }
+        }
+    } else {
+        active = (pmix_pnet_fabric_t*)fabric->module;
+        module = (pmix_pnet_module_t*)active->module;
+    }
+    if (NULL == module) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    if (NULL != module->update_fabric) {
+        rc = module->update_fabric(fabric);
+    }
+    return rc;
+}
+
+pmix_status_t pmix_pnet_base_deregister_fabric(pmix_fabric_t *fabric)
+{
+    pmix_status_t rc = PMIX_SUCCESS;
+    pmix_pnet_fabric_t *active;
+    pmix_pnet_module_t *module = NULL;
+    pmix_pnet_fabric_t *ft;
+
+    /* protect against bozo input */
+    if (NULL == fabric) {
+        return PMIX_ERR_BAD_PARAM;
+    } else if (NULL == fabric->module) {
+        /* this might be a remote request, so look at the
+         * list of fabrics we have registered locally and
+         * see if we have one with the matching index */
+        PMIX_LIST_FOREACH(ft, &pmix_pnet_globals.fabrics, pmix_pnet_fabric_t) {
+            if (fabric->index == ft->index) {
+                module = ft->module;
+            } else if (NULL != fabric->name && NULL != ft->name
+                       && 0 == strcmp(ft->name, fabric->name)) {
+                module = ft->module;
+            }
+        }
+    } else {
+        active = (pmix_pnet_fabric_t*)fabric->module;
+        module = (pmix_pnet_module_t*)active->module;
+    }
+    if (NULL == module) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+
+    if (NULL != module->deregister_fabric) {
+        rc = module->deregister_fabric(fabric);
+    }
+    return rc;
+}
+
 
 static pmix_status_t process_maps(char *nspace, char **nodes, char **procs)
 {
@@ -544,13 +739,10 @@
     pmix_pnet_local_procs_t *lp;
     bool needcheck;
 
-    PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock);
-
     /* bozo check */
     if (pmix_argv_count(nodes) != pmix_argv_count(procs)) {
         rc = PMIX_ERR_BAD_PARAM;
         PMIX_ERROR_LOG(rc);
-        PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
         return rc;
     }
 
@@ -626,6 +818,5 @@
         pmix_argv_free(ranks);
     }
 
-    PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock);
     return PMIX_SUCCESS;
 }
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/base/pnet_base_frame.c pmix-4.0.0/src/mca/pnet/base/pnet_base_frame.c
--- pmix-3.2.2~rc1/src/mca/pnet/base/pnet_base_frame.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/base/pnet_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -53,7 +53,10 @@
     .local_app_finalized = pmix_pnet_base_local_app_finalized,
     .deregister_nspace = pmix_pnet_base_deregister_nspace,
     .collect_inventory = pmix_pnet_base_collect_inventory,
-    .deliver_inventory = pmix_pnet_base_deliver_inventory
+    .deliver_inventory = pmix_pnet_base_deliver_inventory,
+    .register_fabric = pmix_pnet_base_register_fabric,
+    .update_fabric = pmix_pnet_base_update_fabric,
+    .deregister_fabric = pmix_pnet_base_deregister_fabric
 };
 
 static pmix_status_t pmix_pnet_close(void)
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/pnet.h pmix-4.0.0/src/mca/pnet/pnet.h
--- pmix-3.2.2~rc1/src/mca/pnet/pnet.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/pnet.h	2021-01-02 08:56:17.000000000 +0000
@@ -146,6 +146,27 @@
                                                                       pmix_info_t directives[], size_t ndirs,
                                                                       pmix_op_cbfunc_t cbfunc, void *cbdata);
 
+
+/* Register to provide cost information
+ *
+ * Scan the provided directives to identify if the module
+ * should service this request - directives could include
+ * ordered specification of fabric type or a direct request
+ * for a specific component
+ */
+typedef pmix_status_t (*pmix_pnet_base_module_register_fabric_fn_t)(pmix_fabric_t *fabric,
+                                                                    const pmix_info_t directives[],
+                                                                    size_t ndirs,
+                                                                    pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+
+/* Update the fabric information */
+typedef pmix_status_t (*pmix_pnet_base_module_update_fabric_fn_t)(pmix_fabric_t *fabric);
+
+
+/* Deregister the fabric, giving the associated module a chance to cleanup */
+typedef pmix_status_t (*pmix_pnet_base_module_deregister_fabric_fn_t)(pmix_fabric_t *fabric);
+
 /**
  * Base structure for a PNET module. Each component should malloc a
  * copy of the module structure for each fabric plane they support.
@@ -165,6 +186,9 @@
     pmix_pnet_base_module_dregister_nspace_fn_t     deregister_nspace;
     pmix_pnet_base_module_collect_inventory_fn_t    collect_inventory;
     pmix_pnet_base_module_deliver_inventory_fn_t    deliver_inventory;
+    pmix_pnet_base_module_register_fabric_fn_t      register_fabric;
+    pmix_pnet_base_module_update_fabric_fn_t        update_fabric;
+    pmix_pnet_base_module_deregister_fabric_fn_t    deregister_fabric;
 } pmix_pnet_module_t;
 
 
@@ -204,6 +228,9 @@
     pmix_pnet_base_API_deregister_nspace_fn_t       deregister_nspace;
     pmix_pnet_base_API_collect_inventory_fn_t       collect_inventory;
     pmix_pnet_base_API_deliver_inventory_fn_t       deliver_inventory;
+    pmix_pnet_base_module_register_fabric_fn_t      register_fabric;
+    pmix_pnet_base_module_update_fabric_fn_t        update_fabric;
+    pmix_pnet_base_module_deregister_fabric_fn_t    deregister_fabric;
 } pmix_pnet_API_module_t;
 
 
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/simptest/configure.m4 pmix-4.0.0/src/mca/pnet/simptest/configure.m4
--- pmix-3.2.2~rc1/src/mca/pnet/simptest/configure.m4	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/simptest/configure.m4	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,26 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_pmix_pnet_simptest_CONFIG([action-if-can-compile],
+#                               [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_pmix_pnet_simptest_CONFIG], [
+    AC_CONFIG_FILES([src/mca/pnet/simptest/Makefile])
+
+    AC_ARG_WITH([simptest], [AC_HELP_STRING([--with-simptest], [Include simptest fabric support])],
+                [pmix_want_simptest=yes], [pmix_want_simptest=no])
+
+    AS_IF([test "$pmix_want_simptest" = "yes"],
+          [$1
+           PMIX_SUMMARY_ADD([[Transports]],[[Simptest]],[[pnet_simptest]],[$pmix_want_simptest])],
+          [$2])
+
+
+])dnl
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/simptest/Makefile.am pmix-4.0.0/src/mca/pnet/simptest/Makefile.am
--- pmix-3.2.2~rc1/src/mca/pnet/simptest/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/simptest/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,55 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = pnet_simptest.h
+sources = \
+        pnet_simptest_component.c \
+        pnet_simptest.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_pnet_simptest_DSO
+lib =
+lib_sources =
+component = mca_pnet_simptest.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_pnet_simptest.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_pnet_simptest_la_SOURCES = $(component_sources)
+mca_pnet_simptest_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_pnet_simptest_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_pnet_simptest_la_SOURCES = $(lib_sources)
+libmca_pnet_simptest_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/simptest/pnet_simptest.c pmix-4.0.0/src/mca/pnet/simptest/pnet_simptest.c
--- pmix-3.2.2~rc1/src/mca/pnet/simptest/pnet_simptest.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/simptest/pnet_simptest.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,444 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include <time.h>
+
+#include "include/pmix_common.h"
+
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/include/pmix_socket_errno.h"
+#include "src/include/pmix_globals.h"
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/util/alfg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/util/output.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/show_help.h"
+#include "src/mca/preg/preg.h"
+
+#include "src/mca/pnet/pnet.h"
+#include "src/mca/pnet/base/base.h"
+#include "pnet_simptest.h"
+
+static pmix_status_t simptest_init(void);
+static void simptest_finalize(void);
+static pmix_status_t allocate(pmix_namespace_t *nptr,
+                              pmix_info_t info[], size_t ninfo,
+                              pmix_list_t *ilist);
+static pmix_status_t setup_local_network(pmix_namespace_t *nptr,
+                                         pmix_info_t info[],
+                                         size_t ninfo);
+
+pmix_pnet_module_t pmix_simptest_module = {
+    .name = "simptest",
+    .init = simptest_init,
+    .finalize = simptest_finalize,
+    .allocate = allocate,
+    .setup_local_network = setup_local_network
+};
+
+/* internal tracking structures */
+typedef struct {
+    pmix_list_item_t super;
+    char *name;
+    pmix_geometry_t *devices;
+    size_t ndevices;
+    pmix_endpoint_t *endpts;
+    size_t nendpts;
+    pmix_device_distance_t *distances;
+    size_t ndists;
+} pnet_node_t;
+static void ndcon(pnet_node_t *p)
+{
+    p->name = NULL;
+    p->devices = NULL;
+    p->endpts = NULL;
+    p->distances = NULL;
+}
+static void nddes(pnet_node_t *p)
+{
+    if (NULL != p->name) {
+        free(p->name);
+    }
+    if (NULL != p->devices) {
+        PMIX_GEOMETRY_FREE(p->devices, p->ndevices);
+    }
+    if (NULL != p->endpts) {
+        PMIX_ENDPOINT_FREE(p->endpts, p->nendpts);
+    }
+    if (NULL != p->distances) {
+        PMIX_DEVICE_DIST_FREE(p->distances, p->ndists);
+    }
+}
+static PMIX_CLASS_INSTANCE(pnet_node_t,
+                           pmix_list_item_t,
+                           ndcon, nddes);
+
+/* internal variables */
+static pmix_list_t mynodes;
+
+#define PMIX_SIMPTEST_MAX_LINE_LENGTH 1024
+
+static char *localgetline(FILE *fp)
+{
+    char *ret, *buff;
+    char input[PMIX_SIMPTEST_MAX_LINE_LENGTH];
+    int i = 0;
+
+    ret = fgets(input, PMIX_SIMPTEST_MAX_LINE_LENGTH, fp);
+    if (NULL != ret) {
+        if ('\0' != input[0]) {
+            input[strlen(input)-1] = '\0';  /* remove newline */
+            /* strip any leading whitespace */
+             while (' ' == input[i] || '\t' == input[i]) {
+                i++;
+            }
+        }
+       buff = strdup(&input[i]);
+       return buff;
+    }
+
+    return NULL;
+}
+
+static pmix_status_t simptest_init(void)
+{
+    FILE *fp = NULL;
+    char *line, **tmp;
+    pnet_node_t *nd;
+    int n, cache[1024];
+
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet: simptest init");
+
+    PMIX_CONSTRUCT(&mynodes, pmix_list_t);
+
+    /* if the configuration was given in a file, then build
+     * the topology so we can respond to requests */
+    if (NULL == mca_pnet_simptest_component.configfile) {
+        /* we cannot function */
+        return PMIX_ERR_INIT;
+    }
+
+    fp = fopen(mca_pnet_simptest_component.configfile, "r");
+    if (NULL == fp) {
+        pmix_show_help("help-pnet-simptest.txt", "missing-file",
+                       true, mca_pnet_simptest_component.configfile);
+        return PMIX_ERR_FATAL;
+    }
+    while (NULL != (line = localgetline(fp))) {
+        /* if the line starts with a '#' or is blank, then
+         * it is a comment and we ignore it */
+        if (0 == strlen(line) || '#' == line[0]) {
+            free(line);
+            continue;
+        }
+        tmp = pmix_argv_split(line, ' ');
+        nd = PMIX_NEW(pnet_node_t);
+        nd->name = strdup(tmp[0]);
+        pmix_list_append(&mynodes, &nd->super);
+        n = 0;
+        while (n < 1024 && NULL != tmp[n+1]) {
+            cache[n] = strtol(tmp[n+1], NULL, 10);
+            ++n;
+        }
+
+        nd->coord.dims = n;
+        nd->coord.coord = (int*)malloc(nd->coord.dims * sizeof(int));
+        memcpy(nd->coord.coord, cache, nd->coord.dims * sizeof(int));
+        free(line);
+        pmix_argv_free(tmp);
+    }
+
+    if (NULL != fp) {
+        fclose(fp);
+    }
+    return PMIX_SUCCESS;
+}
+
+static void simptest_finalize(void)
+{
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet: simptest finalize");
+
+    PMIX_LIST_DESTRUCT(&mynodes);
+}
+
+
+/* NOTE: if there is any binary data to be transferred, then
+ * this function MUST pack it for transport as the host will
+ * not know how to do so */
+static pmix_status_t allocate(pmix_namespace_t *nptr,
+                              pmix_info_t info[], size_t ninfo,
+                              pmix_list_t *ilist)
+{
+    size_t m, n, q;
+    char **procs = NULL;
+    char **nodes = NULL;
+    pmix_status_t rc;
+    pmix_list_t mylist;
+    char **locals;
+    pnet_node_t *nd, *nd2;
+    pmix_kval_t *kv;
+    pmix_info_t *iptr, *ip2;
+    pmix_data_array_t *darray, *d2, *d3;
+    pmix_rank_t rank;
+    pmix_buffer_t buf;
+    pmix_byte_object_t *bptr;
+
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet:simptest:allocate for nspace %s",
+                        nptr->nspace);
+
+   /* if I am not the scheduler, then ignore this call - should never
+     * happen, but check to be safe */
+    if (!PMIX_PEER_IS_SCHEDULER(pmix_globals.mypeer)) {
+        return PMIX_SUCCESS;
+    }
+
+    if (NULL == info) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+
+    /* check directives to see if a crypto key and/or
+     * fabric resource allocations requested */
+    for (n=0; n < ninfo; n++) {
+        pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                            "pnet:simptest:allocate processing key %s",
+                            info[n].key);
+        if (PMIX_CHECK_KEY(&info[n], PMIX_PROC_MAP)) {
+            rc = pmix_preg.parse_procs(info[n].value.data.string, &procs);
+            if (PMIX_SUCCESS != rc) {
+                return PMIX_ERR_BAD_PARAM;
+            }
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_NODE_MAP)) {
+            rc = pmix_preg.parse_nodes(info[n].value.data.string, &nodes);
+            if (PMIX_SUCCESS != rc) {
+                return PMIX_ERR_BAD_PARAM;
+            }
+        }
+    }
+
+    if (NULL == procs || NULL == nodes) {
+        pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                            "pnet:simptest:allocate missing proc/node map for nspace %s",
+                            nptr->nspace);
+        /* not an error - continue to next active component */
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    PMIX_CONSTRUCT(&mylist, pmix_list_t);
+
+
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet:simptest:allocate assigning endpoints for nspace %s",
+                        nptr->nspace);
+
+    /* cycle across the nodes and add the endpoints
+     * for each proc on the node - we assume the same
+     * list of static endpoints on each node */
+    for (n=0; NULL != nodes[n]; n++) {
+        /* split the procs for this node */
+        locals = pmix_argv_split(procs[n], ',');
+        if (NULL == locals) {
+            /* aren't any on this node */
+            continue;
+        }
+        /* find this node in our list */
+        nd = NULL;
+        PMIX_LIST_FOREACH(nd2, &mynodes, pnet_node_t) {
+            if (0 == strcmp(nd2->name, nodes[n])) {
+                nd = nd2;
+                break;
+            }
+        }
+        if (NULL == nd) {
+            /* should be impossible */
+            rc = PMIX_ERR_NOT_FOUND;
+            PMIX_ERROR_LOG(rc);
+            goto cleanup;
+        }
+        kv = PMIX_NEW(pmix_kval_t);
+        if (NULL == kv) {
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        kv->key = strdup(PMIX_ALLOC_FABRIC_ENDPTS);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        if (NULL == kv->value) {
+            PMIX_RELEASE(kv);
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        kv->value->type = PMIX_DATA_ARRAY;
+        /* for each proc, we will assign an endpt
+         * for each NIC on the node */
+        q = pmix_argv_count(locals);
+        PMIX_DATA_ARRAY_CREATE(darray, q, PMIX_INFO);
+        kv->value->data.darray = darray;
+        iptr = (pmix_info_t*)darray->array;
+        for (m=0; NULL != locals[m]; m++) {
+            /* each proc has one endpoint and one coord corresponding to the
+             * node they upon which they are executing */
+            PMIX_LOAD_KEY(iptr[m].key, PMIX_PROC_DATA);
+            PMIX_DATA_ARRAY_CREATE(d2, 3, PMIX_INFO);
+            iptr[m].value.type = PMIX_DATA_ARRAY;
+            iptr[m].value.data.darray = d2;
+            ip2 = (pmix_info_t*)d2->array;
+            /* start with the rank */
+            rank = strtoul(locals[m], NULL, 10);
+            pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                                "pnet:simptest:allocate assigning %d endpoints for rank %u",
+                                (int)q, rank);
+            PMIX_INFO_LOAD(&ip2[0], PMIX_RANK, &rank, PMIX_PROC_RANK);
+            /* the second element in this array is the coord */
+            PMIX_INFO_LOAD(&ip2[1], PMIX_FABRIC_COORDINATE, &nd->coord, PMIX_COORD);
+            /* third element is the endpt */
+            PMIX_DATA_ARRAY_CREATE(d3, 1, PMIX_BYTE_OBJECT);
+            PMIX_LOAD_KEY(ip2[2].key, PMIX_FABRIC_ENDPT);
+            ip2[2].value.type = PMIX_DATA_ARRAY;
+            ip2[2].value.data.darray = d3;
+            bptr = (pmix_byte_object_t*)d3->array;
+            bptr[0].bytes = strdup(nd->endpt.bytes);
+            bptr[0].size = nd->endpt.size;
+        }
+        pmix_argv_free(locals);
+        locals = NULL;
+        pmix_list_append(&mylist, &kv->super);
+    }
+
+    /* pack all our results into a buffer for xmission to the backend */
+    n = pmix_list_get_size(&mylist);
+    if (0 < n) {
+        PMIX_CONSTRUCT(&buf, pmix_buffer_t);
+        /* cycle across the list and pack the kvals */
+        while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(&mylist))) {
+            PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, kv, 1, PMIX_KVAL);
+            PMIX_RELEASE(kv);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_DESTRUCT(&buf);
+                goto cleanup;
+            }
+        }
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup("pmix-pnet-simptest-blob");
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        if (NULL == kv->value) {
+            PMIX_RELEASE(kv);
+            PMIX_DESTRUCT(&buf);
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        kv->value->type = PMIX_BYTE_OBJECT;
+        PMIX_UNLOAD_BUFFER(&buf, kv->value->data.bo.bytes, kv->value->data.bo.size);
+        PMIX_DESTRUCT(&buf);
+        pmix_list_append(ilist, &kv->super);
+    }
+
+  cleanup:
+    PMIX_LIST_DESTRUCT(&mylist);
+    if (NULL != nodes) {
+        pmix_argv_free(nodes);
+    }
+    if (NULL != procs) {
+        pmix_argv_free(procs);
+    }
+    if (NULL != locals) {
+        pmix_argv_free(locals);
+    }
+    return rc;
+}
+
+static pmix_status_t setup_local_network(pmix_namespace_t *nptr,
+                                         pmix_info_t info[],
+                                         size_t ninfo)
+{
+    size_t n, nvals;
+    pmix_buffer_t bkt;
+    int32_t cnt;
+    pmix_kval_t *kv;
+    pmix_status_t rc;
+    pmix_info_t *iptr;
+
+    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                        "pnet:simptest:setup_local_network with %lu info", (unsigned long)ninfo);
+
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            /* look for my key */
+            if (PMIX_CHECK_KEY(&info[n], "pmix-pnet-simptest-blob")) {
+                pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                                    "pnet:simptest:setup_local_network found my blob");
+                /* this macro NULLs and zero's the incoming bo */
+                PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt,
+                                 info[n].value.data.bo.bytes,
+                                 info[n].value.data.bo.size);
+                /* cycle thru the blob and extract the kvals */
+                kv = PMIX_NEW(pmix_kval_t);
+                cnt = 1;
+                PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
+                                   &bkt, kv, &cnt, PMIX_KVAL);
+                while (PMIX_SUCCESS == rc) {
+                    pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                                        "recvd KEY %s %s", kv->key, PMIx_Data_type_string(kv->value->type));
+                    /* check for the fabric ID */
+                    if (PMIX_CHECK_KEY(kv, PMIX_ALLOC_FABRIC_ENDPTS)) {
+                        iptr = (pmix_info_t*)kv->value->data.darray->array;
+                        nvals = kv->value->data.darray->size;
+                        /* each element in this array is itself an array containing
+                         * the rank and the endpts and coords assigned to that rank. This is
+                         * precisely the data we need to cache for the job, so
+                         * just do so) */
+                        pmix_output_verbose(2, pmix_pnet_base_framework.framework_output,
+                                            "pnet:simptest:setup_local_network caching %d endpts", (int)nvals);
+                        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, iptr, nvals);
+                        if (PMIX_SUCCESS != rc) {
+                            PMIX_RELEASE(kv);
+                            return rc;
+                        }
+                    }
+                    PMIX_RELEASE(kv);
+                    kv = PMIX_NEW(pmix_kval_t);
+                    cnt = 1;
+                    PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer,
+                                       &bkt, kv, &cnt, PMIX_KVAL);
+                }
+                PMIX_RELEASE(kv);
+                /* restore the incoming data */
+                info[n].value.data.bo.bytes = bkt.base_ptr;
+                info[n].value.data.bo.size = bkt.bytes_used;
+                bkt.base_ptr = NULL;
+                bkt.bytes_used = 0;
+            }
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/simptest/pnet_simptest_component.c pmix-4.0.0/src/mca/pnet/simptest/pnet_simptest_component.c
--- pmix-3.2.2~rc1/src/mca/pnet/simptest/pnet_simptest_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/simptest/pnet_simptest_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,122 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/util/argv.h"
+#include "src/mca/pnet/pnet.h"
+#include "pnet_simptest.h"
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_close(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+static pmix_status_t component_register(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_pnet_simptest_component_t mca_pnet_simptest_component = {
+    .super = {
+        .base = {
+            PMIX_PNET_BASE_VERSION_1_0_0,
+
+            /* Component name and version */
+            .pmix_mca_component_name = "simptest",
+            PMIX_MCA_BASE_MAKE_VERSION(component,
+                                       PMIX_MAJOR_VERSION,
+                                       PMIX_MINOR_VERSION,
+                                       PMIX_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .pmix_mca_open_component = component_open,
+            .pmix_mca_close_component = component_close,
+            .pmix_mca_query_component = component_query,
+            .pmix_mca_register_component_params = component_register
+        },
+        .data = {
+            /* The component is checkpoint ready */
+            PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+        }
+    },
+    .configfile = NULL
+};
+
+static pmix_status_t component_register(void)
+{
+    pmix_mca_base_component_t *component = &mca_pnet_simptest_component.super.base;
+
+    (void)pmix_mca_base_component_var_register(component, "config_file",
+                                               "Path of file containing network coordinate configuration",
+                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                               PMIX_INFO_LVL_2,
+                                               PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                               &mca_pnet_simptest_component.configfile);
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t component_open(void)
+{
+    int index;
+    const pmix_mca_base_var_storage_t *value=NULL;
+
+    if (NULL == mca_pnet_simptest_component.configfile ||
+        !PMIX_PROC_IS_SERVER(&pmix_globals.mypeer->proc_type)) {
+        /* nothing we can do without a description
+         * of the fabric topology */
+        return PMIX_ERROR;
+    }
+
+    /* we only allow ourselves to be considered IF the user
+     * specifically requested so */
+    if (0 > (index = pmix_mca_base_var_find("pmix", "pnet", NULL, NULL))) {
+        return PMIX_ERROR;
+    }
+    pmix_mca_base_var_get_value(index, &value, NULL, NULL);
+    if (NULL != value && NULL != value->stringval && '\0' != value->stringval[0]) {
+        if (NULL != strcasestr(value->stringval, "simptest")) {
+            return PMIX_SUCCESS;
+        }
+    }
+    return PMIX_ERROR;
+}
+
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 0;
+    *module = (pmix_mca_base_module_t *)&pmix_simptest_module;
+    return PMIX_SUCCESS;
+}
+
+
+static pmix_status_t component_close(void)
+{
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/pnet/simptest/pnet_simptest.h pmix-4.0.0/src/mca/pnet/simptest/pnet_simptest.h
--- pmix-3.2.2~rc1/src/mca/pnet/simptest/pnet_simptest.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/pnet/simptest/pnet_simptest.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PNET_simptest_H
+#define PMIX_PNET_simptest_H
+
+#include "src/include/pmix_config.h"
+
+
+#include "src/mca/pnet/pnet.h"
+
+BEGIN_C_DECLS
+
+typedef struct {
+    pmix_pnet_base_component_t super;
+    char *configfile;
+} pmix_pnet_simptest_component_t;
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_pnet_simptest_component_t mca_pnet_simptest_component;
+extern pmix_pnet_module_t pmix_simptest_module;
+
+/* define a key for any blob we need to send in a launch msg */
+#define PMIX_PNET_SIMPTEST_BLOB  "pmix.pnet.simptest.blob"
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/preg/compress/preg_compress.c pmix-4.0.0/src/mca/preg/compress/preg_compress.c
--- pmix-3.2.2~rc1/src/mca/preg/compress/preg_compress.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/preg/compress/preg_compress.c	2021-01-02 08:56:17.000000000 +0000
@@ -133,9 +133,6 @@
         return PMIX_ERR_NOMEM;
     }
 
-    if (NULL == tmp) {
-        return PMIX_ERR_NOMEM;
-    }
     rc = pack_blob(tmp, len, regexp);
     free(tmp);
 
diff -Nru pmix-3.2.2~rc1/src/mca/preg/raw/Makefile.am pmix-4.0.0/src/mca/preg/raw/Makefile.am
--- pmix-3.2.2~rc1/src/mca/preg/raw/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/preg/raw/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = preg_raw.h
+sources = \
+        preg_raw_component.c \
+        preg_raw.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_preg_raw_DSO
+lib =
+lib_sources =
+component = mca_preg_raw.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_preg_raw.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_preg_raw_la_SOURCES = $(component_sources)
+mca_preg_raw_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_preg_raw_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_preg_raw_la_SOURCES = $(lib_sources)
+libmca_preg_raw_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/preg/raw/preg_raw.c pmix-4.0.0/src/mca/preg/raw/preg_raw.c
--- pmix-3.2.2~rc1/src/mca/preg/raw/preg_raw.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/preg/raw/preg_raw.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016-2019 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+#include <fcntl.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#include <ctype.h>
+
+
+#include "include/pmix_common.h"
+#include "include/pmix.h"
+
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/printf.h"
+#include "src/mca/bfrops/base/base.h"
+
+#include "src/mca/preg/base/base.h"
+#include "preg_raw.h"
+
+static pmix_status_t generate_node_regex(const char *input,
+                                         char **regex);
+static pmix_status_t generate_ppn(const char *input,
+                                  char **ppn);
+static pmix_status_t parse_nodes(const char *regexp,
+                                 char ***names);
+static pmix_status_t parse_procs(const char *regexp,
+                                 char ***procs);
+static pmix_status_t copy(char **dest, size_t *len, const char *input);
+static pmix_status_t pack(pmix_buffer_t *buffer, const char *input);
+static pmix_status_t unpack(pmix_buffer_t *buffer, char **regex);
+
+pmix_preg_module_t pmix_preg_raw_module = {
+    .name = "raw",
+    .generate_node_regex = generate_node_regex,
+    .generate_ppn = generate_ppn,
+    .parse_nodes = parse_nodes,
+    .parse_procs = parse_procs,
+    .copy = copy,
+    .pack = pack,
+    .unpack = unpack
+};
+
+static pmix_status_t generate_node_regex(const char *input,
+                                         char **regexp)
+{
+    if (0 == strncmp(input, "raw:", 4)) {
+        *regexp = strdup(input);
+    } else {
+        pmix_asprintf(regexp, "raw:%s", input);
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t generate_ppn(const char *input,
+                                  char **regexp)
+{
+    if (0 == strncmp(input, "raw:", 4)) {
+        *regexp = strdup(input);
+    } else {
+        pmix_asprintf(regexp, "raw:%s", input);
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t parse_nodes(const char *regexp,
+                                 char ***names)
+{
+    if (0 != strncmp(regexp, "raw:", 4)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    *names = pmix_argv_split(&regexp[4], ',');
+    return PMIX_SUCCESS;
+
+}
+static pmix_status_t parse_procs(const char *regexp,
+                                 char ***procs)
+{
+    if (0 != strncmp(regexp, "raw:", 4)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    *procs = pmix_argv_split(&regexp[4], ';');
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t copy(char **dest, size_t *len, const char *input)
+{
+    if (0 != strncmp(input, "raw:", 4)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    *dest = strdup(input);
+    *len = strlen(input) + 1;
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t pack(pmix_buffer_t *buffer, const char *input)
+{
+    size_t slen;
+    char *ptr;
+
+    if (0 != strncmp(input, "raw:", 4)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    /* extract the size */
+    slen = strlen(input) + 1;  // retain the NULL terminator
+
+    /* ensure the buffer has enough space */
+    ptr = pmix_bfrop_buffer_extend(buffer, slen);
+    if (NULL == ptr) {
+        return PMIX_ERR_NOMEM;
+    }
+
+    /* xfer the data */
+    memcpy(ptr, input, slen);
+    buffer->bytes_used += slen;
+    buffer->pack_ptr += slen;
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t unpack(pmix_buffer_t *buffer, char **regex)
+{
+    char *ptr;
+
+    ptr = buffer->unpack_ptr;
+
+    if (0 != strncmp(ptr, "raw:", 4)) {
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    *regex = strdup(ptr);
+    buffer->unpack_ptr += strlen(ptr) + 1;
+
+    if (NULL == *regex) {
+        return PMIX_ERR_NOMEM;
+    }
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/preg/raw/preg_raw_component.c pmix-4.0.0/src/mca/preg/raw/preg_raw_component.c
--- pmix-3.2.2~rc1/src/mca/preg/raw/preg_raw_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/preg/raw/preg_raw_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+
+#include "src/mca/preg/preg.h"
+#include "preg_raw.h"
+
+static pmix_status_t component_open(void);
+static pmix_status_t component_close(void);
+static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_mca_base_component_t mca_preg_raw_component = {
+    PMIX_PREG_BASE_VERSION_1_0_0,
+
+    /* Component name and version */
+    .pmix_mca_component_name = "raw",
+    PMIX_MCA_BASE_MAKE_VERSION(component,
+                               PMIX_MAJOR_VERSION,
+                               PMIX_MINOR_VERSION,
+                               PMIX_RELEASE_VERSION),
+
+    /* Component open and close functions */
+    .pmix_mca_open_component = component_open,
+    .pmix_mca_close_component = component_close,
+    .pmix_mca_query_component = component_query,
+};
+
+
+static int component_open(void)
+{
+    return PMIX_SUCCESS;
+}
+
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    *priority = 40;
+    *module = (pmix_mca_base_module_t *)&pmix_preg_raw_module;
+    return PMIX_SUCCESS;
+}
+
+
+static int component_close(void)
+{
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/preg/raw/preg_raw.h pmix-4.0.0/src/mca/preg/raw/preg_raw.h
--- pmix-3.2.2~rc1/src/mca/preg/raw/preg_raw.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/preg/raw/preg_raw.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PREG_RAW_H
+#define PMIX_PREG_RAW_H
+
+#include "src/include/pmix_config.h"
+
+
+#include "src/mca/preg/preg.h"
+
+BEGIN_C_DECLS
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_mca_base_component_t mca_preg_raw_component;
+extern pmix_preg_module_t pmix_preg_raw_module;
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/prm/base/base.h pmix-4.0.0/src/mca/prm/base/base.h
--- pmix-3.2.2~rc1/src/mca/prm/base/base.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,103 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PMIX_PRM_BASE_H_
+#define PMIX_PRM_BASE_H_
+
+#include "src/include/pmix_config.h"
+
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h> /* for struct timeval */
+#endif
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/class/pmix_pointer_array.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+
+#include "src/mca/prm/prm.h"
+
+
+BEGIN_C_DECLS
+
+/*
+ * MCA Framework
+ */
+PMIX_EXPORT extern pmix_mca_base_framework_t pmix_prm_base_framework;
+/**
+ * PRM select function
+ *
+ * Cycle across available components and construct the list
+ * of active modules
+ */
+PMIX_EXPORT pmix_status_t pmix_prm_base_select(void);
+
+/**
+ * Track an active component / module
+ */
+struct pmix_prm_base_active_module_t {
+    pmix_list_item_t super;
+    int pri;
+    pmix_prm_module_t *module;
+    pmix_prm_base_component_t *component;
+};
+typedef struct pmix_prm_base_active_module_t pmix_prm_base_active_module_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_prm_base_active_module_t);
+
+/* define an object for rolling up operations */
+typedef struct {
+    pmix_object_t super;
+    pmix_lock_t lock;
+    pmix_event_t ev;
+    pmix_status_t status;
+    int requests;
+    int replies;
+    pmix_op_cbfunc_t cbfunc;
+    void *cbdata;
+} pmix_prm_rollup_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_prm_rollup_t);
+
+/* framework globals */
+struct pmix_prm_globals_t {
+    pmix_lock_t lock;
+    pmix_list_t actives;
+    bool initialized;
+    bool selected;
+};
+typedef struct pmix_prm_globals_t pmix_prm_globals_t;
+
+PMIX_EXPORT extern pmix_prm_globals_t pmix_prm_globals;
+
+PMIX_EXPORT pmix_status_t pmix_prm_base_notify(pmix_status_t status,
+                                               const pmix_proc_t *source,
+                                               pmix_data_range_t range,
+                                               const pmix_info_t info[], size_t ninfo,
+                                               pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/prm/base/help-prm.txt pmix-4.0.0/src/mca/prm/base/help-prm.txt
--- pmix-3.2.2~rc1/src/mca/prm/base/help-prm.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/base/help-prm.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,56 @@
+# -*- text -*-
+#
+# Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is a US/English help file
+#
+[reqd-not-found]
+The plog_base_order MCA parameter included a required logging
+channel that is not available:
+
+  Channel:  %s
+
+Please update the parameter and try again.
+#
+[syslog:unrec-level]
+An unrecognized syslog level was given:
+
+  Level:  %s
+
+Please see "man syslog" for a list of defined levels. Input
+parameter strings and their corresponding syslog levels
+recognized by PMIx include:
+
+  Parameter       Level
+   err           LOG_ERR  (default)
+   alert         LOG_ALERT
+   crit          LOG_CRIT
+   emerg         LOG_EMERG
+   warn          LOG_WARNING
+   not           LOG_NOTICE
+   info          LOG_INFO
+   debug         LOG_DEBUG
+
+Please redefine the MCA parameter and try again.
+#
+[syslog:unrec-facility]
+An unsupported or unrecognized value was given for the
+syslog facility (i.e., the type of program calling syslog):
+
+  Value:  %s
+
+Please see "man syslog" for a list of defined facility values.
+PMIx currently supports only the following designations:
+
+   Parameter       Level
+    auth          LOG_AUTH
+    priv          LOG_AUTHPRIV
+    daemon        LOG_DAEMON
+    user          LOG_USER  (default)
+
+Please redefine the MCA parameter and try again.
diff -Nru pmix-3.2.2~rc1/src/mca/prm/base/Makefile.include pmix-4.0.0/src/mca/prm/base/Makefile.include
--- pmix-3.2.2~rc1/src/mca/prm/base/Makefile.include	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/base/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This makefile.am does not stand on its own - it is included from
+# src/Makefile.am
+
+headers += \
+         base/base.h
+
+sources += \
+        base/prm_base_frame.c \
+        base/prm_base_select.c \
+        base/prm_base_stubs.c
+
+dist_pmixdata_DATA = base/help-prm.txt
diff -Nru pmix-3.2.2~rc1/src/mca/prm/base/prm_base_frame.c pmix-4.0.0/src/mca/prm/base/prm_base_frame.c
--- pmix-3.2.2~rc1/src/mca/prm/base/prm_base_frame.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/base/prm_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,111 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2009 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+/** @file:
+ *
+ */
+#include "src/include/pmix_config.h"
+
+#include "include/pmix_common.h"
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "src/class/pmix_list.h"
+#include "src/mca/base/base.h"
+#include "src/mca/prm/base/base.h"
+
+/*
+ * The following file was created by configure.  It contains extern
+ * statements and the definition of an array of pointers to each
+ * component's public mca_base_component_t struct.
+ */
+
+#include "src/mca/prm/base/static-components.h"
+
+/* Instantiate the global vars */
+pmix_prm_globals_t pmix_prm_globals = {{0}};
+pmix_prm_API_module_t pmix_prm = {
+    .notify = pmix_prm_base_notify
+};
+
+static pmix_status_t pmix_prm_close(void)
+{
+  pmix_prm_base_active_module_t *active, *prev;
+
+    if (!pmix_prm_globals.initialized) {
+        return PMIX_SUCCESS;
+    }
+    pmix_prm_globals.initialized = false;
+    pmix_prm_globals.selected = false;
+
+    PMIX_LIST_FOREACH_SAFE(active, prev, &pmix_prm_globals.actives, pmix_prm_base_active_module_t) {
+      pmix_list_remove_item(&pmix_prm_globals.actives, &active->super);
+      if (NULL != active->module->finalize) {
+        active->module->finalize();
+      }
+      PMIX_RELEASE(active);
+    }
+    PMIX_DESTRUCT(&pmix_prm_globals.actives);
+
+    PMIX_DESTRUCT_LOCK(&pmix_prm_globals.lock);
+    return pmix_mca_base_framework_components_close(&pmix_prm_base_framework, NULL);
+}
+
+static pmix_status_t pmix_prm_open(pmix_mca_base_open_flag_t flags)
+{
+    /* initialize globals */
+    pmix_prm_globals.initialized = true;
+    PMIX_CONSTRUCT_LOCK(&pmix_prm_globals.lock);
+    pmix_prm_globals.lock.active = false;
+    PMIX_CONSTRUCT(&pmix_prm_globals.actives, pmix_list_t);
+
+    /* Open up all available components */
+    return pmix_mca_base_framework_components_open(&pmix_prm_base_framework, flags);
+}
+
+PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, prm, "PMIx Network Operations",
+                                NULL, pmix_prm_open, pmix_prm_close,
+                                mca_prm_base_static_components, 0);
+
+PMIX_CLASS_INSTANCE(pmix_prm_base_active_module_t,
+                    pmix_list_item_t,
+                    NULL, NULL);
+
+static void rlcon(pmix_prm_rollup_t *p)
+{
+    PMIX_CONSTRUCT_LOCK(&p->lock);
+    p->lock.active = false;
+    p->status = PMIX_SUCCESS;
+    p->requests = 0;
+    p->replies = 0;
+    p->cbfunc = NULL;
+    p->cbdata = NULL;
+}
+static void rldes(pmix_prm_rollup_t *p)
+{
+    PMIX_DESTRUCT_LOCK(&p->lock);
+}
+PMIX_CLASS_INSTANCE(pmix_prm_rollup_t,
+                    pmix_object_t,
+                    rlcon, rldes);
diff -Nru pmix-3.2.2~rc1/src/mca/prm/base/prm_base_select.c pmix-4.0.0/src/mca/prm/base/prm_base_select.c
--- pmix-3.2.2~rc1/src/mca/prm/base/prm_base_select.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/base/prm_base_select.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <string.h>
+
+#include "src/mca/mca.h"
+#include "src/mca/base/base.h"
+
+#include "src/mca/prm/base/base.h"
+
+/* Function for selecting a prioritized list of components
+ * from all those that are available. */
+int pmix_prm_base_select(void)
+{
+    pmix_mca_base_component_list_item_t *cli = NULL;
+    pmix_mca_base_component_t *component = NULL;
+    pmix_mca_base_module_t *module = NULL;
+    pmix_prm_module_t *nmodule;
+    pmix_prm_base_active_module_t *newmodule, *mod;
+    int rc, priority;
+    bool inserted;
+
+    if (pmix_prm_globals.selected) {
+        /* ensure we don't do this twice */
+        return PMIX_SUCCESS;
+    }
+    pmix_prm_globals.selected = true;
+
+    /* Query all available components and ask if they have a module */
+    PMIX_LIST_FOREACH(cli, &pmix_prm_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
+        component = (pmix_mca_base_component_t *) cli->cli_component;
+
+        pmix_output_verbose(5, pmix_prm_base_framework.framework_output,
+                            "mca:prm:select: checking available component %s", component->pmix_mca_component_name);
+
+        /* If there's no query function, skip it */
+        if (NULL == component->pmix_mca_query_component) {
+            pmix_output_verbose(5, pmix_prm_base_framework.framework_output,
+                                "mca:prm:select: Skipping component [%s]. It does not implement a query function",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* Query the component */
+        pmix_output_verbose(5, pmix_prm_base_framework.framework_output,
+                            "mca:prm:select: Querying component [%s]",
+                            component->pmix_mca_component_name);
+        rc = component->pmix_mca_query_component(&module, &priority);
+
+        /* If no module was returned, then skip component */
+        if (PMIX_SUCCESS != rc || NULL == module) {
+            pmix_output_verbose(5, pmix_prm_base_framework.framework_output,
+                                "mca:prm:select: Skipping component [%s]. Query failed to return a module",
+                                component->pmix_mca_component_name );
+            continue;
+        }
+
+        /* If we got a module, keep it */
+        nmodule = (pmix_prm_module_t*) module;
+        /* let it initialize */
+        if (NULL != nmodule->init && PMIX_SUCCESS != nmodule->init()) {
+            continue;
+        }
+        /* add to the list of selected modules */
+        newmodule = PMIX_NEW(pmix_prm_base_active_module_t);
+        newmodule->pri = priority;
+        newmodule->module = nmodule;
+        newmodule->component = (pmix_prm_base_component_t*)cli->cli_component;
+
+        /* maintain priority order */
+        inserted = false;
+        PMIX_LIST_FOREACH(mod, &pmix_prm_globals.actives, pmix_prm_base_active_module_t) {
+            if (priority > mod->pri) {
+                pmix_list_insert_pos(&pmix_prm_globals.actives,
+                                     (pmix_list_item_t*)mod, &newmodule->super);
+                inserted = true;
+                break;
+            }
+        }
+        if (!inserted) {
+            /* must be lowest priority - add to end */
+            pmix_list_append(&pmix_prm_globals.actives, &newmodule->super);
+        }
+    }
+
+    if (4 < pmix_output_get_verbosity(pmix_prm_base_framework.framework_output)) {
+        pmix_output(0, "Final prm priorities");
+        /* show the prioritized list */
+        PMIX_LIST_FOREACH(mod, &pmix_prm_globals.actives, pmix_prm_base_active_module_t) {
+            pmix_output(0, "\tprm: %s Priority: %d", mod->component->base.pmix_mca_component_name, mod->pri);
+        }
+    }
+
+    return PMIX_SUCCESS;;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/prm/base/prm_base_stubs.c pmix-4.0.0/src/mca/prm/base/prm_base_stubs.c
--- pmix-3.2.2~rc1/src/mca/prm/base/prm_base_stubs.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/base/prm_base_stubs.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,144 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_UTSNAME_H
+#include <sys/utsname.h>
+#endif
+#include <time.h>
+
+#include "include/pmix_common.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/preg/preg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/pmix_environ.h"
+#include "src/server/pmix_server_ops.h"
+
+#include "src/mca/prm/base/base.h"
+
+static void cicbfunc(pmix_status_t status,
+                     void *cbdata)
+{
+    pmix_prm_rollup_t *rollup = (pmix_prm_rollup_t*)cbdata;
+
+    PMIX_ACQUIRE_THREAD(&rollup->lock);
+    /* check if they had an error */
+    if (PMIX_SUCCESS != status && PMIX_SUCCESS == rollup->status) {
+        rollup->status = status;
+    }
+    /* record that we got a reply */
+    rollup->replies++;
+    /* see if all have replied */
+    if (rollup->replies < rollup->requests) {
+        /* nope - need to wait */
+        PMIX_RELEASE_THREAD(&rollup->lock);
+        return;
+    }
+
+    /* if we get here, then operation is complete */
+    PMIX_RELEASE_THREAD(&rollup->lock);
+    if (NULL != rollup->cbfunc) {
+        rollup->cbfunc(rollup->status, rollup->cbdata);
+    }
+    PMIX_RELEASE(rollup);
+    return;
+}
+
+pmix_status_t pmix_prm_base_notify(pmix_status_t status,
+                                   const pmix_proc_t *source,
+                                   pmix_data_range_t range,
+                                   const pmix_info_t info[], size_t ninfo,
+                                   pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    pmix_prm_base_active_module_t *active;
+    pmix_prm_rollup_t *myrollup;
+    pmix_status_t rc;
+
+    /* we cannot block here as each plugin could take some time to
+     * complete the request. So instead, we call each active plugin
+     * and get their immediate response - if "in progress", then
+     * we record that we have to wait for their answer before providing
+     * the caller with a response. If "error", then we know we
+     * won't be getting a response from them */
+
+    if (!pmix_prm_globals.initialized) {
+        return PMIX_ERR_INIT;
+    }
+
+    /* create the rollup object */
+    myrollup = PMIX_NEW(pmix_prm_rollup_t);
+    if (NULL == myrollup) {
+        return PMIX_ERR_NOMEM;
+    }
+    myrollup->cbfunc = cbfunc;
+    myrollup->cbdata = cbdata;
+
+    /* hold the lock until all active modules have been called
+     * to avoid race condition where replies come in before
+     * the requests counter has been fully updated */
+    PMIX_ACQUIRE_THREAD(&myrollup->lock);
+
+    PMIX_LIST_FOREACH(active, &pmix_prm_globals.actives, pmix_prm_base_active_module_t) {
+        if (NULL != active->module->notify) {
+            pmix_output_verbose(5, pmix_prm_base_framework.framework_output,
+                                "NOTIFYING %s", active->module->name);
+            rc = active->module->notify(status, source, range, info, ninfo, cicbfunc, (void*)myrollup);
+            /* if they return succeeded, then nothing
+             * to wait for here */
+            if (PMIX_OPERATION_IN_PROGRESS == rc) {
+                myrollup->requests++;
+            } else if (PMIX_OPERATION_SUCCEEDED == rc ||
+                       PMIX_ERR_TAKE_NEXT_OPTION == rc ||
+                       PMIX_ERR_NOT_SUPPORTED == rc) {
+                continue;
+            } else {
+                /* a true error - we need to wait for
+                 * all pending requests to complete
+                 * and then notify the caller of the error */
+                if (PMIX_SUCCESS == myrollup->status) {
+                    myrollup->status = rc;
+                }
+            }
+        }
+    }
+    if (0 == myrollup->requests) {
+        /* report back */
+        PMIX_RELEASE_THREAD(&myrollup->lock);
+        rc = myrollup->status;
+        PMIX_RELEASE(myrollup);
+        return PMIX_OPERATION_SUCCEEDED;
+    }
+
+    PMIX_RELEASE_THREAD(&myrollup->lock);
+    /* return success to indicate we are processing it */
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/prm/default/Makefile.am pmix-4.0.0/src/mca/prm/default/Makefile.am
--- pmix-3.2.2~rc1/src/mca/prm/default/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/default/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,42 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = prm_default.h
+sources = \
+        prm_default_component.c \
+        prm_default.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_prm_default_DSO
+lib =
+lib_sources =
+component = mca_prm_default.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_prm_default.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_prm_default_la_SOURCES = $(component_sources)
+mca_prm_default_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_prm_default_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_prm_default_la_SOURCES = $(lib_sources)
+libmca_prm_default_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/prm/default/prm_default.c pmix-4.0.0/src/mca/prm/default/prm_default.c
--- pmix-3.2.2~rc1/src/mca/prm/default/prm_default.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/default/prm_default.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include <time.h>
+
+#include "include/pmix_common.h"
+
+#include "src/include/pmix_socket_errno.h"
+#include "src/include/pmix_globals.h"
+#include "src/class/pmix_list.h"
+#include "src/util/alfg.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/util/parse_options.h"
+#include "src/util/pif.h"
+#include "src/util/pmix_environ.h"
+#include "src/mca/preg/preg.h"
+
+#include "src/mca/prm/base/base.h"
+#include "prm_default.h"
+
+static pmix_status_t default_notify(pmix_status_t status,
+                                    const pmix_proc_t *source,
+                                    pmix_data_range_t range,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+pmix_prm_module_t pmix_prm_default_module = {
+    .name = "default",
+    .notify = default_notify
+};
+
+
+static pmix_status_t default_notify(pmix_status_t status,
+                                    const pmix_proc_t *source,
+                                    pmix_data_range_t range,
+                                    const pmix_info_t info[], size_t ninfo,
+                                    pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    int rc;
+
+    /* if the server has provided the notify_event function
+     * entry, then just call it */
+    if (NULL != pmix_host_server.notify_event) {
+        rc = pmix_host_server.notify_event(status, source, range,
+                                           (pmix_info_t*)info, ninfo, cbfunc, cbdata);
+    } else {
+        rc = PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/prm/default/prm_default_component.c pmix-4.0.0/src/mca/prm/default/prm_default_component.c
--- pmix-3.2.2~rc1/src/mca/prm/default/prm_default_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/default/prm_default_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,56 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/mca/prm/prm.h"
+#include "prm_default.h"
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module,
+                                     int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+pmix_prm_base_component_t mca_prm_default_component = {
+    .base = {
+        PMIX_PRM_BASE_VERSION_1_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "default",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_query_component = component_query,
+    },
+    .data = {
+        /* The component is checkpoint ready */
+        PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
+    }
+};
+
+static pmix_status_t component_query(pmix_mca_base_module_t **module,
+                                     int *priority)
+{
+    *priority = 5;
+    *module = (pmix_mca_base_module_t *)&pmix_prm_default_module;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/prm/default/prm_default.h pmix-4.0.0/src/mca/prm/default/prm_default.h
--- pmix-3.2.2~rc1/src/mca/prm/default/prm_default.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/default/prm_default.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PRM_DEFAULT_H
+#define PMIX_PRM_DEFAULT_H
+
+#include "src/include/pmix_config.h"
+
+
+#include "src/mca/prm/prm.h"
+
+BEGIN_C_DECLS
+
+/* the component must be visible data for the linker to find it */
+PMIX_EXPORT extern pmix_prm_base_component_t mca_prm_default_component;
+extern pmix_prm_module_t pmix_prm_default_module;
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/prm/Makefile.am pmix-4.0.0/src/mca/prm/Makefile.am
--- pmix-3.2.2~rc1/src/mca/prm/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,44 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = $(LTDLINCL)
+
+# main library setup
+noinst_LTLIBRARIES = libmca_prm.la
+libmca_prm_la_SOURCES =
+
+# local files
+headers = prm.h
+sources =
+
+# Conditionally install the header files
+if WANT_INSTALL_HEADERS
+pmixdir = $(pmixincludedir)/$(subdir)
+nobase_pmix_HEADERS = $(headers)
+endif
+
+include base/Makefile.include
+
+libmca_prm_la_SOURCES += $(headers) $(sources)
+
+distclean-local:
+	rm -f base/static-components.h
diff -Nru pmix-3.2.2~rc1/src/mca/prm/prm.h pmix-4.0.0/src/mca/prm/prm.h
--- pmix-3.2.2~rc1/src/mca/prm/prm.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/prm/prm.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,109 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2007-2008 Cisco Systems, Inc.  All rights reserved.
+ *
+ * Copyright (c) 2015-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/**
+ * @file
+ *
+ * This interface is for use by PMIx servers to interpret/translate/interact
+ * from/to/with their host environment
+ *
+ * Available plugins may be defined at runtime via the typical MCA parameter
+ * syntax.
+ */
+
+#ifndef PMIX_PRM_H
+#define PMIX_PRM_H
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/class/pmix_list.h"
+#include "src/mca/mca.h"
+#include "src/mca/base/pmix_mca_base_var.h"
+#include "src/mca/base/pmix_mca_base_framework.h"
+#include "src/include/pmix_globals.h"
+#include "src/server/pmix_server_ops.h"
+
+BEGIN_C_DECLS
+
+/******    MODULE DEFINITION    ******/
+
+/**
+ * Initialize the module. Returns an error if the module cannot
+ * run, success if it can and wants to be used.
+ */
+typedef pmix_status_t (*pmix_prm_base_module_init_fn_t)(void);
+
+/**
+ * Finalize the module. Tear down any allocated storage, disconnect
+ * from any system support (e.g., LDAP server)
+ */
+typedef void (*pmix_prm_base_module_fini_fn_t)(void);
+
+/**
+ * Pass an event to the host system for transport. If the host system
+ * has called PMIx_server_init and provided an entry for the event
+ * notification upcall, then the default plugin will execute that
+ * pathway. However, some systems have chosen to utilize a "backdoor"
+ * channel for transporting the event - e.g., by calling some RM-provided
+ * API to inject the event into their transport.
+ */
+typedef pmix_status_t (*pmix_prm_base_module_notify_fn_t)(pmix_status_t status,
+                                                          const pmix_proc_t *source,
+                                                          pmix_data_range_t range,
+                                                          const pmix_info_t info[], size_t ninfo,
+                                                          pmix_op_cbfunc_t cbfunc, void *cbdata);
+
+/**
+ * Base structure for a PRM module. Each component should malloc a
+ * copy of the module structure for each fabric plane they support.
+ */
+typedef struct {
+    char *name;
+    /* init/finalize */
+    pmix_prm_base_module_init_fn_t                 init;
+    pmix_prm_base_module_fini_fn_t                 finalize;
+    pmix_prm_base_module_notify_fn_t               notify;
+} pmix_prm_module_t;
+
+
+/**
+ * Base structure for a PRM API - don't expose the init/finalize fns
+ */
+typedef struct {
+    pmix_prm_base_module_notify_fn_t               notify;
+} pmix_prm_API_module_t;
+
+
+/* declare the global APIs */
+PMIX_EXPORT extern pmix_prm_API_module_t pmix_prm;
+
+/*
+ * the standard component data structure
+ */
+struct pmix_prm_base_component_t {
+    pmix_mca_base_component_t                        base;
+    pmix_mca_base_component_data_t                   data;
+};
+typedef struct pmix_prm_base_component_t pmix_prm_base_component_t;
+
+/*
+ * Macro for use in components that are of type prm
+ */
+#define PMIX_PRM_BASE_VERSION_1_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("prm", 1, 0, 0)
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/psensor/heartbeat/psensor_heartbeat.c pmix-4.0.0/src/mca/psensor/heartbeat/psensor_heartbeat.c
--- pmix-3.2.2~rc1/src/mca/psensor/heartbeat/psensor_heartbeat.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/psensor/heartbeat/psensor_heartbeat.c	2021-01-02 08:56:17.000000000 +0000
@@ -212,7 +212,7 @@
         rcv->tag = PMIX_PTL_TAG_HEARTBEAT;
         rcv->cbfunc = pmix_psensor_heartbeat_recv_beats;
         /* add it to the beginning of the list of recvs */
-        pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &rcv->super);
+        pmix_list_prepend(&pmix_ptl_base.posted_recvs, &rcv->super);
         mca_psensor_heartbeat_component.recv_active = true;
     }
 
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/base.h pmix-4.0.0/src/mca/ptl/base/base.h
--- pmix-3.2.2~rc1/src/mca/ptl/base/base.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/base.h	2021-01-02 08:56:17.000000000 +0000
@@ -56,56 +56,80 @@
  */
 PMIX_EXPORT pmix_status_t pmix_ptl_base_select(void);
 
-/**
- * Track an active component
- */
-struct pmix_ptl_base_active_t {
-    pmix_list_item_t super;
-    pmix_status_t pri;
-    pmix_ptl_base_component_t *component;
-    pmix_ptl_module_t *module;
-};
-typedef struct pmix_ptl_base_active_t pmix_ptl_base_active_t;
-PMIX_CLASS_DECLARATION(pmix_ptl_base_active_t);
-
-
 /* framework globals */
-struct pmix_ptl_globals_t {
-    pmix_list_t actives;
+struct pmix_ptl_base_t {
     bool initialized;
     bool selected;
     pmix_list_t posted_recvs;     // list of pmix_ptl_posted_recv_t
     pmix_list_t unexpected_msgs;
     int stop_thread[2];
     bool listen_thread_active;
-    pmix_list_t listeners;
+    pmix_listener_t listener;
+    struct sockaddr_storage connection;
     uint32_t current_tag;
     size_t max_msg_size;
+    char *session_tmpdir;
+    char *system_tmpdir;
+    char *report_uri;
+    char *uri;
+    char *urifile;
+    char *system_filename;
+    char *session_filename;
+    char *nspace_filename;
+    char *pid_filename;
+    char *rendezvous_filename;
+    bool created_rendezvous_file;
+    bool created_session_tmpdir;
+    bool created_system_tmpdir;
+    bool created_system_filename;
+    bool created_session_filename;
+    bool created_nspace_filename;
+    bool created_pid_filename;
+    bool created_urifile;
+    bool remote_connections;
+    bool system_tool;
+    bool session_tool;
+    bool tool_support;
+    char *if_include;
+    char *if_exclude;
+    int ipv4_port;
+    bool disable_ipv4_family;
+    int ipv6_port;
+    bool disable_ipv6_family;
+    int max_retries;
+    int wait_to_connect;
+    int handshake_wait_time;
+    int handshake_max_retries;
 };
-typedef struct pmix_ptl_globals_t pmix_ptl_globals_t;
+typedef struct pmix_ptl_base_t pmix_ptl_base_t;
 
-PMIX_EXPORT extern pmix_ptl_globals_t pmix_ptl_globals;
+PMIX_EXPORT extern pmix_ptl_base_t pmix_ptl_base;
 
 /* API stubs */
 PMIX_EXPORT pmix_status_t pmix_ptl_base_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc);
-PMIX_EXPORT char* pmix_ptl_base_get_available_modules(void);
-PMIX_EXPORT pmix_ptl_module_t* pmix_ptl_base_assign_module(void);
 PMIX_EXPORT pmix_status_t pmix_ptl_base_connect_to_peer(struct pmix_peer_t *peer,
                                                         pmix_info_t info[], size_t ninfo);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_parse_uri_file(char *filename,
+                                                       char **uri,
+                                                       char **nspace,
+                                                       pmix_rank_t *rank,
+                                                       pmix_peer_t *peer);
+
+PMIX_EXPORT pmix_status_t pmix_ptl_base_setup_connection(char *uri,
+                                                         struct sockaddr_storage *connection,
+                                                         size_t *len);
 
-PMIX_EXPORT pmix_status_t pmix_ptl_base_register_recv(struct pmix_peer_t *peer,
-                                                      pmix_ptl_cbfunc_t cbfunc,
-                                                      pmix_ptl_tag_t tag);
-PMIX_EXPORT pmix_status_t pmix_ptl_base_cancel_recv(struct pmix_peer_t *peer,
-                                                    pmix_ptl_tag_t tag);
+PMIX_EXPORT void pmix_ptl_base_post_recv(int fd, short args, void *cbdata);
+PMIX_EXPORT void pmix_ptl_base_cancel_recv(int sd, short args, void *cbdata);
 
-PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t info[], size_t ninfo);
 PMIX_EXPORT void pmix_ptl_base_stop_listening(void);
-PMIX_EXPORT pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env);
+PMIX_EXPORT pmix_status_t pmix_base_write_rndz_file(char *filename, char *uri, bool *created);
 
 /* base support functions */
-PMIX_EXPORT void pmix_ptl_base_send(int sd, short args, void *cbdata);
-PMIX_EXPORT void pmix_ptl_base_send_recv(int sd, short args, void *cbdata);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_check_server_uris(pmix_peer_t *peer, char **evar);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_check_directives(pmix_info_t *info, size_t ninfo);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env);
 PMIX_EXPORT void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata);
 PMIX_EXPORT void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata);
 PMIX_EXPORT void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata);
@@ -116,10 +140,36 @@
 PMIX_EXPORT pmix_status_t pmix_ptl_base_connect(struct sockaddr_storage *addr,
                                                 pmix_socklen_t len, int *fd);
 PMIX_EXPORT void pmix_ptl_base_connection_handler(int sd, short args, void *cbdata);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_setup_listener(void);
 PMIX_EXPORT pmix_status_t pmix_ptl_base_send_connect_ack(int sd);
 PMIX_EXPORT pmix_status_t pmix_ptl_base_recv_connect_ack(int sd);
 PMIX_EXPORT void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err);
 PMIX_EXPORT bool pmix_ptl_base_peer_is_earlier(pmix_peer_t *peer, uint8_t major, uint8_t minor, uint8_t release);
+PMIX_EXPORT void pmix_ptl_base_query_servers(int sd, short args, void *cbdata);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_parse_uri(const char *evar, char **nspace,
+                                                  pmix_rank_t *rank, char **suri);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_df_search(char *dirname, char *prefix,
+                                                  pmix_info_t info[], size_t ninfo,
+                                                  int *sd, char **nspace,
+                                                  pmix_rank_t *rank, char **uri,
+                                                  pmix_peer_t *peer);
+PMIX_EXPORT uint8_t pmix_ptl_base_set_flag(size_t *sz);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_make_connection(pmix_peer_t *peer, char *suri,
+                                                        pmix_info_t *iptr, size_t niptr);
+PMIX_EXPORT void pmix_ptl_base_complete_connection(pmix_peer_t *peer, char *nspace,
+                                                   pmix_rank_t rank, char *uri);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_construct_message(pmix_peer_t *peer,
+                                                          char **msgout, size_t *sz,
+                                                          pmix_info_t *iptr, size_t niptr);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_set_timeout(pmix_peer_t *peer,
+                                                    struct timeval *save,
+                                                    pmix_socklen_t *sz,
+                                                    bool *sockopt);
+PMIX_EXPORT void pmix_ptl_base_setup_socket(pmix_peer_t *peer);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_client_handshake(pmix_peer_t *peer,
+                                                         pmix_status_t reply);
+PMIX_EXPORT pmix_status_t pmix_ptl_base_tool_handshake(pmix_peer_t *peer, pmix_status_t rp);
+PMIX_EXPORT char **pmix_ptl_base_split_and_resolve(char **orig_str, char *name);
 
 END_C_DECLS
 
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/help-ptl-base.txt pmix-4.0.0/src/mca/ptl/base/help-ptl-base.txt
--- pmix-3.2.2~rc1/src/mca/ptl/base/help-ptl-base.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/help-ptl-base.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,113 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2006 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2015-2020 Cisco Systems, Inc.  All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+[static-and-dynamic]
+Both static and dynamic port ranges were specified for the
+PMIx communication subsystem:
+
+Static ports:   %s
+Dynamic ports:  %s
+
+Only one can be specified. Please choose either static or
+dynamic ports and try again.
+#
+[include-exclude]
+Both TCP interface include and exclude lists were specified:
+
+  Include: %s
+  Exclude: %s
+
+Only one of these can be given.
+#
+[not-parseable]
+The specified network is not parseable. Since we cannot determine
+your desired intent, we cannot establish a TCP socket for PMIx
+communications and will therefore abort. Please correct the network
+specification and retry.
+#
+[no-included-found]
+None of the TCP networks specified to be included for communications
+could be found:
+
+  Value given: %s
+
+Please revise the specification and try again.
+#
+[excluded-all]
+The specified list of networks to be excluded for TCP communications
+resulted in no networks being available:
+
+  Value given: %s
+
+Please revise the specification and try again.
+#
+[invalid if_inexclude]
+WARNING: An invalid value was given for pmix_ptl_base_if_%s.  This
+value will be ignored.
+
+  Local host: %s
+  Value:      %s
+  Message:    %s
+#
+[authent-fail]
+An attempt was made to make a TCP connection between two hosts:
+
+  Initiating host:  %s
+  Receiving host:   %s
+
+Unfortunately, the connection was refused due to a failure to
+authenticate. This is usually caused by a mis-match between
+the security domains of the two hosts - e.g., one might be
+using Munge while the other is not. This can typically be
+resolved by specifying the desired security method. For
+example, adding "--mca psec native" to your command line.
+#
+[accept failed]
+WARNING: The accept(3) system call failed on a TCP socket.  While this
+should generally never happen on a well-configured HPC system, the
+most common causes when it does occur are:
+
+  * The process ran out of file descriptors
+  * The operating system ran out of file descriptors
+  * The operating system ran out of memory
+
+Your job may hang until the failure reason is fixed
+(e.g., more file descriptors and/or memory becomes available), and may
+eventually timeout / abort.
+
+  Local host:     %s
+  Errno:          %d (%s)
+  Probable cause: %s
+#
+[privilege failure]
+An attempt was made to initiate a TCP connection from an
+unprivileged source while we are operating at privileged
+levels.
+
+  Local host:      %s
+  Listening port:  %d
+  Remote host:     %s
+  Remote port:    %d
+
+The connection was rejected.
+#
+[no-listeners]
+No sockets were able to be opened on the available protocols
+(IPv4 and/or IPv6). Please check your network and retry.
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/Makefile.include pmix-4.0.0/src/mca/ptl/base/Makefile.include
--- pmix-3.2.2~rc1/src/mca/ptl/base/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
 # Copyright (c) 2016      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -23,8 +23,11 @@
 # This makefile.include does not stand on its own - it is included from
 # src/Makefile.am
 
+dist_pmixdata_DATA = base/help-ptl-base.txt
+
 headers += \
-         base/base.h
+         base/base.h \
+         base/ptl_base_handshake.h
 
 sources += \
         base/ptl_base_frame.c \
@@ -32,4 +35,6 @@
         base/ptl_base_sendrecv.c \
         base/ptl_base_listener.c \
         base/ptl_base_stubs.c \
-        base/ptl_base_connect.c
+        base/ptl_base_connect.c \
+        base/ptl_base_fns.c \
+        base/ptl_base_connection_hdlr.c
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_connect.c pmix-4.0.0/src/mca/ptl/base/ptl_base_connect.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_connect.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_connect.c	2021-01-02 08:56:17.000000000 +0000
@@ -184,7 +184,7 @@
 pmix_status_t pmix_ptl_base_connect(struct sockaddr_storage *addr,
                                     pmix_socklen_t addrlen, int *fd)
 {
-    int sd = -1;
+    int sd = -1, sd2;
     int retries = 0;
 
     pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
@@ -208,7 +208,11 @@
                 /* The server may be too busy to accept new connections */
                 pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                                     "timeout connecting to server");
+                /* get a different socket, but do that BEFORE we release the current
+                 * one so we don't just get the same socket handed back to us */
+                sd2 = socket(addr->ss_family, SOCK_STREAM, 0);
                 CLOSE_THE_SOCKET(sd);
+                sd = sd2;
                 continue;
             }
 
@@ -220,13 +224,21 @@
             if (ECONNABORTED == pmix_socket_errno) {
                 pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                                     "connection to server aborted by OS - retrying");
+                /* get a different socket, but do that BEFORE we release the current
+                 * one so we don't just get the same socket handed back to us */
+                sd2 = socket(addr->ss_family, SOCK_STREAM, 0);
                 CLOSE_THE_SOCKET(sd);
+                sd = sd2;
                 continue;
             } else {
                 pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
                                     "Connect failed: %s (%d)", strerror(pmix_socket_errno),
                                     pmix_socket_errno);
+                /* get a different socket, but do that BEFORE we release the current
+                 * one so we don't just get the same socket handed back to us */
+                sd2 = socket(addr->ss_family, SOCK_STREAM, 0);
                 CLOSE_THE_SOCKET(sd);
+                sd = sd2;
                 continue;
             }
         } else {
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_connection_hdlr.c pmix-4.0.0/src/mca/ptl/base/ptl_base_connection_hdlr.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_connection_hdlr.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_connection_hdlr.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,896 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_stdint.h"
+
+#include <stdio.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+#include "include/pmix_socket_errno.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/getid.h"
+#include "src/util/strnlen.h"
+#include "src/include/pmix_globals.h"
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/mca/bfrops/base/base.h"
+#include "src/mca/gds/base/base.h"
+
+#include "src/mca/ptl/base/ptl_base_handshake.h"
+#include "src/mca/ptl/base/base.h"
+
+static void process_cbfunc(int sd, short args, void *cbdata);
+static void cnct_cbfunc(pmix_status_t status,
+                        pmix_proc_t *proc, void *cbdata);
+static void _check_cached_events(pmix_peer_t *peer);
+static pmix_status_t process_tool_request(pmix_pending_connection_t *pnd,
+                                          char *mg, size_t cnt);
+
+void pmix_ptl_base_connection_handler(int sd, short args, void *cbdata)
+{
+    pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
+    pmix_ptl_hdr_t hdr;
+    pmix_peer_t *peer = NULL;
+    pmix_status_t rc, reply;
+    char *msg = NULL, *mg, *p;
+    uint32_t u32;
+    size_t cnt;
+    pmix_namespace_t *nptr, *tmp;
+    pmix_rank_info_t *info = NULL, *iptr;
+    pmix_proc_t proc;
+    pmix_info_t ginfo;
+    pmix_byte_object_t cred;
+    uint8_t major, minor, release;
+
+    /* acquire the object */
+    PMIX_ACQUIRE_OBJECT(pnd);
+
+    pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
+                        "ptl:base:connection_handler: new connection: %d",
+                        pnd->sd);
+
+    /* ensure the socket is in blocking mode */
+    pmix_ptl_base_set_blocking(pnd->sd);
+
+    /* ensure all is zero'd */
+    memset(&hdr, 0, sizeof(pmix_ptl_hdr_t));
+
+    /* get the header */
+    if (PMIX_SUCCESS != pmix_ptl_base_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_ptl_hdr_t))) {
+        goto error;
+    }
+
+    /* get the id, authentication and version payload (and possibly
+     * security credential) - to guard against potential attacks,
+     * we'll set an arbitrary limit per a define */
+    if (PMIX_MAX_CRED_SIZE < hdr.nbytes) {
+        goto error;
+    }
+    if (NULL == (msg = (char*)malloc(hdr.nbytes))) {
+        goto error;
+    }
+    if (PMIX_SUCCESS != pmix_ptl_base_recv_blocking(pnd->sd, msg, hdr.nbytes)) {
+        /* unable to complete the recv */
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool:connection_handler unable to complete recv of connect-ack with client ON SOCKET %d",
+                            pnd->sd);
+        goto error;
+    }
+
+    cnt = hdr.nbytes;
+    mg = msg;
+    /* extract the name of the sec module they used */
+    PMIX_PTL_GET_STRING(pnd->psec);
+
+    /* extract any credential so we can validate this connection
+     * before doing anything else */
+    PMIX_PTL_GET_U32(pnd->len);
+
+    /* if a credential is present, then create space and
+     * extract it for processing */
+    PMIX_PTL_GET_BLOB(pnd->cred, pnd->len);
+
+    /* get the process type of the connecting peer */
+    PMIX_PTL_GET_U8(pnd->flag);
+
+    switch(pnd->flag) {
+        case 0:
+            /* simple client process */
+            PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_CLIENT);
+            /* get their identifier */
+            PMIX_PTL_GET_PROCID(pnd->proc);
+            break;
+
+        case 1:
+            /* legacy tool - may or may not have an identifier */
+            PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_TOOL);
+            /* get their uid/gid */
+            PMIX_PTL_GET_U32(pnd->uid);
+            PMIX_PTL_GET_U32(pnd->gid);
+            break;
+
+        case 2:
+            /* legacy launcher - may or may not have an identifier */
+            PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_LAUNCHER);
+            /* get their uid/gid */
+            PMIX_PTL_GET_U32(pnd->uid);
+            PMIX_PTL_GET_U32(pnd->gid);
+            break;
+
+        case 3:
+        case 6:
+            /* self-started tool/launcher process that needs an identifier */
+            if (3 == pnd->flag) {
+                PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_TOOL);
+            } else {
+                PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_LAUNCHER);
+            }
+            /* get their uid/gid */
+            PMIX_PTL_GET_U32(pnd->uid);
+            PMIX_PTL_GET_U32(pnd->gid);
+            /* they need an id */
+            pnd->need_id = true;
+            break;
+
+        case 4:
+        case 7:
+            /* self-started tool/launcher process that was given an identifier by caller */
+            if (4 == pnd->flag) {
+                PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_TOOL);
+            } else {
+                PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_LAUNCHER);
+            }
+            /* get their uid/gid */
+            PMIX_PTL_GET_U32(pnd->uid);
+            PMIX_PTL_GET_U32(pnd->gid);
+            /* get their identifier */
+            PMIX_PTL_GET_PROCID(pnd->proc);
+            break;
+
+        case 5:
+        case 8:
+            /* tool/launcher that was started by a PMIx server - identifier specified by server */
+            if (5 == pnd->flag) {
+                PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_TOOL);
+            } else {
+                PMIX_SET_PROC_TYPE(&pnd->proc_type, PMIX_PROC_LAUNCHER);
+            }
+            /* get their uid/gid */
+            PMIX_PTL_GET_U32(pnd->uid);
+            PMIX_PTL_GET_U32(pnd->gid);
+            /* get their identifier */
+            PMIX_PTL_GET_PROCID(pnd->proc);
+            break;
+
+        default:
+            /* we don't know what they are! */
+            PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+            goto error;
+    }
+
+    /* extract their VERSION */
+    PMIX_PTL_GET_STRING(pnd->version);
+    major = strtoul(pnd->version, &p, 10);
+    ++p;
+    minor = strtoul(p, &p, 10);
+    ++p;
+    release = strtoul(p, NULL, 10);
+    PMIX_SET_PROC_MAJOR(&pnd->proc_type, major);
+    PMIX_SET_PROC_MINOR(&pnd->proc_type, minor);
+    PMIX_SET_PROC_RELEASE(&pnd->proc_type, release);
+
+    if (2 == major && 0 == minor) {
+        /* the 2.0 release handshake ends with the version string */
+        pnd->bfrops = strdup("v20");
+        pnd->buffer_type = pmix_bfrops_globals.default_type;  // we can't know any better
+        pnd->gds = strdup("ds12,hash");
+    } else {
+        /* extract the name of the bfrops module they used */
+        PMIX_PTL_GET_STRING(pnd->bfrops);
+
+        /* extract the type of buffer they used */
+        PMIX_PTL_GET_U8(pnd->buffer_type);
+
+        /* extract the name of the gds module they used */
+        PMIX_PTL_GET_STRING(pnd->gds);
+    }
+
+    /* see if this is a tool connection request */
+    if (0 != pnd->flag) {
+        /* nope, it's for a tool, so process it
+         * separately - it is a 2-step procedure */
+        rc = process_tool_request(pnd, mg, cnt);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto error;
+        }
+        return;
+    }
+
+    /* it is a client that is connecting, so it should have
+     * been registered with us prior to being started.
+     * See if we know this nspace */
+    nptr = NULL;
+    PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_namespace_t) {
+        if (0 == strcmp(tmp->nspace, pnd->proc.nspace)) {
+            nptr = tmp;
+            break;
+        }
+    }
+    if (NULL == nptr) {
+        /* we don't know this namespace, reject it */
+        rc = PMIX_ERR_NOT_FOUND;
+        goto error;
+    }
+
+    /* likewise, we should have this peer in our list */
+    info = NULL;
+    PMIX_LIST_FOREACH(iptr, &nptr->ranks, pmix_rank_info_t) {
+        if (iptr->pname.rank == pnd->proc.rank) {
+            info = iptr;
+            break;
+        }
+    }
+    if (NULL == info) {
+        /* rank unknown, reject it */
+        rc = PMIX_ERR_NOT_FOUND;
+        goto error;
+    }
+
+    /* a peer can connect on multiple sockets since it can fork/exec
+     * a child that also calls PMIX_Init, so add it here if necessary.
+     * Create the tracker for this peer */
+    peer = PMIX_NEW(pmix_peer_t);
+    if (NULL == peer) {
+        goto error;
+    }
+    /* mark that this peer is a client of the given type */
+    memcpy(&peer->proc_type, &pnd->proc_type, sizeof(pmix_proc_type_t));
+    /* save the protocol */
+    peer->protocol = pnd->protocol;
+    /* add in the nspace pointer */
+    PMIX_RETAIN(nptr);
+    peer->nptr = nptr;
+    PMIX_RETAIN(info);
+    peer->info = info;
+    /* update the epilog fields */
+    peer->epilog.uid = info->uid;
+    peer->epilog.gid = info->gid;
+    /* ensure the nspace epilog is updated too */
+    nptr->epilog.uid = info->uid;
+    nptr->epilog.gid = info->gid;
+    info->proc_cnt++; /* increase number of processes on this rank */
+    peer->sd = pnd->sd;
+    if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) {
+        goto error;
+    }
+    info->peerid = peer->index;
+
+    /* set the sec module to match this peer */
+    peer->nptr->compat.psec = pmix_psec_base_assign_module(pnd->psec);
+    if (NULL == peer->nptr->compat.psec) {
+        goto error;
+    }
+
+    /* set the bfrops module to match this peer */
+    peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(pnd->bfrops);
+    if (NULL == peer->nptr->compat.bfrops) {
+        goto error;
+    }
+    /* and the buffer type to match */
+    peer->nptr->compat.type = pnd->buffer_type;
+
+    /* set the gds module to match this peer */
+    if (NULL != pnd->gds) {
+        PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, pnd->gds, PMIX_STRING);
+        peer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
+        PMIX_INFO_DESTRUCT(&ginfo);
+    } else {
+        peer->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0);
+    }
+    if (NULL == peer->nptr->compat.gds) {
+        goto error;
+    }
+
+    /* if we haven't previously stored the version for this
+     * nspace, do so now */
+    if (!nptr->version_stored) {
+        PMIX_INFO_LOAD(&ginfo, PMIX_BFROPS_MODULE, pnd->version, PMIX_STRING);
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, peer->nptr, &ginfo, 1);
+        PMIX_INFO_DESTRUCT(&ginfo);
+        nptr->version_stored = true;
+    }
+
+    free(msg);  // can now release the data buffer
+    msg = NULL;
+
+    /* validate the connection */
+    cred.bytes = pnd->cred;
+    cred.size = pnd->len;
+    PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, NULL, 0, NULL, NULL, &cred);
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "client connection validated with status=%d", reply);
+
+    /* tell the client all is good */
+    u32 = htonl(reply);
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+    /* If needed, perform the handshake. The macro will update reply */
+    PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred);
+
+    /* It is possible that connection validation failed
+     * We need to reply to the client first and cleanup after */
+    if (PMIX_SUCCESS != reply) {
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "validation of client connection failed");
+        /* send an error reply to the client */
+        goto error;
+    }
+
+    /* send the client's array index */
+    u32 = htonl(peer->index);
+      if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
+          PMIX_ERROR_LOG(rc);
+          goto error;
+      }
+
+      pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                          "connect-ack from client completed");
+
+    /* let the host server know that this client has connected */
+    if (NULL != pmix_host_server.client_connected2) {
+        PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank);
+        rc = pmix_host_server.client_connected2(&proc, peer->info->server_object, NULL, 0, NULL, NULL);
+        if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+    } else if (NULL != pmix_host_server.client_connected) {
+        PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank);
+        rc = pmix_host_server.client_connected(&proc, peer->info->server_object, NULL, NULL);
+        if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto error;
+        }
+      }
+
+    pmix_ptl_base_set_nonblocking(pnd->sd);
+
+    /* start the events for this client */
+    pmix_event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd,
+                      EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, peer);
+    pmix_event_add(&peer->recv_event, NULL);
+    peer->recv_ev_active = true;
+    pmix_event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd,
+                      EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, peer);
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:server client %s:%u has connected on socket %d",
+                        peer->info->pname.nspace, peer->info->pname.rank, peer->sd);
+    PMIX_RELEASE(pnd);
+
+    /* check the cached events and update the client */
+    _check_cached_events(peer);
+
+    return;
+
+  error:
+    if (NULL != info) {
+        info->proc_cnt--;
+        PMIX_RELEASE(info);
+    }
+    if (NULL != msg) {
+        free(msg);
+    }
+    if (NULL != peer) {
+        pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
+        PMIX_RELEASE(peer);
+    }
+    CLOSE_THE_SOCKET(pnd->sd);
+    PMIX_RELEASE(pnd);
+    return;
+}
+
+/* process the host's callback with tool connection info */
+static void process_cbfunc(int sd, short args, void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata;
+    pmix_namespace_t *nptr;
+    pmix_rank_info_t *info;
+    pmix_peer_t *peer;
+    pmix_status_t rc, reply;
+    uint32_t u32;
+    pmix_info_t ginfo;
+    pmix_byte_object_t cred;
+    pmix_iof_req_t *req = NULL;
+
+    /* acquire the object */
+    PMIX_ACQUIRE_OBJECT(cd);
+    /* shortcuts */
+    peer = (pmix_peer_t*)pnd->peer;
+    nptr = peer->nptr;
+
+    /* send this status so they don't hang */
+    u32 = ntohl(cd->status);
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+
+    /* if the request failed, then we are done */
+    if (PMIX_SUCCESS != cd->status) {
+        goto error;
+    }
+
+    /* if we got an identifier, send it back to the tool */
+    if (pnd->need_id) {
+        /* start with the nspace */
+        if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) {
+            PMIX_ERROR_LOG(rc);
+            goto error;
+        }
+
+        /* now the rank, suitably converted */
+        u32 = ntohl(cd->proc.rank);
+        if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
+            PMIX_ERROR_LOG(rc);
+            goto error;
+        }
+    }
+
+    /* send my nspace back to the tool */
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+
+    /* send my rank back to the tool */
+    u32 = ntohl(pmix_globals.myid.rank);
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+
+    /* if this tool wasn't initially registered as a client,
+     * then add some required structures */
+    if (5 != pnd->flag && 8 != pnd->flag) {
+        PMIX_RETAIN(nptr);
+        nptr->nspace = strdup(cd->proc.nspace);
+        pmix_list_append(&pmix_globals.nspaces, &nptr->super);
+        info = PMIX_NEW(pmix_rank_info_t);
+        info->pname.nspace = strdup(nptr->nspace);
+        info->pname.rank = cd->proc.rank;
+        info->uid = pnd->uid;
+        info->gid = pnd->gid;
+        pmix_list_append(&nptr->ranks, &info->super);
+        PMIX_RETAIN(info);
+        peer->info = info;
+    }
+
+    /* mark the peer proc type */
+    memcpy(&peer->proc_type, &pnd->proc_type, sizeof(pmix_proc_type_t));
+    /* save the protocol */
+    peer->protocol = pnd->protocol;
+    /* save the uid/gid */
+    peer->epilog.uid = peer->info->uid;
+    peer->epilog.gid = peer->info->gid;
+    nptr->epilog.uid = peer->info->uid;
+    nptr->epilog.gid = peer->info->gid;
+    peer->proc_cnt = 1;
+    peer->sd = pnd->sd;
+
+    /* get the appropriate compatibility modules based on the
+     * info provided by the tool during the initial connection request */
+    peer->nptr->compat.psec = pmix_psec_base_assign_module(pnd->psec);
+    if (NULL == peer->nptr->compat.psec) {
+        goto error;
+    }
+    /* set the gds */
+    PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, pnd->gds, PMIX_STRING);
+    peer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
+    PMIX_INFO_DESTRUCT(&ginfo);
+    if (NULL == peer->nptr->compat.gds) {
+        goto error;
+    }
+
+    /* if we haven't previously stored the version for this
+     * nspace, do so now */
+    if (!peer->nptr->version_stored) {
+        PMIX_INFO_LOAD(&ginfo, PMIX_BFROPS_MODULE, pnd->version, PMIX_STRING);
+        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, peer->nptr, &ginfo, 1);
+        PMIX_INFO_DESTRUCT(&ginfo);
+        nptr->version_stored = true;
+    }
+
+    /* automatically setup to forward output to the tool */
+    req = PMIX_NEW(pmix_iof_req_t);
+    if (NULL == req) {
+        goto error;
+    }
+    PMIX_RETAIN(peer);
+    req->requestor = peer;
+    req->nprocs = 1;
+    PMIX_PROC_CREATE(req->procs, req->nprocs);
+    PMIX_LOAD_PROCID(&req->procs[0], pmix_globals.myid.nspace, pmix_globals.myid.rank);
+    req->channels = PMIX_FWD_STDOUT_CHANNEL | PMIX_FWD_STDERR_CHANNEL | PMIX_FWD_STDDIAG_CHANNEL;
+    req->remote_id = 0;     // default ID for tool during init
+    req->local_id = pmix_pointer_array_add(&pmix_globals.iof_requests, req);
+
+    /* validate the connection */
+    cred.bytes = pnd->cred;
+    cred.size = pnd->len;
+    PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, NULL, 0, NULL, NULL, &cred);
+    /* communicate the result to the other side */
+    u32 = htonl(reply);
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+
+    /* If needed perform the handshake. The macro will update reply */
+    PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred);
+
+    /* If verification wasn't successful - stop here */
+    if (PMIX_SUCCESS != reply) {
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "validation of tool credentials failed: %s",
+                            PMIx_Error_string(rc));
+        goto error;
+    }
+
+    /* set the socket non-blocking for all further operations */
+    pmix_ptl_base_set_nonblocking(pnd->sd);
+
+    if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) {
+        goto error;
+    }
+    peer->info->peerid = peer->index;
+
+    /* start the events for this tool */
+    pmix_event_assign(&peer->recv_event, pmix_globals.evbase, peer->sd,
+                      EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, peer);
+    pmix_event_add(&peer->recv_event, NULL);
+    peer->recv_ev_active = true;
+    pmix_event_assign(&peer->send_event, pmix_globals.evbase, peer->sd,
+                      EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, peer);
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:server tool %s:%d has connected on socket %d",
+                        peer->info->pname.nspace, peer->info->pname.rank, peer->sd);
+
+    /* check the cached events and update the tool */
+    _check_cached_events(peer);
+    PMIX_RELEASE(pnd);
+    PMIX_RELEASE(cd);
+    return;
+
+  error:
+    CLOSE_THE_SOCKET(pnd->sd);
+    PMIX_RELEASE(pnd);
+    PMIX_RELEASE(peer);
+    pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
+    PMIX_RELEASE(nptr);  // will release the info object
+    PMIX_RELEASE(cd);
+    if (NULL != req) {
+        pmix_pointer_array_set_item(&pmix_globals.iof_requests, req->local_id, NULL);
+        PMIX_RELEASE(req);
+    }
+}
+
+/* receive a callback from the host RM with an nspace
+ * for a connecting tool */
+static void cnct_cbfunc(pmix_status_t status,
+                        pmix_proc_t *proc, void *cbdata)
+{
+    pmix_setup_caddy_t *cd;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:tool:cnct_cbfunc returning %s:%d %s",
+                        proc->nspace, proc->rank, PMIx_Error_string(status));
+
+    /* need to thread-shift this into our context */
+    cd = PMIX_NEW(pmix_setup_caddy_t);
+    if (NULL == cd) {
+        PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+        return;
+    }
+    cd->status = status;
+    PMIX_LOAD_PROCID(&cd->proc, proc->nspace, proc->rank);
+    cd->cbdata = cbdata;
+    PMIX_THREADSHIFT(cd, process_cbfunc);
+}
+
+static pmix_status_t process_tool_request(pmix_pending_connection_t *pnd,
+                                          char *mg, size_t cnt)
+{
+    pmix_peer_t *peer;
+    pmix_namespace_t *nptr, *tmp;
+    pmix_rank_info_t *info;
+    bool found;
+    size_t n;
+    pmix_buffer_t buf;
+    pmix_status_t rc;
+
+    peer = PMIX_NEW(pmix_peer_t);
+    if (NULL == peer) {
+        return PMIX_ERR_NOMEM;
+    }
+    pnd->peer = peer;
+    /* if this is a tool we launched, then the host may
+     * have already registered it as a client - so check
+     * to see if we already have a peer for it */
+    if (5 == pnd->flag || 8 == pnd->flag) {
+        /* registration only adds the nspace and a rank in that
+         * nspace - it doesn't add the peer object to our array
+         * of local clients. So let's start by searching for
+         * the nspace object */
+        nptr = NULL;
+        PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_namespace_t) {
+            if (0 == strcmp(tmp->nspace, pnd->proc.nspace)) {
+                nptr = tmp;
+                break;
+            }
+        }
+        if (NULL == nptr) {
+            /* it is possible that this is a tool inside of
+             * a job-script as part of a multi-spawn operation.
+             * Since each tool invocation may have finalized and
+             * terminated, the tool will appear to "terminate", thus
+             * causing us to cleanup all references to it, and then
+             * reappear. So we don't reject this connection request.
+             * Instead, we create the nspace and rank objects for
+             * it and let the RM/host decide if this behavior
+             * is allowed */
+            nptr = PMIX_NEW(pmix_namespace_t);
+            if (NULL == nptr) {
+                return PMIX_ERR_NOMEM;
+            }
+            nptr->nspace = strdup(pnd->proc.nspace);
+        }
+        /* now look for the rank */
+        info = NULL;
+        found = false;
+        PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) {
+            if (info->pname.rank == pnd->proc.rank) {
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            /* see above note about not finding nspace */
+            info = PMIX_NEW(pmix_rank_info_t);
+            info->pname.nspace = strdup(pnd->proc.nspace);
+            info->pname.rank = pnd->proc.rank;
+            info->uid = pnd->uid;
+            info->gid = pnd->gid;
+            pmix_list_append(&nptr->ranks, &info->super);
+        }
+        PMIX_RETAIN(info);
+        peer->info = info;
+        PMIX_RETAIN(nptr);
+    } else {
+        nptr = PMIX_NEW(pmix_namespace_t);
+        if (NULL == nptr) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+            PMIX_RELEASE(peer);
+            return PMIX_ERR_NOMEM;
+        }
+    }
+    peer->nptr = nptr;
+    /* select their bfrops compat module so we can unpack
+     * any provided pmix_info_t structs */
+    peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(pnd->bfrops);
+    if (NULL == peer->nptr->compat.bfrops) {
+        PMIX_RELEASE(peer);
+        return PMIX_ERR_NOMEM;
+    }
+    /* set the buffer type */
+    peer->nptr->compat.type = pnd->buffer_type;
+    n = 0;
+    /* if info structs need to be passed along, then unpack them */
+    if (0 < cnt) {
+        int32_t foo;
+        PMIX_CONSTRUCT(&buf, pmix_buffer_t);
+        PMIX_LOAD_BUFFER(peer, &buf, mg, cnt);
+        foo = 1;
+        PMIX_BFROPS_UNPACK(rc, peer, &buf, &pnd->ninfo, &foo, PMIX_SIZE);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_RELEASE(peer);
+            PMIX_DESTRUCT(&buf);
+            return rc;
+        }
+        foo = (int32_t)pnd->ninfo;
+        /* if we have an identifier, then we leave room to pass it */
+        if (!pnd->need_id) {
+            pnd->ninfo += 5;
+        } else {
+            pnd->ninfo += 3;
+        }
+        PMIX_INFO_CREATE(pnd->info, pnd->ninfo);
+        PMIX_BFROPS_UNPACK(rc, peer, &buf, pnd->info, &foo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_RELEASE(peer);
+            PMIX_DESTRUCT(&buf);
+            return rc;
+        }
+        n = foo;
+    } else {
+        if (!pnd->need_id) {
+            pnd->ninfo = 5;
+        } else {
+            pnd->ninfo = 3;
+        }
+        PMIX_INFO_CREATE(pnd->info, pnd->ninfo);
+    }
+
+    /* does the server support tool connections? */
+    if (NULL == pmix_host_server.tool_connected) {
+        if (pnd->need_id) {
+            /* we need someone to provide the tool with an
+             * identifier and they aren't available */
+            /* send an error reply to the client */
+            PMIX_RELEASE(peer);
+            return PMIX_ERR_NOT_SUPPORTED;
+        } else {
+            /* just process it locally */
+            cnct_cbfunc(PMIX_SUCCESS, &pnd->proc, (void*)pnd);
+            /* release the msg */
+            return PMIX_SUCCESS;
+        }
+    }
+
+    /* setup the info array to pass the relevant info
+     * to the server */
+    /* provide the version */
+    PMIX_INFO_LOAD(&pnd->info[n], PMIX_VERSION_INFO, pnd->version, PMIX_STRING);
+    ++n;
+    /* provide the user id */
+    PMIX_INFO_LOAD(&pnd->info[n], PMIX_USERID, &pnd->uid, PMIX_UINT32);
+    ++n;
+    /* and the group id */
+    PMIX_INFO_LOAD(&pnd->info[n], PMIX_GRPID, &pnd->gid, PMIX_UINT32);
+    ++n;
+    /* if we have it, pass along their ID */
+    if (!pnd->need_id) {
+        PMIX_INFO_LOAD(&pnd->info[n], PMIX_NSPACE, pnd->proc.nspace, PMIX_STRING);
+        ++n;
+        PMIX_INFO_LOAD(&pnd->info[n], PMIX_RANK, &pnd->proc.rank, PMIX_PROC_RANK);
+        ++n;
+    }
+
+    /* pass it up for processing */
+    pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd);
+    return PMIX_SUCCESS;
+}
+
+static void _check_cached_events(pmix_peer_t *peer)
+{
+    pmix_notify_caddy_t *cd;
+    int i;
+    size_t n;
+    pmix_range_trkr_t rngtrk;
+    pmix_buffer_t *relay;
+    pmix_proc_t proc;
+    pmix_status_t ret;
+    pmix_cmd_t cmd = PMIX_NOTIFY_CMD;
+    bool matched, found;
+
+    PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank);
+
+    for (i=0; i < pmix_globals.max_events; i++) {
+        pmix_hotel_knock(&pmix_globals.notifications, i, (void**)&cd);
+        if (NULL == cd) {
+            continue;
+        }
+        /* check the range */
+        if (NULL == cd->targets) {
+            rngtrk.procs = &cd->source;
+            rngtrk.nprocs = 1;
+        } else {
+            rngtrk.procs = cd->targets;
+            rngtrk.nprocs = cd->ntargets;
+        }
+        rngtrk.range = cd->range;
+        if (!pmix_notify_check_range(&rngtrk, &proc)) {
+            continue;
+        }
+        found = false;
+        /* if we were given specific targets, check if this is one */
+        if (NULL != cd->targets) {
+            matched = false;
+            for (n=0; n < cd->ntargets; n++) {
+                if (PMIX_CHECK_PROCID(&proc, &cd->targets[n])) {
+                    matched = true;
+                    /* track the number of targets we have left to notify */
+                    --cd->nleft;
+                    /* if this is the last one, then evict this event
+                     * from the cache */
+                    if (0 == cd->nleft) {
+                        pmix_hotel_checkout(&pmix_globals.notifications, cd->room);
+                        found = true;  // mark that we should release cd
+                    }
+                    break;
+                }
+            }
+            if (!matched) {
+                /* do not notify this one */
+                continue;
+            }
+        }
+
+        /* all matches - notify */
+        relay = PMIX_NEW(pmix_buffer_t);
+        if (NULL == relay) {
+            /* nothing we can do */
+            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+            break;
+        }
+        /* pack the info data stored in the event */
+        PMIX_BFROPS_PACK(ret, peer, relay, &cmd, 1, PMIX_COMMAND);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_RELEASE(relay);
+            break;
+        }
+        PMIX_BFROPS_PACK(ret, peer, relay, &cd->status, 1, PMIX_STATUS);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_RELEASE(relay);
+            break;
+        }
+        PMIX_BFROPS_PACK(ret, peer, relay, &cd->source, 1, PMIX_PROC);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_RELEASE(relay);
+            PMIX_ERROR_LOG(ret);
+            break;
+        }
+        PMIX_BFROPS_PACK(ret, peer, relay, &cd->ninfo, 1, PMIX_SIZE);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_RELEASE(relay);
+            break;
+        }
+        if (0 < cd->ninfo) {
+            PMIX_BFROPS_PACK(ret, peer, relay, cd->info, cd->ninfo, PMIX_INFO);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+                PMIX_RELEASE(relay);
+                break;
+            }
+        }
+        PMIX_SERVER_QUEUE_REPLY(ret, peer, 0, relay);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_RELEASE(relay);
+        }
+        if (found) {
+            PMIX_RELEASE(cd);
+        }
+    }
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_fns.c pmix-4.0.0/src/mca/ptl/base/ptl_base_fns.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_fns.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_fns.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1477 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include <stdio.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+#ifdef HAVE_SYS_SYSCTL_H
+#include <sys/sysctl.h>
+#endif
+#include <ctype.h>
+
+#include "src/include/pmix_socket_errno.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/net.h"
+#include "src/util/os_path.h"
+#include "src/util/pif.h"
+#include "src/util/printf.h"
+#include "src/util/show_help.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/mca/ptl/base/ptl_base_handshake.h"
+#include "src/mca/ptl/base/base.h"
+
+/****    SUPPORTING FUNCTIONS    ****/
+static void timeout(int sd, short args, void *cbdata);
+static char *pmix_getline(FILE *fp);
+
+pmix_status_t pmix_ptl_base_check_server_uris(pmix_peer_t *peer, char **ev)
+{
+    char *evar, *vrs;
+    pmix_status_t rc;
+
+    vrs = getenv("PMIX_VERSION");
+
+    if (NULL != (evar = getenv("PMIX_SERVER_URI4"))) {
+        /* we are talking to a v4 server */
+        PMIX_SET_PEER_TYPE(peer, PMIX_PROC_SERVER);
+        PMIX_SET_PEER_VERSION(peer, vrs, 4, 0);
+
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "V4 SERVER DETECTED");
+
+        /* must use the latest bfrops module */
+        PMIX_BFROPS_SET_MODULE(rc, pmix_globals.mypeer, peer, NULL);
+        *ev = evar;
+        return rc;
+    }
+
+    if (NULL != (evar = getenv("PMIX_SERVER_URI3"))) {
+        /* we are talking to a v3 server */
+        PMIX_SET_PEER_TYPE(peer, PMIX_PROC_SERVER);
+        PMIX_SET_PEER_VERSION(peer, vrs, 3, 0);
+
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "V3 SERVER DETECTED");
+
+        /* must use the v3 bfrops module */
+        PMIX_BFROPS_SET_MODULE(rc, pmix_globals.mypeer, peer, "v3");
+        *ev = evar;
+        return rc;
+    }
+
+    if (NULL != (evar = getenv("PMIX_SERVER_URI21"))) {
+        /* we are talking to a v2.1 server */
+        PMIX_SET_PEER_TYPE(peer, PMIX_PROC_SERVER);
+        PMIX_SET_PEER_VERSION(peer, vrs, 2, 1);
+
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "V21 SERVER DETECTED");
+
+        /* must use the v21 bfrops module */
+        PMIX_BFROPS_SET_MODULE(rc, pmix_globals.mypeer, peer, "v21");
+        *ev = evar;
+        return rc;
+    }
+
+    if (NULL != (evar = getenv("PMIX_SERVER_URI2"))) {
+        /* we are talking to a v2.0 server */
+        PMIX_SET_PEER_TYPE(peer, PMIX_PROC_SERVER);
+        PMIX_SET_PEER_VERSION(peer, vrs, 2, 0);
+
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "V20 SERVER DETECTED");
+
+        /* must use the v20 bfrops module */
+        PMIX_BFROPS_SET_MODULE(rc, pmix_globals.mypeer, peer, "v20");
+        *ev = evar;
+        return rc;
+    }
+
+    return PMIX_ERR_UNREACH;
+}
+
+pmix_status_t pmix_ptl_base_check_directives(pmix_info_t *info, size_t ninfo)
+{
+    size_t n;
+    pmix_status_t rc;
+
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_INCLUDE)) {
+            if (NULL != pmix_ptl_base.if_include) {
+                free(pmix_ptl_base.if_include);
+            }
+            pmix_ptl_base.if_include = strdup(info[n].value.data.string);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_EXCLUDE)) {
+            if (NULL != pmix_ptl_base.if_exclude) {
+                free(pmix_ptl_base.if_exclude);
+            }
+            pmix_ptl_base.if_exclude = strdup(info[n].value.data.string);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV4_PORT)) {
+            pmix_ptl_base.ipv4_port = info[n].value.data.integer;
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV6_PORT)) {
+            pmix_ptl_base.ipv6_port = info[n].value.data.integer;
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV4)) {
+            pmix_ptl_base.disable_ipv4_family = PMIX_INFO_TRUE(&info[n]);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV6)) {
+            pmix_ptl_base.disable_ipv6_family = PMIX_INFO_TRUE(&info[n]);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_URI) ||
+                   PMIX_CHECK_KEY(&info[n], PMIX_SERVER_URI)) {
+            if (NULL != pmix_ptl_base.uri) {
+                free(pmix_ptl_base.uri);
+            }
+            pmix_ptl_base.uri = strdup(info[n].value.data.string);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_REPORT_URI)) {
+            if (NULL != pmix_ptl_base.report_uri) {
+                free(pmix_ptl_base.report_uri);
+            }
+            pmix_ptl_base.report_uri = strdup(info[n].value.data.string);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_TMPDIR)) {
+            if (NULL != pmix_ptl_base.session_tmpdir) {
+                free(pmix_ptl_base.session_tmpdir);
+            }
+            pmix_ptl_base.session_tmpdir = strdup(info[n].value.data.string);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_SYSTEM_TMPDIR)) {
+            if (NULL != pmix_ptl_base.system_tmpdir) {
+                free(pmix_ptl_base.system_tmpdir);
+            }
+            pmix_ptl_base.system_tmpdir = strdup(info[n].value.data.string);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_MAX_RETRIES)) {
+             PMIX_VALUE_GET_NUMBER(rc, &info[n].value, pmix_ptl_base.max_retries, int);
+             if (PMIX_SUCCESS != rc) {
+                return rc;
+             }
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_RETRY_DELAY)) {
+             PMIX_VALUE_GET_NUMBER(rc, &info[n].value, pmix_ptl_base.wait_to_connect, int);
+             if (PMIX_SUCCESS != rc) {
+                return rc;
+             }
+        }
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env)
+{
+    pmix_setenv("PMIX_SERVER_TMPDIR", pmix_ptl_base.session_tmpdir, true, env);
+    pmix_setenv("PMIX_SYSTEM_TMPDIR", pmix_ptl_base.system_tmpdir, true, env);
+
+    return PMIX_SUCCESS;
+}
+
+
+pmix_status_t pmix_ptl_base_parse_uri(const char *evar, char **nspace,
+                                      pmix_rank_t *rank, char **suri)
+{
+    char **uri;
+    char *p;
+
+    uri = pmix_argv_split(evar, ';');
+    if (2 != pmix_argv_count(uri)) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        pmix_argv_free(uri);
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* set the server nspace - the rank is appended
+     * to the end with a '.' separator. NOTE: we
+     * cannot search from the FRONT as that would
+     * stop on any FQDN or IPv4 separations */
+    if (NULL == (p = strrchr(uri[0], '.'))) {
+        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+        pmix_argv_free(uri);
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+    *p = '\0';
+    ++p;
+    *nspace = strdup(uri[0]);
+    /* set the server rank */
+    *rank = strtoull(p, NULL, 10);
+    if (NULL != suri) {
+        *suri = strdup(uri[1]);
+    }
+
+    pmix_argv_free(uri);
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_ptl_base_parse_uri_file(char *filename,
+                                           char **uri,
+                                           char **nspace,
+                                           pmix_rank_t *rank,
+                                           pmix_peer_t *peer)
+{
+    FILE *fp;
+    char *srvr, *p;
+    pmix_lock_t lock;
+    pmix_event_t ev;
+    struct timeval tv;
+    int retries;
+    pmix_status_t rc;
+
+     /* if we cannot open the file, then the server must not
+     * be configured to support tool connections, or this
+     * user isn't authorized to access it - or it may just
+     * not exist yet! Check for existence */
+    /* coverity[toctou] */
+    if (0 != access(filename, R_OK)) {
+        if (ENOENT == errno) {
+            /* the file does not exist, so give it
+             * a little time to see if the server
+             * is still starting up */
+            retries = 0;
+            do {
+                ++retries;
+                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                    "WAITING FOR CONNECTION FILE %s", filename);
+                PMIX_CONSTRUCT_LOCK(&lock);
+                if (0 < pmix_ptl_base.wait_to_connect) {
+                    tv.tv_sec = pmix_ptl_base.wait_to_connect;
+                    tv.tv_usec = 0;
+                    pmix_event_evtimer_set(pmix_globals.evbase, &ev,
+                                           timeout, &lock);
+                    PMIX_POST_OBJECT(&ev);
+                    pmix_event_evtimer_add(&ev, &tv);
+                } else {
+                    tv.tv_sec = 0;
+                    tv.tv_usec = 10000;  // use 0.01 sec as default
+                    pmix_event_evtimer_set(pmix_globals.evbase, &ev,
+                                           timeout, &lock);
+                    PMIX_POST_OBJECT(&ev);
+                    pmix_event_evtimer_add(&ev, &tv);
+                }
+                PMIX_WAIT_THREAD(&lock);
+                PMIX_DESTRUCT_LOCK(&lock);
+                /* coverity[toctou] */
+                if (0 == access(filename, R_OK)) {
+                    goto process;
+                }
+            } while (retries < pmix_ptl_base.max_retries);
+            /* otherwise, mark it as unreachable */
+        }
+        return PMIX_ERR_UNREACH;
+    }
+
+  process:
+    fp = fopen(filename, "r");
+    if (NULL == fp) {
+        return PMIX_ERR_UNREACH;
+    }
+    /* get the URI - might seem crazy, but there is actually
+     * a race condition here where the server may have created
+     * the file but not yet finished writing into it. So give
+     * us a chance to get the required info */
+    for (retries=0; retries < 3; retries++) {
+        srvr = pmix_getline(fp);
+        if (NULL != srvr) {
+            break;
+        }
+        fclose(fp);
+        tv.tv_sec = 0;
+        tv.tv_usec = 10000;  // use 0.01 sec as default
+        pmix_event_evtimer_set(pmix_globals.evbase, &ev,
+                               timeout, &lock);
+        PMIX_POST_OBJECT(&ev);
+        pmix_event_evtimer_add(&ev, &tv);
+        PMIX_WAIT_THREAD(&lock);
+        PMIX_DESTRUCT_LOCK(&lock);
+        fp = fopen(filename, "r");
+        if (NULL == fp) {
+            return PMIX_ERR_UNREACH;
+        }
+    }
+    if (NULL == srvr) {
+        PMIX_ERROR_LOG(PMIX_ERR_FILE_READ_FAILURE);
+        fclose(fp);
+        return PMIX_ERR_UNREACH;
+    }
+
+    peer->protocol = PMIX_PROTOCOL_V2;
+    /* see if this file contains the server's version */
+    p = pmix_getline(fp);
+    PMIX_SET_PEER_VERSION(peer, p, 2, 0);
+    if (NULL != p) {
+        free(p);
+    }
+    fclose(fp);
+
+    /* parse the URI */
+    rc = pmix_ptl_base_parse_uri(srvr, nspace, rank, uri);
+    free(srvr);
+
+    return rc;
+}
+
+pmix_status_t pmix_ptl_base_df_search(char *dirname, char *prefix,
+                                      pmix_info_t info[], size_t ninfo,
+                                      int *sd, char **nspace,
+                                      pmix_rank_t *rank, char **uri,
+                                      pmix_peer_t *peer)
+{
+    char *suri, *nsp, *newdir;
+    pmix_rank_t rk;
+    pmix_status_t rc;
+    struct stat buf;
+    DIR *cur_dirp;
+    struct dirent *dir_entry;
+
+    if (NULL == (cur_dirp = opendir(dirname))) {
+        return PMIX_ERR_NOT_FOUND;
+    }
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:ptl: searching directory %s", dirname);
+
+    /* search the entries for something that starts with the provided prefix */
+    while (NULL != (dir_entry = readdir(cur_dirp))) {
+        /* ignore the . and .. entries */
+        if (0 == strcmp(dir_entry->d_name, ".") ||
+            0 == strcmp(dir_entry->d_name, "..")) {
+            continue;
+        }
+        newdir = pmix_os_path(false, dirname, dir_entry->d_name, NULL);
+        /* coverity[toctou] */
+        if (-1 == stat(newdir, &buf)) {
+            free(newdir);
+            continue;
+        }
+        /* if it is a directory, down search */
+        if (S_ISDIR(buf.st_mode)) {
+            rc = pmix_ptl_base_df_search(newdir, prefix, info, ninfo,
+                                         sd, nspace, rank, uri, peer);
+            free(newdir);
+            if (PMIX_SUCCESS == rc) {
+                closedir(cur_dirp);
+                return rc;
+            }
+            continue;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "pmix:tool: checking %s vs %s", dir_entry->d_name, prefix);
+        /* see if it starts with our prefix */
+        if (0 == strncmp(dir_entry->d_name, prefix, strlen(prefix))) {
+            /* try to read this file */
+            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                "pmix:tool: reading file %s", newdir);
+            rc = pmix_ptl_base_parse_uri_file(newdir, &suri, &nsp, &rk, peer);
+            if (PMIX_SUCCESS == rc) {
+                /* go ahead and try to connect */
+                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                    "pmix:tool: attempting to connect to %s", suri);
+                rc = pmix_ptl_base_make_connection(peer, suri, info, ninfo);
+                if (PMIX_SUCCESS == rc) {
+                    (*nspace) = nsp;
+                    *rank = rk;
+                    closedir(cur_dirp);
+                    *uri = suri;
+                    free(newdir);
+                    return PMIX_SUCCESS;
+                }
+                free(suri);
+                free(nsp);
+            }
+        }
+        free(newdir);
+    }
+    closedir(cur_dirp);
+    return PMIX_ERR_NOT_FOUND;
+}
+
+pmix_status_t pmix_ptl_base_setup_connection(char *uri,
+                                             struct sockaddr_storage *connection,
+                                             size_t *len)
+{
+    char *p = NULL, *p2, *host;
+    struct sockaddr_in *in;
+    struct sockaddr_in6 *in6;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:base setup connection to %s", uri);
+
+    memset(connection, 0, sizeof(struct sockaddr_storage));
+    if (0 == strncmp(uri, "tcp4", 4)) {
+        /* need to skip the tcp4: part */
+        p = strdup(&uri[7]);
+        if (NULL == p) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+            return PMIX_ERR_NOMEM;
+        }
+
+        /* separate the IP address from the port */
+        p2 = strrchr(p, ':');
+        if (NULL == p2) {
+            free(p);
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        *p2 = '\0';
+        p2++;
+        host = p;
+        /* load the address */
+        in = (struct sockaddr_in*)connection;
+        in->sin_family = AF_INET;
+        in->sin_addr.s_addr = inet_addr(host);
+        if (in->sin_addr.s_addr == INADDR_NONE) {
+            free(p);
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        in->sin_port = htons(atoi(p2));
+        *len = sizeof(struct sockaddr_in);
+    } else {
+        /* need to skip the tcp6: part */
+        p = strdup(&uri[7]);
+        if (NULL == p) {
+            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+            return PMIX_ERR_NOMEM;
+        }
+
+        p2 = strrchr(p, ':');
+        if (NULL == p2) {
+            free(p);
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        *p2 = '\0';
+        if (']' == p[strlen(p)-1]) {
+            p[strlen(p)-1] = '\0';
+        }
+        if ('[' == p[0]) {
+            host = &p[1];
+        } else {
+            host = &p[0];
+        }
+        /* load the address */
+        in6 = (struct sockaddr_in6*)connection;
+        in6->sin6_family = AF_INET6;
+        if (0 == inet_pton(AF_INET6, host, (void*)&in6->sin6_addr)) {
+            pmix_output (0, "ptl_tcp_parse_uri: Could not convert %s\n", host);
+            free(p);
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
+            return PMIX_ERR_BAD_PARAM;
+        }
+        in6->sin6_port = htons(atoi(p2));
+        *len = sizeof(struct sockaddr_in6);
+    }
+    if (NULL != p) {
+        free(p);
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static pmix_status_t send_connect_ack(pmix_peer_t *peer,
+                                      pmix_info_t iptr[],
+                                      size_t niptr)
+{
+    char *msg;
+    size_t sdsize=0;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:ptl SEND CONNECT ACK");
+
+    /* allow space for a marker indicating client vs tool */
+    sdsize = 1;
+
+    /* set our ID flag and compute the required handshake size */
+    peer->proc_type.flag = pmix_ptl_base_set_flag(&sdsize);
+
+    /* construct the contact message */
+    rc = pmix_ptl_base_construct_message(peer, &msg, &sdsize, iptr, niptr);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* send the entire message across */
+    if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(peer->sd, msg, sdsize)) {
+        free(msg);
+        return PMIX_ERR_UNREACH;
+    }
+    free(msg);
+    return PMIX_SUCCESS;
+}
+
+/* we receive a connection acknowledgment from the server,
+ * consisting of nothing more than a status report. If success,
+ * then we initiate authentication method */
+static pmix_status_t recv_connect_ack(pmix_peer_t *peer)
+{
+    pmix_status_t reply;
+    pmix_status_t rc;
+    struct timeval save;
+    pmix_socklen_t sz;
+    bool sockopt = true;
+    uint32_t u32;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix: RECV CONNECT ACK FROM SERVER");
+
+    /* set the socket timeout so we don't hang on blocking recv */
+    rc = pmix_ptl_base_set_timeout(peer, &save, &sz, &sockopt);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* receive the status reply */
+    rc = pmix_ptl_base_recv_blocking(peer->sd, (char*)&u32, sizeof(uint32_t));
+    if (PMIX_SUCCESS != rc) {
+        if (sockopt) {
+            /* return the socket to normal */
+            if (0 != setsockopt(peer->sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
+                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                    "pmix: could not reset setsockopt SO_RCVTIMEO");
+            }
+        }
+        return rc;
+    }
+    reply = ntohl(u32);
+
+    if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer) &&
+        !PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        rc = pmix_ptl_base_client_handshake(peer, reply);
+    } else {  // we are a tool
+        rc = pmix_ptl_base_tool_handshake(peer, reply);
+    }
+
+    if (sockopt) {
+        /* return the socket to normal */
+        if (0 != setsockopt(peer->sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
+            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                "pmix: could not reset setsockopt SO_RCVTIMEO");
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_ptl_base_make_connection(pmix_peer_t *peer, char *suri,
+                                            pmix_info_t *iptr, size_t niptr)
+{
+    struct sockaddr_storage myconnection;
+    pmix_status_t rc;
+    size_t len;
+    int retries = 0;
+
+    /* setup the connection */
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_setup_connection(suri, &myconnection, &len))) {
+        return rc;
+    }
+
+  retry:
+    /* try to connect */
+    if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&myconnection, len, &peer->sd))) {
+        /* do not error log - might just be a stale connection point */
+        return rc;
+    }
+
+    /* send our identity and any authentication credentials to the server */
+    if (PMIX_SUCCESS != (rc = send_connect_ack(peer, iptr, niptr))) {
+        PMIX_ERROR_LOG(rc);
+        CLOSE_THE_SOCKET(peer->sd);
+        return rc;
+    }
+
+    /* do whatever handshake is required */
+    if (PMIX_SUCCESS != (rc = recv_connect_ack(peer))) {
+        CLOSE_THE_SOCKET(peer->sd);
+        if (PMIX_ERR_TEMP_UNAVAILABLE == rc) {
+            ++retries;
+            if( retries < pmix_ptl_base.handshake_max_retries ) {
+                goto retry;
+            }
+        }
+        return rc;
+    }
+
+    return PMIX_SUCCESS;
+}
+
+void pmix_ptl_base_complete_connection(pmix_peer_t *peer, char *nspace,
+                                       pmix_rank_t rank, char *suri)
+{
+    pmix_kval_t *urikv;
+    pmix_status_t rc;
+
+    pmix_globals.connected = true;
+
+    /* setup the server info */
+    if (NULL == peer->info) {
+        peer->info = PMIX_NEW(pmix_rank_info_t);
+    }
+    if (NULL == peer->nptr) {
+        peer->nptr = PMIX_NEW(pmix_namespace_t);
+    }
+    if (NULL != peer->nptr->nspace) {
+        free(peer->nptr->nspace);
+    }
+    peer->nptr->nspace = strdup(nspace);
+
+    if (NULL != peer->info->pname.nspace) {
+        free(peer->info->pname.nspace);
+    }
+    peer->info->pname.nspace = strdup(peer->nptr->nspace);
+    peer->info->pname.rank = rank;
+
+    /* store the URI for subsequent lookups */
+    PMIX_KVAL_NEW(urikv, PMIX_SERVER_URI);
+    urikv->value->type = PMIX_STRING;
+    asprintf(&urikv->value->data.string, "%s.%u;%s", nspace, rank, suri);
+    PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
+                      &pmix_globals.myid, PMIX_INTERNAL,
+                      urikv);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+    }
+    PMIX_RELEASE(urikv);  // maintain accounting
+
+    pmix_ptl_base_set_nonblocking(peer->sd);
+
+    /* setup recv event */
+    pmix_event_assign(&peer->recv_event,
+                      pmix_globals.evbase,
+                      peer->sd,
+                      EV_READ | EV_PERSIST,
+                      pmix_ptl_base_recv_handler, peer);
+    peer->recv_ev_active = true;
+    PMIX_POST_OBJECT(peer);
+    pmix_event_add(&peer->recv_event, 0);
+
+    /* setup send event */
+    pmix_event_assign(&peer->send_event,
+                      pmix_globals.evbase,
+                      peer->sd,
+                      EV_WRITE|EV_PERSIST,
+                      pmix_ptl_base_send_handler, peer);
+    peer->send_ev_active = false;
+}
+
+uint8_t pmix_ptl_base_set_flag(size_t *sz)
+{
+    uint8_t flag;
+    size_t sdsize = 0;
+
+    /* Defined marker values:
+     *
+     * 0 => simple client process
+     * 1 => legacy tool - may or may not have an identifier
+     * 2 => legacy launcher - may or may not have an identifier
+     * ------------------------------------------
+     * 3 => self-started tool process that needs an identifier
+     * 4 => self-started tool process that was given an identifier by caller
+     * 5 => tool that was started by a PMIx server - identifier specified by server
+     * 6 => self-started launcher that needs an identifier
+     * 7 => self-started launcher that was given an identifier by caller
+     * 8 => launcher that was started by a PMIx server - identifier specified by server
+     */
+    if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
+            /* if we are both launcher and client, then we need
+             * to tell the server we are both */
+            flag = 8;
+            /* add space for our uid/gid for ACL purposes */
+            sdsize += 2*sizeof(uint32_t);
+            /* add space for our identifier */
+            sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
+        } else {
+            /* add space for our uid/gid for ACL purposes */
+            sdsize += 2*sizeof(uint32_t);
+            /* if they gave us an identifier, we need to pass it */
+            if (0 < strlen(pmix_globals.myid.nspace) &&
+                PMIX_RANK_INVALID != pmix_globals.myid.rank) {
+                flag = 7;
+                sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
+            } else {
+                flag = 6;
+            }
+        }
+
+    } else if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer) &&
+               !PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        /* we are a simple client */
+        flag = 0;
+        /* reserve space for our nspace and rank info */
+        sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
+
+    } else {  // must be a tool of some sort
+        /* add space for our uid/gid for ACL purposes */
+        sdsize += 2*sizeof(uint32_t);
+        if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
+            /* if we are both tool and client, then we need
+             * to tell the server we are both */
+            flag = 5;
+            /* add space for our identifier */
+            sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
+        } else if (0 < strlen(pmix_globals.myid.nspace) &&
+            PMIX_RANK_INVALID != pmix_globals.myid.rank) {
+            /* we were given an identifier by the caller, pass it */
+            sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
+            flag = 4;
+        } else {
+            /* we are a self-started tool that needs an identifier */
+            flag = 3;
+        }
+    }
+
+    *sz += sdsize;
+    return flag;
+}
+
+pmix_status_t pmix_ptl_base_construct_message(pmix_peer_t *peer,
+                                              char **msgout, size_t *sz,
+                                              pmix_info_t *iptr, size_t niptr)
+{
+    char *msg;
+    char *sec, *bfrops, *gds;
+    pmix_bfrop_buffer_type_t bftype;
+    uid_t euid;
+    gid_t egid;
+    pmix_buffer_t buf;
+    pmix_status_t rc;
+    pmix_ptl_hdr_t hdr;
+    size_t sdsize, csize;
+    pmix_byte_object_t cred;
+
+    sdsize = *sz;
+
+   /* setup the header */
+    memset(&hdr, 0, sizeof(pmix_ptl_hdr_t));
+    hdr.pindex = -1;
+    hdr.tag = UINT32_MAX;
+
+    /* a security module was assigned to us during rte_init based
+     * on a list of available security modules provided by our
+     * local PMIx server, if known. Now use that module to
+     * get a credential, if the security system provides one. Not
+     * every psec module will do so, thus we must first check */
+    PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
+    PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer,
+                          NULL, 0, NULL, 0, &cred);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_BYTE_OBJECT_DESTRUCT(&cred);
+        return rc;
+    }
+
+    /* add the name of our active sec module - we selected it
+     * in pmix_client.c prior to entering here */
+    sec = pmix_globals.mypeer->nptr->compat.psec->name;
+
+    /* add our active bfrops module name */
+    bfrops = pmix_globals.mypeer->nptr->compat.bfrops->version;
+    /* and the type of buffer we are using */
+    bftype = pmix_globals.mypeer->nptr->compat.type;
+
+    /* add our active gds module for working with the server */
+    gds = (char*)peer->nptr->compat.gds->name;
+
+    /* if we were given info structs to pass to the server, pack them */
+    PMIX_CONSTRUCT(&buf, pmix_buffer_t);
+    if (NULL != iptr) {
+        PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &niptr, 1, PMIX_SIZE);
+        PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, iptr, niptr, PMIX_INFO);
+    }
+
+    /* set the number of bytes to be read beyond the header - must
+     * NULL terminate the strings! */
+    hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \
+                + strlen(bfrops) + 1 + sizeof(bftype) \
+                + strlen(gds) + 1 + sizeof(uint32_t) + cred.size \
+                + buf.bytes_used;
+
+    /* create a space for our message */
+    sdsize = sizeof(hdr) + hdr.nbytes;
+    if (NULL == (msg = (char*)malloc(sdsize))) {
+        PMIX_BYTE_OBJECT_DESTRUCT(&cred);
+        free(sec);
+        PMIX_DESTRUCT(&buf);
+        return PMIX_ERR_OUT_OF_RESOURCE;
+    }
+    memset(msg, 0, sdsize);
+
+    /* load the header */
+    csize=0;
+    memcpy(msg, &hdr, sizeof(pmix_ptl_hdr_t));
+    csize += sizeof(pmix_ptl_hdr_t);
+
+    /* provide our active psec module */
+    PMIX_PTL_PUT_STRING(sec);
+
+    /* load the length of the credential */
+    PMIX_PTL_PUT_U32(cred.size);
+
+    /* load the credential */
+    PMIX_PTL_PUT_BLOB(cred.bytes, cred.size);
+    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
+
+    /* load our process type - this is a single byte,
+     * so no worry about heterogeneity here */
+    PMIX_PTL_PUT_U8(peer->proc_type.flag);
+
+    switch(peer->proc_type.flag) {
+        case 0:
+            /* simple client process */
+            PMIX_PTL_PUT_PROCID(pmix_globals.myid);
+            break;
+
+        /* we cannot have cases 1 or 2 because those are only
+         * for legacy processes */
+
+        case 3:
+        case 6:
+            /* self-started tool/launcher process that needs an identifier */
+            euid = geteuid();
+            PMIX_PTL_PUT_U32(euid);
+            egid = getegid();
+            PMIX_PTL_PUT_U32(egid);
+            break;
+
+        case 4:
+        case 7:
+            /* self-started tool/launcher process that was given an identifier by caller */
+            euid = geteuid();
+            PMIX_PTL_PUT_U32(euid);
+            egid = getegid();
+            PMIX_PTL_PUT_U32(egid);
+            /* add our identifier */
+            PMIX_PTL_PUT_PROCID(pmix_globals.myid);
+            break;
+
+        case 5:
+        case 8:
+            /* tool/launcher that was started by a PMIx server - identifier specified by server */
+            euid = geteuid();
+            PMIX_PTL_PUT_U32(euid);
+            egid = getegid();
+            PMIX_PTL_PUT_U32(egid);
+            /* add our identifier */
+            PMIX_PTL_PUT_PROCID(pmix_globals.myid);
+            break;
+
+        default:
+            /* we don't know what they are! */
+            PMIX_DESTRUCT(&buf);
+            free(msg);
+            return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* provide our version */
+    PMIX_PTL_PUT_STRING(PMIX_VERSION);
+
+    /* provide our active bfrops module */
+    PMIX_PTL_PUT_STRING(bfrops);
+
+    /* provide the bfrops type */
+    PMIX_PTL_PUT_U8(bftype);
+
+    /* provide the gds module */
+    PMIX_PTL_PUT_STRING(gds);
+
+    /* provide the info struct bytes */
+    PMIX_PTL_PUT_BLOB(buf.base_ptr, buf.bytes_used);
+    PMIX_DESTRUCT(&buf);
+
+    *msgout = msg;
+    *sz = sdsize;
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t pmix_ptl_base_set_timeout(pmix_peer_t *peer,
+                                        struct timeval *save,
+                                        pmix_socklen_t *sz,
+                                        bool *sockopt)
+{
+    struct timeval tv;
+
+    /* get the current timeout value so we can reset to it */
+    if (0 != getsockopt(peer->sd, SOL_SOCKET, SO_RCVTIMEO, (void*)save, sz)) {
+        *sockopt = false;
+   } else {
+        /* set a timeout on the blocking recv so we don't hang */
+        tv.tv_sec  = pmix_ptl_base.handshake_wait_time;
+        tv.tv_usec = 0;
+        if (0 != setsockopt(peer->sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
+            *sockopt = false;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+void pmix_ptl_base_setup_socket(pmix_peer_t *peer)
+{
+#if defined(TCP_NODELAY)
+    int optval;
+    optval = 1;
+    if (setsockopt(peer->sd, IPPROTO_TCP, TCP_NODELAY, (char *)&optval, sizeof(optval)) < 0) {
+        opal_backtrace_print(stderr, NULL, 1);
+        pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
+                            "[%s:%d] setsockopt(TCP_NODELAY) failed: %s (%d)",
+                            __FILE__, __LINE__,
+                            strerror(pmix_socket_errno),
+                            pmix_socket_errno);
+    }
+#endif
+#if defined(SO_NOSIGPIPE)
+    /* Some BSD flavors generate EPIPE when we write to a disconnected peer. We need
+     * the prevent this signal to be able to trap socket shutdown and cleanly release
+     * the endpoint.
+     */
+    int optval2 = 1;
+    if (setsockopt(peer->sd, SOL_SOCKET, SO_NOSIGPIPE, (char *)&optval2, sizeof(optval2)) < 0) {
+            pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
+                                "[%s:%d] setsockopt(SO_NOSIGPIPE) failed: %s (%d)",
+                                 __FILE__, __LINE__,
+                                strerror(pmix_socket_errno), pmix_socket_errno);
+    }
+#endif
+}
+
+pmix_status_t pmix_ptl_base_client_handshake(pmix_peer_t *peer, pmix_status_t reply)
+{
+    pmix_status_t rc;
+
+    /* see if they want us to do the handshake */
+    if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
+        PMIX_PSEC_CLIENT_HANDSHAKE(rc, peer, peer->sd);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+    } else if (PMIX_SUCCESS != reply) {
+        return reply;
+    }
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix: RECV CONNECT CONFIRMATION");
+
+    /* receive our index into the peer's client array */
+    PMIX_PTL_RECV_U32(peer->sd, pmix_globals.pindex);
+    return PMIX_SUCCESS;
+}
+
+/* the "peer" object passed into this function is that of the SERVER
+ * to which the tool is connecting - it is NOT the peer of the tool itself*/
+pmix_status_t pmix_ptl_base_tool_handshake(pmix_peer_t *peer, pmix_status_t rp)
+{
+    pmix_nspace_t nspace;
+    pmix_rank_t rank;
+    pmix_status_t reply;
+
+    /* if the status indicates an error, then we are done */
+    if (PMIX_SUCCESS != rp) {
+        return rp;
+    }
+
+    /* if we need an identifier, it comes next */
+    if (3 == peer->proc_type.flag || 6 == peer->proc_type.flag) {
+        PMIX_PTL_RECV_NSPACE(peer->sd, pmix_globals.myid.nspace);
+        PMIX_PTL_RECV_U32(peer->sd, pmix_globals.myid.rank);
+    }
+
+    /* get the server's nspace and rank so we can send to it */
+    if (NULL == peer->info) {
+        peer->info = PMIX_NEW(pmix_rank_info_t);
+    }
+    if (NULL == peer->nptr) {
+        peer->nptr = PMIX_NEW(pmix_namespace_t);
+    }
+    PMIX_PTL_RECV_NSPACE(peer->sd, nspace);
+    PMIX_PTL_RECV_U32(peer->sd, rank);
+
+    if (NULL != peer->nptr->nspace) {
+        free(peer->nptr->nspace);
+    }
+    peer->nptr->nspace = strdup(nspace);
+    if (NULL != peer->info->pname.nspace) {
+        free(peer->info->pname.nspace);
+    }
+    peer->info->pname.nspace = strdup(nspace);
+    peer->info->pname.rank = rank;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank,
+                        peer->info->pname.nspace,
+                        peer->info->pname.rank);
+
+    /* get the returned status from the security handshake */
+    PMIX_PTL_RECV_U32(peer->sd, reply);
+    if (PMIX_SUCCESS != reply) {
+        /* see if they want us to do the handshake */
+        if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
+            PMIX_PSEC_CLIENT_HANDSHAKE(reply, peer, peer->sd);
+            if (PMIX_SUCCESS != reply) {
+                return reply;
+            }
+            /* if the handshake succeeded, then fall thru to the next step */
+        } else {
+            return reply;
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static void check_server(char *filename,
+                         pmix_list_t *servers)
+{
+    FILE *fp;
+    char *srvr, *p, *p2;
+    pmix_lock_t lock;
+    pmix_event_t ev;
+    struct timeval tv;
+    int retries;
+    pmix_info_t *sdata;
+    size_t ndata, n;
+    pmix_infolist_t *iptr, *ians;
+    char *nspace=NULL, *version=NULL;
+    pmix_rank_t rank;
+    pmix_list_t mylist;
+    uint32_t u32;
+    pmix_status_t rc;
+
+     /* if we cannot open the file, then the server must not
+     * be configured to support tool connections, or this
+     * user isn't authorized to access it - or it may just
+     * not exist yet! Check for existence */
+    /* coverity[toctou] */
+    if (0 == access(filename, R_OK)) {
+        goto process;
+    } else {
+        if (ENOENT == errno) {
+            /* the file does not exist, so give it
+             * a little time to see if the server
+             * is still starting up */
+            retries = 0;
+            do {
+                ++retries;
+                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                    "WAITING FOR CONNECTION FILE %s", filename);
+                PMIX_CONSTRUCT_LOCK(&lock);
+                if (0 < pmix_ptl_base.wait_to_connect) {
+                    tv.tv_sec = pmix_ptl_base.wait_to_connect;
+                    tv.tv_usec = 0;
+                    pmix_event_evtimer_set(pmix_globals.evbase, &ev,
+                                           timeout, &lock);
+                    PMIX_POST_OBJECT(&ev);
+                    pmix_event_evtimer_add(&ev, &tv);
+                } else {
+                    tv.tv_sec = 0;
+                    tv.tv_usec = 10000;  // use 0.01 sec as default
+                    pmix_event_evtimer_set(pmix_globals.evbase, &ev,
+                                           timeout, &lock);
+                    PMIX_POST_OBJECT(&ev);
+                    pmix_event_evtimer_add(&ev, &tv);
+                }
+                PMIX_WAIT_THREAD(&lock);
+                PMIX_DESTRUCT_LOCK(&lock);
+                /* coverity[toctou] */
+                if (0 == access(filename, R_OK)) {
+                    goto process;
+                }
+            } while (retries < pmix_ptl_base.max_retries);
+            /* otherwise, it is unreachable */
+        }
+    }
+    return;
+
+  process:
+    fp = fopen(filename, "r");
+    if (NULL == fp) {
+        return;
+    }
+    /* get the URI - might seem crazy, but there is actually
+     * a race condition here where the server may have created
+     * the file but not yet finished writing into it. So give
+     * us a chance to get the required info */
+    for (retries=0; retries < 3; retries++) {
+        srvr = pmix_getline(fp);
+        if (NULL != srvr) {
+            break;
+        }
+        fclose(fp);
+        tv.tv_sec = 0;
+        tv.tv_usec = 10000;  // use 0.01 sec as default
+        pmix_event_evtimer_set(pmix_globals.evbase, &ev,
+                               timeout, &lock);
+        PMIX_POST_OBJECT(&ev);
+        pmix_event_evtimer_add(&ev, &tv);
+        PMIX_WAIT_THREAD(&lock);
+        PMIX_DESTRUCT_LOCK(&lock);
+        fp = fopen(filename, "r");
+        if (NULL == fp) {
+            return;
+        }
+    }
+    if (NULL == srvr) {
+        PMIX_ERROR_LOG(PMIX_ERR_FILE_READ_FAILURE);
+        fclose(fp);
+        return;
+    }
+    rc = pmix_ptl_base_parse_uri(srvr, &nspace, &rank, NULL);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        fclose(fp);
+        if (NULL != nspace) {
+            free(nspace);
+        }
+        return;
+    }
+
+    /* see if we already have this server in our list */
+    PMIX_LIST_FOREACH(iptr, servers, pmix_infolist_t) {
+        /* each item contains an array starting with the server nspace */
+        sdata = (pmix_info_t*)iptr->info.value.data.darray->array;
+        ndata = iptr->info.value.data.darray->size;
+        if (0 == strcmp(sdata[0].value.data.string, nspace) &&
+            sdata[1].value.data.rank == rank) {
+            /* already have this one */
+            fclose(fp);
+            free(srvr);
+            free(nspace);
+            return;
+        }
+    }
+
+    /* begin collecting data for the new entry */
+    PMIX_CONSTRUCT(&mylist, pmix_list_t);
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_SERVER_NSPACE, nspace, PMIX_STRING);
+    pmix_list_append(&mylist, &iptr->super);
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_SERVER_RANK, &rank, PMIX_PROC_RANK);
+    pmix_list_append(&mylist, &iptr->super);
+
+    free(srvr);
+    free(nspace);
+
+    /* see if this file contains the server's version */
+    p2 = pmix_getline(fp);
+    if (NULL == p2) {
+        version = strdup("v2.0");
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "V20 SERVER DETECTED");
+    } else {
+        version = p2;
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "VERSION %s SERVER DETECTED", p2);
+    }
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_VERSION_INFO, version, PMIX_STRING);
+    pmix_list_append(&mylist, &iptr->super);
+    free(p2);
+
+    /* see if the file contains the pid */
+    p2 = pmix_getline(fp);
+    if (NULL == p2) {
+        goto complete;
+    }
+    u32 = strtoul(p2, NULL, 10);
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_SERVER_PIDINFO, &u32, PMIX_UINT32);
+    pmix_list_append(&mylist, &iptr->super);
+    free(p2);
+
+    /* check for uid:gid */
+    p2 = pmix_getline(fp);
+    if (NULL == p2) {
+        goto complete;
+    }
+    /* find the colon */
+    if (NULL == (p = strchr(p2, ':'))) {
+        /* bad format */
+        free(p2);
+        goto complete;
+    }
+    *p = '\0';
+    ++p;
+    u32 = strtoul(p2, NULL, 10);
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_USERID, &u32, PMIX_UINT32);
+    pmix_list_append(&mylist, &iptr->super);
+    u32 = strtoul(p, NULL, 10);
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_GRPID, &u32, PMIX_UINT32);
+    pmix_list_append(&mylist, &iptr->super);
+    free(p2);
+
+    /* check for timestamp */
+    p2 = pmix_getline(fp);
+    if (NULL == p2) {
+        goto complete;
+    }
+    iptr = PMIX_NEW(pmix_infolist_t);
+    PMIX_INFO_LOAD(&iptr->info, PMIX_SERVER_START_TIME, p2, PMIX_STRING);
+    pmix_list_append(&mylist, &iptr->super);
+    free(p2);
+
+  complete:
+    fclose(fp);
+
+    /* convert the list to an array */
+    if (0 < (ndata = pmix_list_get_size(&mylist))) {
+        ians = PMIX_NEW(pmix_infolist_t);
+        PMIX_LOAD_KEY(&ians->info.key, PMIX_SERVER_INFO_ARRAY);
+        ians->info.value.type = PMIX_DATA_ARRAY;
+        PMIX_DATA_ARRAY_CREATE(ians->info.value.data.darray, ndata, PMIX_INFO);
+        sdata = (pmix_info_t*)ians->info.value.data.darray->array;
+        n = 0;
+        PMIX_LIST_FOREACH(iptr, &mylist, pmix_infolist_t) {
+            PMIX_INFO_XFER(&sdata[n], &iptr->info);
+            ++n;
+        }
+        PMIX_LIST_DESTRUCT(&mylist);
+        pmix_list_append(servers, &ians->super);
+    }
+}
+
+static void query_servers(char *dirname, pmix_list_t *servers)
+{
+    char *newdir, *dname;
+    struct stat buf;
+    DIR *cur_dirp;
+    struct dirent *dir_entry;
+
+    /* search the system tmpdir directory tree for files
+     * beginning with "pmix." as these can be potential
+     * servers */
+
+    if (NULL == dirname) {
+        /* we first check the system tmpdir to see if a system-level
+         * server is present */
+        dname = pmix_ptl_base.system_tmpdir;
+    } else {
+        dname = dirname;
+    }
+    cur_dirp = opendir(dname);
+    if (NULL == cur_dirp) {
+        return;
+    }
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "pmix:tcp: searching directory %s",
+                        (NULL == dirname) ? pmix_ptl_base.system_tmpdir : dirname);
+
+    /* search the entries for something that starts with the "pmix." prefix */
+    while (NULL != (dir_entry = readdir(cur_dirp))) {
+        /* ignore the . and .. entries */
+        if (0 == strcmp(dir_entry->d_name, ".") ||
+            0 == strcmp(dir_entry->d_name, "..")) {
+            continue;
+        }
+        newdir = pmix_os_path(false, dname, dir_entry->d_name, NULL);
+        /* coverity[toctou] */
+        if (-1 == stat(newdir, &buf)) {
+            free(newdir);
+            continue;
+        }
+        /* if it is a directory, down search */
+        if (S_ISDIR(buf.st_mode)) {
+            query_servers(newdir, servers);
+            free(newdir);
+            continue;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "pmix:tcp: checking %s", dir_entry->d_name);
+        /* see if it starts with our prefix */
+        if (0 == strncmp(dir_entry->d_name, "pmix.", strlen("pmix."))) {
+            /* try to read this file */
+            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                "pmix:tcp: reading file %s", newdir);
+            check_server(newdir, servers);
+        }
+        free(newdir);
+    }
+    closedir(cur_dirp);
+}
+
+static void _local_relcb(void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+
+    if (NULL != cd->info) {
+        PMIX_INFO_FREE(cd->info, cd->ninfo);
+    }
+    PMIX_RELEASE(cd);
+}
+
+void pmix_ptl_base_query_servers(int sd, short args, void *cbdata)
+{
+    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
+    pmix_list_t servers;
+    size_t n;
+    pmix_infolist_t *iptr;
+    pmix_status_t rc;
+
+    PMIX_CONSTRUCT(&servers, pmix_list_t);
+
+    query_servers(NULL, &servers);
+
+    /* convert the list to an array of pmix_info_t */
+    cd->ninfo = pmix_list_get_size(&servers);
+    if (0 == cd->ninfo) {
+        rc = PMIX_ERR_NOT_FOUND;
+    } else {
+        PMIX_INFO_CREATE(cd->info, cd->ninfo);
+        n = 0;
+        PMIX_LIST_FOREACH(iptr, &servers, pmix_infolist_t) {
+            PMIX_INFO_XFER(&cd->info[n], &iptr->info);
+            ++n;
+        }
+        rc = PMIX_SUCCESS;
+    }
+    PMIX_LIST_DESTRUCT(&servers);
+
+    /* execute the callback function */
+    cd->cbfunc(rc, cd->info, cd->ninfo, cd->cbdata, _local_relcb, cd);
+}
+
+static void timeout(int sd, short args, void *cbdata)
+{
+    pmix_lock_t *lock = (pmix_lock_t*)cbdata;
+    PMIX_WAKEUP_THREAD(lock);
+}
+
+static char *pmix_getline(FILE *fp)
+{
+    char *ret, *buff;
+    char input[1024];
+
+    ret = fgets(input, 1024, fp);
+    if (NULL != ret) {
+       input[strlen(input)-1] = '\0';  /* remove newline */
+       buff = strdup(input);
+       return buff;
+    }
+
+    return NULL;
+}
+
+/*
+ * Go through a list of argv; if there are any subnet specifications
+ * (a.b.c.d/e), resolve them to an interface name (Currently only
+ * supporting IPv4).  If unresolvable, warn and remove.
+ */
+char **pmix_ptl_base_split_and_resolve(char **orig_str, char *name)
+{
+    int i, ret, save, if_index;
+    char **argv, *str, *tmp;
+    char if_name[PMIX_IF_NAMESIZE];
+    struct sockaddr_storage argv_inaddr, if_inaddr;
+    uint32_t argv_prefix;
+
+    /* Sanity check */
+    if (NULL == orig_str || NULL == *orig_str) {
+        return NULL;
+    }
+
+    argv = pmix_argv_split(*orig_str, ',');
+    if (NULL == argv) {
+        return NULL;
+    }
+    for (save = i = 0; NULL != argv[i]; ++i) {
+        if (isalpha(argv[i][0])) {
+            argv[save++] = argv[i];
+            continue;
+        }
+
+        /* Found a subnet notation.  Convert it to an IP
+           address/netmask.  Get the prefix first. */
+        argv_prefix = 0;
+        tmp = strdup(argv[i]);
+        str = strchr(argv[i], '/');
+        if (NULL == str) {
+            pmix_show_help("help-ptl-tool.txt", "invalid if_inexclude",
+                           true, name, tmp, "Invalid specification (missing \"/\")");
+            free(argv[i]);
+            free(tmp);
+            continue;
+        }
+        *str = '\0';
+        argv_prefix = atoi(str + 1);
+
+        /* Now convert the IPv4 address */
+        ((struct sockaddr*) &argv_inaddr)->sa_family = AF_INET;
+        ret = inet_pton(AF_INET, argv[i],
+                        &((struct sockaddr_in*) &argv_inaddr)->sin_addr);
+        free(argv[i]);
+
+        if (1 != ret) {
+            pmix_show_help("help-ptl-tool.txt", "invalid if_inexclude",
+                           true, name, tmp,
+                           "Invalid specification (inet_pton() failed)");
+            free(tmp);
+            continue;
+        }
+        pmix_output_verbose(20, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool: Searching for %s address+prefix: %s / %u",
+                            name,
+                            pmix_net_get_hostname((struct sockaddr*) &argv_inaddr),
+                            argv_prefix);
+
+        /* Go through all interfaces and see if we can find a match */
+        for (if_index = pmix_ifbegin(); if_index >= 0;
+                           if_index = pmix_ifnext(if_index)) {
+            pmix_ifindextoaddr(if_index,
+                               (struct sockaddr*) &if_inaddr,
+                               sizeof(if_inaddr));
+            if (pmix_net_samenetwork((struct sockaddr*) &argv_inaddr,
+                                     (struct sockaddr*) &if_inaddr,
+                                     argv_prefix)) {
+                break;
+            }
+        }
+        /* If we didn't find a match, keep trying */
+        if (if_index < 0) {
+            pmix_show_help("help-ptl-tool.txt", "invalid if_inexclude",
+                           true, name, tmp,
+                           "Did not find interface matching this subnet");
+            free(tmp);
+            continue;
+        }
+
+        /* We found a match; get the name and replace it in the
+           argv */
+        pmix_ifindextoname(if_index, if_name, sizeof(if_name));
+        pmix_output_verbose(20, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool: Found match: %s (%s)",
+                            pmix_net_get_hostname((struct sockaddr*) &if_inaddr),
+                            if_name);
+        argv[save++] = strdup(if_name);
+        free(tmp);
+    }
+
+    /* The list may have been compressed if there were invalid
+       entries, so ensure we end it with a NULL entry */
+    argv[save] = NULL;
+    free(*orig_str);
+    *orig_str = pmix_argv_join(argv, ',');
+    return argv;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_frame.c pmix-4.0.0/src/mca/ptl/base/ptl_base_frame.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_frame.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_frame.c	2021-01-02 08:56:17.000000000 +0000
@@ -43,6 +43,12 @@
 #include "src/mca/base/pmix_mca_base_framework.h"
 #include "src/class/pmix_list.h"
 #include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/util/error.h"
+#include "src/util/os_dirpath.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/show_help.h"
+
 #include "src/mca/ptl/ptl_types.h"
 #include "src/mca/ptl/base/base.h"
 
@@ -57,13 +63,16 @@
 #define PMIX_MAX_MSG_SIZE   16
 
 /* Instantiate the global vars */
-pmix_ptl_globals_t pmix_ptl_globals = {{{0}}};
+pmix_ptl_base_t pmix_ptl_base = {0};
 int pmix_ptl_base_output = -1;
+pmix_ptl_module_t pmix_ptl = {0};
 
 static size_t max_msg_size = PMIX_MAX_MSG_SIZE;
 
 static int pmix_ptl_register(pmix_mca_base_register_flag_t flags)
 {
+    int idx;
+
     (void)flags;
     pmix_mca_base_var_register("pmix", "ptl", "base", "max_msg_size",
                                "Max size (in Mbytes) of a client/server msg",
@@ -71,17 +80,119 @@
                                PMIX_INFO_LVL_2,
                                PMIX_MCA_BASE_VAR_SCOPE_READONLY,
                                &max_msg_size);
-    pmix_ptl_globals.max_msg_size = max_msg_size * 1024 * 1024;
+    pmix_ptl_base.max_msg_size = max_msg_size * 1024 * 1024;
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "if_include",
+                                     "Comma-delimited list of devices and/or CIDR notation of TCP networks "
+                                     "(e.g., \"eth0,192.168.0.0/16\").  Mutually exclusive with ptl_tcp_if_exclude.",
+                                     PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                     PMIX_INFO_LVL_2,
+                                     PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                     &pmix_ptl_base.if_include);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "if_include", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "if_exclude",
+                                     "Comma-delimited list of devices and/or CIDR notation of TCP networks to NOT use "
+                                     "-- all devices not matching these specifications will be used (e.g., \"eth0,192.168.0.0/16\"). "
+                                     "If set to a non-default value, it is mutually exclusive with ptl_tcp_if_include.",
+                                     PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                     PMIX_INFO_LVL_2,
+                                     PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                     &pmix_ptl_base.if_exclude);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "if_exclude", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    /* if_include and if_exclude need to be mutually exclusive */
+    if (NULL != pmix_ptl_base.if_include &&
+        NULL != pmix_ptl_base.if_exclude) {
+        pmix_show_help("help-ptl-base.txt", "include-exclude", true,
+                       pmix_ptl_base.if_include,
+                       pmix_ptl_base.if_exclude);
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "ipv4_port",
+                                     "IPv4 port to be used",
+                                     PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.ipv4_port);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "ipv4_port", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "ipv6_port",
+                                     "IPv6 port to be used",
+                                     PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.ipv6_port);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "ipv6_port", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "disable_ipv4_family",
+                                     "Disable the IPv4 interfaces",
+                                     PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.disable_ipv4_family);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "disable_ipv4_family", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "disable_ipv6_family",
+                                     "Disable the IPv6 interfaces",
+                                     PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.disable_ipv6_family);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "disable_ipv6_family", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "connection_wait_time",
+                                     "Number of seconds to wait for the server connection file to appear",
+                                     PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.wait_to_connect);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "connection_wait_time", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "max_retries",
+                                     "Number of times to look for the connection file before quitting",
+                                     PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.max_retries);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "max_retries", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "handshake_wait_time",
+                                     "Number of seconds to wait for the server reply to the handshake request",
+                                     PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.handshake_wait_time);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "handshake_wait_time", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "handshake_max_retries",
+                                     "Number of times to retry the handshake request before giving up",
+                                     PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                     PMIX_INFO_LVL_4,
+                                     PMIX_MCA_BASE_VAR_SCOPE_READONLY,
+                                     &pmix_ptl_base.handshake_max_retries);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "handshake_max_retries", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    idx = pmix_mca_base_var_register("pmix", "ptl", "base", "report_uri",
+                                     "Output URI [- => stdout, + => stderr, or filename]",
+                                     PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                     PMIX_INFO_LVL_2,
+                                     PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
+                                     &pmix_ptl_base.report_uri);
+    (void)pmix_mca_base_var_register_synonym(idx, "pmix", "ptl", "tcp", "report_uri", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+
     return PMIX_SUCCESS;
 }
 
 static pmix_status_t pmix_ptl_close(void)
 {
-    if (!pmix_ptl_globals.initialized) {
+    if (!pmix_ptl_base.initialized) {
         return PMIX_SUCCESS;
     }
-    pmix_ptl_globals.initialized = false;
-    pmix_ptl_globals.selected = false;
+    pmix_ptl_base.initialized = false;
+    pmix_ptl_base.selected = false;
 
     /* ensure the listen thread has been shut down */
     pmix_ptl_base_stop_listening();
@@ -93,11 +204,67 @@
         }
     }
 
-    /* the components will cleanup when closed */
-    PMIX_LIST_DESTRUCT(&pmix_ptl_globals.actives);
-    PMIX_LIST_DESTRUCT(&pmix_ptl_globals.posted_recvs);
-    PMIX_LIST_DESTRUCT(&pmix_ptl_globals.unexpected_msgs);
-    PMIX_LIST_DESTRUCT(&pmix_ptl_globals.listeners);
+    /* the component will cleanup when closed */
+    PMIX_LIST_DESTRUCT(&pmix_ptl_base.posted_recvs);
+    PMIX_LIST_DESTRUCT(&pmix_ptl_base.unexpected_msgs);
+    PMIX_DESTRUCT(&pmix_ptl_base.listener);
+
+    if (NULL != pmix_ptl_base.system_filename) {
+        if (pmix_ptl_base.created_system_filename) {
+            remove(pmix_ptl_base.system_filename);
+        }
+        free(pmix_ptl_base.system_filename);
+    }
+    if (NULL != pmix_ptl_base.session_filename) {
+        if (pmix_ptl_base.created_session_filename) {
+            remove(pmix_ptl_base.session_filename);
+        }
+        free(pmix_ptl_base.session_filename);
+    }
+    if (NULL != pmix_ptl_base.nspace_filename) {
+        if (pmix_ptl_base.created_nspace_filename) {
+            remove(pmix_ptl_base.nspace_filename);
+        }
+        free(pmix_ptl_base.nspace_filename);
+    }
+    if (NULL != pmix_ptl_base.pid_filename) {
+        if (pmix_ptl_base.created_pid_filename) {
+            remove(pmix_ptl_base.pid_filename);
+        }
+        free(pmix_ptl_base.pid_filename);
+    }
+    if (NULL != pmix_ptl_base.rendezvous_filename) {
+        if (pmix_ptl_base.created_rendezvous_file) {
+            remove(pmix_ptl_base.rendezvous_filename);
+        }
+        free(pmix_ptl_base.rendezvous_filename);
+    }
+    if (NULL != pmix_ptl_base.uri) {
+        free(pmix_ptl_base.uri);
+    }
+    if (NULL != pmix_ptl_base.urifile) {
+        if (pmix_ptl_base.created_urifile) {
+            /* remove the file */
+            remove(pmix_ptl_base.urifile);
+        }
+        free(pmix_ptl_base.urifile);
+        pmix_ptl_base.urifile = NULL;
+    }
+    if (NULL != pmix_ptl_base.session_tmpdir) {
+        /* if I created the session tmpdir, then remove it if empty */
+        if (pmix_ptl_base.created_session_tmpdir) {
+            pmix_os_dirpath_destroy(pmix_ptl_base.session_tmpdir,
+                                    true, NULL);
+        }
+        free(pmix_ptl_base.session_tmpdir);
+    }
+    if (NULL != pmix_ptl_base.system_tmpdir) {
+        if (pmix_ptl_base.created_system_tmpdir) {
+            pmix_os_dirpath_destroy(pmix_ptl_base.system_tmpdir,
+                                    true, NULL);
+        }
+        free(pmix_ptl_base.system_tmpdir);
+    }
 
     return pmix_mca_base_framework_components_close(&pmix_ptl_base_framework, NULL);
 }
@@ -105,15 +272,52 @@
 static pmix_status_t pmix_ptl_open(pmix_mca_base_open_flag_t flags)
 {
     pmix_status_t rc;
+    char *tdir;
 
     /* initialize globals */
-    pmix_ptl_globals.initialized = true;
-    PMIX_CONSTRUCT(&pmix_ptl_globals.actives, pmix_list_t);
-    PMIX_CONSTRUCT(&pmix_ptl_globals.posted_recvs, pmix_list_t);
-    PMIX_CONSTRUCT(&pmix_ptl_globals.unexpected_msgs, pmix_list_t);
-    pmix_ptl_globals.listen_thread_active = false;
-    PMIX_CONSTRUCT(&pmix_ptl_globals.listeners, pmix_list_t);
-    pmix_ptl_globals.current_tag = PMIX_PTL_TAG_DYNAMIC;
+    pmix_ptl_base.initialized = true;
+    PMIX_CONSTRUCT(&pmix_ptl_base.posted_recvs, pmix_list_t);
+    PMIX_CONSTRUCT(&pmix_ptl_base.unexpected_msgs, pmix_list_t);
+    pmix_ptl_base.listen_thread_active = false;
+    PMIX_CONSTRUCT(&pmix_ptl_base.listener, pmix_listener_t);
+    pmix_ptl_base.current_tag = PMIX_PTL_TAG_DYNAMIC;
+
+    /* check for environ-based directives
+     * on system tmpdir to use */
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
+        PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        pmix_ptl_base.session_tmpdir = strdup(pmix_server_globals.tmpdir);
+    } else {
+        if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) {
+            pmix_ptl_base.session_tmpdir = strdup(tdir);
+        } else {
+            pmix_ptl_base.session_tmpdir = strdup(pmix_tmp_directory());
+        }
+    }
+
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
+        PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        pmix_ptl_base.system_tmpdir = strdup(pmix_server_globals.system_tmpdir);
+    } else {
+        if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) {
+            pmix_ptl_base.system_tmpdir = strdup(tdir);
+        } else {
+            pmix_ptl_base.system_tmpdir = strdup(pmix_tmp_directory());
+        }
+    }
+
+    if (NULL != pmix_ptl_base.report_uri &&
+        0 != strcmp(pmix_ptl_base.report_uri, "-") &&
+        0 != strcmp(pmix_ptl_base.report_uri, "+")) {
+        pmix_ptl_base.urifile = strdup(pmix_ptl_base.report_uri);
+    }
+
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
+        PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        if (NULL != (tdir = getenv("PMIX_LAUNCHER_RENDEZVOUS_FILE"))) {
+            pmix_ptl_base.rendezvous_filename = strdup(tdir);
+        }
+    }
 
     /* Open up all available components */
     rc = pmix_mca_base_framework_components_open(&pmix_ptl_base_framework, flags);
@@ -126,10 +330,6 @@
                                 mca_ptl_base_static_components, 0);
 
 /***   INSTANTIATE INTERNAL CLASSES   ***/
-PMIX_CLASS_INSTANCE(pmix_ptl_base_active_t,
-                    pmix_list_item_t,
-                    NULL, NULL);
-
 static void scon(pmix_ptl_send_t *p)
 {
     memset(&p->hdr, 0, sizeof(pmix_ptl_hdr_t));
@@ -202,26 +402,29 @@
 static void pccon(pmix_pending_connection_t *p)
 {
     p->need_id = false;
-    memset(p->nspace, 0, PMIX_MAX_NSLEN+1);
+    PMIX_LOAD_PROCID(&p->proc, NULL, PMIX_RANK_UNDEF);
     p->info = NULL;
     p->ninfo = 0;
     p->peer = NULL;
+    p->version = NULL;
     p->bfrops = NULL;
     p->psec = NULL;
     p->gds = NULL;
-    p->ptl = NULL;
     p->cred = NULL;
     p->proc_type.type = PMIX_PROC_UNDEF;
     p->proc_type.major = PMIX_MAJOR_WILDCARD;
     p->proc_type.minor = PMIX_MINOR_WILDCARD;
     p->proc_type.release = PMIX_RELEASE_WILDCARD;
-    p->proc_type.padding = 0;
+    p->proc_type.flag = 0;
 }
 static void pcdes(pmix_pending_connection_t *p)
 {
     if (NULL != p->info) {
         PMIX_INFO_FREE(p->info, p->ninfo);
     }
+    if (NULL != p->version) {
+        free(p->version);
+    }
     if (NULL != p->bfrops) {
         free(p->bfrops);
     }
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_handshake.h pmix-4.0.0/src/mca/ptl/base/ptl_base_handshake.h
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_handshake.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_handshake.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,169 @@
+/* -*- C -*-
+ *
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+#ifndef PMIX_PTL_BASE_HANDSHAKE_H_
+#define PMIX_PTL_BASE_HANDSHAKE_H_
+
+#include "src/include/pmix_config.h"
+
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+
+/* The following macros are for use in the PTL connection
+ * handler. For simplicity, they include variables that are
+ * defined in the functions where they are used - thus, these
+ * macros are NOT portable for use elsewhere
+ *
+ * A symmetric pair of macros are provided specifically to
+ * make it easier to read and compare the two ends of the
+ * handshake */
+
+BEGIN_C_DECLS
+
+/* The following macros are used in the ptl_base_connection_hdlr.c
+ * file to parse the handshake message and extract its fields */
+
+#define PMIX_PTL_GET_STRING(n)                  \
+    do {                                        \
+        size_t _l;                              \
+        PMIX_STRNLEN(_l, mg, cnt);              \
+        if (_l < cnt) {                         \
+            (n) = strdup(mg);                   \
+            mg += strlen((n)) + 1;              \
+            cnt -= strlen((n)) + 1;             \
+        } else {                                \
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); \
+            goto error;                         \
+        }                                       \
+    } while(0)
+
+/* numerical value must be converted back to host
+ * order, so we use an intermediate storage place
+ * for that purpose */
+#define PMIX_PTL_GET_U32(n)                     \
+    do {                                        \
+        uint32_t _u;                            \
+        if (sizeof(uint32_t) <= cnt) {          \
+            memcpy(&_u, mg, sizeof(uint32_t));  \
+            (n) = ntohl(_u);                    \
+            mg += sizeof(uint32_t);             \
+            cnt -= sizeof(uint32_t);            \
+        } else {                                \
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); \
+            goto error;                         \
+        }                                       \
+    } while(0)
+
+#define PMIX_PTL_GET_BLOB(b, l)                 \
+    do {                                        \
+        if (0 < (l)) {                          \
+            (b) = (char*)malloc((l));           \
+            if (NULL == (b)) {                  \
+                PMIX_ERROR_LOG(PMIX_ERR_NOMEM); \
+                goto error;                     \
+            }                                   \
+            memcpy((b), mg, (l));               \
+            mg += (l);                          \
+            cnt -= (l);                         \
+        }                                       \
+    } while(0)
+
+#define PMIX_PTL_GET_U8(n)                      \
+    do {                                        \
+        if (sizeof(uint8_t) <= cnt) {           \
+            memcpy(&(n), mg, sizeof(uint8_t));  \
+            mg += sizeof(uint8_t);              \
+            cnt -= sizeof(uint8_t);             \
+        } else {                                \
+            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); \
+            goto error;                         \
+        }                                       \
+    } while(0)
+
+#define PMIX_PTL_GET_PROCID(p)          \
+    do {                                \
+        char *n;                        \
+        uint32_t r;                     \
+        PMIX_PTL_GET_STRING(n);         \
+        PMIX_PTL_GET_U32(r);            \
+        PMIX_LOAD_PROCID(&(p), n, r);   \
+        free(n);                        \
+    } while(0)
+
+
+
+/* The following macros are for use in the ptl_base_fns.c
+ * when constructing the handshake message. */
+#define PMIX_PTL_PUT_STRING(n)                  \
+    do {                                        \
+        memcpy(msg+csize, (n), strlen((n)));    \
+        csize += strlen((n))+1;                 \
+    } while(0)
+
+/* numerical value must be converted to network byte
+ * order, so we use an intermediate storage place
+ * for that purpose */
+#define PMIX_PTL_PUT_U32(n)                         \
+    do {                                            \
+        uint32_t _u;                                \
+        _u = htonl((uint32_t)(n));                  \
+        memcpy(msg+csize, &_u, sizeof(uint32_t));   \
+        csize += sizeof(uint32_t);                  \
+    } while(0)
+
+#define PMIX_PTL_PUT_BLOB(b, l)             \
+    do {                                    \
+        if (0 < (l)) {                      \
+            memcpy(msg+csize, (b), (l));    \
+            csize += (l);                   \
+        }                                   \
+    } while(0)
+
+#define PMIX_PTL_PUT_U8(n)                          \
+    do {                                            \
+        memcpy(msg+csize, &(n), sizeof(uint8_t));   \
+        csize += sizeof(uint8_t);                   \
+    } while(0)
+
+#define PMIX_PTL_PUT_PROCID(p)              \
+    do {                                    \
+        PMIX_PTL_PUT_STRING((p).nspace);    \
+        PMIX_PTL_PUT_U32((p).rank);         \
+    } while(0)
+
+
+/* the following macros are for use in the ptl_base_fns.c
+ * when sending/recving values during the handshake */
+#define PMIX_PTL_RECV_NSPACE(s, n)                                              \
+    do {                                                                        \
+        pmix_status_t r;                                                        \
+        r = pmix_ptl_base_recv_blocking((s), (char*)(n), PMIX_MAX_NSLEN+1);     \
+        if (PMIX_SUCCESS != r) {                                                \
+            return r;                                                           \
+        }                                                                       \
+    } while(0)
+
+#define PMIX_PTL_RECV_U32(s, n)                                                 \
+    do {                                                                        \
+        pmix_status_t r;                                                        \
+        uint32_t _u;                                                            \
+        r = pmix_ptl_base_recv_blocking((s), (char*)&_u, sizeof(uint32_t));     \
+        if (PMIX_SUCCESS != r) {                                                \
+            return r;                                                           \
+        }                                                                       \
+        (n) = htonl(_u);                                                        \
+    } while(0)
+
+END_C_DECLS
+
+#endif
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_listener.c pmix-4.0.0/src/mca/ptl/base/ptl_base_listener.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_listener.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_listener.c	2021-01-02 08:56:17.000000000 +0000
@@ -48,10 +48,15 @@
 #include <pthread.h>
 
 #include "src/class/pmix_list.h"
+#include "src/util/basename.h"
 #include "src/util/error.h"
 #include "src/util/fd.h"
+#include "src/util/net.h"
 #include "src/util/output.h"
+#include "src/util/os_dirpath.h"
+#include "src/util/pif.h"
 #include "src/util/pmix_environ.h"
+#include "src/util/show_help.h"
 
  #include "src/mca/ptl/base/base.h"
 
@@ -60,88 +65,40 @@
 static pthread_t engine;
 static bool setup_complete = false;
 
-/* Cycle across all available plugins and provide them with
- * an opportunity to register rendezvous points (server-side
- * function). Function is to return PMIX_SUCCESS if at least
- * one rendezvous can be defined. */
-static pmix_status_t setup_listeners(pmix_info_t *info, size_t ninfo, bool *need_listener)
-{
-    pmix_ptl_base_active_t *active;
-    pmix_status_t rc;
-    size_t n;
-    bool single = false;
-
-    if (!pmix_ptl_globals.initialized) {
-        return PMIX_ERR_INIT;
-    }
-
-    /* scan the directives to see if they want only one listener setup */
-    if (NULL != info) {
-        for (n=0; n < ninfo; n++) {
-            if (0 == strncmp(info[n].key, PMIX_SINGLE_LISTENER, PMIX_MAX_KEYLEN)) {
-                single = PMIX_INFO_TRUE(&info[n]);
-                break;
-            }
-        }
-    }
-
-    PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
-        if (NULL != active->component->setup_listener) {
-            rc = active->component->setup_listener(info, ninfo, need_listener);
-            if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) {
-                return rc;
-            }
-            if (single) {
-                return PMIX_SUCCESS;
-            }
-        }
-    }
-    /* we must have at least one listener */
-    if (0 == pmix_list_get_size(&pmix_ptl_globals.listeners)) {
-        return PMIX_ERR_INIT;
-    }
-    return PMIX_SUCCESS;
-}
-
 /*
  * start listening thread
  */
-pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo)
+pmix_status_t pmix_ptl_base_start_listening(pmix_info_t info[], size_t ninfo)
 {
     pmix_status_t rc;
-    bool need_listener = false;
 
-    /* setup the listeners */
+    /* setup the listener */
     if (!setup_complete) {
-        if (PMIX_SUCCESS != (rc = setup_listeners(info, ninfo, &need_listener))) {
+        rc = pmix_ptl.setup_listener(info, ninfo);
+        if (PMIX_SUCCESS != rc) {
             return rc;
         }
     }
     setup_complete = true;
 
-    /* if we don't need a listener thread, then we are done */
-    if (!need_listener) {
-        return PMIX_SUCCESS;
-    }
-
     /*** spawn internal listener thread */
-    if (0 > pipe(pmix_ptl_globals.stop_thread)) {
+    if (0 > pipe(pmix_ptl_base.stop_thread)) {
         PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO);
         return PMIX_ERR_OUT_OF_RESOURCE;
     }
     /* Make sure the pipe FDs are set to close-on-exec so that
        they don't leak into children */
-    if (pmix_fd_set_cloexec(pmix_ptl_globals.stop_thread[0]) != PMIX_SUCCESS ||
-        pmix_fd_set_cloexec(pmix_ptl_globals.stop_thread[1]) != PMIX_SUCCESS) {
+    if (pmix_fd_set_cloexec(pmix_ptl_base.stop_thread[0]) != PMIX_SUCCESS ||
+        pmix_fd_set_cloexec(pmix_ptl_base.stop_thread[1]) != PMIX_SUCCESS) {
         PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO);
-        close(pmix_ptl_globals.stop_thread[0]);
-        close(pmix_ptl_globals.stop_thread[1]);
+        close(pmix_ptl_base.stop_thread[0]);
+        close(pmix_ptl_base.stop_thread[1]);
         return PMIX_ERR_OUT_OF_RESOURCE;
     }
     /* fork off the listener thread */
-    pmix_ptl_globals.listen_thread_active = true;
+    pmix_ptl_base.listen_thread_active = true;
     if (0 > pthread_create(&engine, NULL, listen_thread, NULL)) {
-        pmix_ptl_globals.listen_thread_active = false;
+        pmix_ptl_base.listen_thread_active = false;
         return PMIX_ERROR;
     }
 
@@ -151,58 +108,54 @@
 void pmix_ptl_base_stop_listening(void)
 {
     int i;
-    pmix_listener_t *lt;
+    pmix_listener_t *lt = &pmix_ptl_base.listener;
 
     pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
                         "listen_thread: shutdown");
 
-    if (!pmix_ptl_globals.listen_thread_active) {
+    if (!pmix_ptl_base.listen_thread_active) {
         /* nothing we can do */
         return;
     }
 
     /* mark it as inactive */
-    pmix_ptl_globals.listen_thread_active = false;
+    pmix_ptl_base.listen_thread_active = false;
     /* use the block to break it loose just in
      * case the thread is blocked in a call to select for
      * a long time */
     i=1;
-    if (0 > write(pmix_ptl_globals.stop_thread[1], &i, sizeof(int))) {
+    if (0 > write(pmix_ptl_base.stop_thread[1], &i, sizeof(int))) {
         return;
     }
     /* wait for thread to exit */
     pthread_join(engine, NULL);
-    /* close the sockets to remove the connection points */
-    PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
-        CLOSE_THE_SOCKET(lt->socket);
-        lt->socket = -1;
-    }
+    /* close the socket to remove the connection points */
+    CLOSE_THE_SOCKET(lt->socket);
+    lt->socket = -1;
 }
 
 static void* listen_thread(void *obj)
 {
     (void)obj;
-    int rc, max, accepted_connections;
+    int rc, max;
     socklen_t addrlen = sizeof(struct sockaddr_storage);
     pmix_pending_connection_t *pending_connection;
     struct timeval timeout;
     fd_set readfds;
-    pmix_listener_t *lt;
+    pmix_listener_t *lt = &pmix_ptl_base.listener;
 
     pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
                         "listen_thread: active");
 
 
-    while (pmix_ptl_globals.listen_thread_active) {
+    while (pmix_ptl_base.listen_thread_active) {
         FD_ZERO(&readfds);
-        max = -1;
-        PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
-            FD_SET(lt->socket, &readfds);
-            max = (lt->socket > max) ? lt->socket : max;
-        }
+        FD_SET(lt->socket, &readfds);
+        max = lt->socket;
+
         /* add the stop_thread fd */
-        FD_SET(pmix_ptl_globals.stop_thread[0], &readfds);
-        max = (pmix_ptl_globals.stop_thread[0] > max) ? pmix_ptl_globals.stop_thread[0] : max;
+        FD_SET(pmix_ptl_base.stop_thread[0], &readfds);
+        max = (pmix_ptl_base.stop_thread[0] > max) ? pmix_ptl_base.stop_thread[0] : max;
 
         /* set timeout interval */
         timeout.tv_sec = 2;
@@ -212,87 +165,503 @@
          * comes in, we'll get woken up right away.
          */
         rc = select(max + 1, &readfds, NULL, NULL, &timeout);
-        if (!pmix_ptl_globals.listen_thread_active) {
+        if (!pmix_ptl_base.listen_thread_active) {
             /* we've been asked to terminate */
-            close(pmix_ptl_globals.stop_thread[0]);
-            close(pmix_ptl_globals.stop_thread[1]);
+            close(pmix_ptl_base.stop_thread[0]);
+            close(pmix_ptl_base.stop_thread[1]);
             return NULL;
         }
         if (rc < 0) {
             continue;
         }
 
-        /* Spin accepting connections until all active listen sockets
-         * do not have any incoming connections, pushing each connection
-         * onto the event queue for processing
+
+        /* according to the man pages, select replaces the given descriptor
+         * set with a subset consisting of those descriptors that are ready
+         * for the specified operation - in this case, a read. So we need to
+         * first check to see if this file descriptor is included in the
+         * returned subset
          */
-        do {
-            accepted_connections = 0;
-            PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
-
-                /* according to the man pages, select replaces the given descriptor
-                 * set with a subset consisting of those descriptors that are ready
-                 * for the specified operation - in this case, a read. So we need to
-                 * first check to see if this file descriptor is included in the
-                 * returned subset
-                 */
-                if (0 == FD_ISSET(lt->socket, &readfds)) {
-                    /* this descriptor is not included */
+        if (0 == FD_ISSET(lt->socket, &readfds)) {
+            /* this descriptor is not included */
+            continue;
+        }
+
+        /* this descriptor is ready to be read, which means a connection
+         * request has been received - so harvest it. All we want to do
+         * here is accept the connection and push the info onto the event
+         * library for subsequent processing - we don't want to actually
+         * process the connection here as it takes too long, and so the
+         * OS might start rejecting connections due to timeout.
+         */
+        pending_connection = PMIX_NEW(pmix_pending_connection_t);
+        pending_connection->protocol = lt->protocol;
+        pmix_event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
+                          EV_WRITE, lt->cbfunc, pending_connection);
+        pending_connection->sd = accept(lt->socket,
+                                        (struct sockaddr*)&(pending_connection->addr),
+                                        &addrlen);
+        if (pending_connection->sd < 0) {
+            PMIX_RELEASE(pending_connection);
+            if (pmix_socket_errno != EAGAIN ||
+                pmix_socket_errno != EWOULDBLOCK) {
+                if (EMFILE == pmix_socket_errno ||
+                    ENOBUFS == pmix_socket_errno ||
+                    ENOMEM == pmix_socket_errno) {
+                    PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
+                } else if (EINVAL == pmix_socket_errno ||
+                           EINTR == pmix_socket_errno) {
+                    /* race condition at finalize */
+                    goto done;
+                } else if (ECONNABORTED == pmix_socket_errno) {
+                    /* they aborted the attempt */
                     continue;
+                } else {
+                    pmix_output(0, "listen_thread: accept() failed: %s (%d).",
+                                strerror(pmix_socket_errno), pmix_socket_errno);
                 }
+                goto done;
+            }
+            continue;
+        }
+
+        pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
+                            "listen_thread: new connection: (%d, %d)",
+                            pending_connection->sd, pmix_socket_errno);
+        /* post the object */
+        PMIX_POST_OBJECT(pending_connection);
+        /* activate the event */
+        pmix_event_active(&pending_connection->ev, EV_WRITE, 1);
+    }
+
+ done:
+    pmix_ptl_base.listen_thread_active = false;
+    return NULL;
+}
+
+pmix_status_t pmix_base_write_rndz_file(char *filename, char *uri, bool *created)
+{
+    FILE *fp;
+    char *dirname;
+    time_t mytime;
+
+    dirname = pmix_dirname(filename);
+    if (NULL != dirname) {
+        if (0 != pmix_os_dirpath_access(dirname, 0755)) {
+            if (0 != pmix_os_dirpath_create(dirname, 0755)) {
+                pmix_output(0, "System tmpdir %s could not be created\n", dirname);
+                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
+                free(dirname);
+                return PMIX_ERR_FILE_OPEN_FAILURE;
+            }
+            *created = true;
+        }
+        free(dirname);
+    }
+
+    fp = fopen(filename, "w");
+    if (NULL == fp) {
+        pmix_output(0, "Impossible to open the file %s in write mode\n", filename);
+        PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
+        return PMIX_ERR_FILE_OPEN_FAILURE;
+    }
+
+    /* output the URI */
+    fprintf(fp, "%s\n", uri);
+    /* add the version */
+    fprintf(fp, "%s\n", PMIX_VERSION);
+    /* output our pid */
+    fprintf(fp, "%lu\n", (unsigned long)getpid());
+    /* output our effective uid and gid */
+    fprintf(fp, "%lu:%lu\n", (unsigned long)geteuid(), (unsigned long)getegid());
+    /* output the time */
+    mytime = time(NULL);
+    fprintf(fp, "%s\n", ctime(&mytime));
+    fclose(fp);
+    /* set the file mode */
+    if (0 != chmod(filename, S_IRUSR | S_IWUSR | S_IRGRP)) {
+        PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
+        return PMIX_ERR_FILE_OPEN_FAILURE;
+    }
+    return PMIX_SUCCESS;
+}
+
+/* discover the available
+ * interfaces, filter them thru any given directives, and select
+ * the one we will listen on for connection requests. This will
+ * be a loopback device by default, unless we are asked to support
+ * tool connections - in that case, we will take a non-loopback
+ * device by default, if one is available after filtering directives
+ *
+ * If we are a tool and were give a rendezvous file, then we first
+ * check to see if it already exists. If it does, then this is the
+ * connection info we are to use. If it doesn't, then this is the
+ * name of the file we are to use to store our listener info.
+ *
+ * If we are a server and are given a rendezvous file, then that is
+ * is the name of the file we are to use to store our listener info.
+ */
+pmix_status_t pmix_ptl_base_setup_listener(void)
+{
+    int flags = 0;
+    pmix_listener_t *lt;
+    int i, rc=0, saveindex = -1;
+    char **interfaces = NULL;
+    bool including = false;
+    char name[32];
+    struct sockaddr_storage my_ss;
+    int kindex;
+    pmix_socklen_t addrlen;
+    char *prefix;
+    char myconnhost[PMIX_MAXHOSTNAMELEN] = {0};
+    int myport;
+    pmix_kval_t *urikv;
+    pid_t mypid;
+    pmix_socklen_t socklen;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "ptl:tool setup_listener");
+
+    lt = &pmix_ptl_base.listener;
+
+    /* if interface include was given, construct a list
+     * of those interfaces which match the specifications - remember,
+     * the includes could be given as named interfaces, IP addrs, or
+     * subnet+mask
+     */
+    if (NULL != pmix_ptl_base.if_include) {
+        interfaces = pmix_ptl_base_split_and_resolve(&pmix_ptl_base.if_include, "include");
+        including = true;
+    } else if (NULL != pmix_ptl_base.if_exclude) {
+        interfaces = pmix_ptl_base_split_and_resolve(&pmix_ptl_base.if_exclude, "exclude");
+        including = false;
+    }
+
+    /* look at all available interfaces and pick one - we default to a
+     * loopback interface if available, but otherwise pick the first
+     * available interface since we are only talking locally */
+    for (i = pmix_ifbegin(); i >= 0; i = pmix_ifnext(i)) {
+        if (PMIX_SUCCESS != pmix_ifindextoaddr(i, (struct sockaddr*)&my_ss, sizeof(my_ss))) {
+            pmix_output (0, "ptl_tool: problems getting address for index %i (kernel index %i)\n",
+                         i, pmix_ifindextokindex(i));
+            continue;
+        }
+        /* ignore non-ip4/6 interfaces */
+        if (AF_INET != my_ss.ss_family &&
+            AF_INET6 != my_ss.ss_family) {
+            continue;
+        }
+        /* get the name for diagnostic purposes */
+        pmix_ifindextoname(i, name, sizeof(name));
 
-                /* this descriptor is ready to be read, which means a connection
-                 * request has been received - so harvest it. All we want to do
-                 * here is accept the connection and push the info onto the event
-                 * library for subsequent processing - we don't want to actually
-                 * process the connection here as it takes too long, and so the
-                 * OS might start rejecting connections due to timeout.
-                 */
-                pending_connection = PMIX_NEW(pmix_pending_connection_t);
-                pending_connection->protocol = lt->protocol;
-                pending_connection->ptl = lt->ptl;
-                pmix_event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
-                                  EV_WRITE, lt->cbfunc, pending_connection);
-                pending_connection->sd = accept(lt->socket,
-                                                (struct sockaddr*)&(pending_connection->addr),
-                                                &addrlen);
-                if (pending_connection->sd < 0) {
-                    PMIX_RELEASE(pending_connection);
-                    if (pmix_socket_errno != EAGAIN ||
-                        pmix_socket_errno != EWOULDBLOCK) {
-                        if (EMFILE == pmix_socket_errno ||
-                            ENOBUFS == pmix_socket_errno ||
-                            ENOMEM == pmix_socket_errno) {
-                            PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
-                        } else if (EINVAL == pmix_socket_errno ||
-                                   EINTR == pmix_socket_errno) {
-                            /* race condition at finalize */
-                            goto done;
-                        } else if (ECONNABORTED == pmix_socket_errno) {
-                            /* they aborted the attempt */
-                            continue;
-                        } else {
-                            pmix_output(0, "listen_thread: accept() failed: %s (%d).",
-                                        strerror(pmix_socket_errno), pmix_socket_errno);
-                        }
-                        goto done;
-                    }
+        /* ignore any virtual interfaces */
+        if (0 == strncmp(name, "vir", 3)) {
+            continue;
+        }
+        /* ignore any interfaces in a disabled family */
+        if (AF_INET == my_ss.ss_family &&
+            pmix_ptl_base.disable_ipv4_family) {
+            continue;
+        } else if (AF_INET6 == my_ss.ss_family &&
+                   pmix_ptl_base.disable_ipv6_family) {
+            continue;
+        }
+        /* get the kernel index */
+        kindex = pmix_ifindextokindex(i);
+        if (kindex <= 0) {
+            continue;
+        }
+        pmix_output_verbose(10, pmix_ptl_base_framework.framework_output,
+                            "WORKING INTERFACE %d KERNEL INDEX %d FAMILY: %s", i, kindex,
+                            (AF_INET == my_ss.ss_family) ? "V4" : "V6");
+        /* handle include/exclude directives */
+        if (NULL != interfaces) {
+            /* check for match */
+            rc = pmix_ifmatches(kindex, interfaces);
+            /* if one of the network specifications isn't parseable, then
+             * error out as we can't do what was requested
+             */
+            if (PMIX_ERR_FABRIC_NOT_PARSEABLE == rc) {
+                pmix_show_help("help-ptl-tool.txt", "not-parseable", true);
+                pmix_argv_free(interfaces);
+                return PMIX_ERR_BAD_PARAM;
+            }
+            /* if we are including, then ignore this if not present */
+            if (including) {
+                if (PMIX_SUCCESS != rc) {
+                    pmix_output_verbose(10, pmix_ptl_base_framework.framework_output,
+                                        "ptl:tool:init rejecting interface %s (not in include list)", name);
                     continue;
                 }
+            } else {
+                /* we are excluding, so ignore if present */
+                if (PMIX_SUCCESS == rc) {
+                    pmix_output_verbose(10, pmix_ptl_base_framework.framework_output,
+                                        "ptl:tool:init rejecting interface %s (in exclude list)", name);
+                    continue;
+                }
+            }
+        }
 
-                pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
-                                    "listen_thread: new connection: (%d, %d)",
-                                    pending_connection->sd, pmix_socket_errno);
-                /* post the object */
-                PMIX_POST_OBJECT(pending_connection);
-                /* activate the event */
-                pmix_event_active(&pending_connection->ev, EV_WRITE, 1);
-                accepted_connections++;
+        /* if this is the loopback device and they didn't enable
+         * remote connections, then we are done */
+        if (pmix_ifisloopback(i)) {
+            if (pmix_ptl_base.remote_connections) {
+                /* ignore loopback */
+                continue;
+            } else {
+                pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
+                                    "ptl:tool:init loopback interface %s selected", name);
+                saveindex = i;
+                break;
+            }
+        } else {
+            /* if this is the first one we found, then hang on to it - we
+             * will use it if a loopback device is not found */
+            if (saveindex < 0) {
+                saveindex = i;
             }
-        } while (accepted_connections > 0);
+        }
+    }
+    /* cleanup */
+    if (NULL != interfaces) {
+        pmix_argv_free(interfaces);
+    }
+
+    /* if we didn't find anything, then we cannot operate */
+    if (saveindex < 0) {
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+
+    /* save the connection */
+    if (PMIX_SUCCESS != pmix_ifindextoaddr(saveindex,
+                                           (struct sockaddr*)&pmix_ptl_base.connection,
+                                           sizeof(struct sockaddr))) {
+        pmix_output (0, "ptl:base: problems getting address for kernel index %i\n",
+                     pmix_ifindextokindex(saveindex));
+        return PMIX_ERR_NOT_AVAILABLE;
+    }
+
+    /* set the port */
+    if (AF_INET == pmix_ptl_base.connection.ss_family) {
+        ((struct sockaddr_in*) &pmix_ptl_base.connection)->sin_port = htons(pmix_ptl_base.ipv4_port);
+        if (0 != pmix_ptl_base.ipv4_port) {
+            flags = 1;
+        }
+    } else if (AF_INET6 == pmix_ptl_base.connection.ss_family) {
+        ((struct sockaddr_in6*) &pmix_ptl_base.connection)->sin6_port = htons(pmix_ptl_base.ipv6_port);
+        if (0 != pmix_ptl_base.ipv6_port) {
+            flags = 1;
+        }
     }
 
- done:
-    pmix_ptl_globals.listen_thread_active = false;
-    return NULL;
+    lt->varname = strdup("PMIX_SERVER_URI4:PMIX_SERVER_URI3:PMIX_SERVER_URI2:PMIX_SERVER_URI21");
+    lt->protocol = PMIX_PROTOCOL_V2;
+    lt->cbfunc = pmix_ptl_base_connection_handler;
+
+    addrlen = sizeof(struct sockaddr_storage);
+    /* create a listen socket for incoming connection attempts */
+    lt->socket = socket(pmix_ptl_base.connection.ss_family, SOCK_STREAM, 0);
+    if (lt->socket < 0) {
+        printf("%s:%d socket() failed\n", __FILE__, __LINE__);
+        goto sockerror;
+    }
+
+    /* set reusing ports flag */
+    if (setsockopt (lt->socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&flags, sizeof(flags)) < 0) {
+        pmix_output(0, "ptl:base:create_listen: unable to set the "
+                    "SO_REUSEADDR option (%s:%d)\n",
+                    strerror(pmix_socket_errno), pmix_socket_errno);
+        goto sockerror;
+    }
+
+    /* Set the socket to close-on-exec so that no children inherit
+     * this FD */
+    if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) {
+        goto sockerror;
+    }
+
+#if PMIX_HAVE_APPLE
+    socklen = sizeof(struct sockaddr);
+#else
+    socklen = sizeof(struct sockaddr_storage);
+#endif
+    if (bind(lt->socket, (struct sockaddr*)&pmix_ptl_base.connection, socklen) < 0) {
+        printf("%s:%d bind() failed for socket %d size %u: %s\n",
+               __FILE__, __LINE__, lt->socket, (unsigned)sizeof(struct sockaddr_storage), strerror(errno));
+        goto sockerror;
+    }
+
+    /* resolve assigned port */
+    if (getsockname(lt->socket, (struct sockaddr*)&pmix_ptl_base.connection, &addrlen) < 0) {
+        pmix_output(0, "ptl:tool:create_listen: getsockname(): %s (%d)",
+                    strerror(pmix_socket_errno), pmix_socket_errno);
+        goto sockerror;
+    }
+
+    /* setup listen backlog to maximum allowed by kernel */
+    if (listen(lt->socket, SOMAXCONN) < 0) {
+        printf("%s:%d listen() failed\n", __FILE__, __LINE__);
+        goto sockerror;
+    }
+
+    /* set socket up to be non-blocking, otherwise accept could block */
+    if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) {
+        printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__);
+        goto sockerror;
+    }
+    flags |= O_NONBLOCK;
+    if (fcntl(lt->socket, F_SETFL, flags) < 0) {
+        printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__);
+        goto sockerror;
+    }
+
+    if (AF_INET == pmix_ptl_base.connection.ss_family) {
+        prefix = "tcp4://";
+        myport = ntohs(((struct sockaddr_in*) &pmix_ptl_base.connection)->sin_port);
+        inet_ntop(AF_INET, &((struct sockaddr_in*) &pmix_ptl_base.connection)->sin_addr,
+                  myconnhost, PMIX_MAXHOSTNAMELEN-1);
+    } else if (AF_INET6 == pmix_ptl_base.connection.ss_family) {
+        prefix = "tcp6://";
+        myport = ntohs(((struct sockaddr_in6*) &pmix_ptl_base.connection)->sin6_port);
+        inet_ntop(AF_INET6, &((struct sockaddr_in6*) &pmix_ptl_base.connection)->sin6_addr,
+                  myconnhost, PMIX_MAXHOSTNAMELEN-1);
+    } else {
+        goto sockerror;
+    }
+
+    rc = asprintf(&lt->uri, "%s.%d;%s%s:%d", pmix_globals.myid.nspace, pmix_globals.myid.rank, prefix, myconnhost, myport);
+    if (0 > rc || NULL == lt->uri) {
+        goto sockerror;
+    }
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "ptl:base URI %s", lt->uri);
+
+    /* save the URI internally so we can report it */
+    urikv = PMIX_NEW(pmix_kval_t);
+    urikv->key = strdup(PMIX_SERVER_URI);
+    PMIX_VALUE_CREATE(urikv->value, 1);
+    PMIX_VALUE_LOAD(urikv->value, lt->uri, PMIX_STRING);
+    PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
+                      &pmix_globals.myid, PMIX_INTERNAL,
+                      urikv);
+    PMIX_RELEASE(urikv);  // maintain accounting
+
+    if (NULL != pmix_ptl_base.report_uri) {
+        /* if the string is a "-", then output to stdout */
+        if (0 == strcmp(pmix_ptl_base.report_uri, "-")) {
+            fprintf(stdout, "%s\n", lt->uri);
+        } else if (0 == strcmp(pmix_ptl_base.report_uri, "+")) {
+            /* output to stderr */
+            fprintf(stderr, "%s\n", lt->uri);
+        } else {
+            /* must be a file */
+            FILE *fp;
+            fp = fopen(pmix_ptl_base.report_uri, "w");
+            if (NULL == fp) {
+                pmix_output(0, "Impossible to open the file %s in write mode\n", pmix_ptl_base.report_uri);
+                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
+                goto sockerror;
+            }
+            /* output my nspace and rank plus the URI */
+            fprintf(fp, "%s\n", lt->uri);
+            /* add a flag that indicates we accept v2.1 protocols */
+            fprintf(fp, "v%s\n", PMIX_VERSION);
+            fclose(fp);
+            pmix_ptl_base.created_urifile = true;
+        }
+    }
+
+    /* if we were given a rendezvous file, then drop it */
+    if (NULL != pmix_ptl_base.rendezvous_filename) {
+        /* if we are a tool and the file already exists, then we
+         * just use it as providing the rendezvous info for our
+         * server */
+        if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+            struct stat buf;
+            /* coverity[toctou] */
+            if (0 == stat(pmix_ptl_base.rendezvous_filename, &buf)) {
+                goto nextstep;
+            }
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "WRITING RENDEZVOUS FILE %s",
+                            pmix_ptl_base.rendezvous_filename);
+        rc = pmix_base_write_rndz_file(pmix_ptl_base.rendezvous_filename, lt->uri,
+                                       &pmix_ptl_base.created_rendezvous_file);
+        if (PMIX_SUCCESS != rc) {
+            goto sockerror;
+        }
+    }
+
+  nextstep:
+    /* if we are going to support tools, then drop contact file(s) */
+    if (pmix_ptl_base.system_tool) {
+        if (0 > asprintf(&pmix_ptl_base.system_filename, "%s/pmix.sys.%s",
+                         pmix_ptl_base.system_tmpdir, pmix_globals.hostname)) {
+            goto sockerror;
+        }
+        rc = pmix_base_write_rndz_file(pmix_ptl_base.system_filename, lt->uri,
+                                       &pmix_ptl_base.created_system_tmpdir);
+        if (PMIX_SUCCESS != rc) {
+            goto sockerror;
+        }
+        pmix_ptl_base.created_system_filename = true;
+    }
+
+    if (pmix_ptl_base.session_tool) {
+        /* first output to a std file */
+        if (0 > asprintf(&pmix_ptl_base.session_filename, "%s/pmix.%s.tool",
+                         pmix_ptl_base.session_tmpdir, pmix_globals.hostname)) {
+            goto sockerror;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "WRITING SESSION TOOL FILE %s",
+                            pmix_ptl_base.session_filename);
+        rc = pmix_base_write_rndz_file(pmix_ptl_base.session_filename, lt->uri,
+                                       &pmix_ptl_base.created_session_tmpdir);
+        if (PMIX_SUCCESS != rc) {
+            goto sockerror;
+        }
+        pmix_ptl_base.created_session_filename = true;
+    }
+
+    if (pmix_ptl_base.tool_support) {
+        /* now output to a file based on pid */
+        mypid = getpid();
+        if (0 > asprintf(&pmix_ptl_base.pid_filename, "%s/pmix.%s.tool.%d",
+                         pmix_ptl_base.session_tmpdir, pmix_globals.hostname, mypid)) {
+            goto sockerror;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "WRITING PID TOOL FILE %s",
+                            pmix_ptl_base.pid_filename);
+        rc = pmix_base_write_rndz_file(pmix_ptl_base.pid_filename, lt->uri,
+                                       &pmix_ptl_base.created_session_tmpdir);
+        if (PMIX_SUCCESS != rc) {
+            goto sockerror;
+        }
+        pmix_ptl_base.created_pid_filename = true;
+
+        /* now output it into a file based on my nspace */
+        if (0 > asprintf(&pmix_ptl_base.nspace_filename, "%s/pmix.%s.tool.%s",
+                         pmix_ptl_base.session_tmpdir, pmix_globals.hostname, pmix_globals.myid.nspace)) {
+            goto sockerror;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "WRITING NSPACE TOOL FILE %s",
+                            pmix_ptl_base.nspace_filename);
+        rc = pmix_base_write_rndz_file(pmix_ptl_base.nspace_filename, lt->uri,
+                                       &pmix_ptl_base.created_session_tmpdir);
+        if (PMIX_SUCCESS != rc) {
+            goto sockerror;
+        }
+        pmix_ptl_base.created_nspace_filename = true;
+    }
+
+    return PMIX_SUCCESS;
+
+  sockerror:
+    CLOSE_THE_SOCKET(lt->socket);
+    return rc;
 }
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_select.c pmix-4.0.0/src/mca/ptl/base/ptl_base_select.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_select.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_select.c	2021-01-02 08:56:17.000000000 +0000
@@ -37,16 +37,16 @@
 {
     pmix_mca_base_component_list_item_t *cli = NULL;
     pmix_ptl_base_component_t *component = NULL;
-    pmix_ptl_base_active_t *newactive, *active;
     pmix_mca_base_module_t *mod;
-    int pri;
-    bool inserted;
+    pmix_ptl_module_t *pmod;
+    int rc, pri, best_pri = -1;;
+    bool inserted = false;
 
-    if (pmix_ptl_globals.selected) {
+    if (pmix_ptl_base.selected) {
         /* ensure we don't do this twice */
         return PMIX_SUCCESS;
     }
-    pmix_ptl_globals.selected = true;
+    pmix_ptl_base.selected = true;
 
     /* Query all available components and ask if they have a module */
     PMIX_LIST_FOREACH(cli, &pmix_ptl_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
@@ -57,46 +57,39 @@
                             component->base.pmix_mca_component_name);
 
         /* get the module for this component */
-        if (PMIX_SUCCESS != component->base.pmix_mca_query_component(&mod, &pri)) {
+        rc =component->base.pmix_mca_query_component(&mod, &pri);
+        if (PMIX_SUCCESS != rc || NULL == mod) {
             continue;
         }
 
-        /* add to our prioritized list of available actives */
-        newactive = PMIX_NEW(pmix_ptl_base_active_t);
-        newactive->pri = component->priority;
-        newactive->component = component;
-        newactive->module = (pmix_ptl_module_t*)mod;
-
-        /* maintain priority order */
-        inserted = false;
-        PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
-            if (newactive->pri > active->pri) {
-                pmix_list_insert_pos(&pmix_ptl_globals.actives,
-                                     &active->super, &newactive->super);
-                inserted = true;
-                break;
-            }
+        /* If we got a module, try to initialize it */
+        pmod = (pmix_ptl_module_t*)mod;
+        if (NULL != pmod->init && PMIX_SUCCESS != pmod->init()) {
+            continue;
         }
-        if (!inserted) {
-            /* must be lowest priority - add to end */
-            pmix_list_append(&pmix_ptl_globals.actives, &newactive->super);
+
+        /* keep only the highest priority module */
+        if (best_pri < pri) {
+            best_pri = pri;
+            /* give any prior module a chance to finalize */
+            if (NULL != pmix_ptl.finalize) {
+                pmix_ptl.finalize();
+            }
+            pmix_ptl = *pmod;
+            inserted = true;
         }
+
     }
 
     /* if no modules were found, then that's an error as we require at least one */
-    if (0 == pmix_list_get_size(&pmix_ptl_globals.actives)) {
+    if (!inserted) {
         pmix_show_help("help-pmix-runtime.txt", "no-plugins", true, "PTL");
         return PMIX_ERR_SILENT;
     }
 
-    if (4 < pmix_output_get_verbosity(pmix_ptl_base_framework.framework_output)) {
-        pmix_output(0, "Final PTL priorities");
-        /* show the prioritized list */
-        PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
-            pmix_output(0, "\tPTL: %s Priority: %d",
-                        active->component->base.pmix_mca_component_name, active->pri);
-        }
-    }
+    pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
+                        "mca:ptl:select: using component %s",
+                        pmix_ptl.name);
 
     return PMIX_SUCCESS;;
 }
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_sendrecv.c pmix-4.0.0/src/mca/ptl/base/ptl_base_sendrecv.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_sendrecv.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_sendrecv.c	2021-01-02 08:56:17.000000000 +0000
@@ -115,15 +115,15 @@
                          * as otherwise the collective will never complete */
                         if (PMIX_FENCENB_CMD == trk->type) {
                             if (NULL != trk->modexcbfunc) {
-                                trk->modexcbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, 0, trk, NULL, NULL);
+                                trk->modexcbfunc(PMIX_ERR_LOST_CONNECTION, NULL, 0, trk, NULL, NULL);
                             }
                         } else if (PMIX_CONNECTNB_CMD == trk->type) {
                             if (NULL != trk->op_cbfunc) {
-                                trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk);
+                                trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION, trk);
                             }
                         } else if (PMIX_DISCONNECTNB_CMD == trk->type) {
                             if (NULL != trk->op_cbfunc) {
-                                trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk);
+                                trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION, trk);
                             }
                         }
                     } else {
@@ -207,7 +207,7 @@
          * one connection we can have */
         pmix_globals.connected = false;
          /* set the public error status */
-        err = PMIX_ERR_LOST_CONNECTION_TO_SERVER;
+        err = PMIX_ERR_LOST_CONNECTION;
         /* it is possible that we have sendrecv's in progress where
          * we are waiting for a response to arrive. Since we have
          * lost connection to the server, that will never happen.
@@ -219,7 +219,7 @@
         /* must set the buffer type so it doesn't fail in unpack */
         buf.type = pmix_client_globals.myserver->nptr->compat.type;
         hdr.nbytes = 0; // initialize the hdr to something safe
-        PMIX_LIST_FOREACH(rcv, &pmix_ptl_globals.posted_recvs, pmix_ptl_posted_recv_t) {
+        PMIX_LIST_FOREACH(rcv, &pmix_ptl_base.posted_recvs, pmix_ptl_posted_recv_t) {
             if (UINT_MAX != rcv->tag && NULL != rcv->cbfunc) {
                 /* construct and load the buffer */
                 hdr.tag = rcv->tag;
@@ -521,10 +521,10 @@
                                     "ptl:base:recv:handler allocate data region of size %lu",
                                     (unsigned long)peer->recv_msg->hdr.nbytes);
                 /* allocate the data region */
-                if (pmix_ptl_globals.max_msg_size < peer->recv_msg->hdr.nbytes) {
+                if (pmix_ptl_base.max_msg_size < peer->recv_msg->hdr.nbytes) {
                     pmix_show_help("help-pmix-runtime.txt", "ptl:msg_size", true,
                                    (unsigned long)peer->recv_msg->hdr.nbytes,
-                                   (unsigned long)pmix_ptl_globals.max_msg_size);
+                                   (unsigned long)pmix_ptl_base.max_msg_size);
                     goto err_close;
                 }
                 peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes);
@@ -631,10 +631,12 @@
     }
 
     pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "[%s:%d] send to %s:%u on tag %d",
+                        "[%s:%d] send to %s:%u of size %u on tag %d",
                         __FILE__, __LINE__,
                         (queue->peer)->info->pname.nspace,
-                        (queue->peer)->info->pname.rank, (queue->tag));
+                        (queue->peer)->info->pname.rank,
+                        (NULL == queue->buf) ? 0 : (unsigned)queue->buf->bytes_used,
+                        (queue->tag));
 
     if (NULL == queue->buf) {
         /* nothing to send? */
@@ -697,11 +699,11 @@
     }
 
     /* take the next tag in the sequence */
-    pmix_ptl_globals.current_tag++;
-    if (UINT32_MAX == pmix_ptl_globals.current_tag ) {
-        pmix_ptl_globals.current_tag = PMIX_PTL_TAG_DYNAMIC;
+    pmix_ptl_base.current_tag++;
+    if (UINT32_MAX == pmix_ptl_base.current_tag ) {
+        pmix_ptl_base.current_tag = PMIX_PTL_TAG_DYNAMIC;
     }
-    tag = pmix_ptl_globals.current_tag;
+    tag = pmix_ptl_base.current_tag;
 
     if (NULL != ms->cbfunc) {
         /* if a callback msg is expected, setup a recv for it */
@@ -715,12 +717,13 @@
         /* add it to the list of recvs - we cannot have unexpected messages
          * in this subsystem as the server never sends us something that
          * we didn't previously request */
-        pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super);
+        pmix_list_prepend(&pmix_ptl_base.posted_recvs, &req->super);
     }
 
     pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "QUEIENG MSG TO SERVER OF SIZE %d",
-                        (int)ms->bfr->bytes_used);
+                        "QUEING MSG TO SERVER %s ON SOCKET %d OF SIZE %d",
+                        PMIX_PNAME_PRINT(&ms->peer->info->pname),
+                        ms->peer->sd, (int)ms->bfr->bytes_used);
     snd = PMIX_NEW(pmix_ptl_send_t);
     snd->hdr.pindex = htonl(pmix_globals.pindex);
     snd->hdr.tag = htonl(tag);
@@ -743,6 +746,7 @@
         PMIX_POST_OBJECT(snd);
         pmix_event_add(&ms->peer->send_event, 0);
     }
+
     /* cleanup */
     PMIX_RELEASE(ms);
     PMIX_POST_OBJECT(snd);
@@ -765,7 +769,7 @@
                         (int)msg->hdr.nbytes, msg->hdr.tag, msg->sd);
 
     /* see if we have a waiting recv for this message */
-    PMIX_LIST_FOREACH(rcv, &pmix_ptl_globals.posted_recvs, pmix_ptl_posted_recv_t) {
+    PMIX_LIST_FOREACH(rcv, &pmix_ptl_base.posted_recvs, pmix_ptl_posted_recv_t) {
         pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
                             "checking msg on tag %u for tag %u",
                             msg->hdr.tag, rcv->tag);
@@ -794,7 +798,7 @@
             }
             /* done with the recv if it is a dynamic tag */
             if (PMIX_PTL_TAG_DYNAMIC <= rcv->tag && UINT_MAX != rcv->tag) {
-                pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super);
+                pmix_list_remove_item(&pmix_ptl_base.posted_recvs, &rcv->super);
                 PMIX_RELEASE(rcv);
             }
             PMIX_RELEASE(msg);
@@ -815,7 +819,7 @@
 
     /* it is possible that someone may post a recv for this message
      * at some point, so we have to hold onto it */
-    pmix_list_append(&pmix_ptl_globals.unexpected_msgs, &msg->super);
+    pmix_list_append(&pmix_ptl_base.unexpected_msgs, &msg->super);
     /* ensure we post the modified object before another thread
      * picks it back up */
     PMIX_POST_OBJECT(msg);
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_stubs.c pmix-4.0.0/src/mca/ptl/base/ptl_base_stubs.c
--- pmix-3.2.2~rc1/src/mca/ptl/base/ptl_base_stubs.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/base/ptl_base_stubs.c	2021-01-02 08:56:17.000000000 +0000
@@ -76,26 +76,6 @@
     return false;
 }
 
-pmix_status_t pmix_ptl_base_setup_fork(const pmix_proc_t *proc, char ***env)
-{
-    pmix_ptl_base_active_t *active;
-    pmix_status_t rc;
-
-    if (!pmix_ptl_globals.initialized) {
-        return PMIX_ERR_INIT;
-    }
-
-    PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
-        if (NULL != active->component->setup_fork) {
-            rc = active->component->setup_fork(proc, env);
-            if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) {
-                return rc;
-            }
-        }
-    }
-    return PMIX_SUCCESS;
-}
-
 pmix_status_t pmix_ptl_base_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc)
 {
     pmix_ptl_posted_recv_t *req;
@@ -113,62 +93,11 @@
     /* add it to the list of recvs - we cannot have unexpected messages
      * in this subsystem as the server never sends us something that
      * we didn't previously request */
-    pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super);
+    pmix_list_prepend(&pmix_ptl_base.posted_recvs, &req->super);
     return PMIX_SUCCESS;
 }
 
-char* pmix_ptl_base_get_available_modules(void)
-{
-    pmix_ptl_base_active_t *active;
-    char **tmp=NULL, *reply=NULL;
-
-    if (!pmix_ptl_globals.initialized) {
-        return NULL;
-    }
-
-    PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
-        pmix_argv_append_nosize(&tmp, active->component->base.pmix_mca_component_name);
-    }
-    if (NULL != tmp) {
-        reply = pmix_argv_join(tmp, ',');
-        pmix_argv_free(tmp);
-    }
-    return reply;
-}
-
-/* return the highest priority module */
-pmix_ptl_module_t* pmix_ptl_base_assign_module(void)
-{
-    pmix_ptl_base_active_t *active;
-
-    if (!pmix_ptl_globals.initialized) {
-        return NULL;
-    }
-
-    active = (pmix_ptl_base_active_t*)pmix_list_get_first(&pmix_ptl_globals.actives);
-    return active->module;
-}
-
-pmix_status_t pmix_ptl_base_connect_to_peer(struct pmix_peer_t *peer,
-                                            pmix_info_t info[], size_t ninfo)
-{
-    pmix_peer_t *pr = (pmix_peer_t*)peer;
-    pmix_ptl_base_active_t *active;
-
-    PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
-        if (NULL != active->module->connect_to_peer) {
-            if (PMIX_SUCCESS == active->module->connect_to_peer(peer, info, ninfo)) {
-                pr->nptr->compat.ptl = active->module;
-                return PMIX_SUCCESS;
-            }
-        }
-    }
-
-    return PMIX_ERR_UNREACH;
-}
-
-
-static void post_recv(int fd, short args, void *cbdata)
+void pmix_ptl_base_post_recv(int fd, short args, void *cbdata)
 {
     (void)fd;
     (void)args;
@@ -180,11 +109,11 @@
                         "posting recv on tag %d", req->tag);
 
     /* add it to the list of recvs */
-    pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super);
+    pmix_list_append(&pmix_ptl_base.posted_recvs, &req->super);
 
     /* now check the unexpected msg queue to see if we already
      * recvd something for it */
-    PMIX_LIST_FOREACH_SAFE(msg, nmsg, &pmix_ptl_globals.unexpected_msgs, pmix_ptl_recv_t) {
+    PMIX_LIST_FOREACH_SAFE(msg, nmsg, &pmix_ptl_base.unexpected_msgs, pmix_ptl_recv_t) {
         if (msg->hdr.tag == req->tag || UINT_MAX == req->tag) {
             if (NULL != req->cbfunc) {
                 /* construct and load the buffer */
@@ -199,43 +128,22 @@
                 req->cbfunc(msg->peer, &msg->hdr, &buf, req->cbdata);
                 PMIX_DESTRUCT(&buf);  // free's the msg data
             }
-            pmix_list_remove_item(&pmix_ptl_globals.unexpected_msgs, &msg->super);
+            pmix_list_remove_item(&pmix_ptl_base.unexpected_msgs, &msg->super);
             PMIX_RELEASE(msg);
         }
     }
 }
 
-pmix_status_t pmix_ptl_base_register_recv(struct pmix_peer_t *peer,
-                                          pmix_ptl_cbfunc_t cbfunc,
-                                          pmix_ptl_tag_t tag)
-{
-    (void)peer;
-    pmix_ptl_posted_recv_t *req;
-
-    req = PMIX_NEW(pmix_ptl_posted_recv_t);
-    if (NULL == req) {
-        return PMIX_ERR_NOMEM;
-    }
-    req->tag = tag;
-    req->cbfunc = cbfunc;
-    /* have to push this into an event so we can add this
-     * to the list of posted recvs */
-    pmix_event_assign(&(req->ev), pmix_globals.evbase, -1,
-                      EV_WRITE, post_recv, req);
-    pmix_event_active(&(req->ev), EV_WRITE, 1);
-    return PMIX_SUCCESS;
-}
-
-static void cancel_recv(int fd, short args, void *cbdata)
+void pmix_ptl_base_cancel_recv(int fd, short args, void *cbdata)
 {
     (void)fd;
     (void)args;
     pmix_ptl_posted_recv_t *req = (pmix_ptl_posted_recv_t*)cbdata;
     pmix_ptl_posted_recv_t *rcv;
 
-    PMIX_LIST_FOREACH(rcv, &pmix_ptl_globals.posted_recvs, pmix_ptl_posted_recv_t) {
+    PMIX_LIST_FOREACH(rcv, &pmix_ptl_base.posted_recvs, pmix_ptl_posted_recv_t) {
         if (rcv->tag == req->tag) {
-            pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super);
+            pmix_list_remove_item(&pmix_ptl_base.posted_recvs, &rcv->super);
             PMIX_RELEASE(rcv);
             PMIX_RELEASE(req);
             return;
@@ -243,22 +151,3 @@
     }
     PMIX_RELEASE(req);
 }
-
-pmix_status_t pmix_ptl_base_cancel_recv(struct pmix_peer_t *peer,
-                                        pmix_ptl_tag_t tag)
-{
-    (void)peer;
-    pmix_ptl_posted_recv_t *req;
-
-    req = PMIX_NEW(pmix_ptl_posted_recv_t);
-    if (NULL == req) {
-        return PMIX_ERR_NOMEM;
-    }
-    req->tag = tag;
-    /* have to push this into an event so we can modify
-     * the list of posted recvs */
-    pmix_event_assign(&(req->ev), pmix_globals.evbase, -1,
-                      EV_WRITE, cancel_recv, req);
-    pmix_event_active(&(req->ev), EV_WRITE, 1);
-    return PMIX_SUCCESS;
-}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/client/Makefile.am pmix-4.0.0/src/mca/ptl/client/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ptl/client/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/client/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = ptl_client.h
+sources = \
+        ptl_client_component.c \
+        ptl_client.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_ptl_client_DSO
+lib =
+lib_sources =
+component = mca_ptl_client.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_ptl_client.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_ptl_client_la_SOURCES = $(component_sources)
+mca_ptl_client_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_ptl_client_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_ptl_client_la_SOURCES = $(lib_sources)
+libmca_ptl_client_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/client/ptl_client.c pmix-4.0.0/src/mca/ptl/client/ptl_client.c
--- pmix-3.2.2~rc1/src/mca/ptl/client/ptl_client.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/client/ptl_client.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "src/include/pmix_globals.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+#ifdef HAVE_SYS_SYSCTL_H
+#include <sys/sysctl.h>
+#endif
+
+#include "src/include/pmix_socket_errno.h"
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/os_path.h"
+#include "src/util/show_help.h"
+#include "src/mca/bfrops/base/base.h"
+#include "src/mca/gds/gds.h"
+
+#include "src/mca/ptl/base/base.h"
+#include "ptl_client.h"
+
+static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
+                                     pmix_info_t *info, size_t ninfo);
+
+pmix_ptl_module_t pmix_ptl_client_module = {
+    .name = "client",
+    .connect_to_peer = connect_to_peer
+};
+
+static pmix_status_t connect_to_peer(struct pmix_peer_t *pr,
+                                     pmix_info_t *info, size_t ninfo)
+{
+    char *evar, *suri = NULL;
+    char *nspace=NULL;
+    pmix_rank_t rank = PMIX_RANK_WILDCARD;
+    char **server_nspace = NULL, *rendfile = NULL;
+    pmix_status_t rc;
+    pmix_peer_t *peer = (pmix_peer_t*)pr;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "ptl:tcp: connecting to server");
+
+    rc = pmix_ptl_base_check_server_uris(peer, &evar);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* the URI consists of the following elements:
+    *    - server nspace.rank
+    *    - ptl rendezvous URI
+    */
+    rc = pmix_ptl_base_parse_uri(evar, &nspace, &rank, &suri);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "ptl:tcp:client attempt connect to %s:%u at %s", nspace, rank, suri);
+
+    rc = pmix_ptl_base_make_connection(peer, suri, NULL, 0);
+    if (PMIX_SUCCESS != rc) {
+        free(nspace);
+        free(suri);
+        return rc;
+    }
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "tcp_peer_try_connect: Connection across to peer %s:%u succeeded",
+                        nspace, rank);
+
+    /* mark the connection as made */
+    pmix_ptl_base_complete_connection(peer, nspace, rank, suri);
+
+    if (NULL != nspace) {
+        free(nspace);
+    }
+    if (NULL != rendfile) {
+        free(rendfile);
+    }
+    if (NULL != suri) {
+        free(suri);
+    }
+    if (NULL != server_nspace) {
+        free(server_nspace);
+    }
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/client/ptl_client_component.c pmix-4.0.0/src/mca/ptl/client/ptl_client_component.c
--- pmix-3.2.2~rc1/src/mca/ptl/client/ptl_client_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/client/ptl_client_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennptlee and The University
+ *                         of Tennptlee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2020 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2019      Mellanox Technologies, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+
+#include "src/mca/ptl/base/base.h"
+#include "src/mca/ptl/client/ptl_client.h"
+
+static int component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+ PMIX_EXPORT pmix_ptl_base_component_t mca_ptl_client_component = {
+    .base = {
+        PMIX_PTL_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "client",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_query_component = component_query
+    },
+    .priority = 50
+};
+
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    /* if I am not a client, then look elsewhere */
+    if (!PMIX_PEER_IS_CLIENT(pmix_globals.mypeer) ||
+        PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        *module = NULL;
+        *priority = 0;
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    *module = (pmix_mca_base_module_t*)&pmix_ptl_client_module;
+    *priority = mca_ptl_client_component.priority;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/client/ptl_client.h pmix-4.0.0/src/mca/ptl/client/ptl_client.h
--- pmix-3.2.2~rc1/src/mca/ptl/client/ptl_client.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/client/ptl_client.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PTL_CLIENT_H
+#define PMIX_PTL_CLIENT_H
+
+#include "src/mca/ptl/ptl.h"
+
+BEGIN_C_DECLS
+
+extern pmix_ptl_base_component_t mca_ptl_client_component;
+
+extern pmix_ptl_module_t pmix_ptl_client_module;
+
+END_C_DECLS
+
+#endif /* PMIX_PTL_CLIENT_H */
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/ptl.h pmix-4.0.0/src/mca/ptl/ptl.h
--- pmix-3.2.2~rc1/src/mca/ptl/ptl.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/ptl.h	2021-01-02 08:56:17.000000000 +0000
@@ -56,60 +56,23 @@
  * (b) using different transports as necessitated by different
  *     environments.
  *
- * This is a mult-select framework - i.e., multiple components
- * are selected and "active" at the same time. The intent is
+ * This is a single-select framework - i.e., only one component
+ * is selected and "active" at a time. The intent is
  * to have one component for each use-case, with the
  * expectation that the community will do its best not to revise
  * communications in manners that expand components to support (a).
  * Thus, new variations should be rare, and only a few components
  * will exist.
- *
- * The framework itself reflects the fact that any given peer
- * will utilize only one messaging method.
- * Thus, once a peer is identified, it will pass its version string
- * to this framework's "assign_module" function, which will then
- * pass it to each component until one returns a module capable of
- * processing the given version. This module is then "attached" to
- * the pmix_peer_t object so it can be used for all subsequent
- * communication to/from that peer.
- *
- * Accordingly, there are two levels of APIs defined for this
- * framework:
- *
- * (a) component level - these allow for init/finalize of the
- *     component, and assignment of a module to a given peer
- *     based on the version that peer is using
- *
- * (b) module level - implement send/recv/etc. Note that the
- *     module only needs to provide those functions that differ
- *     from the base functions - they don't need to duplicate
- *     all that code!
  */
 
 /****    MODULE INTERFACE DEFINITION    ****/
 
-/* initialize an active plugin - note that servers may have
- * multiple active plugins, while clients can only have one */
+/* initialize the active plugin */
 typedef pmix_status_t (*pmix_ptl_init_fn_t)(void);
 
-/* finalize an active plugin */
+/* finalize the active plugin */
 typedef void (*pmix_ptl_finalize_fn_t)(void);
 
-/* (TWO-WAY) send a message to the peer, and get a response delivered
- * to the specified callback function. The buffer will be free'd
- * at the completion of the send, and the cbfunc will be called
- * when the corresponding reply is received */
-typedef pmix_status_t (*pmix_ptl_send_recv_fn_t)(struct pmix_peer_t *peer,
-                                                 pmix_buffer_t *bfr,
-                                                 pmix_ptl_cbfunc_t cbfunc,
-                                                 void *cbdata);
-
-/* (ONE-WAY) send a message to the peer. The buffer will be free'd
- * at the completion of the send */
-typedef pmix_status_t (*pmix_ptl_send_fn_t)(struct pmix_peer_t *peer,
-                                            pmix_buffer_t *bfr,
-                                            pmix_ptl_tag_t tag);
-
 /* (ONE-WAY) register a persistent recv */
 typedef pmix_status_t (*pmix_ptl_recv_fn_t)(struct pmix_peer_t *peer,
                                             pmix_ptl_cbfunc_t cbfunc,
@@ -120,74 +83,124 @@
                                               pmix_ptl_tag_t tag);
 
 /* connect to a peer - this is a blocking function
- * to establish a connection to a peer. It assigns
- * the corresponding module to the peer's compat
- * structure for future use */
+ * to establish a connection to a peer*/
 typedef pmix_status_t (*pmix_ptl_connect_to_peer_fn_t)(struct pmix_peer_t *peer,
                                                        pmix_info_t info[], size_t ninfo);
 
 
+/* query available servers on the local node */
+typedef void (*pmix_ptl_query_servers_fn_t)(char *dirname, pmix_list_t *servers);
+
+/* define an API for establishing a
+ * communication rendezvous point for local procs. The active component
+ * is given an opportunity to register a listener with the
+ * PTL base */
+typedef pmix_status_t (*pmix_ptl_setup_listener_fn_t)(pmix_info_t info[], size_t ninfo);
+
+/* define an API for obtaining any envars that are to
+ * be passed to client procs upon fork */
+typedef pmix_status_t (*pmix_ptl_setup_fork_fn_t)(const pmix_proc_t *proc, char ***env);
+
 
 /**
  * Base structure for a PTL module
  */
 struct pmix_ptl_module_t {
+    char *name;
     pmix_ptl_init_fn_t                  init;
     pmix_ptl_finalize_fn_t              finalize;
-    pmix_ptl_send_recv_fn_t             send_recv;
-    pmix_ptl_send_fn_t                  send;
     pmix_ptl_recv_fn_t                  recv;
     pmix_ptl_cancel_fn_t                cancel;
     pmix_ptl_connect_to_peer_fn_t       connect_to_peer;
+    pmix_ptl_query_servers_fn_t         query_servers;
+    pmix_ptl_setup_listener_fn_t        setup_listener;
+    pmix_ptl_setup_fork_fn_t            setup_fork;
 };
 typedef struct pmix_ptl_module_t pmix_ptl_module_t;
 
 
 /*****    MACROS FOR EXECUTING PTL FUNCTIONS    *****/
-#define PMIX_PTL_SEND_RECV(r, p, b, c, d)                                               \
-    do {                                                                                \
-        if ((p)->finalized) {                                                           \
-            (r) = PMIX_ERR_UNREACH;                                                     \
-        } else {                                                                        \
-            (r) = (p)->nptr->compat.ptl->send_recv((struct pmix_peer_t*)(p), b, c, d);  \
+
+/* (TWO-WAY) send a message to the peer, and get a response delivered
+ * to the specified callback function. The buffer will be free'd
+ * at the completion of the send, and the cbfunc will be called
+ * when the corresponding reply is received */
+#define PMIX_PTL_SEND_RECV(r, p, b, c, d)                       \
+    do {                                                        \
+        pmix_ptl_sr_t *ms;                                      \
+        pmix_peer_t *pr = (pmix_peer_t*)(p);                    \
+        if ((p)->finalized) {                                   \
+            (r) = PMIX_ERR_UNREACH;                             \
+        } else {                                                \
+                ms = PMIX_NEW(pmix_ptl_sr_t);                   \
+                PMIX_RETAIN(pr);                                \
+                ms->peer = pr;                                  \
+                ms->bfr = (b);                                  \
+                ms->cbfunc = (c);                               \
+                ms->cbdata = (d);                               \
+                PMIX_THREADSHIFT(ms, pmix_ptl_base_send_recv);  \
+                (r) = PMIX_SUCCESS;                             \
         }                                                                               \
     } while(0)
 
-#define PMIX_PTL_SEND_ONEWAY(r, p, b, t)                                        \
-    do {                                                                        \
-        if ((p)->finalized) {                                                   \
-            (r) = PMIX_ERR_UNREACH;                                             \
-        } else {                                                                \
-            (r) = (p)->nptr->compat.ptl->send((struct pmix_peer_t*)(p), b, t);  \
-        }                                                                       \
+/* (ONE-WAY) send a message to the peer. The buffer will be free'd
+ * at the completion of the send */
+#define PMIX_PTL_SEND_ONEWAY(r, p, b, t)                \
+    do {                                                \
+        pmix_ptl_queue_t *q;                            \
+        pmix_peer_t *pr = (pmix_peer_t*)(p);            \
+        if ((p)->finalized) {                           \
+            (r) = PMIX_ERR_UNREACH;                     \
+        } else {                                        \
+            q = PMIX_NEW(pmix_ptl_queue_t);             \
+            PMIX_RETAIN(pr);                            \
+            q->peer = pr;                               \
+            q->buf = (b);                               \
+            q->tag = (t);                               \
+            PMIX_THREADSHIFT(q, pmix_ptl_base_send);    \
+            (r) = PMIX_SUCCESS;                         \
+        }                                               \
     } while(0)
 
-#define PMIX_PTL_RECV(r, p, c, t)      \
-    (r) = (p)->nptr->compat.ptl->recv((struct pmix_peer_t*)(p), c, t)
-
-#define PMIX_PTL_CANCEL(r, p, t)                        \
-    (r) = (p)->nptr->compat.ptl->cancel((struct pmix_peer_t*)(p), t)
+#define PMIX_PTL_RECV(r, c, t)                                          \
+    do {                                                                \
+        pmix_ptl_posted_recv_t *req;                                    \
+        req = PMIX_NEW(pmix_ptl_posted_recv_t);                         \
+        if (NULL == req) {                                              \
+            (r) = PMIX_ERR_NOMEM;                                       \
+        } else {                                                        \
+            req->tag = (t);                                             \
+            req->cbfunc = (c);                                          \
+            pmix_event_assign(&(req->ev), pmix_globals.evbase, -1,      \
+                              EV_WRITE, pmix_ptl_base_post_recv, req);  \
+            pmix_event_active(&(req->ev), EV_WRITE, 1);                 \
+            (r) = PMIX_SUCCESS;                                         \
+        }                                                               \
+    } while(0)
 
-PMIX_EXPORT extern pmix_status_t pmix_ptl_base_connect_to_peer(struct pmix_peer_t* peer,
-                                                               pmix_info_t info[], size_t ninfo);
+#define PMIX_PTL_CANCEL(r, t)                                               \
+    do {                                                                    \
+        pmix_ptl_posted_recv_t *req;                                        \
+        req = PMIX_NEW(pmix_ptl_posted_recv_t);                             \
+        if (NULL == req) {                                                  \
+            (r) = PMIX_ERR_NOMEM;                                           \
+        } else {                                                            \
+            req->tag = (t);                                                 \
+            pmix_event_assign(&(req->ev), pmix_globals.evbase, -1,          \
+                              EV_WRITE, pmix_ptl_base_cancel_recv, req);    \
+            pmix_event_active(&(req->ev), EV_WRITE, 1);                     \
+            (r) = PMIX_SUCCESS;                                             \
+        }                                                                   \
+    } while(0)
 
+/* expose functions used by the macros */
+PMIX_EXPORT extern void pmix_ptl_base_send(int sd, short args, void *cbdata);
+PMIX_EXPORT extern void pmix_ptl_base_send_recv(int sd, short args, void *cbdata);
+PMIX_EXPORT extern void pmix_ptl_base_register_recv(int sd, short args, void *cbdata);
+PMIX_EXPORT extern void pmix_ptl_base_cancel_recv(int sd, short args, void *cbdata);
 
 /****    COMPONENT STRUCTURE DEFINITION    ****/
 
-/* define a component-level API for establishing a
- * communication rendezvous point for local procs. Each active component
- * would be given an opportunity to register a listener with the
- * PTL base, and/or to establish their own method for handling
- * connection requests. The component sets the need_listener flag
- * to true if a listener thread is required - otherwise, it does _not_
- * modify this parameter */
-typedef pmix_status_t (*pmix_ptl_base_setup_listener_fn_t)(pmix_info_t info[], size_t ninfo,
-                                                           bool *need_listener);
-
-/* define a component-level API for obtaining any envars that are to
- * be passed to client procs upon fork */
-typedef pmix_status_t (*pmix_ptl_base_setup_fork_fn_t)(const pmix_proc_t *proc, char ***env);
-
 /*
  * the standard component data structure
  */
@@ -196,18 +209,17 @@
     pmix_mca_base_component_data_t                  data;
     int                                             priority;
     char*                                           uri;
-    pmix_ptl_base_setup_listener_fn_t               setup_listener;
-    pmix_ptl_base_setup_fork_fn_t                   setup_fork;
-
 };
 typedef struct pmix_ptl_base_component_t pmix_ptl_base_component_t;
 
+/* export the PTL module struct */
+PMIX_EXPORT extern pmix_ptl_module_t pmix_ptl;
 
 /*
  * Macro for use in components that are of type ptl
  */
-#define PMIX_PTL_BASE_VERSION_1_0_0 \
-    PMIX_MCA_BASE_VERSION_1_0_0("ptl", 1, 0, 0)
+#define PMIX_PTL_BASE_VERSION_2_0_0 \
+    PMIX_MCA_BASE_VERSION_1_0_0("ptl", 2, 0, 0)
 
 END_C_DECLS
 
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/ptl_types.h pmix-4.0.0/src/mca/ptl/ptl_types.h
--- pmix-3.2.2~rc1/src/mca/ptl/ptl_types.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/ptl_types.h	2021-01-02 08:56:17.000000000 +0000
@@ -67,7 +67,7 @@
     uint8_t major;
     uint8_t minor;
     uint8_t release;
-    uint8_t padding;  // make the struct be 64-bits for addressing
+    uint8_t flag;  // see pmix_ptl_base_set_flag for definition of values
 } pmix_proc_type_t;
 
 #define PMIX_MAJOR_WILDCARD         255
@@ -81,7 +81,7 @@
         .major = PMIX_MAJOR_WILDCARD,       \
         .minor = PMIX_MINOR_WILDCARD,       \
         .release = PMIX_RELEASE_WILDCARD,   \
-        .padding = 0                        \
+        .flag = 0                           \
     }
 
 /* Define process types - we use a bit-mask as procs can
@@ -126,6 +126,29 @@
 
 /* provide macros for setting the major, minor, and release values
  * just so people don't have to deal with the details of the struct */
+#define PMIX_SET_PEER_VERSION(p, e, a, b)       \
+    do {                                        \
+        char *e2;                               \
+        unsigned long mj, mn, rl;               \
+        if (NULL != e) {                        \
+            if ('v' == e[0]) {                  \
+                mj = strtoul(&e[1], &e2, 10);   \
+            } else {                            \
+                mj = strtoul(e, &e2, 10);       \
+            }                                   \
+            ++e2;                               \
+            mn = strtoul(e2, &e2, 10);          \
+            ++e2;                               \
+            rl = strtoul(e2, NULL, 10);         \
+            PMIX_SET_PEER_MAJOR((p), mj);       \
+            PMIX_SET_PEER_MINOR((p), mn);       \
+            PMIX_SET_PEER_RELEASE((p), rl);     \
+        } else {                                \
+            PMIX_SET_PEER_MAJOR((p), (a));      \
+            PMIX_SET_PEER_MINOR((p), (b));      \
+        }                                       \
+    } while(0)
+
 #define PMIX_SET_PEER_MAJOR(p, a)   \
     (p)->proc_type.major = (a)
 #define PMIX_SET_PEER_MINOR(p, a)   \
@@ -276,16 +299,16 @@
     int sd;
     bool need_id;
     uint8_t flag;
-    char nspace[PMIX_MAX_NSLEN+1];
+    pmix_proc_t proc;
     pmix_info_t *info;
     size_t ninfo;
     pmix_status_t status;
     struct sockaddr_storage addr;
     struct pmix_peer_t *peer;
+    char *version;
     char *bfrops;
     char *psec;
     char *gds;
-    struct pmix_ptl_module_t *ptl;
     pmix_bfrop_buffer_type_t buffer_type;
     char *cred;
     size_t len;
@@ -299,7 +322,6 @@
 typedef struct pmix_listener_t {
     pmix_list_item_t super;
     pmix_listener_protocol_t protocol;
-    struct pmix_ptl_module_t *ptl;
     int socket;
     char *varname;
     char *uri;
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/server/Makefile.am pmix-4.0.0/src/mca/ptl/server/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ptl/server/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/server/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = ptl_server.h
+sources = \
+        ptl_server_component.c \
+        ptl_server.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_ptl_server_DSO
+lib =
+lib_sources =
+component = mca_ptl_server.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_ptl_server.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_ptl_server_la_SOURCES = $(component_sources)
+mca_ptl_server_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_ptl_server_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_ptl_server_la_SOURCES = $(lib_sources)
+libmca_ptl_server_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/server/ptl_server.c pmix-4.0.0/src/mca/ptl/server/ptl_server.c
--- pmix-3.2.2~rc1/src/mca/ptl/server/ptl_server.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/server/ptl_server.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "src/include/pmix_globals.h"
+
+#include "src/mca/ptl/base/base.h"
+#include "ptl_server.h"
+
+static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo);
+
+pmix_ptl_module_t pmix_ptl_server_module = {
+    .name = "server",
+    .setup_fork = pmix_ptl_base_setup_fork,
+    .setup_listener = setup_listener
+};
+
+static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo)
+{
+    pmix_status_t rc;
+    size_t n;
+
+    for (n=0; n < ninfo; n++) {
+        if (0 == strcmp(info[n].key, PMIX_SERVER_SESSION_SUPPORT)) {
+            pmix_ptl_base.session_tool = PMIX_INFO_TRUE(&info[n]);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SYSTEM_SUPPORT)) {
+            pmix_ptl_base.system_tool = PMIX_INFO_TRUE(&info[n]);
+        } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) {
+            pmix_ptl_base.tool_support = PMIX_INFO_TRUE(&info[n]);
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_REMOTE_CONNECTIONS)) {
+            pmix_ptl_base.remote_connections = PMIX_INFO_TRUE(&info[n]);
+        }
+    }
+
+    rc = pmix_ptl_base_setup_listener();
+    return rc;
+}
+
+
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/server/ptl_server_component.c pmix-4.0.0/src/mca/ptl/server/ptl_server_component.c
--- pmix-3.2.2~rc1/src/mca/ptl/server/ptl_server_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/server/ptl_server_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,78 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennptlee and The University
+ *                         of Tennptlee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2020 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2019      Mellanox Technologies, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/include/pmix_globals.h"
+#include "src/util/error.h"
+#include "src/mca/ptl/ptl.h"
+#include "src/mca/ptl/server/ptl_server.h"
+
+static int component_query(pmix_mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+ PMIX_EXPORT pmix_ptl_base_component_t mca_ptl_server_component = {
+    .base = {
+        PMIX_PTL_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "server",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_query_component = component_query
+    },
+    .priority = 30,
+    .uri = NULL,
+};
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    /* if I am a server and not a tool, then take me */
+    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) &&
+        !PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        *module = (pmix_mca_base_module_t*)&pmix_ptl_server_module;
+        *priority = mca_ptl_server_component.priority;
+        return PMIX_SUCCESS;
+    }
+
+    *module = NULL;
+    *priority = -1;
+    return PMIX_ERR_TAKE_NEXT_OPTION;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/server/ptl_server.h pmix-4.0.0/src/mca/ptl/server/ptl_server.h
--- pmix-3.2.2~rc1/src/mca/ptl/server/ptl_server.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/server/ptl_server.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PTL_SERVER_H
+#define PMIX_PTL_SERVER_H
+
+#include "src/mca/ptl/ptl.h"
+
+BEGIN_C_DECLS
+
+extern pmix_ptl_base_component_t mca_ptl_server_component;
+
+extern pmix_ptl_module_t pmix_ptl_server_module;
+
+END_C_DECLS
+
+#endif /* PMIX_PTL_SERVER_H */
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tcp/Makefile.am pmix-4.0.0/src/mca/ptl/tcp/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ptl/tcp/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tcp/Makefile.am	1970-01-01 00:00:00.000000000 +0000
@@ -1,53 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-headers = ptl_tcp.h
-sources = \
-        ptl_tcp_component.c \
-        ptl_tcp.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_pmix_ptl_tcp_DSO
-lib =
-lib_sources =
-component = mca_ptl_tcp.la
-component_sources = $(headers) $(sources)
-else
-lib = libmca_ptl_tcp.la
-lib_sources = $(headers) $(sources)
-component =
-component_sources =
-endif
-
-mcacomponentdir = $(pmixlibdir)
-mcacomponent_LTLIBRARIES = $(component)
-mca_ptl_tcp_la_SOURCES = $(component_sources)
-mca_ptl_tcp_la_LDFLAGS = -module -avoid-version
-if NEED_LIBPMIX
-mca_ptl_tcp_la_LIBADD = $(top_builddir)/src/libpmix.la
-endif
-
-noinst_LTLIBRARIES = $(lib)
-libmca_ptl_tcp_la_SOURCES = $(lib_sources)
-libmca_ptl_tcp_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tcp/ptl_tcp.c pmix-4.0.0/src/mca/ptl/tcp/ptl_tcp.c
--- pmix-3.2.2~rc1/src/mca/ptl/tcp/ptl_tcp.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tcp/ptl_tcp.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,1495 +0,0 @@
-/*
- * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2018      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-#include "src/include/pmix_config.h"
-#include "src/include/pmix_globals.h"
-
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_DIRENT_H
-#include <dirent.h>
-#endif
-#ifdef HAVE_SYS_SYSCTL_H
-#include <sys/sysctl.h>
-#endif
-
-#include "src/include/pmix_socket_errno.h"
-#include "src/client/pmix_client_ops.h"
-#include "src/server/pmix_server_ops.h"
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/util/os_path.h"
-#include "src/util/show_help.h"
-#include "src/mca/bfrops/base/base.h"
-#include "src/mca/gds/gds.h"
-
-#include "src/mca/ptl/base/base.h"
-#include "ptl_tcp.h"
-
-static pmix_status_t init(void);
-static void finalize(void);
-static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
-                                     pmix_info_t *info, size_t ninfo);
-static pmix_status_t send_recv(struct pmix_peer_t *peer,
-                               pmix_buffer_t *bfr,
-                               pmix_ptl_cbfunc_t cbfunc,
-                               void *cbdata);
-static pmix_status_t send_oneway(struct pmix_peer_t *peer,
-                                 pmix_buffer_t *bfr,
-                                 pmix_ptl_tag_t tag);
-
-pmix_ptl_module_t pmix_ptl_tcp_module = {
-    .init = init,
-    .finalize = finalize,
-    .send_recv = send_recv,
-    .send = send_oneway,
-    .connect_to_peer = connect_to_peer
-};
-
-static pmix_status_t recv_connect_ack(int sd, uint8_t myflag);
-static pmix_status_t send_connect_ack(int sd, uint8_t *myflag, pmix_info_t info[], size_t ninfo);
-
-
-static pmix_status_t init(void)
-{
-    return PMIX_SUCCESS;
-}
-
-static void finalize(void)
-{
-}
-
-static char *pmix_getline(FILE *fp)
-{
-    char *ret, *buff;
-    char input[1024];
-
-    ret = fgets(input, 1024, fp);
-    if (NULL != ret) {
-       input[strlen(input)-1] = '\0';  /* remove newline */
-       buff = strdup(input);
-       return buff;
-    }
-
-    return NULL;
-}
-
-static pmix_status_t parse_uri_file(char *filename,
-                                    char **uri,
-                                    char **nspace,
-                                    pmix_rank_t *rank);
-static pmix_status_t try_connect(char *uri, int *sd, pmix_info_t info[], size_t ninfo);
-static pmix_status_t df_search(char *dirname, char *prefix,
-                               pmix_info_t info[], size_t ninfo,
-                               int *sd, char **nspace,
-                               pmix_rank_t *rank, char **uri);
-
-static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
-                                     pmix_info_t *info, size_t ninfo)
-{
-    char *evar, **uri, *suri = NULL, *suri2 = NULL;
-    char *filename, *nspace=NULL;
-    pmix_rank_t rank = PMIX_RANK_WILDCARD;
-    char *p = NULL, *p2, *server_nspace = NULL, *rendfile = NULL;
-    int sd, rc;
-    size_t n;
-    char myhost[PMIX_MAXHOSTNAMELEN] = {0};
-    bool system_level = false;
-    bool system_level_only = false;
-    bool reconnect = false;
-    pid_t pid = 0, mypid;
-    pmix_list_t ilist;
-    pmix_info_caddy_t *kv;
-    pmix_info_t *iptr = NULL, mypidinfo, mycmdlineinfo, launcher;
-    size_t niptr = 0;
-    pmix_kval_t *urikv = NULL;
-    int major, minor, release;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "ptl:tcp: connecting to server");
-
-    /* see if the connection info is in the info array - if
-     * so, then that overrides all other options */
-
-
-    /* if I am a client, then we need to look for the appropriate
-     * connection info in the environment */
-    if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
-        if (NULL != (evar = getenv("PMIX_SERVER_URI3"))) {
-            /* we are talking to a v3 server */
-            PMIX_SET_PEER_TYPE(pmix_client_globals.myserver, PMIX_PROC_SERVER);
-            PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, 3);
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "V3 SERVER DETECTED");
-            /* must use the v3 bfrops module */
-            pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module("v3");
-            if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
-                return PMIX_ERR_INIT;
-            }
-        } else if (NULL != (evar = getenv("PMIX_SERVER_URI21"))) {
-            /* we are talking to a v2.1 server */
-            PMIX_SET_PEER_TYPE(pmix_client_globals.myserver, PMIX_PROC_SERVER);
-            PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, 2);
-            PMIX_SET_PEER_MINOR(pmix_client_globals.myserver, 1);
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "V21 SERVER DETECTED");
-            /* must use the v21 bfrops module */
-            pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module("v21");
-            if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
-                return PMIX_ERR_INIT;
-            }
-        } else if (NULL != (evar = getenv("PMIX_SERVER_URI2"))) {
-            /* we are talking to a v2.0 server */
-            PMIX_SET_PEER_TYPE(pmix_client_globals.myserver, PMIX_PROC_SERVER);
-            PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, 2);
-            PMIX_SET_PEER_MINOR(pmix_client_globals.myserver, 0);
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "V20 SERVER DETECTED");
-            /* must use the v20 bfrops module */
-            pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module("v20");
-            if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
-                return PMIX_ERR_INIT;
-            }
-        } else {
-            /* not us */
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-        /* the server will be using the same bfrops as us */
-        pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
-        /* mark that we are using the V2 (i.e., tcp) protocol */
-        pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2;
-
-        /* see if they set their version in the env */
-        if (NULL != (p2 = getenv("PMIX_VERSION"))) {
-            major = strtoul(p2, &p, 10);
-            ++p;
-            minor = strtoul(p, &p, 10);
-            ++p;
-            release = strtoul(p, NULL, 10);
-            PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, major);
-            PMIX_SET_PEER_MINOR(pmix_client_globals.myserver, minor);
-            PMIX_SET_PEER_RELEASE(pmix_client_globals.myserver, release);
-        }
-
-        /* the URI consists of the following elements:
-        *    - server nspace.rank
-        *    - ptl rendezvous URI
-        */
-        uri = pmix_argv_split(evar, ';');
-        if (2 != pmix_argv_count(uri)) {
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            pmix_argv_free(uri);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-
-        /* set the server nspace */
-        p = uri[0];
-        if (NULL == (p2 = strchr(p, '.'))) {
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            pmix_argv_free(uri);
-            return PMIX_ERR_NOT_SUPPORTED;
-        }
-        *p2 = '\0';
-        ++p2;
-        nspace = strdup(p);
-        rank = strtoull(p2, NULL, 10);
-        suri = strdup(uri[1]);
-
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp:client attempt connect to %s", uri[1]);
-
-        /* go ahead and try to connect */
-        if (PMIX_SUCCESS != (rc = try_connect(uri[1], &sd, info, ninfo))) {
-            free(nspace);
-            pmix_argv_free(uri);
-            free(suri);
-            return rc;
-        }
-        pmix_argv_free(uri);
-        goto complete;
-
-    }
-
-    /* get here if we are a tool - check any provided directives
-     * to see where they want us to connect to */
-    suri = NULL;
-    PMIX_CONSTRUCT(&ilist, pmix_list_t);
-    if (NULL != info) {
-        for (n=0; n < ninfo; n++) {
-            if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_TO_SYSTEM)) {
-                system_level_only = PMIX_INFO_TRUE(&info[n]);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_SYSTEM_FIRST)) {
-                /* try the system-level */
-                system_level = PMIX_INFO_TRUE(&info[n]);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_PIDINFO)) {
-                pid = info[n].value.data.pid;
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_NSPACE)) {
-                if (NULL != server_nspace) {
-                    /* they included it more than once */
-                    if (0 == strcmp(server_nspace, info[n].value.data.string)) {
-                        /* same value, so ignore it */
-                        continue;
-                    }
-                    /* otherwise, we don't know which one to use */
-                    rc = PMIX_ERR_BAD_PARAM;
-                    goto cleanup;
-                }
-                server_nspace = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_URI)) {
-                if (NULL != suri) {
-                    /* they included it more than once */
-                    if (0 == strcmp(suri, info[n].value.data.string)) {
-                        /* same value, so ignore it */
-                        continue;
-                    }
-                    /* otherwise, we don't know which one to use */
-                    rc = PMIX_ERR_BAD_PARAM;
-                    goto cleanup;
-                }
-                suri = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_RETRY_DELAY)) {
-                mca_ptl_tcp_component.wait_to_connect = info[n].value.data.uint32;
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_MAX_RETRIES)) {
-                mca_ptl_tcp_component.max_retries = info[n].value.data.uint32;
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_RECONNECT_SERVER)) {
-                reconnect = true;
-            } else {
-                /* need to pass this to server */
-                kv = PMIX_NEW(pmix_info_caddy_t);
-                kv->info = &info[n];
-                pmix_list_append(&ilist, &kv->super);
-            }
-        }
-    }
-    /* add our pid to the array */
-    kv = PMIX_NEW(pmix_info_caddy_t);
-    mypid = getpid();
-    PMIX_INFO_LOAD(&mypidinfo, PMIX_PROC_PID, &mypid, PMIX_PID);
-    kv->info = &mypidinfo;
-    pmix_list_append(&ilist, &kv->super);
-
-    /* if I am a launcher, tell them so */
-    if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        kv = PMIX_NEW(pmix_info_caddy_t);
-        PMIX_INFO_LOAD(&launcher, PMIX_LAUNCHER, NULL, PMIX_BOOL);
-        kv->info = &launcher;
-        pmix_list_append(&ilist, &kv->super);
-    }
-
-    /* add our cmd line to the array */
-#if PMIX_HAVE_APPLE
-    int mib[3], argmax, nargs, num;
-    size_t size;
-    char *procargs, *cp, *cptr;
-    char **stack = NULL;
-
-    /* Get the maximum process arguments size. */
-    mib[0] = CTL_KERN;
-    mib[1] = KERN_ARGMAX;
-    size = sizeof(argmax);
-
-    if (sysctl(mib, 2, &argmax, &size, NULL, 0) == -1) {
-        fprintf(stderr, "sysctl() argmax failed\n");
-        rc = PMIX_ERR_NO_PERMISSIONS;
-        goto cleanup;
-    }
-
-    /* Allocate space for the arguments. */
-    procargs = (char *)malloc(argmax);
-    if (procargs == NULL) {
-        rc = -1;
-        goto cleanup;
-    }
-
-    /* Make a sysctl() call to get the raw argument space of the process. */
-    mib[0] = CTL_KERN;
-    mib[1] = KERN_PROCARGS2;
-    mib[2] = getpid();
-
-    size = (size_t)argmax;
-
-    if (sysctl(mib, 3, procargs, &size, NULL, 0) == -1) {
-        fprintf(stderr, "Lacked permissions\n");;
-        rc = PMIX_ERR_NO_PERMISSIONS;
-        goto cleanup;
-    }
-
-    memcpy(&nargs, procargs, sizeof(nargs));
-    /* this points to the executable - skip over that to get the rest */
-    cp = procargs + sizeof(nargs);
-    cp += strlen(cp);
-    /* this is the first argv */
-    pmix_argv_append_nosize(&stack, cp);
-    /* skip any embedded NULLs */
-    while (cp < &procargs[size] && '\0' == *cp) {
-        ++cp;
-    }
-    if (cp != &procargs[size]) {
-        /* from this point, we have the argv separated by NULLs - split them out */
-        cptr = cp;
-        num = 0;
-        while (cp < &procargs[size] && num < nargs) {
-            if ('\0' == *cp) {
-                pmix_argv_append_nosize(&stack, cptr);
-                ++cp;  // skip over the NULL
-                cptr = cp;
-                ++num;
-            } else {
-                ++cp;
-            }
-        }
-    }
-    p = pmix_argv_join(stack, ' ');
-    pmix_argv_free(stack);
-    free(procargs);
-#else
-    char tmp[512];
-    FILE *fp;
-
-    /* open the pid's info file */
-    snprintf(tmp, 512, "/proc/%lu/cmdline", (unsigned long)mypid);
-    fp = fopen(tmp, "r");
-    if (NULL != fp) {
-        /* read the cmd line */
-        fgets(tmp, 512, fp);
-        fclose(fp);
-        p = strdup(tmp);
-    }
-#endif
-    /* pass it along */
-    kv = PMIX_NEW(pmix_info_caddy_t);
-    PMIX_INFO_LOAD(&mycmdlineinfo, PMIX_CMD_LINE, p, PMIX_STRING);
-    kv->info = &mycmdlineinfo;
-    pmix_list_append(&ilist, &kv->super);
-    free(p);
-
-    /* if we need to pass anything, setup an array */
-    if (0 < (niptr = pmix_list_get_size(&ilist))) {
-        PMIX_INFO_CREATE(iptr, niptr);
-        n = 0;
-        while (NULL != (kv = (pmix_info_caddy_t*)pmix_list_remove_first(&ilist))) {
-            PMIX_INFO_XFER(&iptr[n], kv->info);
-            PMIX_RELEASE(kv);
-            ++n;
-        }
-    }
-    PMIX_LIST_DESTRUCT(&ilist);
-
-    if (NULL == suri && !reconnect && NULL != mca_ptl_tcp_component.super.uri) {
-        suri = strdup(mca_ptl_tcp_component.super.uri);
-    }
-
-    /* mark that we are using the V2 protocol */
-    pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2;
-    gethostname(myhost, sizeof(myhost)-1);
-    /* if we were given a URI via MCA param, then look no further */
-    if (NULL != suri) {
-        /* if the string starts with "file:", then they are pointing
-         * us to a file we need to read to get the URI itself */
-        if (0 == strncmp(suri, "file:", 5)) {
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "ptl:tcp:tool getting connection info from %s", suri);
-            nspace = NULL;
-            rc = parse_uri_file(&suri[5], &suri2, &nspace, &rank);
-            if (PMIX_SUCCESS != rc) {
-                rc = PMIX_ERR_UNREACH;
-                goto cleanup;
-            }
-            free(suri);
-            suri = suri2;
-        } else {
-            /* we need to extract the nspace/rank of the server from the string */
-            p = strchr(suri, ';');
-            if (NULL == p) {
-                rc = PMIX_ERR_BAD_PARAM;
-                goto cleanup;
-            }
-            *p = '\0';
-            p++;
-            suri2 = strdup(p); // save the uri portion
-            /* the '.' in the first part of the original string separates
-             * nspace from rank */
-            p = strchr(suri, '.');
-            if (NULL == p) {
-                free(suri2);
-                rc = PMIX_ERR_BAD_PARAM;
-                goto cleanup;
-            }
-            *p = '\0';
-            p++;
-            nspace = strdup(suri);
-            rank = strtoull(p, NULL, 10);
-            free(suri);
-            suri = suri2;
-            /* now update the URI */
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp:tool attempt connect using given URI %s", suri);
-        /* go ahead and try to connect */
-        if (PMIX_SUCCESS != (rc = try_connect(suri, &sd, iptr, niptr))) {
-            goto cleanup;
-        }
-        /* cleanup */
-        goto complete;
-    }
-
-    /* if they gave us a rendezvous file, use it */
-    if (NULL != rendfile) {
-        /* try to read the file */
-        rc = parse_uri_file(rendfile, &suri, &nspace, &rank);
-        free(rendfile);
-        rendfile = NULL;
-        if (PMIX_SUCCESS == rc) {
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "ptl:tcp:tool attempt connect to rendezvous server at %s", suri);
-            /* go ahead and try to connect */
-            if (PMIX_SUCCESS == try_connect(suri, &sd, iptr, niptr)) {
-                /* don't free nspace - we will use it below */
-                if (NULL != iptr) {
-                    PMIX_INFO_FREE(iptr, niptr);
-                }
-                goto complete;
-            }
-        }
-        /* since they gave us a specific rendfile and we couldn't
-         * connect to it, return an error */
-        rc = PMIX_ERR_UNREACH;
-        goto cleanup;
-    }
-
-    /* if they asked for system-level first or only, we start there */
-    if (system_level || system_level_only) {
-        if (0 > asprintf(&filename, "%s/pmix.sys.%s", mca_ptl_tcp_component.system_tmpdir, myhost)) {
-            rc = PMIX_ERR_NOMEM;
-            goto cleanup;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp:tool looking for system server at %s",
-                            filename);
-        /* try to read the file */
-        rc = parse_uri_file(filename, &suri, &nspace, &rank);
-        free(filename);
-        if (PMIX_SUCCESS == rc) {
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "ptl:tcp:tool attempt connect to system server at %s", suri);
-            /* go ahead and try to connect */
-            if (PMIX_SUCCESS == try_connect(suri, &sd, iptr, niptr)) {
-                /* don't free nspace - we will use it below */
-                goto complete;
-            }
-            free(nspace);
-            nspace = NULL;
-        }
-    }
-
-    /* we get here if they either didn't ask for a system-level connection,
-     * or they asked for it and it didn't succeed. If they _only_ wanted
-     * a system-level connection, then we are done */
-    if (system_level_only) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp: connecting to system failed");
-        rc = PMIX_ERR_UNREACH;
-        goto cleanup;
-    }
-
-    /* if they gave us a pid, then look for it */
-    if (0 != pid) {
-        if (0 > asprintf(&filename, "pmix.%s.tool.%d", myhost, pid)) {
-            rc = PMIX_ERR_NOMEM;
-            goto cleanup;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp:tool searching for given session server %s",
-                            filename);
-        nspace = NULL;
-        rc = df_search(mca_ptl_tcp_component.system_tmpdir,
-                       filename, iptr, niptr, &sd, &nspace, &rank, &suri);
-        free(filename);
-        if (PMIX_SUCCESS == rc) {
-            goto complete;
-        }
-        /* since they gave us a specific pid and we couldn't
-         * connect to it, return an error */
-        rc = PMIX_ERR_UNREACH;
-        goto cleanup;
-    }
-
-    /* if they gave us an nspace, then look for it */
-    if (NULL != server_nspace) {
-        if (0 > asprintf(&filename, "pmix.%s.tool.%s", myhost, server_nspace)) {
-            rc = PMIX_ERR_NOMEM;
-            goto cleanup;
-        }
-        free(server_nspace);
-        server_nspace = NULL;
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp:tool searching for given session server %s",
-                            filename);
-        nspace = NULL;
-        rc = df_search(mca_ptl_tcp_component.system_tmpdir,
-                       filename, iptr, niptr, &sd, &nspace, &rank, &suri);
-        free(filename);
-        if (PMIX_SUCCESS == rc) {
-            goto complete;
-        }
-        /* since they gave us a specific nspace and we couldn't
-         * connect to it, return an error */
-        rc = PMIX_ERR_UNREACH;
-        goto cleanup;
-    }
-
-    /* they didn't give us a pid, so we will search to see what session-level
-     * tools are available to this user. We will take the first connection
-     * that succeeds - this is based on the likelihood that there is only
-     * one session per user on a node */
-
-    if (0 > asprintf(&filename, "pmix.%s.tool", myhost)) {
-        rc = PMIX_ERR_NOMEM;
-        goto cleanup;
-    }
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "ptl:tcp:tool searching for session server %s",
-                        filename);
-    nspace = NULL;
-    rc = df_search(mca_ptl_tcp_component.system_tmpdir,
-                   filename, iptr, niptr, &sd, &nspace, &rank, &suri);
-    free(filename);
-    if (PMIX_SUCCESS != rc) {
-        rc = PMIX_ERR_UNREACH;
-        goto cleanup;
-    }
-
-  complete:
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "tcp_peer_try_connect: Connection across to server succeeded");
-
-    /* do a final bozo check */
-    if (NULL == nspace || PMIX_RANK_WILDCARD == rank) {
-        CLOSE_THE_SOCKET(sd);
-        rc = PMIX_ERR_UNREACH;
-        goto cleanup;
-    }
-    /* mark the connection as made */
-    pmix_globals.connected = true;
-    pmix_client_globals.myserver->sd = sd;
-
-    /* tools setup their server info in try_connect because they
-     * utilize a broader handshake */
-    if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
-        /* setup the server info */
-        if (NULL == pmix_client_globals.myserver->info) {
-            pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t);
-        }
-        if (NULL == pmix_client_globals.myserver->nptr) {
-            pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t);
-        }
-        if (NULL != pmix_client_globals.myserver->nptr->nspace) {
-            free(pmix_client_globals.myserver->nptr->nspace);
-        }
-        pmix_client_globals.myserver->nptr->nspace = strdup(nspace);
-
-        if (NULL != pmix_client_globals.myserver->info->pname.nspace) {
-            free(pmix_client_globals.myserver->info->pname.nspace);
-        }
-        pmix_client_globals.myserver->info->pname.nspace = strdup(pmix_client_globals.myserver->nptr->nspace);
-        pmix_client_globals.myserver->info->pname.rank = rank;
-    }
-    /* store the URI for subsequent lookups */
-    urikv = PMIX_NEW(pmix_kval_t);
-    urikv->key = strdup(PMIX_SERVER_URI);
-    PMIX_VALUE_CREATE(urikv->value, 1);
-    urikv->value->type = PMIX_STRING;
-    asprintf(&urikv->value->data.string, "%s.%u;%s", nspace, rank, suri);
-    PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid, PMIX_INTERNAL,
-                      urikv);
-    PMIX_RELEASE(urikv);  // maintain accounting
-
-    pmix_ptl_base_set_nonblocking(sd);
-
-    /* setup recv event */
-    pmix_event_assign(&pmix_client_globals.myserver->recv_event,
-                      pmix_globals.evbase,
-                      pmix_client_globals.myserver->sd,
-                      EV_READ | EV_PERSIST,
-                      pmix_ptl_base_recv_handler, pmix_client_globals.myserver);
-    pmix_client_globals.myserver->recv_ev_active = true;
-    PMIX_POST_OBJECT(pmix_client_globals.myserver);
-    pmix_event_add(&pmix_client_globals.myserver->recv_event, 0);
-
-    /* setup send event */
-    pmix_event_assign(&pmix_client_globals.myserver->send_event,
-                      pmix_globals.evbase,
-                      pmix_client_globals.myserver->sd,
-                      EV_WRITE|EV_PERSIST,
-                      pmix_ptl_base_send_handler, pmix_client_globals.myserver);
-    pmix_client_globals.myserver->send_ev_active = false;
-
-  cleanup:
-    if (NULL != nspace) {
-        free(nspace);
-    }
-    if (NULL != iptr) {
-        PMIX_INFO_FREE(iptr, niptr);
-    }
-    if (NULL != rendfile) {
-        free(rendfile);
-    }
-    if (NULL != suri) {
-        free(suri);
-    }
-    if (NULL != server_nspace) {
-        free(server_nspace);
-    }
-    return rc;
-}
-
-static pmix_status_t send_recv(struct pmix_peer_t *peer,
-                               pmix_buffer_t *bfr,
-                               pmix_ptl_cbfunc_t cbfunc,
-                               void *cbdata)
-{
-    pmix_ptl_sr_t *ms;
-    pmix_peer_t *pr = (pmix_peer_t*)peer;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "[%s:%d] post send to server",
-                        __FILE__, __LINE__);
-
-    ms = PMIX_NEW(pmix_ptl_sr_t);
-    PMIX_RETAIN(pr);
-    ms->peer = pr;
-    ms->bfr = bfr;
-    ms->cbfunc = cbfunc;
-    ms->cbdata = cbdata;
-    PMIX_THREADSHIFT(ms, pmix_ptl_base_send_recv);
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t send_oneway(struct pmix_peer_t *peer,
-                                 pmix_buffer_t *bfr,
-                                 pmix_ptl_tag_t tag)
-{
-    pmix_ptl_queue_t *q;
-    pmix_peer_t *pr = (pmix_peer_t*)peer;
-
-    /* we have to transfer this to an event for thread
-     * safety as we need to post this message on the
-     * peer's send queue */
-    q = PMIX_NEW(pmix_ptl_queue_t);
-    PMIX_RETAIN(pr);
-    q->peer = pr;
-    q->buf = bfr;
-    q->tag = tag;
-    PMIX_THREADSHIFT(q, pmix_ptl_base_send);
-    return PMIX_SUCCESS;
-}
-
-static void timeout(int sd, short args, void *cbdata)
-{
-    pmix_lock_t *lock = (pmix_lock_t*)cbdata;
-    PMIX_WAKEUP_THREAD(lock);
-}
-
-/****    SUPPORTING FUNCTIONS    ****/
-static pmix_status_t parse_uri_file(char *filename,
-                                    char **uri,
-                                    char **nspace,
-                                    pmix_rank_t *rank)
-{
-    FILE *fp;
-    char *srvr, *p, *p2, *p3;
-    pmix_lock_t lock;
-    pmix_event_t ev;
-    struct timeval tv;
-    int retries;
-    int major, minor, release;
-
-     /* if we cannot open the file, then the server must not
-     * be configured to support tool connections, or this
-     * user isn't authorized to access it - or it may just
-     * not exist yet! Check for existence */
-    /* coverity[toctou] */
-    if (0 == access(filename, R_OK)) {
-        goto process;
-    } else {
-        if (ENOENT == errno) {
-            /* the file does not exist, so give it
-             * a little time to see if the server
-             * is still starting up */
-            retries = 0;
-            do {
-                ++retries;
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "WAITING FOR CONNECTION FILE %s", filename);
-                PMIX_CONSTRUCT_LOCK(&lock);
-                if (0 < mca_ptl_tcp_component.wait_to_connect) {
-                    tv.tv_sec = mca_ptl_tcp_component.wait_to_connect;
-                    tv.tv_usec = 0;
-                    pmix_event_evtimer_set(pmix_globals.evbase, &ev,
-                                           timeout, &lock);
-                    PMIX_POST_OBJECT(&ev);
-                    pmix_event_evtimer_add(&ev, &tv);
-                } else {
-                    tv.tv_sec = 0;
-                    tv.tv_usec = 10000;  // use 0.01 sec as default
-                    pmix_event_evtimer_set(pmix_globals.evbase, &ev,
-                                           timeout, &lock);
-                    PMIX_POST_OBJECT(&ev);
-                    pmix_event_evtimer_add(&ev, &tv);
-                }
-                PMIX_WAIT_THREAD(&lock);
-                PMIX_DESTRUCT_LOCK(&lock);
-                /* coverity[toctou] */
-                if (0 == access(filename, R_OK)) {
-                    goto process;
-                }
-            } while (retries < mca_ptl_tcp_component.max_retries);
-            /* otherwise, mark it as unreachable */
-        }
-    }
-    return PMIX_ERR_UNREACH;
-
-  process:
-    fp = fopen(filename, "r");
-    if (NULL == fp) {
-        return PMIX_ERR_UNREACH;
-    }
-    /* get the URI - might seem crazy, but there is actually
-     * a race condition here where the server may have created
-     * the file but not yet finished writing into it. So give
-     * us a chance to get the required info */
-    for (retries=0; retries < 3; retries++) {
-        srvr = pmix_getline(fp);
-        if (NULL != srvr) {
-            break;
-        }
-        fclose(fp);
-        tv.tv_sec = 0;
-        tv.tv_usec = 10000;  // use 0.01 sec as default
-        pmix_event_evtimer_set(pmix_globals.evbase, &ev,
-                               timeout, &lock);
-        PMIX_POST_OBJECT(&ev);
-        pmix_event_evtimer_add(&ev, &tv);
-        PMIX_WAIT_THREAD(&lock);
-        PMIX_DESTRUCT_LOCK(&lock);
-        fp = fopen(filename, "r");
-        if (NULL == fp) {
-            return PMIX_ERR_UNREACH;
-        }
-    }
-    if (NULL == srvr) {
-        PMIX_ERROR_LOG(PMIX_ERR_FILE_READ_FAILURE);
-        fclose(fp);
-        return PMIX_ERR_UNREACH;
-    }
-    /* see if this file contains the server's version */
-    p2 = pmix_getline(fp);
-    if (NULL == p2) {
-        PMIX_SET_PEER_TYPE(pmix_client_globals.myserver, PMIX_PROC_SERVER);
-        PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, 2);
-        PMIX_SET_PEER_MINOR(pmix_client_globals.myserver, 0);
-        pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2;
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "V20 SERVER DETECTED");
-    } else {
-        /* convert the version to a number */
-        if ('v' == p2[0]) {
-            major = strtoul(&p2[1], &p3, 10);
-        } else {
-            major = strtoul(p2, &p3, 10);
-        }
-        minor = strtoul(p3, &p3, 10);
-        release = strtoul(p3, NULL, 10);
-        PMIX_SET_PEER_TYPE(pmix_client_globals.myserver, PMIX_PROC_SERVER);
-        PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, major);
-        PMIX_SET_PEER_MINOR(pmix_client_globals.myserver, minor);
-        PMIX_SET_PEER_RELEASE(pmix_client_globals.myserver, release);
-        if (2 <= major) {
-            pmix_client_globals.myserver->protocol = PMIX_PROTOCOL_V2;
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "V2 PROTOCOL SERVER DETECTED");
-        }
-    }
-    if (NULL != p2) {
-        free(p2);
-    }
-
-    fclose(fp);
-    /* up to the first ';' is the server nspace/rank */
-    if (NULL == (p = strchr(srvr, ';'))) {
-        /* malformed */
-        free(srvr);
-        return PMIX_ERR_UNREACH;
-    }
-    *p = '\0';
-    ++p;  // move past the semicolon
-    /* the nspace is the section up to the '.' */
-    if (NULL == (p2 = strchr(srvr, '.'))) {
-        /* malformed */
-        free(srvr);
-        return PMIX_ERR_UNREACH;
-    }
-    *p2 = '\0';
-    ++p2;
-    /* set the server nspace/rank */
-    *nspace = strdup(srvr);
-    *rank = strtoull(p2, NULL, 10);
-
-    /* now parse the uri itself */
-    *uri = strdup(p);
-    free(srvr);
-
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t try_connect(char *uri, int *sd, pmix_info_t iptr[], size_t niptr)
-{
-    char *p, *p2, *host;
-    struct sockaddr_in *in;
-    struct sockaddr_in6 *in6;
-    size_t len;
-    pmix_status_t rc;
-    int retries = 0;
-    uint8_t myflag;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:tcp try connect to %s", uri);
-
-    /* mark that we are the active module for this server */
-    pmix_client_globals.myserver->nptr->compat.ptl = &pmix_ptl_tcp_module;
-
-    /* setup the path to the daemon rendezvous point */
-    memset(&mca_ptl_tcp_component.connection, 0, sizeof(struct sockaddr_storage));
-    if (0 == strncmp(uri, "tcp4", 4)) {
-        /* need to skip the tcp4: part */
-        p = strdup(&uri[7]);
-        if (NULL == p) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
-            return PMIX_ERR_NOMEM;
-        }
-
-        /* separate the IP address from the port */
-        p2 = strchr(p, ':');
-        if (NULL == p2) {
-            free(p);
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            return PMIX_ERR_BAD_PARAM;
-        }
-        *p2 = '\0';
-        p2++;
-        host = p;
-        /* load the address */
-        in = (struct sockaddr_in*)&mca_ptl_tcp_component.connection;
-        in->sin_family = AF_INET;
-        in->sin_addr.s_addr = inet_addr(host);
-        if (in->sin_addr.s_addr == INADDR_NONE) {
-            free(p);
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            return PMIX_ERR_BAD_PARAM;
-        }
-        in->sin_port = htons(atoi(p2));
-        len = sizeof(struct sockaddr_in);
-    } else {
-        /* need to skip the tcp6: part */
-        p = strdup(&uri[7]);
-        if (NULL == p) {
-            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
-            return PMIX_ERR_NOMEM;
-        }
-
-        p2 = strchr(p, ':');
-        if (NULL == p2) {
-            free(p);
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            return PMIX_ERR_BAD_PARAM;
-        }
-        *p2 = '\0';
-        if (']' == p[strlen(p)-1]) {
-            p[strlen(p)-1] = '\0';
-        }
-        if ('[' == p[0]) {
-            host = &p[1];
-        } else {
-            host = &p[0];
-        }
-        /* load the address */
-        in6 = (struct sockaddr_in6*)&mca_ptl_tcp_component.connection;
-        in6->sin6_family = AF_INET6;
-        if (0 == inet_pton(AF_INET6, host, (void*)&in6->sin6_addr)) {
-            pmix_output (0, "ptl_tcp_parse_uri: Could not convert %s\n", host);
-            free(p);
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            return PMIX_ERR_BAD_PARAM;
-        }
-        in6->sin6_port = htons(atoi(p2));
-        len = sizeof(struct sockaddr_in6);
-    }
-    free(p);
-
-  retry:
-    /* establish the connection */
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_tcp_component.connection, len, sd))) {
-        /* do not error log - might just be a stale connection point */
-        return rc;
-    }
-
-    /* send our identity and any authentication credentials to the server */
-    if (PMIX_SUCCESS != (rc = send_connect_ack(*sd, &myflag, iptr, niptr))) {
-        PMIX_ERROR_LOG(rc);
-        CLOSE_THE_SOCKET(*sd);
-        return rc;
-    }
-
-    /* do whatever handshake is required */
-    if (PMIX_SUCCESS != (rc = recv_connect_ack(*sd, myflag))) {
-        CLOSE_THE_SOCKET(*sd);
-        if (PMIX_ERR_TEMP_UNAVAILABLE == rc) {
-            ++retries;
-            if( retries < mca_ptl_tcp_component.handshake_max_retries ) {
-                goto retry;
-            }
-        }
-        return rc;
-    }
-
-    return PMIX_SUCCESS;
-}
-static pmix_status_t send_connect_ack(int sd, uint8_t *myflag,
-                                      pmix_info_t iptr[], size_t niptr)
-{
-    char *msg;
-    pmix_ptl_hdr_t hdr;
-    size_t sdsize=0, csize=0;
-    pmix_byte_object_t cred;
-    char *sec, *bfrops, *gds;
-    pmix_bfrop_buffer_type_t bftype;
-    pmix_status_t rc;
-    uint8_t flag;
-    uid_t euid;
-    gid_t egid;
-    uint32_t u32;
-    pmix_buffer_t buf;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:tcp SEND CONNECT ACK");
-
-    /* if we are a server, then we shouldn't be here */
-    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) &&
-        !PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-        return PMIX_ERR_NOT_SUPPORTED;
-    }
-
-    /* setup the header */
-    memset(&hdr, 0, sizeof(pmix_ptl_hdr_t));
-    hdr.pindex = -1;
-    hdr.tag = UINT32_MAX;
-
-    /* a security module was assigned to us during rte_init based
-     * on a list of available security modules provided by our
-     * local PMIx server, if known. Now use that module to
-     * get a credential, if the security system provides one. Not
-     * every psec module will do so, thus we must first check */
-    PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
-    PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer,
-                          NULL, 0, NULL, 0, &cred);
-    if (PMIX_SUCCESS != rc) {
-        return rc;
-    }
-
-    /* allow space for a marker indicating client vs tool */
-    sdsize = 1;
-
-    /* Defined marker values:
-     *
-     * 0 => simple client process
-     * 1 => legacy tool - may or may not have an identifier
-     * 2 => legacy launcher - may or may not have an identifier
-     * ------------------------------------------
-     * 3 => self-started tool process that needs an identifier
-     * 4 => self-started tool process that was given an identifier by caller
-     * 5 => tool that was started by a PMIx server - identifier specified by server
-     * 6 => self-started launcher that needs an identifier
-     * 7 => self-started launcher that was given an identifier by caller
-     * 8 => launcher that was started by a PMIx server - identifier specified by server
-     */
-    if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
-            /* if we are both launcher and client, then we need
-             * to tell the server we are both */
-            flag = 8;
-            /* add space for our uid/gid for ACL purposes */
-            sdsize += 2*sizeof(uint32_t);
-            /* add space for our identifier */
-            sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
-        } else {
-            /* add space for our uid/gid for ACL purposes */
-            sdsize += 2*sizeof(uint32_t);
-            /* if they gave us an identifier, we need to pass it */
-            if (0 < strlen(pmix_globals.myid.nspace) &&
-                PMIX_RANK_INVALID != pmix_globals.myid.rank) {
-                flag = 7;
-                sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
-            } else {
-                flag = 6;
-            }
-        }
-
-    } else if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer) &&
-               !PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
-        /* we are a simple client */
-        flag = 0;
-        /* reserve space for our nspace and rank info */
-        sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
-
-    } else {  // must be a tool of some sort
-        /* add space for our uid/gid for ACL purposes */
-        sdsize += 2*sizeof(uint32_t);
-        if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
-            /* if we are both tool and client, then we need
-             * to tell the server we are both */
-            flag = 5;
-            /* add space for our identifier */
-            sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
-        } else if (0 < strlen(pmix_globals.myid.nspace) &&
-            PMIX_RANK_INVALID != pmix_globals.myid.rank) {
-            /* we were given an identifier by the caller, pass it */
-            sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t);
-            flag = 4;
-        } else {
-            /* we are a self-started tool that needs an identifier */
-            flag = 3;
-        }
-    }
-    *myflag = flag;
-
-    /* add the name of our active sec module - we selected it
-     * in pmix_client.c prior to entering here */
-    sec = pmix_globals.mypeer->nptr->compat.psec->name;
-
-    /* add our active bfrops module name */
-    bfrops = pmix_globals.mypeer->nptr->compat.bfrops->version;
-    /* and the type of buffer we are using */
-    bftype = pmix_globals.mypeer->nptr->compat.type;
-
-    /* add our active gds module for working with the server */
-    gds = (char*)pmix_client_globals.myserver->nptr->compat.gds->name;
-
-    /* if we were given info structs to pass to the server, pack them */
-    PMIX_CONSTRUCT(&buf, pmix_buffer_t);
-    if (NULL != iptr) {
-        PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &niptr, 1, PMIX_SIZE);
-        PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, iptr, niptr, PMIX_INFO);
-    }
-
-    /* set the number of bytes to be read beyond the header - must
-     * NULL terminate the strings! */
-    hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \
-                + strlen(bfrops) + 1 + sizeof(bftype) \
-                + strlen(gds) + 1 + sizeof(uint32_t) + cred.size \
-                + buf.bytes_used;
-
-    /* create a space for our message */
-    sdsize = (sizeof(hdr) + hdr.nbytes);
-    if (NULL == (msg = (char*)malloc(sdsize))) {
-        PMIX_BYTE_OBJECT_DESTRUCT(&cred);
-        free(sec);
-        PMIX_DESTRUCT(&buf);
-        return PMIX_ERR_OUT_OF_RESOURCE;
-    }
-    memset(msg, 0, sdsize);
-
-    /* load the message */
-    csize=0;
-    memcpy(msg, &hdr, sizeof(pmix_ptl_hdr_t));
-    csize += sizeof(pmix_ptl_hdr_t);
-
-    /* provide our active psec module */
-    memcpy(msg+csize, sec, strlen(sec));
-    csize += strlen(sec)+1;
-
-    /* load the length of the credential - we put this in uint32_t
-     * format as that is a fixed size, and convert to network
-     * byte order for heterogeneity */
-    u32 = htonl((uint32_t)cred.size);
-    memcpy(msg+csize, &u32, sizeof(uint32_t));
-    csize += sizeof(uint32_t);
-    /* load the credential */
-    if (0 < u32) {
-        memcpy(msg+csize, cred.bytes, cred.size);
-        csize += cred.size;
-    }
-    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
-
-    /* load our process type - this is a single byte,
-     * so no worry about heterogeneity here */
-    memcpy(msg+csize, &flag, 1);
-    csize += 1;
-
-    if (0 == flag) {
-        /* if we are a client, provide our nspace/rank */
-        memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
-        csize += strlen(pmix_globals.myid.nspace)+1;
-        /* again, need to convert */
-        u32 = htonl((uint32_t)pmix_globals.myid.rank);
-        memcpy(msg+csize, &u32, sizeof(uint32_t));
-        csize += sizeof(uint32_t);
-    } else if (3 == flag || 6 == flag) {
-        /* we are a tool or launcher that needs an identifier - add our ACLs */
-        euid = geteuid();
-        u32 = htonl(euid);
-        memcpy(msg+csize, &u32, sizeof(uint32_t));
-        csize += sizeof(uint32_t);
-        egid = getegid();
-        u32 = htonl(egid);
-        memcpy(msg+csize, &u32, sizeof(uint32_t));
-        csize += sizeof(uint32_t);
-    } else if (4 == flag || 5 == flag || 7 == flag || 8 == flag) {
-        /* we are a tool or launcher that has an identifier - start with our ACLs */
-        euid = geteuid();
-        u32 = htonl(euid);
-        memcpy(msg+csize, &u32, sizeof(uint32_t));
-        csize += sizeof(uint32_t);
-        egid = getegid();
-        u32 = htonl(egid);
-        memcpy(msg+csize, &u32, sizeof(uint32_t));
-        csize += sizeof(uint32_t);
-        /* now add our identifier */
-        memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
-        csize += strlen(pmix_globals.myid.nspace)+1;
-        /* again, need to convert */
-        u32 = htonl((uint32_t)pmix_globals.myid.rank);
-        memcpy(msg+csize, &u32, sizeof(uint32_t));
-        csize += sizeof(uint32_t);
-    } else {
-        /* not a valid flag */
-        PMIX_DESTRUCT(&buf);
-        return PMIX_ERR_NOT_SUPPORTED;
-    }
-
-    /* provide our version */
-    memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
-    csize += strlen(PMIX_VERSION)+1;
-
-    /* provide our active bfrops module */
-    memcpy(msg+csize, bfrops, strlen(bfrops));
-    csize += strlen(bfrops)+1;
-
-    /* provide the bfrops type */
-    memcpy(msg+csize, &bftype, sizeof(bftype));
-    csize += sizeof(bftype);
-
-    /* provide the gds module */
-    memcpy(msg+csize, gds, strlen(gds));
-    csize += strlen(gds)+1;
-
-    /* provide the info struct bytes */
-    memcpy(msg+csize, buf.base_ptr, buf.bytes_used);
-    csize += buf.bytes_used;
-
-    /* send the entire message across */
-    if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
-        free(msg);
-        PMIX_DESTRUCT(&buf);
-        return PMIX_ERR_UNREACH;
-    }
-    free(msg);
-    PMIX_DESTRUCT(&buf);
-    return PMIX_SUCCESS;
-}
-
-/* we receive a connection acknowledgement from the server,
- * consisting of nothing more than a status report. If success,
- * then we initiate authentication method */
-static pmix_status_t recv_connect_ack(int sd, uint8_t myflag)
-{
-    pmix_status_t reply;
-    pmix_status_t rc;
-    struct timeval tv, save;
-    pmix_socklen_t sz;
-    bool sockopt = true;
-    pmix_nspace_t nspace;
-    uint32_t u32;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix: RECV CONNECT ACK FROM SERVER");
-
-    /* get the current timeout value so we can reset to it */
-    sz = sizeof(save);
-    if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) {
-        if (ENOPROTOOPT == errno || EOPNOTSUPP == errno) {
-            sockopt = false;
-        } else {
-           return PMIX_ERR_UNREACH;
-       }
-   } else {
-        /* set a timeout on the blocking recv so we don't hang */
-        tv.tv_sec  = mca_ptl_tcp_component.handshake_wait_time;
-        tv.tv_usec = 0;
-        if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
-            if (ENOPROTOOPT == errno || EOPNOTSUPP == errno) {
-                sockopt = false;
-            } else {
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
-                return PMIX_ERR_UNREACH;
-            }
-        }
-    }
-
-    /* receive the status reply */
-    rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t));
-    if (PMIX_SUCCESS != rc) {
-        if (sockopt) {
-            /* return the socket to normal */
-            if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
-                return PMIX_ERR_UNREACH;
-            }
-        }
-        return rc;
-    }
-    reply = ntohl(u32);
-
-    if (0 == myflag) {
-        /* see if they want us to do the handshake */
-        if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
-            PMIX_PSEC_CLIENT_HANDSHAKE(rc, pmix_client_globals.myserver, sd);
-            if (PMIX_SUCCESS != rc) {
-                return rc;
-            }
-        } else if (PMIX_SUCCESS != reply) {
-            return reply;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "pmix: RECV CONNECT CONFIRMATION");
-
-        /* receive our index into the server's client array */
-        rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t));
-        if (PMIX_SUCCESS != rc) {
-            return rc;
-        }
-        pmix_globals.pindex = ntohl(u32);
-    } else {  // we are a tool
-        /* if the status indicates an error, then we are done */
-        if (PMIX_SUCCESS != reply) {
-            return reply;
-        }
-        /* if we needed an identifier, recv it */
-        if (3 == myflag || 6 == myflag) {
-            /* first the nspace */
-            rc = pmix_ptl_base_recv_blocking(sd, (char*)&nspace, PMIX_MAX_NSLEN+1);
-            if (PMIX_SUCCESS != rc) {
-                return rc;
-            }
-            PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, nspace);
-            /* now the rank */
-            rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t));
-            if (PMIX_SUCCESS != rc) {
-                return rc;
-            }
-            /* convert and store */
-            pmix_globals.myid.rank = htonl(u32);
-        }
-
-        /* get the server's nspace and rank so we can send to it */
-        if (NULL == pmix_client_globals.myserver->info) {
-            pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t);
-        }
-        if (NULL == pmix_client_globals.myserver->nptr) {
-            pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t);
-        }
-        rc = pmix_ptl_base_recv_blocking(sd, (char*)nspace, PMIX_MAX_NSLEN+1);
-        if (PMIX_SUCCESS != rc) {
-            return rc;
-        }
-        if (NULL != pmix_client_globals.myserver->nptr->nspace) {
-            free(pmix_client_globals.myserver->nptr->nspace);
-        }
-        pmix_client_globals.myserver->nptr->nspace = strdup(nspace);
-        if (NULL != pmix_client_globals.myserver->info->pname.nspace) {
-            free(pmix_client_globals.myserver->info->pname.nspace);
-        }
-        pmix_client_globals.myserver->info->pname.nspace = strdup(nspace);
-        rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t));
-        if (PMIX_SUCCESS != rc) {
-            return rc;
-        }
-        pmix_client_globals.myserver->info->pname.rank = htonl(u32);
-
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d",
-                            pmix_globals.myid.nspace, pmix_globals.myid.rank,
-                            pmix_client_globals.myserver->info->pname.nspace,
-                            pmix_client_globals.myserver->info->pname.rank);
-
-        /* get the returned status from the security handshake */
-        rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(pmix_status_t));
-        if (PMIX_SUCCESS != rc) {
-            if (sockopt) {
-                /* return the socket to normal */
-                if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
-                    return PMIX_ERR_UNREACH;
-                }
-            }
-            return rc;
-        }
-
-        reply = ntohl(u32);
-        if (PMIX_SUCCESS != reply) {
-            /* see if they want us to do the handshake */
-            if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
-                PMIX_PSEC_CLIENT_HANDSHAKE(reply, pmix_client_globals.myserver, sd);
-                if (PMIX_SUCCESS != reply) {
-                    return reply;
-                }
-                /* if the handshake succeeded, then fall thru to the next step */
-            } else {
-                return reply;
-            }
-        }
-    }
-
-    if (sockopt) {
-        if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
-            return PMIX_ERR_UNREACH;
-        }
-    }
-#if defined(TCP_NODELAY)
-    int optval;
-    optval = 1;
-    if (setsockopt(sd, IPPROTO_TCP, TCP_NODELAY, (char *)&optval, sizeof(optval)) < 0) {
-        opal_backtrace_print(stderr, NULL, 1);
-        pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
-                            "[%s:%d] setsockopt(TCP_NODELAY) failed: %s (%d)",
-                            __FILE__, __LINE__,
-                            strerror(pmix_socket_errno),
-                            pmix_socket_errno);
-    }
-#endif
-#if defined(SO_NOSIGPIPE)
-    /* Some BSD flavors generate EPIPE when we write to a disconnected peer. We need
-     * the prevent this signal to be able to trap socket shutdown and cleanly release
-     * the endpoint.
-     */
-    int optval2 = 1;
-    if (setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, (char *)&optval2, sizeof(optval2)) < 0) {
-            pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
-                                "[%s:%d] setsockopt(SO_NOSIGPIPE) failed: %s (%d)",
-                                 __FILE__, __LINE__,
-                                strerror(pmix_socket_errno), pmix_socket_errno);
-    }
-#endif
-
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t df_search(char *dirname, char *prefix,
-                               pmix_info_t info[], size_t ninfo,
-                               int *sd, char **nspace,
-                               pmix_rank_t *rank, char **uri)
-{
-    char *suri, *nsp, *newdir;
-    pmix_rank_t rk;
-    pmix_status_t rc;
-    struct stat buf;
-    DIR *cur_dirp;
-    struct dirent *dir_entry;
-
-    if (NULL == (cur_dirp = opendir(dirname))) {
-        return PMIX_ERR_NOT_FOUND;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:tcp: searching directory %s", dirname);
-
-    /* search the entries for something that starts with the provided prefix */
-    while (NULL != (dir_entry = readdir(cur_dirp))) {
-        /* ignore the . and .. entries */
-        if (0 == strcmp(dir_entry->d_name, ".") ||
-            0 == strcmp(dir_entry->d_name, "..")) {
-            continue;
-        }
-        newdir = pmix_os_path(false, dirname, dir_entry->d_name, NULL);
-        /* coverity[toctou] */
-        if (-1 == stat(newdir, &buf)) {
-            free(newdir);
-            continue;
-        }
-        /* if it is a directory, down search */
-        if (S_ISDIR(buf.st_mode)) {
-            rc = df_search(newdir, prefix, info, ninfo, sd, nspace, rank, uri);
-            free(newdir);
-            if (PMIX_SUCCESS == rc) {
-                closedir(cur_dirp);
-                return rc;
-            }
-            continue;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "pmix:tcp: checking %s vs %s", dir_entry->d_name, prefix);
-        /* see if it starts with our prefix */
-        if (0 == strncmp(dir_entry->d_name, prefix, strlen(prefix))) {
-            /* try to read this file */
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "pmix:tcp: reading file %s", newdir);
-            rc = parse_uri_file(newdir, &suri, &nsp, &rk);
-            if (PMIX_SUCCESS == rc) {
-                /* go ahead and try to connect */
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "pmix:tcp: attempting to connect to %s", suri);
-                if (PMIX_SUCCESS == try_connect(suri, sd, info, ninfo)) {
-                    (*nspace) = nsp;
-                    *rank = rk;
-                    closedir(cur_dirp);
-                    *uri = suri;
-                    free(newdir);
-                    return PMIX_SUCCESS;
-                }
-                free(suri);
-                free(nsp);
-            }
-        }
-        free(newdir);
-    }
-    closedir(cur_dirp);
-    return PMIX_ERR_NOT_FOUND;
-}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tcp/ptl_tcp_component.c pmix-4.0.0/src/mca/ptl/tcp/ptl_tcp_component.c
--- pmix-3.2.2~rc1/src/mca/ptl/tcp/ptl_tcp_component.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tcp/ptl_tcp_component.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,2211 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennptlee and The University
- *                         of Tennptlee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2017-2018 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018-2020 IBM Corporation.  All rights reserved.
- * Copyright (c) 2019      Mellanox Technologies, Inc. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * These symbols are in a file by themselves to provide nice linker
- * semantics.  Since linkers generally pull in symbols by object
- * files, keeping these symbols as the only symbols in this file
- * prevents utility programs such as "ompi_info" from having to import
- * entire components just to query their version and parameters.
- */
-
-#include "src/include/pmix_config.h"
-#include "include/pmix_common.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#include <fcntl.h>
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
-#endif
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-#include <ctype.h>
-#include <stdio.h>
-
-#include "src/include/pmix_socket_errno.h"
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/util/fd.h"
-#include "src/util/net.h"
-#include "src/util/name_fns.h"
-#include "src/util/os_dirpath.h"
-#include "src/util/os_path.h"
-#include "src/util/parse_options.h"
-#include "src/util/pif.h"
-#include "src/util/pmix_environ.h"
-#include "src/util/show_help.h"
-#include "src/util/strnlen.h"
-#include "src/event/pmix_event.h"
-#include "src/common/pmix_iof.h"
-#include "src/server/pmix_server_ops.h"
-#include "src/mca/bfrops/base/base.h"
-#include "src/mca/gds/base/base.h"
-#include "src/mca/psec/base/base.h"
-
-#include "src/mca/ptl/base/base.h"
-#include "src/mca/ptl/tcp/ptl_tcp.h"
-
-static pmix_status_t component_open(void);
-static pmix_status_t component_close(void);
-static int component_register(void);
-static int component_query(pmix_mca_base_module_t **module, int *priority);
-static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo,
-                                    bool *need_listener);
-static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env);
-/*
- * Instantiate the public struct with all of our public information
- * and pointers to our public functions in it
- */
- PMIX_EXPORT pmix_ptl_tcp_component_t mca_ptl_tcp_component = {
-    .super = {
-        .base = {
-            PMIX_PTL_BASE_VERSION_1_0_0,
-
-            /* Component name and version */
-            .pmix_mca_component_name = "tcp",
-            PMIX_MCA_BASE_MAKE_VERSION(component,
-                                       PMIX_MAJOR_VERSION,
-                                       PMIX_MINOR_VERSION,
-                                       PMIX_RELEASE_VERSION),
-
-            /* Component open and close functions */
-            .pmix_mca_open_component = component_open,
-            .pmix_mca_close_component = component_close,
-            .pmix_mca_register_component_params = component_register,
-            .pmix_mca_query_component = component_query
-        },
-        .priority = 30,
-        .uri = NULL,
-        .setup_listener = setup_listener,
-        .setup_fork = setup_fork
-    },
-    .session_tmpdir = NULL,
-    .system_tmpdir = NULL,
-    .if_include = NULL,
-    .if_exclude = NULL,
-    .ipv4_port = 0,
-    .ipv6_port = 0,
-    .disable_ipv4_family = false,
-    .disable_ipv6_family = true,
-    .session_filename = NULL,
-    .nspace_filename = NULL,
-    .pid_filename = NULL,
-    .system_filename = NULL,
-    .rendezvous_filename = NULL,
-    .wait_to_connect = 4,
-    .max_retries = 2,
-    .report_uri = NULL,
-    .remote_connections = false,
-    .handshake_wait_time = 4,
-    .handshake_max_retries = 2
-};
-
-static char **split_and_resolve(char **orig_str, char *name);
-static void connection_handler(int sd, short args, void *cbdata);
-static void cnct_cbfunc(pmix_status_t status,
-                        pmix_proc_t *proc, void *cbdata);
-static void _check_cached_events(pmix_peer_t *peer);
-
-static int component_register(void)
-{
-    pmix_mca_base_component_t *component = &mca_ptl_tcp_component.super.base;
-
-    (void)pmix_mca_base_component_var_register(component, "server_uri",
-                                               "URI of a server a tool wishes to connect to - either the "
-                                               "URI itself, or file:path-to-file-containing-uri",
-                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                               PMIX_INFO_LVL_2,
-                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &mca_ptl_tcp_component.super.uri);
-
-    (void)pmix_mca_base_component_var_register(component, "report_uri",
-                                               "Output URI [- => stdout, + => stderr, or filename]",
-                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                               PMIX_INFO_LVL_2,
-                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &mca_ptl_tcp_component.report_uri);
-
-    (void)pmix_mca_base_component_var_register(component, "remote_connections",
-                                               "Enable connections from remote tools",
-                                               PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                               PMIX_INFO_LVL_2,
-                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &mca_ptl_tcp_component.remote_connections);
-
-    (void)pmix_mca_base_component_var_register(component, "if_include",
-                                               "Comma-delimited list of devices and/or CIDR notation of TCP networks (e.g., \"eth0,192.168.0.0/16\").  Mutually exclusive with ptl_tcp_if_exclude.",
-                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                               PMIX_INFO_LVL_2,
-                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &mca_ptl_tcp_component.if_include);
-
-    (void)pmix_mca_base_component_var_register(component, "if_exclude",
-                                               "Comma-delimited list of devices and/or CIDR notation of TCP networks to NOT use -- all devices not matching these specifications will be used (e.g., \"eth0,192.168.0.0/16\").  If set to a non-default value, it is mutually exclusive with ptl_tcp_if_include.",
-                                               PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                               PMIX_INFO_LVL_2,
-                                               PMIX_MCA_BASE_VAR_SCOPE_LOCAL,
-                                               &mca_ptl_tcp_component.if_exclude);
-
-    /* if_include and if_exclude need to be mutually exclusive */
-    if (NULL != mca_ptl_tcp_component.if_include &&
-        NULL != mca_ptl_tcp_component.if_exclude) {
-        /* Return ERR_NOT_AVAILABLE so that a warning message about
-           "open" failing is not printed */
-        pmix_show_help("help-ptl-tcp.txt", "include-exclude", true,
-                       mca_ptl_tcp_component.if_include,
-                       mca_ptl_tcp_component.if_exclude);
-        return PMIX_ERR_NOT_AVAILABLE;
-    }
-
-    (void)pmix_mca_base_component_var_register(component, "ipv4_port",
-                                          "IPv4 port to be used",
-                                          PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.ipv4_port);
-
-    (void)pmix_mca_base_component_var_register(component, "ipv6_port",
-                                          "IPv6 port to be used",
-                                          PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.ipv6_port);
-
-    (void)pmix_mca_base_component_var_register(component, "disable_ipv4_family",
-                                          "Disable the IPv4 interfaces",
-                                          PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.disable_ipv4_family);
-
-    (void)pmix_mca_base_component_var_register(component, "disable_ipv6_family",
-                                          "Disable the IPv6 interfaces",
-                                          PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.disable_ipv6_family);
-
-    (void)pmix_mca_base_component_var_register(component, "connection_wait_time",
-                                          "Number of seconds to wait for the server connection file to appear",
-                                          PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.wait_to_connect);
-
-    (void)pmix_mca_base_component_var_register(component, "max_retries",
-                                          "Number of times to look for the connection file before quitting",
-                                          PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.max_retries);
-
-    (void)pmix_mca_base_component_var_register(component, "handshake_wait_time",
-                                          "Number of seconds to wait for the server reply to the handshake request",
-                                          PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.handshake_wait_time);
-
-    (void)pmix_mca_base_component_var_register(component, "handshake_max_retries",
-                                          "Number of times to retry the handshake request before giving up",
-                                          PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                          PMIX_INFO_LVL_4,
-                                          PMIX_MCA_BASE_VAR_SCOPE_READONLY,
-                                          &mca_ptl_tcp_component.handshake_max_retries);
-
-    return PMIX_SUCCESS;
-}
-
-static char *urifile = NULL;
-static bool created_rendezvous_file = false;
-static bool created_session_tmpdir = false;
-static bool created_system_tmpdir = false;
-static bool created_system_filename = false;
-static bool created_session_filename = false;
-static bool created_nspace_filename = false;
-static bool created_pid_filename = false;
-static bool created_urifile = false;
-
-static pmix_status_t component_open(void)
-{
-    char *tdir;
-
-    memset(&mca_ptl_tcp_component.connection, 0, sizeof(mca_ptl_tcp_component.connection));
-
-    /* check for environ-based directives
-     * on system tmpdir to use */
-    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
-        PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        mca_ptl_tcp_component.session_tmpdir = strdup(pmix_server_globals.tmpdir);
-    } else {
-        if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) {
-            mca_ptl_tcp_component.session_tmpdir = strdup(tdir);
-        } else {
-            mca_ptl_tcp_component.session_tmpdir = strdup(pmix_tmp_directory());
-        }
-    }
-
-    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
-        PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        mca_ptl_tcp_component.system_tmpdir = strdup(pmix_server_globals.system_tmpdir);
-    } else {
-        if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) {
-            mca_ptl_tcp_component.system_tmpdir = strdup(tdir);
-        } else {
-            mca_ptl_tcp_component.system_tmpdir = strdup(pmix_tmp_directory());
-        }
-    }
-
-    if (NULL != mca_ptl_tcp_component.report_uri &&
-        0 != strcmp(mca_ptl_tcp_component.report_uri, "-") &&
-        0 != strcmp(mca_ptl_tcp_component.report_uri, "+")) {
-        urifile = strdup(mca_ptl_tcp_component.report_uri);
-    }
-
-    if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer) ||
-        PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        if (NULL != (tdir = getenv("PMIX_LAUNCHER_RENDEZVOUS_FILE"))) {
-            mca_ptl_tcp_component.rendezvous_filename = strdup(tdir);
-        }
-    }
-
-    return PMIX_SUCCESS;
-}
-
-
-pmix_status_t component_close(void)
-{
-    if (NULL != mca_ptl_tcp_component.system_filename) {
-        if (created_system_filename) {
-            remove(mca_ptl_tcp_component.system_filename);
-        }
-        free(mca_ptl_tcp_component.system_filename);
-    }
-    if (NULL != mca_ptl_tcp_component.session_filename) {
-        if (created_session_filename) {
-            remove(mca_ptl_tcp_component.session_filename);
-        }
-        free(mca_ptl_tcp_component.session_filename);
-    }
-    if (NULL != mca_ptl_tcp_component.nspace_filename) {
-        if (created_nspace_filename) {
-            remove(mca_ptl_tcp_component.nspace_filename);
-        }
-        free(mca_ptl_tcp_component.nspace_filename);
-    }
-    if (NULL != mca_ptl_tcp_component.pid_filename) {
-        if (created_pid_filename) {
-            remove(mca_ptl_tcp_component.pid_filename);
-        }
-        free(mca_ptl_tcp_component.pid_filename);
-    }
-    if (NULL != mca_ptl_tcp_component.rendezvous_filename) {
-        if (created_rendezvous_file) {
-            remove(mca_ptl_tcp_component.rendezvous_filename);
-        }
-        free(mca_ptl_tcp_component.rendezvous_filename);
-    }
-    if (NULL != urifile) {
-        if (created_urifile) {
-            /* remove the file */
-            remove(urifile);
-        }
-        free(urifile);
-        urifile = NULL;
-    }
-    if (NULL != mca_ptl_tcp_component.session_tmpdir) {
-        /* if I am a tool or a server, then remove my session directory if empty */
-        if (created_session_tmpdir) {
-            pmix_os_dirpath_destroy(mca_ptl_tcp_component.session_tmpdir,
-                                    true, NULL);
-        }
-        free(mca_ptl_tcp_component.session_tmpdir);
-    }
-    if (NULL != mca_ptl_tcp_component.system_tmpdir) {
-        if (created_system_tmpdir) {
-            pmix_os_dirpath_destroy(mca_ptl_tcp_component.system_tmpdir,
-                                    true, NULL);
-        }
-        free(mca_ptl_tcp_component.system_tmpdir);
-    }
-    return PMIX_SUCCESS;
-}
-
-static int component_query(pmix_mca_base_module_t **module, int *priority)
-{
-    *module = (pmix_mca_base_module_t*)&pmix_ptl_tcp_module;
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env)
-{
-    pmix_setenv("PMIX_SERVER_TMPDIR", mca_ptl_tcp_component.session_tmpdir, true, env);
-    pmix_setenv("PMIX_SYSTEM_TMPDIR", mca_ptl_tcp_component.system_tmpdir, true, env);
-
-    return PMIX_SUCCESS;
-}
-
-/* if we are the server, then we need to discover the available
- * interfaces, filter them thru any given directives, and select
- * the one we will listen on for connection requests. This will
- * be a loopback device by default, unless we are asked to support
- * tool connections - in that case, we will take a non-loopback
- * device by default, if one is available after filtering directives
- *
- * If we are a tool and were give a rendezvous file, then we first
- * check to see if it already exists. If it does, then this is the
- * connection info we are to use. If it doesn't, then this is the
- * name of the file we are to use to store our listener info.
- *
- * If we are a server and are given a rendezvous file, then that is
- * is the name of the file we are to use to store our listener info.
- *
- * NOTE: we accept MCA parameters, but info keys override them
- */
-static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo,
-                                    bool *need_listener)
-{
-    int flags = 0;
-    pmix_listener_t *lt;
-    int i, rc, saveindex = -1;
-    char **interfaces = NULL;
-    bool including = false;
-    char name[32];
-    struct sockaddr_storage my_ss;
-    int kindex;
-    size_t n;
-    bool system_tool = false;
-    bool tool_support = false;
-    pmix_socklen_t addrlen;
-    char *prefix, myhost[PMIX_MAXHOSTNAMELEN] = {0};
-    char myconnhost[PMIX_MAXHOSTNAMELEN] = {0};
-    int myport;
-    pmix_kval_t *urikv;
-    FILE *fp;
-    pid_t mypid;
-    struct stat sbuf;
-    time_t mytime;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "ptl:tcp setup_listener");
-
-    /* if we are not a server, then we shouldn't be doing this */
-    if (!PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
-        return PMIX_ERR_NOT_SUPPORTED;
-    }
-
-    /* scan the info keys and process any override instructions */
-    if (NULL != info) {
-        for (n=0; n < ninfo; n++) {
-            if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_INCLUDE)) {
-                if (NULL != mca_ptl_tcp_component.if_include) {
-                    free(mca_ptl_tcp_component.if_include);
-                }
-                mca_ptl_tcp_component.if_include = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_EXCLUDE)) {
-                if (NULL != mca_ptl_tcp_component.if_exclude) {
-                    free(mca_ptl_tcp_component.if_exclude);
-                }
-                mca_ptl_tcp_component.if_exclude = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV4_PORT)) {
-                mca_ptl_tcp_component.ipv4_port = info[n].value.data.integer;
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV6_PORT)) {
-                mca_ptl_tcp_component.ipv6_port = info[n].value.data.integer;
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV4)) {
-                mca_ptl_tcp_component.disable_ipv4_family = PMIX_INFO_TRUE(&info[n]);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV6)) {
-                mca_ptl_tcp_component.disable_ipv6_family = PMIX_INFO_TRUE(&info[n]);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_REMOTE_CONNECTIONS)) {
-                mca_ptl_tcp_component.remote_connections = PMIX_INFO_TRUE(&info[n]);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_URI)) {
-                if (NULL != mca_ptl_tcp_component.super.uri) {
-                    free(mca_ptl_tcp_component.super.uri);
-                }
-                mca_ptl_tcp_component.super.uri = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_REPORT_URI)) {
-                if (NULL != mca_ptl_tcp_component.report_uri) {
-                    free(mca_ptl_tcp_component.report_uri);
-                }
-                mca_ptl_tcp_component.report_uri = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_TMPDIR)) {
-                if (NULL != mca_ptl_tcp_component.session_tmpdir) {
-                    free(mca_ptl_tcp_component.session_tmpdir);
-                }
-                mca_ptl_tcp_component.session_tmpdir = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SYSTEM_TMPDIR)) {
-                if (NULL != mca_ptl_tcp_component.system_tmpdir) {
-                    free(mca_ptl_tcp_component.system_tmpdir);
-                }
-                mca_ptl_tcp_component.system_tmpdir = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SYSTEM_SUPPORT)) {
-                system_tool = PMIX_INFO_TRUE(&info[n]);
-            } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) {
-                tool_support = PMIX_INFO_TRUE(&info[n]);
-            } else if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer) &&
-                       PMIX_CHECK_KEY(&info[n], PMIX_LAUNCHER_RENDEZVOUS_FILE)) {
-                if (NULL != mca_ptl_tcp_component.rendezvous_filename) {
-                    free(mca_ptl_tcp_component.rendezvous_filename);
-                }
-                mca_ptl_tcp_component.rendezvous_filename = strdup(info[n].value.data.string);
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_MAX_RETRIES)) {
-                 PMIX_VALUE_GET_NUMBER(rc, &info[n].value, mca_ptl_tcp_component.max_retries, int);
-                 if (PMIX_SUCCESS != rc) {
-                    return rc;
-                 }
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_RETRY_DELAY)) {
-                 PMIX_VALUE_GET_NUMBER(rc, &info[n].value, mca_ptl_tcp_component.wait_to_connect, int);
-                 if (PMIX_SUCCESS != rc) {
-                    return rc;
-                 }
-            }
-        }
-    }
-
-    /* if interface include was given, construct a list
-     * of those interfaces which match the specifications - remember,
-     * the includes could be given as named interfaces, IP addrs, or
-     * subnet+mask
-     */
-    if (NULL != mca_ptl_tcp_component.if_include) {
-        interfaces = split_and_resolve(&mca_ptl_tcp_component.if_include,
-                                       "include");
-        including = true;
-    } else if (NULL != mca_ptl_tcp_component.if_exclude) {
-        interfaces = split_and_resolve(&mca_ptl_tcp_component.if_exclude,
-                                       "exclude");
-        including = false;
-    }
-
-    /* look at all available interfaces and pick one - we default to a
-     * loopback interface if available, but otherwise pick the first
-     * available interface since we are only talking locally */
-    for (i = pmix_ifbegin(); i >= 0; i = pmix_ifnext(i)) {
-        if (PMIX_SUCCESS != pmix_ifindextoaddr(i, (struct sockaddr*)&my_ss, sizeof(my_ss))) {
-            pmix_output (0, "ptl_tcp: problems getting address for index %i (kernel index %i)\n",
-                         i, pmix_ifindextokindex(i));
-            continue;
-        }
-        /* ignore non-ip4/6 interfaces */
-        if (AF_INET != my_ss.ss_family &&
-            AF_INET6 != my_ss.ss_family) {
-            continue;
-        }
-        /* get the name for diagnostic purposes */
-        pmix_ifindextoname(i, name, sizeof(name));
-
-        /* ignore any virtual interfaces */
-        if (0 == strncmp(name, "vir", 3)) {
-            continue;
-        }
-        /* ignore any interfaces in a disabled family */
-        if (AF_INET == my_ss.ss_family &&
-            mca_ptl_tcp_component.disable_ipv4_family) {
-            continue;
-        } else if (AF_INET6 == my_ss.ss_family &&
-                   mca_ptl_tcp_component.disable_ipv6_family) {
-            continue;
-        }
-        /* get the kernel index */
-        kindex = pmix_ifindextokindex(i);
-        if (kindex <= 0) {
-            continue;
-        }
-        pmix_output_verbose(10, pmix_ptl_base_framework.framework_output,
-                            "WORKING INTERFACE %d KERNEL INDEX %d FAMILY: %s", i, kindex,
-                            (AF_INET == my_ss.ss_family) ? "V4" : "V6");
-        /* handle include/exclude directives */
-        if (NULL != interfaces) {
-            /* check for match */
-            rc = pmix_ifmatches(kindex, interfaces);
-            /* if one of the network specifications isn't parseable, then
-             * error out as we can't do what was requested
-             */
-            if (PMIX_ERR_FABRIC_NOT_PARSEABLE == rc) {
-                pmix_show_help("help-ptl-tcp.txt", "not-parseable", true);
-                pmix_argv_free(interfaces);
-                return PMIX_ERR_BAD_PARAM;
-            }
-            /* if we are including, then ignore this if not present */
-            if (including) {
-                if (PMIX_SUCCESS != rc) {
-                    pmix_output_verbose(10, pmix_ptl_base_framework.framework_output,
-                                        "ptl:tcp:init rejecting interface %s (not in include list)", name);
-                    continue;
-                }
-            } else {
-                /* we are excluding, so ignore if present */
-                if (PMIX_SUCCESS == rc) {
-                    pmix_output_verbose(10, pmix_ptl_base_framework.framework_output,
-                                        "ptl:tcp:init rejecting interface %s (in exclude list)", name);
-                    continue;
-                }
-            }
-        }
-
-        /* if this is the loopback device and they didn't enable
-         * remote connections, then we are done */
-        if (pmix_ifisloopback(i)) {
-            if (mca_ptl_tcp_component.remote_connections) {
-                /* ignore loopback */
-                continue;
-            } else {
-                pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
-                                    "ptl:tcp:init loopback interface %s selected", name);
-                saveindex = i;
-                break;
-            }
-        } else {
-            /* if this is the first one we found, then hang on to it - we
-             * will use it if a loopback device is not found */
-            if (saveindex < 0) {
-                saveindex = i;
-            }
-        }
-    }
-    /* cleanup */
-    if (NULL != interfaces) {
-        pmix_argv_free(interfaces);
-    }
-
-    /* if we didn't find anything, then we cannot operate */
-    if (saveindex < 0) {
-        return PMIX_ERR_NOT_AVAILABLE;
-    }
-
-    /* save the connection */
-    if (PMIX_SUCCESS != pmix_ifindextoaddr(saveindex,
-                                           (struct sockaddr*)&mca_ptl_tcp_component.connection,
-                                           sizeof(struct sockaddr))) {
-        pmix_output (0, "ptl:tcp: problems getting address for kernel index %i\n",
-                     pmix_ifindextokindex(saveindex));
-        return PMIX_ERR_NOT_AVAILABLE;
-    }
-
-    /* set the port */
-    if (AF_INET == mca_ptl_tcp_component.connection.ss_family) {
-        ((struct sockaddr_in*) &mca_ptl_tcp_component.connection)->sin_port = htons(mca_ptl_tcp_component.ipv4_port);
-        if (0 != mca_ptl_tcp_component.ipv4_port) {
-            flags = 1;
-        }
-    } else if (AF_INET6 == mca_ptl_tcp_component.connection.ss_family) {
-        ((struct sockaddr_in6*) &mca_ptl_tcp_component.connection)->sin6_port = htons(mca_ptl_tcp_component.ipv6_port);
-        if (0 != mca_ptl_tcp_component.ipv6_port) {
-            flags = 1;
-        }
-    }
-
-    lt = PMIX_NEW(pmix_listener_t);
-    lt->varname = strdup("PMIX_SERVER_URI3:PMIX_SERVER_URI2:PMIX_SERVER_URI21");
-    lt->protocol = PMIX_PROTOCOL_V2;
-    lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_tcp_module;
-    lt->cbfunc = connection_handler;
-
-    addrlen = sizeof(struct sockaddr_storage);
-    /* create a listen socket for incoming connection attempts */
-    lt->socket = socket(mca_ptl_tcp_component.connection.ss_family, SOCK_STREAM, 0);
-    if (lt->socket < 0) {
-        printf("%s:%d socket() failed\n", __FILE__, __LINE__);
-        goto sockerror;
-    }
-
-    /* set reusing ports flag */
-    if (setsockopt (lt->socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&flags, sizeof(flags)) < 0) {
-        pmix_output(0, "ptl:tcp:create_listen: unable to set the "
-                    "SO_REUSEADDR option (%s:%d)\n",
-                    strerror(pmix_socket_errno), pmix_socket_errno);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* Set the socket to close-on-exec so that no children inherit
-     * this FD */
-    if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) {
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    if (bind(lt->socket, (struct sockaddr*)&mca_ptl_tcp_component.connection, sizeof(struct sockaddr)) < 0) {
-        printf("%s:%d bind() failed: %s\n", __FILE__, __LINE__, strerror(errno));
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* resolve assigned port */
-    if (getsockname(lt->socket, (struct sockaddr*)&mca_ptl_tcp_component.connection, &addrlen) < 0) {
-        pmix_output(0, "ptl:tcp:create_listen: getsockname(): %s (%d)",
-                    strerror(pmix_socket_errno), pmix_socket_errno);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* setup listen backlog to maximum allowed by kernel */
-    if (listen(lt->socket, SOMAXCONN) < 0) {
-        printf("%s:%d listen() failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* set socket up to be non-blocking, otherwise accept could block */
-    if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) {
-        printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-    flags |= O_NONBLOCK;
-    if (fcntl(lt->socket, F_SETFL, flags) < 0) {
-        printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    gethostname(myhost, sizeof(myhost)-1);
-    if (AF_INET == mca_ptl_tcp_component.connection.ss_family) {
-        prefix = "tcp4://";
-        myport = ntohs(((struct sockaddr_in*) &mca_ptl_tcp_component.connection)->sin_port);
-        inet_ntop(AF_INET, &((struct sockaddr_in*) &mca_ptl_tcp_component.connection)->sin_addr,
-                  myconnhost, PMIX_MAXHOSTNAMELEN-1);
-    } else if (AF_INET6 == mca_ptl_tcp_component.connection.ss_family) {
-        prefix = "tcp6://";
-        myport = ntohs(((struct sockaddr_in6*) &mca_ptl_tcp_component.connection)->sin6_port);
-        inet_ntop(AF_INET6, &((struct sockaddr_in6*) &mca_ptl_tcp_component.connection)->sin6_addr,
-                  myconnhost, PMIX_MAXHOSTNAMELEN-1);
-    } else {
-        goto sockerror;
-    }
-
-    rc = asprintf(&lt->uri, "%s.%d;%s%s:%d", pmix_globals.myid.nspace, pmix_globals.myid.rank, prefix, myconnhost, myport);
-    if (0 > rc || NULL == lt->uri) {
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "ptl:tcp URI %s", lt->uri);
-
-    /* save the URI internally so we can report it */
-    urikv = PMIX_NEW(pmix_kval_t);
-    urikv->key = strdup(PMIX_SERVER_URI);
-    PMIX_VALUE_CREATE(urikv->value, 1);
-    PMIX_VALUE_LOAD(urikv->value, lt->uri, PMIX_STRING);
-    PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                      &pmix_globals.myid, PMIX_INTERNAL,
-                      urikv);
-    PMIX_RELEASE(urikv);  // maintain accounting
-
-    if (NULL != mca_ptl_tcp_component.report_uri) {
-        /* if the string is a "-", then output to stdout */
-        if (0 == strcmp(mca_ptl_tcp_component.report_uri, "-")) {
-            fprintf(stdout, "%s\n", lt->uri);
-        } else if (0 == strcmp(mca_ptl_tcp_component.report_uri, "+")) {
-            /* output to stderr */
-            fprintf(stderr, "%s\n", lt->uri);
-        } else {
-            /* must be a file */
-            FILE *fp;
-            fp = fopen(mca_ptl_tcp_component.report_uri, "w");
-            if (NULL == fp) {
-                pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.report_uri);
-                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-                CLOSE_THE_SOCKET(lt->socket);
-                free(mca_ptl_tcp_component.system_filename);
-                mca_ptl_tcp_component.system_filename = NULL;
-                goto sockerror;
-            }
-            /* output my nspace and rank plus the URI */
-            fprintf(fp, "%s\n", lt->uri);
-            /* add a flag that indicates we accept v2.1 protocols */
-            fprintf(fp, "v%s\n", PMIX_VERSION);
-            fclose(fp);
-            created_urifile = true;
-        }
-    }
-
-    /* if we were given a rendezvous file, then drop it */
-    if (NULL != mca_ptl_tcp_component.rendezvous_filename) {
-        FILE *fp;
-        /* if we are a tool and the file already exists, then we
-         * just use it as providing the rendezvous info for our
-         * server */
-        if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
-            struct stat buf;
-            /* coverity[toctou] */
-            if (0 == stat(mca_ptl_tcp_component.rendezvous_filename, &buf)) {
-                goto nextstep;
-            }
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "WRITING RENDEZVOUS FILE %s",
-                            mca_ptl_tcp_component.rendezvous_filename);
-        fp = fopen(mca_ptl_tcp_component.rendezvous_filename, "w");
-        if (NULL == fp) {
-            pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.rendezvous_filename);
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.rendezvous_filename);
-            mca_ptl_tcp_component.rendezvous_filename = NULL;
-            goto sockerror;
-        }
-
-        /* output my nspace and rank plus the URI */
-        fprintf(fp, "%s\n", lt->uri);
-        /* add a flag that indicates we accept v3.0 protocols */
-        fprintf(fp, "v%s\n", PMIX_VERSION);
-        fclose(fp);
-        /* set the file mode */
-        if (0 != chmod(mca_ptl_tcp_component.rendezvous_filename, S_IRUSR | S_IWUSR | S_IRGRP)) {
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.rendezvous_filename);
-            mca_ptl_tcp_component.rendezvous_filename = NULL;
-            goto sockerror;
-        }
-        created_rendezvous_file = true;
-    }
-
-  nextstep:
-    /* if we are going to support tools, then drop contact file(s) */
-    if (system_tool) {
-        if (0 == stat(mca_ptl_tcp_component.system_tmpdir, &sbuf)) {
-            /* already exists - check if it is a directory */
-            if (! S_ISDIR(sbuf.st_mode)) {
-                /* nope - we are hosed */
-                pmix_output(0, "System tmpdir %s is not a directory\n", mca_ptl_tcp_component.system_tmpdir);
-                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-                CLOSE_THE_SOCKET(lt->socket);
-                goto sockerror;
-            }
-        } else {
-            /* need to create it */
-            rc = mkdir(mca_ptl_tcp_component.system_tmpdir, 0755);
-            if (0 != rc) {
-                pmix_output(0, "System tmpdir %s could not be created\n", mca_ptl_tcp_component.system_tmpdir);
-                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-                CLOSE_THE_SOCKET(lt->socket);
-                goto sockerror;
-            }
-            created_system_tmpdir = true;
-        }
-        if (0 > asprintf(&mca_ptl_tcp_component.system_filename, "%s/pmix.sys.%s",
-                         mca_ptl_tcp_component.system_tmpdir, myhost)) {
-            CLOSE_THE_SOCKET(lt->socket);
-            goto sockerror;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "WRITING SYSTEM FILE %s",
-                            mca_ptl_tcp_component.system_filename);
-        fp = fopen(mca_ptl_tcp_component.system_filename, "w");
-        if (NULL == fp) {
-            pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.system_filename);
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.system_filename);
-            mca_ptl_tcp_component.system_filename = NULL;
-            goto sockerror;
-        }
-
-        /* output my nspace and rank plus the URI */
-        fprintf(fp, "%s\n", lt->uri);
-        /* add a flag that indicates we accept v3.0 protocols */
-        fprintf(fp, "v%s\n", PMIX_VERSION);
-        /* output our pid */
-        fprintf(fp, "%lu\n", (unsigned long)getpid());
-        /* output our effective uid and gid */
-        fprintf(fp, "%lu:%lu\n", (unsigned long)geteuid(), (unsigned long)getegid());
-        /* output the time */
-        mytime = time(NULL);
-        fprintf(fp, "%s\n", ctime(&mytime));
-        fclose(fp);
-        /* set the file mode */
-        if (0 != chmod(mca_ptl_tcp_component.system_filename, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) {
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.system_filename);
-            mca_ptl_tcp_component.system_filename = NULL;
-            goto sockerror;
-        }
-        created_system_filename = true;
-    }
-
-    if (tool_support) {
-        if (0 == stat(mca_ptl_tcp_component.session_tmpdir, &sbuf)) {
-            /* already exists - check if it is a directory */
-            if (! S_ISDIR(sbuf.st_mode)) {
-                /* nope - we are hosed */
-                pmix_output(0, "Session tmpdir %s is not a directory\n", mca_ptl_tcp_component.session_tmpdir);
-                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-                CLOSE_THE_SOCKET(lt->socket);
-                goto sockerror;
-            }
-        } else {
-            /* need to create it */
-            rc = mkdir(mca_ptl_tcp_component.session_tmpdir, 0755);
-            if (0 != rc) {
-                pmix_output(0, "Session tmpdir %s could not be created\n", mca_ptl_tcp_component.session_tmpdir);
-                PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-                CLOSE_THE_SOCKET(lt->socket);
-                goto sockerror;
-            }
-            created_session_tmpdir = true;
-        }
-        /* now output to a file based on pid */
-        mypid = getpid();
-        if (0 > asprintf(&mca_ptl_tcp_component.pid_filename, "%s/pmix.%s.tool.%d",
-                         mca_ptl_tcp_component.session_tmpdir, myhost, mypid)) {
-            CLOSE_THE_SOCKET(lt->socket);
-            goto sockerror;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "WRITING PID TOOL FILE %s",
-                            mca_ptl_tcp_component.pid_filename);
-        fp = fopen(mca_ptl_tcp_component.pid_filename, "w");
-        if (NULL == fp) {
-            pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.pid_filename);
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.pid_filename);
-            mca_ptl_tcp_component.pid_filename = NULL;
-            goto sockerror;
-        }
-
-        /* output my URI */
-        fprintf(fp, "%s\n", lt->uri);
-        /* add a flag that indicates we accept v2.1 protocols */
-        fprintf(fp, "%s\n", PMIX_VERSION);
-        /* output our pid */
-        fprintf(fp, "%lu\n", (unsigned long)getpid());
-        /* output our effective uid and gid */
-        fprintf(fp, "%lu:%lu\n", (unsigned long)geteuid(), (unsigned long)getegid());
-        /* output the time */
-        mytime = time(NULL);
-        fprintf(fp, "%s\n", ctime(&mytime));
-        fclose(fp);
-        /* set the file mode */
-        if (0 != chmod(mca_ptl_tcp_component.pid_filename, S_IRUSR | S_IWUSR | S_IRGRP)) {
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.pid_filename);
-            mca_ptl_tcp_component.pid_filename = NULL;
-            goto sockerror;
-        }
-        created_pid_filename = true;
-
-        /* now output it into a file based on my nspace */
-
-        if (0 > asprintf(&mca_ptl_tcp_component.nspace_filename, "%s/pmix.%s.tool.%s",
-                         mca_ptl_tcp_component.session_tmpdir, myhost, pmix_globals.myid.nspace)) {
-            CLOSE_THE_SOCKET(lt->socket);
-            goto sockerror;
-        }
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "WRITING NSPACE TOOL FILE %s",
-                            mca_ptl_tcp_component.nspace_filename);
-        fp = fopen(mca_ptl_tcp_component.nspace_filename, "w");
-        if (NULL == fp) {
-            pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.nspace_filename);
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.nspace_filename);
-            mca_ptl_tcp_component.nspace_filename = NULL;
-            goto sockerror;
-        }
-
-        /* output my URI */
-        fprintf(fp, "%s\n", lt->uri);
-        /* add a flag that indicates we accept v2.1 protocols */
-        fprintf(fp, "%s\n", PMIX_VERSION);
-        /* output our pid */
-        fprintf(fp, "%lu\n", (unsigned long)getpid());
-        /* output our effective uid and gid */
-        fprintf(fp, "%lu:%lu\n", (unsigned long)geteuid(), (unsigned long)getegid());
-        /* output the time */
-        mytime = time(NULL);
-        fprintf(fp, "%s\n", ctime(&mytime));
-        fclose(fp);
-        /* set the file mode */
-        if (0 != chmod(mca_ptl_tcp_component.nspace_filename, S_IRUSR | S_IWUSR | S_IRGRP)) {
-            PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE);
-            CLOSE_THE_SOCKET(lt->socket);
-            free(mca_ptl_tcp_component.nspace_filename);
-            mca_ptl_tcp_component.nspace_filename = NULL;
-            goto sockerror;
-        }
-        created_nspace_filename = true;
-    }
-    /* if we are a tool and connected, then register any rendezvous files for cleanup */
-    if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer) && pmix_globals.connected) {
-        char **clnup = NULL, *cptr = NULL;
-        pmix_info_t dir;
-        if (NULL != mca_ptl_tcp_component.nspace_filename) {
-            pmix_argv_append_nosize(&clnup, mca_ptl_tcp_component.nspace_filename);
-        }
-        if (NULL != mca_ptl_tcp_component.session_filename) {
-            pmix_argv_append_nosize(&clnup, mca_ptl_tcp_component.session_filename);
-        }
-        if (NULL != clnup) {
-            cptr = pmix_argv_join(clnup, ',');
-            pmix_argv_free(clnup);
-            PMIX_INFO_LOAD(&dir, PMIX_REGISTER_CLEANUP, cptr, PMIX_STRING);
-            free(cptr);
-            PMIx_Job_control_nb(&pmix_globals.myid, 1, &dir, 1, NULL, NULL);
-            PMIX_INFO_DESTRUCT(&dir);
-        }
-    }
-
-    /* we need listener thread support */
-    *need_listener = true;
-    pmix_list_append(&pmix_ptl_globals.listeners, &lt->super);
-
-    return PMIX_SUCCESS;
-
-  sockerror:
-    PMIX_RELEASE(lt);
-    return PMIX_ERROR;
-}
-
-/*
- * Go through a list of argv; if there are any subnet specifications
- * (a.b.c.d/e), resolve them to an interface name (Currently only
- * supporting IPv4).  If unresolvable, warn and remove.
- */
-static char **split_and_resolve(char **orig_str, char *name)
-{
-    int i, ret, save, if_index;
-    char **argv, *str, *tmp;
-    char if_name[PMIX_IF_NAMESIZE];
-    struct sockaddr_storage argv_inaddr, if_inaddr;
-    uint32_t argv_prefix;
-
-    /* Sanity check */
-    if (NULL == orig_str || NULL == *orig_str) {
-        return NULL;
-    }
-
-    argv = pmix_argv_split(*orig_str, ',');
-    if (NULL == argv) {
-        return NULL;
-    }
-    for (save = i = 0; NULL != argv[i]; ++i) {
-        if (isalpha(argv[i][0])) {
-            argv[save++] = argv[i];
-            continue;
-        }
-
-        /* Found a subnet notation.  Convert it to an IP
-           address/netmask.  Get the prefix first. */
-        argv_prefix = 0;
-        tmp = strdup(argv[i]);
-        str = strchr(argv[i], '/');
-        if (NULL == str) {
-            pmix_show_help("help-ptl-tcp.txt", "invalid if_inexclude",
-                           true, name, tmp, "Invalid specification (missing \"/\")");
-            free(argv[i]);
-            free(tmp);
-            continue;
-        }
-        *str = '\0';
-        argv_prefix = atoi(str + 1);
-
-        /* Now convert the IPv4 address */
-        ((struct sockaddr*) &argv_inaddr)->sa_family = AF_INET;
-        ret = inet_pton(AF_INET, argv[i],
-                        &((struct sockaddr_in*) &argv_inaddr)->sin_addr);
-        free(argv[i]);
-
-        if (1 != ret) {
-            pmix_show_help("help-ptl-tcp.txt", "invalid if_inexclude",
-                           true, name, tmp,
-                           "Invalid specification (inet_pton() failed)");
-            free(tmp);
-            continue;
-        }
-        pmix_output_verbose(20, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp: Searching for %s address+prefix: %s / %u",
-                            name,
-                            pmix_net_get_hostname((struct sockaddr*) &argv_inaddr),
-                            argv_prefix);
-
-        /* Go through all interfaces and see if we can find a match */
-        for (if_index = pmix_ifbegin(); if_index >= 0;
-                           if_index = pmix_ifnext(if_index)) {
-            pmix_ifindextoaddr(if_index,
-                               (struct sockaddr*) &if_inaddr,
-                               sizeof(if_inaddr));
-            if (pmix_net_samenetwork((struct sockaddr*) &argv_inaddr,
-                                     (struct sockaddr*) &if_inaddr,
-                                     argv_prefix)) {
-                break;
-            }
-        }
-        /* If we didn't find a match, keep trying */
-        if (if_index < 0) {
-            pmix_show_help("help-ptl-tcp.txt", "invalid if_inexclude",
-                           true, name, tmp,
-                           "Did not find interface matching this subnet");
-            free(tmp);
-            continue;
-        }
-
-        /* We found a match; get the name and replace it in the
-           argv */
-        pmix_ifindextoname(if_index, if_name, sizeof(if_name));
-        pmix_output_verbose(20, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp: Found match: %s (%s)",
-                            pmix_net_get_hostname((struct sockaddr*) &if_inaddr),
-                            if_name);
-        argv[save++] = strdup(if_name);
-        free(tmp);
-    }
-
-    /* The list may have been compressed if there were invalid
-       entries, so ensure we end it with a NULL entry */
-    argv[save] = NULL;
-    free(*orig_str);
-    *orig_str = pmix_argv_join(argv, ',');
-    return argv;
-}
-
-static void connection_handler(int sd, short args, void *cbdata)
-{
-    pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
-    pmix_ptl_hdr_t hdr;
-    pmix_peer_t *peer;
-    pmix_rank_t rank=0;
-    pmix_status_t rc, reply;
-    char *msg, *mg, *version;
-    char *sec, *bfrops, *gds;
-    pmix_bfrop_buffer_type_t bftype;
-    char *nspace = NULL;
-    uint32_t len, u32;
-    size_t cnt, msglen, n;
-    pmix_namespace_t *nptr, *tmp;
-    bool found;
-    pmix_rank_info_t *info;
-    pmix_proc_t proc;
-    pmix_info_t ginfo;
-    pmix_proc_type_t proc_type = PMIX_PROC_TYPE_STATIC_INIT;
-    pmix_byte_object_t cred;
-    pmix_buffer_t buf;
-    uint8_t major, minor, release;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(pnd);
-
-    pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
-                        "ptl:tcp:connection_handler: new connection: %d",
-                        pnd->sd);
-
-    /* ensure the socket is in blocking mode */
-    pmix_ptl_base_set_blocking(pnd->sd);
-
-    /* ensure all is zero'd */
-    memset(&hdr, 0, sizeof(pmix_ptl_hdr_t));
-
-    /* get the header */
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_ptl_hdr_t)))) {
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get the id, authentication and version payload (and possibly
-     * security credential) - to guard against potential attacks,
-     * we'll set an arbitrary limit per a define */
-    if (PMIX_MAX_CRED_SIZE < hdr.nbytes) {
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    if (NULL == (msg = (char*)malloc(hdr.nbytes))) {
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(pnd->sd, msg, hdr.nbytes))) {
-        /* unable to complete the recv */
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:tcp:connection_handler unable to complete recv of connect-ack with client ON SOCKET %d",
-                            pnd->sd);
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    cnt = hdr.nbytes;
-    mg = msg;
-    /* extract the name of the sec module they used */
-    PMIX_STRNLEN(msglen, mg, cnt);
-    if (msglen < cnt) {
-        sec = mg;
-        mg += strlen(sec) + 1;
-        cnt -= strlen(sec) + 1;
-    } else {
-        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_BAD_PARAM;
-        goto error;
-    }
-
-    /* extract any credential so we can validate this connection
-     * before doing anything else */
-    if (sizeof(uint32_t) <= cnt) {
-        memcpy(&len, mg, sizeof(uint32_t));
-        mg += sizeof(uint32_t);
-        cnt -= sizeof(uint32_t);
-    } else {
-        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_BAD_PARAM;
-        goto error;
-    }
-    /* convert it to host byte order */
-    pnd->len = ntohl(len);
-    /* if a credential is present, then create space and
-     * extract it for processing */
-    if (0 < pnd->len) {
-        pnd->cred = (char*)malloc(pnd->len);
-        if (NULL == pnd->cred) {
-            /* probably cannot send an error reply if we are out of memory */
-            free(msg);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd);
-            return;
-        }
-        memcpy(pnd->cred, mg, pnd->len);
-        mg += pnd->len;
-        cnt -= pnd->len;
-    }
-
-    /* get the process type of the connecting peer */
-    if (1 <= cnt) {
-        memcpy(&pnd->flag, mg, 1);
-        ++mg;
-        --cnt;
-    } else {
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_BAD_PARAM;
-        goto error;
-    }
-
-    if (0 == pnd->flag) {
-        /* they must be a client, so get their nspace/rank */
-        PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_CLIENT);
-        PMIX_STRNLEN(msglen, mg, cnt);
-        if (msglen < cnt) {
-            nspace = mg;
-            mg += strlen(nspace) + 1;
-            cnt -= strlen(nspace) + 1;
-        } else {
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-
-        if (sizeof(pmix_rank_t) <= cnt) {
-            /* have to convert this to host order */
-            memcpy(&u32, mg, sizeof(uint32_t));
-            rank = ntohl(u32);
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-        } else {
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-    } else if (1 == pnd->flag) {
-        /* they are a tool */
-        PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_TOOL);
-        /* extract the uid/gid */
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->uid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->gid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-    } else if (2 == pnd->flag) {
-        /* they are a launcher */
-        PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_LAUNCHER);
-        /* extract the uid/gid */
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->uid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->gid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-    } else if (3 == pnd->flag || 6 == pnd->flag) {
-        /* they are a tool or launcher that needs an identifier */
-        if (3 == pnd->flag) {
-            PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_TOOL);
-        } else {
-            PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_LAUNCHER);
-        }
-        /* extract the uid/gid */
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->uid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->gid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-        /* they need an id */
-        pnd->need_id = true;
-    } else if (4 == pnd->flag || 5 == pnd->flag || 7 == pnd->flag || 8 == pnd->flag) {
-        /* they are a tool or launcher that has an identifier - start with our ACLs */
-        if (4 == pnd->flag || 5 == pnd->flag) {
-            PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_TOOL);
-        } else {
-            PMIX_SET_PROC_TYPE(&proc_type, PMIX_PROC_LAUNCHER);
-        }
-        /* extract the uid/gid */
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->uid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-        if (sizeof(uint32_t) <= cnt) {
-            memcpy(&u32, mg, sizeof(uint32_t));
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-            pnd->gid = ntohl(u32);
-        } else {
-           free(msg);
-           /* send an error reply to the client */
-           rc = PMIX_ERR_BAD_PARAM;
-           goto error;
-        }
-        PMIX_STRNLEN(msglen, mg, cnt);
-        if (msglen < cnt) {
-            nspace = mg;
-            mg += strlen(nspace) + 1;
-            cnt -= strlen(nspace) + 1;
-        } else {
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-
-        if (sizeof(pmix_rank_t) <= cnt) {
-            /* have to convert this to host order */
-            memcpy(&u32, mg, sizeof(uint32_t));
-            rank = ntohl(u32);
-            mg += sizeof(uint32_t);
-            cnt -= sizeof(uint32_t);
-        } else {
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-    } else {
-        /* we don't know what they are! */
-        PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
-        rc = PMIX_ERR_NOT_SUPPORTED;
-        free(msg);
-        goto error;
-    }
-
-    /* extract their VERSION */
-    PMIX_STRNLEN(msglen, mg, cnt);
-    if (msglen < cnt) {
-        version = mg;
-        mg += strlen(version) + 1;
-        cnt -= strlen(version) + 1;
-    } else {
-        PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_BAD_PARAM;
-        goto error;
-    }
-    major = strtoul(version, &version, 10);
-    ++version;
-    minor = strtoul(version, &version, 10);
-    ++version;
-    release = strtoul(version, NULL, 10);
-    PMIX_SET_PROC_MAJOR(&proc_type, major);
-    PMIX_SET_PROC_MINOR(&proc_type, minor);
-    PMIX_SET_PROC_RELEASE(&proc_type, release);
-
-    if (2 == major && 0 == minor) {
-        /* the 2.0 release handshake ends with the version string */
-        bfrops = "v20";
-        bftype = pmix_bfrops_globals.default_type;  // we can't know any better
-        gds = "ds12,hash";
-    } else {
-        /* extract the name of the bfrops module they used */
-        PMIX_STRNLEN(msglen, mg, cnt);
-        if (msglen < cnt) {
-            bfrops = mg;
-            mg += strlen(bfrops) + 1;
-            cnt -= strlen(bfrops) + 1;
-        } else {
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-
-        /* extract the type of buffer they used */
-        if (sizeof(bftype) < cnt) {
-            memcpy(&bftype, mg, sizeof(bftype));
-            mg += sizeof(bftype);
-            cnt -= sizeof(bftype);
-        } else {
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-
-        /* extract the name of the gds module they used */
-        PMIX_STRNLEN(msglen, mg, cnt);
-        if (msglen < cnt) {
-            gds = mg;
-            mg += strlen(gds) + 1;
-            cnt -= strlen(gds) + 1;
-        } else {
-            PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
-            free(msg);
-            /* send an error reply to the client */
-            rc = PMIX_ERR_BAD_PARAM;
-            goto error;
-        }
-    }
-
-    /* see if this is a tool connection request */
-    if (0 != pnd->flag) {
-        peer = PMIX_NEW(pmix_peer_t);
-        if (NULL == peer) {
-            /* probably cannot send an error reply if we are out of memory */
-            free(msg);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd);
-            return;
-        }
-        pnd->peer = peer;
-        /* if this is a tool we launched, then the host may
-         * have already registered it as a client - so check
-         * to see if we already have a peer for it */
-        if (5 == pnd->flag || 8 == pnd->flag) {
-            /* registration only adds the nspace and a rank in that
-             * nspace - it doesn't add the peer object to our array
-             * of local clients. So let's start by searching for
-             * the nspace object */
-            nptr = NULL;
-            PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_namespace_t) {
-                if (0 == strcmp(tmp->nspace, nspace)) {
-                    nptr = tmp;
-                    break;
-                }
-            }
-            if (NULL == nptr) {
-                /* it is possible that this is a tool inside of
-                 * a job-script as part of a multi-spawn operation.
-                 * Since each tool invocation may have finalized and
-                 * terminated, the tool will appear to "terminate", thus
-                 * causing us to cleanup all references to it, and then
-                 * reappear. So we don't reject this connection request.
-                 * Instead, we create the nspace and rank objects for
-                 * it and let the RM/host decide if this behavior
-                 * is allowed */
-                nptr = PMIX_NEW(pmix_namespace_t);
-                if (NULL == nptr) {
-                    rc = PMIX_ERR_NOMEM;
-                    goto error;
-                }
-                nptr->nspace = strdup(nspace);
-            }
-            /* now look for the rank */
-            info = NULL;
-            found = false;
-            PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) {
-                if (info->pname.rank == rank) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found) {
-                /* see above note about not finding nspace */
-                info = PMIX_NEW(pmix_rank_info_t);
-                info->pname.nspace = strdup(nspace);
-                info->pname.rank = rank;
-                info->uid = pnd->uid;
-                info->gid = pnd->gid;
-                pmix_list_append(&nptr->ranks, &info->super);
-            }
-            PMIX_RETAIN(info);
-            peer->info = info;
-            PMIX_RETAIN(nptr);
-        } else {
-            nptr = PMIX_NEW(pmix_namespace_t);
-            if (NULL == nptr) {
-                PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
-                CLOSE_THE_SOCKET(pnd->sd);
-                PMIX_RELEASE(pnd);
-                PMIX_RELEASE(peer);
-                return;
-            }
-        }
-        peer->nptr = nptr;
-        /* select their bfrops compat module */
-        peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(bfrops);
-        if (NULL == peer->nptr->compat.bfrops) {
-            PMIX_RELEASE(peer);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd);
-            return;
-        }
-        /* set the buffer type */
-        peer->nptr->compat.type = bftype;
-        n = 0;
-        /* if info structs need to be passed along, then unpack them */
-        if (0 < cnt) {
-            int32_t foo;
-            PMIX_CONSTRUCT(&buf, pmix_buffer_t);
-            PMIX_LOAD_BUFFER(peer, &buf, mg, cnt);
-            foo = 1;
-            PMIX_BFROPS_UNPACK(rc, peer, &buf, &pnd->ninfo, &foo, PMIX_SIZE);
-            foo = (int32_t)pnd->ninfo;
-            /* if we have an identifier, then we leave room to pass it */
-            if (!pnd->need_id) {
-                pnd->ninfo += 5;
-            } else {
-                pnd->ninfo += 3;
-            }
-            PMIX_INFO_CREATE(pnd->info, pnd->ninfo);
-            PMIX_BFROPS_UNPACK(rc, peer, &buf, pnd->info, &foo, PMIX_INFO);
-            n = foo;
-        } else {
-            if (!pnd->need_id) {
-                pnd->ninfo = 5;
-            } else {
-                pnd->ninfo = 3;
-            }
-            PMIX_INFO_CREATE(pnd->info, pnd->ninfo);
-        }
-
-        /* pass along the proc_type */
-        memcpy(&pnd->proc_type, &proc_type, sizeof(pmix_proc_type_t));
-        /* pass along the bfrop, buffer_type, and sec fields so
-         * we can assign them once we create a peer object */
-        pnd->psec = strdup(sec);
-        if (NULL != gds) {
-            pnd->gds = strdup(gds);
-        }
-
-        /* does the server support tool connections? */
-        if (NULL == pmix_host_server.tool_connected) {
-            if (pnd->need_id) {
-                /* we need someone to provide the tool with an
-                 * identifier and they aren't available */
-                /* send an error reply to the client */
-                rc = PMIX_ERR_NOT_SUPPORTED;
-                PMIX_RELEASE(peer);
-                /* release the msg */
-                free(msg);
-                goto error;
-            } else {
-                /* just process it locally */
-                PMIX_LOAD_PROCID(&proc, nspace, rank);
-                cnct_cbfunc(PMIX_SUCCESS, &proc, (void*)pnd);
-                /* release the msg */
-                free(msg);
-                return;
-            }
-        }
-
-        /* setup the info array to pass the relevant info
-         * to the server */
-        /* provide the version */
-        PMIX_INFO_LOAD(&pnd->info[n], PMIX_VERSION_INFO, version, PMIX_STRING);
-        ++n;
-        /* provide the user id */
-        PMIX_INFO_LOAD(&pnd->info[n], PMIX_USERID, &pnd->uid, PMIX_UINT32);
-        ++n;
-        /* and the group id */
-        PMIX_INFO_LOAD(&pnd->info[n], PMIX_GRPID, &pnd->gid, PMIX_UINT32);
-        ++n;
-        /* if we have it, pass along their ID */
-        if (!pnd->need_id) {
-            PMIX_INFO_LOAD(&pnd->info[n], PMIX_NSPACE, nspace, PMIX_STRING);
-            ++n;
-            PMIX_INFO_LOAD(&pnd->info[n], PMIX_RANK, &rank, PMIX_PROC_RANK);
-            ++n;
-        }
-        /* release the msg */
-        free(msg);
-
-        /* pass it up for processing */
-        pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd);
-        return;
-    }
-
-    /* see if we know this nspace */
-    nptr = NULL;
-    PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_namespace_t) {
-        if (0 == strcmp(tmp->nspace, nspace)) {
-            nptr = tmp;
-            break;
-        }
-    }
-    if (NULL == nptr) {
-        /* we don't know this namespace, reject it */
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_NOT_FOUND;
-        goto error;
-    }
-
-    /* see if we have this peer in our list */
-    info = NULL;
-    found = false;
-    PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) {
-        if (info->pname.rank == rank) {
-            found = true;
-            break;
-        }
-    }
-    if (!found) {
-        /* rank unknown, reject it */
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_NOT_FOUND;
-        goto error;
-    }
-
-    /* a peer can connect on multiple sockets since it can fork/exec
-     * a child that also calls PMIX_Init, so add it here if necessary.
-     * Create the tracker for this peer */
-    peer = PMIX_NEW(pmix_peer_t);
-    if (NULL == peer) {
-        /* probably cannot send an error reply if we are out of memory */
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    /* mark that this peer is a client of the given type */
-    memcpy(&peer->proc_type, &proc_type, sizeof(pmix_proc_type_t));
-    /* save the protocol */
-    peer->protocol = pnd->protocol;
-    /* add in the nspace pointer */
-    PMIX_RETAIN(nptr);
-    peer->nptr = nptr;
-    PMIX_RETAIN(info);
-    peer->info = info;
-    /* update the epilog fields */
-    peer->epilog.uid = info->uid;
-    peer->epilog.gid = info->gid;
-    /* ensure the nspace epilog is updated too */
-    nptr->epilog.uid = info->uid;
-    nptr->epilog.gid = info->gid;
-    info->proc_cnt++; /* increase number of processes on this rank */
-    peer->sd = pnd->sd;
-    if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) {
-        free(msg);
-        info->proc_cnt--;
-        PMIX_RELEASE(peer);
-        /* probably cannot send an error reply if we are out of memory */
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    info->peerid = peer->index;
-
-    /* set the sec module to match this peer */
-    peer->nptr->compat.psec = pmix_psec_base_assign_module(sec);
-    if (NULL == peer->nptr->compat.psec) {
-        free(msg);
-        info->proc_cnt--;
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-        PMIX_RELEASE(peer);
-        /* send an error reply to the client */
-        goto error;
-    }
-
-    /* set the bfrops module to match this peer */
-    peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(bfrops);
-    if (NULL == peer->nptr->compat.bfrops) {
-        free(msg);
-        info->proc_cnt--;
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-        PMIX_RELEASE(peer);
-        /* send an error reply to the client */
-        goto error;
-    }
-    /* and the buffer type to match */
-    peer->nptr->compat.type = bftype;
-
-    /* set the gds module to match this peer */
-    if (NULL != gds) {
-        PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, gds, PMIX_STRING);
-        peer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
-        PMIX_INFO_DESTRUCT(&ginfo);
-    } else {
-        peer->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0);
-    }
-    if (NULL == peer->nptr->compat.gds) {
-        free(msg);
-        info->proc_cnt--;
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-        PMIX_RELEASE(peer);
-        /* send an error reply to the client */
-        goto error;
-    }
-
-    /* if we haven't previously stored the version for this
-     * nspace, do so now */
-    if (!nptr->version_stored) {
-        PMIX_INFO_LOAD(&ginfo, PMIX_BFROPS_MODULE, peer->nptr->compat.bfrops->version, PMIX_STRING);
-        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, peer->nptr, &ginfo, 1);
-        PMIX_INFO_DESTRUCT(&ginfo);
-        nptr->version_stored = true;
-    }
-
-    free(msg);  // can now release the data buffer
-
-    /* the choice of PTL module is obviously us */
-    peer->nptr->compat.ptl = &pmix_ptl_tcp_module;
-
-    /* validate the connection */
-    cred.bytes = pnd->cred;
-    cred.size = pnd->len;
-    PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, NULL, 0, NULL, NULL, &cred);
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "client connection validated with status=%d", reply);
-
-    /* tell the client all is good */
-    u32 = htonl(reply);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-        PMIX_ERROR_LOG(rc);
-        info->proc_cnt--;
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-        PMIX_RELEASE(peer);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    /* If needed perform the handshake. The macro will update reply */
-    PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred);
-
-    /* It is possible that connection validation failed
-     * We need to reply to the client first and cleanup after */
-    if (PMIX_SUCCESS != reply) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "validation of client connection failed");
-        info->proc_cnt--;
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-        PMIX_RELEASE(peer);
-        /* send an error reply to the client */
-        goto error;
-    }
-
-
-      /* send the client's array index */
-    u32 = htonl(peer->index);
-      if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-          PMIX_ERROR_LOG(rc);
-          info->proc_cnt--;
-          pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-          PMIX_RELEASE(peer);
-          goto error;
-      }
-
-      pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                          "connect-ack from client completed");
-
-      /* let the host server know that this client has connected */
-      if (NULL != pmix_host_server.client_connected) {
-          pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN);
-          proc.rank = peer->info->pname.rank;
-          rc = pmix_host_server.client_connected(&proc, peer->info->server_object,
-                                                 NULL, NULL);
-          if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
-              PMIX_ERROR_LOG(rc);
-              info->proc_cnt--;
-              pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL);
-              PMIX_RELEASE(peer);
-              goto error;
-          }
-      }
-
-    pmix_ptl_base_set_nonblocking(pnd->sd);
-
-    /* start the events for this client */
-    pmix_event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd,
-                      EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, peer);
-    pmix_event_add(&peer->recv_event, NULL);
-    peer->recv_ev_active = true;
-    pmix_event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd,
-                      EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, peer);
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:server client %s:%u has connected on socket %d",
-                        peer->info->pname.nspace, peer->info->pname.rank, peer->sd);
-    PMIX_RELEASE(pnd);
-
-    /* check the cached events and update the client */
-    _check_cached_events(peer);
-
-    return;
-
-  error:
-    /* send an error reply to the client */
-    u32 = htonl(rc);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(int)))) {
-        PMIX_ERROR_LOG(rc);
-        CLOSE_THE_SOCKET(pnd->sd);
-    }
-    PMIX_RELEASE(pnd);
-    return;
-}
-
-/* process the callback with tool connection info */
-static void process_cbfunc(int sd, short args, void *cbdata)
-{
-    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
-    pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata;
-    pmix_namespace_t *nptr;
-    pmix_rank_info_t *info;
-    pmix_peer_t *peer;
-    pmix_status_t rc, reply;
-    uint32_t u32;
-    pmix_info_t ginfo;
-    pmix_byte_object_t cred;
-    pmix_iof_req_t *req;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(cd);
-
-    /* send this status so they don't hang */
-    u32 = ntohl(cd->status);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-        PMIX_ERROR_LOG(rc);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd->peer);
-        PMIX_RELEASE(pnd);
-        PMIX_RELEASE(cd);
-        return;
-    }
-
-    /* if the request failed, then we are done */
-    if (PMIX_SUCCESS != cd->status) {
-        PMIX_RELEASE(pnd->peer);
-        PMIX_RELEASE(pnd);
-        PMIX_RELEASE(cd);
-        return;
-    }
-
-    /* if we got an identifier, send it back to the tool */
-    if (pnd->need_id) {
-        /* start with the nspace */
-        if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) {
-            PMIX_ERROR_LOG(rc);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd->peer);
-            PMIX_RELEASE(pnd);
-            PMIX_RELEASE(cd);
-            return;
-        }
-
-        /* now the rank, suitably converted */
-        u32 = ntohl(cd->proc.rank);
-        if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-            PMIX_ERROR_LOG(rc);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd->peer);
-            PMIX_RELEASE(pnd);
-            PMIX_RELEASE(cd);
-            return;
-        }
-    }
-
-    /* send my nspace back to the tool */
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) {
-        PMIX_ERROR_LOG(rc);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd->peer);
-        PMIX_RELEASE(pnd);
-        PMIX_RELEASE(cd);
-        return;
-    }
-
-    /* send my rank back to the tool */
-    u32 = ntohl(pmix_globals.myid.rank);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-        PMIX_ERROR_LOG(rc);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd->peer);
-        PMIX_RELEASE(pnd);
-        PMIX_RELEASE(cd);
-        return;
-    }
-
-    /* shortcuts */
-    peer = (pmix_peer_t*)pnd->peer;
-    nptr = peer->nptr;
-
-    /* if this tool wasn't initially registered as a client,
-     * then add some required structures */
-    if (5 != pnd->flag && 8 != pnd->flag) {
-        PMIX_RETAIN(nptr);
-        nptr->nspace = strdup(cd->proc.nspace);
-        pmix_list_append(&pmix_globals.nspaces, &nptr->super);
-        info = PMIX_NEW(pmix_rank_info_t);
-        info->pname.nspace = strdup(nptr->nspace);
-        info->pname.rank = cd->proc.rank;
-        info->uid = pnd->uid;
-        info->gid = pnd->gid;
-        pmix_list_append(&nptr->ranks, &info->super);
-        PMIX_RETAIN(info);
-        peer->info = info;
-    }
-
-    /* mark the peer proc type */
-    memcpy(&peer->proc_type, &pnd->proc_type, sizeof(pmix_proc_type_t));
-    /* save the protocol */
-    peer->protocol = pnd->protocol;
-    /* save the uid/gid */
-    peer->epilog.uid = peer->info->uid;
-    peer->epilog.gid = peer->info->gid;
-    nptr->epilog.uid = peer->info->uid;
-    nptr->epilog.gid = peer->info->gid;
-    peer->proc_cnt = 1;
-    peer->sd = pnd->sd;
-
-    /* get the appropriate compatibility modules based on the
-     * info provided by the tool during the initial connection request */
-    peer->nptr->compat.psec = pmix_psec_base_assign_module(pnd->psec);
-    if (NULL == peer->nptr->compat.psec) {
-        PMIX_RELEASE(peer);
-        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
-        PMIX_RELEASE(nptr);  // will release the info object
-        CLOSE_THE_SOCKET(pnd->sd);
-        goto done;
-    }
-    /* the choice of PTL module was obviously made by the connecting
-     * tool as we received this request via that channel, so simply
-     * record it here for future use */
-    peer->nptr->compat.ptl = &pmix_ptl_tcp_module;
-    /* set the gds */
-    PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, pnd->gds, PMIX_STRING);
-    peer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
-    PMIX_INFO_DESTRUCT(&ginfo);
-    if (NULL == peer->nptr->compat.gds) {
-        PMIX_RELEASE(peer);
-        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
-        PMIX_RELEASE(nptr);  // will release the info object
-        CLOSE_THE_SOCKET(pnd->sd);
-        goto done;
-    }
-
-    /* if we haven't previously stored the version for this
-     * nspace, do so now */
-    if (!peer->nptr->version_stored) {
-        PMIX_INFO_LOAD(&ginfo, PMIX_BFROPS_MODULE, peer->nptr->compat.bfrops->version, PMIX_STRING);
-        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, peer->nptr, &ginfo, 1);
-        PMIX_INFO_DESTRUCT(&ginfo);
-        nptr->version_stored = true;
-    }
-
-    /* automatically setup to forward output to the tool */
-    req = PMIX_NEW(pmix_iof_req_t);
-    if (NULL == req) {
-        PMIX_RELEASE(peer);
-        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
-        PMIX_RELEASE(nptr);  // will release the info object
-        CLOSE_THE_SOCKET(pnd->sd);
-        goto done;
-    }
-    PMIX_RETAIN(peer);
-    req->requestor = peer;
-    req->nprocs = 1;
-    PMIX_PROC_CREATE(req->procs, req->nprocs);
-    PMIX_LOAD_PROCID(&req->procs[0], pmix_globals.myid.nspace, pmix_globals.myid.rank);
-    req->channels = PMIX_FWD_STDOUT_CHANNEL | PMIX_FWD_STDERR_CHANNEL | PMIX_FWD_STDDIAG_CHANNEL;
-    req->remote_id = 0;     // default ID for tool during init
-    req->local_id = pmix_pointer_array_add(&pmix_globals.iof_requests, req);
-
-    /* validate the connection */
-    cred.bytes = pnd->cred;
-    cred.size = pnd->len;
-    PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, NULL, 0, NULL, NULL, &cred);
-    /* communicate the result to the other side */
-    u32 = htonl(reply);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-        PMIX_ERROR_LOG(rc);
-        PMIX_RELEASE(peer);
-        pmix_list_remove_item(&pmix_server_globals.nspaces, &nptr->super);
-        PMIX_RELEASE(nptr);  // will release the info object
-        CLOSE_THE_SOCKET(pnd->sd);
-        goto done;
-    }
-
-    /* If needed perform the handshake. The macro will update reply */
-    PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred);
-
-    /* If verification wasn't successful - stop here */
-    if (PMIX_SUCCESS != reply) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "validation of tool credentials failed: %s",
-                            PMIx_Error_string(rc));
-        PMIX_RELEASE(peer);
-        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
-        PMIX_RELEASE(nptr);  // will release the info object
-        CLOSE_THE_SOCKET(pnd->sd);
-        goto done;
-    }
-
-    /* set the socket non-blocking for all further operations */
-    pmix_ptl_base_set_nonblocking(pnd->sd);
-
-    if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) {
-        PMIX_RELEASE(pnd);
-        PMIX_RELEASE(cd);
-        PMIX_RELEASE(peer);
-        pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
-        PMIX_RELEASE(nptr);  // will release the info object
-        /* probably cannot send an error reply if we are out of memory */
-        return;
-    }
-    peer->info->peerid = peer->index;
-
-    /* start the events for this tool */
-    pmix_event_assign(&peer->recv_event, pmix_globals.evbase, peer->sd,
-                      EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, peer);
-    pmix_event_add(&peer->recv_event, NULL);
-    peer->recv_ev_active = true;
-    pmix_event_assign(&peer->send_event, pmix_globals.evbase, peer->sd,
-                      EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, peer);
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:server tool %s:%d has connected on socket %d",
-                        peer->info->pname.nspace, peer->info->pname.rank, peer->sd);
-
-    /* check the cached events and update the tool */
-    _check_cached_events(peer);
-
-  done:
-    PMIX_RELEASE(pnd);
-    PMIX_RELEASE(cd);
-}
-
-/* receive a callback from the host RM with an nspace
- * for a connecting tool */
-static void cnct_cbfunc(pmix_status_t status,
-                        pmix_proc_t *proc, void *cbdata)
-{
-    pmix_setup_caddy_t *cd;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:tcp:cnct_cbfunc returning %s:%d %s",
-                        proc->nspace, proc->rank, PMIx_Error_string(status));
-
-    /* need to thread-shift this into our context */
-    cd = PMIX_NEW(pmix_setup_caddy_t);
-    if (NULL == cd) {
-        PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
-        return;
-    }
-    cd->status = status;
-    PMIX_LOAD_PROCID(&cd->proc, proc->nspace, proc->rank);
-    cd->cbdata = cbdata;
-    PMIX_THREADSHIFT(cd, process_cbfunc);
-}
-
-static void _check_cached_events(pmix_peer_t *peer)
-{
-    pmix_notify_caddy_t *cd;
-    int i;
-    size_t n;
-    pmix_range_trkr_t rngtrk;
-    pmix_buffer_t *relay;
-    pmix_proc_t proc;
-    pmix_status_t ret;
-    pmix_cmd_t cmd = PMIX_NOTIFY_CMD;
-    bool matched, found;
-
-    PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank);
-
-    for (i=0; i < pmix_globals.max_events; i++) {
-        pmix_hotel_knock(&pmix_globals.notifications, i, (void**)&cd);
-        if (NULL == cd) {
-            continue;
-        }
-        /* check the range */
-        if (NULL == cd->targets) {
-            rngtrk.procs = &cd->source;
-            rngtrk.nprocs = 1;
-        } else {
-            rngtrk.procs = cd->targets;
-            rngtrk.nprocs = cd->ntargets;
-        }
-        rngtrk.range = cd->range;
-        if (!pmix_notify_check_range(&rngtrk, &proc)) {
-            continue;
-        }
-        found = false;
-        /* if we were given specific targets, check if this is one */
-        if (NULL != cd->targets) {
-            matched = false;
-            for (n=0; n < cd->ntargets; n++) {
-                if (PMIX_CHECK_PROCID(&proc, &cd->targets[n])) {
-                    matched = true;
-                    /* track the number of targets we have left to notify */
-                    --cd->nleft;
-                    /* if this is the last one, then evict this event
-                     * from the cache */
-                    if (0 == cd->nleft) {
-                        pmix_hotel_checkout(&pmix_globals.notifications, cd->room);
-                        found = true;  // mark that we should release cd
-                    }
-                    break;
-                }
-            }
-            if (!matched) {
-                /* do not notify this one */
-                continue;
-            }
-        }
-
-        /* all matches - notify */
-        relay = PMIX_NEW(pmix_buffer_t);
-        if (NULL == relay) {
-            /* nothing we can do */
-            PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
-            break;
-        }
-        /* pack the info data stored in the event */
-        PMIX_BFROPS_PACK(ret, peer, relay, &cmd, 1, PMIX_COMMAND);
-        if (PMIX_SUCCESS != ret) {
-            PMIX_ERROR_LOG(ret);
-            PMIX_RELEASE(relay);
-            break;
-        }
-        PMIX_BFROPS_PACK(ret, peer, relay, &cd->status, 1, PMIX_STATUS);
-        if (PMIX_SUCCESS != ret) {
-            PMIX_ERROR_LOG(ret);
-            PMIX_RELEASE(relay);
-            break;
-        }
-        PMIX_BFROPS_PACK(ret, peer, relay, &cd->source, 1, PMIX_PROC);
-        if (PMIX_SUCCESS != ret) {
-            PMIX_RELEASE(relay);
-            PMIX_ERROR_LOG(ret);
-            break;
-        }
-        PMIX_BFROPS_PACK(ret, peer, relay, &cd->ninfo, 1, PMIX_SIZE);
-        if (PMIX_SUCCESS != ret) {
-            PMIX_ERROR_LOG(ret);
-            PMIX_RELEASE(relay);
-            break;
-        }
-        if (0 < cd->ninfo) {
-            PMIX_BFROPS_PACK(ret, peer, relay, cd->info, cd->ninfo, PMIX_INFO);
-            if (PMIX_SUCCESS != ret) {
-                PMIX_ERROR_LOG(ret);
-                PMIX_RELEASE(relay);
-                break;
-            }
-        }
-        PMIX_SERVER_QUEUE_REPLY(ret, peer, 0, relay);
-        if (PMIX_SUCCESS != ret) {
-            PMIX_RELEASE(relay);
-        }
-        if (found) {
-            PMIX_RELEASE(cd);
-        }
-    }
-}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tcp/ptl_tcp.h pmix-4.0.0/src/mca/ptl/tcp/ptl_tcp.h
--- pmix-3.2.2~rc1/src/mca/ptl/tcp/ptl_tcp.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tcp/ptl_tcp.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2016-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2018      IBM Corporation.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef PMIX_PTL_TCP_H
-#define PMIX_PTL_TCP_H
-
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
-#endif
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-
-#include "src/mca/ptl/ptl.h"
-
-BEGIN_C_DECLS
-
-typedef struct {
-    pmix_ptl_base_component_t super;
-    char *session_tmpdir;
-    char *system_tmpdir;
-    char *if_include;
-    char *if_exclude;
-    int ipv4_port;
-    int ipv6_port;
-    bool disable_ipv4_family;
-    bool disable_ipv6_family;
-    struct sockaddr_storage connection;
-    char *session_filename;
-    char *nspace_filename;
-    char *pid_filename;
-    char *system_filename;
-    char *rendezvous_filename;
-    int wait_to_connect;
-    int max_retries;
-    char *report_uri;
-    bool remote_connections;
-    int handshake_wait_time;
-    int handshake_max_retries;
-} pmix_ptl_tcp_component_t;
-
-extern pmix_ptl_tcp_component_t mca_ptl_tcp_component;
-
-extern pmix_ptl_module_t pmix_ptl_tcp_module;
-
-END_C_DECLS
-
-#endif /* PMIX_PTL_TCP_H */
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tool/Makefile.am pmix-4.0.0/src/mca/ptl/tool/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ptl/tool/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tool/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+# -*- makefile -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
+# Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+headers = ptl_tool.h
+sources = \
+        ptl_tool_component.c \
+        ptl_tool.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_pmix_ptl_tool_DSO
+lib =
+lib_sources =
+component = mca_ptl_tool.la
+component_sources = $(headers) $(sources)
+else
+lib = libmca_ptl_tool.la
+lib_sources = $(headers) $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pmixlibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_ptl_tool_la_SOURCES = $(component_sources)
+mca_ptl_tool_la_LDFLAGS = -module -avoid-version
+if NEED_LIBPMIX
+mca_ptl_tool_la_LIBADD = $(top_builddir)/src/libpmix.la
+endif
+
+noinst_LTLIBRARIES = $(lib)
+libmca_ptl_tool_la_SOURCES = $(lib_sources)
+libmca_ptl_tool_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tool/ptl_tool.c pmix-4.0.0/src/mca/ptl/tool/ptl_tool.c
--- pmix-3.2.2~rc1/src/mca/ptl/tool/ptl_tool.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tool/ptl_tool.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,513 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "src/include/pmix_globals.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+#ifdef HAVE_SYS_SYSCTL_H
+#include <sys/sysctl.h>
+#endif
+
+#include "src/include/pmix_socket_errno.h"
+#include "src/client/pmix_client_ops.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/os_path.h"
+#include "src/util/show_help.h"
+#include "src/mca/bfrops/base/base.h"
+#include "src/mca/gds/gds.h"
+
+#include "src/mca/ptl/base/base.h"
+#include "ptl_tool.h"
+
+static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
+                                     pmix_info_t *info, size_t ninfo);
+static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo);
+
+pmix_ptl_module_t pmix_ptl_tool_module = {
+    .name = "tool",
+    .connect_to_peer = connect_to_peer,
+    .setup_fork = pmix_ptl_base_setup_fork,
+    .setup_listener = setup_listener
+};
+
+static pmix_status_t connect_to_peer(struct pmix_peer_t *pr,
+                                     pmix_info_t *info, size_t ninfo)
+{
+    char *suri = NULL, *st, *evar;
+    char *filename, *nspace=NULL;
+    pmix_rank_t rank = PMIX_RANK_WILDCARD;
+    char *p = NULL, *server_nspace = NULL, *rendfile = NULL;
+    int sd, rc;
+    size_t n;
+    bool system_level = false;
+    bool system_level_only = false;
+    pid_t pid = 0, mypid;
+    pmix_list_t ilist;
+    pmix_info_caddy_t *kv;
+    pmix_info_t *iptr = NULL, mypidinfo, mycmdlineinfo, launcher;
+    size_t niptr = 0;
+    pmix_peer_t *peer = (pmix_peer_t*)pr;
+
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "ptl:tool: connecting to server");
+
+    /* check for common directives */
+    rc = pmix_ptl_base_check_directives(info, ninfo);
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* check any provided directives
+     * to see where they want us to connect to */
+    PMIX_CONSTRUCT(&ilist, pmix_list_t);
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_TO_SYSTEM)) {
+                system_level_only = PMIX_INFO_TRUE(&info[n]);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_SYSTEM_FIRST)) {
+                /* try the system-level */
+                system_level = PMIX_INFO_TRUE(&info[n]);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_PIDINFO)) {
+                pid = info[n].value.data.pid;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_NSPACE)) {
+                if (NULL != server_nspace) {
+                    /* they included it more than once */
+                    if (0 == strcmp(server_nspace, info[n].value.data.string)) {
+                        /* same value, so ignore it */
+                        continue;
+                    }
+                    /* otherwise, we don't know which one to use */
+                    rc = PMIX_ERR_BAD_PARAM;
+                    goto cleanup;
+                }
+                server_nspace = strdup(info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TOOL_ATTACHMENT_FILE)) {
+                if (NULL != rendfile) {
+                    free(rendfile);
+                }
+                rendfile = strdup(info[n].value.data.string);
+            } else if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer) &&
+                       PMIX_CHECK_KEY(&info[n], PMIX_LAUNCHER_RENDEZVOUS_FILE)) {
+                if (NULL != pmix_ptl_base.rendezvous_filename) {
+                    free(pmix_ptl_base.rendezvous_filename);
+                }
+                pmix_ptl_base.rendezvous_filename = strdup(info[n].value.data.string);
+            } else {
+                /* need to pass this to server */
+                kv = PMIX_NEW(pmix_info_caddy_t);
+                kv->info = &info[n];
+                pmix_list_append(&ilist, &kv->super);
+            }
+        }
+    }
+
+    /* add our pid to the array */
+    kv = PMIX_NEW(pmix_info_caddy_t);
+    mypid = getpid();
+    PMIX_INFO_LOAD(&mypidinfo, PMIX_PROC_PID, &mypid, PMIX_PID);
+    kv->info = &mypidinfo;
+    pmix_list_append(&ilist, &kv->super);
+
+    /* if I am a launcher, tell them so */
+    if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        kv = PMIX_NEW(pmix_info_caddy_t);
+        PMIX_INFO_LOAD(&launcher, PMIX_LAUNCHER, NULL, PMIX_BOOL);
+        kv->info = &launcher;
+        pmix_list_append(&ilist, &kv->super);
+    }
+
+    /* add our cmd line to the array */
+#if PMIX_HAVE_APPLE
+    int mib[3], argmax, nargs, num;
+    size_t size;
+    char *procargs, *cp, *cptr;
+    char **stack = NULL;
+
+    /* Get the maximum process arguments size. */
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_ARGMAX;
+    size = sizeof(argmax);
+
+    if (sysctl(mib, 2, &argmax, &size, NULL, 0) == -1) {
+        fprintf(stderr, "sysctl() argmax failed\n");
+        rc = PMIX_ERR_NO_PERMISSIONS;
+        goto cleanup;
+    }
+
+    /* Allocate space for the arguments. */
+    procargs = (char *)malloc(argmax);
+    if (procargs == NULL) {
+        rc = -1;
+        goto cleanup;
+    }
+
+    /* Make a sysctl() call to get the raw argument space of the process. */
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_PROCARGS2;
+    mib[2] = getpid();
+
+    size = (size_t)argmax;
+
+    if (sysctl(mib, 3, procargs, &size, NULL, 0) == -1) {
+        fprintf(stderr, "Lacked permissions\n");;
+        rc = PMIX_ERR_NO_PERMISSIONS;
+        goto cleanup;
+    }
+
+    memcpy(&nargs, procargs, sizeof(nargs));
+    /* this points to the executable - skip over that to get the rest */
+    cp = procargs + sizeof(nargs);
+    cp += strlen(cp);
+    /* this is the first argv */
+    pmix_argv_append_nosize(&stack, cp);
+    /* skip any embedded NULLs */
+    while (cp < &procargs[size] && '\0' == *cp) {
+        ++cp;
+    }
+    if (cp != &procargs[size]) {
+        /* from this point, we have the argv separated by NULLs - split them out */
+        cptr = cp;
+        num = 0;
+        while (cp < &procargs[size] && num < nargs) {
+            if ('\0' == *cp) {
+                pmix_argv_append_nosize(&stack, cptr);
+                ++cp;  // skip over the NULL
+                cptr = cp;
+                ++num;
+            } else {
+                ++cp;
+            }
+        }
+    }
+    p = pmix_argv_join(stack, ' ');
+    pmix_argv_free(stack);
+    free(procargs);
+#else
+    char tmp[512];
+    FILE *fp;
+
+    /* open the pid's info file */
+    snprintf(tmp, 512, "/proc/%lu/cmdline", (unsigned long)mypid);
+    fp = fopen(tmp, "r");
+    if (NULL != fp) {
+        /* read the cmd line */
+        fgets(tmp, 512, fp);
+        fclose(fp);
+        p = strdup(tmp);
+    }
+#endif
+    /* pass it along */
+    kv = PMIX_NEW(pmix_info_caddy_t);
+    PMIX_INFO_LOAD(&mycmdlineinfo, PMIX_CMD_LINE, p, PMIX_STRING);
+    kv->info = &mycmdlineinfo;
+    pmix_list_append(&ilist, &kv->super);
+    free(p);
+
+    /* if we need to pass anything, setup an array */
+    if (0 < (niptr = pmix_list_get_size(&ilist))) {
+        PMIX_INFO_CREATE(iptr, niptr);
+        n = 0;
+        while (NULL != (kv = (pmix_info_caddy_t*)pmix_list_remove_first(&ilist))) {
+            PMIX_INFO_XFER(&iptr[n], kv->info);
+            PMIX_RELEASE(kv);
+            ++n;
+        }
+    }
+    PMIX_LIST_DESTRUCT(&ilist);
+
+    /* mark that we are using the V2 protocol */
+    pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V2;
+    /* if we were given a URI, then look no further */
+    if (NULL != pmix_ptl_base.uri) {
+        /* if the string starts with "file:", then they are pointing
+         * us to a file we need to read to get the URI itself */
+        if (0 == strncmp(pmix_ptl_base.uri, "file:", 5)) {
+            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                "ptl:tool:tool getting connection info from %s",
+                                pmix_ptl_base.uri);
+            nspace = NULL;
+            rc = pmix_ptl_base_parse_uri_file(&pmix_ptl_base.uri[5], &suri, &nspace, &rank, peer);
+            if (PMIX_SUCCESS != rc) {
+                rc = PMIX_ERR_UNREACH;
+                goto cleanup;
+            }
+        } else {
+            st = strdup(pmix_ptl_base.uri);
+            /* we need to extract the nspace/rank of the server from the string */
+            p = strchr(st, ';');
+            if (NULL == p) {
+                free(st);
+                rc = PMIX_ERR_BAD_PARAM;
+                goto cleanup;
+            }
+            *p = '\0';
+            p++;
+            suri = strdup(p); // save the uri portion
+            /* the '.' in the first part of the original string separates
+             * nspace from rank */
+            p = strchr(st, '.');
+            if (NULL == p) {
+                free(st);
+                rc = PMIX_ERR_BAD_PARAM;
+                goto cleanup;
+            }
+            *p = '\0';
+            p++;
+            nspace = strdup(st);
+            rank = strtoull(p, NULL, 10);
+            /* now update the URI */
+            free(st);
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool:tool attempt connect using given URI %s", suri);
+        /* go ahead and try to connect */
+        rc = pmix_ptl_base_make_connection(peer, suri, iptr, niptr);
+        if (PMIX_SUCCESS != rc) {
+            goto cleanup;
+        }
+        /* cleanup */
+        goto complete;
+    }
+
+    /* if they gave us a rendezvous file, use it */
+    if (NULL != rendfile) {
+        /* try to read the file */
+        rc = pmix_ptl_base_parse_uri_file(rendfile, &suri, &nspace, &rank, peer);
+        if (PMIX_SUCCESS == rc) {
+            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                "ptl:tool:tool attempt connect to rendezvous server at %s", suri);
+            /* go ahead and try to connect */
+            rc = pmix_ptl_base_make_connection(peer, suri, iptr, niptr);
+            if (PMIX_SUCCESS == rc) {
+                /* don't free nspace - we will use it below */
+                if (NULL != iptr) {
+                    PMIX_INFO_FREE(iptr, niptr);
+                }
+                goto complete;
+            }
+        }
+        /* since they gave us a specific rendfile and we couldn't
+         * connect to it, return an error */
+        rc = PMIX_ERR_UNREACH;
+        goto cleanup;
+    }
+
+    /* if they asked for system-level first or only, we start there */
+    if (system_level || system_level_only) {
+        if (0 > asprintf(&filename, "%s/pmix.sys.%s", pmix_ptl_base.system_tmpdir, pmix_globals.hostname)) {
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool:tool looking for system server at %s",
+                            filename);
+        /* try to read the file */
+        rc = pmix_ptl_base_parse_uri_file(filename, &suri, &nspace, &rank, peer);
+        free(filename);
+        if (PMIX_SUCCESS == rc) {
+            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                                "ptl:tool:tool attempt connect to system server at %s", suri);
+            /* go ahead and try to connect */
+            rc = pmix_ptl_base_make_connection(peer, suri, iptr, niptr);
+            if (PMIX_SUCCESS == rc) {
+                /* don't free nspace - we will use it below */
+                goto complete;
+            }
+            free(nspace);
+            nspace = NULL;
+        }
+    }
+
+    /* we get here if they either didn't ask for a system-level connection,
+     * or they asked for it and it didn't succeed. If they _only_ wanted
+     * a system-level connection, then we are done */
+    if (system_level_only) {
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool: connecting to system failed");
+        rc = PMIX_ERR_UNREACH;
+        goto cleanup;
+    }
+
+    /* if they gave us a pid, then look for it */
+    if (0 != pid) {
+        if (0 > asprintf(&filename, "pmix.%s.tool.%d", pmix_globals.hostname, pid)) {
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool:tool searching for given session server %s",
+                            filename);
+        nspace = NULL;
+        rc = pmix_ptl_base_df_search(pmix_ptl_base.system_tmpdir,
+                                     filename, iptr, niptr, &sd, &nspace,
+                                     &rank, &suri, peer);
+        free(filename);
+        if (PMIX_SUCCESS == rc) {
+            goto complete;
+        }
+        /* since they gave us a specific pid and we couldn't
+         * connect to it, return an error */
+        rc = PMIX_ERR_UNREACH;
+        goto cleanup;
+    }
+
+    /* if they gave us an nspace, then look for it */
+    if (NULL != server_nspace) {
+        if (0 > asprintf(&filename, "pmix.%s.tool.%s", pmix_globals.hostname, server_nspace)) {
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool:tool searching for given session server %s",
+                            filename);
+        nspace = NULL;
+        rc = pmix_ptl_base_df_search(pmix_ptl_base.system_tmpdir,
+                                     filename, iptr, niptr, &sd, &nspace,
+                                     &rank, &suri, peer);
+        free(filename);
+        if (PMIX_SUCCESS == rc) {
+            goto complete;
+        }
+        /* since they gave us a specific nspace and we couldn't
+         * connect to it, return an error */
+        rc = PMIX_ERR_UNREACH;
+        goto cleanup;
+    }
+
+    /* see if we are a client of some server */
+    rc = pmix_ptl_base_check_server_uris(peer, &evar);
+    if (PMIX_SUCCESS == rc) {
+        PMIX_SET_PEER_TYPE(pmix_globals.mypeer, PMIX_PROC_CLIENT_TOOL);
+        rc = pmix_ptl_base_parse_uri(evar, &nspace, &rank, &suri);
+        if (PMIX_SUCCESS != rc) {
+            goto cleanup;
+        }
+        rc = pmix_ptl_base_make_connection(peer, suri, iptr, niptr);
+        if (PMIX_SUCCESS != rc) {
+            goto cleanup;
+        }
+    } else if (PMIX_ERR_UNREACH != rc) {
+        goto cleanup;
+    } else {
+        /* we aren't a client, so we will search to see what session-level
+         * tools are available to this user. We will take the first connection
+         * that succeeds - this is based on the likelihood that there is only
+         * one session per user on a node */
+
+        if (0 > asprintf(&filename, "pmix.%s.tool", pmix_globals.hostname)) {
+            rc = PMIX_ERR_NOMEM;
+            goto cleanup;
+        }
+        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                            "ptl:tool:tool searching for session server %s",
+                            filename);
+        nspace = NULL;
+        rc = pmix_ptl_base_df_search(pmix_ptl_base.system_tmpdir,
+                                     filename, iptr, niptr, &sd, &nspace,
+                                     &rank, &suri, peer);
+        free(filename);
+        if (PMIX_SUCCESS != rc) {
+            rc = PMIX_ERR_UNREACH;
+            goto cleanup;
+        }
+    }
+
+  complete:
+    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
+                        "tool_peer_try_connect: Connection across to server succeeded");
+
+    pmix_ptl_base_complete_connection(peer, nspace, rank, suri);
+
+  cleanup:
+    if (NULL != nspace) {
+        free(nspace);
+    }
+    if (NULL != iptr) {
+        PMIX_INFO_FREE(iptr, niptr);
+    }
+    if (NULL != rendfile) {
+        free(rendfile);
+    }
+    free(suri);
+    if (NULL != server_nspace) {
+        free(server_nspace);
+    }
+    return rc;
+}
+
+static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo)
+{
+    pmix_status_t rc;
+    char **clnup = NULL, *cptr = NULL;
+    pmix_info_t dir;
+
+    rc = pmix_ptl_base_setup_listener();
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* if we are connected, then register any rendezvous files for cleanup */
+    if (pmix_globals.connected) {
+        if (NULL != pmix_ptl_base.nspace_filename) {
+            pmix_argv_append_nosize(&clnup, pmix_ptl_base.nspace_filename);
+        }
+        if (NULL != pmix_ptl_base.session_filename) {
+            pmix_argv_append_nosize(&clnup, pmix_ptl_base.session_filename);
+        }
+        if (NULL != clnup) {
+            cptr = pmix_argv_join(clnup, ',');
+            pmix_argv_free(clnup);
+            PMIX_INFO_LOAD(&dir, PMIX_REGISTER_CLEANUP, cptr, PMIX_STRING);
+            free(cptr);
+            PMIx_Job_control_nb(&pmix_globals.myid, 1, &dir, 1, NULL, NULL);
+            PMIX_INFO_DESTRUCT(&dir);
+        }
+    }
+
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tool/ptl_tool_component.c pmix-4.0.0/src/mca/ptl/tool/ptl_tool_component.c
--- pmix-3.2.2~rc1/src/mca/ptl/tool/ptl_tool_component.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tool/ptl_tool_component.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,77 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennptlee and The University
+ *                         of Tennptlee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2017-2018 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018-2020 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2019      Mellanox Technologies, Inc. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_common.h"
+
+#include "src/include/pmix_globals.h"
+#include "src/util/error.h"
+#include "src/util/output.h"
+#include "src/mca/ptl/base/base.h"
+#include "src/mca/ptl/tool/ptl_tool.h"
+
+static int component_query(pmix_mca_base_module_t **module, int *priority);
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+ PMIX_EXPORT pmix_ptl_base_component_t mca_ptl_tool_component = {
+    .base = {
+        PMIX_PTL_BASE_VERSION_2_0_0,
+
+        /* Component name and version */
+        .pmix_mca_component_name = "tool",
+        PMIX_MCA_BASE_MAKE_VERSION(component,
+                                   PMIX_MAJOR_VERSION,
+                                   PMIX_MINOR_VERSION,
+                                   PMIX_RELEASE_VERSION),
+
+        /* Component open and close functions */
+        .pmix_mca_query_component = component_query
+    },
+    .priority = 40,
+    .uri = NULL
+};
+
+static int component_query(pmix_mca_base_module_t **module, int *priority)
+{
+    /* if I am not a tool, then look elsewhere */
+    if (!PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        *module = NULL;
+        *priority = 0;
+        return PMIX_ERR_TAKE_NEXT_OPTION;
+    }
+
+    *module = (pmix_mca_base_module_t*)&pmix_ptl_tool_module;
+    *priority = mca_ptl_tool_component.priority;
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/tool/ptl_tool.h pmix-4.0.0/src/mca/ptl/tool/ptl_tool.h
--- pmix-3.2.2~rc1/src/mca/ptl/tool/ptl_tool.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/tool/ptl_tool.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2018      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef PMIX_PTL_TOOL_H
+#define PMIX_PTL_TOOL_H
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+
+#include "src/mca/ptl/ptl.h"
+
+BEGIN_C_DECLS
+
+extern pmix_ptl_base_component_t mca_ptl_tool_component;
+
+extern pmix_ptl_module_t pmix_ptl_tool_module;
+
+END_C_DECLS
+
+#endif /* PMIX_PTL_TOOL_H */
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/usock/Makefile.am pmix-4.0.0/src/mca/ptl/usock/Makefile.am
--- pmix-3.2.2~rc1/src/mca/ptl/usock/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/usock/Makefile.am	1970-01-01 00:00:00.000000000 +0000
@@ -1,53 +0,0 @@
-# -*- makefile -*-
-#
-# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
-#                         University Research and Technology
-#                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
-#                         of Tennessee Research Foundation.  All rights
-#                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
-# Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
-# $COPYRIGHT$
-#
-# Additional copyrights may follow
-#
-# $HEADER$
-#
-
-headers = ptl_usock.h
-sources = \
-        ptl_usock_component.c \
-        ptl_usock.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_pmix_ptl_usock_DSO
-lib =
-lib_sources =
-component = mca_ptl_usock.la
-component_sources = $(headers) $(sources)
-else
-lib = libmca_ptl_usock.la
-lib_sources = $(headers) $(sources)
-component =
-component_sources =
-endif
-
-mcacomponentdir = $(pmixlibdir)
-mcacomponent_LTLIBRARIES = $(component)
-mca_ptl_usock_la_SOURCES = $(component_sources)
-mca_ptl_usock_la_LDFLAGS = -module -avoid-version
-if NEED_LIBPMIX
-mca_ptl_usock_la_LIBADD = $(top_builddir)/src/libpmix.la
-endif
-
-noinst_LTLIBRARIES = $(lib)
-libmca_ptl_usock_la_SOURCES = $(lib_sources)
-libmca_ptl_usock_la_LDFLAGS = -module -avoid-version
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/usock/ptl_usock.c pmix-4.0.0/src/mca/ptl/usock/ptl_usock.c
--- pmix-3.2.2~rc1/src/mca/ptl/usock/ptl_usock.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/usock/ptl_usock.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,950 +0,0 @@
-/*
- * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2011-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
- *                         reserved.
- * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2019      Research Organization for Information Science
- *                         and Technology (RIST).  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-#include "src/include/pmix_config.h"
-#include "include/pmix_common.h"
-
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/client/pmix_client_ops.h"
-#include "src/include/pmix_globals.h"
-#include "src/include/pmix_socket_errno.h"
-#include "src/mca/bfrops/base/base.h"
-#include "src/mca/psec/base/base.h"
-
-#include "src/mca/ptl/base/base.h"
-#include "ptl_usock.h"
-
-static pmix_status_t init(void);
-static void finalize(void);
-static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
-                                     pmix_info_t *info, size_t ninfo);
-static pmix_status_t send_recv(struct pmix_peer_t *peer,
-                               pmix_buffer_t *bfr,
-                               pmix_ptl_cbfunc_t cbfunc,
-                               void *cbdata);
-static pmix_status_t send_oneway(struct pmix_peer_t *peer,
-                                 pmix_buffer_t *bfr,
-                                 pmix_ptl_tag_t tag);
-
-pmix_ptl_module_t pmix_ptl_usock_module = {
-    .init = init,
-    .finalize = finalize,
-    .send_recv = send_recv,
-    .send = send_oneway,
-    .connect_to_peer = connect_to_peer
-};
-
-static pmix_status_t recv_connect_ack(int sd);
-static pmix_status_t send_connect_ack(int sd);
-
-static pmix_status_t init(void)
-{
-    return PMIX_SUCCESS;
-}
-
-static void finalize(void)
-{
-}
-
-static void pmix_usock_send_recv(int fd, short args, void *cbdata);
-static void pmix_usock_send(int fd, short args, void *cbdata);
-
-static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
-                                     pmix_info_t *info, size_t ninfo)
-{
-    struct sockaddr_un *address;
-    char *evar, **uri;
-    pmix_status_t rc;
-    int sd;
-    pmix_socklen_t len;
-    bool retried = false;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "[%s:%d] connect to server",
-                        __FILE__, __LINE__);
-
-    /* if we are not a client, there is nothing we can do */
-    if (!PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
-        return PMIX_ERR_NOT_SUPPORTED;
-    }
-
-    PMIX_SET_PEER_TYPE(pmix_client_globals.myserver, PMIX_PROC_SERVER);
-    /* if we don't have a path to the daemon rendezvous point,
-     * then we need to return an error */
-    if (NULL != (evar = getenv("PMIX_SERVER_URI2USOCK"))) {
-        /* this is a v2.1+ server */
-        pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module("v21");
-        PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, 2);
-        if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
-            return PMIX_ERR_INIT;
-        }
-    } else if (NULL != (evar = getenv("PMIX_SERVER_URI"))) {
-        /* this is a pre-v2.1 server - must use the v12 bfrops module */
-        pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module("v12");
-        PMIX_SET_PEER_MAJOR(pmix_client_globals.myserver, 1);
-        PMIX_SET_PEER_MINOR(pmix_client_globals.myserver, 2);
-        if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
-            return PMIX_ERR_INIT;
-        }
-    } else {
-        /* let the caller know that the server isn't available */
-        return PMIX_ERR_SERVER_NOT_AVAIL;
-    }
-    /* the server will be using the same bfrops as us */
-    pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
-    /* mark that we are using the V1 protocol */
-    pmix_globals.mypeer->protocol = PMIX_PROTOCOL_V1;
-
-    uri = pmix_argv_split(evar, ':');
-    if (3 != pmix_argv_count(uri)) {
-        pmix_argv_free(uri);
-        PMIX_ERROR_LOG(PMIX_ERROR);
-        return PMIX_ERROR;
-    }
-    /* set the server nspace */
-    if (NULL == pmix_client_globals.myserver->info) {
-        pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t);
-    }
-    if (NULL == pmix_client_globals.myserver->nptr) {
-        pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t);
-    }
-    if (NULL == pmix_client_globals.myserver->nptr->nspace) {
-        pmix_client_globals.myserver->nptr->nspace = strdup(uri[0]);
-    }
-    if (NULL == pmix_client_globals.myserver->info->pname.nspace) {
-        pmix_client_globals.myserver->info->pname.nspace = strdup(uri[0]);
-    }
-
-    /* set the server rank */
-    pmix_client_globals.myserver->info->pname.rank = strtoull(uri[1], NULL, 10);
-
-    /* setup the path to the daemon rendezvous point */
-    memset(&mca_ptl_usock_component.connection, 0, sizeof(struct sockaddr_storage));
-    address = (struct sockaddr_un*)&mca_ptl_usock_component.connection;
-    address->sun_family = AF_UNIX;
-    snprintf(address->sun_path, sizeof(address->sun_path)-1, "%s", uri[2]);
-    /* if the rendezvous file doesn't exist, that's an error */
-    if (0 != access(uri[2], R_OK)) {
-        pmix_argv_free(uri);
-        PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND);
-        return PMIX_ERR_NOT_FOUND;
-    }
-    pmix_argv_free(uri);
-
-  retry:
-    /* establish the connection */
-    len = sizeof(struct sockaddr_un);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_usock_component.connection, len, &sd))) {
-        PMIX_ERROR_LOG(rc);
-        return rc;
-    }
-    pmix_client_globals.myserver->sd = sd;
-
-    /* send our identity and any authentication credentials to the server */
-    if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) {
-        CLOSE_THE_SOCKET(sd);
-        return rc;
-    }
-
-    /* do whatever handshake is required */
-    if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) {
-        CLOSE_THE_SOCKET(sd);
-        if (PMIX_ERR_TEMP_UNAVAILABLE == rc) {
-            /* give it two tries */
-            if (!retried) {
-                retried = true;
-                goto retry;
-            }
-        }
-        return rc;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "sock_peer_try_connect: Connection across to server succeeded");
-
-    /* mark the connection as made */
-    pmix_globals.connected = true;
-
-    pmix_ptl_base_set_nonblocking(sd);
-
-    /* setup recv event */
-    pmix_event_assign(&pmix_client_globals.myserver->recv_event,
-                      pmix_globals.evbase,
-                      pmix_client_globals.myserver->sd,
-                      EV_READ | EV_PERSIST,
-                      pmix_usock_recv_handler, pmix_client_globals.myserver);
-    pmix_event_add(&pmix_client_globals.myserver->recv_event, 0);
-    pmix_client_globals.myserver->recv_ev_active = true;
-    PMIX_POST_OBJECT(pmix_client_globals.myserver);
-    pmix_event_add(&pmix_client_globals.myserver->recv_event, 0);
-
-    /* setup send event */
-    pmix_event_assign(&pmix_client_globals.myserver->send_event,
-                      pmix_globals.evbase,
-                      pmix_client_globals.myserver->sd,
-                      EV_WRITE|EV_PERSIST,
-                      pmix_usock_send_handler, pmix_client_globals.myserver);
-    pmix_client_globals.myserver->send_ev_active = false;
-
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t send_recv(struct pmix_peer_t *peer,
-                               pmix_buffer_t *bfr,
-                               pmix_ptl_cbfunc_t cbfunc,
-                               void *cbdata)
-{
-    pmix_ptl_sr_t *ms;
-    pmix_peer_t *pr = (pmix_peer_t*)peer;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "[%s:%d] post send to server",
-                        __FILE__, __LINE__);
-
-    ms = PMIX_NEW(pmix_ptl_sr_t);
-    PMIX_RETAIN(pr);
-    ms->peer = pr;
-    ms->bfr = bfr;
-    ms->cbfunc = cbfunc;
-    ms->cbdata = cbdata;
-    PMIX_THREADSHIFT(ms, pmix_usock_send_recv);
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t send_oneway(struct pmix_peer_t *peer,
-                                 pmix_buffer_t *bfr,
-                                 pmix_ptl_tag_t tag)
-{
-    pmix_ptl_queue_t *q;
-    pmix_peer_t *pr = (pmix_peer_t*)peer;
-
-    /* we have to transfer this to an event for thread
-     * safety as we need to post this message on the
-     * peer's send queue */
-    q = PMIX_NEW(pmix_ptl_queue_t);
-    PMIX_RETAIN(pr);
-    q->peer = peer;
-    q->buf = bfr;
-    q->tag = tag;
-    PMIX_THREADSHIFT(q, pmix_usock_send);
-
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t send_connect_ack(int sd)
-{
-    char *msg;
-    pmix_usock_hdr_t hdr;
-    size_t sdsize=0, csize=0;
-    pmix_byte_object_t cred;
-    pmix_status_t rc;
-    char *sec, *bfrops, *gds;
-    pmix_bfrop_buffer_type_t bftype;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix: SEND CONNECT ACK");
-
-    /* setup the header */
-    memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
-    hdr.pindex = -1;
-    hdr.tag = UINT32_MAX;
-
-    /* reserve space for the nspace and rank info */
-    sdsize = strlen(pmix_globals.myid.nspace) + 1 + sizeof(int);
-
-    /* get a credential, if the security system provides one. Not
-     * every SPC will do so, thus we must first check */
-    PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
-    PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer,
-                          NULL, 0, NULL, 0, &cred);
-    if (PMIX_SUCCESS != rc) {
-        return rc;
-    }
-
-    /* add the name of our active sec module - we selected it
-     * in pmix_client.c prior to entering here */
-    sec = pmix_globals.mypeer->nptr->compat.psec->name;
-
-    /* add our active bfrops module name */
-    bfrops = pmix_globals.mypeer->nptr->compat.bfrops->version;
-    /* and the type of buffer we are using */
-    bftype = pmix_globals.mypeer->nptr->compat.type;
-
-    /* add our active gds module for working with the server */
-    gds = (char*)pmix_client_globals.myserver->nptr->compat.gds->name;
-
-    /* set the number of bytes to be read beyond the header */
-    hdr.nbytes = sdsize + (strlen(PMIX_VERSION) + 1) + \
-                (sizeof(size_t) + cred.size) + \
-                (strlen(sec) + 1) + \
-                (strlen(bfrops) + 1) + sizeof(bftype) + \
-                (strlen(gds) + 1);  // must NULL terminate the strings!
-
-    /* create a space for our message */
-    sdsize = (sizeof(hdr) + hdr.nbytes);
-    if (NULL == (msg = (char*)malloc(sdsize))) {
-        PMIX_BYTE_OBJECT_DESTRUCT(&cred);
-        return PMIX_ERR_OUT_OF_RESOURCE;
-    }
-    memset(msg, 0, sdsize);
-
-    /* load the message */
-    csize=0;
-    memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t));
-    csize += sizeof(pmix_usock_hdr_t);
-    /* pass our nspace */
-    memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
-    csize += strlen(pmix_globals.myid.nspace)+1;
-    /* pass our rank */
-    memcpy(msg+csize, &pmix_globals.myid.rank, sizeof(int));
-    csize += sizeof(int);
-
-    /* pass our version string */
-    memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
-    csize += strlen(PMIX_VERSION)+1;
-
-    /* pass the size of the credential */
-    memcpy(msg+csize, &cred.size, sizeof(size_t));
-    csize += sizeof(size_t);
-    if (0 < cred.size) {
-        memcpy(msg+csize, cred.bytes, cred.size);
-        csize += cred.size;
-    }
-    PMIX_BYTE_OBJECT_DESTRUCT(&cred);
-
-    /* pass our active sec module */
-    memcpy(msg+csize, sec, strlen(sec));
-    csize += strlen(sec)+1;
-
-    /* provide our active bfrops module */
-    memcpy(msg+csize, bfrops, strlen(bfrops));
-    csize += strlen(bfrops)+1;
-
-    /* provide the bfrops type */
-    memcpy(msg+csize, &bftype, sizeof(bftype));
-    csize += sizeof(bftype);
-
-    /* provide the gds module */
-    memcpy(msg+csize, gds, strlen(gds));
-
-    /* send the entire msg across */
-    if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
-        free(msg);
-        return PMIX_ERR_UNREACH;
-    }
-    free(msg);
-    return PMIX_SUCCESS;
-}
-
-/* we receive a connection acknowledgement from the server,
- * consisting of nothing more than a status report. If success,
- * then we initiate authentication method */
-static pmix_status_t recv_connect_ack(int sd)
-{
-    pmix_status_t reply;
-    pmix_status_t rc;
-    struct timeval tv, save;
-    pmix_socklen_t sz;
-    bool sockopt = true;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix: RECV CONNECT ACK FROM SERVER");
-
-    /* get the current timeout value so we can reset to it */
-    sz = sizeof(save);
-    if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) {
-        if (ENOPROTOOPT == errno || EOPNOTSUPP == errno) {
-            sockopt = false;
-        } else {
-             return PMIX_ERR_UNREACH;
-        }
-    } else {
-        /* set a timeout on the blocking recv so we don't hang */
-        tv.tv_sec  = 2;
-        tv.tv_usec = 0;
-        if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
-            return PMIX_ERR_UNREACH;
-        }
-    }
-
-    /* receive the status reply */
-    rc = pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(int));
-    if (PMIX_SUCCESS != rc) {
-        if (sockopt) {
-            /* return the socket to normal */
-            if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
-                return PMIX_ERR_UNREACH;
-            }
-        }
-        return rc;
-    }
-
-    /* see if they want us to do the handshake */
-    if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
-        PMIX_PSEC_CLIENT_HANDSHAKE(rc, pmix_client_globals.myserver, sd);
-        if (PMIX_SUCCESS != rc) {
-            return rc;
-        }
-    } else if (PMIX_SUCCESS != reply) {
-        return reply;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix: RECV CONNECT CONFIRMATION");
-
-    /* receive our index into the server's client array */
-    rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int));
-    if (PMIX_SUCCESS != rc) {
-        return rc;
-    }
-    if (sockopt) {
-        /* return the socket to normal */
-        if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
-            return PMIX_ERR_UNREACH;
-        }
-    }
-
-    return PMIX_SUCCESS;
-}
-
-static pmix_status_t send_bytes(int sd, char **buf, size_t *remain)
-{
-    pmix_status_t ret = PMIX_SUCCESS;
-    int rc;
-    char *ptr = *buf;
-    while (0 < *remain) {
-        rc = write(sd, ptr, *remain);
-        if (rc < 0) {
-            if (pmix_socket_errno == EINTR) {
-                continue;
-            } else if (pmix_socket_errno == EAGAIN) {
-                /* tell the caller to keep this message on active,
-                 * but let the event lib cycle so other messages
-                 * can progress while this socket is busy
-                 */
-                ret = PMIX_ERR_RESOURCE_BUSY;
-                goto exit;
-            } else if (pmix_socket_errno == EWOULDBLOCK) {
-                /* tell the caller to keep this message on active,
-                 * but let the event lib cycle so other messages
-                 * can progress while this socket is busy
-                 */
-                ret = PMIX_ERR_WOULD_BLOCK;
-                goto exit;
-            }
-            /* we hit an error and cannot progress this message */
-            pmix_output(0, "pmix_usock_msg_send_bytes: write failed: %s (%d) [sd = %d]",
-                        strerror(pmix_socket_errno),
-                        pmix_socket_errno, sd);
-            ret = PMIX_ERR_COMM_FAILURE;
-            goto exit;
-        }
-        /* update location */
-        (*remain) -= rc;
-        ptr += rc;
-    }
-    /* we sent the full data block */
-exit:
-    *buf = ptr;
-    return ret;
-}
-
-static pmix_status_t read_bytes(int sd, char **buf, size_t *remain)
-{
-    pmix_status_t ret = PMIX_SUCCESS;
-    int rc;
-    char *ptr = *buf;
-
-    /* read until all bytes recvd or error */
-    while (0 < *remain) {
-        rc = read(sd, ptr, *remain);
-        if (rc < 0) {
-            if(pmix_socket_errno == EINTR) {
-                continue;
-            } else if (pmix_socket_errno == EAGAIN) {
-                /* tell the caller to keep this message on active,
-                 * but let the event lib cycle so other messages
-                 * can progress while this socket is busy
-                 */
-                ret = PMIX_ERR_RESOURCE_BUSY;
-                goto exit;
-            } else if (pmix_socket_errno == EWOULDBLOCK) {
-                /* tell the caller to keep this message on active,
-                 * but let the event lib cycle so other messages
-                 * can progress while this socket is busy
-                 */
-                ret = PMIX_ERR_WOULD_BLOCK;
-                goto exit;
-            }
-            /* we hit an error and cannot progress this message - report
-             * the error back to the RML and let the caller know
-             * to abort this message
-             */
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "pmix_usock_msg_recv: readv failed: %s (%d)",
-                                strerror(pmix_socket_errno),
-                                pmix_socket_errno);
-            ret = PMIX_ERR_UNREACH;
-            goto exit;
-        } else if (0 == rc) {
-            /* the remote peer closed the connection */
-            ret = PMIX_ERR_UNREACH;
-            goto exit;
-        }
-        /* we were able to read something, so adjust counters and location */
-        *remain -= rc;
-        ptr += rc;
-    }
-    /* we read the full data block */
-exit:
-    *buf = ptr;
-    return ret;
-}
-
-/*
- * A file descriptor is available/ready for send. Check the state
- * of the socket and take the appropriate action.
- */
-void pmix_usock_send_handler(int sd, short flags, void *cbdata)
-{
-    pmix_peer_t *peer = (pmix_peer_t*)cbdata;
-    pmix_ptl_send_t *msg = peer->send_msg;
-    pmix_status_t rc;
-    uint32_t nbytes;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(peer);
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "%s:%d usock:send_handler SENDING TO PEER %s:%d tag %u with %s msg",
-                        pmix_globals.myid.nspace, pmix_globals.myid.rank,
-                        peer->info->pname.nspace, peer->info->pname.rank,
-                        (NULL == msg) ? UINT_MAX : msg->hdr.tag,
-                        (NULL == msg) ? "NULL" : "NON-NULL");
-
-    if (NULL != msg) {
-        if (!msg->hdr_sent) {
-            if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
-            /* we have to convert the header back to host-byte order */
-                msg->hdr.pindex = ntohl(msg->hdr.pindex);
-                msg->hdr.tag = ntohl(msg->hdr.tag);
-                nbytes = msg->hdr.nbytes;
-                msg->hdr.nbytes = ntohl(nbytes);
-            }
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "usock:send_handler SENDING HEADER WITH MSG IDX %d TAG %d SIZE %lu",
-                                msg->hdr.pindex, msg->hdr.tag, (unsigned long)msg->hdr.nbytes);
-            if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) {
-                /* header is completely sent */
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "usock:send_handler HEADER SENT");
-                msg->hdr_sent = true;
-                /* setup to send the data */
-                if (NULL == msg->data) {
-                    /* this was a zero-byte msg - nothing more to do */
-                    PMIX_RELEASE(msg);
-                    peer->send_msg = NULL;
-                    goto next;
-                } else {
-                    /* send the data as a single block */
-                    msg->sdptr = msg->data->base_ptr;
-                    msg->sdbytes = msg->hdr.nbytes;
-                }
-                /* fall thru and let the send progress */
-            } else if (PMIX_ERR_RESOURCE_BUSY == rc ||
-                       PMIX_ERR_WOULD_BLOCK == rc) {
-                /* exit this event and let the event lib progress */
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "usock:send_handler RES BUSY OR WOULD BLOCK");
-                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
-                    /* have to convert back again so we are correct when we re-enter */
-                    msg->hdr.pindex = htonl(msg->hdr.pindex);
-                    msg->hdr.tag = htonl(msg->hdr.tag);
-                    nbytes = msg->hdr.nbytes;
-                    msg->hdr.nbytes = htonl(nbytes);
-                }
-                /* ensure we post the modified peer object before another thread
-                 * picks it back up */
-                PMIX_POST_OBJECT(peer);
-                return;
-            } else {
-                // report the error
-                pmix_event_del(&peer->send_event);
-                peer->send_ev_active = false;
-                PMIX_RELEASE(msg);
-                peer->send_msg = NULL;
-                pmix_ptl_base_lost_connection(peer, rc);
-                /* ensure we post the modified peer object before another thread
-                 * picks it back up */
-                PMIX_POST_OBJECT(peer);
-                return;
-            }
-        }
-
-        if (msg->hdr_sent) {
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "usock:send_handler SENDING BODY OF MSG");
-            if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) {
-                // message is complete
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "usock:send_handler BODY SENT");
-                PMIX_RELEASE(msg);
-                peer->send_msg = NULL;
-            } else if (PMIX_ERR_RESOURCE_BUSY == rc ||
-                       PMIX_ERR_WOULD_BLOCK == rc) {
-                /* exit this event and let the event lib progress */
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "usock:send_handler RES BUSY OR WOULD BLOCK");
-                /* ensure we post the modified peer object before another thread
-                 * picks it back up */
-                PMIX_POST_OBJECT(peer);
-                return;
-            } else {
-                // report the error
-                pmix_output(0, "pmix_usock_peer_send_handler: unable to send message ON SOCKET %d",
-                            peer->sd);
-                pmix_event_del(&peer->send_event);
-                peer->send_ev_active = false;
-                PMIX_RELEASE(msg);
-                peer->send_msg = NULL;
-                pmix_ptl_base_lost_connection(peer, rc);
-                /* ensure we post the modified peer object before another thread
-                 * picks it back up */
-                PMIX_POST_OBJECT(peer);
-                return;
-            }
-        }
-
-    next:
-        /* if current message completed - progress any pending sends by
-         * moving the next in the queue into the "on-deck" position. Note
-         * that this doesn't mean we send the message right now - we will
-         * wait for another send_event to fire before doing so. This gives
-         * us a chance to service any pending recvs.
-         */
-        peer->send_msg = (pmix_ptl_send_t*)
-            pmix_list_remove_first(&peer->send_queue);
-    }
-
-    /* if nothing else to do unregister for send event notifications */
-    if (NULL == peer->send_msg && peer->send_ev_active) {
-        pmix_event_del(&peer->send_event);
-        peer->send_ev_active = false;
-    }
-
-    /* ensure we post the modified peer object before another thread
-     * picks it back up */
-    PMIX_POST_OBJECT(peer);
-}
-
-/*
- * Dispatch to the appropriate action routine based on the state
- * of the connection with the peer.
- */
-
-void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
-{
-    pmix_status_t rc;
-    pmix_peer_t *peer = (pmix_peer_t*)cbdata;
-    pmix_ptl_recv_t *msg = NULL;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(peer);
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "usock:recv:handler called with peer %s:%d",
-                        (NULL == peer) ? "NULL" : peer->info->pname.nspace,
-                        (NULL == peer) ? PMIX_RANK_UNDEF : peer->info->pname.rank);
-
-    if (NULL == peer) {
-        return;
-    }
-    /* allocate a new message and setup for recv */
-    if (NULL == peer->recv_msg) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "usock:recv:handler allocate new recv msg");
-        peer->recv_msg = PMIX_NEW(pmix_ptl_recv_t);
-        if (NULL == peer->recv_msg) {
-            pmix_output(0, "usock_recv_handler: unable to allocate recv message\n");
-            goto err_close;
-        }
-        PMIX_RETAIN(peer);
-        peer->recv_msg->peer = peer;  // provide a handle back to the peer object
-        /* start by reading the header */
-        peer->recv_msg->rdptr = (char*)&peer->recv_msg->hdr;
-        peer->recv_msg->rdbytes = sizeof(pmix_usock_hdr_t);
-    }
-    msg = peer->recv_msg;
-    msg->sd = sd;
-    /* if the header hasn't been completely read, read it */
-    if (!msg->hdr_recvd) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "usock:recv:handler read hdr on socket %d", peer->sd);
-        if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &msg->rdptr, &msg->rdbytes))) {
-            /* completed reading the header */
-            peer->recv_msg->hdr_recvd = true;
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "RECVD MSG FOR TAG %d SIZE %d",
-                                (int)peer->recv_msg->hdr.tag,
-                                (int)peer->recv_msg->hdr.nbytes);
-            /* if this is a zero-byte message, then we are done */
-            if (0 == peer->recv_msg->hdr.nbytes) {
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "RECVD ZERO-BYTE MESSAGE FROM %s:%d for tag %d",
-                                    peer->info->pname.nspace, peer->info->pname.rank,
-                                    peer->recv_msg->hdr.tag);
-                peer->recv_msg->data = NULL;  // make sure
-                peer->recv_msg->rdptr = NULL;
-                peer->recv_msg->rdbytes = 0;
-                /* post it for delivery */
-                PMIX_ACTIVATE_POST_MSG(peer->recv_msg);
-                peer->recv_msg = NULL;
-                PMIX_POST_OBJECT(peer);
-                return;
-            } else {
-                pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                    "usock:recv:handler allocate data region of size %lu",
-                                    (unsigned long)peer->recv_msg->hdr.nbytes);
-                /* allocate the data region */
-                peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes);
-                memset(peer->recv_msg->data, 0, peer->recv_msg->hdr.nbytes);
-                /* point to it */
-                peer->recv_msg->rdptr = peer->recv_msg->data;
-                peer->recv_msg->rdbytes = peer->recv_msg->hdr.nbytes;
-            }
-            /* fall thru and attempt to read the data */
-        } else if (PMIX_ERR_RESOURCE_BUSY == rc ||
-                   PMIX_ERR_WOULD_BLOCK == rc) {
-            /* exit this event and let the event lib progress */
-            return;
-        } else {
-            /* the remote peer closed the connection - report that condition
-             * and let the caller know
-             */
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "pmix_usock_msg_recv: peer closed connection");
-            goto err_close;
-        }
-    }
-
-    if (peer->recv_msg->hdr_recvd) {
-        /* continue to read the data block - we start from
-         * wherever we left off, which could be at the
-         * beginning or somewhere in the message
-         */
-        if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &msg->rdptr, &msg->rdbytes))) {
-            /* we recvd all of the message */
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "RECVD COMPLETE MESSAGE FROM SERVER OF %d BYTES FOR TAG %d ON PEER SOCKET %d",
-                                (int)peer->recv_msg->hdr.nbytes,
-                                peer->recv_msg->hdr.tag, peer->sd);
-            /* post it for delivery */
-            PMIX_ACTIVATE_POST_MSG(peer->recv_msg);
-            peer->recv_msg = NULL;
-            /* ensure we post the modified peer object before another thread
-             * picks it back up */
-            PMIX_POST_OBJECT(peer);
-            return;
-        } else if (PMIX_ERR_RESOURCE_BUSY == rc ||
-                   PMIX_ERR_WOULD_BLOCK == rc) {
-            /* exit this event and let the event lib progress */
-            /* ensure we post the modified peer object before another thread
-             * picks it back up */
-            PMIX_POST_OBJECT(peer);
-            return;
-        } else {
-            /* the remote peer closed the connection - report that condition
-             * and let the caller know
-             */
-            pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                                "pmix_usock_msg_recv: peer closed connection");
-            goto err_close;
-        }
-    }
-    /* success */
-    return;
-
-  err_close:
-    /* stop all events */
-    if (peer->recv_ev_active) {
-        pmix_event_del(&peer->recv_event);
-        peer->recv_ev_active = false;
-    }
-    if (peer->send_ev_active) {
-        pmix_event_del(&peer->send_event);
-        peer->send_ev_active = false;
-    }
-    if (NULL != peer->recv_msg) {
-        PMIX_RELEASE(peer->recv_msg);
-        peer->recv_msg = NULL;
-    }
-    pmix_ptl_base_lost_connection(peer, PMIX_ERR_UNREACH);
-    /* ensure we post the modified peer object before another thread
-     * picks it back up */
-    PMIX_POST_OBJECT(peer);
-}
-
-void pmix_usock_send_recv(int fd, short args, void *cbdata)
-{
-    pmix_ptl_sr_t *ms = (pmix_ptl_sr_t*)cbdata;
-    pmix_ptl_posted_recv_t *req;
-    pmix_ptl_send_t *snd;
-    uint32_t tag;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(ms);
-
-    if (ms->peer->sd < 0) {
-        /* this peer's socket has been closed */
-        PMIX_RELEASE(ms);
-        /* ensure we post the object before another thread
-         * picks it back up */
-        PMIX_POST_OBJECT(NULL);
-        return;
-    }
-
-    /* take the next tag in the sequence */
-    pmix_ptl_globals.current_tag++;
-    if (UINT32_MAX == pmix_ptl_globals.current_tag ) {
-        pmix_ptl_globals.current_tag = PMIX_PTL_TAG_DYNAMIC;
-    }
-    tag = pmix_ptl_globals.current_tag;
-
-    if (NULL != ms->cbfunc) {
-        /* if a callback msg is expected, setup a recv for it */
-        req = PMIX_NEW(pmix_ptl_posted_recv_t);
-        req->tag = tag;
-        req->cbfunc = ms->cbfunc;
-        req->cbdata = ms->cbdata;
-        pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
-                            "posting recv on tag %d", req->tag);
-        /* add it to the list of recvs - we cannot have unexpected messages
-         * in this subsystem as the server never sends us something that
-         * we didn't previously request */
-        pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super);
-    }
-
-    snd = PMIX_NEW(pmix_ptl_send_t);
-    snd->hdr.pindex = pmix_globals.pindex;
-    snd->hdr.tag = tag;
-    snd->hdr.nbytes = ms->bfr->bytes_used;
-    snd->data = ms->bfr;
-    /* always start with the header */
-    snd->sdptr = (char*)&snd->hdr;
-    snd->sdbytes = sizeof(pmix_usock_hdr_t);
-
-    /* if there is no message on-deck, put this one there */
-    if (NULL == ms->peer->send_msg) {
-        ms->peer->send_msg = snd;
-    } else {
-        /* add it to the queue */
-        pmix_list_append(&ms->peer->send_queue, &snd->super);
-    }
-    /* ensure the send event is active */
-    if (!ms->peer->send_ev_active) {
-        ms->peer->send_ev_active = true;
-        PMIX_POST_OBJECT(snd);
-        pmix_event_add(&ms->peer->send_event, 0);
-    }
-    /* cleanup */
-    PMIX_RELEASE(ms);
-    PMIX_POST_OBJECT(snd);
-}
-
-static void pmix_usock_send(int sd, short args, void *cbdata)
-{
-    pmix_ptl_queue_t *queue = (pmix_ptl_queue_t*)cbdata;
-    pmix_ptl_send_t *snd;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(queue);
-
-    if (NULL == queue->peer || queue->peer->sd < 0 ||
-        NULL == queue->peer->info || NULL == queue->peer->nptr) {
-        /* this peer has lost connection */
-        PMIX_RELEASE(queue);
-        /* ensure we post the object before another thread
-         * picks it back up */
-        PMIX_POST_OBJECT(queue);
-        return;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "[%s:%d] send to %s:%u on tag %d",
-                        __FILE__, __LINE__,
-                        (queue->peer)->info->pname.nspace,
-                        (queue->peer)->info->pname.rank, (queue->tag));
-
-    snd = PMIX_NEW(pmix_ptl_send_t);
-    snd->hdr.pindex = htonl(pmix_globals.pindex);
-    snd->hdr.tag = htonl(queue->tag);
-    snd->hdr.nbytes = htonl((queue->buf)->bytes_used);
-    snd->data = (queue->buf);
-    /* always start with the header */
-    snd->sdptr = (char*)&snd->hdr;
-    snd->sdbytes = sizeof(pmix_ptl_hdr_t);
-
-    /* if there is no message on-deck, put this one there */
-    if (NULL == (queue->peer)->send_msg) {
-        (queue->peer)->send_msg = snd;
-    } else {
-        /* add it to the queue */
-        pmix_list_append(&(queue->peer)->send_queue, &snd->super);
-    }
-    /* ensure the send event is active */
-    if (!(queue->peer)->send_ev_active) {
-        (queue->peer)->send_ev_active = true;
-        PMIX_POST_OBJECT(queue->peer);
-        pmix_event_add(&(queue->peer)->send_event, 0);
-    }
-    PMIX_RELEASE(queue);
-    PMIX_POST_OBJECT(snd);
-}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/usock/ptl_usock_component.c pmix-4.0.0/src/mca/ptl/usock/ptl_usock_component.c
--- pmix-3.2.2~rc1/src/mca/ptl/usock/ptl_usock_component.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/usock/ptl_usock_component.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,774 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennptlee and The University
- *                         of Tennptlee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2017      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2018      IBM Corporation.  All rights reserved.
- * Copyright (c) 2019      Mellanox Technologies, Inc. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- * These symbols are in a file by themselves to provide nice linker
- * semantics.  Since linkers generally pull in symbols by object
- * files, keeping these symbols as the only symbols in this file
- * prevents utility programs such as "ompi_info" from having to import
- * entire components just to query their version and parameters.
- */
-
-#include "src/include/pmix_config.h"
-#include "include/pmix_common.h"
-
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_UIO_H
-#include <sys/uio.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#include <errno.h>
-
-#include "src/util/argv.h"
-#include "src/util/error.h"
-#include "src/util/fd.h"
-#include "src/util/show_help.h"
-#include "src/util/strnlen.h"
-#include "src/mca/bfrops/base/base.h"
-#include "src/mca/gds/base/base.h"
-#include "src/mca/psec/base/base.h"
-#include "src/server/pmix_server_ops.h"
-
-#include "src/mca/ptl/base/base.h"
-#include "src/mca/ptl/usock/ptl_usock.h"
-
-static pmix_status_t component_open(void);
-static pmix_status_t component_close(void);
-static int component_query(pmix_mca_base_module_t **module, int *priority);
-static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo,
-                                    bool *need_listener);
-
-/*
- * Instantiate the public struct with all of our public information
- * and pointers to our public functions in it
- */
-PMIX_EXPORT pmix_ptl_usock_component_t mca_ptl_usock_component = {
-    .super = {
-        .base = {
-            PMIX_PTL_BASE_VERSION_1_0_0,
-
-            /* Component name and version */
-            .pmix_mca_component_name = "usock",
-            PMIX_MCA_BASE_MAKE_VERSION(component,
-                                       PMIX_MAJOR_VERSION,
-                                       PMIX_MINOR_VERSION,
-                                       PMIX_RELEASE_VERSION),
-
-            /* Component open and close functions */
-            .pmix_mca_open_component = component_open,
-            .pmix_mca_close_component = component_close,
-            .pmix_mca_query_component = component_query
-        },
-        .priority = 15,
-        .uri = NULL,
-        .setup_listener = setup_listener
-    },
-    .tmpdir = NULL,
-    .filename = NULL
-};
-
-static void connection_handler(int sd, short args, void *cbdata);
-static void listener_cb(int incoming_sd, void *cbdata);
-
-pmix_status_t component_open(void)
-{
-    char *tdir;
-
-    memset(&mca_ptl_usock_component.connection, 0, sizeof(mca_ptl_usock_component.connection));
-
-    /* check for environ-based directives
-     * on system tmpdir to use */
-    if (NULL == (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) {
-        if (NULL == (tdir = getenv("TMPDIR"))) {
-            if (NULL == (tdir = getenv("TEMP"))) {
-                if (NULL == (tdir = getenv("TMP"))) {
-                    tdir = "/tmp";
-                }
-            }
-        }
-    }
-    if (NULL != tdir) {
-        mca_ptl_usock_component.tmpdir = strdup(tdir);
-    }
-
-    return PMIX_SUCCESS;
-}
-
-
-pmix_status_t component_close(void)
-{
-    if (NULL != mca_ptl_usock_component.tmpdir) {
-        free(mca_ptl_usock_component.tmpdir);
-    }
-    if (NULL != mca_ptl_usock_component.super.uri) {
-        free(mca_ptl_usock_component.super.uri);
-    }
-    if (NULL != mca_ptl_usock_component.filename) {
-        /* remove the file */
-        unlink(mca_ptl_usock_component.filename);
-        free(mca_ptl_usock_component.filename);
-    }
-
-    return PMIX_SUCCESS;
-}
-
-static int component_query(pmix_mca_base_module_t **module, int *priority)
-{
-    if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
-        return PMIX_ERR_NOT_SUPPORTED;
-    }
-    *module = (pmix_mca_base_module_t*)&pmix_ptl_usock_module;
-    *priority = mca_ptl_usock_component.super.priority;
-    return PMIX_SUCCESS;
-}
-
-/* if we are the server, then we need to setup a usock rendezvous
- * point for legacy releases, but only do so if requested as some
- * systems may not wish to support older releases. The system can,
- * of course, simply use the MCA param method to disable this
- * component (PMIX_MCA_ptl=^usock), or can tell us to disqualify
- * ourselves using an info key to this API.
- *
- * NOTE: we accept MCA parameters, but info keys override them
- */
-static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo,
-                                    bool *need_listener)
-{
-    int flags;
-    size_t n;
-    pmix_listener_t *lt;
-    pmix_status_t rc;
-    socklen_t addrlen;
-    struct sockaddr_un *address;
-    bool disabled = true;
-    char *pmix_pid;
-    pid_t mypid;
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "ptl:usock setup_listener");
-
-    /* if we are not a server, then we shouldn't be doing this */
-    if (!PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
-        return PMIX_ERR_NOT_SUPPORTED;
-    }
-
-    /* scan the info keys and process any override instructions */
-    if (NULL != info) {
-        for (n=0; n < ninfo; n++) {
-            if (0 == strcmp(info[n].key, PMIX_USOCK_DISABLE)) {
-                disabled = PMIX_INFO_TRUE(&info[n]);;
-                break;
-            }
-        }
-    }
-
-    /* see if we have been disabled */
-    if (disabled) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "ptl:usock not available");
-        return PMIX_ERR_NOT_AVAILABLE;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "ptl:usock setting up listener");
-
-    addrlen = sizeof(struct sockaddr_un);
-    address = (struct sockaddr_un*)&mca_ptl_usock_component.connection;
-    address->sun_family = AF_UNIX;
-
-    /* define the listener */
-    lt = PMIX_NEW(pmix_listener_t);
-
-    /* for now, just setup the v1.1 series rendezvous point
-     * we use the pid to reduce collisions */
-    mypid = getpid();
-    if (0 > asprintf(&pmix_pid, "%s/pmix-%d", mca_ptl_usock_component.tmpdir, mypid)) {
-        PMIX_RELEASE(lt);
-        return PMIX_ERR_NOMEM;
-    }
-    if ((strlen(pmix_pid) + 1) > sizeof(address->sun_path)-1) {
-        pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true,
-                       mca_ptl_usock_component.tmpdir, pmix_pid);
-        free(pmix_pid);
-        PMIX_RELEASE(lt);
-        return PMIX_ERR_INVALID_LENGTH;
-    }
-    snprintf(address->sun_path, sizeof(address->sun_path)-1, "%s", pmix_pid);
-    free(pmix_pid);
-    /* set the URI */
-    lt->varname = strdup("PMIX_SERVER_URI:PMIX_SERVER_URI2USOCK");
-    if (0 > asprintf(&lt->uri, "%s:%lu:%s", pmix_globals.myid.nspace,
-                    (unsigned long)pmix_globals.myid.rank, address->sun_path)) {
-        PMIX_RELEASE(lt);
-        return PMIX_ERR_NOMEM;
-    }
-    /* save the rendezvous filename for later removal */
-    mca_ptl_usock_component.filename = strdup(address->sun_path);
-
-    lt->protocol = PMIX_PROTOCOL_V1;
-    lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_usock_module;
-    lt->cbfunc = connection_handler;
-    pmix_list_append(&pmix_ptl_globals.listeners, &lt->super);
-
-    /* create a listen socket for incoming connection attempts */
-    lt->socket = socket(PF_UNIX, SOCK_STREAM, 0);
-    if (lt->socket < 0) {
-        printf("%s:%d socket() failed\n", __FILE__, __LINE__);
-        goto sockerror;
-    }
-    /* Set the socket to close-on-exec so that no children inherit
-     * this FD */
-    if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) {
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    if (bind(lt->socket, (struct sockaddr*)address, addrlen) < 0) {
-        printf("%s:%d bind() failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-    /* chown as required */
-    if (lt->owner_given) {
-        if (0 != chown(address->sun_path, lt->owner, -1)) {
-            pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno));
-            CLOSE_THE_SOCKET(lt->socket);
-            goto sockerror;
-        }
-    }
-    if (lt->group_given) {
-        if (0 != chown(address->sun_path, -1, lt->group)) {
-            pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno));
-            CLOSE_THE_SOCKET(lt->socket);
-            goto sockerror;
-        }
-    }
-    /* set the mode as required */
-    if (0 != chmod(address->sun_path, lt->mode)) {
-        pmix_output(0, "CANNOT CHMOD socket %s: %s", address->sun_path, strerror (errno));
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* setup listen backlog to maximum allowed by kernel */
-    if (listen(lt->socket, SOMAXCONN) < 0) {
-        printf("%s:%d listen() failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* set socket up to be non-blocking, otherwise accept could block */
-    if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) {
-        printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-    flags |= O_NONBLOCK;
-    if (fcntl(lt->socket, F_SETFL, flags) < 0) {
-        printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__);
-        CLOSE_THE_SOCKET(lt->socket);
-        goto sockerror;
-    }
-
-    /* if the server will listen for us, then ask it to do so now */
-    rc = PMIX_ERR_NOT_SUPPORTED;
-    if (NULL != pmix_host_server.listener) {
-        rc = pmix_host_server.listener(lt->socket, listener_cb, (void*)lt);
-    }
-
-    if (PMIX_SUCCESS != rc) {
-        *need_listener = true;
-    }
-
-    return PMIX_SUCCESS;
-
-  sockerror:
-      pmix_list_remove_item(&pmix_ptl_globals.listeners, &lt->super);
-      PMIX_RELEASE(lt);
-      return PMIX_ERROR;
-}
-
-static void listener_cb(int incoming_sd, void *cbdata)
-{
-    pmix_pending_connection_t *pending_connection;
-
-    /* throw it into our event library for processing */
-    pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
-                        "listen_cb: pushing new connection %d into evbase",
-                        incoming_sd);
-    pending_connection = PMIX_NEW(pmix_pending_connection_t);
-    pending_connection->sd = incoming_sd;
-    pmix_event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
-                      EV_WRITE, connection_handler, pending_connection);
-    pmix_event_active(&pending_connection->ev, EV_WRITE, 1);
-}
-
-static void connection_handler(int sd, short args, void *cbdata)
-{
-    pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
-    char *msg, *ptr, *nspace, *version, *sec, *bfrops, *gds;
-    pmix_status_t rc, reply;
-    unsigned int rank;
-    pmix_usock_hdr_t hdr;
-    pmix_namespace_t *nptr, *tmp;
-    pmix_rank_info_t *info;
-    pmix_peer_t *psave = NULL;
-    bool found;
-    pmix_proc_t proc;
-    size_t len;
-    pmix_bfrop_buffer_type_t bftype;
-    char **vers;
-    int major, minor, rel;
-    unsigned int msglen;
-    pmix_info_t ginfo;
-    pmix_byte_object_t cred;
-    uint32_t u32;
-
-    /* acquire the object */
-    PMIX_ACQUIRE_OBJECT(pnd);
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "USOCK CONNECTION FROM PEER ON SOCKET %d", pnd->sd);
-
-    /* ensure all is zero'd */
-    memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
-
-    /* get the header */
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_usock_hdr_t)))) {
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get the id, authentication and version payload (and possibly
-     * security credential) - to guard against potential attacks,
-     * we'll set an arbitrary limit per a define */
-    if (PMIX_MAX_CRED_SIZE < hdr.nbytes) {
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    if (NULL == (msg = (char*)malloc(hdr.nbytes))) {
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    if (PMIX_SUCCESS != pmix_ptl_base_recv_blocking(pnd->sd, msg, hdr.nbytes)) {
-        /* unable to complete the recv */
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "unable to complete recv of connect-ack with client ON SOCKET %d", pnd->sd);
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    len = hdr.nbytes;
-    ptr = msg;
-
-    /* extract the nspace of the requestor */
-    PMIX_STRNLEN(msglen, ptr, len);
-    if (msglen < len) {
-        nspace = ptr;
-        ptr += strlen(nspace) + 1;
-        len -= strlen(nspace) + 1;
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* extract the rank */
-    PMIX_STRNLEN(msglen, ptr, len);
-    if (msglen <= len) {
-        memcpy(&rank, ptr, sizeof(int));
-        ptr += sizeof(int);
-        len -= sizeof(int);
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get their version string */
-    PMIX_STRNLEN(msglen, ptr, len);
-    if (msglen < len) {
-        version = ptr;
-        ptr += strlen(version) + 1;
-        len -= strlen(version) + 1;
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* check the version - we do NOT support anything less than
-     * v1.2.5 */
-    vers = pmix_argv_split(version, '.');
-    major = strtol(vers[0], NULL, 10);
-    minor = strtol(vers[1], NULL, 10);
-    rel = strtol(vers[2], NULL, 10);
-    pmix_argv_free(vers);
-    if (1 == major && (2 != minor || 5 > rel)) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "connection request from client of unsupported version %s", version);
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get any provided credential */
-    if (1 == major) {
-        /* credential is always a string */
-        PMIX_STRNLEN(msglen, ptr, len);
-        if (msglen < len) {
-            cred.bytes = ptr;
-            cred.size = strlen(ptr);
-            ptr += cred.size + 1;
-            len -= cred.size + 1;
-        } else {
-            free(msg);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd);
-            return;
-        }
-    } else {
-        /* credential could be something else */
-        if (sizeof(size_t) < len) {
-            memcpy(&cred.size, ptr, sizeof(size_t));
-            ptr += sizeof(size_t);
-            len -= sizeof(size_t);
-        } else {
-            free(msg);
-            CLOSE_THE_SOCKET(pnd->sd);
-            PMIX_RELEASE(pnd);
-            return;
-        }
-        if (0 < cred.size) {
-            cred.bytes = ptr;
-            ptr += cred.size;
-            len -= cred.size;
-        } else {
-            /* set cred pointer to NULL to guard against validation
-             * methods that assume a zero length credential is NULL */
-            cred.bytes = NULL;
-        }
-    }
-
-    /* get their sec module */
-    PMIX_STRNLEN(msglen, ptr, len);
-    if (msglen < len) {
-        sec = ptr;
-        ptr += strlen(sec) + 1;
-        len -= strlen(sec) + 1;
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get their bfrops module */
-    PMIX_STRNLEN(msglen, ptr, len);
-    if (msglen < len) {
-        bfrops = ptr;
-        ptr += strlen(bfrops) + 1;
-        len -= strlen(bfrops) + 1;
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get their buffer type */
-    if (0 < len) {
-        bftype = ptr[0];
-        ptr += 1;
-        len -= 1;
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* get their gds module */
-    PMIX_STRNLEN(msglen, ptr, len);
-    if (msglen < len) {
-        gds = ptr;
-        ptr += strlen(gds) + 1;
-        len -= strlen(gds) + 1;
-    } else {
-        free(msg);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "connect-ack recvd from peer %s:%d:%s on socket %d",
-                        nspace, rank, version, pnd->sd);
-
-    /* see if we know this nspace */
-    nptr = NULL;
-    PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_namespace_t) {
-        if (0 == strcmp(tmp->nspace, nspace)) {
-            nptr = tmp;
-            break;
-        }
-    }
-    if (NULL == nptr) {
-        /* we don't know this namespace, reject it */
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_NOT_FOUND;
-        goto error;
-    }
-
-    /* see if we have this peer in our list */
-    info = NULL;
-    found = false;
-    PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) {
-        if (info->pname.rank == rank) {
-            found = true;
-            break;
-        }
-    }
-    if (!found) {
-        /* rank unknown, reject it */
-        free(msg);
-        /* send an error reply to the client */
-        rc = PMIX_ERR_NOT_FOUND;
-        goto error;
-    }
-    /* a peer can connect on multiple sockets since it can fork/exec
-     * a child that also calls PMIx_Init, so add it here if necessary.
-     * Create the tracker for this peer */
-    psave = PMIX_NEW(pmix_peer_t);
-    if (NULL == psave) {
-        free(msg);
-        rc = PMIX_ERR_NOMEM;
-        goto error;
-    }
-    /* mark it as being a client of the correct type */
-    PMIX_SET_PROC_TYPE(&psave->proc_type, PMIX_PROC_CLIENT);
-    PMIX_SET_PROC_MAJOR(&psave->proc_type, major);
-    PMIX_SET_PROC_MINOR(&psave->proc_type, minor);
-    PMIX_SET_PROC_RELEASE(&psave->proc_type, rel);
-
-    /* save the protocol */
-    psave->protocol = pnd->protocol;
-    /* add the nspace tracker */
-    PMIX_RETAIN(nptr);
-    psave->nptr = nptr;
-    PMIX_RETAIN(info);
-    psave->info = info;
-    /* save the epilog info */
-    psave->epilog.uid = info->uid;
-    psave->epilog.gid = info->gid;
-    nptr->epilog.uid = info->uid;
-    nptr->epilog.gid = info->gid;
-    info->proc_cnt++; /* increase number of processes on this rank */
-    psave->sd = pnd->sd;
-    if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) {
-        free(msg);
-        info->proc_cnt--;
-        PMIX_RELEASE(info);
-        PMIX_RELEASE(psave);
-        /* probably cannot send an error reply if we are out of memory */
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-    info->peerid = psave->index;
-
-    /* get the appropriate compatibility modules */
-    nptr->compat.psec = pmix_psec_base_assign_module(sec);
-    if (NULL == nptr->compat.psec) {
-        free(msg);
-        info->proc_cnt--;
-        PMIX_RELEASE(info);
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-        PMIX_RELEASE(psave);
-        /* send an error reply to the client */
-        goto error;
-    }
-
-    /* set the bfrops module to match this peer */
-    nptr->compat.bfrops = pmix_bfrops_base_assign_module(bfrops);
-    if (NULL == nptr->compat.bfrops) {
-        free(msg);
-        info->proc_cnt--;
-        PMIX_RELEASE(info);
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-        PMIX_RELEASE(psave);
-       /* send an error reply to the client */
-        goto error;
-    }
-    /* set the buffer type */
-    nptr->compat.type = bftype;
-
-    /* set the gds module to match this peer */
-    PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, gds, PMIX_STRING);
-    nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
-    PMIX_INFO_DESTRUCT(&ginfo);
-    if (NULL == nptr->compat.gds) {
-        free(msg);
-        info->proc_cnt--;
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-        PMIX_RELEASE(psave);
-        /* send an error reply to the client */
-        goto error;
-    }
-
-    /* if we haven't previously stored the version for this
-     * nspace, do so now */
-    if (!nptr->version_stored) {
-        PMIX_INFO_LOAD(&ginfo, PMIX_BFROPS_MODULE, nptr->compat.bfrops->version, PMIX_STRING);
-        PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, &ginfo, 1);
-        PMIX_INFO_DESTRUCT(&ginfo);
-        nptr->version_stored = true;
-    }
-
-    /* the choice of PTL module was obviously made by the connecting
-     * tool as we received this request via that channel, so simply
-     * record it here for future use */
-    nptr->compat.ptl = &pmix_ptl_usock_module;
-
-    /* now done with the msg */
-    free(msg);
-
-    /* validate the connection - the macro will send the status result to the client */
-    PMIX_PSEC_VALIDATE_CONNECTION(reply, psave, NULL, 0, NULL, 0, &cred);
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "client connection validated with status=%d", reply);
-
-    /* Communicate the result of validation to the client */
-    u32 = htonl(reply);
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) {
-        PMIX_ERROR_LOG(rc);
-        info->proc_cnt--;
-        PMIX_RELEASE(info);
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-        PMIX_RELEASE(psave);
-        /* error reply was sent by the above macro */
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    /* If needed perform the handshake. The macro will update reply */
-    PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, psave, NULL, 0, NULL, 0, &cred);
-
-    /* It is possible that connection validation failed
-     * We need to reply to the client first and cleanup after */
-    if (PMIX_SUCCESS != reply) {
-        pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                            "validation of client credentials failed: %s",
-                            PMIx_Error_string(rc));
-        info->proc_cnt--;
-        PMIX_RELEASE(info);
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-        PMIX_RELEASE(psave);
-        /* error reply was sent by the above macro */
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-
-
-    /* send the client's array index */
-    if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) {
-        PMIX_ERROR_LOG(rc);
-        info->proc_cnt--;
-        PMIX_RELEASE(info);
-        pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-        PMIX_RELEASE(psave);
-        CLOSE_THE_SOCKET(pnd->sd);
-        PMIX_RELEASE(pnd);
-        return;
-    }
-
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "connect-ack from client completed");
-
-    /* let the host server know that this client has connected */
-    if (NULL != pmix_host_server.client_connected) {
-        pmix_strncpy(proc.nspace, psave->info->pname.nspace, PMIX_MAX_NSLEN);
-        proc.rank = psave->info->pname.rank;
-        rc = pmix_host_server.client_connected(&proc, psave->info->server_object, NULL, NULL);
-        if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
-            PMIX_ERROR_LOG(rc);
-            info->proc_cnt--;
-            PMIX_RELEASE(info);
-            pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
-            PMIX_RELEASE(psave);
-            CLOSE_THE_SOCKET(pnd->sd);
-        }
-    }
-
-    /* start the events for this client */
-    pmix_event_assign(&psave->recv_event, pmix_globals.evbase, pnd->sd,
-                      EV_READ|EV_PERSIST, pmix_usock_recv_handler, psave);
-    pmix_event_add(&psave->recv_event, NULL);
-    psave->recv_ev_active = true;
-    pmix_event_assign(&psave->send_event, pmix_globals.evbase, pnd->sd,
-                      EV_WRITE|EV_PERSIST, pmix_usock_send_handler, psave);
-    pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
-                        "pmix:server client %s:%u has connected on socket %d",
-                        psave->info->pname.nspace, psave->info->pname.rank, psave->sd);
-
-    PMIX_RELEASE(pnd);
-    return;
-
-  error:
-    /* send an error reply to the client */
-    if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) {
-        PMIX_ERROR_LOG(rc);
-    }
-    CLOSE_THE_SOCKET(pnd->sd);
-    PMIX_RELEASE(pnd);
-    return;
-}
diff -Nru pmix-3.2.2~rc1/src/mca/ptl/usock/ptl_usock.h pmix-4.0.0/src/mca/ptl/usock/ptl_usock.h
--- pmix-3.2.2~rc1/src/mca/ptl/usock/ptl_usock.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/mca/ptl/usock/ptl_usock.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2016-2017 Intel, Inc. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#ifndef PMIX_PTL_USOCK_H
-#define PMIX_PTL_USOCK_H
-
-#include "src/mca/ptl/ptl.h"
-
-BEGIN_C_DECLS
-
-typedef struct {
-    pmix_ptl_base_component_t super;
-    char *tmpdir;
-    struct sockaddr_storage connection;
-    char *filename;
-} pmix_ptl_usock_component_t;
-
-/* header for messages */
-typedef struct {
-    int pindex;
-    uint32_t tag;
-    size_t nbytes;
-} pmix_usock_hdr_t;
-
-extern pmix_ptl_usock_component_t mca_ptl_usock_component;
-
-extern pmix_ptl_module_t pmix_ptl_usock_module;
-
-void pmix_usock_send_handler(int sd, short args, void *cbdata);
-
-void pmix_usock_recv_handler(int sd, short args, void *cbdata);
-
-END_C_DECLS
-
-#endif /* PMIX_PTL_USOCK_H */
diff -Nru pmix-3.2.2~rc1/src/runtime/pmix_finalize.c pmix-4.0.0/src/runtime/pmix_finalize.c
--- pmix-3.2.2~rc1/src/runtime/pmix_finalize.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/runtime/pmix_finalize.c	2021-01-02 08:56:17.000000000 +0000
@@ -28,6 +28,7 @@
 
 #include "src/class/pmix_object.h"
 #include "src/client/pmix_client_ops.h"
+#include "src/common/pmix_attributes.h"
 #include "src/util/output.h"
 #include "src/util/keyval_parse.h"
 #include "src/util/show_help.h"
@@ -38,6 +39,7 @@
 #include "src/mca/gds/base/base.h"
 #include "src/mca/pif/base/base.h"
 #include "src/mca/pinstalldirs/base/base.h"
+#include "src/mca/ploc/base/base.h"
 #include "src/mca/plog/base/base.h"
 #include "src/mca/pnet/base/base.h"
 #include "src/mca/preg/base/base.h"
@@ -66,6 +68,11 @@
         return;
     }
 
+    /* release the attribute support trackers */
+    pmix_release_registered_attrs();
+
+    /* close ploc */
+    (void)pmix_mca_base_framework_close(&pmix_ploc_base_framework);
 
     /* close plog */
     (void)pmix_mca_base_framework_close(&pmix_plog_base_framework);
@@ -138,8 +145,6 @@
     PMIX_LIST_DESTRUCT(&pmix_globals.nspaces);
 
     /* now safe to release the event base */
-    if (!pmix_globals.external_evbase) {
-        (void)pmix_progress_thread_stop(NULL);
-    }
+    (void)pmix_progress_thread_stop(NULL);
 
 }
diff -Nru pmix-3.2.2~rc1/src/runtime/pmix_init.c pmix-4.0.0/src/runtime/pmix_init.c
--- pmix-3.2.2~rc1/src/runtime/pmix_init.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/runtime/pmix_init.c	2021-01-02 08:56:17.000000000 +0000
@@ -34,8 +34,9 @@
 #endif
 
 #include "src/include/pmix_globals.h"
+#include "src/util/net.h"
+#include "src/util/name_fns.h"
 #include "src/util/output.h"
-#include "src/util/pmix_environ.h"
 #include "src/util/show_help.h"
 #include "src/mca/base/base.h"
 #include "src/mca/base/pmix_mca_base_var.h"
@@ -44,6 +45,7 @@
 #include "src/mca/gds/base/base.h"
 #include "src/mca/pif/base/base.h"
 #include "src/mca/pinstalldirs/base/base.h"
+#include "src/mca/ploc/base/base.h"
 #include "src/mca/plog/base/base.h"
 #include "src/mca/pnet/base/base.h"
 #include "src/mca/preg/base/base.h"
@@ -52,6 +54,7 @@
 #include "src/mca/ptl/base/base.h"
 
 #include "src/client/pmix_client_ops.h"
+#include "src/common/pmix_attributes.h"
 #include "src/event/pmix_event.h"
 #include "src/include/types.h"
 #include "src/util/error.h"
@@ -77,12 +80,14 @@
     .nodeid = UINT32_MAX,
     .pindex = 0,
     .evbase = NULL,
-    .external_evbase = false,
     .debug_output = -1,
     .connected = false,
     .commits_pending = false,
     .mygds = NULL,
-    .pushstdin = false
+    .pushstdin = false,
+    .topology = {NULL, NULL},
+    .external_topology = false,
+    .external_progress = false
 };
 
 
@@ -101,9 +106,12 @@
 {
     int ret, debug_level;
     char *error = NULL, *evar;
-    size_t n;
+    size_t n, m;
     char hostname[PMIX_MAXHOSTNAMELEN] = {0};
     char *gds = NULL;
+    pmix_info_t *iptr;
+    size_t minfo;
+    bool keepfqdn = false;
 
     if( ++pmix_initialized != 1 ) {
         if( pmix_initialized < 1 ) {
@@ -168,17 +176,14 @@
         goto return_error;
     }
 
-    /* if an external event base wasn't provide, create one */
-    if (!pmix_globals.external_evbase) {
-        /* tell libevent that we need thread support */
-        pmix_event_use_threads();
-
-        /* create an event base and progress thread for us */
-        if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) {
-            error = "progress thread";
-            ret = PMIX_ERROR;
-            goto return_error;
-        }
+    /* tell libevent that we need thread support */
+    pmix_event_use_threads();
+
+    /* create an event base and progress thread for us */
+    if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) {
+        error = "progress thread";
+        ret = PMIX_ERROR;
+        goto return_error;
     }
 
     /* setup the globals structure */
@@ -202,13 +207,6 @@
         error = "notification hotel init";
         goto return_error;
     }
-    /* if we were given a hostname in our environment, use it */
-    if (NULL != (evar = getenv("PMIX_HOSTNAME"))) {
-        pmix_globals.hostname = strdup(evar);
-    } else {
-        gethostname(hostname, PMIX_MAXHOSTNAMELEN-1);
-        pmix_globals.hostname = strdup(hostname);
-    }
 
     /* and setup the iof request tracking list */
     PMIX_CONSTRUCT(&pmix_globals.iof_requests, pmix_pointer_array_t);
@@ -276,7 +274,7 @@
         ret = PMIX_ERR_NOMEM;
         goto return_error;
     }
-    /* whatever our declared proc type, we are definitely v3.0 */
+    /* whatever our declared proc type, set our version */
     PMIX_SET_PEER_TYPE(pmix_globals.mypeer, type);
     PMIX_SET_PEER_MAJOR(pmix_globals.mypeer, PMIX_VERSION_MAJOR);
     PMIX_SET_PEER_MINOR(pmix_globals.mypeer, PMIX_VERSION_MINOR);
@@ -293,10 +291,7 @@
     /* scan incoming info for directives */
     if (NULL != info) {
         for (n=0; n < ninfo; n++) {
-            if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_BASE)) {
-                pmix_globals.evbase = (pmix_event_base_t*)info[n].value.data.ptr;
-                pmix_globals.external_evbase = true;
-            } else if (PMIX_CHECK_KEY(&info[n], PMIX_HOSTNAME)) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_HOSTNAME)) {
                 if (NULL != pmix_globals.hostname) {
                     free(pmix_globals.hostname);
                 }
@@ -308,10 +303,44 @@
                 }
             } else if (PMIX_CHECK_KEY(&info[n], PMIX_GDS_MODULE)) {
                 gds = info[n].value.data.string;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_NODE_INFO_ARRAY)) {
+                /* contains info about our node */
+                iptr = (pmix_info_t*)info[n].value.data.darray->array;
+                minfo = info[n].value.data.darray->size;
+                for (m=0; m < minfo; m++) {
+                    if (PMIX_CHECK_KEY(&iptr[m], PMIX_HOSTNAME)) {
+                        if (NULL != pmix_globals.hostname) {
+                            free(pmix_globals.hostname);
+                        }
+                        pmix_globals.hostname = strdup(iptr[m].value.data.string);
+                    } else if (PMIX_CHECK_KEY(&iptr[m], PMIX_NODEID)) {
+                        PMIX_VALUE_GET_NUMBER(ret, &iptr[m].value, pmix_globals.nodeid, uint32_t);
+                        if (PMIX_SUCCESS != ret) {
+                            goto return_error;
+                        }
+                    }
+                }
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_EXTERNAL_PROGRESS)) {
+                pmix_globals.external_progress = PMIX_INFO_TRUE(&info[n]);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_HOSTNAME_KEEP_FQDN)) {
+                keepfqdn = PMIX_INFO_TRUE(&info[n]);
             }
         }
     }
 
+    /* if we were given a hostname in our environment, use it */
+    if (NULL != (evar = getenv("PMIX_HOSTNAME"))) {
+        pmix_globals.hostname = strdup(evar);
+    } else {
+        gethostname(hostname, PMIX_MAXHOSTNAMELEN-1);
+        /* strip the FQDN unless told to keep it */
+        if (!keepfqdn && !pmix_net_isaddr(hostname) &&
+            NULL != (evar = strchr(hostname, '.'))) {
+            *evar = '\0';
+        }
+        pmix_globals.hostname = strdup(hostname);
+    }
+
     /* the choice of modules to use when communicating with a peer
      * will be done by the individual init functions and at the
      * time of connection to that peer */
@@ -353,10 +382,6 @@
     }
 
     /* open the ptl and select the active plugins */
-    if (NULL != (evar = getenv("PMIX_PTL_MODULE"))) {
-        /* convert to an MCA param, but don't overwrite something already there */
-        pmix_setenv("PMIX_MCA_ptl", evar, false, &environ);
-    }
     if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) {
         error = "pmix_ptl_base_open";
         goto return_error;
@@ -427,12 +452,23 @@
         goto return_error;
     }
 
-    if (!pmix_globals.external_evbase) {
-        /* start progressing the event library */
-        if (PMIX_SUCCESS != (ret = pmix_progress_thread_start(NULL))) {
-            error = "pmix_progress_thread_start";
-            goto return_error;
-        }
+    /* open the ploc and select the active plugins */
+    if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ploc_base_framework, 0)) ) {
+        error = "pmix_ploc_base_open";
+        goto return_error;
+    }
+    if (PMIX_SUCCESS != (ret = pmix_ploc_base_select()) ) {
+        error = "pmix_ploc_base_select";
+        goto return_error;
+    }
+
+    /* initialize the attribute support system */
+    pmix_init_registered_attrs();
+
+    /* start progressing the event library */
+    if (PMIX_SUCCESS != (ret = pmix_progress_thread_start(NULL))) {
+        error = "pmix_progress_thread_start";
+        goto return_error;
     }
 
     return PMIX_SUCCESS;
diff -Nru pmix-3.2.2~rc1/src/runtime/pmix_progress_threads.c pmix-4.0.0/src/runtime/pmix_progress_threads.c
--- pmix-3.2.2~rc1/src/runtime/pmix_progress_threads.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/runtime/pmix_progress_threads.c	2021-01-02 08:56:17.000000000 +0000
@@ -25,10 +25,9 @@
 #include "src/threads/threads.h"
 #include "src/util/error.h"
 #include "src/util/fd.h"
-
+#include "src/include/pmix_globals.h"
 #include "src/runtime/pmix_progress_threads.h"
 
-
 /* create a tracking object for progress threads */
 typedef struct {
     pmix_list_item_t super;
@@ -93,6 +92,17 @@
                           tracker_constructor,
                           tracker_destructor);
 
+/* LOCAL VARIABLES */
+static bool inited = false;
+static pmix_list_t tracking;
+static struct timeval long_timeout = {
+    .tv_sec = 3600,
+    .tv_usec = 0
+};
+static const char *shared_thread_name = "PMIX-wide async progress thread";
+static pmix_progress_tracker_t *shared_thread_tracker = NULL;
+
+
 #if PMIX_HAVE_LIBEV
 
 typedef enum {
@@ -201,14 +211,6 @@
 }
 #endif
 
-static bool inited = false;
-static pmix_list_t tracking;
-static struct timeval long_timeout = {
-    .tv_sec = 3600,
-    .tv_usec = 0
-};
-static const char *shared_thread_name = "PMIX-wide async progress thread";
-
 /*
  * If this event is fired, just restart it so that this event base
  * continues to have something to block on.
@@ -235,6 +237,14 @@
     return PMIX_THREAD_CANCELLED;
 }
 
+void PMIx_Progress(void)
+{
+    if (NULL == shared_thread_tracker) {
+        return;
+    }
+    pmix_event_loop(shared_thread_tracker->ev_base, PMIX_EVLOOP_ONCE);
+}
+
 static void stop_progress_engine(pmix_progress_tracker_t *trk)
 {
     assert(trk->ev_active);
@@ -321,6 +331,10 @@
     trk->engine_constructed = true;
     pmix_list_append(&tracking, &trk->super);
 
+    if (0 == strcmp(name, shared_thread_name)) {
+        shared_thread_tracker = trk;
+    }
+
     return trk->ev_base;
 }
 
@@ -334,7 +348,10 @@
         return PMIX_ERR_NOT_FOUND;
     }
 
-    if (NULL == name) {
+    if (NULL == name || 0 == strcmp(name, shared_thread_name)) {
+        if (pmix_globals.external_progress) {
+            return PMIX_SUCCESS;
+        }
         name = shared_thread_name;
     }
 
@@ -366,7 +383,10 @@
         return PMIX_ERR_NOT_FOUND;
     }
 
-    if (NULL == name) {
+    if (NULL == name || 0 == strcmp(name, shared_thread_name)) {
+        if (pmix_globals.external_progress) {
+            return PMIX_SUCCESS;
+        }
         name = shared_thread_name;
     }
 
@@ -403,7 +423,10 @@
         return PMIX_ERR_NOT_FOUND;
     }
 
-    if (NULL == name) {
+    if (NULL == name || 0 == strcmp(name, shared_thread_name)) {
+        if (pmix_globals.external_progress) {
+            return PMIX_SUCCESS;
+        }
         name = shared_thread_name;
     }
 
@@ -436,7 +459,10 @@
         return PMIX_ERR_NOT_FOUND;
     }
 
-    if (NULL == name) {
+    if (NULL == name || 0 == strcmp(name, shared_thread_name)) {
+        if (pmix_globals.external_progress) {
+            return PMIX_SUCCESS;
+        }
         name = shared_thread_name;
     }
 
@@ -478,7 +504,10 @@
         return PMIX_ERR_NOT_FOUND;
     }
 
-    if (NULL == name) {
+    if (NULL == name || 0 == strcmp(name, shared_thread_name)) {
+        if (pmix_globals.external_progress) {
+            return PMIX_SUCCESS;
+        }
         name = shared_thread_name;
     }
 
diff -Nru pmix-3.2.2~rc1/src/server/pmix_server.c pmix-4.0.0/src/server/pmix_server.c
--- pmix-3.2.2~rc1/src/server/pmix_server.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/server/pmix_server.c	2021-01-02 08:56:17.000000000 +0000
@@ -49,6 +49,7 @@
 #include <sys/stat.h>
 
 
+#include "src/common/pmix_attributes.h"
 #include "src/util/argv.h"
 #include "src/util/error.h"
 #include "src/util/name_fns.h"
@@ -62,11 +63,13 @@
 #include "src/runtime/pmix_rte.h"
 #include "src/mca/bfrops/base/base.h"
 #include "src/mca/gds/base/base.h"
+#include "src/mca/ploc/base/base.h"
+#include "src/mca/pmdl/base/base.h"
 #include "src/mca/pnet/base/base.h"
 #include "src/mca/preg/preg.h"
 #include "src/mca/psensor/base/base.h"
 #include "src/mca/ptl/base/base.h"
-#include "src/hwloc/hwloc-internal.h"
+#include "src/tool/pmix_tool_ops.h"
 
 /* the server also needs access to client operations
  * as it can, and often does, behave as a client */
@@ -77,11 +80,112 @@
 pmix_server_globals_t pmix_server_globals = {{{0}}};
 
 // local variables
+static pmix_event_t parentdied;
 static char *security_mode = NULL;
-static char *ptl_mode = NULL;
 static char *bfrops_mode = NULL;
 static char *gds_mode = NULL;
 static pid_t mypid;
+static pmix_proc_t myparent;
+
+static void pdiedfn(int fd, short flags, void *arg)
+{
+    pmix_info_t info[3];
+
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &myparent, PMIX_PROC);
+
+    /* generate a job-terminated event */
+    PMIx_Notify_event(PMIX_ERR_JOB_TERMINATED, &pmix_globals.myid,
+                      PMIX_RANGE_PROC_LOCAL,
+                      info, 3, NULL, NULL);
+}
+
+/* callback to receive job info */
+static void job_data(struct pmix_peer_t *pr,
+                     pmix_ptl_hdr_t *hdr,
+                     pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_status_t rc;
+    char *nspace;
+    int32_t cnt = 1;
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+
+    /* unpack the nspace - should be same as our own */
+    PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver,
+                       buf, &nspace, &cnt, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        cb->status = PMIX_ERROR;
+        PMIX_POST_OBJECT(cb);
+        PMIX_WAKEUP_THREAD(&cb->lock);
+        return;
+    }
+
+    /* decode it */
+    PMIX_GDS_STORE_JOB_INFO(cb->status,
+                            pmix_client_globals.myserver,
+                            nspace, buf);
+    cb->status = PMIX_SUCCESS;
+    PMIX_POST_OBJECT(cb);
+    PMIX_WAKEUP_THREAD(&cb->lock);
+}
+
+/* event handler registration callback */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    pmix_lock_t *lock = (pmix_lock_t*)cbdata;
+
+    lock->status = status;
+    PMIX_WAKEUP_THREAD(lock);
+}
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    pmix_lock_t *lock=NULL;
+    char *name = NULL;
+    size_t n;
+
+    pmix_output_verbose(2, pmix_client_globals.base_output,
+                        "[%s:%d] DEBUGGER RELEASE RECVD",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank);
+    if (NULL != info) {
+        lock = NULL;
+        for (n=0; n < ninfo; n++) {
+            if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
+                lock = (pmix_lock_t*)info[n].value.data.ptr;
+            } else if (0 == strncmp(info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
+                name = info[n].value.data.string;
+            }
+        }
+        /* if the object wasn't returned, then that is an error */
+        if (NULL == lock) {
+            pmix_output_verbose(2, pmix_client_globals.base_output,
+                                "event handler %s failed to return object",
+                                (NULL == name) ? "NULL" : name);
+            /* let the event handler progress */
+            if (NULL != cbfunc) {
+                cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+            }
+            return;
+        }
+    }
+    if (NULL != lock) {
+        PMIX_WAKEUP_THREAD(lock);
+    }
+
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
 
 // local functions for connection support
 pmix_status_t pmix_server_initialize(void)
@@ -96,6 +200,7 @@
     PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t);
     PMIX_CONSTRUCT(&pmix_server_globals.groups, pmix_list_t);
     PMIX_CONSTRUCT(&pmix_server_globals.iof, pmix_list_t);
+    PMIX_CONSTRUCT(&pmix_server_globals.psets, pmix_list_t);
 
     pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server init called");
@@ -154,9 +259,6 @@
     /* get our available security modules */
     security_mode = pmix_psec_base_get_available_modules();
 
-    /* get our available ptl modules */
-    ptl_mode = pmix_ptl_base_get_available_modules();
-
     /* get our available bfrop modules */
     bfrops_mode = pmix_bfrops_base_get_available_modules();
 
@@ -173,28 +275,48 @@
 {
     pmix_ptl_posted_recv_t *req;
     pmix_status_t rc;
-    size_t n, m;
-    pmix_kval_t *kv;
-    bool protect, nspace_given = false, rank_given = false;
-    pmix_info_t ginfo;
-    char *protected[] = {
-        PMIX_USERID,
-        PMIX_GRPID,
-        PMIX_SOCKET_MODE,
-        PMIX_SERVER_TOOL_SUPPORT,
-        PMIX_SERVER_SYSTEM_SUPPORT,
-        PMIX_SERVER_GATEWAY,
-        NULL
-    };
-    char *evar;
+    size_t n;
+    bool nspace_given = false, rank_given = false;
+    bool share_topo = false;
+    pmix_info_t ginfo, *iptr, evinfo[2];
+    char *evar, *nspace = NULL;
+    pmix_rank_t rank = PMIX_RANK_INVALID;
     pmix_rank_info_t *rinfo;
     pmix_proc_type_t ptype = PMIX_PROC_TYPE_STATIC_INIT;
+    pmix_topology_t *topo = NULL;
+    pmix_value_t value, *val;
+    pmix_proc_t myproc, wildcard;
+    pmix_buffer_t *bfr;
+    pmix_cmd_t cmd;
+    pmix_cb_t cb;
+    pmix_lock_t reglock, releaselock;
+    pmix_status_t code;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
     pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server init called");
 
+
+    /* backward compatibility fix - remove any directive to use
+     * the old usock component so we avoid a warning message */
+    if (NULL != (evar = getenv("PMIX_MCA_ptl"))) {
+        if (0 == strcmp(evar, "usock")) {
+            /* we cannot support a usock-only environment */
+            PMIX_RELEASE_THREAD(&pmix_global_lock);
+            fprintf(stderr, "-------------------------------------------------------------------\n");
+            fprintf(stderr, "PMIx no longer supports the \"usock\" transport for client-server\n");
+            fprintf(stderr, "communication. A directive was detected that only allows that mode.\n");
+            fprintf(stderr, "We cannot continue - please remove that constraint and try again.\n");
+            fprintf(stderr, "-------------------------------------------------------------------\n");
+            return PMIX_ERR_INIT;
+        }
+        /* anything else should just be cleared */
+        pmix_unsetenv("PMIX_MCA_ptl", &environ);
+    }
+    /* init the parent procid to something innocuous */
+    PMIX_LOAD_PROCID(&myparent, NULL, PMIX_RANK_UNDEF);
+
     PMIX_SET_PROC_TYPE(&ptype, PMIX_PROC_SERVER);
     /* setup the function pointers */
     if (NULL == module) {
@@ -209,10 +331,37 @@
                 if (PMIX_INFO_TRUE(&info[n])) {
                     PMIX_SET_PROC_TYPE(&ptype, PMIX_PROC_GATEWAY);
                 }
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SCHEDULER)) {
+                if (PMIX_INFO_TRUE(&info[n])) {
+                    PMIX_SET_PROC_TYPE(&ptype, PMIX_PROC_SCHEDULER);
+                }
             } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_TMPDIR)) {
                 pmix_server_globals.tmpdir = strdup(info[n].value.data.string);
             } else if (PMIX_CHECK_KEY(&info[n], PMIX_SYSTEM_TMPDIR)) {
                 pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_NSPACE)) {
+                nspace = info[n].value.data.string;
+                nspace_given = true;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_RANK)) {
+                rank = info[n].value.data.rank;
+                rank_given = true;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TOPOLOGY2)) {
+                if (NULL != topo && pmix_globals.external_topology) {
+                    /* must have come from PMIX_TOPOLOGY entry */
+                    free(pmix_globals.topology.source);
+                }
+                topo = (pmix_topology_t*)info[n].value.data.ptr;
+                pmix_globals.topology.source = strdup(topo->source);
+                pmix_globals.topology.topology = topo->topology;
+                pmix_globals.external_topology = true;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_TOPOLOGY)) {
+                if (NULL == topo) {  // prefer PMIX_TOPOLOGY2
+                    pmix_globals.topology.source = strdup("hwloc");
+                    pmix_globals.topology.topology = info[n].value.data.ptr;
+                    pmix_globals.external_topology = true;
+                }
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SHARE_TOPOLOGY)) {
+                share_topo = true;
             }
         }
     }
@@ -239,6 +388,16 @@
         return rc;
     }
 
+    /* if we were given a keepalive pipe, register an
+     * event to capture the event */
+    if (NULL != (evar = getenv("PMIX_KEEPALIVE_PIPE"))) {
+        rc = strtol(evar, NULL, 10);
+        pmix_event_set(pmix_globals.evbase, &parentdied, rc, PMIX_EV_READ, pdiedfn, NULL);
+        pmix_event_add(&parentdied, NULL);
+        pmix_unsetenv("PMIX_KEEPALIVE_PIPE", &environ);
+        pmix_fd_set_cloexec(rc);  // don't let children inherit this
+    }
+
     /* assign our internal bfrops module */
     pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL);
     if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
@@ -257,14 +416,6 @@
         return rc;
     }
 
-    /* assign our internal ptl module */
-    pmix_globals.mypeer->nptr->compat.ptl = pmix_ptl_base_assign_module();
-    if (NULL == pmix_globals.mypeer->nptr->compat.ptl) {
-        PMIX_ERROR_LOG(rc);
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return rc;
-    }
-
     /* assign our internal gds module */
     PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, "hash", PMIX_STRING);
     pmix_globals.mypeer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
@@ -288,57 +439,20 @@
         return rc;
     }
 
-    /* check the info keys for info we
-     * need to provide to every client and
-     * directives aimed at us */
-    if (NULL != info) {
-        for (n=0; n < ninfo; n++) {
-            if (0 == strncmp(info[n].key, PMIX_SERVER_NSPACE, PMIX_MAX_KEYLEN)) {
-                PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, info[n].value.data.string);
-                nspace_given = true;
-            } else if (0 == strncmp(info[n].key, PMIX_SERVER_RANK, PMIX_MAX_KEYLEN)) {
-                pmix_globals.myid.rank = info[n].value.data.rank;
-                rank_given = true;
-            } else {
-                /* check the list of protected keys */
-                protect = false;
-                for (m=0; NULL != protected[m]; m++) {
-                    if (0 == strcmp(info[n].key, protected[m])) {
-                        protect = true;
-                        break;
-                    }
-                }
-                if (protect) {
-                    continue;
-                }
-                /* store and pass along to every client */
-                kv = PMIX_NEW(pmix_kval_t);
-                kv->key = strdup(info[n].key);
-                PMIX_VALUE_CREATE(kv->value, 1);
-                PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer,
-                                       kv->value, &info[n].value);
-                if (PMIX_SUCCESS != rc) {
-                    PMIX_RELEASE(kv);
-                    PMIX_ERROR_LOG(rc);
-                    PMIX_RELEASE_THREAD(&pmix_global_lock);
-                    return rc;
-                }
-                pmix_list_append(&pmix_server_globals.gdata, &kv->super);
-            }
-        }
-    }
-
-    if (!nspace_given) {
+    if (nspace_given) {
+        PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, nspace);
+    } else {
         /* look for our namespace, if one was given */
         if (NULL == (evar = getenv("PMIX_SERVER_NAMESPACE"))) {
             /* use a fake namespace */
             PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, "pmix-server");
         } else {
-            pmix_output(0, "NSPACE FROM ENV %s", evar);
             PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, evar);
         }
     }
-    if (!rank_given) {
+    if (rank_given) {
+        pmix_globals.myid.rank = rank;
+    } else {
         /* look for our rank, if one was given */
         mypid = getpid();
         if (NULL == (evar = getenv("PMIX_SERVER_RANK"))) {
@@ -370,28 +484,32 @@
     PMIX_RETAIN(pmix_globals.mypeer->info);
     pmix_client_globals.myserver->info = pmix_globals.mypeer->info;
 
-    /* open the pnet framework and select the active modules for this environment */
-    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) {
+    /* open the pmdl framework and select the active modules for this environment */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pmdl_base_framework, 0))) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
-    if (PMIX_SUCCESS != (rc = pmix_pnet_base_select())) {
+    if (PMIX_SUCCESS != (rc = pmix_pmdl_base_select())) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
 
-    /* if requested, setup the topology */
-    if (PMIX_SUCCESS != (rc = pmix_hwloc_get_topology(info, ninfo))) {
+    /* open the psensor framework */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_psensor_base_framework, 0))) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
+    }
+    if (PMIX_SUCCESS != (rc = pmix_psensor_base_select())) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
 
-    /* open the psensor framework */
-    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_psensor_base_framework, 0))) {
+    /* open the ploc framework */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_ploc_base_framework, 0))) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
-    if (PMIX_SUCCESS != (rc = pmix_psensor_base_select())) {
+    if (PMIX_SUCCESS != (rc = pmix_ploc_base_select())) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
@@ -401,7 +519,7 @@
     req->tag = UINT32_MAX;
     req->cbfunc = pmix_server_message_handler;
     /* add it to the end of the list of recvs */
-    pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super);
+    pmix_list_append(&pmix_ptl_base.posted_recvs, &req->super);
 
     /* if we are a gateway, setup our IOF events */
     if (PMIX_PEER_IS_GATEWAY(pmix_globals.mypeer)) {
@@ -412,17 +530,35 @@
                              2, PMIX_FWD_STDERR_CHANNEL, pmix_iof_write_handler);
     }
 
-#if PMIX_HAVE_HWLOC
+    /* register our attributes */
+    if (PMIX_SUCCESS != (rc = pmix_register_server_attrs())) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
+    }
+
     /* if we don't know our topology, we better get it now as we
      * increasingly rely on it - note that our host will hopefully
      * have passed it to us so we don't duplicate their storage! */
-    if (NULL == pmix_hwloc_topology) {
-        if (PMIX_SUCCESS != (rc = pmix_hwloc_get_topology(info, ninfo))) {
+    if (PMIX_SUCCESS != (rc = pmix_ploc.setup_topology(info, ninfo))) {
+        /* if they told us to share our topology and we cannot do so,
+         * then that is a reportable error */
+        if (share_topo) {
             PMIX_RELEASE_THREAD(&pmix_global_lock);
             return rc;
         }
     }
-#endif
+
+    /* open the pnet framework and select the active modules for this environment
+     * Do this after setting up the topology so the components can check to see
+     * if they have any local assets */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
+    }
+    if (PMIX_SUCCESS != (rc = pmix_pnet_base_select())) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
+    }
 
     /* start listening for connections */
     if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) {
@@ -435,6 +571,93 @@
     ++pmix_globals.init_cntr;
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
+    /* save the topology internally in case our host wants it */
+    PMIX_LOAD_PROCID(&myproc, pmix_globals.myid.nspace, pmix_globals.myid.rank);
+    value.type = PMIX_TOPO;
+    value.data.topo = &pmix_globals.topology;
+    rc = PMIx_Store_internal(&myproc, PMIX_TOPOLOGY2, &value);
+
+    /* see if they gave us a rendezvous URI to which we are to call back */
+    evar = getenv("PMIX_LAUNCHER_RNDZ_URI");
+    if (NULL != evar) {
+        /* attach to the specified server so it can
+         * tell us what we are to do */
+        PMIX_INFO_CREATE(iptr, 3);
+        PMIX_INFO_LOAD(&iptr[0], PMIX_SERVER_URI, evar, PMIX_STRING);
+        rc = 2;  // give us two seconds to connect
+        PMIX_INFO_LOAD(&iptr[1], PMIX_TIMEOUT, &rc, PMIX_INT);
+        PMIX_INFO_LOAD(&iptr[2], PMIX_PRIMARY_SERVER, NULL, PMIX_BOOL);
+        rc = PMIx_tool_attach_to_server(NULL, &myparent, iptr, 3);
+        if (PMIX_SUCCESS != rc) {
+            return PMIX_ERR_UNREACH;
+        }
+
+        /* save our parent ID */
+        value.type = PMIX_PROC;
+        value.data.proc = &myparent;
+        rc = PMIx_Store_internal(&pmix_globals.myid, PMIX_PARENT_ID, &value);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+        /* retrieve any job info it has for us */
+        bfr = PMIX_NEW(pmix_buffer_t);
+        cmd = PMIX_REQ_CMD;
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         bfr, &cmd, 1, PMIX_COMMAND);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(bfr);
+            return rc;
+        }
+        /* send to the server */
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                           bfr, job_data, (void*)&cb);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+        /* wait for the data to return */
+        PMIX_WAIT_THREAD(&cb.lock);
+        rc = cb.status;
+        PMIX_DESTRUCT(&cb);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+        /* restore our original primary server */
+        rc = PMIx_tool_set_server(&myproc, NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+    }
+
+    /* see if we were asked to stop in init */
+    PMIX_LOAD_PROCID(&wildcard, pmix_globals.myid.nspace, PMIX_RANK_WILDCARD);
+    PMIX_INFO_LOAD(&ginfo, PMIX_OPTIONAL, NULL, PMIX_BOOL);
+    rc = PMIx_Get(&wildcard, PMIX_DEBUG_STOP_IN_INIT, &ginfo, 1, &val);
+    if (PMIX_SUCCESS == rc) {
+        /* if the value was found, then we need to wait for debugger attach here */
+        /* register for the debugger release notification */
+        PMIX_CONSTRUCT_LOCK(&reglock);
+        PMIX_CONSTRUCT_LOCK(&releaselock);
+        PMIX_INFO_LOAD(&evinfo[0], PMIX_EVENT_RETURN_OBJECT, &releaselock, PMIX_POINTER);
+        PMIX_INFO_LOAD(&evinfo[1], PMIX_EVENT_HDLR_NAME, "WAIT-FOR-RELEASE", PMIX_STRING);
+        pmix_output_verbose(2, pmix_client_globals.event_output,
+                            "[%s:%d] WAITING IN INIT FOR RELEASE",
+                            pmix_globals.myid.nspace, pmix_globals.myid.rank);
+        code = PMIX_ERR_DEBUGGER_RELEASE;
+        PMIx_Register_event_handler(&code, 1, evinfo, 2,
+                                    notification_fn, evhandler_reg_callbk, (void*)&reglock);
+        /* wait for registration to complete */
+        PMIX_WAIT_THREAD(&reglock);
+        PMIX_DESTRUCT_LOCK(&reglock);
+        PMIX_INFO_DESTRUCT(&evinfo[0]);
+        PMIX_INFO_DESTRUCT(&evinfo[1]);
+        /* wait for release to arrive */
+        PMIX_WAIT_THREAD(&releaselock);
+        PMIX_DESTRUCT_LOCK(&releaselock);
+        PMIX_VALUE_RELEASE(val);
+    }
+
     return PMIX_SUCCESS;
 }
 
@@ -460,13 +683,11 @@
     pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server finalize called");
 
-    if (!pmix_globals.external_evbase) {
-        /* stop the progress thread, but leave the event base
-         * still constructed. This will allow us to safely
-         * tear down the infrastructure, including removal
-         * of any events objects may be holding */
-        (void)pmix_progress_thread_pause(NULL);
-    }
+    /* stop the progress thread, but leave the event base
+     * still constructed. This will allow us to safely
+     * tear down the infrastructure, including removal
+     * of any events objects may be holding */
+    (void)pmix_progress_thread_pause(NULL);
 
     /* flush anything that is still trying to be written out */
     pmix_iof_static_dump_output(&pmix_client_globals.iof_stdout);
@@ -497,17 +718,12 @@
     }
     PMIX_LIST_DESTRUCT(&pmix_server_globals.groups);
     PMIX_LIST_DESTRUCT(&pmix_server_globals.iof);
-
-    pmix_hwloc_cleanup();
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.psets);
 
     if (NULL != security_mode) {
         free(security_mode);
     }
 
-    if (NULL != ptl_mode) {
-        free(ptl_mode);
-    }
-
     if (NULL != bfrops_mode) {
         free(bfrops_mode);
     }
@@ -612,6 +828,12 @@
         goto release;
     }
 
+    /* give the programming models a chance to add anything they need */
+    rc = pmix_pmdl.register_nspace(nptr);
+    if (PMIX_SUCCESS != rc) {
+        goto release;
+    }
+
     /* check any pending trackers to see if they are
      * waiting for us. There is a slight race condition whereby
      * the host server could have spawned the local client and
@@ -754,7 +976,7 @@
     PMIX_LIST_FOREACH_SAFE(reginfo, regnext, &pmix_server_globals.events, pmix_regevents_info_t) {
         PMIX_LIST_FOREACH_SAFE(prev, pnext, &reginfo->peers, pmix_peer_events_info_t) {
             if ((NULL != peer && prev->peer == peer) ||
-                (NULL != proc && PMIX_CHECK_PROCID(proc, &prev->peer->info->pname))) {
+                (NULL != proc && NULL != prev->peer->info && PMIX_CHECK_PROCID(proc, &prev->peer->info->pname))) {
                 pmix_list_remove_item(&reginfo->peers, &prev->super);
                 PMIX_RELEASE(prev);
                 if (0 == pmix_list_get_size(&reginfo->peers)) {
@@ -772,7 +994,16 @@
         if (NULL == (req = (pmix_iof_req_t*)pmix_pointer_array_get_item(&pmix_globals.iof_requests, i))) {
             continue;
         }
-        if ((NULL != peer && PMIX_CHECK_PROCID(&req->requestor->info->pname, &peer->info->pname)) ||
+        /* protect against errors */
+        if (NULL == req->requestor || NULL == req->requestor->info) {
+            pmix_pointer_array_set_item(&pmix_globals.iof_requests, i, NULL);
+            PMIX_RELEASE(req);
+            continue;
+        }
+        if (NULL != peer && NULL == peer->info) {
+            continue;
+        }
+        if ((NULL != peer && NULL != peer->info && PMIX_CHECK_PROCID(&req->requestor->info->pname, &peer->info->pname)) ||
             (NULL != proc && PMIX_CHECK_PROCID(&req->requestor->info->pname, proc))) {
             pmix_pointer_array_set_item(&pmix_globals.iof_requests, i, NULL);
             PMIX_RELEASE(req);
@@ -781,7 +1012,7 @@
 
     /* see if this proc is involved in any direct modex requests */
     PMIX_LIST_FOREACH_SAFE(dlcd, dnxt, &pmix_server_globals.local_reqs, pmix_dmdx_local_t) {
-        if ((NULL != peer && PMIX_CHECK_PROCID(&peer->info->pname, &dlcd->proc)) ||
+        if ((NULL != peer && NULL != peer->info && PMIX_CHECK_PROCID(&peer->info->pname, &dlcd->proc)) ||
             (NULL != proc && PMIX_CHECK_PROCID(proc, &dlcd->proc))) {
                 /* cleanup this request */
             pmix_list_remove_item(&pmix_server_globals.local_reqs, &dlcd->super);
@@ -798,7 +1029,7 @@
         if (NULL != ncd && NULL != ncd->targets && 0 < ncd->ntargets) {
             tgt = NULL;
             for (n=0; n < ncd->ntargets; n++) {
-                if ((NULL != peer && PMIX_CHECK_PROCID(&peer->info->pname, &ncd->targets[n])) ||
+                if ((NULL != peer && NULL != peer->info && PMIX_CHECK_PROCID(&peer->info->pname, &ncd->targets[n])) ||
                     (NULL != proc && PMIX_CHECK_PROCID(proc, &ncd->targets[n]))) {
                     tgt = &ncd->targets[n];
                     break;
@@ -851,6 +1082,9 @@
     /* release any job-level network resources */
     pmix_pnet.deregister_nspace(cd->proc.nspace);
 
+    /* release any programming model info */
+    pmix_pmdl.deregister_nspace(cd->proc.nspace);
+
     /* let our local storage clean up */
     PMIX_GDS_DEL_NSPACE(rc, cd->proc.nspace);
 
@@ -918,6 +1152,143 @@
     PMIX_THREADSHIFT(cd, _deregister_nspace);
 }
 
+static void _register_resources(int sd, short args, void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    pmix_kval_t *kv;
+    size_t n;
+    pmix_status_t rc = PMIX_SUCCESS;
+
+    /* add any provided data to our global cache for all nspaces */
+    for (n=0; n < cd->ninfo; n++) {
+        kv = PMIX_NEW(pmix_kval_t);
+        kv->key = strdup(cd->info[n].key);
+        kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t));
+        PMIX_VALUE_XFER(rc, kv->value, &cd->info[n].value);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_RELEASE(kv);
+            break;
+        }
+        pmix_list_append(&pmix_server_globals.gdata, &kv->super);
+    }
+
+    cd->opcbfunc(rc, cd->cbdata);
+    PMIX_RELEASE(cd);
+}
+
+pmix_status_t PMIx_server_register_resources(pmix_info_t info[], size_t ninfo,
+                                             pmix_op_cbfunc_t cbfunc,
+                                             void *cbdata)
+{
+    pmix_setup_caddy_t *cd;
+    pmix_lock_t mylock;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_server_globals.base_output,
+                        "pmix:server register resources");
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    cd = PMIX_NEW(pmix_setup_caddy_t);
+    cd->info = info;
+    cd->ninfo = ninfo;
+    cd->opcbfunc = cbfunc;
+    cd->cbdata = cbdata;
+
+    /* if the provided callback is NULL, then substitute
+     * our own internal cbfunc and block here */
+    if (NULL == cbfunc) {
+        PMIX_CONSTRUCT_LOCK(&mylock);
+        cd->opcbfunc = opcbfunc;
+        cd->cbdata = &mylock;
+        PMIX_THREADSHIFT(cd, _register_resources);
+        PMIX_WAIT_THREAD(&mylock);
+        rc = mylock.status;
+        if (PMIX_SUCCESS == rc) {
+            rc = PMIX_OPERATION_SUCCEEDED;
+        }
+        PMIX_DESTRUCT_LOCK(&mylock);
+        return rc;
+    }
+
+    /* we have to push this into our event library to avoid
+     * potential threading issues */
+    PMIX_THREADSHIFT(cd, _register_resources);
+    return PMIX_SUCCESS;
+}
+
+static void _deregister_resources(int sd, short args, void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    pmix_kval_t *kv;
+    size_t n;
+
+    /* find any matches in our global cache and remove them */
+    for (n=0; n < cd->ninfo; n++) {
+        PMIX_LIST_FOREACH(kv, &pmix_server_globals.gdata, pmix_kval_t) {
+            if (PMIX_CHECK_KEY(kv, cd->info[n].key)) {
+                pmix_list_remove_item(&pmix_server_globals.gdata, &kv->super);
+                PMIX_RELEASE(kv);
+                break;
+            }
+        }
+    }
+
+    cd->opcbfunc(PMIX_SUCCESS, cd->cbdata);
+    PMIX_RELEASE(cd);
+}
+
+pmix_status_t PMIx_server_deregister_resources(pmix_info_t info[], size_t ninfo,
+                                               pmix_op_cbfunc_t cbfunc,
+                                               void *cbdata)
+{
+    pmix_setup_caddy_t *cd;
+    pmix_lock_t mylock;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_server_globals.base_output,
+                        "pmix:server deregister resources");
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    cd = PMIX_NEW(pmix_setup_caddy_t);
+    cd->info = info;
+    cd->ninfo = ninfo;
+    cd->opcbfunc = cbfunc;
+    cd->cbdata = cbdata;
+
+    /* if the provided callback is NULL, then substitute
+     * our own internal cbfunc and block here */
+    if (NULL == cbfunc) {
+        PMIX_CONSTRUCT_LOCK(&mylock);
+        cd->opcbfunc = opcbfunc;
+        cd->cbdata = &mylock;
+        PMIX_THREADSHIFT(cd, _deregister_resources);
+        PMIX_WAIT_THREAD(&mylock);
+        rc = mylock.status;
+        if (PMIX_SUCCESS == rc) {
+            rc = PMIX_OPERATION_SUCCEEDED;
+        }
+        PMIX_DESTRUCT_LOCK(&mylock);
+        return rc;
+    }
+
+    /* we have to push this into our event library to avoid
+     * potential threading issues */
+    PMIX_THREADSHIFT(cd, _deregister_resources);
+    return PMIX_SUCCESS;
+}
+
 void pmix_server_execute_collective(int sd, short args, void *cbdata)
 {
     pmix_trkr_caddy_t *tcd = (pmix_trkr_caddy_t*)cbdata;
@@ -1408,28 +1779,26 @@
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
     pmix_output_verbose(2, pmix_server_globals.base_output,
-                        "pmix:server setup_fork for nspace %s rank %d",
+                        "pmix:server setup_fork for nspace %s rank %u",
                         proc->nspace, proc->rank);
 
     /* pass the nspace */
     pmix_setenv("PMIX_NAMESPACE", proc->nspace, true, env);
     /* pass the rank */
-    (void)snprintf(rankstr, 127, "%d", proc->rank);
+    (void)snprintf(rankstr, 127, "%u", proc->rank);
     pmix_setenv("PMIX_RANK", rankstr, true, env);
     /* pass our rendezvous info */
-    PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
-        if (NULL != lt->uri && NULL != lt->varname) {
-            varnames = pmix_argv_split(lt->varname, ':');
-            for (n=0; NULL != varnames[n]; n++) {
-                pmix_setenv(varnames[n], lt->uri, true, env);
-            }
-            pmix_argv_free(varnames);
+    lt = &pmix_ptl_base.listener;
+    if (NULL != lt->uri && NULL != lt->varname) {
+        varnames = pmix_argv_split(lt->varname, ':');
+        for (n=0; NULL != varnames[n]; n++) {
+            pmix_setenv(varnames[n], lt->uri, true, env);
         }
+        pmix_argv_free(varnames);
     }
+
     /* pass our active security modules */
     pmix_setenv("PMIX_SECURITY_MODE", security_mode, true, env);
-    /* pass our available ptl modules */
-    pmix_setenv("PMIX_PTL_MODULE", ptl_mode, true, env);
     /* pass the type of buffer we are using */
     if (PMIX_BFROP_BUFFER_FULLY_DESC == pmix_globals.mypeer->nptr->compat.type) {
         pmix_setenv("PMIX_BFROP_BUFFER_TYPE", "PMIX_BFROP_BUFFER_FULLY_DESC", true, env);
@@ -1457,8 +1826,14 @@
         return rc;
     }
 
-    /* ensure we agree on our hostname - typically only important in
-     * test scenarios where we are faking multiple nodes */
+    /* get any contribution for the specific programming
+     * model/implementation, if known */
+    if (PMIX_SUCCESS != (rc = pmix_pmdl.setup_fork(proc, env))) {
+        PMIX_ERROR_LOG(rc);
+        return rc;
+    }
+
+    /* ensure we agree on our hostname */
     pmix_setenv("PMIX_HOSTNAME", pmix_globals.hostname, true, env);
 
     /* communicate our version */
@@ -1751,6 +2126,13 @@
         goto depart;
     }
 
+    /* pass to the programming model libraries */
+    if (PMIX_SUCCESS != (rc = pmix_pmdl.harvest_envars(cd->nspace,
+                                                       cd->info, cd->ninfo,
+                                                       &ilist))) {
+        goto depart;
+    }
+
     /* setup the return callback */
     fcd = PMIX_NEW(pmix_setup_caddy_t);
     if (NULL == fcd) {
@@ -1849,6 +2231,8 @@
                                               pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_setup_caddy_t *cd;
+    pmix_status_t rc;
+    pmix_lock_t mylock;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
@@ -1869,6 +2253,23 @@
     cd->ninfo = ninfo;
     cd->opcbfunc = cbfunc;
     cd->cbdata = cbdata;
+
+    /* if the provided callback is NULL, then substitute
+     * our own internal cbfunc and block here */
+    if (NULL == cbfunc) {
+        PMIX_CONSTRUCT_LOCK(&mylock);
+        cd->opcbfunc = opcbfunc;
+        cd->cbdata = &mylock;
+        PMIX_THREADSHIFT(cd, _setup_local_support);
+        PMIX_WAIT_THREAD(&mylock);
+        rc = mylock.status;
+        PMIX_DESTRUCT_LOCK(&mylock);
+        if (PMIX_SUCCESS == rc) {
+            rc = PMIX_OPERATION_SUCCEEDED;
+        }
+        return rc;
+    }
+
     PMIX_THREADSHIFT(cd, _setup_local_support);
 
     return PMIX_SUCCESS;
@@ -1951,6 +2352,8 @@
                                       pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_setup_caddy_t *cd;
+    pmix_lock_t mylock;
+    pmix_status_t rc;
 
     /* need to threadshift this request */
     cd = PMIX_NEW(pmix_setup_caddy_t);
@@ -1965,6 +2368,23 @@
     cd->ninfo = ninfo;
     cd->opcbfunc = cbfunc;
     cd->cbdata = cbdata;
+
+    /* if the provided callback is NULL, then substitute
+     * our own internal cbfunc and block here */
+    if (NULL == cbfunc) {
+        PMIX_CONSTRUCT_LOCK(&mylock);
+        cd->opcbfunc = opcbfunc;
+        cd->cbdata = &mylock;
+        PMIX_THREADSHIFT(cd, _iofdeliver);
+        PMIX_WAIT_THREAD(&mylock);
+        rc = mylock.status;
+        PMIX_DESTRUCT_LOCK(&mylock);
+        if (PMIX_SUCCESS == rc) {
+            rc = PMIX_OPERATION_SUCCEEDED;
+        }
+        return rc;
+    }
+
     PMIX_THREADSHIFT(cd, _iofdeliver);
     return PMIX_SUCCESS;
 }
@@ -2135,6 +2555,8 @@
                                             pmix_op_cbfunc_t cbfunc, void *cbdata)
 {
     pmix_shift_caddy_t *cd;
+    pmix_lock_t mylock;
+    pmix_status_t rc;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
@@ -2155,12 +2577,187 @@
     cd->ndirs = ndirs;
     cd->cbfunc.opcbfn = cbfunc;
     cd->cbdata = cbdata;
+
+    /* if the provided callback is NULL, then substitute
+     * our own internal cbfunc and block here */
+    if (NULL == cbfunc) {
+        PMIX_CONSTRUCT_LOCK(&mylock);
+        cd->cbfunc.opcbfn = opcbfunc;
+        cd->cbdata = &mylock;
+        PMIX_THREADSHIFT(cd, dlinv);
+        PMIX_WAIT_THREAD(&mylock);
+        rc = mylock.status;
+        PMIX_DESTRUCT_LOCK(&mylock);
+        if (PMIX_SUCCESS == rc) {
+            rc = PMIX_OPERATION_SUCCEEDED;
+        }
+        return rc;
+    }
+
     PMIX_THREADSHIFT(cd, dlinv);
 
     return PMIX_SUCCESS;
 
 }
 
+pmix_status_t PMIx_server_generate_locality_string(const pmix_cpuset_t *cpuset,
+                                                   char **locality)
+{
+    pmix_status_t rc;
+
+    /* just pass this down */
+    rc = pmix_ploc.generate_locality_string(cpuset, locality);
+    return rc;
+}
+
+pmix_status_t PMIx_server_generate_cpuset_string(const pmix_cpuset_t *cpuset,
+                                                 char **cpuset_string)
+{
+    pmix_status_t rc;
+
+    /* just pass this down */
+    rc = pmix_ploc.generate_cpuset_string(cpuset, cpuset_string);
+    return rc;
+}
+
+typedef struct {
+    pmix_info_t *info;
+    size_t ninfo;
+} mydata_t;
+
+static void release_info(pmix_status_t status, void *cbdata)
+{
+    mydata_t *cd = (mydata_t*)cbdata;
+    PMIX_INFO_FREE(cd->info, cd->ninfo);
+    free(cd);
+}
+
+static void psetdef(int sd, short args, void *cbdata)
+{
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    /* the PMIx server needs to cache the process sets so it can
+     * respond to queries for names and memberships */
+    mydata_t *mydat;
+    pmix_data_array_t *darray;
+    pmix_proc_t *ptr;
+    pmix_pset_t *ps;
+
+    mydat = (mydata_t*)malloc(sizeof(mydata_t));
+    mydat->ninfo = 3;
+    PMIX_INFO_CREATE(mydat->info, mydat->ninfo);
+    PMIX_INFO_LOAD(&mydat->info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&mydat->info[1], PMIX_PSET_NAME, cd->nspace, PMIX_STRING);
+    PMIX_DATA_ARRAY_CREATE(darray, cd->nprocs, PMIX_PROC);
+    PMIX_LOAD_KEY(mydat->info[2].key, PMIX_PSET_MEMBERS);
+    mydat->info[2].value.type = PMIX_DATA_ARRAY;
+    mydat->info[2].value.data.darray = darray;
+    ptr = (pmix_proc_t*)darray->array;
+    memcpy(ptr, cd->procs, cd->nprocs * sizeof(pmix_proc_t));
+
+    PMIx_Notify_event(PMIX_PROCESS_SET_DEFINE,
+                      &pmix_globals.myid, PMIX_RANGE_LOCAL,
+                      mydat->info, mydat->ninfo, release_info, (void*)mydat);
+
+    /* now record the process set */
+    ps = PMIX_NEW(pmix_pset_t);
+    ps->name = strdup(cd->nspace);
+    ps->members = (pmix_proc_t*)malloc(cd->nprocs * sizeof(pmix_proc_t));
+    memcpy(ps->members, cd->procs, cd->nprocs * sizeof(pmix_proc_t));
+    ps->nmembers = cd->nprocs;
+    pmix_list_append(&pmix_server_globals.psets, &ps->super);
+
+    PMIX_WAKEUP_THREAD(&cd->lock);
+}
+
+pmix_status_t PMIx_server_define_process_set(const pmix_proc_t *members,
+                                             size_t nmembers, char *pset_name)
+{
+    pmix_setup_caddy_t cd;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* need to threadshift this request */
+    PMIX_CONSTRUCT(&cd, pmix_setup_caddy_t);
+    cd.nspace = pset_name;
+    cd.procs = (pmix_proc_t*)members;
+    cd.nprocs = nmembers;
+    cd.opcbfunc = opcbfunc;
+    cd.cbdata = &cd.lock;
+    PMIX_THREADSHIFT(&cd, psetdef);
+    PMIX_WAIT_THREAD(&cd.lock);
+    rc = cd.lock.status;
+    /* protect the input */
+    cd.procs = NULL;
+    cd.nprocs = 0;
+    PMIX_DESTRUCT(&cd);
+    if (PMIX_SUCCESS == rc) {
+        rc = PMIX_OPERATION_SUCCEEDED;
+    }
+    return rc;
+}
+
+static void psetdel(int sd, short args, void *cbdata)
+{
+    /* the PMIx server needs to delete the process set from its list */
+    pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
+    mydata_t *mydat;
+    pmix_pset_t *ps;
+
+    mydat = (mydata_t*)malloc(sizeof(mydata_t));
+    mydat->ninfo = 2;
+    PMIX_INFO_CREATE(mydat->info, mydat->ninfo);
+    PMIX_INFO_LOAD(&mydat->info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&mydat->info[1], PMIX_PSET_NAME, cd->nspace, PMIX_STRING);
+
+    PMIx_Notify_event(PMIX_PROCESS_SET_DELETE,
+                      &pmix_globals.myid, PMIX_RANGE_LOCAL,
+                      mydat->info, mydat->ninfo, release_info, (void*)mydat);
+
+    /* now find this process set */
+    PMIX_LIST_FOREACH(ps, &pmix_server_globals.psets, pmix_pset_t) {
+        if (0 == strcmp(cd->nspace, ps->name)) {
+            pmix_list_remove_item(&pmix_server_globals.psets, &ps->super);
+            PMIX_RELEASE(ps);
+            break;
+        }
+    }
+    PMIX_WAKEUP_THREAD(&cd->lock);
+}
+
+pmix_status_t PMIx_server_delete_process_set(char *pset_name)
+{
+    pmix_setup_caddy_t cd;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* need to threadshift this request */
+    PMIX_CONSTRUCT(&cd, pmix_setup_caddy_t);
+    cd.nspace = pset_name;
+    cd.opcbfunc = opcbfunc;
+    cd.cbdata = &cd.lock;
+    PMIX_THREADSHIFT(&cd, psetdel);
+    PMIX_WAIT_THREAD(&cd.lock);
+    rc = cd.lock.status;
+    PMIX_DESTRUCT(&cd);
+    if (PMIX_SUCCESS == rc) {
+        rc = PMIX_OPERATION_SUCCEEDED;
+    }
+    return rc;
+}
+
+
 /****    THE FOLLOWING CALLBACK FUNCTIONS ARE USED BY THE HOST SERVER    ****
  ****    THEY THEREFORE CAN OCCUR IN EITHER THE HOST SERVER'S THREAD     ****
  ****    CONTEXT, OR IN OUR OWN THREAD CONTEXT IF THE CALLBACK OCCURS    ****
@@ -3374,6 +3971,65 @@
     PMIX_THREADSHIFT(cd, _iofreg);
 }
 
+static void fabric_cbfunc(pmix_status_t status,
+                          pmix_info_t *info, size_t ninfo,
+                          void *cbdata,
+                          pmix_release_cbfunc_t release_fn,
+                          void *release_cbdata)
+{
+    pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata;
+    pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata;
+    pmix_buffer_t *reply;
+    pmix_status_t rc;
+
+    pmix_output_verbose(2, pmix_server_globals.base_output,
+                        "pmix:fabric callback with status %d", status);
+
+    reply = PMIX_NEW(pmix_buffer_t);
+    if (NULL == reply) {
+        PMIX_ERROR_LOG(PMIX_ERR_NOMEM);
+        PMIX_RELEASE(cd);
+        return;
+    }
+    PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    /* pack the returned data */
+    PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto complete;
+    }
+    if (0 < ninfo) {
+        PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+        }
+    }
+
+  complete:
+    // send reply
+    PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply);
+        if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(reply);
+    }
+
+    // cleanup
+    if (NULL != qcd->queries) {
+        PMIX_QUERY_FREE(qcd->queries, qcd->nqueries);
+    }
+    if (NULL != qcd->info) {
+        PMIX_INFO_FREE(qcd->info, qcd->ninfo);
+    }
+    PMIX_RELEASE(qcd);
+    PMIX_RELEASE(cd);
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+}
+
 /* the switchyard is the primary message handling function. It's purpose
  * is to take incoming commands (packed into a buffer), unpack them,
  * and then call the corresponding host server's function to execute
@@ -3409,8 +4065,24 @@
         return rc;
     }
     pmix_output_verbose(2, pmix_server_globals.base_output,
-                        "recvd pmix cmd %s from %s:%u",
-                        pmix_command_string(cmd), peer->info->pname.nspace, peer->info->pname.rank);
+                        "recvd pmix cmd %s from %s:%u bytes %u",
+                        pmix_command_string(cmd),
+                        peer->info->pname.nspace, peer->info->pname.rank,
+                        (unsigned int)buf->bytes_used);
+
+    /* if I am a tool, all I can do is relay this to my primary server
+     * if I am connected - if not connected, then I must return an error */
+    if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
+        rc = pmix_tool_relay_op(cmd, peer, buf, tag);
+        if (PMIX_ERR_NOT_SUPPORTED != rc) {
+            return rc;
+        }
+        /* if the tool relay doesn't support it, let it
+         * be processed by the logic tree */
+    }
+
+    /* if I am a server, then redirect the cmd to the appropriate
+     * function for processing */
 
     if (PMIX_REQ_CMD == cmd) {
         reply = PMIX_NEW(pmix_buffer_t);
@@ -3680,6 +4352,38 @@
             PMIX_RELEASE(cd);
         }
         return rc;
+    }
+
+    if (PMIX_GROUP_CONSTRUCT_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        if (PMIX_SUCCESS != (rc = pmix_server_grpconstruct(cd, buf))) {
+            PMIX_RELEASE(cd);
+        }
+        return rc;
+    }
+
+    if (PMIX_GROUP_DESTRUCT_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        if (PMIX_SUCCESS != (rc = pmix_server_grpdestruct(cd, buf))) {
+            PMIX_RELEASE(cd);
+        }
+        return rc;
+    }
+
+    if (PMIX_FABRIC_REGISTER_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        if (PMIX_SUCCESS != (rc = pmix_server_fabric_register(cd, buf, fabric_cbfunc))) {
+            PMIX_RELEASE(cd);
+        }
+        return rc;
+    }
+
+    if (PMIX_FABRIC_UPDATE_CMD == cmd) {
+        PMIX_GDS_CADDY(cd, peer, tag);
+        if (PMIX_SUCCESS != (rc = pmix_server_fabric_update(cd, buf, fabric_cbfunc))) {
+            PMIX_RELEASE(cd);
+        }
+        return rc;
     }
 
     return PMIX_ERR_NOT_SUPPORTED;
diff -Nru pmix-3.2.2~rc1/src/server/pmix_server_get.c pmix-4.0.0/src/server/pmix_server_get.c
--- pmix-3.2.2~rc1/src/server/pmix_server_get.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/server/pmix_server_get.c	2021-01-02 08:56:17.000000000 +0000
@@ -415,9 +415,15 @@
         cb.ninfo = cd->ninfo;
         cb.key = key;
         PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
+        /* if the requested key was found, but in a different scope,
+         * then we report this back as there is no point in waiting */
+        if (PMIX_ERR_EXISTS_OUTSIDE_SCOPE == rc) {
+            PMIX_DESTRUCT(&cb);
+            return PMIX_ERR_NOT_FOUND;
+        }
         /* A local client may send a get request concurrently with
          * a commit request from another client, but the server may
-         * have processed the commit request earlyer than the get
+         * have processed the commit request earlier than the get
          * request. In this case, we create a local tracker for
          * possibly existing keys that are added with the completed
          * commit request. Thus, the get request will be pended in
@@ -435,6 +441,11 @@
             }
         }
         PMIX_DESTRUCT(&cb);  // does not release info or key
+        /* if the requested key was found, but in a different scope,
+         * then we report this back as there is no point in waiting */
+        if (PMIX_ERR_EXISTS_OUTSIDE_SCOPE == rc) {
+            return PMIX_ERR_NOT_FOUND;
+        }
         if (PMIX_SUCCESS != rc) {
             /* if the target proc is local, then we just need to wait */
             if (local) {
diff -Nru pmix-3.2.2~rc1/src/server/pmix_server_ops.c pmix-4.0.0/src/server/pmix_server_ops.c
--- pmix-3.2.2~rc1/src/server/pmix_server_ops.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/server/pmix_server_ops.c	2021-01-02 08:56:17.000000000 +0000
@@ -52,9 +52,11 @@
 
 #include "src/class/pmix_hotel.h"
 #include "src/class/pmix_list.h"
+#include "src/common/pmix_attributes.h"
 #include "src/mca/bfrops/bfrops.h"
 #include "src/mca/plog/plog.h"
 #include "src/mca/pnet/pnet.h"
+#include "src/mca/prm/prm.h"
 #include "src/mca/psensor/psensor.h"
 #include "src/mca/ptl/base/base.h"
 #include "src/util/argv.h"
@@ -1039,9 +1041,7 @@
     if (0 < tv.tv_sec) {
         PMIX_RETAIN(trk);
         cd->trk = trk;
-        pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev,
-                               fence_timeout, cd);
-        pmix_event_evtimer_add(&cd->ev, &tv);
+        PMIX_THREADSHIFT_DELAY(cd, fence_timeout, tv.tv_sec);
         cd->event_active = true;
     }
 
@@ -2010,9 +2010,7 @@
     if (PMIX_SUCCESS == rc && 0 < tv.tv_sec) {
         PMIX_RETAIN(trk);
         cd->trk = trk;
-        pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev,
-                               connect_timeout, cd);
-        pmix_event_evtimer_add(&cd->ev, &tv);
+        PMIX_THREADSHIFT_DELAY(cd, connect_timeout, tv.tv_sec);
         cd->event_active = true;
     }
 
@@ -2548,21 +2546,9 @@
         goto complete;
     }
 
-    if (NULL == pmix_host_server.notify_event) {
-        rc = PMIX_ERR_NOT_SUPPORTED;
-        goto complete;
-    }
-
-    /* since our host is going to send this everywhere, it may well
-     * come back to us. We already processed it, so mark it here
-     * to ensure we don't do it again. We previously inserted the
-     * PMIX_SERVER_INTERNAL_NOTIFY key at the very end of the
-     * info array - just overwrite that position */
-    PMIX_INFO_LOAD(&cd->info[cd->ninfo-1], PMIX_EVENT_PROXY, &pmix_globals.myid, PMIX_PROC);
-
     /* pass it to our host RM for distribution */
-    rc = pmix_host_server.notify_event(cd->status, &cd->source, cd->range,
-                                       cd->info, cd->ninfo, local_cbfunc, cd);
+    rc = pmix_prm.notify(cd->status, &cd->source, cd->range,
+                         cd->info, cd->ninfo, local_cbfunc, cd);
     if (PMIX_SUCCESS == rc) {
         /* let the callback function respond for us */
         return;
@@ -2733,6 +2719,15 @@
     PMIX_CONSTRUCT(&results, pmix_list_t);
 
     for (n=0; n < cd->nqueries; n++) {
+        /* if they are asking for information on support, then go get it */
+        if (0 == strcmp(cd->queries[n].keys[0], PMIX_QUERY_ATTRIBUTE_SUPPORT)) {
+            /* we are already in an event, but shift it as the handler expects to */
+            cd->cbfunc = cbfunc;
+            PMIX_RETAIN(cd); // protect against early release
+            PMIX_THREADSHIFT(cd, pmix_attrs_query_support);
+            PMIX_LIST_DESTRUCT(&results);
+            return PMIX_SUCCESS;
+        }
         for (p=0; p < cd->queries[n].nqual; p++) {
             if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_QUERY_REFRESH_CACHE)) {
                 if (PMIX_INFO_TRUE(&cd->queries[n].qualifiers[p])) {
@@ -3838,6 +3833,880 @@
     return rc;
 }
 
+static void grp_timeout(int sd, short args, void *cbdata)
+{
+    pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata;
+    pmix_buffer_t *reply;
+    pmix_status_t ret, rc = PMIX_ERR_TIMEOUT;
+
+    pmix_output_verbose(2, pmix_server_globals.fence_output,
+                        "ALERT: grp construct timeout fired");
+
+    /* send this requestor the reply */
+    reply = PMIX_NEW(pmix_buffer_t);
+    if (NULL == reply) {
+        goto error;
+    }
+    /* setup the reply, starting with the returned status */
+    PMIX_BFROPS_PACK(ret, cd->peer, reply, &rc, 1, PMIX_STATUS);
+    if (PMIX_SUCCESS != ret) {
+        PMIX_ERROR_LOG(ret);
+        PMIX_RELEASE(reply);
+        goto error;
+    }
+    pmix_output_verbose(2, pmix_server_globals.base_output,
+                        "server:grp_timeout reply being sent to %s:%u",
+                        cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
+    PMIX_SERVER_QUEUE_REPLY(ret, cd->peer, cd->hdr.tag, reply);
+    if (PMIX_SUCCESS != ret) {
+        PMIX_RELEASE(reply);
+    }
+
+  error:
+    cd->event_active = false;
+    /* remove it from the list */
+    pmix_list_remove_item(&cd->trk->local_cbs, &cd->super);
+    PMIX_RELEASE(cd);
+}
+
+static void _grpcbfunc(int sd, short argc, void *cbdata)
+{
+    pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata;
+    pmix_server_trkr_t *trk = scd->tracker;
+    pmix_server_caddy_t *cd;
+    pmix_buffer_t *reply, xfer;
+    pmix_status_t ret;
+    size_t n, ctxid = SIZE_MAX;
+    pmix_group_t *grp;
+    pmix_byte_object_t *bo = NULL;
+    pmix_nspace_caddy_t *nptr;
+    pmix_list_t nslist;
+    bool found;
+
+    PMIX_ACQUIRE_OBJECT(scd);
+
+    if (NULL == trk) {
+        /* give them a release if they want it - this should
+         * never happen, but protect against the possibility */
+        if (NULL != scd->cbfunc.relfn) {
+            scd->cbfunc.relfn(scd->cbdata);
+        }
+        PMIX_RELEASE(scd);
+        return;
+    }
+
+    pmix_output_verbose(2, pmix_server_globals.connect_output,
+                        "server:grpcbfunc processing WITH %d MEMBERS",
+                        (int)pmix_list_get_size(&trk->local_cbs));
+
+    /* if the timer is active, clear it */
+    if (trk->event_active) {
+        pmix_event_del(&trk->ev);
+    }
+    grp = (pmix_group_t*)trk->cbdata;
+
+    /* the tracker's "hybrid" field is used to indicate construct
+     * vs destruct */
+    if (trk->hybrid) {
+        /* we destructed the group */
+        if (NULL != grp) {
+            pmix_list_remove_item(&pmix_server_globals.groups, &grp->super);
+            PMIX_RELEASE(grp);
+        }
+    } else {
+        /* see if this group was assigned a context ID or collected data */
+        for (n=0; n < scd->ninfo; n++) {
+            if (PMIX_CHECK_KEY(&scd->info[n], PMIX_GROUP_CONTEXT_ID)) {
+                PMIX_VALUE_GET_NUMBER(ret, &scd->info[n].value, ctxid, size_t);
+            } else if (PMIX_CHECK_KEY(&scd->info[n], PMIX_GROUP_ENDPT_DATA)) {
+                bo = &scd->info[n].value.data.bo;
+            }
+        }
+    }
+
+    /* if data was returned, then we need to have the modex cbfunc
+     * store it for us before releasing the group members */
+    if (NULL != bo) {
+        PMIX_CONSTRUCT(&xfer, pmix_buffer_t);
+        PMIX_CONSTRUCT(&nslist, pmix_list_t);
+        /* Collect the nptr list with uniq GDS components of all local
+         * participants. It does not allow multiple storing to the
+         * same GDS if participants have mutual GDS. */
+        PMIX_LIST_FOREACH(cd, &trk->local_cbs, pmix_server_caddy_t) {
+            // see if we already have this nspace
+            found = false;
+            PMIX_LIST_FOREACH(nptr, &nslist, pmix_nspace_caddy_t) {
+                if (0 == strcmp(nptr->ns->compat.gds->name,
+                            cd->peer->nptr->compat.gds->name)) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                // add it
+                nptr = PMIX_NEW(pmix_nspace_caddy_t);
+                PMIX_RETAIN(cd->peer->nptr);
+                nptr->ns = cd->peer->nptr;
+                pmix_list_append(&nslist, &nptr->super);
+            }
+        }
+
+        PMIX_LIST_FOREACH(nptr, &nslist, pmix_nspace_caddy_t) {
+            PMIX_LOAD_BUFFER(pmix_globals.mypeer, &xfer, bo->bytes, bo->size);
+            PMIX_GDS_STORE_MODEX(ret, nptr->ns, &xfer, trk);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+                break;
+            }
+        }
+    }
+
+    /* loop across all procs in the tracker, sending them the reply */
+    PMIX_LIST_FOREACH(cd, &trk->local_cbs, pmix_server_caddy_t) {
+        reply = PMIX_NEW(pmix_buffer_t);
+        if (NULL == reply) {
+            break;
+        }
+        /* setup the reply, starting with the returned status */
+        PMIX_BFROPS_PACK(ret, cd->peer, reply, &scd->status, 1, PMIX_STATUS);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_ERROR_LOG(ret);
+            PMIX_RELEASE(reply);
+            break;
+        }
+        if (!trk->hybrid) {
+            /* if a ctxid was provided, pass it along */
+            PMIX_BFROPS_PACK(ret, cd->peer, reply, &ctxid, 1, PMIX_SIZE);
+            if (PMIX_SUCCESS != ret) {
+                PMIX_ERROR_LOG(ret);
+                PMIX_RELEASE(reply);
+                break;
+            }
+        }
+        pmix_output_verbose(2, pmix_server_globals.connect_output,
+                            "server:grp_cbfunc reply being sent to %s:%u",
+                            cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
+        PMIX_SERVER_QUEUE_REPLY(ret, cd->peer, cd->hdr.tag, reply);
+        if (PMIX_SUCCESS != ret) {
+            PMIX_RELEASE(reply);
+        }
+    }
+
+    /* remove the tracker from the list */
+    pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super);
+    PMIX_RELEASE(trk);
+
+    /* we are done */
+    if (NULL != scd->cbfunc.relfn) {
+        scd->cbfunc.relfn(scd->cbdata);
+    }
+    PMIX_RELEASE(scd);
+}
+
+
+static void grpcbfunc(pmix_status_t status,
+                      pmix_info_t *info, size_t ninfo,
+                      void *cbdata,
+                      pmix_release_cbfunc_t relfn,
+                      void *relcbd)
+{
+    pmix_server_trkr_t *tracker = (pmix_server_trkr_t*)cbdata;
+    pmix_shift_caddy_t *scd;
+
+    pmix_output_verbose(2, pmix_server_globals.connect_output,
+                        "server:grpcbfunc called with %d info", (int)ninfo);
+
+    if (NULL == tracker) {
+        /* nothing to do - but be sure to give them
+         * a release if they want it */
+        if (NULL != relfn) {
+            relfn(relcbd);
+        }
+        return;
+    }
+
+    /* need to thread-shift this callback as it accesses global data */
+    scd = PMIX_NEW(pmix_shift_caddy_t);
+    if (NULL == scd) {
+        /* nothing we can do */
+        if (NULL != relfn) {
+            relfn(cbdata);
+        }
+        return;
+    }
+    scd->status = status;
+    scd->info = info;
+    scd->ninfo = ninfo;
+    scd->tracker = tracker;
+    scd->cbfunc.relfn = relfn;
+    scd->cbdata = relcbd;
+    PMIX_THREADSHIFT(scd, _grpcbfunc);
+}
+
+/* we are being called from the PMIx server's switchyard function,
+ * which means we are in an event and can access global data */
+pmix_status_t pmix_server_grpconstruct(pmix_server_caddy_t *cd,
+                                       pmix_buffer_t *buf)
+{
+    pmix_peer_t *peer = (pmix_peer_t*)cd->peer;
+    pmix_peer_t *pr;
+    int32_t cnt, m;
+    pmix_status_t rc;
+    char *grpid;
+    pmix_proc_t *procs;
+    pmix_group_t *grp, *pgrp;
+    pmix_info_t *info = NULL, *iptr;
+    size_t n, ninfo, nprocs, n2;
+    pmix_server_trkr_t *trk;
+    struct timeval tv = {0, 0};
+    bool need_cxtid = false;
+    bool match, force_local = false;
+    bool embed_barrier = false;
+    bool barrier_directive_included = false;
+    pmix_buffer_t bucket;
+    pmix_byte_object_t bo;
+    pmix_list_t mbrs;
+    pmix_namelist_t *nm;
+    bool expanded = false;
+
+    pmix_output_verbose(2, pmix_server_globals.connect_output,
+                        "recvd grpconstruct cmd");
+
+    /* unpack the group ID */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &grpid, &cnt, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+
+    /* see if we already have this group */
+    grp = NULL;
+    PMIX_LIST_FOREACH(pgrp, &pmix_server_globals.groups, pmix_group_t) {
+        if (0 == strcmp(grpid, pgrp->grpid)) {
+            grp = pgrp;
+            break;
+        }
+    }
+    if (NULL == grp) {
+        /* create a new entry */
+        grp = PMIX_NEW(pmix_group_t);
+        if (NULL == grp) {
+            rc = PMIX_ERR_NOMEM;
+            goto error;
+        }
+        grp->grpid = grpid;
+        pmix_list_append(&pmix_server_globals.groups, &grp->super);
+    } else {
+        free(grpid);
+    }
+
+    /* unpack the number of procs */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &nprocs, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+    if (0 == nprocs) {
+        return PMIX_ERR_BAD_PARAM;
+    }
+    PMIX_PROC_CREATE(procs, nprocs);
+    if (NULL == procs) {
+        rc = PMIX_ERR_NOMEM;
+        goto error;
+    }
+    cnt = nprocs;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, procs, &cnt, PMIX_PROC);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_PROC_FREE(procs, nprocs);
+        goto error;
+    }
+    if (NULL == grp->members) {
+        /* see if they used a local proc or local peer
+         * wildcard - if they did, then we need to expand
+         * it here */
+        PMIX_CONSTRUCT(&mbrs, pmix_list_t);
+        for (n=0; n < nprocs; n++) {
+            if (PMIX_RANK_LOCAL_PEERS == procs[n].rank) {
+                expanded = true;
+                /* expand to all local procs in this nspace */
+                for (m=0; m < pmix_server_globals.clients.size; m++) {
+                    if (NULL == (pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, m))) {
+                        continue;
+                    }
+                    if (PMIX_CHECK_NSPACE(procs[n].nspace, pr->info->pname.nspace)) {
+                        nm = PMIX_NEW(pmix_namelist_t);
+                        nm->pname = &pr->info->pname;
+                        pmix_list_append(&mbrs, &nm->super);
+                    }
+                }
+            } else if (PMIX_RANK_LOCAL_NODE == procs[n].rank) {
+                expanded = true;
+                /* add in all procs on the node */
+                for (m=0; m < pmix_server_globals.clients.size; m++) {
+                    if (NULL == (pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, m))) {
+                        continue;
+                    }
+                    nm = PMIX_NEW(pmix_namelist_t);
+                    nm->pname = &pr->info->pname;
+                    pmix_list_append(&mbrs, &nm->super);
+                }
+            } else {
+                nm = PMIX_NEW(pmix_namelist_t);
+                /* have to duplicate the name here */
+                nm->pname = (pmix_name_t*)malloc(sizeof(pmix_name_t));
+                nm->pname->nspace = strdup(procs[n].nspace);
+                nm->pname->rank = procs[n].rank;
+                pmix_list_append(&mbrs, &nm->super);
+            }
+        }
+        if (expanded) {
+            PMIX_PROC_FREE(procs, nprocs);
+            nprocs = pmix_list_get_size(&mbrs);
+            PMIX_PROC_CREATE(procs, nprocs);
+            n=0;
+            while (NULL != (nm = (pmix_namelist_t*)pmix_list_remove_first(&mbrs))) {
+                PMIX_LOAD_PROCID(&procs[n], nm->pname->nspace, nm->pname->rank);
+                PMIX_RELEASE(nm);
+            }
+            PMIX_DESTRUCT(&mbrs);
+        }
+        grp->members = procs;
+        grp->nmbrs = nprocs;
+    } else {
+        PMIX_PROC_FREE(procs, nprocs);
+    }
+
+    /* unpack the number of directives */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(info, ninfo);
+        cnt = ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto error;
+        }
+    }
+
+    /* find/create the local tracker for this operation */
+    if (NULL == (trk = get_tracker(grp->grpid, grp->members, grp->nmbrs, PMIX_GROUP_CONSTRUCT_CMD))) {
+        /* If no tracker was found - create and initialize it once */
+        if (NULL == (trk = new_tracker(grp->grpid, grp->members, grp->nmbrs, PMIX_GROUP_CONSTRUCT_CMD))) {
+            /* only if a bozo error occurs */
+            PMIX_ERROR_LOG(PMIX_ERROR);
+            rc = PMIX_ERROR;
+            goto error;
+        }
+        /* group members must have access to all endpoint info
+         * upon completion of the construct operation */
+        trk->collect_type = PMIX_COLLECT_YES;
+        /* mark as being a construct operation */
+        trk->hybrid = false;
+        /* pass along the grp object */
+        trk->cbdata = grp;
+        /* we only save the info structs from the first caller
+         * who provides them - it is a user error to provide
+         * different values from different participants */
+        trk->info = info;
+        trk->ninfo = ninfo;
+        /* see if we are to enforce a timeout or if they want
+         * a context ID created - we don't internally care
+         * about any other directives */
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+                tv.tv_sec = info[n].value.data.uint32;
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_GROUP_ASSIGN_CONTEXT_ID)) {
+                need_cxtid = PMIX_INFO_TRUE(&info[n]);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_GROUP_LOCAL_ONLY)) {
+                force_local = PMIX_INFO_TRUE(&info[n]);
+            } else if (PMIX_CHECK_KEY(&info[n], PMIX_EMBED_BARRIER)) {
+                embed_barrier = PMIX_INFO_TRUE(&info[n]);
+                barrier_directive_included = true;
+            }
+        }
+        /* see if this constructor only references local processes and isn't
+         * requesting a context ID - if both conditions are met, then we
+         * can just locally process the request without bothering the host.
+         * This is meant to provide an optimized path for a fairly common
+         * operation */
+        if (force_local) {
+            trk->local = true;
+        } else if (need_cxtid) {
+            trk->local = false;
+        } else {
+            trk->local = true;
+            for (n=0; n < grp->nmbrs; n++) {
+                /* if this entry references the local procs, then
+                 * we can skip it */
+                if (PMIX_RANK_LOCAL_PEERS == grp->members[n].rank ||
+                    PMIX_RANK_LOCAL_NODE == grp->members[n].rank) {
+                    continue;
+                }
+                /* see if it references a specific local proc */
+                match = false;
+                for (m=0; m < pmix_server_globals.clients.size; m++) {
+                    if (NULL == (pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, m))) {
+                        continue;
+                    }
+                    if (PMIX_CHECK_PROCID(&grp->members[n], &pr->info->pname)) {
+                        match = true;
+                        break;
+                    }
+                }
+                if (!match) {
+                    /* this requires a non_local operation */
+                    trk->local = false;
+                    break;
+                }
+            }
+        }
+    } else {
+        /* cleanup */
+        PMIX_INFO_FREE(info, ninfo);
+        info = NULL;
+    }
+
+    /* add this contributor to the tracker so they get
+     * notified when we are done */
+    pmix_list_append(&trk->local_cbs, &cd->super);
+
+    /* if a timeout was specified, set it */
+    if (0 < tv.tv_sec) {
+        PMIX_THREADSHIFT_DELAY(trk, grp_timeout, tv.tv_sec);
+        trk->event_active = true;
+    }
+
+    /* if all local contributions have been received,
+     * let the local host's server know that we are at the
+     * "fence" point - they will callback once the barrier
+     * across all participants has been completed */
+    if (trk->def_complete &&
+        pmix_list_get_size(&trk->local_cbs) == trk->nlocal) {
+        pmix_output_verbose(2, pmix_server_globals.base_output,
+                            "local group op complete with %d procs", (int)trk->npcs);
+
+        if (trk->local) {
+            /* nothing further needs to be done - we have
+             * created the local group. let the grpcbfunc
+             * threadshift the result */
+            grpcbfunc(PMIX_SUCCESS, NULL, 0, trk, NULL, NULL);
+            return PMIX_SUCCESS;
+        }
+
+        /* check if our host supports group operations */
+        if (NULL == pmix_host_server.group) {
+            /* remove the tracker from the list */
+            pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super);
+            PMIX_RELEASE(trk);
+            return PMIX_ERR_NOT_SUPPORTED;
+        }
+
+        /* if they direct us to not embed a barrier, then we won't gather
+         * the data for distribution */
+        if (!barrier_directive_included ||
+            (barrier_directive_included && embed_barrier)) {
+            /* collect any remote contributions provided by group members */
+            PMIX_CONSTRUCT(&bucket, pmix_buffer_t);
+            rc = _collect_data(trk, &bucket);
+            if (PMIX_SUCCESS != rc) {
+                if (trk->event_active) {
+                    pmix_event_del(&trk->ev);
+                }
+                /* remove the tracker from the list */
+                pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super);
+                PMIX_RELEASE(trk);
+                PMIX_DESTRUCT(&bucket);
+                return rc;
+            }
+            /* xfer the results to a byte object */
+            PMIX_UNLOAD_BUFFER(&bucket, bo.bytes, bo.size);
+            PMIX_DESTRUCT(&bucket);
+            /* load any results into a data object for inclusion in the
+             * fence operation */
+            n2 = trk->ninfo + 1;
+            PMIX_INFO_CREATE(iptr, n2);
+            for (n=0; n < trk->ninfo; n++) {
+                PMIX_INFO_XFER(&iptr[n], &trk->info[n]);
+            }
+            PMIX_INFO_LOAD(&iptr[ninfo], PMIX_GROUP_ENDPT_DATA, &bo, PMIX_BYTE_OBJECT);
+            PMIX_BYTE_OBJECT_DESTRUCT(&bo);
+            PMIX_INFO_FREE(trk->info, trk->ninfo);
+            trk->info = iptr;
+            trk->ninfo = n2;
+        }
+        rc = pmix_host_server.group(PMIX_GROUP_CONSTRUCT, grp->grpid,
+                                    trk->pcs, trk->npcs,
+                                    trk->info, trk->ninfo,
+                                    grpcbfunc, trk);
+        if (PMIX_SUCCESS != rc) {
+            if (trk->event_active) {
+                pmix_event_del(&trk->ev);
+            }
+            if (PMIX_OPERATION_SUCCEEDED == rc) {
+                /* let the grpcbfunc threadshift the result */
+                grpcbfunc(PMIX_SUCCESS, NULL, 0, trk, NULL, NULL);
+                return PMIX_SUCCESS;
+            }
+            /* remove the tracker from the list */
+            pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super);
+            PMIX_RELEASE(trk);
+            return rc;
+        }
+    }
+
+    return PMIX_SUCCESS;
+
+  error:
+    if (NULL != info) {
+        PMIX_INFO_FREE(info, ninfo);
+    }
+    return rc;
+}
+
+/* we are being called from the PMIx server's switchyard function,
+ * which means we are in an event and can access global data */
+pmix_status_t pmix_server_grpdestruct(pmix_server_caddy_t *cd,
+                                      pmix_buffer_t *buf)
+{
+    pmix_peer_t *peer = (pmix_peer_t*)cd->peer;
+    int32_t cnt;
+    pmix_status_t rc;
+    char *grpid;
+    pmix_info_t *info = NULL;
+    size_t n, ninfo;
+    pmix_server_trkr_t *trk;
+    pmix_group_t *grp, *pgrp;
+    struct timeval tv = {0, 0};
+
+    pmix_output_verbose(2, pmix_server_globals.iof_output,
+                        "recvd grpdestruct cmd");
+
+    if (NULL == pmix_host_server.group) {
+        PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED);
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    /* unpack the group ID */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &grpid, &cnt, PMIX_STRING);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+
+    /* find this group in our list */
+    grp = NULL;
+    PMIX_LIST_FOREACH(pgrp, &pmix_server_globals.groups, pmix_group_t) {
+        if (0 == strcmp(grpid, pgrp->grpid)) {
+            grp = pgrp;
+            break;
+        }
+    }
+    free(grpid);
+
+    /* if not found, then this is an error - we cannot
+     * destruct a group we don't know about */
+    if (NULL == grp) {
+        rc = PMIX_ERR_NOT_FOUND;
+        goto error;
+    }
+
+    /* unpack the number of directives */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, peer, buf, &ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto error;
+    }
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(info, ninfo);
+        cnt = ninfo;
+        PMIX_BFROPS_UNPACK(rc, peer, buf, info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            goto error;
+        }
+        /* see if we are to enforce a timeout - we don't internally care
+         * about any other directives */
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+                tv.tv_sec = info[n].value.data.uint32;
+                break;
+            }
+        }
+    }
+
+    /* find/create the local tracker for this operation */
+    if (NULL == (trk = get_tracker(grp->grpid, grp->members, grp->nmbrs, PMIX_GROUP_DESTRUCT_CMD))) {
+        /* If no tracker was found - create and initialize it once */
+        if (NULL == (trk = new_tracker(grp->grpid, grp->members, grp->nmbrs, PMIX_GROUP_DESTRUCT_CMD))) {
+            /* only if a bozo error occurs */
+            PMIX_ERROR_LOG(PMIX_ERROR);
+            rc = PMIX_ERROR;
+            goto error;
+        }
+        trk->collect_type = PMIX_COLLECT_NO;
+        /* mark as being a destruct operation */
+        trk->hybrid = true;
+        /* pass along the group object */
+        trk->cbdata = grp;
+    }
+
+    /* we only save the info structs from the first caller
+     * who provides them - it is a user error to provide
+     * different values from different participants */
+    if (NULL == trk->info) {
+        trk->info = info;
+        trk->ninfo = ninfo;
+    } else {
+        /* cleanup */
+        PMIX_INFO_FREE(info, ninfo);
+        info = NULL;
+    }
+
+    /* add this contributor to the tracker so they get
+     * notified when we are done */
+    pmix_list_append(&trk->local_cbs, &cd->super);
+
+    /* if a timeout was specified, set it */
+    if (0 < tv.tv_sec) {
+        PMIX_THREADSHIFT_DELAY(trk, grp_timeout, tv.tv_sec);
+        trk->event_active = true;
+    }
+
+    /* if all local contributions have been received,
+     * let the local host's server know that we are at the
+     * "fence" point - they will callback once the barrier
+     * across all participants has been completed */
+    if (trk->def_complete &&
+        pmix_list_get_size(&trk->local_cbs) == trk->nlocal) {
+        pmix_output_verbose(2, pmix_server_globals.base_output,
+                            "local group op complete %d", (int)trk->nlocal);
+
+        rc = pmix_host_server.group(PMIX_GROUP_DESTRUCT, grp->grpid,
+                                    grp->members, grp->nmbrs,
+                                    trk->info, trk->ninfo,
+                                    grpcbfunc, trk);
+        if (PMIX_SUCCESS != rc) {
+            if (trk->event_active) {
+                pmix_event_del(&trk->ev);
+            }
+            if (PMIX_OPERATION_SUCCEEDED == rc) {
+                /* let the grpcbfunc threadshift the result */
+                grpcbfunc(PMIX_SUCCESS, NULL, 0, trk, NULL, NULL);
+                return PMIX_SUCCESS;
+            }
+            /* remove the tracker from the list */
+            pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super);
+            PMIX_RELEASE(trk);
+            return rc;
+        }
+    }
+
+    return PMIX_SUCCESS;
+
+  error:
+    if (NULL != info) {
+        PMIX_INFO_FREE(info, ninfo);
+    }
+    return rc;
+}
+
+static void _fabric_response(int sd, short args, void *cbdata)
+{
+    pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata;
+
+    qcd->cbfunc(PMIX_SUCCESS, qcd->info, qcd->ninfo,
+                qcd->cbdata, NULL, NULL);
+    PMIX_RELEASE(qcd);
+}
+
+static void frcbfunc(pmix_status_t status, void *cbdata)
+{
+    pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata;
+
+    PMIX_ACQUIRE_OBJECT(qcd);
+    qcd->status = status;
+    PMIX_POST_OBJECT(qcd);
+
+    PMIX_WAKEUP_THREAD(&qcd->lock);
+
+}
+/* we are being called from the PMIx server's switchyard function,
+ * which means we are in an event and can access global data */
+pmix_status_t pmix_server_fabric_register(pmix_server_caddy_t *cd,
+                                          pmix_buffer_t *buf,
+                                          pmix_info_cbfunc_t cbfunc)
+{
+    int32_t cnt;
+    pmix_status_t rc;
+    pmix_query_caddy_t *qcd = NULL;
+    pmix_proc_t proc;
+    pmix_fabric_t fabric;
+
+    pmix_output_verbose(2, pmix_server_globals.base_output,
+                        "recvd register_fabric request from client");
+
+
+    qcd = PMIX_NEW(pmix_query_caddy_t);
+    if (NULL == qcd) {
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_RETAIN(cd);
+    qcd->cbfunc = cbfunc;
+    qcd->cbdata = cd;
+
+    /* unpack the number of directives */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &qcd->ninfo, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(qcd);
+        goto exit;
+    }
+    /* unpack the directives */
+    if (0 < qcd->ninfo) {
+        PMIX_INFO_CREATE(cd->info, qcd->ninfo);
+        cnt = qcd->ninfo;
+        PMIX_BFROPS_UNPACK(rc, cd->peer, buf, qcd->info, &cnt, PMIX_INFO);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(qcd);
+            goto exit;
+        }
+    }
+
+    /* see if we support this request ourselves */
+    PMIX_FABRIC_CONSTRUCT(&fabric);
+    rc = pmix_pnet.register_fabric(&fabric, qcd->info, qcd->ninfo, frcbfunc, qcd);
+    if (PMIX_OPERATION_SUCCEEDED == rc) {
+        /* we need to respond, but we want to ensure
+         * that occurs _after_ the client returns from its API */
+        if (NULL != qcd->info) {
+            PMIX_INFO_FREE(qcd->info, qcd->ninfo);
+        }
+        qcd->info = fabric.info;
+        qcd->ninfo = fabric.ninfo;
+        PMIX_THREADSHIFT(qcd, _fabric_response);
+        return PMIX_SUCCESS;
+    } else if (PMIX_SUCCESS == rc) {
+        PMIX_WAIT_THREAD(&qcd->lock);
+        /* we need to respond, but we want to ensure
+         * that occurs _after_ the client returns from its API */
+        if (NULL != qcd->info) {
+            PMIX_INFO_FREE(qcd->info, qcd->ninfo);
+        }
+        qcd->info = fabric.info;
+        qcd->ninfo = fabric.ninfo;
+        PMIX_THREADSHIFT(qcd, _fabric_response);
+        return PMIX_SUCCESS;
+    }
+
+    /* if we don't internally support it, see if
+     * our host does */
+    if (NULL == pmix_host_server.fabric) {
+        rc = PMIX_ERR_NOT_SUPPORTED;
+        goto exit;
+    }
+
+    /* setup the requesting peer name */
+    PMIX_LOAD_PROCID(&proc, cd->peer->info->pname.nspace, cd->peer->info->pname.rank);
+
+    /* ask the host to execute the request */
+    if (PMIX_SUCCESS != (rc = pmix_host_server.fabric(&proc, PMIX_FABRIC_REQUEST_INFO,
+                                                       qcd->info, qcd->ninfo,
+                                                       cbfunc, qcd))) {
+        goto exit;
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    if (NULL != qcd) {
+        PMIX_RELEASE(qcd);
+    }
+    return rc;
+}
+
+pmix_status_t pmix_server_fabric_update(pmix_server_caddy_t *cd,
+                                        pmix_buffer_t *buf,
+                                        pmix_info_cbfunc_t cbfunc)
+{
+    int32_t cnt;
+    size_t index;
+    pmix_status_t rc;
+    pmix_query_caddy_t *qcd;
+    pmix_proc_t proc;
+    pmix_fabric_t fabric;
+
+    pmix_output_verbose(2, pmix_server_globals.base_output,
+                        "recvd update_fabric request from client");
+
+
+    qcd = PMIX_NEW(pmix_query_caddy_t);
+    if (NULL == qcd) {
+        return PMIX_ERR_NOMEM;
+    }
+    PMIX_RETAIN(cd);
+    qcd->cbdata = cd;
+
+    /* unpack the fabric index */
+    cnt = 1;
+    PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &index, &cnt, PMIX_SIZE);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        goto exit;
+    }
+
+    /* see if we support this request ourselves */
+    PMIX_FABRIC_CONSTRUCT(&fabric);
+    fabric.index = index;
+    rc = pmix_pnet.update_fabric(&fabric);
+    if (PMIX_SUCCESS == rc) {
+        /* we need to respond, but we want to ensure
+         * that occurs _after_ the client returns from its API */
+        if (NULL != qcd->info) {
+            PMIX_INFO_FREE(qcd->info, qcd->ninfo);
+        }
+        qcd->info = fabric.info;
+        qcd->ninfo = fabric.ninfo;
+        PMIX_THREADSHIFT(qcd, _fabric_response);
+        return rc;
+    }
+
+    /* if we don't internally support it, see if
+     * our host does */
+    if (NULL == pmix_host_server.fabric) {
+        rc = PMIX_ERR_NOT_SUPPORTED;
+        goto exit;
+    }
+
+    /* setup the requesting peer name */
+    pmix_strncpy(proc.nspace, cd->peer->info->pname.nspace, PMIX_MAX_NSLEN);
+    proc.rank = cd->peer->info->pname.rank;
+    /* add the index */
+    qcd->ninfo = 1;
+    PMIX_INFO_CREATE(qcd->info, qcd->ninfo);
+    PMIX_INFO_LOAD(&qcd->info[0], PMIX_FABRIC_INDEX, &index, PMIX_SIZE);
+
+    /* ask the host to execute the request */
+    if (PMIX_SUCCESS != (rc = pmix_host_server.fabric(&proc, PMIX_FABRIC_UPDATE_INFO,
+                                                       qcd->info, qcd->ninfo,
+                                                       cbfunc, qcd))) {
+        goto exit;
+    }
+    return PMIX_SUCCESS;
+
+  exit:
+    return rc;
+}
 
 /*****    INSTANCE SERVER LIBRARY CLASSES    *****/
 static void tcon(pmix_server_trkr_t *t)
@@ -4108,3 +4977,22 @@
 PMIX_CLASS_INSTANCE(pmix_iof_cache_t,
                     pmix_list_item_t,
                     iocon, iodes);
+
+static void pscon(pmix_pset_t *p)
+{
+    p->name = NULL;
+    p->members = NULL;
+    p->nmembers = 0;
+}
+static void psdes(pmix_pset_t *p)
+{
+    if (NULL != p->name) {
+        free(p->name);
+    }
+    if (NULL != p->members) {
+        free(p->members);
+    }
+}
+PMIX_CLASS_INSTANCE(pmix_pset_t,
+                    pmix_list_item_t,
+                    pscon, psdes);
diff -Nru pmix-3.2.2~rc1/src/server/pmix_server_ops.h pmix-4.0.0/src/server/pmix_server_ops.h
--- pmix-3.2.2~rc1/src/server/pmix_server_ops.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/server/pmix_server_ops.h	2021-01-02 08:56:17.000000000 +0000
@@ -170,6 +170,14 @@
 PMIX_CLASS_DECLARATION(pmix_iof_cache_t);
 
 typedef struct {
+    pmix_list_item_t super;
+    char *name;
+    pmix_proc_t *members;
+    size_t nmembers;
+} pmix_pset_t;
+PMIX_CLASS_DECLARATION(pmix_pset_t);
+
+typedef struct {
     pmix_list_t nspaces;                    // list of pmix_nspace_t for the nspaces we know about
     pmix_pointer_array_t clients;           // array of pmix_peer_t local clients
     pmix_list_t collectives;                // list of active pmix_server_trkr_t
@@ -179,6 +187,7 @@
     pmix_list_t events;                     // list of pmix_regevents_info_t registered events
     pmix_list_t groups;                     // list of pmix_group_t group memberships
     pmix_list_t iof;                        // IO to be forwarded to clients
+    pmix_list_t psets;                      // list of known psets and memberships
     size_t max_iof_cache;                   // max number of IOF messages to cache
     bool tool_connections_allowed;
     char *tmpdir;                           // temporary directory for this server
@@ -348,6 +357,12 @@
                                                pmix_op_cbfunc_t cbfunc,
                                                void *cbdata);
 
+PMIX_EXPORT pmix_status_t pmix_server_grpconstruct(pmix_server_caddy_t *cd,
+                                                   pmix_buffer_t *buf);
+
+PMIX_EXPORT pmix_status_t pmix_server_grpdestruct(pmix_server_caddy_t *cd,
+                                                  pmix_buffer_t *buf);
+
 PMIX_EXPORT pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer,
                                                               pmix_buffer_t *buf,
                                                               pmix_op_cbfunc_t cbfunc,
@@ -363,6 +378,22 @@
 PMIX_EXPORT void pmix_server_purge_events(pmix_peer_t *peer,
                                           pmix_proc_t *proc);
 
+PMIX_EXPORT pmix_status_t pmix_server_fabric_register(pmix_server_caddy_t *cd,
+                                                      pmix_buffer_t *buf,
+                                                      pmix_info_cbfunc_t cbfunc);
+
+PMIX_EXPORT pmix_status_t pmix_server_fabric_update(pmix_server_caddy_t *cd,
+                                                    pmix_buffer_t *buf,
+                                                    pmix_info_cbfunc_t cbfunc);
+
+PMIX_EXPORT pmix_status_t pmix_server_fabric_get_vertex_info(pmix_server_caddy_t *cd,
+                                                             pmix_buffer_t *buf,
+                                                             pmix_info_cbfunc_t cbfunc);
+
+PMIX_EXPORT pmix_status_t pmix_server_fabric_get_device_index(pmix_server_caddy_t *cd,
+                                                              pmix_buffer_t *buf,
+                                                              pmix_info_cbfunc_t cbfunc);
+
 PMIX_EXPORT extern pmix_server_module_t pmix_host_server;
 PMIX_EXPORT extern pmix_server_globals_t pmix_server_globals;
 
diff -Nru pmix-3.2.2~rc1/src/tool/Makefile.include pmix-4.0.0/src/tool/Makefile.include
--- pmix-3.2.2~rc1/src/tool/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tool/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+# Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -7,5 +7,9 @@
 # $HEADER$
 #
 
+headers += \
+        tool/pmix_tool_ops.h
+
 sources += \
-        tool/pmix_tool.c
+        tool/pmix_tool.c \
+        tool/pmix_tool_ops.c
diff -Nru pmix-3.2.2~rc1/src/tool/pmix_tool.c pmix-4.0.0/src/tool/pmix_tool.c
--- pmix-3.2.2~rc1/src/tool/pmix_tool.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tool/pmix_tool.c	2021-01-02 08:56:17.000000000 +0000
@@ -51,6 +51,7 @@
 #include "src/util/argv.h"
 #include "src/util/error.h"
 #include "src/util/hash.h"
+#include "src/util/name_fns.h"
 #include "src/util/output.h"
 #include "src/util/pmix_environ.h"
 #include "src/util/show_help.h"
@@ -58,18 +59,37 @@
 #include "src/runtime/pmix_rte.h"
 #include "src/mca/bfrops/base/base.h"
 #include "src/mca/gds/base/base.h"
+#include "src/mca/pfexec/base/base.h"
+#include "src/mca/ploc/base/base.h"
+#include "src/mca/pmdl/base/base.h"
 #include "src/mca/pnet/base/base.h"
 #include "src/mca/ptl/base/base.h"
 #include "src/mca/psec/psec.h"
 #include "src/include/pmix_globals.h"
+#include "src/common/pmix_attributes.h"
 #include "src/common/pmix_iof.h"
 #include "src/client/pmix_client_ops.h"
 #include "src/server/pmix_server_ops.h"
 
 #define PMIX_MAX_RETRIES 10
 
-static pmix_event_t stdinsig;
+static pmix_event_t stdinsig, parentdied;
 static pmix_iof_read_event_t stdinev;
+static pmix_proc_t myparent;
+
+
+static void pdiedfn(int fd, short flags, void *arg)
+{
+    pmix_info_t info[3];
+
+    PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &myparent, PMIX_PROC);
+
+    /* generate a job-terminated event */
+    PMIx_Notify_event(PMIX_ERR_JOB_TERMINATED, &pmix_globals.myid,
+                      PMIX_RANGE_PROC_LOCAL,
+                      info, 3, NULL, NULL);
+}
 
 static void _notify_complete(pmix_status_t status, void *cbdata)
 {
@@ -206,15 +226,21 @@
             goto error;
         }
     }
-    /* prep the chain for processing */
-    pmix_prep_event_chain(chain, chain->info, ninfo, false);
 
     pmix_output_verbose(2, pmix_client_globals.event_output,
                         "[%s:%d] pmix:tool_notify_recv - processing event %s from source %s:%d, calling errhandler",
                         pmix_globals.myid.nspace, pmix_globals.myid.rank, PMIx_Error_string(chain->status),
                         chain->source.nspace, chain->source.rank);
 
-    pmix_invoke_local_event_hdlr(chain);
+    rc = pmix_server_notify_client_of_event(chain->status, &chain->source,
+                                            PMIX_RANGE_LOCAL,  // will be ignored
+                                            chain->info, chain->ninfo,
+                                            _notify_complete, chain);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(chain);
+        goto error;
+    }
     return;
 
     error:
@@ -337,28 +363,87 @@
     PMIX_WAKEUP_THREAD(&cb->lock);
 }
 
+/* event handler registration callback */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    pmix_lock_t *lock = (pmix_lock_t*)cbdata;
+
+    lock->status = status;
+    PMIX_WAKEUP_THREAD(lock);
+}
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    pmix_lock_t *lock=NULL;
+    char *name = NULL;
+    size_t n;
+
+    pmix_output_verbose(2, pmix_client_globals.base_output,
+                        "[%s:%d] DEBUGGER RELEASE RECVD",
+                        pmix_globals.myid.nspace, pmix_globals.myid.rank);
+    if (NULL != info) {
+        lock = NULL;
+        for (n=0; n < ninfo; n++) {
+            if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) {
+                lock = (pmix_lock_t*)info[n].value.data.ptr;
+            } else if (0 == strncmp(info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) {
+                name = info[n].value.data.string;
+            }
+        }
+        /* if the object wasn't returned, then that is an error */
+        if (NULL == lock) {
+            pmix_output_verbose(2, pmix_client_globals.base_output,
+                                "event handler %s failed to return object",
+                                (NULL == name) ? "NULL" : name);
+            /* let the event handler progress */
+            if (NULL != cbfunc) {
+                cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+            }
+            return;
+        }
+    }
+    if (NULL != lock) {
+        PMIX_WAKEUP_THREAD(lock);
+    }
+
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
 PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
                                pmix_info_t info[], size_t ninfo)
 {
     pmix_status_t rc;
     char *evar, *nspace = NULL;
     pmix_rank_t rank = PMIX_RANK_UNDEF;
-    bool gdsfound, do_not_connect = false;
+    bool do_not_connect = false;
     bool nspace_given = false;
     bool nspace_in_enviro = false;
     bool rank_given = false;
     bool fwd_stdin = false;
     bool connect_optional = false;
-    pmix_info_t ginfo;
+    pmix_info_t ginfo, *iptr, evinfo[2];
     size_t n;
     pmix_ptl_posted_recv_t *rcv;
-    pmix_proc_t wildcard;
+    pmix_proc_t wildcard, myserver;
     int fd;
     pmix_proc_type_t ptype = PMIX_PROC_TYPE_STATIC_INIT;
     pmix_cb_t cb;
     pmix_buffer_t *req;
-    pmix_cmd_t cmd = PMIX_REQ_CMD;
+    pmix_cmd_t cmd;
     pmix_iof_req_t *iofreq;
+    pmix_lock_t reglock, releaselock;
+    pmix_status_t code;
+    pmix_value_t *val, value;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
 
@@ -377,24 +462,36 @@
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_SUCCESS;
     }
+    /* init the parent procid to something innocuous */
+    PMIX_LOAD_PROCID(&myparent, NULL, PMIX_RANK_UNDEF);
+
+    /* backward compatibility fix - remove any directive to use
+     * the old usock component so we avoid a warning message */
+    if (NULL != (evar = getenv("PMIX_MCA_ptl"))) {
+        if (0 == strcmp(evar, "usock")) {
+            /* we cannot support a usock-only environment */
+            PMIX_RELEASE_THREAD(&pmix_global_lock);
+            fprintf(stderr, "-------------------------------------------------------------------\n");
+            fprintf(stderr, "PMIx no longer supports the \"usock\" transport for client-server\n");
+            fprintf(stderr, "communication. A directive was detected that only allows that mode.\n");
+            fprintf(stderr, "We cannot continue - please remove that constraint and try again.\n");
+            fprintf(stderr, "-------------------------------------------------------------------\n");
+            return PMIX_ERR_INIT;
+        }
+        /* anything else should just be cleared */
+        pmix_unsetenv("PMIX_MCA_ptl", &environ);
+    }
 
     /* parse the input directives */
-    gdsfound = false;
     PMIX_SET_PROC_TYPE(&ptype, PMIX_PROC_TOOL);
     if (NULL != info) {
         for (n=0; n < ninfo; n++) {
-            if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) {
-                PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, info[n].value.data.string, PMIX_STRING);
-                gdsfound = true;
-            } else if (0 == strncmp(info[n].key, PMIX_TOOL_DO_NOT_CONNECT, PMIX_MAX_KEYLEN)) {
+            if (0 == strncmp(info[n].key, PMIX_TOOL_DO_NOT_CONNECT, PMIX_MAX_KEYLEN)) {
                 do_not_connect = PMIX_INFO_TRUE(&info[n]);
             } else if (0 == strncmp(info[n].key, PMIX_TOOL_NSPACE, PMIX_MAX_KEYLEN)) {
                 if (NULL != nspace) {
                     /* cannot define it twice */
                     free(nspace);
-                    if (gdsfound) {
-                        PMIX_INFO_DESTRUCT(&ginfo);
-                    }
                     PMIX_RELEASE_THREAD(&pmix_global_lock);
                     return PMIX_ERR_BAD_PARAM;
                 }
@@ -414,6 +511,8 @@
                 pmix_server_globals.tmpdir = strdup(info[n].value.data.string);
             } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) {
                 pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string);
+            } else if (0 == strncmp(info[n].key, PMIX_TOOL_CONNECT_OPTIONAL, PMIX_MAX_KEYLEN)) {
+                connect_optional = PMIX_INFO_TRUE(&info[n]);
             }
         }
     }
@@ -439,9 +538,6 @@
         if (NULL != nspace) {
             free(nspace);
         }
-        if (gdsfound) {
-            PMIX_INFO_DESTRUCT(&ginfo);
-        }
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_BAD_PARAM;
     }
@@ -482,9 +578,6 @@
             if (NULL != nspace) {
                 free(nspace);
             }
-            if (gdsfound) {
-                PMIX_INFO_DESTRUCT(&ginfo);
-            }
             PMIX_RELEASE_THREAD(&pmix_global_lock);
             return PMIX_ERR_BAD_PARAM;
         }
@@ -498,17 +591,24 @@
         if (NULL != nspace) {
             free(nspace);
         }
-        if (gdsfound) {
-            PMIX_INFO_DESTRUCT(&ginfo);
-        }
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return rc;
     }
+
+    /* if we were given a keepalive pipe, register an
+     * event to capture the event */
+    if (NULL != (evar = getenv("PMIX_KEEPALIVE_PIPE"))) {
+        rc = strtol(evar, NULL, 10);
+        pmix_event_set(pmix_globals.evbase, &parentdied, rc, PMIX_EV_READ, pdiedfn, NULL);
+        pmix_event_add(&parentdied, NULL);
+        pmix_unsetenv("PMIX_KEEPALIVE_PIPE", &environ);
+        pmix_fd_set_cloexec(rc);  // don't let children inherit this
+    }
+
     /* if we were given a name, then set it now */
     if (nspace_given || nspace_in_enviro) {
-        pmix_strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN);
+        PMIX_LOAD_PROCID(&pmix_globals.myid, nspace, rank);
         free(nspace);
-        pmix_globals.myid.rank = rank;
     }
 
     /* setup the IO Forwarding recv */
@@ -516,7 +616,7 @@
     rcv->tag = PMIX_PTL_TAG_IOF;
     rcv->cbfunc = tool_iof_handler;
     /* add it to the end of the list of recvs */
-    pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super);
+    pmix_list_append(&pmix_ptl_base.posted_recvs, &rcv->super);
 
 
     /* setup the globals */
@@ -525,27 +625,18 @@
     pmix_pointer_array_init(&pmix_client_globals.peers, 1, INT_MAX, 1);
     pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t);
     if (NULL == pmix_client_globals.myserver) {
-        if (gdsfound) {
-            PMIX_INFO_DESTRUCT(&ginfo);
-        }
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_NOMEM;
     }
     pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t);
     if (NULL == pmix_client_globals.myserver->nptr) {
         PMIX_RELEASE(pmix_client_globals.myserver);
-        if (gdsfound) {
-            PMIX_INFO_DESTRUCT(&ginfo);
-        }
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_NOMEM;
     }
     pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t);
     if (NULL == pmix_client_globals.myserver->info) {
         PMIX_RELEASE(pmix_client_globals.myserver);
-        if (gdsfound) {
-            PMIX_INFO_DESTRUCT(&ginfo);
-        }
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_NOMEM;
     }
@@ -560,38 +651,28 @@
         /* setup a rank_info object for us */
         pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t);
         if (NULL == pmix_globals.mypeer->info) {
-            if (gdsfound) {
-                PMIX_INFO_DESTRUCT(&ginfo);
-            }
             PMIX_RELEASE_THREAD(&pmix_global_lock);
             return PMIX_ERR_NOMEM;
         }
         pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace);
         pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank;
-        /* our bfrops module will be set when we connect to the server */
-    } else {
-        /* select our bfrops compat module */
-        pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL);
-        if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
-            if (gdsfound) {
-                PMIX_INFO_DESTRUCT(&ginfo);
-            }
-            PMIX_RELEASE_THREAD(&pmix_global_lock);
-            return PMIX_ERR_INIT;
-        }
-        /* the server will be using the same */
-        pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
     }
 
+    /* select our bfrops compat module */
+    pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL);
+    if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    /* the server will be using the same */
+    pmix_client_globals.myserver->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
+
     /* select our psec compat module - the selection may be based
      * on the corresponding envars that should have been passed
      * to us at launch */
     evar = getenv("PMIX_SECURITY_MODE");
     pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(evar);
     if (NULL == pmix_globals.mypeer->nptr->compat.psec) {
-        if (gdsfound) {
-            PMIX_INFO_DESTRUCT(&ginfo);
-        }
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_INIT;
     }
@@ -613,67 +694,54 @@
     /* the server will be using the same */
     pmix_client_globals.myserver->nptr->compat.type = pmix_globals.mypeer->nptr->compat.type;
 
-    /* select a GDS module for our own internal use - the user may
-     * have passed down a directive for this purpose. If they did, then
-     * use it. Otherwise, we want the "hash" module */
-    if (!gdsfound) {
-        PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, "hash", PMIX_STRING);
-    }
+    /* tools are restricted to the "hash" component for interacting
+     * with a server's GDS framework */
+    PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, "hash", PMIX_STRING);
     pmix_globals.mypeer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
-    if (NULL == pmix_globals.mypeer->nptr->compat.gds) {
-        PMIX_INFO_DESTRUCT(&ginfo);
-        PMIX_RELEASE_THREAD(&pmix_global_lock);
-        return PMIX_ERR_INIT;
-    }
     PMIX_INFO_DESTRUCT(&ginfo);
-    /* select the gds compat module we will use to interact with
-     * our server- the selection will be based
-     * on the corresponding envars that should have been passed
-     * to us at launch */
-    evar = getenv("PMIX_GDS_MODULE");
-    if (NULL != evar) {
-        PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, evar, PMIX_STRING);
-        pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1);
-        PMIX_INFO_DESTRUCT(&ginfo);
-    } else {
-        pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0);
-    }
-    if (NULL == pmix_client_globals.myserver->nptr->compat.gds) {
+    if (NULL == pmix_globals.mypeer->nptr->compat.gds) {
         PMIX_RELEASE_THREAD(&pmix_global_lock);
         return PMIX_ERR_INIT;
     }
+    pmix_client_globals.myserver->nptr->compat.gds = pmix_globals.mypeer->nptr->compat.gds;
 
-    /* if we are a launcher, then we also need to act as a server,
-     * so setup the server-related structures here */
-    if (PMIX_PROC_IS_LAUNCHER(&ptype) ||
-        PMIX_PROC_IS_CLIENT_LAUNCHER(&ptype)) {
-        if (PMIX_SUCCESS != (rc = pmix_server_initialize())) {
-            PMIX_ERROR_LOG(rc);
-            if (NULL != nspace) {
-                free(nspace);
-            }
-            if (gdsfound) {
-                PMIX_INFO_DESTRUCT(&ginfo);
-            }
-            PMIX_RELEASE_THREAD(&pmix_global_lock);
-            return rc;
+    /* tools can, in some scenarios, act as servers,
+     * so initialize the server globals too */
+    if (PMIX_SUCCESS != (rc = pmix_server_initialize())) {
+        PMIX_ERROR_LOG(rc);
+        if (NULL != nspace) {
+            free(nspace);
         }
-        /* setup the function pointers */
-        memset(&pmix_host_server, 0, sizeof(pmix_server_module_t));
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
     }
+    /* setup the function pointers */
+    memset(&pmix_host_server, 0, sizeof(pmix_server_module_t));
 
     if (do_not_connect) {
         /* ensure we mark that we are not connected */
         pmix_globals.connected = false;
-        /* it is an error if we were not given an nspace/rank */
+        /* it is not an error if we were not given an nspace/rank */
         if (!nspace_given || !rank_given) {
-            PMIX_RELEASE_THREAD(&pmix_global_lock);
-            return PMIX_ERR_UNREACH;
+            /* self-assign a namespace and rank for ourselves. Use our hostname:pid
+             * for the nspace, and rank clearly is 0 */
+            snprintf(pmix_globals.myid.nspace, PMIX_MAX_NSLEN-1, "%s:%lu", pmix_globals.hostname, (unsigned long)pmix_globals.pid);
+            pmix_globals.myid.rank = 0;
+            nspace_given = false;
+            rank_given = false;
+            /* also setup the client myserver to point to ourselves */
+            pmix_client_globals.myserver->nptr->nspace = strdup(pmix_globals.myid.nspace);
+            pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t);
+            pmix_client_globals.myserver->info->pname.nspace = strdup(pmix_globals.myid.nspace);
+            pmix_client_globals.myserver->info->pname.rank = pmix_globals.myid.rank;
+            pmix_client_globals.myserver->info->uid = pmix_globals.uid;
+            pmix_client_globals.myserver->info->gid = pmix_globals.gid;
         }
     } else {
         /* connect to the server */
-        rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo);
+        rc = pmix_ptl.connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo);
         if (PMIX_SUCCESS != rc) {
+            /* if connection wasn't optional, then error out */
             if (!connect_optional) {
                 PMIX_RELEASE_THREAD(&pmix_global_lock);
                 return rc;
@@ -694,8 +762,13 @@
             pmix_client_globals.myserver->info->gid = pmix_globals.gid;
         }
     }
+    /* setup the wildcard ID */
+    PMIX_LOAD_PROCID(&wildcard, pmix_globals.myid.nspace, PMIX_RANK_WILDCARD);
     /* pass back the ID */
     PMIX_LOAD_PROCID(proc, pmix_globals.myid.nspace, pmix_globals.myid.rank);
+    /* cache the server in case we later want to call "set_server" on it */
+    PMIX_RETAIN(pmix_client_globals.myserver);
+    pmix_pointer_array_add(&pmix_server_globals.clients, pmix_client_globals.myserver);
 
     /* load into our own peer object */
     if (NULL == pmix_globals.mypeer->nptr->nspace) {
@@ -709,14 +782,25 @@
     }
     pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace);
     pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank;
-    /* if we are acting as a server, then start listening */
+    /* if we are acting as a server, then setup the global recv */
     if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
         /* setup the wildcard recv for inbound messages from clients */
         rcv = PMIX_NEW(pmix_ptl_posted_recv_t);
         rcv->tag = UINT32_MAX;
         rcv->cbfunc = pmix_server_message_handler;
         /* add it to the end of the list of recvs */
-        pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super);
+        pmix_list_append(&pmix_ptl_base.posted_recvs, &rcv->super);
+    }
+
+    /* open the pmdl framework and select the active modules for this environment
+     * as we might need them if we are asking a server to launch something for us */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pmdl_base_framework, 0))) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
+    }
+    if (PMIX_SUCCESS != (rc = pmix_pmdl_base_select())) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return rc;
     }
 
     /* setup IOF */
@@ -807,8 +891,9 @@
      * job info - we do this as a non-blocking
      * transaction because some systems cannot handle very large
      * blocking operations and error out if we try them. */
-    if (PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
+    if (!do_not_connect && PMIX_PEER_IS_CLIENT(pmix_globals.mypeer)) {
          req = PMIX_NEW(pmix_buffer_t);
+         cmd = PMIX_REQ_CMD;
          PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                           req, &cmd, 1, PMIX_COMMAND);
          if (PMIX_SUCCESS != rc) {
@@ -841,8 +926,6 @@
          * well-known */
         pmix_cb_t cb;
         PMIX_CONSTRUCT(&cb, pmix_cb_t);
-        pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
-        wildcard.rank = PMIX_RANK_WILDCARD;
         cb.proc = &wildcard;
         cb.copy = true;
         PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb);
@@ -870,13 +953,34 @@
 
     /* if we are acting as a server, then start listening */
     if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        /* setup the fork/exec framework */
+        if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pfexec_base_framework, 0)) ) {
+            return rc;
+        }
+        if (PMIX_SUCCESS != (rc = pmix_pfexec_base_select()) ) {
+            return rc;
+        }
+
+        /* open the ploc framework */
+        if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_ploc_base_framework, 0))) {
+            return rc;
+        }
+        if (PMIX_SUCCESS != (rc = pmix_ploc_base_select())) {
+            return rc;
+        }
+
+        /* if we don't know our topology, we better get it now as we
+         * increasingly rely on it - note that our host will hopefully
+         * have passed it to us so we don't duplicate their storage! */
+        if (PMIX_SUCCESS != (rc = pmix_ploc.setup_topology(info, ninfo))) {
+            return rc;
+        }
+
         /* open the pnet framework and select the active modules for this environment */
         if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0))) {
-            PMIX_RELEASE_THREAD(&pmix_global_lock);
             return rc;
         }
         if (PMIX_SUCCESS != (rc = pmix_pnet_base_select())) {
-            PMIX_RELEASE_THREAD(&pmix_global_lock);
             return rc;
         }
 
@@ -887,6 +991,95 @@
         }
     }
 
+    /* see if they gave us a rendezvous URI to which we are to call back */
+    evar = getenv("PMIX_LAUNCHER_RNDZ_URI");
+    if (NULL != evar) {
+        /* attach to the specified server so it can
+         * tell us what we are to do - save our
+         * current server for now */
+        PMIX_LOAD_PROCID(&myserver, pmix_client_globals.myserver->info->pname.nspace,
+                         pmix_client_globals.myserver->info->pname.rank);
+        PMIX_INFO_CREATE(iptr, 3);
+        PMIX_INFO_LOAD(&iptr[0], PMIX_SERVER_URI, evar, PMIX_STRING);
+        rc = 2;  // give us two seconds to connect
+        PMIX_INFO_LOAD(&iptr[1], PMIX_TIMEOUT, &rc, PMIX_INT);
+        PMIX_INFO_LOAD(&iptr[2], PMIX_PRIMARY_SERVER, NULL, PMIX_BOOL);
+        rc = PMIx_tool_attach_to_server(NULL, &myparent, iptr, 3);
+        if (PMIX_SUCCESS != rc) {
+            return PMIX_ERR_UNREACH;
+        }
+
+        /* save our parent ID */
+        value.type = PMIX_PROC;
+        value.data.proc = &myparent;
+        rc = PMIx_Store_internal(&pmix_globals.myid, PMIX_PARENT_ID, &value);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+        /* retrieve any job info it has for us */
+        req = PMIX_NEW(pmix_buffer_t);
+        cmd = PMIX_REQ_CMD;
+        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
+                         req, &cmd, 1, PMIX_COMMAND);
+        if (PMIX_SUCCESS != rc) {
+            PMIX_ERROR_LOG(rc);
+            PMIX_RELEASE(req);
+            return rc;
+        }
+        /* send to the server */
+        PMIX_CONSTRUCT(&cb, pmix_cb_t);
+        PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
+                           req, job_data, (void*)&cb);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+        /* wait for the data to return */
+        PMIX_WAIT_THREAD(&cb.lock);
+        rc = cb.status;
+        PMIX_DESTRUCT(&cb);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+        /* restore our original primary server */
+        rc = PMIx_tool_set_server(&myserver, NULL, 0);
+        if (PMIX_SUCCESS != rc) {
+            return rc;
+        }
+    }
+
+    /* register the tool supported attrs */
+    rc = pmix_register_tool_attrs();
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* see if we were asked to stop in init */
+    PMIX_INFO_LOAD(&ginfo, PMIX_OPTIONAL, NULL, PMIX_BOOL);
+    rc = PMIx_Get(&wildcard, PMIX_DEBUG_STOP_IN_INIT, &ginfo, 1, &val);
+    if (PMIX_SUCCESS == rc) {
+        /* if the value was found, then we need to wait for debugger attach here */
+        /* register for the debugger release notification */
+        PMIX_CONSTRUCT_LOCK(&reglock);
+        PMIX_CONSTRUCT_LOCK(&releaselock);
+        PMIX_INFO_LOAD(&evinfo[0], PMIX_EVENT_RETURN_OBJECT, &releaselock, PMIX_POINTER);
+        PMIX_INFO_LOAD(&evinfo[1], PMIX_EVENT_HDLR_NAME, "WAIT-FOR-RELEASE", PMIX_STRING);
+        pmix_output_verbose(2, pmix_client_globals.event_output,
+                            "[%s:%d] WAITING IN INIT FOR RELEASE",
+                            pmix_globals.myid.nspace, pmix_globals.myid.rank);
+        code = PMIX_ERR_DEBUGGER_RELEASE;
+        PMIx_Register_event_handler(&code, 1, evinfo, 2,
+                                    notification_fn, evhandler_reg_callbk, (void*)&reglock);
+        /* wait for registration to complete */
+        PMIX_WAIT_THREAD(&reglock);
+        PMIX_DESTRUCT_LOCK(&reglock);
+        PMIX_INFO_DESTRUCT(&evinfo[0]);
+        PMIX_INFO_DESTRUCT(&evinfo[1]);
+        /* wait for release to arrive */
+        PMIX_WAIT_THREAD(&releaselock);
+        PMIX_DESTRUCT_LOCK(&releaselock);
+        PMIX_VALUE_RELEASE(val);
+    }
+
     return PMIX_SUCCESS;
 }
 
@@ -895,10 +1088,8 @@
     pmix_kval_t *kptr;
     pmix_status_t rc;
     pmix_proc_t wildcard;
-    char hostname[PMIX_MAXHOSTNAMELEN] = {0};
 
-    pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
-    wildcard.rank = PMIX_RANK_WILDCARD;
+    PMIX_LOAD_PROCID(&wildcard, pmix_globals.myid.nspace, PMIX_RANK_WILDCARD);
 
     /* the jobid is just our nspace */
     kptr = PMIX_NEW(pmix_kval_t);
@@ -1132,16 +1323,11 @@
      * and load it from there */
 
     /* hostname */
-    if (NULL != pmix_globals.hostname) {
-        pmix_strncpy(hostname, pmix_globals.hostname, PMIX_MAXHOSTNAMELEN);
-    } else {
-        gethostname(hostname, PMIX_MAXHOSTNAMELEN-1);
-    }
     kptr = PMIX_NEW(pmix_kval_t);
     kptr->key = strdup(PMIX_HOSTNAME);
     PMIX_VALUE_CREATE(kptr->value, 1);
     kptr->value->type = PMIX_STRING;
-    kptr->value->data.string = strdup(hostname);
+    kptr->value->data.string = strdup(pmix_globals.hostname);
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
                       &pmix_globals.myid,
                       PMIX_INTERNAL, kptr);
@@ -1161,7 +1347,7 @@
     kptr->key = strdup(PMIX_NODE_MAP);
     PMIX_VALUE_CREATE(kptr->value, 1);
     kptr->value->type = PMIX_STRING;
-    kptr->value->data.string = strdup(hostname);
+    kptr->value->data.string = strdup(pmix_globals.hostname);
     PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
                       &wildcard,
                       PMIX_INTERNAL, kptr);
@@ -1268,6 +1454,7 @@
     struct timeval tv = {5, 0};
     int n;
     pmix_peer_t *peer;
+    pmix_pfexec_child_t *child;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (1 != pmix_globals.init_cntr) {
@@ -1331,14 +1518,25 @@
 
     }
 
-    if (!pmix_globals.external_evbase) {
-        /* stop the progress thread, but leave the event base
-         * still constructed. This will allow us to safely
-         * tear down the infrastructure, including removal
-         * of any events objects may be holding */
-        (void)pmix_progress_thread_pause(NULL);
+    if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
+        /* if we have launched children, then we need to cleanly
+         * terminate them - do this before stopping our progress
+         * thread as we need it for terminating procs */
+        if (pmix_pfexec_globals.active) {
+            pmix_event_del(pmix_pfexec_globals.handler);
+            pmix_pfexec_globals.active = false;
+        }
+        PMIX_LIST_FOREACH(child, &pmix_pfexec_globals.children, pmix_pfexec_child_t) {
+            pmix_pfexec.kill_proc(&child->proc);
+        }
     }
 
+    /* stop the progress thread, but leave the event base
+     * still constructed. This will allow us to safely
+     * tear down the infrastructure, including removal
+     * of any events objects may be holding */
+    (void)pmix_progress_thread_pause(NULL);
+
     PMIX_RELEASE(pmix_client_globals.myserver);
     PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests);
     for (n=0; n < pmix_client_globals.peers.size; n++) {
@@ -1347,27 +1545,27 @@
         }
     }
 
-    if (PMIX_PEER_IS_LAUNCHER(pmix_globals.mypeer)) {
-        pmix_ptl_base_stop_listening();
+    pmix_ptl_base_stop_listening();
 
-        for (n=0; n < pmix_server_globals.clients.size; n++) {
-            if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n))) {
-                PMIX_RELEASE(peer);
-            }
+    for (n=0; n < pmix_server_globals.clients.size; n++) {
+        if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n))) {
+            PMIX_RELEASE(peer);
         }
-
-        (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework);
-        PMIX_DESTRUCT(&pmix_server_globals.clients);
-        PMIX_LIST_DESTRUCT(&pmix_server_globals.collectives);
-        PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd);
-        PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs);
-        PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata);
-        PMIX_LIST_DESTRUCT(&pmix_server_globals.events);
-        PMIX_LIST_DESTRUCT(&pmix_server_globals.iof);
-
-        (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework);
     }
 
+    (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework);
+    PMIX_DESTRUCT(&pmix_server_globals.clients);
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.collectives);
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd);
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs);
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata);
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.events);
+    PMIX_LIST_DESTRUCT(&pmix_server_globals.iof);
+
+    (void)pmix_mca_base_framework_close(&pmix_pfexec_base_framework);
+    (void)pmix_mca_base_framework_close(&pmix_pmdl_base_framework);
+    (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework);
+
     pmix_rte_finalize();
     if (NULL != pmix_globals.mypeer) {
         PMIX_RELEASE(pmix_globals.mypeer);
@@ -1382,13 +1580,96 @@
 pmix_status_t PMIx_tool_connect_to_server(pmix_proc_t *proc,
                                           pmix_info_t info[], size_t ninfo)
 {
-    pmix_buffer_t *msg;
-    pmix_cmd_t cmd = PMIX_FINALIZE_CMD;
     pmix_status_t rc;
-    pmix_tool_timeout_t tev;
-    struct timeval tv = {2, 0};
-    pmix_event_base_t *evbase_save;
+
+    rc = PMIx_tool_attach_to_server(proc, NULL, info, ninfo);
+    return rc;
+}
+
+static void retry_attach(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
     pmix_kval_t *kptr;
+    pmix_peer_t *peer;
+    size_t n;
+    pmix_status_t rc;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+
+    /* check for directives */
+    cb->checked = false;
+    for (n=0; n < cb->ninfo; n++) {
+        if (PMIX_CHECK_KEY(&cb->info[n], PMIX_PRIMARY_SERVER)) {
+            cb->checked = PMIX_INFO_TRUE(&cb->info[n]);
+            break;
+        }
+    }
+
+    /* ask the ptl to establish connection to the new server */
+    peer = PMIX_NEW(pmix_peer_t);
+    /* setup the infrastructure - assume this new server will follow
+     * same rules as our current one */
+    peer->nptr = PMIX_NEW(pmix_namespace_t);
+    peer->info = PMIX_NEW(pmix_rank_info_t);
+    peer->nptr->compat.bfrops = pmix_globals.mypeer->nptr->compat.bfrops;
+    peer->nptr->compat.psec = pmix_globals.mypeer->nptr->compat.psec;
+    peer->nptr->compat.type = pmix_globals.mypeer->nptr->compat.type;
+    peer->nptr->compat.gds = pmix_globals.mypeer->nptr->compat.gds;
+
+    cb->status = pmix_ptl.connect_to_peer((struct pmix_peer_t*)peer, cb->info, cb->ninfo);
+
+    if (PMIX_SUCCESS == cb->status) {
+        /* return the name */
+        cb->pname.nspace = strdup(peer->info->pname.nspace);
+        cb->pname.rank = peer->info->pname.rank;
+        /* add the peer to our known clients */
+        pmix_pointer_array_add(&pmix_server_globals.clients, peer);
+        if (cb->checked) {
+            /* point our active server at this new one */
+            pmix_client_globals.myserver = peer;
+            /* mark that we are connected */
+            pmix_globals.connected = true;
+            /* update our active server's ID in the local key-value store */
+            kptr = PMIX_NEW(pmix_kval_t);
+            kptr->key = strdup(PMIX_SERVER_NSPACE);
+            PMIX_VALUE_CREATE(kptr->value, 1);
+            kptr->value->type = PMIX_STRING;
+            kptr->value->data.string = strdup(peer->info->pname.nspace);
+            PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
+                              &pmix_globals.myid,
+                              PMIX_INTERNAL, kptr);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            PMIX_RELEASE(kptr); // maintain accounting
+            kptr = PMIX_NEW(pmix_kval_t);
+            kptr->key = strdup(PMIX_SERVER_RANK);
+            PMIX_VALUE_CREATE(kptr->value, 1);
+            kptr->value->type = PMIX_PROC_RANK;
+            kptr->value->data.rank = peer->info->pname.rank;
+            PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
+                              &pmix_globals.myid,
+                              PMIX_INTERNAL, kptr);
+            if (PMIX_SUCCESS != rc) {
+                PMIX_ERROR_LOG(rc);
+            }
+            PMIX_RELEASE(kptr); // maintain accounting
+        }
+
+    } else {
+        PMIX_RELEASE(peer);
+    }
+
+    PMIX_WAKEUP_THREAD(&cb->lock);
+    PMIX_POST_OBJECT(cb);
+    return;
+}
+
+pmix_status_t PMIx_tool_attach_to_server(pmix_proc_t *myproc, pmix_proc_t *server,
+                                         pmix_info_t info[], size_t ninfo)
+{
+    pmix_status_t rc;
+    pmix_cb_t *cb;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
@@ -1397,138 +1678,301 @@
     }
     PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    /* check for bozo error - we do this
-     * prior to breaking any existing connection to avoid
-     * becoming stranded due to an incorrect function call */
+    /* check for bozo error */
     if (NULL == info || 0 == ninfo) {
         pmix_show_help("help-pmix-runtime.txt", "tool:no-server", true);
         return PMIX_ERR_BAD_PARAM;
     }
 
-    /* break any existing connection */
-    if (pmix_globals.connected) {
-        /* gracefully terminate this connection */
-        msg = PMIX_NEW(pmix_buffer_t);
-        /* pack the cmd */
-        PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
-                         msg, &cmd, 1, PMIX_COMMAND);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            PMIX_RELEASE(msg);
-            return rc;
-        }
+    cb = PMIX_NEW(pmix_cb_t);
+    cb->info = info;
+    cb->ninfo = ninfo;
+    PMIX_THREADSHIFT(cb, retry_attach);
 
+    PMIX_WAIT_THREAD(&cb->lock);
+    rc = cb->status;
 
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "pmix:tool:reconnect sending finalize sync to server");
+    /* if they gave us an address, we pass back our name */
+    if (NULL != myproc) {
+        memcpy(myproc, &pmix_globals.myid, sizeof(pmix_proc_t));
+    }
 
-        /* setup a timer to protect ourselves should the server be unable
-         * to answer for some reason */
-        PMIX_CONSTRUCT_LOCK(&tev.lock);
-        pmix_event_assign(&tev.ev, pmix_globals.evbase, -1, 0,
-                          fin_timeout, &tev);
-        tev.active = true;
-        PMIX_POST_OBJECT(&tev);
-        pmix_event_add(&tev.ev, &tv);
-        PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg,
-                           finwait_cbfunc, (void*)&tev);
-        if (PMIX_SUCCESS != rc) {
-            if (tev.active) {
-                pmix_event_del(&tev.ev);
-            }
-            return rc;
-        }
+    /* if the transition didn't succeed, then return at this point */
+    if (PMIX_SUCCESS != rc) {
+        return rc;
+    }
 
-        /* wait for the ack to return */
-        PMIX_WAIT_THREAD(&tev.lock);
-        PMIX_DESTRUCT_LOCK(&tev.lock);
-        if (tev.active) {
-            pmix_event_del(&tev.ev);
+    /* if they gave us an address, return the new server's ID */
+    if (NULL != server) {
+        PMIX_LOAD_PROCID(server, cb->pname.nspace, cb->pname.rank);
+    }
+
+    return PMIX_SUCCESS;
+}
+
+static void disc(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    pmix_peer_t *peer = NULL, *pr;
+    int n;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+
+    if (NULL == cb->proc) {
+        pmix_globals.connected = false;
+        cb->status = PMIX_SUCCESS;
+        PMIX_WAKEUP_THREAD(&cb->lock);
+        PMIX_POST_OBJECT(cb);
+        return;
+    }
+
+    /* see if we have this server */
+    for (n=0; n < pmix_server_globals.clients.size; n++) {
+        pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n);
+        if (NULL == pr) {
+            continue;
+        }
+        if (PMIX_CHECK_NSPACE(cb->proc->nspace, pr->info->pname.nspace) &&
+            PMIX_CHECK_RANK(cb->proc->rank, pr->info->pname.rank)) {
+            peer = pr;
+            pmix_pointer_array_set_item(&pmix_server_globals.clients, n, NULL);
+            break;
         }
-        pmix_output_verbose(2, pmix_globals.debug_output,
-                            "pmix:tool:reconnect finalize sync received");
+    }
+    if (NULL == peer) {
+        cb->status = PMIX_ERR_NOT_FOUND;
+        PMIX_WAKEUP_THREAD(&cb->lock);
+        PMIX_POST_OBJECT(cb);
+        return;
     }
 
-    /* stop the existing progress thread */
-    (void)pmix_progress_thread_pause(NULL);
+    /* if we are disconnecting from the active server, then we enter a
+     * "disconnected" state where we point the active server back at
+     * ourselves - effectively the same as when we init without connecting */
+    if (peer == pmix_client_globals.myserver) {
+        PMIX_RETAIN(pmix_globals.mypeer);
+        /* switch servers - we are in an event, so it is
+         * safe to do so */
+        pmix_client_globals.myserver = pmix_globals.mypeer;
+        pmix_globals.connected = false;
+    }
 
-    /* save that event base */
-    evbase_save = pmix_globals.evbase;
+    /* now drop the connection */
+    PMIX_RELEASE(peer);
 
-    /* create a new progress thread */
-    pmix_globals.evbase = pmix_progress_thread_init("reconnect");
-    pmix_progress_thread_start("reconnect");
-
-    /* now ask the ptl to establish connection to the new server */
-    rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo);
-
-    /* once that activity has all completed, then stop the new progress thread */
-    pmix_progress_thread_stop("reconnect");
-    pmix_progress_thread_finalize("reconnect");
-
-    /* restore the original progress thread */
-    pmix_globals.evbase = evbase_save;
-    /* restore the communication events */
-    pmix_event_assign(&pmix_client_globals.myserver->recv_event,
-                      pmix_globals.evbase,
-                      pmix_client_globals.myserver->sd,
-                      EV_READ | EV_PERSIST,
-                      pmix_ptl_base_recv_handler, pmix_client_globals.myserver);
-    pmix_client_globals.myserver->recv_ev_active = true;
-    PMIX_POST_OBJECT(pmix_client_globals.myserver);
-    pmix_event_add(&pmix_client_globals.myserver->recv_event, 0);
-
-    /* setup send event */
-    pmix_event_assign(&pmix_client_globals.myserver->send_event,
-                      pmix_globals.evbase,
-                      pmix_client_globals.myserver->sd,
-                      EV_WRITE|EV_PERSIST,
-                      pmix_ptl_base_send_handler, pmix_client_globals.myserver);
-    pmix_client_globals.myserver->send_ev_active = false;
-    /* resume processing events */
-    pmix_progress_thread_resume(NULL);
+    cb->status = PMIX_SUCCESS;
+    PMIX_WAKEUP_THREAD(&cb->lock);
 
-    /* if they gave us an address, we pass back our name */
-    if (NULL != proc) {
-        memcpy(proc, &pmix_globals.myid, sizeof(pmix_proc_t));
+    PMIX_POST_OBJECT(cb);
+    return;
+}
+
+pmix_status_t PMIx_tool_disconnect(const pmix_proc_t *server)
+{
+    pmix_status_t rc;
+    pmix_cb_t *cb;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
     }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
 
-    /* if the transition didn't succeed, then return at this point */
-    if (PMIX_SUCCESS != rc) {
-        return rc;
+    cb = PMIX_NEW(pmix_cb_t);
+    cb->proc = (pmix_proc_t*)server;
+    PMIX_THREADSHIFT(cb, disc);
+
+    PMIX_WAIT_THREAD(&cb->lock);
+    rc = cb->status;
+    cb->proc = NULL;
+    PMIX_RELEASE(cb);
+
+    return rc;
+}
+
+static void getsrvrs(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    int n;
+    size_t ns;
+    pmix_list_t srvrs;
+    pmix_proclist_t *ps;
+    pmix_peer_t *pr;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+
+    /* get servers */
+    PMIX_CONSTRUCT(&srvrs, pmix_list_t);
+    /* put our current active server at the front */
+    if (pmix_globals.mypeer != pmix_client_globals.myserver) {
+        ps = PMIX_NEW(pmix_proclist_t);
+        PMIX_LOAD_PROCID(&ps->proc, pmix_client_globals.myserver->info->pname.nspace, pmix_client_globals.myserver->info->pname.rank);
+        pmix_list_append(&srvrs, &ps->super);
+    }
+
+    for (n=0; n < pmix_server_globals.clients.size; n++) {
+        pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n);
+        if (NULL == pr) {
+            continue;
+        }
+        /* if it is our current primary server, ignore it */
+        if (pr == pmix_client_globals.myserver) {
+            continue;
+        }
+        /* record it */
+        ps = PMIX_NEW(pmix_proclist_t);
+        PMIX_LOAD_PROCID(&ps->proc, pr->info->pname.nspace, pr->info->pname.rank);
+        pmix_list_append(&srvrs, &ps->super);
+    }
+
+    ns = pmix_list_get_size(&srvrs);
+
+    if (0 == ns) {
+        /* we aren't connected to anyone */
+        cb->status = PMIX_ERR_UNREACH;
+        cb->nprocs = 0;
+        cb->procs = NULL;
+        PMIX_DESTRUCT(&srvrs);
+        PMIX_WAKEUP_THREAD(&cb->lock);
+        PMIX_POST_OBJECT(cb);
+        return;
     }
 
-    /* update our server's ID */
-    if (NULL != pmix_client_globals.myserver &&
-        NULL != pmix_client_globals.myserver->info &&
-        NULL != pmix_client_globals.myserver->info->pname.nspace) {
-        kptr = PMIX_NEW(pmix_kval_t);
-        kptr->key = strdup(PMIX_SERVER_NSPACE);
-        PMIX_VALUE_CREATE(kptr->value, 1);
-        kptr->value->type = PMIX_STRING;
-        kptr->value->data.string = strdup(pmix_client_globals.myserver->info->pname.nspace);
-        PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                          &pmix_globals.myid,
-                          PMIX_INTERNAL, kptr);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            return rc;
-        }
-        PMIX_RELEASE(kptr); // maintain accounting
-        kptr = PMIX_NEW(pmix_kval_t);
-        kptr->key = strdup(PMIX_SERVER_RANK);
-        PMIX_VALUE_CREATE(kptr->value, 1);
-        kptr->value->type = PMIX_PROC_RANK;
-        kptr->value->data.rank = pmix_client_globals.myserver->info->pname.rank;
-        PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer,
-                          &pmix_globals.myid,
-                          PMIX_INTERNAL, kptr);
-        if (PMIX_SUCCESS != rc) {
-            PMIX_ERROR_LOG(rc);
-            return rc;
+    /* allocate the array */
+    PMIX_PROC_CREATE(cb->procs, ns);
+    cb->nprocs = ns;
+
+    /* now load the array */
+    n=0;
+    PMIX_LIST_FOREACH(ps, &srvrs, pmix_proclist_t) {
+        memcpy(&cb->procs[n], &ps->proc, sizeof(pmix_proc_t));
+        ++n;
+    }
+    cb->status = PMIX_SUCCESS;
+    PMIX_LIST_DESTRUCT(&srvrs);
+
+    PMIX_WAKEUP_THREAD(&cb->lock);
+    PMIX_POST_OBJECT(cb);
+    return;
+}
+pmix_status_t PMIx_tool_get_servers(pmix_proc_t *servers[], size_t *nservers)
+{
+    pmix_status_t rc;
+    pmix_cb_t *cb;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    cb = PMIX_NEW(pmix_cb_t);
+
+    PMIX_THREADSHIFT(cb, getsrvrs);
+    PMIX_WAIT_THREAD(&cb->lock);
+    rc = cb->status;
+    *servers = cb->procs;
+    *nservers = cb->nprocs;
+
+    cb->procs = NULL;   // protect the array
+    cb->nprocs = 0;
+    PMIX_RELEASE(cb);
+
+    return rc;
+}
+
+static void retry_set(int sd, short args, void *cbdata)
+{
+    pmix_cb_t *cb = (pmix_cb_t*)cbdata;
+    int n;
+    pmix_peer_t *peer = NULL, *pr;
+
+    PMIX_ACQUIRE_OBJECT(cb);
+
+    /* see if we have this server */
+    for (n=0; n < pmix_server_globals.clients.size; n++) {
+        pr = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n);
+        if (NULL == pr) {
+            continue;
+        }
+        if (PMIX_CHECK_NSPACE(cb->proc->nspace, pr->info->pname.nspace) &&
+            PMIX_CHECK_RANK(cb->proc->rank, pr->info->pname.rank)) {
+            peer = pr;
+            break;
+        }
+    }
+    if (NULL == peer) {
+        /* do they want us to wait? */
+        if (cb->checked) {
+            /* have we timed out? */
+            --cb->status;
+            if (cb->status < 0) {
+                cb->status = PMIX_ERR_NOT_FOUND;
+                PMIX_WAKEUP_THREAD(&cb->lock);
+                return;
+            }
+            PMIX_THREADSHIFT_DELAY(cb, retry_set, 0.25);
+        } else {
+            /* no - so just return failure */
+            cb->status = PMIX_ERR_UNREACH;
+            PMIX_WAKEUP_THREAD(&cb->lock);
         }
-        PMIX_RELEASE(kptr); // maintain accounting
+        PMIX_POST_OBJECT(cb);
+        return;
     }
 
-    return PMIX_SUCCESS;
+    /* if this is the current active server, then ignore the request */
+    if (peer == pmix_client_globals.myserver) {
+        pmix_globals.connected = true;  // just ensure we mark ourselves as connected
+        cb->status = PMIX_SUCCESS;
+        PMIX_WAKEUP_THREAD(&cb->lock);
+        PMIX_POST_OBJECT(cb);
+        return;
+    }
+
+    /* switch the active server - we are in an event, so
+     * it is safe to do so */
+    pmix_client_globals.myserver = peer;
+    pmix_globals.connected = true;
+
+    cb->status = PMIX_SUCCESS;
+    PMIX_WAKEUP_THREAD(&cb->lock);
+    PMIX_POST_OBJECT(cb);
+    return;
+}
+
+pmix_status_t PMIx_tool_set_server(const pmix_proc_t *server,
+                                   pmix_info_t info[], size_t ninfo)
+{
+    pmix_status_t rc;
+    pmix_cb_t *cb;
+    size_t n;
+
+    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
+    if (pmix_globals.init_cntr <= 0) {
+        PMIX_RELEASE_THREAD(&pmix_global_lock);
+        return PMIX_ERR_INIT;
+    }
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+
+    /* threadshift this so we can access global structures */
+    cb = PMIX_NEW(pmix_cb_t);
+    cb->proc = (pmix_proc_t*)server;
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+            cb->status = 4 * info[n].value.data.integer;
+        } else if (PMIX_CHECK_KEY(&info[n], PMIX_WAIT_FOR_CONNECTION)) {
+            cb->checked = PMIX_INFO_TRUE(&info[n]);
+        }
+    }
+    PMIX_THREADSHIFT(cb, retry_set);
+
+    /* wait for completion */
+    PMIX_WAIT_THREAD(&cb->lock);
+    rc = cb->status;
+    PMIX_RELEASE(cb);
+
+    return rc;
 }
diff -Nru pmix-3.2.2~rc1/src/tool/pmix_tool_ops.c pmix-4.0.0/src/tool/pmix_tool_ops.c
--- pmix-3.2.2~rc1/src/tool/pmix_tool_ops.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tool/pmix_tool_ops.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,139 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "src/include/pmix_config.h"
+
+#include "src/client/pmix_client_ops.h"
+#include "src/include/pmix_globals.h"
+#include "src/util/error.h"
+#include "src/util/name_fns.h"
+#include "src/mca/bfrops/bfrops.h"
+#include "src/mca/ptl/ptl.h"
+
+#include "pmix_tool_ops.h"
+
+static void tool_switchyard(struct pmix_peer_t *pr,
+                            pmix_ptl_hdr_t *hdr,
+                            pmix_buffer_t *buf, void *cbdata);
+
+/* the following function is used by the PMIx server
+ * switchyard to process commands sent to one tool by
+ * another tool. Tool processes cannot have PMIx clients
+ * as they are tools and NOT servers. Thus, the only
+ * case we must handle is that where one tool has
+ * connected to another tool as its primary server
+ * (for example, a debugger fork/exec'ing a launcher).
+ * In this case, the tool must relay the request to
+ * a server that can process it since it will itself
+ * lack the ability to do so.
+ *
+ * For example, a "spawn" command must be relayed to
+ * the local system server so it can be executed as
+ * the tool itself has no mechanism by which it can
+ * do so
+ *
+ * Input params:
+ * cmd - the command we received
+ *
+ * peer - the tool that sent us the request
+ *
+ * bfr - the buffer containing the request
+ */
+pmix_status_t pmix_tool_relay_op(pmix_cmd_t cmd, pmix_peer_t *peer,
+                                 pmix_buffer_t *bfr, uint32_t tag)
+{
+    pmix_shift_caddy_t *s;
+    pmix_status_t rc;
+    pmix_buffer_t *relay;
+    pmix_cmd_t relaycmds[] = {
+        PMIX_SPAWNNB_CMD
+    };
+    bool found=false;
+    size_t nrelaycmds, n;
+
+    /***** IF IT IS THE SPAWN COMMAND, WE SEND IT WITH A
+    SEPARATE CBFUNC SO WE CAN INTERCEPT THE RESPONSE *****/
+
+    /* there are some commands we just cannot relay as
+     * it makes no sense to do so */
+    nrelaycmds = sizeof(relaycmds) / sizeof(pmix_cmd_t);
+    for (n=0; n < nrelaycmds; n++) {
+        if (cmd == relaycmds[n]) {
+            found = true;
+            break;
+        }
+    }
+    if (!found) {
+        return PMIX_ERR_NOT_SUPPORTED;
+    }
+
+    if (!pmix_globals.connected) {
+        return PMIX_ERR_UNREACH;
+    }
+
+    s = PMIX_NEW(pmix_shift_caddy_t);
+    PMIX_RETAIN(peer);
+    s->peer = peer;
+    s->ncodes = tag;
+    /* reset the buffer's pointers so the entire
+     * message is resent */
+    bfr->unpack_ptr = bfr->base_ptr;
+    relay = PMIX_NEW(pmix_buffer_t);
+    PMIX_BFROPS_COPY_PAYLOAD(rc, peer, relay, bfr);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(relay);
+        PMIX_RELEASE(s);
+        return rc;
+    }
+
+    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, relay,
+                           tool_switchyard, (void*)s);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_RELEASE(relay);
+        PMIX_RELEASE(s);
+        return rc;
+    }
+    return PMIX_SUCCESS;
+}
+
+static void tool_switchyard(struct pmix_peer_t *pr,
+                            pmix_ptl_hdr_t *hdr,
+                            pmix_buffer_t *buf, void *cbdata)
+{
+    pmix_shift_caddy_t *s = (pmix_shift_caddy_t*)cbdata;
+    pmix_buffer_t *relay;
+    pmix_status_t rc;
+    uint32_t tag = (uint32_t)s->ncodes;
+
+    /* the tag for the original sender was stored in ncodes, and
+     * the server would have packed the buffer using my
+     * bfrops plugin */
+
+    relay = PMIX_NEW(pmix_buffer_t);
+    PMIX_BFROPS_COPY_PAYLOAD(rc, pmix_globals.mypeer, relay, buf);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(relay);
+        return;
+    }
+    PMIX_SERVER_QUEUE_REPLY(rc, s->peer, tag, relay);
+    if (PMIX_SUCCESS != rc) {
+        PMIX_ERROR_LOG(rc);
+        PMIX_RELEASE(relay);
+    }
+    PMIX_RELEASE(s);
+}
diff -Nru pmix-3.2.2~rc1/src/tool/pmix_tool_ops.h pmix-4.0.0/src/tool/pmix_tool_ops.h
--- pmix-3.2.2~rc1/src/tool/pmix_tool_ops.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tool/pmix_tool_ops.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,21 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ *
+ * $COPYRIGHT$
+ */
+
+#ifndef PMIX_TOOL_OPS_H
+#define PMIX_TOOL_OPS_H
+
+#include "src/include/pmix_config.h"
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+
+PMIX_EXPORT pmix_status_t pmix_tool_relay_op(pmix_cmd_t cmd,
+                                             pmix_peer_t *peer,
+                                             pmix_buffer_t *bfr,
+                                             uint32_t tag);
+
+#endif // PMIX_TOOL_OPS_H
diff -Nru pmix-3.2.2~rc1/src/tools/Makefile.include pmix-4.0.0/src/tools/Makefile.include
--- pmix-3.2.2~rc1/src/tools/Makefile.include	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tools/Makefile.include	2021-01-02 08:56:17.000000000 +0000
@@ -28,10 +28,16 @@
     tools/pevent \
     tools/pmix_info \
     tools/plookup \
-    tools/pps
+    tools/pps \
+    tools/pattrs \
+    tools/pquery \
+    tools/wrapper
 
 DIST_SUBDIRS += \
     tools/pevent \
     tools/pmix_info \
     tools/plookup \
-    tools/pps
+    tools/pps \
+    tools/pattrs \
+    tools/pquery \
+    tools/wrapper
diff -Nru pmix-3.2.2~rc1/src/tools/pattrs/help-pattrs.txt pmix-4.0.0/src/tools/pattrs/help-pattrs.txt
--- pmix-3.2.2~rc1/src/tools/pattrs/help-pattrs.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/pattrs/help-pattrs.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,27 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017-2019 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for pattrs
+#
+[usage]
+pattrs [OPTIONS]
+  PMIx Supported Attributes and Functions tool
+
+%s
diff -Nru pmix-3.2.2~rc1/src/tools/pattrs/Makefile.am pmix-4.0.0/src/tools/pattrs/Makefile.am
--- pmix-3.2.2~rc1/src/tools/pattrs/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/pattrs/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CFLAGS = \
+            -DPMIX_CONFIGURE_USER="\"@PMIX_CONFIGURE_USER@\"" \
+            -DPMIX_CONFIGURE_HOST="\"@PMIX_CONFIGURE_HOST@\"" \
+            -DPMIX_CONFIGURE_DATE="\"@PMIX_CONFIGURE_DATE@\"" \
+            -DPMIX_BUILD_USER="\"$$USER\"" \
+            -DPMIX_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPMIX_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
+            -DPMIX_BUILD_CFLAGS="\"@CFLAGS@\"" \
+            -DPMIX_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
+            -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
+            -DPMIX_BUILD_LIBS="\"@LIBS@\"" \
+            -DPMIX_CC_ABSOLUTE="\"@PMIX_CC_ABSOLUTE@\"" \
+            -DPMIX_GREEK_VERSION="\"@PMIX_GREEK_VERSION@\"" \
+            -DPMIX_REPO_REV="\"@PMIX_REPO_REV@\"" \
+            -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\""
+
+if PMIX_INSTALL_BINARIES
+
+bin_PROGRAMS = pattrs
+
+dist_pmixdata_DATA = help-pattrs.txt
+
+endif # PMIX_INSTALL_BINARIES
+
+pattrs_SOURCES = pattrs.c
+pattrs_LDADD = \
+    $(PMIX_EXTRA_LTLIB) \
+	$(top_builddir)/src/libpmix.la
diff -Nru pmix-3.2.2~rc1/src/tools/pattrs/pattrs.c pmix-4.0.0/src/tools/pattrs/pattrs.c
--- pmix-3.2.2~rc1/src/tools/pattrs/pattrs.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/pattrs/pattrs.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,503 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "pmix_config.h"
+#include "include/pmix_common.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <pthread.h>
+
+#include "include/pmix_tool.h"
+#include "src/common/pmix_attributes.h"
+#include "src/mca/base/base.h"
+#include "src/mca/pinstalldirs/base/base.h"
+#include "src/threads/threads.h"
+#include "src/util/cmd_line.h"
+#include "src/util/keyval_parse.h"
+#include "src/util/show_help.h"
+#include "src/runtime/pmix_rte.h"
+
+typedef struct {
+    pmix_lock_t lock;
+    pmix_status_t status;
+} mylock_t;
+
+static pmix_proc_t myproc;
+
+/* define a structure for collecting returned
+ * info from a query */
+typedef struct {
+    pmix_lock_t lock;
+    pmix_status_t status;
+    pmix_info_t *info;
+    size_t ninfo;
+} myquery_data_t;
+
+
+/* this is a callback function for the PMIx_Query
+ * API. The query will callback with a status indicating
+ * if the request could be fully satisfied, partially
+ * satisfied, or completely failed. The info parameter
+ * contains an array of the returned data, with the
+ * info->key field being the key that was provided in
+ * the query call. Thus, you can correlate the returned
+ * data in the info->value field to the requested key.
+ *
+ * Once we have dealt with the returned data, we must
+ * call the release_fn so that the PMIx library can
+ * cleanup */
+static void cbfunc(pmix_status_t status,
+                   pmix_info_t *info, size_t ninfo,
+                   void *cbdata,
+                   pmix_release_cbfunc_t release_fn,
+                   void *release_cbdata)
+{
+    myquery_data_t *mq = (myquery_data_t*)cbdata;
+    size_t n;
+
+    mq->status = status;
+    /* save the returned info - the PMIx library "owns" it
+     * and will release it and perform other cleanup actions
+     * when release_fn is called */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+
+    /* release the block */
+    PMIX_WAKEUP_THREAD(&mq->lock);
+}
+
+/* this is the event notification function we pass down below
+ * when registering for general events - i.e.,, the default
+ * handler. We don't technically need to register one, but it
+ * is usually good practice to catch any events that occur */
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    /* this example doesn't do anything with default events */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+/* event handler registration is done asynchronously because it
+ * may involve the PMIx server registering with the host RM for
+ * external events. So we provide a callback function that returns
+ * the status of the request (success or an error), plus a numerical index
+ * to the registered event. The index is used later on to deregister
+ * an event handler - if we don't explicitly deregister it, then the
+ * PMIx server will do so when it see us exit */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    if (PMIX_SUCCESS != status) {
+        fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+    }
+    lock->status = status;
+    PMIX_WAKEUP_THREAD(&lock->lock);
+}
+
+/*****************************************
+ * Global Vars for Command line Arguments
+ *****************************************/
+typedef struct {
+    bool help;
+    bool verbose;
+    pid_t pid;
+    char *nspace;
+    char *uri;
+    bool sysfirst;
+    bool system;
+    char *client;
+    char *server;
+    char *tool;
+    char *host;
+    bool clientfns;
+    bool serverfns;
+    bool toolfns;
+    bool hostfns;
+} pmix_pattrs_globals_t;
+
+pmix_pattrs_globals_t pmix_pattrs_globals = {0};
+
+pmix_cmd_line_init_t cmd_line_opts[] = {
+    { NULL,
+      'h', NULL, "help",
+      0,
+      &pmix_pattrs_globals.help, PMIX_CMD_LINE_TYPE_BOOL,
+      "This help message" },
+
+    { NULL,
+      'v', NULL, "verbose",
+      0,
+      &pmix_pattrs_globals.verbose, PMIX_CMD_LINE_TYPE_BOOL,
+      "Be Verbose" },
+
+    { NULL,
+      'p', NULL, "pid",
+      1,
+      &pmix_pattrs_globals.pid, PMIX_CMD_LINE_TYPE_INT,
+      "Specify server pid to connect to" },
+
+    { NULL,
+      'n', NULL, "nspace",
+      1,
+      &pmix_pattrs_globals.nspace, PMIX_CMD_LINE_TYPE_STRING,
+      "Specify server nspace to connect to" },
+
+    { NULL,
+      '\0', NULL, "uri",
+      1,
+      &pmix_pattrs_globals.uri, PMIX_CMD_LINE_TYPE_STRING,
+      "Specify URI of server to connect to" },
+
+    { NULL,
+      '\0', NULL, "system-server-first",
+      0,
+      &pmix_pattrs_globals.sysfirst, PMIX_CMD_LINE_TYPE_BOOL,
+      "Look for the system server first" },
+
+    { NULL,
+      '\0', NULL, "system-server",
+      0,
+      &pmix_pattrs_globals.system, PMIX_CMD_LINE_TYPE_BOOL,
+      "Specifically connect to the system server" },
+
+    { NULL,
+      'c', NULL, "client",
+      1,
+      &pmix_pattrs_globals.client, PMIX_CMD_LINE_TYPE_STRING,
+      "Comma-delimited list of client function whose attributes are to be printed (function or all)" },
+
+    { NULL,
+      's', NULL, "server",
+      1,
+      &pmix_pattrs_globals.server, PMIX_CMD_LINE_TYPE_STRING,
+      "Comma-delimited list of server function whose attributes are to be printed (function or all)" },
+
+    { NULL,
+      't', NULL, "tool",
+      1,
+      &pmix_pattrs_globals.tool, PMIX_CMD_LINE_TYPE_STRING,
+      "Comma-delimited list of tool function whose attributes are to be printed (function or all)" },
+
+    { NULL,
+      'h', NULL, "host",
+      1,
+      &pmix_pattrs_globals.host, PMIX_CMD_LINE_TYPE_STRING,
+      "Comma-delimited list of host function whose attributes are to be printed (function or all)" },
+
+    { NULL,
+      '\0', NULL, "client-fns",
+      0,
+      &pmix_pattrs_globals.clientfns, PMIX_CMD_LINE_TYPE_BOOL,
+      "List the functions supported in this client library" },
+
+    { NULL,
+      '\0', NULL, "server-fns",
+      0,
+      &pmix_pattrs_globals.serverfns, PMIX_CMD_LINE_TYPE_BOOL,
+      "List the functions supported in this server library" },
+
+    { NULL,
+      '\0', NULL, "tool-fns",
+      0,
+      &pmix_pattrs_globals.toolfns, PMIX_CMD_LINE_TYPE_BOOL,
+      "List the functions supported in this tool library" },
+
+    { NULL,
+      '\0', NULL, "host-fns",
+      0,
+      &pmix_pattrs_globals.hostfns, PMIX_CMD_LINE_TYPE_BOOL,
+      "List the functions supported by this host environment" },
+
+
+    /* End of list */
+    { NULL,
+      '\0', NULL, NULL,
+      0,
+      NULL, PMIX_CMD_LINE_TYPE_NULL,
+      NULL }
+};
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    pmix_info_t *info;
+    mylock_t mylock;
+    pmix_cmd_line_t cmd_line;
+    char **fns = NULL;
+    size_t n, m;
+    myquery_data_t mq;
+    pmix_query_t query;
+    pmix_regattr_t *reg;
+    char **ans = NULL;
+
+    /* protect against problems if someone passes us thru a pipe
+     * and then abnormally terminates the pipe early */
+    signal(SIGPIPE, SIG_IGN);
+
+    /* initialize the output system */
+    if (!pmix_output_init()) {
+        return PMIX_ERROR;
+    }
+
+    /* initialize install dirs code */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pinstalldirs_base_framework, 0))) {
+        fprintf(stderr, "pmix_pinstalldirs_base_open() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
+                __FILE__, __LINE__, rc);
+        return rc;
+    }
+    if (PMIX_SUCCESS != (rc = pmix_pinstall_dirs_base_init(NULL, 0))) {
+        fprintf(stderr, "pmix_pinstalldirs_base_init() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
+                __FILE__, __LINE__, rc);
+        return rc;
+    }
+
+    /* initialize the help system */
+    pmix_show_help_init();
+
+    /* keyval lex-based parser */
+    if (PMIX_SUCCESS != (rc = pmix_util_keyval_parse_init())) {
+        fprintf(stderr, "pmix_util_keyval_parse_init failed with %d\n", rc);
+        return PMIX_ERROR;
+    }
+
+    /* Setup the parameter system */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_var_init())) {
+        fprintf(stderr, "pmix_mca_base_var_init failed with %d\n", rc);
+        return PMIX_ERROR;
+    }
+
+    /* register params for pmix */
+    if (PMIX_SUCCESS != (rc = pmix_register_params())) {
+        fprintf(stderr, "pmix_register_params failed with %d\n", rc);
+        return PMIX_ERROR;
+    }
+
+    /* Parse the command line options */
+    pmix_cmd_line_create(&cmd_line, cmd_line_opts);
+
+    rc = pmix_cmd_line_parse(&cmd_line, false, false, argc, argv);
+
+    if (PMIX_SUCCESS != rc) {
+        if (PMIX_ERR_SILENT != rc) {
+            fprintf(stderr, "%s: command line error (%s)\n", argv[0],
+                    PMIx_Error_string(rc));
+        }
+        return rc;
+    }
+
+    if (pmix_pattrs_globals.help) {
+        char *str, *args = NULL;
+        args = pmix_cmd_line_get_usage_msg(&cmd_line);
+        str = pmix_show_help_string("help-pattrs.txt", "usage", true,
+                                    args);
+        if (NULL != str) {
+            printf("%s", str);
+            free(str);
+        }
+        free(args);
+        /* If we show the help message, that should be all we do */
+        exit(0);
+    }
+
+    /* cannot list functions and get attributes at same time */
+    if ((pmix_pattrs_globals.clientfns || pmix_pattrs_globals.serverfns || pmix_pattrs_globals.toolfns || pmix_pattrs_globals.hostfns)
+        && (pmix_pattrs_globals.client || pmix_pattrs_globals.server || pmix_pattrs_globals.tool || NULL != pmix_pattrs_globals.host)) {
+        fprintf(stderr, "Cannot request both a list of functions and attributes at same time\n");
+        exit(1);
+    }
+
+    /* if they are asking for client, server, or tool attrs, then
+     * we don't need to connect to anyone - just register the
+     * attrs and report */
+    if (pmix_pattrs_globals.clientfns || pmix_pattrs_globals.serverfns || pmix_pattrs_globals.toolfns ||
+        pmix_pattrs_globals.client || pmix_pattrs_globals.server || pmix_pattrs_globals.tool) {
+        PMIX_INFO_CREATE(info, 1);
+        PMIX_INFO_LOAD(&info[0], PMIX_TOOL_DO_NOT_CONNECT, NULL, PMIX_BOOL);
+        rc = PMIx_tool_init(&myproc, info, 1);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "PMIx_tool_init failed: %s\n", PMIx_Error_string(rc));
+            exit(rc);
+        }
+        if (pmix_pattrs_globals.clientfns) {
+            pmix_register_client_attrs();
+            fns = pmix_attributes_print_functions(PMIX_CLIENT_FUNCTIONS);
+        } else if (pmix_pattrs_globals.serverfns) {
+            pmix_register_server_attrs();
+            fns = pmix_attributes_print_functions(PMIX_SERVER_FUNCTIONS);
+        } else if (pmix_pattrs_globals.toolfns) {
+            pmix_register_tool_attrs();
+            fns = pmix_attributes_print_functions(PMIX_TOOL_FUNCTIONS);
+        } else if (NULL != pmix_pattrs_globals.client) {
+            pmix_register_client_attrs();
+            fns = pmix_attributes_print_attr(PMIX_CLIENT_ATTRIBUTES, pmix_pattrs_globals.client);
+        } else if (NULL != pmix_pattrs_globals.server) {
+            pmix_register_server_attrs();
+            fns = pmix_attributes_print_attr(PMIX_SERVER_ATTRIBUTES, pmix_pattrs_globals.server);
+        } else if (NULL != pmix_pattrs_globals.tool) {
+            pmix_register_tool_attrs();
+            fns = pmix_attributes_print_attr(PMIX_TOOL_ATTRIBUTES, pmix_pattrs_globals.tool);
+        }
+        if (NULL != fns) {
+            for (n=0; NULL != fns[n]; n++) {
+                fprintf(stderr, "%s\n", fns[n]);
+            }
+        }
+        goto done;
+    }
+
+    /* if they didn't give us an option, then we can't do anything */
+    if (!pmix_pattrs_globals.hostfns && NULL == pmix_pattrs_globals.host) {
+        char *str, *args = NULL;
+        args = pmix_cmd_line_get_usage_msg(&cmd_line);
+        str = pmix_show_help_string("help-pattrs.txt", "usage", true,
+                                    args);
+        if (NULL != str) {
+            printf("%s", str);
+            free(str);
+        }
+        free(args);
+        exit(1);
+    }
+
+    /* if we were given the pid of a starter, then direct that
+     * we connect to it */
+    n = 1;
+    PMIX_INFO_CREATE(info, n);
+    if (0 < pmix_pattrs_globals.pid) {
+        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_PIDINFO, &pmix_pattrs_globals.pid, PMIX_PID);
+    } else if (NULL != pmix_pattrs_globals.nspace) {
+        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_NSPACE, pmix_pattrs_globals.nspace, PMIX_STRING);
+    } else if (NULL != pmix_pattrs_globals.uri) {
+        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_URI, pmix_pattrs_globals.uri, PMIX_STRING);
+    } else if (pmix_pattrs_globals.sysfirst) {
+        /* otherwise, use the system connection first, if available */
+        PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_SYSTEM_FIRST, NULL, PMIX_BOOL);
+    } else if (pmix_pattrs_globals.system) {
+        PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_TO_SYSTEM, NULL, PMIX_BOOL);
+    } else {
+        PMIX_INFO_FREE(info, 1);
+        n = 0;
+    }
+    /* init as a tool */
+    if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, info, n))) {
+        fprintf(stderr, "PMIx_tool_init failed: %s\n", PMIx_Error_string(rc));
+        exit(rc);
+    }
+    PMIX_INFO_FREE(info, 1);
+
+    /* register a default event handler */
+    PMIX_CONSTRUCT_LOCK(&mylock.lock);
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, evhandler_reg_callbk, (void*)&mylock);
+    PMIX_WAIT_THREAD(&mylock.lock);
+    if (PMIX_SUCCESS != mylock.status) {
+        fprintf(stderr, "PMIx_Register_event_handler returned bad status: %d\n", mylock.status);
+        PMIX_DESTRUCT_LOCK(&mylock.lock);
+        rc = mylock.status;
+        goto done;
+    }
+    PMIX_DESTRUCT_LOCK(&mylock.lock);
+
+    /* generate the query */
+    PMIX_QUERY_CONSTRUCT(&query);
+    pmix_argv_append_nosize(&query.keys, PMIX_QUERY_ATTRIBUTE_SUPPORT);
+    PMIX_QUERY_QUALIFIERS_CREATE(&query, 1);
+    if (pmix_pattrs_globals.hostfns) {
+        PMIX_INFO_LOAD(&query.qualifiers[0], PMIX_HOST_FUNCTIONS, NULL, PMIX_BOOL);
+    } else {
+        PMIX_INFO_LOAD(&query.qualifiers[0], PMIX_HOST_ATTRIBUTES, pmix_pattrs_globals.host, PMIX_STRING);
+    }
+    PMIX_CONSTRUCT_LOCK(&mq.lock);
+    rc = PMIx_Query_info_nb(&query, 1, cbfunc,(void*)&mq);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "PMIx_Query_info failed: %d\n", rc);
+        goto done;
+    }
+    PMIX_WAIT_THREAD(&mq.lock);
+    PMIX_DESTRUCT_LOCK(&mq.lock);
+    if (PMIX_SUCCESS != mq.status) {
+        fprintf(stderr, "PMIx_Query_info returned: %s\n", PMIx_Error_string(mq.status));
+        rc = mq.status;
+    } else {
+        /* print out the returned value(s) */
+        for (n=0; n < mq.ninfo; n++) {
+            if (PMIX_CHECK_KEY(&mq.info[n], PMIX_HOST_FUNCTIONS)) {
+                fns = pmix_argv_split(mq.info[n].value.data.string, ',');
+                fprintf(stderr, "HOST SUPPORTED FUNCTIONS:\n");
+                for (m=0; NULL != fns[m]; m++) {
+                    fprintf(stderr, "\t%s\n", fns[m]);
+                }
+                pmix_argv_free(fns);
+            } else {
+                pmix_attributes_print_headers(&ans, PMIX_HOST_ATTRIBUTES);
+                if (PMIX_DATA_ARRAY == mq.info[n].value.type) {
+                    info = (pmix_info_t*)mq.info[n].value.data.darray->array;
+                    for (m=0; m < mq.info[n].value.data.darray->size; m++) {
+                        reg = (pmix_regattr_t*)info[m].value.data.darray->array;
+                        pmix_attributes_print_attrs(&ans, info[m].key, reg, info[0].value.data.darray->size);
+                    }
+                } else {
+                    reg = (pmix_regattr_t*)mq.info[n].value.data.ptr;
+                    pmix_attributes_print_attrs(&ans, mq.info[n].key, reg, 1);
+                }
+                for (m=0; NULL != ans[m]; m++) {
+                    fprintf(stderr, "%s\n", ans[m]);
+                }
+                pmix_argv_free(ans);
+                ans = NULL;
+            }
+        }
+    }
+
+  done:
+    PMIx_tool_finalize();
+
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/src/tools/pevent/Makefile.am pmix-4.0.0/src/tools/pevent/Makefile.am
--- pmix-3.2.2~rc1/src/tools/pevent/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tools/pevent/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2017-2018 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,6 +19,22 @@
 # $HEADER$
 #
 
+AM_CFLAGS = \
+            -DPMIX_CONFIGURE_USER="\"@PMIX_CONFIGURE_USER@\"" \
+            -DPMIX_CONFIGURE_HOST="\"@PMIX_CONFIGURE_HOST@\"" \
+            -DPMIX_CONFIGURE_DATE="\"@PMIX_CONFIGURE_DATE@\"" \
+            -DPMIX_BUILD_USER="\"$$USER\"" \
+            -DPMIX_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPMIX_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
+            -DPMIX_BUILD_CFLAGS="\"@CFLAGS@\"" \
+            -DPMIX_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
+            -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
+            -DPMIX_BUILD_LIBS="\"@LIBS@\"" \
+            -DPMIX_CC_ABSOLUTE="\"@PMIX_CC_ABSOLUTE@\"" \
+            -DPMIX_GREEK_VERSION="\"@PMIX_GREEK_VERSION@\"" \
+            -DPMIX_REPO_REV="\"@PMIX_REPO_REV@\"" \
+            -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\""
+
 if PMIX_INSTALL_BINARIES
 
 bin_PROGRAMS = pevent
diff -Nru pmix-3.2.2~rc1/src/tools/plookup/Makefile.am pmix-4.0.0/src/tools/plookup/Makefile.am
--- pmix-3.2.2~rc1/src/tools/plookup/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tools/plookup/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2017-2018 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,6 +19,22 @@
 # $HEADER$
 #
 
+AM_CFLAGS = \
+            -DPMIX_CONFIGURE_USER="\"@PMIX_CONFIGURE_USER@\"" \
+            -DPMIX_CONFIGURE_HOST="\"@PMIX_CONFIGURE_HOST@\"" \
+            -DPMIX_CONFIGURE_DATE="\"@PMIX_CONFIGURE_DATE@\"" \
+            -DPMIX_BUILD_USER="\"$$USER\"" \
+            -DPMIX_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPMIX_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
+            -DPMIX_BUILD_CFLAGS="\"@CFLAGS@\"" \
+            -DPMIX_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
+            -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
+            -DPMIX_BUILD_LIBS="\"@LIBS@\"" \
+            -DPMIX_CC_ABSOLUTE="\"@PMIX_CC_ABSOLUTE@\"" \
+            -DPMIX_GREEK_VERSION="\"@PMIX_GREEK_VERSION@\"" \
+            -DPMIX_REPO_REV="\"@PMIX_REPO_REV@\"" \
+            -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\""
+
 if PMIX_INSTALL_BINARIES
 
 bin_PROGRAMS = plookup
diff -Nru pmix-3.2.2~rc1/src/tools/pmix_info/Makefile.am pmix-4.0.0/src/tools/pmix_info/Makefile.am
--- pmix-3.2.2~rc1/src/tools/pmix_info/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tools/pmix_info/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,22 +19,13 @@
 # $HEADER$
 #
 
-if SOURCE_DATE_EPOCH_SET
-  USER = @PMIX_CONFIGURE_USER@
-  PMIX_BUILD_HOST = @PMIX_CONFIGURE_HOST@
-  PMIX_BUILD_DATE = @PMIX_CONFIGURE_DATE@
-else
-  PMIX_BUILD_HOST = `(hostname || uname -n) 2> /dev/null | sed 1q`
-  PMIX_BUILD_DATE = `date +%Y-%m-%dT%H:%M:%S`
-endif
-
 AM_CFLAGS = \
             -DPMIX_CONFIGURE_USER="\"@PMIX_CONFIGURE_USER@\"" \
             -DPMIX_CONFIGURE_HOST="\"@PMIX_CONFIGURE_HOST@\"" \
             -DPMIX_CONFIGURE_DATE="\"@PMIX_CONFIGURE_DATE@\"" \
-            -DPMIX_BUILD_USER="\"$(USER)\"" \
-            -DPMIX_BUILD_HOST="\"$(PMIX_BUILD_HOST)\"" \
-            -DPMIX_BUILD_DATE="\"$(PMIX_BUILD_DATE)\"" \
+            -DPMIX_BUILD_USER="\"$$USER\"" \
+            -DPMIX_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPMIX_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
             -DPMIX_BUILD_CFLAGS="\"@CFLAGS@\"" \
             -DPMIX_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
             -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
diff -Nru pmix-3.2.2~rc1/src/tools/pmix_info/support.c pmix-4.0.0/src/tools/pmix_info/support.c
--- pmix-3.2.2~rc1/src/tools/pmix_info/support.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tools/pmix_info/support.c	2021-01-02 08:56:17.000000000 +0000
@@ -1327,11 +1327,13 @@
     char *debug;
     char *have_dl;
     char *symbol_visibility;
+    char *manpages;
 
     /* setup the strings that don't require allocations*/
     debug = PMIX_ENABLE_DEBUG ? "yes" : "no";
     have_dl = PMIX_HAVE_PDL_SUPPORT ? "yes" : "no";
     symbol_visibility = PMIX_HAVE_VISIBILITY ? "yes" : "no";
+    manpages = PMIX_ENABLE_MAN_PAGES ? "yes" : "no";
 
     /* output values */
     pmix_info_out("Configured by", "config:user", PMIX_CONFIGURE_USER);
@@ -1373,4 +1375,5 @@
     pmix_info_out("Internal debug support", "option:debug", debug);
     pmix_info_out("dl support", "option:dlopen", have_dl);
     pmix_info_out("Symbol vis. support", "options:visibility", symbol_visibility);
+    pmix_info_out("Manpages built", "options:man-pages", manpages);
 }
diff -Nru pmix-3.2.2~rc1/src/tools/pps/Makefile.am pmix-4.0.0/src/tools/pps/Makefile.am
--- pmix-3.2.2~rc1/src/tools/pps/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/tools/pps/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
-# Copyright (c) 2017-2018 Intel, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -19,6 +19,22 @@
 # $HEADER$
 #
 
+AM_CFLAGS = \
+            -DPMIX_CONFIGURE_USER="\"@PMIX_CONFIGURE_USER@\"" \
+            -DPMIX_CONFIGURE_HOST="\"@PMIX_CONFIGURE_HOST@\"" \
+            -DPMIX_CONFIGURE_DATE="\"@PMIX_CONFIGURE_DATE@\"" \
+            -DPMIX_BUILD_USER="\"$$USER\"" \
+            -DPMIX_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPMIX_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
+            -DPMIX_BUILD_CFLAGS="\"@CFLAGS@\"" \
+            -DPMIX_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
+            -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
+            -DPMIX_BUILD_LIBS="\"@LIBS@\"" \
+            -DPMIX_CC_ABSOLUTE="\"@PMIX_CC_ABSOLUTE@\"" \
+            -DPMIX_GREEK_VERSION="\"@PMIX_GREEK_VERSION@\"" \
+            -DPMIX_REPO_REV="\"@PMIX_REPO_REV@\"" \
+            -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\""
+
 if PMIX_INSTALL_BINARIES
 
 bin_PROGRAMS = pps
diff -Nru pmix-3.2.2~rc1/src/tools/pquery/help-pquery.txt pmix-4.0.0/src/tools/pquery/help-pquery.txt
--- pmix-3.2.2~rc1/src/tools/pquery/help-pquery.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/pquery/help-pquery.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,32 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for pquery
+#
+[usage]
+pquery [OPTIONS] KEY1[qualifiers] KEY2[qualifiers]...
+  PMIx Query tool
+
+%s
+
+Keys passed to pquery may optionally include one or more qualifiers, with the
+individual qualifiers delimited by semi-colons. For example:
+
+PMIX_STORAGE_XFER_SIZE[PMIX_STORAGE_ID=lustre1,lustre2;PMIX_STORAGE_PATH=foo]
diff -Nru pmix-3.2.2~rc1/src/tools/pquery/Makefile.am pmix-4.0.0/src/tools/pquery/Makefile.am
--- pmix-3.2.2~rc1/src/tools/pquery/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/pquery/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+# Copyright (c) 2017-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CFLAGS = \
+            -DPMIX_CONFIGURE_USER="\"@PMIX_CONFIGURE_USER@\"" \
+            -DPMIX_CONFIGURE_HOST="\"@PMIX_CONFIGURE_HOST@\"" \
+            -DPMIX_CONFIGURE_DATE="\"@PMIX_CONFIGURE_DATE@\"" \
+            -DPMIX_BUILD_USER="\"$$USER\"" \
+            -DPMIX_BUILD_HOST="\"$${HOSTNAME:-`(hostname || uname -n) | sed 1q`}\"" \
+            -DPMIX_BUILD_DATE="\"`$(top_srcdir)/config/getdate.sh`\"" \
+            -DPMIX_BUILD_CFLAGS="\"@CFLAGS@\"" \
+            -DPMIX_BUILD_CPPFLAGS="\"@CPPFLAGS@\"" \
+            -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \
+            -DPMIX_BUILD_LIBS="\"@LIBS@\"" \
+            -DPMIX_CC_ABSOLUTE="\"@PMIX_CC_ABSOLUTE@\"" \
+            -DPMIX_GREEK_VERSION="\"@PMIX_GREEK_VERSION@\"" \
+            -DPMIX_REPO_REV="\"@PMIX_REPO_REV@\"" \
+            -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\""
+
+if PMIX_INSTALL_BINARIES
+
+bin_PROGRAMS = pquery
+
+dist_pmixdata_DATA = help-pquery.txt
+
+endif # PMIX_INSTALL_BINARIES
+
+pquery_SOURCES = pquery.c
+pquery_LDADD = \
+    $(PMIX_EXTRA_LTLIB) \
+	$(top_builddir)/src/libpmix.la
diff -Nru pmix-3.2.2~rc1/src/tools/pquery/pquery.c pmix-4.0.0/src/tools/pquery/pquery.c
--- pmix-3.2.2~rc1/src/tools/pquery/pquery.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/pquery/pquery.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,540 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "pmix_config.h"
+#include "include/pmix_common.h"
+#include "include/pmix_server.h"
+#include "include/pmix.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <pthread.h>
+
+#include "include/pmix_tool.h"
+#include "src/common/pmix_attributes.h"
+#include "src/mca/base/base.h"
+#include "src/mca/pinstalldirs/base/base.h"
+#include "src/threads/threads.h"
+#include "src/util/cmd_line.h"
+#include "src/util/keyval_parse.h"
+#include "src/util/show_help.h"
+#include "src/runtime/pmix_rte.h"
+
+typedef struct {
+    pmix_lock_t lock;
+    pmix_status_t status;
+} mylock_t;
+
+static pmix_proc_t myproc;
+
+/* define a structure for collecting returned
+ * info from a query */
+typedef struct {
+    pmix_lock_t lock;
+    pmix_status_t status;
+    pmix_info_t *info;
+    size_t ninfo;
+} myquery_data_t;
+
+
+/* this is a callback function for the PMIx_Query
+ * API. The query will callback with a status indicating
+ * if the request could be fully satisfied, partially
+ * satisfied, or completely failed. The info parameter
+ * contains an array of the returned data, with the
+ * info->key field being the key that was provided in
+ * the query call. Thus, you can correlate the returned
+ * data in the info->value field to the requested key.
+ *
+ * Once we have dealt with the returned data, we must
+ * call the release_fn so that the PMIx library can
+ * cleanup */
+static void cbfunc(pmix_status_t status,
+                   pmix_info_t *info, size_t ninfo,
+                   void *cbdata,
+                   pmix_release_cbfunc_t release_fn,
+                   void *release_cbdata)
+{
+    myquery_data_t *mq = (myquery_data_t*)cbdata;
+    size_t n;
+
+    mq->status = status;
+    /* save the returned info - the PMIx library "owns" it
+     * and will release it and perform other cleanup actions
+     * when release_fn is called */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != release_fn) {
+        release_fn(release_cbdata);
+    }
+
+    /* release the block */
+    PMIX_WAKEUP_THREAD(&mq->lock);
+}
+
+/* this is the event notification function we pass down below
+ * when registering for general events - i.e.,, the default
+ * handler. We don't technically need to register one, but it
+ * is usually good practice to catch any events that occur */
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    /* this example doesn't do anything with default events */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+/* event handler registration is done asynchronously because it
+ * may involve the PMIx server registering with the host RM for
+ * external events. So we provide a callback function that returns
+ * the status of the request (success or an error), plus a numerical index
+ * to the registered event. The index is used later on to deregister
+ * an event handler - if we don't explicitly deregister it, then the
+ * PMIx server will do so when it see us exit */
+static void evhandler_reg_callbk(pmix_status_t status,
+                                 size_t evhandler_ref,
+                                 void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+
+    if (PMIX_SUCCESS != status) {
+        fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref);
+    }
+    lock->status = status;
+    PMIX_WAKEUP_THREAD(&lock->lock);
+}
+
+/*****************************************
+ * Global Vars for Command line Arguments
+ *****************************************/
+typedef struct {
+    bool help;
+    bool verbose;
+    pid_t pid;
+    char *nspace;
+    char *uri;
+    bool sysfirst;
+    bool system;
+    char *client;
+    char *server;
+    char *tool;
+    char *host;
+    bool clientfns;
+    bool serverfns;
+    bool toolfns;
+    bool hostfns;
+} pmix_pquery_globals_t;
+
+pmix_pquery_globals_t pmix_pquery_globals = {0};
+
+pmix_cmd_line_init_t cmd_line_opts[] = {
+    { NULL,
+      'h', NULL, "help",
+      0,
+      &pmix_pquery_globals.help, PMIX_CMD_LINE_TYPE_BOOL,
+      "This help message" },
+
+    { NULL,
+      'v', NULL, "verbose",
+      0,
+      &pmix_pquery_globals.verbose, PMIX_CMD_LINE_TYPE_BOOL,
+      "Be Verbose" },
+
+    { NULL,
+      'p', NULL, "pid",
+      1,
+      &pmix_pquery_globals.pid, PMIX_CMD_LINE_TYPE_INT,
+      "Specify server pid to connect to" },
+
+    { NULL,
+      'n', NULL, "nspace",
+      1,
+      &pmix_pquery_globals.nspace, PMIX_CMD_LINE_TYPE_STRING,
+      "Specify server nspace to connect to" },
+
+    { NULL,
+      '\0', NULL, "uri",
+      1,
+      &pmix_pquery_globals.uri, PMIX_CMD_LINE_TYPE_STRING,
+      "Specify URI of server to connect to" },
+
+    { NULL,
+      '\0', NULL, "system-server-first",
+      0,
+      &pmix_pquery_globals.sysfirst, PMIX_CMD_LINE_TYPE_BOOL,
+      "Look for the system server first" },
+
+    { NULL,
+      '\0', NULL, "system-server",
+      0,
+      &pmix_pquery_globals.system, PMIX_CMD_LINE_TYPE_BOOL,
+      "Specifically connect to the system server" },
+
+
+    /* End of list */
+    { NULL,
+      '\0', NULL, NULL,
+      0,
+      NULL, PMIX_CMD_LINE_TYPE_NULL,
+      NULL }
+};
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    pmix_info_t *info;
+    mylock_t mylock;
+    pmix_cmd_line_t cmd_line;
+    size_t n, m, k, nqueries, ninfo, ndarray;
+    myquery_data_t mq = {0};
+    int count;
+    char **qkeys = NULL;
+    const char *attr;
+    bool server = false;
+    pmix_list_t querylist, qlist;
+    pmix_querylist_t *qry;
+    char **qprs;
+    char *strt, *endp, *kptr;
+    pmix_infolist_t *iptr;
+    char *str, *args = NULL;
+    pmix_query_t *queries;
+    pmix_info_t *infoptr, *ifptr;
+    uint64_t u64;
+
+    /* protect against problems if someone passes us thru a pipe
+     * and then abnormally terminates the pipe early */
+    signal(SIGPIPE, SIG_IGN);
+
+    /* initialize the output system */
+    if (!pmix_output_init()) {
+        return PMIX_ERROR;
+    }
+
+    /* initialize install dirs code */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pinstalldirs_base_framework, 0))) {
+        fprintf(stderr, "pmix_pinstalldirs_base_open() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
+                __FILE__, __LINE__, rc);
+        return rc;
+    }
+    if (PMIX_SUCCESS != (rc = pmix_pinstall_dirs_base_init(NULL, 0))) {
+        fprintf(stderr, "pmix_pinstalldirs_base_init() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
+                __FILE__, __LINE__, rc);
+        return rc;
+    }
+
+    /* initialize the help system */
+    pmix_show_help_init();
+
+    /* keyval lex-based parser */
+    if (PMIX_SUCCESS != (rc = pmix_util_keyval_parse_init())) {
+        fprintf(stderr, "pmix_util_keyval_parse_init failed with %d\n", rc);
+        return PMIX_ERROR;
+    }
+
+    /* Setup the parameter system */
+    if (PMIX_SUCCESS != (rc = pmix_mca_base_var_init())) {
+        fprintf(stderr, "pmix_mca_base_var_init failed with %d\n", rc);
+        return PMIX_ERROR;
+    }
+
+    /* register params for pmix */
+    if (PMIX_SUCCESS != (rc = pmix_register_params())) {
+        fprintf(stderr, "pmix_register_params failed with %d\n", rc);
+        return PMIX_ERROR;
+    }
+
+    /* Parse the command line options */
+    pmix_cmd_line_create(&cmd_line, cmd_line_opts);
+
+    rc = pmix_cmd_line_parse(&cmd_line, true, false, argc, argv);
+
+    if (PMIX_SUCCESS != rc) {
+        if (PMIX_ERR_SILENT != rc) {
+            fprintf(stderr, "%s: command line error (%s)\n", argv[0],
+                    PMIx_Error_string(rc));
+        }
+        return rc;
+    }
+
+    if (pmix_pquery_globals.help) {
+        args = pmix_cmd_line_get_usage_msg(&cmd_line);
+        str = pmix_show_help_string("help-pquery.txt", "usage", true,
+                                    args);
+        if (NULL != str) {
+            printf("%s", str);
+            free(str);
+        }
+        free(args);
+        /* If we show the help message, that should be all we do */
+        exit(0);
+    }
+
+    /* get the argv array of keys they want us to query */
+    pmix_cmd_line_get_tail(&cmd_line, &count, &qkeys);
+
+    /* if they didn't give us an option, then we can't do anything */
+    if (NULL == qkeys) {
+        args = pmix_cmd_line_get_usage_msg(&cmd_line);
+        str = pmix_show_help_string("help-pquery.txt", "usage", true,
+                                    args);
+        if (NULL != str) {
+            printf("%s", str);
+            free(str);
+        }
+        free(args);
+        exit(1);
+    }
+
+    /* they might be querying the system about a client, server, or tool
+     * attribute, so register those - this will allow us to compare the
+     * provided key with the actual name of the attribute so we can
+     * identify it */
+    pmix_init_registered_attrs();
+    pmix_register_client_attrs();
+    pmix_register_server_attrs();
+    pmix_register_tool_attrs();
+
+    /* if we were given the pid of a starter, then direct that
+     * we connect to it */
+    n = 1;
+    PMIX_INFO_CREATE(info, n);
+    if (0 < pmix_pquery_globals.pid) {
+        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_PIDINFO, &pmix_pquery_globals.pid, PMIX_PID);
+    } else if (NULL != pmix_pquery_globals.nspace) {
+        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_NSPACE, pmix_pquery_globals.nspace, PMIX_STRING);
+    } else if (NULL != pmix_pquery_globals.uri) {
+        PMIX_INFO_LOAD(&info[0], PMIX_SERVER_URI, pmix_pquery_globals.uri, PMIX_STRING);
+    } else if (pmix_pquery_globals.sysfirst) {
+        /* otherwise, use the system connection first, if available */
+        PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_SYSTEM_FIRST, NULL, PMIX_BOOL);
+    } else if (pmix_pquery_globals.system) {
+        PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_TO_SYSTEM, NULL, PMIX_BOOL);
+    } else {
+        /* we set ourselves up as a server, but no connections required */
+        PMIX_INFO_LOAD(&info[0], PMIX_GDS_MODULE, "hash", PMIX_STRING);
+        server = true;;
+    }
+
+    if (server) {
+        /* initialize as a server so we can process the query ourselves */
+        rc = PMIx_server_init(NULL, info, n);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "PMIx_server_init failed: %s\n", PMIx_Error_string(rc));
+            exit(rc);
+        }
+    } else {
+        /* init as a tool */
+        rc = PMIx_tool_init(&myproc, info, n);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "PMIx_tool_init failed: %s\n", PMIx_Error_string(rc));
+            exit(rc);
+        }
+        PMIX_INFO_FREE(info, 1);
+    }
+
+    /* register a default event handler */
+    PMIX_CONSTRUCT_LOCK(&mylock.lock);
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, evhandler_reg_callbk, (void*)&mylock);
+    PMIX_WAIT_THREAD(&mylock.lock);
+    if (PMIX_SUCCESS != mylock.status) {
+        fprintf(stderr, "PMIx_Register_event_handler returned bad status: %d\n", mylock.status);
+        PMIX_DESTRUCT_LOCK(&mylock.lock);
+        rc = mylock.status;
+        goto done;
+    }
+    PMIX_DESTRUCT_LOCK(&mylock.lock);
+
+    /* generate the queries */
+    PMIX_CONSTRUCT(&querylist, pmix_list_t);
+    for (n=0; NULL != qkeys[n]; n++) {
+        qry = PMIX_NEW(pmix_querylist_t);
+        PMIX_CONSTRUCT(&qlist, pmix_list_t);
+        /* check for qualifiers: key[qual=foo;qual=bar] */
+        if (NULL != (strt = strchr(qkeys[n], '['))) {
+            /* we have qualifiers - find the end */
+            *strt = '\0';
+            ++strt;
+            if (NULL == (endp = strrchr(strt, ']'))) {
+                str = pmix_show_help_string("help-pquery.txt", "bad-quals", true, qkeys[n]);
+                if (NULL != str) {
+                    printf("%s", str);
+                    free(str);
+                }
+                exit(1);
+            }
+            *endp = '\0';
+            /* break into qual=val pairs */
+            qprs = pmix_argv_split(strt, ';');
+            for (m=0; NULL != qprs[m]; m++) {
+                /* break each pair */
+                if (NULL == (kptr = strchr(qprs[m], '='))) {
+                    str = pmix_show_help_string("help-pquery.txt", "bad-qual", true, qkeys[n], qprs[m]);
+                    if (NULL != str) {
+                        printf("%s", str);
+                        free(str);
+                    }
+                    exit(1);
+                }
+                *kptr = '\0';
+                kptr++;
+                iptr = PMIX_NEW(pmix_infolist_t);
+                if (NULL == (attr = pmix_attributes_lookup(qprs[m]))) {
+                    exit(1);
+                }
+                PMIX_INFO_LOAD(&iptr->info, attr, kptr, PMIX_STRING);
+                pmix_list_append(&qlist, &iptr->super);
+            }
+        }
+        /* convert the key */
+        if (NULL == (attr = pmix_attributes_lookup(qkeys[n]))) {
+            exit(1);
+        }
+        pmix_argv_append_nosize(&qry->query.keys, attr);
+        /* add in any qualifiers */
+        m = pmix_list_get_size(&qlist);
+        if (0 < m) {
+            PMIX_QUERY_QUALIFIERS_CREATE(&qry->query, m);
+            m = 0;
+            PMIX_LIST_FOREACH(iptr, &qlist, pmix_infolist_t) {
+                PMIX_INFO_XFER(&qry->query.qualifiers[m], &iptr->info);
+                ++m;
+            }
+        }
+        pmix_list_append(&querylist, &qry->super);
+        PMIX_LIST_DESTRUCT(&qlist);
+    }
+    /* convert the list of queries into an array */
+    nqueries = pmix_list_get_size(&querylist);
+    PMIX_QUERY_CREATE(queries, nqueries);
+    m = 0;
+    PMIX_LIST_FOREACH(qry, &querylist, pmix_querylist_t) {
+        /* move the queries across */
+        queries[m].keys = qry->query.keys;
+        queries[m].nqual = qry->query.nqual;
+        queries[m].qualifiers = qry->query.qualifiers;
+        ++m;
+    }
+    PMIX_LIST_DESTRUCT(&querylist);
+
+    PMIX_CONSTRUCT_LOCK(&mq.lock);
+    rc = PMIx_Query_info_nb(queries, nqueries, cbfunc,(void*)&mq);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "PMIx_Query_info failed: %d\n", rc);
+        goto done;
+    }
+    PMIX_WAIT_THREAD(&mq.lock);
+    PMIX_DESTRUCT_LOCK(&mq.lock);
+    if (PMIX_SUCCESS != mq.status) {
+        fprintf(stderr, "PMIx_Query_info returned: %s\n", PMIx_Error_string(mq.status));
+        rc = mq.status;
+    } else {
+        /* print out the returned value(s) */
+        for (n=0; n < mq.ninfo; n++) {
+            if (NULL == (attr = pmix_attributes_reverse_lookup(mq.info[n].key))) {
+                fprintf(stdout, "%s: ", mq.info[n].key);
+            } else {
+                fprintf(stdout, "%s: ", attr);
+            }
+            if (PMIX_STRING == mq.info[n].value.type) {
+                fprintf(stdout, "%s\n", mq.info[n].value.data.string);
+            } else if (PMIX_DATA_ARRAY ==  mq.info[n].value.type) {
+                fprintf(stdout, "\n");
+                infoptr = (pmix_info_t*)mq.info[n].value.data.darray->array;
+                ninfo = mq.info[n].value.data.darray->size;
+                for (m=0; m < ninfo; m++) {
+                    if (NULL == (attr = pmix_attributes_reverse_lookup(infoptr[m].key))) {
+                        fprintf(stdout, "\t%s:", infoptr[m].key);
+                    } else {
+                        fprintf(stdout, "\t%s:", attr);
+                    }
+                    if (PMIX_STRING == infoptr[m].value.type) {
+                        fprintf(stdout, "  %s\n", infoptr[m].value.data.string);
+                    } else if (PMIX_DATA_ARRAY == infoptr[m].value.type) {
+                        fprintf(stdout, "\n");
+                        ifptr = (pmix_info_t*)infoptr[m].value.data.darray->array;
+                        ndarray = infoptr[m].value.data.darray->size;
+                        for (k=0; k < ndarray; k++) {
+                            if (NULL == (attr = pmix_attributes_reverse_lookup(ifptr[k].key))) {
+                                fprintf(stdout, "\t\t%s:", ifptr[k].key);
+                            } else {
+                                fprintf(stdout, "\t\t%s:", attr);
+                            }
+                            if (PMIX_STRING == ifptr[k].value.type) {
+                                fprintf(stdout, "  %s\n", ifptr[k].value.data.string);
+                            } else if (PMIX_PROC_RANK == ifptr[k].value.type) {
+                                fprintf(stdout, "  %u\n", ifptr[k].value.data.rank);
+                            } else {
+                                /* see if it is a number */
+                                PMIX_VALUE_GET_NUMBER(rc, &ifptr[k].value, u64, uint64_t);
+                                if (PMIX_SUCCESS == rc) {
+                                    fprintf(stdout, "  %lu\n", (unsigned long)u64);
+                                } else {
+                                    fprintf(stdout, "  Unimplemented value type: %s\n", PMIx_Data_type_string(ifptr[k].value.type));
+                                }
+                            }
+                        }
+                    } else if (PMIX_PROC_RANK == infoptr[m].value.type) {
+                        fprintf(stdout, "  %u\n", infoptr[m].value.data.rank);
+                    } else {
+                        /* see if it is a number */
+                        PMIX_VALUE_GET_NUMBER(rc, &infoptr[m].value, u64, uint64_t);
+                        if (PMIX_SUCCESS == rc) {
+                            fprintf(stdout, "  %lu\n", (unsigned long)infoptr[m].value.data.uint64);
+                        } else {
+                            fprintf(stdout, "  Unimplemented value type: %s\n", PMIx_Data_type_string(infoptr[m].value.type));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+  done:
+    if (server) {
+        PMIx_server_finalize();
+    } else {
+        PMIx_tool_finalize();
+    }
+
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/src/tools/wrapper/.gitignore pmix-4.0.0/src/tools/wrapper/.gitignore
--- pmix-3.2.2~rc1/src/tools/wrapper/.gitignore	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/wrapper/.gitignore	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1 @@
+*.1
diff -Nru pmix-3.2.2~rc1/src/tools/wrapper/help-pmixcc.txt pmix-4.0.0/src/tools/wrapper/help-pmixcc.txt
--- pmix-3.2.2~rc1/src/tools/wrapper/help-pmixcc.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/wrapper/help-pmixcc.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,68 @@
+# -*- text -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2010 Oracle and/or its affiliates.  All rights reserved.
+# Copyright (c) 2012-2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# This is the US/English help file for Open MPI wrapper compiler error
+# messages.
+#
+[no-compiler-found]
+The Open MPI wrapper compiler was unable to find the specified compiler
+%s in your PATH.
+
+Note that this compiler was either specified at configure time or in
+one of several possible environment variables.
+#
+[version]
+%s: %s %s (Language: %s)
+#
+[usage]
+%s [-showme[:<command,compile,link,incdirs,
+              libdirs,libs,version,help>]] args
+
+  -showme:command    Show command used to invoke real compiler
+  -showme:compile    Show flags added when compiling
+  -showme:link       Show flags added when linking
+  -showme:incdirs    Show list of include dirs added when compiling
+  -showme:libdirs    Show list of library dirs added when linking
+  -showme:libs       Show list of libraries added when linking
+  -showme:version    Show version of %s
+  -showme:help       This help message
+#
+[file-not-found]
+%s could not find the file %s, needed for %s support.
+This may indicate an incomplete install and linking will likely fail.
+#
+[spawn-failed]
+Unable to call the compiler (%s). The failure return the error
+%s.
+The failed command was [%s].
+#
+[compiler-failed]
+The child process (%s) exit with error %d.
+The failed command was [%s].
+#
+[no-options-support]
+The %s wrapper compiler was unable to find a block of options in its
+configuration to support your compiler mode.  Your application was not
+compiled. or linked.
+
+The wrapper compiler command line was:
+
+  %s
diff -Nru pmix-3.2.2~rc1/src/tools/wrapper/Makefile.am pmix-4.0.0/src/tools/wrapper/Makefile.am
--- pmix-3.2.2~rc1/src/tools/wrapper/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/wrapper/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2006 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2006-2020 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
+# Copyright (c) 2014      Research Organization for Information Science
+#                         and Technology (RIST). All rights reserved.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
+# Copyright (c) 2020      IBM Corporation.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+include $(top_srcdir)/Makefile.pmix-rules
+
+MD_FILES = pmixcc.1.md
+man_pages_from_md = $(MD_FILES:.1.md=.1)
+EXTRA_DIST = $(MD_FILES) $(man_pages_from_md)
+AM_LDFLAGS = $(PMIX_EXTRA_LIB_LDFLAGS) $(pmix_hwloc_LDFLAGS) $(pmix_libevent_LDFLAGS)
+
+if PMIX_INSTALL_BINARIES
+
+bin_PROGRAMS = pmixcc
+
+if PMIX_ENABLE_MAN_PAGES
+_man_MANS = $(man_pages_from_md)
+endif # PMIX_ENABLE_MAN_PAGES
+
+nodist_pmixdata_DATA = pmixcc-wrapper-data.txt
+
+dist_pmixdata_DATA = help-pmixcc.txt
+
+pmixcc_SOURCES = \
+        pmixcc.c
+
+pmixcc_LDADD = \
+    $(PMIXEXTRA_LTLIB) \
+    $(pmix_libevent_LIBS) \
+    $(pmix_hwloc_LIBS) \
+	$(top_builddir)/src/libpmix.la
+
+endif # PMIX_INSTALL_BINARIES
diff -Nru pmix-3.2.2~rc1/src/tools/wrapper/pmixcc.1.md pmix-4.0.0/src/tools/wrapper/pmixcc.1.md
--- pmix-3.2.2~rc1/src/tools/wrapper/pmixcc.1.md	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/wrapper/pmixcc.1.md	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,32 @@
+# NAME
+
+pmix_wrapper - Back-end PMIx wrapper command
+
+# SYNOPSIS
+
+```
+pmix_wrapper [options]
+```
+
+# DESCRIPTION
+
+`pmix_wrapper` is not meant to be called directly by end users. It is
+automatically invoked as the back-end by the PMIx wrapper command
+`mpicc.`
+
+Some PMIx installations may have additional wrapper commands, and/or
+have renamed the wrapper compilers listed above to avoid executable name
+conflicts with other PMIx implementations.
+
+# SEE ALSO
+
+The following may exist depending on your particular PMIx installation:
+`pmixcc`(1) and the website at https://pmix.org/.
+
+# AUTHORS
+
+The PMIx maintainers -- see https://pmix.org/ or the file AUTHORS.
+
+This manual page was originally contributed by Dirk Eddelbuettel
+<edd@debian.org>, one of the Debian GNU/Linux maintainers for Open
+MPI, and may be used by others.
diff -Nru pmix-3.2.2~rc1/src/tools/wrapper/pmixcc.c pmix-4.0.0/src/tools/wrapper/pmixcc.c
--- pmix-3.2.2~rc1/src/tools/wrapper/pmixcc.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/wrapper/pmixcc.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1000 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+ * Copyright (c) 2018-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2020      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "pmix_config.h"
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif  /* HAVE_SYS_STAT_H */
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif  /* HAVE_SYS_TYPES_H */
+#ifdef HAVE_REGEX_H
+#include <regex.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif  /* HAVE_SYS_WAIT_H */
+#include <string.h>
+
+#include "src/mca/pinstalldirs/pinstalldirs.h"
+#include "src/mca/pinstalldirs/base/base.h"
+#include "src/runtime/pmix_rte.h"
+#include "include/pmix_common.h"
+#include "src/util/argv.h"
+#include "src/util/error.h"
+#include "src/util/keyval_parse.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/show_help.h"
+#include "src/util/path.h"
+#include "src/util/few.h"
+#include "src/util/basename.h"
+#include "src/util/os_path.h"
+#include "src/util/printf.h"
+
+#define PMIX_INCLUDE_FLAG  "-I"
+#define PMIX_LIBDIR_FLAG   "-L"
+
+struct options_data_t {
+    char **compiler_args;
+    char *language;
+    char *project;
+    char *project_short;
+    char *version;
+    char *compiler_env;
+    char *compiler_flags_env;
+    char *compiler;
+    char **preproc_flags;
+    char **comp_flags;
+    char **comp_flags_prefix;
+    char **link_flags;
+    char **libs;
+    char **libs_static;
+    char *dyn_lib_file;
+    char *static_lib_file;
+    char *req_file;
+    char *path_includedir;
+    char *path_libdir;
+    char *path_pmixincludedir;
+    char *path_pmixlibdir;
+};
+
+static struct options_data_t *options_data = NULL;
+/* index used by parser */
+static int parse_options_idx = -1;
+/* index of options specified by user */
+static int user_data_idx = -1;
+/* index of options to use by default */
+static int default_data_idx = -1;
+
+#define COMP_DRY_RUN       0x001
+#define COMP_SHOW_ERROR    0x002
+#define COMP_WANT_COMMAND  0x004
+#define COMP_WANT_PREPROC  0x008
+#define COMP_WANT_COMPILE  0x010
+#define COMP_WANT_LINK     0x020
+#define COMP_WANT_PMPI     0x040
+#define COMP_WANT_STATIC   0x080
+#define COMP_WANT_LINKALL  0x100
+
+static void
+options_data_init(struct options_data_t *data)
+{
+    data->compiler_args = (char **) malloc(sizeof(char*));
+    data->compiler_args[0] = NULL;
+    data->language = NULL;
+    data->compiler = NULL;
+    data->project = NULL;
+    data->project_short = NULL;
+    data->version = NULL;
+    data->compiler_env = NULL;
+    data->compiler_flags_env = NULL;
+    data->preproc_flags = (char **) malloc(sizeof(char*));
+    data->preproc_flags[0] = NULL;
+    data->comp_flags = (char **) malloc(sizeof(char*));
+    data->comp_flags[0] = NULL;
+    data->comp_flags_prefix = (char **) malloc(sizeof(char*));
+    data->comp_flags_prefix[0] = NULL;
+    data->link_flags = (char **) malloc(sizeof(char*));
+    data->link_flags[0] = NULL;
+    data->libs = (char **) malloc(sizeof(char*));
+    data->libs[0] = NULL;
+    data->libs_static = (char **) malloc(sizeof(char*));
+    data->libs_static[0] = NULL;
+    data->dyn_lib_file = NULL;
+    data->static_lib_file = NULL;
+    data->req_file = NULL;
+    data->path_includedir = NULL;
+    data->path_libdir = NULL;
+    data->path_pmixincludedir = NULL;
+    data->path_pmixlibdir = NULL;
+}
+
+static void
+options_data_free(struct options_data_t *data)
+{
+    if (NULL != data->compiler_args) {
+        pmix_argv_free(data->compiler_args);
+    }
+    if (NULL != data->language) free(data->language);
+    if (NULL != data->compiler) free(data->compiler);
+    if (NULL != data->project) free(data->project);
+    if (NULL != data->project_short) free(data->project_short);
+    if (NULL != data->version) free(data->version);
+    if (NULL != data->compiler_env) free(data->compiler_env);
+    if (NULL != data->compiler_flags_env) free(data->compiler_flags_env);
+    pmix_argv_free(data->preproc_flags);
+    pmix_argv_free(data->comp_flags);
+    pmix_argv_free(data->comp_flags_prefix);
+    pmix_argv_free(data->link_flags);
+    pmix_argv_free(data->libs);
+    pmix_argv_free(data->libs_static);
+    if (NULL != data->dyn_lib_file) free(data->dyn_lib_file);
+    if (NULL != data->static_lib_file) free(data->static_lib_file);
+    if (NULL != data->req_file) free(data->req_file);
+    if (NULL != data->path_includedir) free(data->path_includedir);
+    if (NULL != data->path_libdir) free(data->path_libdir);
+    if (NULL != data->path_pmixincludedir) free(data->path_pmixincludedir);
+    if (NULL != data->path_pmixlibdir) free(data->path_pmixlibdir);
+}
+
+static void
+options_data_expand(const char *value)
+{
+    /* make space for the new set of args */
+    parse_options_idx++;
+    options_data = (struct options_data_t *) realloc(options_data, sizeof(struct options_data_t) * (parse_options_idx + 1));
+    options_data_init(&(options_data[parse_options_idx]));
+
+    /* if there are values, this is not the default case.
+       Otherwise, it's the default case... */
+    if (NULL != value && 0 != strcmp(value, "")) {
+        char **values = pmix_argv_split(value, ';');
+        pmix_argv_insert(&(options_data[parse_options_idx].compiler_args),
+                         pmix_argv_count(options_data[parse_options_idx].compiler_args),
+                         values);
+        pmix_argv_free(values);
+    } else {
+        free(options_data[parse_options_idx].compiler_args);
+        options_data[parse_options_idx].compiler_args = NULL;
+        /* this is a default */
+        default_data_idx = parse_options_idx;
+    }
+}
+
+
+static int
+find_options_index(const char *arg)
+{
+    int i, j;
+#ifdef HAVE_REGEXEC
+    int args_count;
+    regex_t res;
+#endif
+
+    for (i = 0 ; i <= parse_options_idx ; ++i) {
+        if (NULL == options_data[i].compiler_args) {
+            continue;
+        }
+
+#ifdef HAVE_REGEXEC
+        args_count = pmix_argv_count(options_data[i].compiler_args);
+        for (j = 0 ; j < args_count ; ++j) {
+            if (0 != regcomp(&res, options_data[i].compiler_args[j], REG_NOSUB)) {
+                return -1;
+            }
+
+            if (0 == regexec(&res, arg, (size_t) 0, NULL, 0)) {
+                regfree(&res);
+                return i;
+            }
+
+            regfree(&res);
+        }
+#else
+        for (j = 0 ; j < pmix_argv_count(options_data[i].compiler_args) ; ++j) {
+            if (0 == strcmp(arg, options_data[i].compiler_args[j])) {
+                return i;
+            }
+        }
+#endif
+    }
+
+    return -1;
+}
+
+
+static void
+expand_flags(char **argv)
+{
+    int i;
+    char *tmp;
+
+    for (i = 0 ; argv[i] != NULL ; ++i) {
+        tmp = pmix_pinstall_dirs_expand(argv[i]);
+        if (tmp != argv[i]) {
+            free(argv[i]);
+            argv[i] = tmp;
+        }
+    }
+}
+
+
+static void
+data_callback(const char *key, const char *value)
+{
+    /* handle case where text file does not contain any special
+       compiler options field */
+    if (parse_options_idx < 0 && 0 != strcmp(key, "compiler_args")) {
+        options_data_expand(NULL);
+    }
+
+    if (0 == strcmp(key, "compiler_args")) {
+        options_data_expand(value);
+    } else if (0 == strcmp(key, "language")) {
+        if (NULL != value) options_data[parse_options_idx].language = strdup(value);
+    } else if (0 == strcmp(key, "compiler")) {
+        if (NULL != value) options_data[parse_options_idx].compiler = strdup(value);
+    } else if (0 == strcmp(key, "project")) {
+        if (NULL != value) options_data[parse_options_idx].project = strdup(value);
+    } else if (0 == strcmp(key, "version")) {
+        if (NULL != value) options_data[parse_options_idx].version = strdup(value);
+    } else if (0 == strcmp(key, "preprocessor_flags")) {
+        char **values = pmix_argv_split(value, ' ');
+        pmix_argv_insert(&options_data[parse_options_idx].preproc_flags,
+                         pmix_argv_count(options_data[parse_options_idx].preproc_flags),
+                         values);
+        expand_flags(options_data[parse_options_idx].preproc_flags);
+        pmix_argv_free(values);
+    } else if (0 == strcmp(key, "compiler_flags")) {
+        char **values = pmix_argv_split(value, ' ');
+        pmix_argv_insert(&options_data[parse_options_idx].comp_flags,
+                         pmix_argv_count(options_data[parse_options_idx].comp_flags),
+                         values);
+        expand_flags(options_data[parse_options_idx].comp_flags);
+        pmix_argv_free(values);
+    } else if (0 == strcmp(key, "compiler_flags_prefix")) {
+        char **values = pmix_argv_split(value, ' ');
+        pmix_argv_insert(&options_data[parse_options_idx].comp_flags_prefix,
+                         pmix_argv_count(options_data[parse_options_idx].comp_flags_prefix),
+                         values);
+        expand_flags(options_data[parse_options_idx].comp_flags_prefix);
+        pmix_argv_free(values);
+    } else if (0 == strcmp(key, "linker_flags")) {
+        char **values = pmix_argv_split(value, ' ');
+        pmix_argv_insert(&options_data[parse_options_idx].link_flags,
+                         pmix_argv_count(options_data[parse_options_idx].link_flags),
+                         values);
+        expand_flags(options_data[parse_options_idx].link_flags);
+        pmix_argv_free(values);
+    } else if (0 == strcmp(key, "libs")) {
+        char **values = pmix_argv_split(value, ' ');
+        pmix_argv_insert(&options_data[parse_options_idx].libs,
+                         pmix_argv_count(options_data[parse_options_idx].libs),
+                         values);
+        pmix_argv_free(values);
+    } else if (0 == strcmp(key, "libs_static")) {
+        char **values = pmix_argv_split(value, ' ');
+        pmix_argv_insert(&options_data[parse_options_idx].libs_static,
+                         pmix_argv_count(options_data[parse_options_idx].libs_static),
+                         values);
+        pmix_argv_free(values);
+    } else if (0 == strcmp(key, "dyn_lib_file")) {
+        if (NULL != value) options_data[parse_options_idx].dyn_lib_file = strdup(value);
+    } else if (0 == strcmp(key, "static_lib_file")) {
+        if (NULL != value) options_data[parse_options_idx].static_lib_file = strdup(value);
+    } else if (0 == strcmp(key, "required_file")) {
+        if (NULL != value) options_data[parse_options_idx].req_file = strdup(value);
+    } else if (0 == strcmp(key, "project_short")) {
+        if (NULL != value) options_data[parse_options_idx].project_short = strdup(value);
+    } else if (0 == strcmp(key, "compiler_env")) {
+        if (NULL != value) options_data[parse_options_idx].compiler_env = strdup(value);
+    } else if (0 == strcmp(key, "compiler_flags_env")) {
+        if (NULL != value) options_data[parse_options_idx].compiler_flags_env = strdup(value);
+    } else if (0 == strcmp(key, "includedir")) {
+        if (NULL != value) {
+            options_data[parse_options_idx].path_includedir =
+                pmix_pinstall_dirs_expand(value);
+            if (0 != strcmp(options_data[parse_options_idx].path_includedir, "/usr/include") ||
+                0 == strncmp(options_data[parse_options_idx].language, "Fortran", strlen("Fortran"))) {
+                char *line;
+                pmix_asprintf(&line, PMIX_INCLUDE_FLAG"%s",
+                         options_data[parse_options_idx].path_includedir);
+                pmix_argv_append_unique_nosize(&options_data[parse_options_idx].preproc_flags, line);
+                free(line);
+            }
+        }
+    } else if (0 == strcmp(key, "libdir")) {
+        if (NULL != value) options_data[parse_options_idx].path_libdir =
+                               pmix_pinstall_dirs_expand(value);
+        if (0 != strcmp(options_data[parse_options_idx].path_libdir, "/usr/lib")) {
+            char *line;
+            pmix_asprintf(&line, PMIX_LIBDIR_FLAG"%s",
+                     options_data[parse_options_idx].path_libdir);
+            pmix_argv_append_unique_nosize(&options_data[parse_options_idx].link_flags, line);
+            free(line);
+        }
+    } else if (0 == strcmp(key, "pmixincludedir")) {
+        printf("EXPANDING!\n");
+        if (NULL != value) {
+            options_data[parse_options_idx].path_pmixincludedir =
+                pmix_pinstall_dirs_expand(value);
+            if (0 != strcmp(options_data[parse_options_idx].path_pmixincludedir, "/usr/include") ||
+                0 == strncmp(options_data[parse_options_idx].language, "Fortran", strlen("Fortran"))) {
+                char *line;
+                pmix_asprintf(&line, PMIX_INCLUDE_FLAG"%s",
+                         options_data[parse_options_idx].path_pmixincludedir);
+                pmix_argv_append_unique_nosize(&options_data[parse_options_idx].preproc_flags, line);
+                free(line);
+            }
+        }
+    } else if (0 == strcmp(key, "pmixlibdir")) {
+        if (NULL != value) options_data[parse_options_idx].path_pmixlibdir =
+                               pmix_pinstall_dirs_expand(value);
+        if (0 != strcmp(options_data[parse_options_idx].path_pmixlibdir, "/usr/lib")) {
+            char *line;
+            pmix_asprintf(&line, PMIX_LIBDIR_FLAG"%s",
+                     options_data[parse_options_idx].path_pmixlibdir);
+            pmix_argv_append_unique_nosize(&options_data[parse_options_idx].link_flags, line);
+            free(line);
+        }
+    }
+}
+
+
+static int
+data_init(const char *appname)
+{
+    int ret;
+    char *datafile;
+
+    /* now load the data */
+    pmix_asprintf(&datafile, "%s%s%s-wrapper-data.txt",
+             pmix_pinstall_dirs.pmixdatadir, PMIX_PATH_SEP, appname);
+    if (NULL == datafile) return PMIX_ERR_OUT_OF_RESOURCE;
+
+    ret = pmix_util_keyval_parse(datafile, data_callback);
+    if( PMIX_SUCCESS != ret ) {
+        fprintf(stderr, "Cannot open configuration file %s\n", datafile );
+    }
+    free(datafile);
+
+    return ret;
+}
+
+
+static int
+data_finalize(void)
+{
+    int i;
+
+    for (i = 0 ; i <= parse_options_idx ; ++i) {
+        options_data_free(&(options_data[i]));
+    }
+    free(options_data);
+
+    return PMIX_SUCCESS;
+}
+
+
+static void
+print_flags(char **args, char *pattern)
+{
+    int i;
+    bool found = false;
+
+    for (i = 0 ; args[i] != NULL ; ++i) {
+        if (0 == strncmp(args[i], pattern, strlen(pattern))) {
+            if (found)  printf(" ");
+            printf("%s", args[i] + strlen(pattern));
+            found = true;
+        }
+    }
+
+    if (found) printf("\n");
+}
+
+
+static void
+load_env_data(const char *project, const char *flag, char **data)
+{
+    char *envname;
+    char *envvalue;
+
+    if (NULL == project || NULL == flag) return;
+
+    pmix_asprintf(&envname, "%s_MPI%s", project, flag);
+    if (NULL == (envvalue = getenv(envname))) {
+        free(envname);
+        pmix_asprintf(&envname, "%s_%s", project, flag);
+        if (NULL == (envvalue = getenv(envname))) {
+            free(envname);
+            return;
+        }
+    }
+    free(envname);
+
+    if (NULL != *data) free(*data);
+    *data = strdup(envvalue);
+}
+
+
+static void
+load_env_data_argv(const char *project, const char *flag, char ***data)
+{
+    char *envname;
+    char *envvalue;
+
+    if (NULL == project || NULL == flag) return;
+
+    pmix_asprintf(&envname, "%s_MPI%s", project, flag);
+    if (NULL == (envvalue = getenv(envname))) {
+        free(envname);
+        pmix_asprintf(&envname, "%s_%s", project, flag);
+        if (NULL == (envvalue = getenv(envname))) {
+            free(envname);
+            return;
+        }
+    }
+    free(envname);
+
+    if (NULL != *data) pmix_argv_free(*data);
+
+    *data = pmix_argv_split(envvalue, ' ');
+}
+
+
+int
+main(int argc, char *argv[])
+{
+    int exit_status = 0, ret, flags = 0, i;
+    int exec_argc = 0, user_argc = 0;
+    char **exec_argv = NULL, **user_argv = NULL;
+    char *exec_command, *base_argv0 = NULL;
+    bool disable_flags = true;
+    bool real_flag = false;
+
+    /* initialize the output system */
+    if (!pmix_output_init()) {
+        return PMIX_ERROR;
+    }
+
+    /* initialize install dirs code */
+    if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pinstalldirs_base_framework, 0))) {
+        fprintf(stderr, "pmix_pinstalldirs_base_open() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
+                __FILE__, __LINE__, ret);
+        return ret;
+    }
+
+    if (PMIX_SUCCESS != (ret = pmix_pinstall_dirs_base_init(NULL, 0))) {
+        fprintf(stderr, "pmix_pinstalldirs_base_init() failed -- process will likely abort (%s:%d, returned %d instead of PMIX_SUCCESS)\n",
+                __FILE__, __LINE__, ret);
+        return ret;
+    }
+
+    /* initialize the help system */
+    pmix_show_help_init();
+
+    /* keyval lex-based parser */
+    if (PMIX_SUCCESS != (ret = pmix_util_keyval_parse_init())) {
+        pmix_show_help( "help-pmix-runtime.txt",
+                        "pmix_init:startup:internal-failure", true,
+                        "pmix_util_keyval_parse_init", ret );
+        return ret;
+    }
+
+    /* initialize the mca */
+    if (PMIX_SUCCESS != (ret = pmix_mca_base_open())) {
+        pmix_show_help( "help-pmix-runtime.txt",
+                        "pmix_init:startup:internal-failure", true,
+                        "pmix_mca_base_open", ret );
+        return ret;
+    }
+
+    /****************************************************
+     *
+     * Setup compiler information
+     *
+     ****************************************************/
+
+    base_argv0 = pmix_basename(argv[0]);
+#if defined(EXEEXT)
+    if( 0 != strlen(EXEEXT) ) {
+        char extension[] = EXEEXT;
+        char* temp = strstr( base_argv0, extension );
+        char* old_match = temp;
+        while( NULL != temp ) {
+            old_match = temp;
+            temp = strstr( temp + 1, extension );
+        }
+        /* Only if there was a match of .exe, erase the last occurence of .exe */
+        if ( NULL != old_match ) {
+            *old_match = '\0';
+        }
+    }
+#endif  /* defined(EXEEXT) */
+
+    if (PMIX_SUCCESS != (ret = data_init(base_argv0))) {
+        fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, PMIx_Error_string(ret));
+        return ret;
+    }
+
+    for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
+        user_data_idx = find_options_index(argv[i]);
+    }
+    /* if we didn't find a match, look for the NULL (base case) options */
+    if (user_data_idx < 0) {
+        user_data_idx = default_data_idx;
+    }
+    /* if we still didn't find a match, abort */
+    if (user_data_idx < 0) {
+        char *flat = pmix_argv_join(argv, ' ');
+        pmix_show_help("help-pmixcc.txt", "no-options-support", true,
+                       base_argv0, flat, NULL);
+        free(flat);
+        exit(1);
+    }
+
+    /* compiler */
+    load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
+
+    /* preprocessor flags */
+    load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
+
+    /* compiler flags */
+    load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
+                       &options_data[user_data_idx].comp_flags);
+
+    /* linker flags */
+    load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
+
+    /* libs */
+    load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
+
+
+    /****************************************************
+     *
+     * Sanity Checks
+     *
+     ****************************************************/
+
+    if (NULL != options_data[user_data_idx].req_file) {
+        /* make sure the language is supported */
+        if (0 == strcmp(options_data[user_data_idx].req_file, "not supported")) {
+            pmix_show_help("help-pmixcc.txt", "no-language-support", true,
+                           options_data[user_data_idx].language, base_argv0, NULL);
+            exit_status = 1;
+            goto cleanup;
+        }
+
+        if (options_data[user_data_idx].req_file[0] != '\0') {
+            char *filename;
+            struct stat buf;
+            filename = pmix_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
+            if (0 != stat(filename, &buf)) {
+                pmix_show_help("help-pmixcc.txt", "file-not-found", true,
+                               base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
+            }
+        }
+    }
+
+    /****************************************************
+     *
+     * Parse user flags
+     *
+     ****************************************************/
+    flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
+        COMP_WANT_COMPILE|COMP_WANT_LINK;
+
+    user_argv = pmix_argv_copy(argv + 1);
+    user_argc = pmix_argv_count(user_argv);
+
+    for (i = 0 ; i < user_argc ; ++i) {
+        if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
+            0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
+            0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
+            0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
+            bool done_now = false;
+
+            /* check for specific things we want to see.  First three
+               still invoke all the building routines.  Last set want
+               to parse out certain flags, so we don't go through the
+               normal build routine - skip to cleanup. */
+            if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
+                0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
+                flags = COMP_WANT_COMMAND;
+                /* we know what we want, so don't process any more args */
+                done_now = true;
+            } else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
+                0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
+                flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
+                /* we know what we want, so don't process any more args */
+                done_now = true;
+            } else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
+                       0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
+                flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
+                /* we know what we want, so don't process any more args */
+                done_now = true;
+            } else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
+                       0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
+                print_flags(options_data[user_data_idx].preproc_flags, PMIX_INCLUDE_FLAG);
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
+                       0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
+                print_flags(options_data[user_data_idx].link_flags, PMIX_LIBDIR_FLAG);
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
+                       0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
+                print_flags(options_data[user_data_idx].libs, "-l");
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
+                       0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
+                char * str;
+                str = pmix_show_help_string("help-pmixcc.txt",
+                                            "version", false,
+                                            argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
+                if (NULL != str) {
+                    printf("%s", str);
+                    free(str);
+                }
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:help", strlen("-showme:help")) ||
+                       0 == strncmp(user_argv[i], "--showme:help", strlen("--showme:help"))) {
+                char *str;
+                str = pmix_show_help_string("help-pmixcc.txt", "usage",
+                                            false, argv[0],
+                                            options_data[user_data_idx].project,
+                                            NULL);
+                if (NULL != str) {
+                    printf("%s", str);
+                    free(str);
+                }
+
+                exit_status = 0;
+                goto cleanup;
+            } else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
+                       0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
+                fprintf(stderr, "%s: unrecognized option: %s\n", argv[0],
+                        user_argv[i]);
+                fprintf(stderr, "Type '%s --showme:help' for usage.\n",
+                        argv[0]);
+                exit_status = 1;
+                goto cleanup;
+            }
+
+            flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
+            /* remove element from user_argv */
+            pmix_argv_delete(&user_argc, &user_argv, i, 1);
+            --i;
+
+            if (done_now) {
+                disable_flags = false;
+                break;
+            }
+
+        } else if (0 == strcmp(user_argv[i], "-c")) {
+            flags &= ~COMP_WANT_LINK;
+            real_flag = true;
+        } else if (0 == strcmp(user_argv[i], "-E") ||
+                   0 == strcmp(user_argv[i], "-M")) {
+            flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
+            real_flag = true;
+        } else if (0 == strcmp(user_argv[i], "-S")) {
+            flags &= ~COMP_WANT_LINK;
+            real_flag = true;
+        } else if (0 == strcmp(user_argv[i], "-lpmpi")) {
+            flags |= COMP_WANT_PMPI;
+
+            /* remove element from user_argv */
+            pmix_argv_delete(&user_argc, &user_argv, i, 1);
+            --i;
+        } else if (0 == strcmp(user_argv[i], "-static") ||
+                   0 == strcmp(user_argv[i], "--static") ||
+                   0 == strcmp(user_argv[i], "-Bstatic") ||
+                   0 == strcmp(user_argv[i], "-Wl,-static") ||
+                   0 == strcmp(user_argv[i], "-Wl,--static") ||
+                   0 == strcmp(user_argv[i], "-Wl,-Bstatic")) {
+            flags |= COMP_WANT_STATIC;
+        } else if (0 == strcmp(user_argv[i], "-dynamic") ||
+                   0 == strcmp(user_argv[i], "--dynamic") ||
+                   0 == strcmp(user_argv[i], "-Bdynamic") ||
+                   0 == strcmp(user_argv[i], "-Wl,-dynamic") ||
+                   0 == strcmp(user_argv[i], "-Wl,--dynamic") ||
+                   0 == strcmp(user_argv[i], "-Wl,-Bdynamic")) {
+            flags &= ~COMP_WANT_STATIC;
+        } else if (0 == strcmp(user_argv[i], "--openmpi:linkall")) {
+            /* This is an intentionally undocummented wrapper compiler
+               switch.  It should only be used by Open MPI developers
+               -- not end users.  It will cause mpicc to use the
+               static library list, even if we're compiling
+               dynamically (i.e., it'll specifically -lopen-rte and
+               -lopen-pal (and all their dependent libs)).  We provide
+               this flag for test MPI applications that also invoke
+               ORTE and/or PMIX function calls.
+
+               On some systems (e.g., OS X), if the top-level
+               application calls ORTE/PMIX functions and you don't -l
+               ORTE and PMIX, then the functions won't be resolved at
+               link time (i.e., the implicit library dependencies of
+               libmpi won't be pulled in at link time), and therefore
+               the link will fail.  This flag will cause the wrapper
+               to explicitly list the ORTE and PMIX libs on the
+               underlying compiler command line, so the application
+               will therefore link properly. */
+            flags |= COMP_WANT_LINKALL;
+
+            /* remove element from user_argv */
+            pmix_argv_delete(&user_argc, &user_argv, i, 1);
+        } else if ('-' != user_argv[i][0]) {
+            disable_flags = false;
+            flags |= COMP_SHOW_ERROR;
+            real_flag = true;
+        } else {
+            /* if the option flag is one that we use to determine
+               which set of compiler data to use, don't count it as a
+               real option */
+            if (find_options_index(user_argv[i]) < 0) {
+                real_flag = true;
+            }
+        }
+    }
+
+    /* clear out the want_flags if we got no arguments not starting
+       with a - (dash) and -showme wasn't given OR -showme was given
+       and we had at least one more non-showme argument that started
+       with a - (dash) and no other non-dash arguments.  Some examples:
+
+       pmix_wrapper                : clear our flags
+       pmix_wrapper -v             : clear our flags
+       pmix_wrapper -E a.c         : don't clear our flags
+       pmix_wrapper a.c            : don't clear our flags
+       pmix_wrapper -showme        : don't clear our flags
+       pmix_wrapper -showme -v     : clear our flags
+       pmix_wrapper -showme -E a.c : don't clear our flags
+       pmix_wrapper -showme a.c    : don't clear our flags
+    */
+    if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
+        flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
+    }
+
+    /****************************************************
+     *
+     * Assemble the command line
+     *
+     ****************************************************/
+
+    /* compiler (may be multiple arguments, so split) */
+    if (flags & COMP_WANT_COMMAND) {
+        exec_argv = pmix_argv_split(options_data[user_data_idx].compiler, ' ');
+        exec_argc = pmix_argv_count(exec_argv);
+    } else {
+        exec_argv = (char **) malloc(sizeof(char*));
+        exec_argv[0] = NULL;
+        exec_argc = 0;
+    }
+
+    /* This error would normally not happen unless the user edits the
+       wrapper data files manually */
+    if (NULL == exec_argv) {
+        pmix_show_help("help-pmixcc.txt", "no-compiler-specified", true);
+        return 1;
+    }
+
+    if (flags & COMP_WANT_COMPILE) {
+        pmix_argv_insert(&exec_argv, exec_argc,
+                         options_data[user_data_idx].comp_flags_prefix);
+        exec_argc = pmix_argv_count(exec_argv);
+    }
+
+    /* Per https://svn.open-mpi.org/trac/ompi/ticket/2201, add all the
+       user arguments before anything else. */
+    pmix_argv_insert(&exec_argv, exec_argc, user_argv);
+    exec_argc = pmix_argv_count(exec_argv);
+
+    /* preproc flags */
+    if (flags & COMP_WANT_PREPROC) {
+        pmix_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].preproc_flags);
+        exec_argc = pmix_argv_count(exec_argv);
+    }
+
+    /* compiler flags */
+    if (flags & COMP_WANT_COMPILE) {
+        pmix_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags);
+        exec_argc = pmix_argv_count(exec_argv);
+    }
+
+    /* link flags and libs */
+    if (flags & COMP_WANT_LINK) {
+        bool have_static_lib;
+        bool have_dyn_lib;
+        bool use_static_libs;
+        char *filename1, *filename2;
+        struct stat buf;
+
+        pmix_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
+        exec_argc = pmix_argv_count(exec_argv);
+
+        /* Are we linking statically?  If so, decide what libraries to
+           list.  It depends on two factors:
+
+           1. Was --static (etc.) specified?
+           2. Does OMPI have static, dynamic, or both libraries installed?
+
+           Here's a matrix showing what we'll do in all 6 cases:
+
+           What's installed    --static    no --static
+           ----------------    ----------  -----------
+           ompi .so libs       -lmpi       -lmpi
+           ompi .a libs        all         all
+           ompi both libs      all         -lmpi
+
+        */
+
+        filename1 = pmix_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].static_lib_file, NULL );
+        if (0 == stat(filename1, &buf)) {
+            have_static_lib = true;
+        } else {
+            have_static_lib = false;
+        }
+
+        filename2 = pmix_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].dyn_lib_file, NULL );
+        if (0 == stat(filename2, &buf)) {
+            have_dyn_lib = true;
+        } else {
+            have_dyn_lib = false;
+        }
+
+        /* Determine which set of libs to use: dynamic or static.  Be
+           pedantic to make the code easy to read. */
+        if (flags & COMP_WANT_LINKALL) {
+            /* If --openmpi:linkall was specified, list all the libs
+               (i.e., the static libs) if they're available, either in
+               static or dynamic form. */
+            if (have_static_lib || have_dyn_lib) {
+                use_static_libs = true;
+            } else {
+                fprintf(stderr, "The linkall option has failed as we were unable to find either static or dynamic libs\n"
+                        "Files looked for:\n  Static: %s\n  Dynamic: %s\n",
+                        filename1, filename2);
+                free(filename1);
+                free(filename2);
+                exit(1);
+            }
+        } else if (flags & COMP_WANT_STATIC) {
+            /* If --static (or something like it) was specified, if we
+               have the static libs, then use them.  Otherwise, use
+               the dynamic libs. */
+            if (have_static_lib) {
+                use_static_libs = true;
+            } else {
+                use_static_libs = false;
+            }
+        } else {
+            /* If --static (or something like it) was NOT specified
+               (or if --dyanic, or something like it, was specified),
+               if we have the dynamic libs, then use them.  Otherwise,
+               use the static libs. */
+            if (have_dyn_lib) {
+                use_static_libs = false;
+            } else {
+                use_static_libs = true;
+            }
+        }
+        free(filename1);
+        free(filename2);
+
+        if (use_static_libs) {
+            pmix_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs_static);
+        } else {
+            pmix_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
+        }
+        exec_argc = pmix_argv_count(exec_argv);
+    }
+
+
+    /****************************************************
+     *
+     * Execute the command
+     *
+     ****************************************************/
+
+    if (flags & COMP_DRY_RUN) {
+        exec_command = pmix_argv_join(exec_argv, ' ');
+        printf("%s\n", exec_command);
+    } else {
+        char *tmp;
+
+#if 0
+        exec_command = pmix_argv_join(exec_argv, ' ');
+        printf("command: %s\n", exec_command);
+#endif
+
+        tmp = pmix_path_findv(exec_argv[0], 0, environ, NULL);
+        if (NULL == tmp) {
+            pmix_show_help("help-pmixcc.txt", "no-compiler-found", true,
+                           exec_argv[0], NULL);
+            errno = 0;
+            exit_status = 1;
+        }  else {
+            int status;
+
+            free(exec_argv[0]);
+            exec_argv[0] = tmp;
+            ret = pmix_few(exec_argv, &status);
+            exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
+                              (WIFSIGNALED(status) ? WTERMSIG(status) :
+                                  (WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
+            if( (PMIX_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
+                char* exec_command = pmix_argv_join(exec_argv, ' ');
+                if( PMIX_SUCCESS != ret ) {
+                    pmix_show_help("help-pmixcc.txt", "spawn-failed", true,
+                                   exec_argv[0], strerror(status), exec_command, NULL);
+                } else {
+#if 0
+                    pmix_show_help("help-pmixcc.txt", "compiler-failed", true,
+                                   exec_argv[0], exit_status, exec_command, NULL);
+#endif
+                }
+                free(exec_command);
+            }
+        }
+    }
+
+    /****************************************************
+     *
+     * Cleanup
+     *
+     ****************************************************/
+ cleanup:
+
+    pmix_argv_free(exec_argv);
+    pmix_argv_free(user_argv);
+    if (NULL != base_argv0) free(base_argv0);
+
+    if (PMIX_SUCCESS != (ret = data_finalize())) {
+        return ret;
+    }
+
+    /* keyval lex-based parser */
+    pmix_util_keyval_parse_finalize();
+
+    (void)pmix_mca_base_framework_close(&pmix_pinstalldirs_base_framework);
+    pmix_mca_base_close();
+    /* finalize the show_help system */
+    pmix_show_help_finalize();
+
+    /* finalize the output system.  This has to come *after* the
+       malloc code, as the malloc code needs to call into this, but
+       the malloc code turning off doesn't affect pmix_output that
+       much */
+    pmix_output_finalize();
+
+    return exit_status;
+}
diff -Nru pmix-3.2.2~rc1/src/tools/wrapper/pmixcc-wrapper-data.txt.in pmix-4.0.0/src/tools/wrapper/pmixcc-wrapper-data.txt.in
--- pmix-3.2.2~rc1/src/tools/wrapper/pmixcc-wrapper-data.txt.in	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/src/tools/wrapper/pmixcc-wrapper-data.txt.in	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,25 @@
+# There can be multiple blocks of configuration data, chosen by
+# compiler flags (using the compiler_args key to chose which block
+# should be activated.  This can be useful for multilib builds.  See the
+# multilib page at:
+#    https://github.com/open-mpi/ompi/wiki/compilerwrapper3264
+# for more information.
+
+project=PMI-Exascale (PMIx)
+project_short=PMIx
+version=@PMIX_VERSION@
+language=C
+compiler_env=CC
+compiler_flags_env=CFLAGS
+compiler=@WRAPPER_CC@
+preprocessor_flags=@PMIX_WRAPPER_EXTRA_CPPFLAGS@
+compiler_flags_prefix=@PMIX_WRAPPER_EXTRA_CFLAGS_PREFIX@
+compiler_flags=@PMIX_WRAPPER_EXTRA_CFLAGS@
+linker_flags=@PMIX_WRAPPER_EXTRA_LDFLAGS@
+libs=-lpmix
+libs_static=-lpmix @PMIX_WRAPPER_EXTRA_LIBS@
+dyn_lib_file=libpmix.@PMIX_DYN_LIB_SUFFIX@
+static_lib_file=libpmix.a
+required_file=
+includedir=${includedir}
+libdir=${libdir}
diff -Nru pmix-3.2.2~rc1/src/util/argv.c pmix-4.0.0/src/util/argv.c
--- pmix-3.2.2~rc1/src/util/argv.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/argv.c	2021-01-02 08:56:17.000000000 +0000
@@ -156,7 +156,7 @@
     return PMIX_SUCCESS;
 }
 
-pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite)
+pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg)
 {
     int i;
 
@@ -170,11 +170,7 @@
     /* see if this arg is already present in the array */
     for (i=0; NULL != (*argv)[i]; i++) {
         if (0 == strcmp(arg, (*argv)[i])) {
-            /* already exists - are we authorized to overwrite? */
-            if (overwrite) {
-                free((*argv)[i]);
-                (*argv)[i] = strdup(arg);
-            }
+            /* already exists */
             return PMIX_SUCCESS;
         }
     }
diff -Nru pmix-3.2.2~rc1/src/util/argv.h pmix-4.0.0/src/util/argv.h
--- pmix-3.2.2~rc1/src/util/argv.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/argv.h	2021-01-02 08:56:17.000000000 +0000
@@ -110,7 +110,6 @@
  *
  * @param argv Pointer to an argv array.
  * @param str Pointer to the string to append.
- * @param bool Whether or not to overwrite a matching value if found
  *
  * @retval PMIX_SUCCESS On success
  * @retval PMIX_ERROR On failure
@@ -119,7 +118,7 @@
  * except that it only appends the provided argument if it does not already
  * exist in the provided array.
  */
-PMIX_EXPORT pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite);
+PMIX_EXPORT pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg);
 
 /**
  * Append to an argv-style array, but only if the provided argument
diff -Nru pmix-3.2.2~rc1/src/util/error.c pmix-4.0.0/src/util/error.c
--- pmix-3.2.2~rc1/src/util/error.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/error.c	2021-01-02 08:56:17.000000000 +0000
@@ -143,6 +143,8 @@
         return "NOT-FOUND";
     case PMIX_ERR_NOT_SUPPORTED:
         return "NOT-SUPPORTED";
+    case PMIX_ERR_PARAM_VALUE_NOT_SUPPORTED:
+        return "PARAM-VALUE-NOT-SUPPORTED";
     case PMIX_ERR_NOT_IMPLEMENTED:
         return "NOT-IMPLEMENTED";
     case PMIX_ERR_COMM_FAILURE:
@@ -151,13 +153,23 @@
         return "UNPACK-PAST-END";
     case PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES:
         return "PMIX CONFLICTING CLEANUP DIRECTIVES";
-
-    case PMIX_ERR_LOST_CONNECTION_TO_SERVER:
-        return "LOST_CONNECTION_TO_SERVER";
-    case PMIX_ERR_LOST_PEER_CONNECTION:
-        return "LOST-PEER-CONNECTION";
-    case PMIX_ERR_LOST_CONNECTION_TO_CLIENT:
-        return "LOST-CONNECTION-TO-CLIENT";
+    case PMIX_ERR_PARTIAL_SUCCESS:
+        return "PARTIAL SUCCESS";
+    case PMIX_ERR_DUPLICATE_KEY:
+        return "DUPLICATE KEY";
+    case PMIX_ERR_EMPTY:
+        return "EMPTY";
+    case PMIX_PROCESS_SET_DEFINE:
+        return "PROCESS SET DEFINED";
+    case PMIX_ERR_IOF_FAILURE:
+        return "IOF FAILURE";
+    case PMIX_ERR_IOF_COMPLETE:
+        return "IOF COMPLETE";
+
+    case PMIX_ERR_LOST_CONNECTION:
+        return "LOST_CONNECTION";
+    case PMIX_ERR_EXISTS_OUTSIDE_SCOPE:
+        return "OUTSIDE-SCOPE";
     case PMIX_QUERY_PARTIAL_SUCCESS:
         return "QUERY-PARTIAL-SUCCESS";
     case PMIX_NOTIFY_ALLOC_COMPLETE:
@@ -179,8 +191,6 @@
 
     case PMIX_ERR_EVENT_REGISTRATION:
         return "EVENT-REGISTRATION";
-    case PMIX_ERR_JOB_TERMINATED:
-        return "PMIX_ERR_JOB_TERMINATED";
     case PMIX_ERR_UPDATE_ENDPOINTS:
         return "UPDATE-ENDPOINTS";
     case PMIX_MODEL_DECLARED:
@@ -196,8 +206,6 @@
     case PMIX_OPENMP_PARALLEL_EXITED:
         return "OPENMP-PARALLEL-EXITED";
 
-    case PMIX_LAUNCH_DIRECTIVE:
-        return "LAUNCH-DIRECTIVE";
     case PMIX_LAUNCHER_READY:
         return "LAUNCHER-READY";
     case PMIX_OPERATION_IN_PROGRESS:
@@ -207,12 +215,46 @@
     case PMIX_ERR_INVALID_OPERATION:
         return "INVALID-OPERATION";
 
-    case PMIX_ERR_NODE_DOWN:
+    case PMIX_GROUP_INVITED:
+        return "GROUP-INVITED";
+    case PMIX_GROUP_LEFT:
+        return "GROUP-LEFT";
+    case PMIX_GROUP_INVITE_ACCEPTED:
+        return "GROUP-INVITE-ACCEPTED";
+    case PMIX_GROUP_INVITE_DECLINED:
+        return "GROUP-INVITE-DECLINED";
+    case PMIX_GROUP_INVITE_FAILED:
+        return "GROUP-INVITE-FAILED";
+    case PMIX_GROUP_MEMBERSHIP_UPDATE:
+        return "GROUP-MEMBERSHIP-UPDATE";
+    case PMIX_GROUP_CONSTRUCT_ABORT:
+        return "GROUP-CONSTRUCT-ABORT";
+    case PMIX_GROUP_CONSTRUCT_COMPLETE:
+        return "GROUP-CONSTRUCT-COMPLETE";
+    case PMIX_GROUP_LEADER_SELECTED:
+        return "GROUP-LEADER-SELECTED";
+    case PMIX_GROUP_LEADER_FAILED:
+        return "GROUP-LEADER-FAILED";
+    case PMIX_GROUP_CONTEXT_ID_ASSIGNED:
+        return "GROUP-CONTEXT-ID-ASSIGNED";
+    case PMIX_GROUP_MEMBER_FAILED:
+        return "GROUP-MEMBER-FAILED";
+
+    case PMIX_FABRIC_UPDATED:
+        return "FABRIC-UPDATED";
+    case PMIX_FABRIC_UPDATE_PENDING:
+        return "FABRIC-UPDATE-PENDING";
+    case PMIX_ERR_REPEAT_ATTR_REGISTRATION:
+        return "REPEAT-ATTRIBUTE-REGISTRATION";
+
+    case PMIX_EVENT_NODE_DOWN:
         return "NODE-DOWN";
-    case PMIX_ERR_NODE_OFFLINE:
+    case PMIX_EVENT_NODE_OFFLINE:
         return "NODE-OFFLINE";
-    case PMIX_ERR_SYS_OTHER:
+    case PMIX_EVENT_SYS_OTHER:
         return "UNDEFINED-SYSTEM-EVENT";
+    case PMIX_EVENT_SYS_BASE:
+        return "SYSTEM BASE EVENT";
 
     case PMIX_EVENT_NO_ACTION_TAKEN:
         return "EVENT-NO-ACTION-TAKEN";
@@ -243,9 +285,52 @@
     case PMIX_ERR_TEMP_UNAVAILABLE:
         return "PMIX TEMPORARILY UNAVAILABLE";
 
+    case PMIX_LAUNCH_COMPLETE:
+        return "PMIX LAUNCH COMPLETE";
+
     case PMIX_MAX_ERR_CONSTANT:
         return "PMIX_ERR_WILDCARD";
 
+    case PMIX_ERR_JOB_APP_NOT_EXECUTABLE:
+        return "APPLICATION NOT EXECUTABLE";
+    case PMIX_ERR_JOB_NO_EXE_SPECIFIED:
+        return "NO EXECUTABLE SPECIFIED";
+    case PMIX_ERR_JOB_FAILED_TO_MAP:
+        return "JOB FAILED TO MAP";
+    case PMIX_ERR_JOB_CANCELED:
+        return "JOB CANCELED";
+    case PMIX_ERR_JOB_FAILED_TO_LAUNCH:
+        return "JOB FAILED TO LAUNCH";
+    case PMIX_ERR_JOB_ABORTED:
+        return "JOB ABORTED";
+    case PMIX_ERR_JOB_KILLED_BY_CMD:
+        return "KILLED BY COMMAND";
+    case PMIX_ERR_JOB_ABORTED_BY_SIG:
+        return "ABORTED BY SIGNAL";
+    case PMIX_ERR_JOB_TERM_WO_SYNC:
+        return "TERMINATED WITHOUT SYNC";
+    case PMIX_ERR_JOB_SENSOR_BOUND_EXCEEDED:
+        return "SENSOR BOUND EXCEEDED";
+    case PMIX_ERR_JOB_NON_ZERO_TERM:
+        return "NON-ZERO TERMINATION";
+    case PMIX_ERR_JOB_ALLOC_FAILED:
+        return "FAILED TO OBTAIN ALLOCATION";
+    case PMIX_ERR_JOB_ABORTED_BY_SYS_EVENT:
+        return "JOB ABORTED BY SYSTEM EVENT";
+    case PMIX_ERR_PROC_TERM_WO_SYNC:
+        return "PROC TERMINATED WITHOUT SYNC";
+
+    case PMIX_EVENT_PROC_TERMINATED:
+        return "PROC TERMINATED";
+    case PMIX_EVENT_JOB_START:
+        return "JOB STARTED";
+    case PMIX_EVENT_JOB_END:
+        return "JOB ENDED";
+    case PMIX_EVENT_SESSION_START:
+        return "SESSION STARTED";
+    case PMIX_EVENT_SESSION_END:
+        return "SESSION ENDED";
+
     default:
         return "ERROR STRING NOT FOUND";
     }
diff -Nru pmix-3.2.2~rc1/src/util/error.h pmix-4.0.0/src/util/error.h
--- pmix-3.2.2~rc1/src/util/error.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/error.h	2021-01-02 08:56:17.000000000 +0000
@@ -28,12 +28,17 @@
 
  BEGIN_C_DECLS
 
+/* define a starting point for PMIx internal error codes
+ * that are never exposed outside the library */
+#define PMIX_INTERNAL_ERR_BASE                      -1330
+
+
 /* internal error codes - never exposed outside of the library */
 #define PMIX_ERR_NOT_AVAILABLE                          (PMIX_INTERNAL_ERR_BASE - 28)
 #define PMIX_ERR_FATAL                                  (PMIX_INTERNAL_ERR_BASE - 29)
 #define PMIX_ERR_VALUE_OUT_OF_BOUNDS                    (PMIX_INTERNAL_ERR_BASE - 30)
 #define PMIX_ERR_PERM                                   (PMIX_INTERNAL_ERR_BASE - 31)
-#define PMIX_ERR_FABRIC_NOT_PARSEABLE                  (PMIX_INTERNAL_ERR_BASE - 33)
+#define PMIX_ERR_FABRIC_NOT_PARSEABLE                   (PMIX_INTERNAL_ERR_BASE - 33)
 #define PMIX_ERR_FILE_OPEN_FAILURE                      (PMIX_INTERNAL_ERR_BASE - 34)
 #define PMIX_ERR_FILE_READ_FAILURE                      (PMIX_INTERNAL_ERR_BASE - 35)
 #define PMIX_ERR_TAKE_NEXT_OPTION                       (PMIX_INTERNAL_ERR_BASE - 36)
diff -Nru pmix-3.2.2~rc1/src/util/keyval_parse.c pmix-4.0.0/src/util/keyval_parse.c
--- pmix-3.2.2~rc1/src/util/keyval_parse.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/keyval_parse.c	2021-01-02 08:56:17.000000000 +0000
@@ -266,21 +266,37 @@
 
 static int add_to_env_str(char *var, char *val)
 {
-    int sz, varsz, valsz;
+    int sz, varsz=0, valsz=0, new_envsize;
     void *tmp;
 
     if (NULL == var) {
         return PMIX_ERR_BAD_PARAM;
     }
 
+    varsz = strlen(var);
+    if (NULL != val) {
+        valsz = strlen(val);
+        /* account for '=' */
+        valsz += 1;
+    }
+    sz = 0;
     if (NULL != env_str) {
-        varsz = strlen(var);
-        valsz = (NULL != val) ? strlen(val) : 0;
-        sz = strlen(env_str)+varsz+valsz+2;
-        if (envsize <= sz) {
-            envsize *=2;
+        sz = strlen(env_str);
+        /* account for ';' */
+        sz += 1;
+    }
+    /* add required new size incl NULL byte */
+    sz += varsz+valsz+1;
 
-            tmp = realloc(env_str, envsize);
+    /* make sure we have sufficient space */
+    new_envsize = envsize;
+    while (new_envsize <= sz) {
+        new_envsize *= 2;
+    }
+
+    if (NULL != env_str) {
+        if (new_envsize > envsize) {
+            tmp = realloc(env_str, new_envsize);
             if (NULL == tmp) {
                 return PMIX_ERR_OUT_OF_RESOURCE;
             }
@@ -288,11 +304,12 @@
         }
         strcat(env_str, ";");
     } else {
-        env_str = calloc(1, envsize);
+        env_str = calloc(1, new_envsize);
         if (NULL == env_str) {
             return PMIX_ERR_OUT_OF_RESOURCE;
         }
     }
+    envsize = new_envsize;
 
     strcat(env_str, var);
     if (NULL != val) {
diff -Nru pmix-3.2.2~rc1/src/util/name_fns.c pmix-4.0.0/src/util/name_fns.c
--- pmix-3.2.2~rc1/src/util/name_fns.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/name_fns.c	2021-01-02 08:56:17.000000000 +0000
@@ -124,7 +124,7 @@
     index = ptr->cntr;
     snprintf(ptr->buffers[index],
              PMIX_PRINT_NAME_ARGS_MAX_SIZE,
-             "[%s:%s]", ns, rank);
+             "[%s.%s]", ns, rank);
     ptr->cntr++;
     if (PMIX_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) {
         ptr->cntr = 0;
diff -Nru pmix-3.2.2~rc1/src/util/pmix_environ.c pmix-4.0.0/src/util/pmix_environ.c
--- pmix-3.2.2~rc1/src/util/pmix_environ.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/src/util/pmix_environ.c	2021-01-02 08:56:17.000000000 +0000
@@ -317,6 +317,9 @@
                 /* see if we already have this envar on the list */
                 duplicate = false;
                 PMIX_LIST_FOREACH(kv, ilist, pmix_kval_t) {
+                    if (PMIX_ENVAR != kv->value->type) {
+                        continue;
+                    }
                     if (0 == strcmp(kv->value->data.envar.envar, cs_env)) {
                         /* if the value is the same, then ignore it */
                         if (0 != strcmp(kv->value->data.envar.value, string_key)) {
diff -Nru pmix-3.2.2~rc1/test/cli_stages.c pmix-4.0.0/test/cli_stages.c
--- pmix-3.2.2~rc1/test/cli_stages.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/cli_stages.c	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
@@ -70,7 +70,7 @@
 
     cli->sd = sd;
     cli->ev = pmix_event_new(ebase, sd,
-                             EV_READ|EV_PERSIST, callback, cli);
+                      EV_READ|EV_PERSIST, callback, cli);
     pmix_event_add(cli->ev,NULL);
     pmix_ptl_base_set_nonblocking(sd);
     TEST_VERBOSE(("Connection accepted from rank %d", cli_rank(cli) ));
@@ -205,12 +205,8 @@
                 pmix_event_notification_cbfunc_fn_t cbfunc,
                 void *cbdata)
 {
-    TEST_ERROR((" PMIX server event handler with status = %d", status));
-    /* notify clients of error */
-    PMIx_Notify_event(status, source,
-                      PMIX_RANGE_NAMESPACE,
-                      NULL, 0,
-                      op_callbk, NULL);
+    TEST_ERROR((" PMIX server event handler for %s:%d with status = %d", source->nspace, source->rank, status));
+    cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
 }
 
 void op_callbk(pmix_status_t status,
diff -Nru pmix-3.2.2~rc1/test/cli_stages.h pmix-4.0.0/test/cli_stages.h
--- pmix-3.2.2~rc1/test/cli_stages.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/cli_stages.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
@@ -15,7 +15,7 @@
 #ifndef CLI_STAGES_H
 #define CLI_STAGES_H
 
-#include <pmix_config.h>
+#include "src/include/pmix_config.h"
 #include <signal.h>
 #include <fcntl.h>
 #include <sys/types.h>
@@ -25,7 +25,6 @@
 #include <sys/wait.h>
 #include <sys/time.h>
 #include <time.h>
-#include PMIX_EVENT_HEADER
 #include <errno.h>
 #include "src/include/pmix_globals.h"
 #include "pmix_server.h"
@@ -59,13 +58,15 @@
 
 int cli_rank(cli_info_t *cli);
 void cli_init(int nprocs);
-void cli_connect(cli_info_t *cli, int sd, pmix_event_base_t * ebase, event_callback_fn callback);
+void cli_connect(cli_info_t *cli, int sd, struct event_base * ebase, event_callback_fn callback);
 void cli_finalize(cli_info_t *cli);
 void cli_disconnect(cli_info_t *cli);
 void cli_terminate(cli_info_t *cli);
 void cli_cleanup(cli_info_t *cli);
 void cli_kill_all(void);
 
+bool test_terminated(void);
+
 void errhandler(size_t evhdlr_registration_id,
                 pmix_status_t status,
                 const pmix_proc_t *source,
diff -Nru pmix-3.2.2~rc1/test/Makefile.am pmix-4.0.0/test/Makefile.am
--- pmix-3.2.2~rc1/test/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -24,7 +24,11 @@
 if !WANT_HIDDEN
 # these tests use internal symbols
 # use --disable-visibility
-SUBDIRS = simple
+SUBDIRS = simple sshot
+
+if WANT_PYTHON_BINDINGS
+SUBDIRS += python
+endif
 
 endif
 
@@ -62,12 +66,6 @@
     pmix_client \
     pmix_regex
 
-if WANT_PMI_BACKWARD
-check_PROGRAMS += \
-    pmi_client \
-    pmi2_client
-endif
-
 TESTS = \
 	run_tests00.pl \
 	run_tests01.pl \
@@ -89,10 +87,6 @@
 
 ##########################
 
-if WANT_PMI_BACKWARD
-noinst_PROGRAMS += pmi_client pmi2_client
-endif
-
 noinst_PROGRAMS += pmix_test pmix_client pmix_regex
 
 pmix_test_SOURCES = $(headers) \
@@ -101,20 +95,6 @@
 pmix_test_LDADD = \
     $(top_builddir)/src/libpmix.la
 
-if WANT_PMI_BACKWARD
-pmi_client_SOURCES = $(headers) \
-        pmi_client.c
-pmi_client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
-pmi_client_LDADD = \
-    $(top_builddir)/src/libpmi.la
-
-pmi2_client_SOURCES = $(headers) \
-        pmi2_client.c
-pmi2_client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
-pmi2_client_LDADD = \
-    $(top_builddir)/src/libpmi2.la
-endif
-
 pmix_client_SOURCES = $(headers) \
         pmix_client.c test_fence.c test_common.c test_publish.c test_spawn.c \
         test_cd.c test_resolve_peers.c test_error.c test_replace.c test_internal.c
diff -Nru pmix-3.2.2~rc1/test/pmi2_client.c pmix-4.0.0/test/pmi2_client.c
--- pmix-3.2.2~rc1/test/pmi2_client.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/pmi2_client.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,572 +0,0 @@
-/*
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <stdarg.h>
-
-#include "pmi2.h"
-
-/* Target is legacy SLURM pmi2 library implementation */
-static int _legacy = 0;
-/* Verbose level 0-silent, 1-fatal, 2-error, 3+ debug*/
-static int _verbose = 1;
-
-static void log_fatal(const char *format, ...)
-{
-    va_list arglist;
-    char *output = NULL;
-
-    va_start(arglist, format);
-    if (_verbose > 0) {
-        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output) {
-            va_end(arglist);
-            return;
-        }
-        fprintf(stderr, "FATAL: %s", output);
-        free(output);
-    }
-    va_end(arglist);
-}
-
-static void log_error(const char *format, ...)
-{
-    va_list arglist;
-    char *output = NULL;
-
-    va_start(arglist, format);
-    if (_verbose > 0) {
-        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output) {
-            va_end(arglist);
-            return;
-        }
-        fprintf(stderr, "ERROR: %s", output);
-        free(output);
-    }
-    va_end(arglist);
-}
-
-static void log_info(const char *format, ...)
-{
-    va_list arglist;
-    char *output = NULL;
-
-    va_start(arglist, format);
-    if (_verbose > 0) {
-        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output) {
-            va_end(arglist);
-            return;
-        }
-        fprintf(stderr, "INFO: %s", output);
-        free(output);
-    }
-    va_end(arglist);
-}
-
-#define log_assert(e, msg) \
-    do {                                                                \
-        if (!(e)) {                                                     \
-            log_fatal("%s at %s:%d\n", msg, __func__, __LINE__);    \
-            rc = -1;                                                    \
-        }                                                               \
-    } while (0)
-
-static inline long random_value(long min_value, long max_value)
-{
-   return ((min_value >= max_value) ? min_value : min_value + (rand() % (max_value - min_value + 1)));
-}
-
-static int test_item1(void);
-static int test_item2(void);
-static int test_item3(void);
-static int test_item4(void);
-static int test_item5(void);
-static int test_item6(void);
-static int test_item7(void);
-static int test_item8(void);
-static int test_item9(void);
-/* several sequence of fences is a buggy case for pmix v1.0 (see https://github.com/open-mpi/pmix/issues/37) */
-static int test_item10(void);
-
-static int spawned, size, rank, appnum;
-static char jobid[100];
-
-
-int main(int argc, char **argv)
-{
-    int ret = 0;
-    int rc;
-    char *str = NULL;
-    int ti = (argc > 1 ? atoi(argv[1]) : 0);
-
-    srand(time(NULL));
-    str = getenv("VERBOSE");
-    _verbose = (str ? atoi(str) : _verbose);
-
-    spawned = random_value(10, 20);
-    size = random_value(10, 20);
-    rank = random_value(10, 20);
-    appnum = random_value(10, 20);
-    if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
-        log_fatal("PMI2_Init failed: %d\n", rc);
-        return rc;
-    }
-
-    str = getenv("PMIX_NAMESPACE");
-    _legacy = (str ? 0 : 1);
-
-    /* this test should be always run */
-    if (1) {
-        rc = test_item1();
-        ret += (rc ? 1 : 0);
-        log_info("TI1  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 2 == ti) {
-        rc = test_item2();
-        ret += (rc ? 1 : 0);
-        log_info("TI2  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 3 == ti) {
-        rc = test_item3();
-        ret += (rc ? 1 : 0);
-        log_info("TI3  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 4 == ti) {
-        rc = test_item4();
-        ret += (rc ? 1 : 0);
-        log_info("TI4  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 5 == ti) {
-        rc = test_item5();
-        ret += (rc ? 1 : 0);
-        log_info("TI5  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 6 == ti) {
-        rc = test_item6();
-        ret += (rc ? 1 : 0);
-        log_info("TI6  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 7 == ti) {
-        rc = test_item7();
-        ret += (rc ? 1 : 0);
-        log_info("TI7  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 8 == ti) {
-        rc = test_item8();
-        ret += (rc ? 1 : 0);
-        log_info("TI8  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 9 == ti) {
-        rc = test_item9();
-        ret += (rc ? 1 : 0);
-        log_info("TI9  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 10 == ti) {
-        rc = test_item10();
-        ret += (rc ? 1 : 0);
-        log_info("TI10  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (PMI2_SUCCESS != (rc = PMI2_Finalize())) {
-        log_fatal("PMI2_Finalize failed: %d\n", rc);
-        return rc;
-    }
-
-    return ret;
-}
-
-static int test_item1(void)
-{
-    int rc = 0;
-
-    log_info("spawned=%d size=%d rank=%d appnum=%d\n", spawned, size, rank, appnum);
-
-    log_assert(spawned == 0 || spawned == 1, "");
-    log_assert(size >= 0, "");
-    log_assert(rank >= 0, "");
-    log_assert(rank < size, "");
-
-    sprintf(jobid, "%s", __func__);
-    if (PMI2_SUCCESS != (rc = PMI2_Job_GetId(jobid, sizeof(jobid)))) {
-        log_fatal("PMI2_Job_GetId failed: %d\n", rc);
-        return rc;
-    }
-
-    log_info("jobid=%s\n", jobid);
-    log_assert(memcmp(jobid, __func__, sizeof(__func__)), "");
-
-    return rc;
-}
-
-static int test_item2(void)
-{
-    int rc = 0;
-    int val = 0;
-
-    log_assert(PMI2_Initialized(), "");
-
-    val = random_value(10, 100);
-    if (PMI2_SUCCESS != (rc = PMI2_Job_GetRank(&val))) {
-        log_fatal("PMI2_Job_GetRank failed: %d\n", rc);
-        return rc;
-    }
-    log_assert(rank == val, "");
-
-    val = -1;
-    if (PMI2_SUCCESS != (rc = PMI2_Info_GetSize(&val))) {
-        log_fatal("PMI2_Info_GetSize failed: %d\n", rc);
-        return rc;
-    }
-    log_assert(0 < val, "");
-
-    return rc;
-}
-
-static int test_item3(void)
-{
-    int rc = 0;
-    char val[PMI2_MAX_VALLEN];
-    int found = 0;
-    /* Predefined Job attributes */
-    const char *tkeys[] = {
-            "universeSize",
-            "hasNameServ",
-            "physTopology",
-            "physTopologyLevels",
-            "cartDims",
-            "isHeterogeneous",
-            NULL
-    };
-    const char **ptr = tkeys;
-
-    if (!_legacy) {
-        log_error("%s\n", "PMIx and SLURM/PMI2 does not set Job Attributes (Do not mark test as failed)");
-        return rc;
-    }
-
-    while (*ptr) {
-        if (PMI2_SUCCESS != (rc = PMI2_Info_GetJobAttr(*ptr, val, sizeof(val), &found))) {
-            log_fatal("PMI2_Info_GetJobAttr: [%s] %d\n", *ptr, rc);
-            return rc;
-        }
-        log_info("key=%s value=%s found=%d\n", *ptr, (found ? val : "N/A"), found);
-        if (!_legacy && !found) {
-            log_error("PMIx does not set: %s (Do not mark test as failed)\n", *ptr);
-        }
-        ptr++;
-    }
-
-    return rc;
-}
-
-static int test_item4(void)
-{
-    int rc = 0;
-    char val[PMI2_MAX_VALLEN];
-    int found = 0;
-    /* Predefined Node attributes */
-    const char *tkeys[] = {
-            "memPoolType",
-            "memSYSVid",
-            "memAnonMMAPfd",
-            "memNTName",
-            NULL
-    };
-    const char **ptr = tkeys;
-
-    if (!_legacy) {
-        log_error("%s\n", "PMIx and SLURM/PMI2 does not set Node Attributes (Do not mark test as failed)");
-        return rc;
-    }
-
-    while (*ptr) {
-        if (PMI2_SUCCESS != (rc = PMI2_Info_GetNodeAttr(*ptr, val, sizeof(val), &found, 1))) {
-            log_fatal("PMI2_Info_GetNodeAttr: [%s] %d\n", *ptr, rc);
-            return rc;
-        }
-        log_info("key=%s value=%s found=%d\n", *ptr, (found ? val : "N/A"), found);
-        if (!_legacy && !found) {
-            log_error("PMIx does not set: %s (Do not mark test as failed)\n", *ptr);
-        }
-        ptr++;
-    }
-
-    return rc;
-}
-
-static int test_item5(void)
-{
-    int rc = 0;
-    char val[PMI2_MAX_VALLEN];
-    int found = 0;
-    const char *tkey = "sharedFilename";
-    const char *tval = "pmix-pmi2-check";
-
-    if (PMI2_SUCCESS != (rc = PMI2_Info_PutNodeAttr(tkey, tval))) {
-        log_fatal("PMI2_Info_PutNodeAttr %d\n", rc);
-        return rc;
-    }
-
-    if (PMI2_SUCCESS != (rc = PMI2_Info_GetNodeAttr(tkey, val, sizeof(val), &found, 1))) {
-        log_fatal("PMI2_Info_GetNodeAttr %d\n", rc);
-        return rc;
-    }
-
-    log_info("tkey=%s tval=%s val=%s found=%d\n", tkey, tval, val, found);
-
-    log_assert(found, "PMI2_Info_GetNodeAttr does not find expected key");
-    log_assert(strlen(tval) == strlen(val), "value does not meet expectation");
-    log_assert(!strcmp(tval, val), "value does not meet expectation");
-
-    return rc;
-}
-
-static int test_item6(void)
-{
-    int rc = 0;
-    char val[PMI2_MAX_VALLEN];
-    int len;
-    const char *tkey = __func__;
-    const char *tval = __FILE__;
-
-    if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) {
-        log_fatal("PMI2_KVS_Put %d\n", rc);
-        return rc;
-    }
-
-    /* expected result: return error status */
-    rc = PMI2_KVS_Get(NULL, PMI2_ID_NULL, tkey, val, sizeof(val), &len);
-    if (PMI2_SUCCESS == rc) {
-        log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len);
-        log_error("%s\n", "PMI2_KVS_Get should not find data w/o commit");
-        return 1;
-    }
-
-    return 0;
-}
-
-static int test_item7(void)
-{
-    int rc = 0;
-    char val[PMI2_MAX_VALLEN];
-    int len;
-    char tkey[PMI2_MAX_VALLEN];
-    char tval[PMI2_MAX_VALLEN];
-
-    sprintf(tkey, "KEY-%d", rank);
-    sprintf(tval, "VALUE-%d", rank);
-    if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) {
-        log_fatal("PMI2_KVS_Put %d\n", rc);
-        return rc;
-    }
-
-    if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) {
-        log_fatal("PMI2_KVS_Fence %d\n", rc);
-        return rc;
-    }
-
-    /* expected result: return error status */
-    rc = PMI2_KVS_Get(jobid, rank, tkey, val, sizeof(val), &len);
-    if (PMI2_SUCCESS != rc) {
-        log_fatal("PMI2_KVS_Get [%s=?] %d\n", tkey, rc);
-        return rc;
-    }
-
-    log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len);
-
-    log_assert((int)strlen(tval) == len, "value does not meet expectation");
-    log_assert(!strcmp(tval, val), "value does not meet expectation");
-
-    return 0;
-}
-
-static int test_item8(void)
-{
-    int rc = 0;
-    int len;
-    char tkey[PMI2_MAX_VALLEN];
-    char tval[PMI2_MAX_VALLEN];
-    char val[PMI2_MAX_VALLEN];
-    int i = 0;
-
-    for (i = 0; i < size; i++) {
-        sprintf(tkey, "KEY-%d", i);
-        sprintf(tval, "VALUE-%d", i);
-        if (i == rank) {
-            if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) {
-                log_fatal("PMI2_KVS_Put [%s=%s] %d\n", tkey, tval, rc);
-                return rc;
-            }
-        }
-    }
-
-    if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) {
-        log_fatal("PMI2_KVS_Fence %d\n", rc);
-        return rc;
-    }
-
-    for (i = 0; i < size; i++) {
-        sprintf(tkey, "KEY-%d", i);
-        sprintf(tval, "VALUE-%d", i);
-        if (PMI2_SUCCESS != (rc = PMI2_KVS_Get(jobid, i, tkey, val, sizeof(val), &len))) {
-            log_fatal("PMI2_KVS_Get [%s=?] %d\n", tkey, rc);
-            return rc;
-        }
-
-        log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len);
-
-        log_assert((int)strlen(tval) == len, "value does not meet expectation");
-        log_assert(!strcmp(tval, val), "value does not meet expectation");
-    }
-
-    return rc;
-}
-
-static int test_item9(void)
-{
-    int rc = 0;
-    int len;
-    char tkey[PMI2_MAX_VALLEN];
-    char tval[PMI2_MAX_VALLEN];
-    char val[PMI2_MAX_VALLEN];
-    int i = 0;
-
-    for (i = 0; i < size; i++) {
-        sprintf(tkey, "KEY-%d", i);
-        sprintf(tval, "VALUE-%d", i);
-        if (i == rank) {
-            log_info("Rank %d executing Put of key %s\n", rank, tkey);
-            if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(tkey, tval))) {
-                log_fatal("PMI2_KVS_Put [%s=%s] %d\n", tkey, tval, rc);
-                return rc;
-            }
-        }
-    }
-
-    log_info("Rank %d executing Fence\n", rank);
-    if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) {
-        log_fatal("PMI2_KVS_Fence %d\n", rc);
-        return rc;
-    }
-
-    for (i = 0; i < size; i++) {
-        sprintf(tkey, "KEY-%d", i);
-        sprintf(tval, "VALUE-%d", i);
-        log_info("Rank %d executing Get of key %s\n", rank, tkey);
-        if (PMI2_SUCCESS != (rc = PMI2_KVS_Get(jobid, PMI2_ID_NULL, tkey, val, sizeof(val), &len))) {
-            log_fatal("PMI2_KVS_Get [%s=?] %d\n", tkey, rc);
-            return rc;
-        }
-
-        log_info("tkey=%s tval=%s val=%s len=%d\n", tkey, tval, val, len);
-
-        log_assert((int)strlen(tval) == len, "value does not meet expectation");
-        log_assert(!strcmp(tval, val), "value does not meet expectation");
-    }
-
-    return rc;
-}
-
-static int test_item10(void)
-{
-    int rc = 0;
-    int i, j, r;
-    char symb, symb_start = 'a';
-    int fence_cnt;
-    int fence_num = 5;
-    int keys_per_fence = 50;
-    int val_size = random_value(10, PMI2_MAX_VALLEN / 10);
-    int keys_total = 0;
-
-    fence_cnt = 0;
-    while (fence_cnt < fence_num) {
-        log_info("fence_cnt=%d of fence_num=%d keys_per_fence=%d keys_total=%d val_size=%d\n",
-                fence_cnt, fence_num, keys_per_fence, keys_total, val_size);
-        symb = symb_start;
-        for (i = 0; i < keys_per_fence; i++) {
-            char key[PMI2_MAX_KEYLEN];
-            char val[PMI2_MAX_VALLEN] = "";
-            sprintf(key, "RANK%d-key-%d", rank, i + keys_total);
-            for (j = 0; j < val_size; j++) {
-                val[j] = symb;
-            }
-            symb++;
-            if (symb > 'z') {
-                symb = 'a';
-            }
-            if (PMI2_SUCCESS != (rc = PMI2_KVS_Put(key, val))) {
-                log_fatal("%d : PMI2_KVS_Put [%s=%s] %d\n", rank, key, val, rc);
-                return rc;
-            }
-            log_info("%d : PMI2_KVS_Put [%s=%s] %d\n", rank, key, val, rc);
-        }
-        symb_start = symb;
-        keys_total += keys_per_fence;
-
-        if (PMI2_SUCCESS != (rc = PMI2_KVS_Fence())) {
-            log_fatal("%d : PMI2_KVS_Fence %d\n", rank, rc);
-            return rc;
-        }
-
-        for (r = 0; r < size; r++) {
-            int len;
-            symb = 'a';
-            for (i = 0; i < keys_total; i++) {
-                char key[PMI2_MAX_KEYLEN];
-                char val[PMI2_MAX_VALLEN] = "";
-                sprintf(key, "RANK%d-key-%d", r, i);
-
-                if (PMI2_SUCCESS != (rc = PMI2_KVS_Get(jobid, r, key, val, sizeof(val), &len))) {
-                    log_fatal("%d : PMI2_KVS_Get [%s=?] %d\n", rank, key, rc);
-                    return rc;
-                }
-
-                log_info("%d : PMI2_KVS_Get from %d [%s=%s] %d\n", r, rank, key, val, rc);
-
-                if (len != val_size) {
-                    log_fatal("%d: failure on rank %d, key #%d: len mismatch:"
-                            " %d instead of %d\n", rank, r, i, len, val_size);
-                }
-
-                for (j = 0; j < val_size; j++) {
-                    if (val[j] != symb) {
-                        log_fatal("%d: failure on rank %d, key #%d: value mismatch"
-                                " at symb %d: \'%c\' instead of \'%c\'\n", rank,
-                                r, i, j, val[j], symb);
-                    }
-                }
-                symb++;
-                if (symb > 'z') {
-                    symb = 'a';
-                }
-            }
-        }
-        fence_cnt++;
-    }
-
-    return rc;
-}
diff -Nru pmix-3.2.2~rc1/test/pmi_client.c pmix-4.0.0/test/pmi_client.c
--- pmix-3.2.2~rc1/test/pmi_client.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/pmi_client.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,445 +0,0 @@
-/*
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Mellanox Technologies, Inc.
- *                         All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <stdarg.h>
-
-#include "pmi.h"
-
-/* Target is legacy SLURM pmi library implementation */
-static int _legacy = 0;
-/* Verbose level 0-silent, 1-fatal, 2-error, 3+ debug*/
-static int _verbose = 1;
-
-static int spawned, size, rank=-1, appnum;
-static char jobid[255];
-
-static void log_fatal(const char *format, ...)
-{
-    va_list arglist;
-    char *output = NULL;
-
-    va_start(arglist, format);
-    if (_verbose > 0) {
-        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output) {
-            va_end(arglist);
-            return;
-        }
-        fprintf(stderr, "%d:FATAL: %s", rank, output);
-        free(output);
-    }
-    va_end(arglist);
-}
-
-static void log_error(const char *format, ...)
-{
-    va_list arglist;
-    char *output = NULL;
-
-    va_start(arglist, format);
-    if (_verbose > 0) {
-        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output) {
-            va_end(arglist);
-            return;
-        }
-        fprintf(stderr, "%d:ERROR: %s", rank, output);
-        free(output);
-    }
-    va_end(arglist);
-}
-
-static void log_info(const char *format, ...)
-{
-    va_list arglist;
-    char *output = NULL;
-
-    va_start(arglist, format);
-    if (_verbose > 0) {
-        if (0 > vasprintf(&output, format, arglist) ||
-            NULL == output) {
-            va_end(arglist);
-            return;
-        }
-        fprintf(stderr, "%d:INFO: %s", rank, output);
-        free(output);
-    }
-    va_end(arglist);
-}
-
-#define log_assert(e, msg) \
-    do {                                                                \
-        if (!(e)) {                                                     \
-            log_fatal("%d:%s at %s:%d\n", rank, msg, __func__, __LINE__);    \
-            rc = -1;                                                    \
-        }                                                               \
-    } while (0)
-
-static inline long random_value(long min_value, long max_value)
-{
-   return ((min_value >= max_value) ? min_value : min_value + (rand() % (max_value - min_value + 1)));
-}
-
-static int test_item1(void);
-static int test_item2(void);
-static int test_item3(void);
-static int test_item4(void);
-static int test_item5(void);
-static int test_item6(void);
-static int test_item7(void);
-
-int main(int argc, char **argv)
-{
-    int ret = 0;
-    int rc;
-    char *str = NULL;
-    int ti = (argc > 1 ? atoi(argv[1]) : 0);
-
-    srand(time(NULL));
-    str = getenv("VERBOSE");
-    _verbose = (str ? atoi(str) : _verbose);
-
-    spawned = random_value(10, 20);
-    size = random_value(10, 20);
-    rank = random_value(10, 20);
-    appnum = random_value(10, 20);
-    if (PMI_SUCCESS != (rc = PMI_Init(&spawned))) {
-        log_fatal("PMI_Init failed: %d\n", rc);
-        return rc;
-    }
-
-    str = getenv("PMIX_NAMESPACE");
-    _legacy = (str ? 0 : 1);
-
-    /* this test should be always run */
-    if (1) {
-        rc = test_item1();
-        ret += (rc ? 1 : 0);
-        log_info("TI1  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 2 == ti) {
-        rc = test_item2();
-        ret += (rc ? 1 : 0);
-        log_info("TI2  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 3 == ti) {
-        rc = test_item3();
-        ret += (rc ? 1 : 0);
-        log_info("TI3  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 4 == ti) {
-        rc = test_item4();
-        ret += (rc ? 1 : 0);
-        log_info("TI4  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 5 == ti) {
-        rc = test_item5();
-        ret += (rc ? 1 : 0);
-        log_info("TI5  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 6 == ti) {
-        rc = test_item6();
-        ret += (rc ? 1 : 0);
-        log_info("TI6  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (!ti || 7 == ti) {
-        rc = test_item7();
-        ret += (rc ? 1 : 0);
-        log_info("TI7  : %s\n", (rc ? "FAIL" : "PASS"));
-    }
-
-    if (PMI_SUCCESS != (rc = PMI_Finalize())) {
-        log_fatal("PMI_Finalize failed: %d\n", rc);
-        return rc;
-    }
-
-    return ret;
-}
-
-static int test_item1(void)
-{
-    int rc = 0;
-    int val = 0;
-
-    log_assert(spawned == PMI_FALSE || spawned == PMI_TRUE, "");
-
-    if (PMI_SUCCESS != (rc = PMI_Get_size(&size))) {
-        log_fatal("PMI_Get_Size failed: %d\n", rc);
-        return rc;
-    }
-    log_assert(size >= 0, "");
-
-    if (PMI_SUCCESS != (rc = PMI_Get_rank(&rank))) {
-        log_fatal("PMI_Get_Rank failed: %d\n", rc);
-        return rc;
-    }
-    log_assert(rank >= 0, "");
-    log_assert(rank < size, "");
-
-    if (PMI_SUCCESS != (rc = PMI_Get_appnum(&appnum))) {
-        log_fatal("PMI_Get_appnum failed: %d\n", rc);
-        return rc;
-    }
-
-    log_info("spawned=%d size=%d rank=%d appnum=%d\n", spawned, size, rank, appnum);
-
-    val = random_value(10, 100);
-    if (PMI_SUCCESS != (rc = PMI_Get_universe_size(&val))) {
-        log_fatal("PMI_Get_universe_size failed: %d\n", rc);
-        return rc;
-    }
-    log_assert(size == val, "");
-
-    val = random_value(10, 100);
-    if (PMI_SUCCESS != (rc = PMI_Get_id_length_max(&val))) {
-        log_fatal("PMI_Get_id_length_max failed: %d\n", rc);
-        return rc;
-    }
-    log_info("PMI_Get_id_length_max=%d\n", val);
-    if (!_legacy) {
-        log_assert(sizeof(jobid) == val, "Check PMIX_MAX_NSLEN value in pmix_common.h");
-    }
-
-    sprintf(jobid, "%s", __func__);
-    if (PMI_SUCCESS != (rc = PMI_Get_id(jobid, sizeof(jobid)))) {
-        log_fatal("PMI_Get_id failed: %d\n", rc);
-        return rc;
-    }
-
-    log_info("jobid=%s\n", jobid);
-    log_assert(memcmp(jobid, __func__, sizeof(__func__)), "");
-
-    sprintf(jobid, "%s", __func__);
-    if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(jobid, sizeof(jobid)))) {
-        log_fatal("PMI_Get_kvs_domain_id failed: %d\n", rc);
-        return rc;
-    }
-
-    log_info("PMI_Get_kvs_domain_id=%s\n", jobid);
-    log_assert(memcmp(jobid, __func__, sizeof(__func__)), "");
-
-    sprintf(jobid, "%s", __func__);
-    if (PMI_SUCCESS != (rc = PMI_KVS_Get_my_name(jobid, sizeof(jobid)))) {
-        log_fatal("PMI_KVS_Get_my_name failed: %d\n", rc);
-        return rc;
-    }
-
-    log_info("PMI_KVS_Get_my_name=%s\n", jobid);
-    log_assert(memcmp(jobid, __func__, sizeof(__func__)), "");
-
-    return rc;
-}
-
-static int test_item2(void)
-{
-    int rc = 0;
-    PMI_BOOL val;
-
-    if (PMI_SUCCESS != (rc = PMI_Initialized(&val))) {
-        log_fatal("PMI_Initialized failed: %d\n", rc);
-        return rc;
-    }
-    log_assert(PMI_TRUE == val, "");
-
-    return rc;
-}
-
-static int test_item3(void)
-{
-    int rc = 0;
-    int val = 0;
-
-    val = random_value(10, 100);
-    if (PMI_SUCCESS != (rc = PMI_KVS_Get_key_length_max(&val))) {
-        log_fatal("PMI_KVS_Get_key_length_max failed: %d\n", rc);
-        return rc;
-    }
-    log_info("PMI_KVS_Get_key_length_max=%d\n", val);
-    if (!_legacy) {
-        log_assert(511 == val, "Check PMIX_MAX_KEYLEN value in pmix_common.h");
-    }
-
-    val = random_value(10, 100);
-    if (PMI_SUCCESS != (rc = PMI_KVS_Get_value_length_max(&val))) {
-        log_fatal("PMI_KVS_Get_value_length_max failed: %d\n", rc);
-        return rc;
-    }
-    log_info("PMI_KVS_Get_value_length_max=%d\n", val);
-    if (!_legacy) {
-        log_assert(4096 == val, "Check limitation for a value");
-    }
-
-    return rc;
-}
-
-static int test_item4(void)
-{
-    int rc = 0;
-    int val = 0;
-    int *ranks = NULL;
-    int i = 0;
-
-    val = -1;
-    if (PMI_SUCCESS != (rc = PMI_Get_clique_size(&val))) {
-        log_fatal("PMI_Get_clique_size failed: %d\n", rc);
-        return rc;
-    }
-    log_info("PMI_Get_clique_size=%d\n", val);
-    log_assert((0 < val) && (val <= size), "");
-
-    ranks = alloca(val*sizeof(int));
-    if (!ranks) {
-        return PMI_FAIL;
-    }
-
-    memset(ranks, (-1), val*sizeof(int));
-    if (PMI_SUCCESS != (rc = PMI_Get_clique_ranks(ranks, val))) {
-        log_fatal("PMI_Get_clique_ranks failed: %d\n", rc);
-        return rc;
-    }
-
-    for (i = 0; i < val; i++) {
-        if (!((0 <= ranks[i]) && (ranks[i] < size))) {
-            log_fatal("found invalid value in ranks array: ranks[%d]=%d\n", i, ranks[i]);
-            return rc;
-        }
-    }
-
-    return rc;
-}
-
-static int test_item5(void)
-{
-    int rc = 0;
-    char *val = NULL;
-    int val_size = 0;
-    /* Predefined Job attributes */
-    const char *tkeys[] = {
-            "PMI_process_mapping",
-            NULL
-    };
-    const char **ptr = tkeys;
-
-    if (!_legacy) {
-        log_error("%s\n", "PMIx and SLURM/PMI1 do not set 'PMI_process_mapping' (Do not mark test as failed)");
-        return rc;
-    }
-
-    if (PMI_SUCCESS != (rc = PMI_KVS_Get_value_length_max(&val_size))) {
-        log_fatal("PMI_KVS_Get_value_length_max failed: %d\n", rc);
-        return rc;
-    }
-
-    val = alloca(val_size);
-    if (!val) {
-        return PMI_FAIL;
-    }
-
-    while (*ptr) {
-        if (PMI_SUCCESS != (rc = PMI_KVS_Get(jobid, *ptr, val, val_size))) {
-            log_fatal("PMI_KVS_Get: [%s] %d\n", *ptr, rc);
-            return rc;
-        }
-        log_info("key=%s value=%.80s\n", *ptr, val);
-        ptr++;
-    }
-
-    return rc;
-}
-
-static int test_item6(void)
-{
-    int rc = 0;
-    char val[100];
-    char *tkey;
-    const char *tval = __FILE__;
-
-    asprintf(&tkey, "%d:%s", rank, __func__);
-    if (PMI_SUCCESS != (rc = PMI_KVS_Put(jobid, tkey, tval))) {
-        log_fatal("PMI_KVS_Put %d\n", rc);
-        free(tkey);
-        return rc;
-    }
-
-    if (PMI_SUCCESS != (rc = PMI_KVS_Get(jobid, tkey, val, sizeof(val)))) {
-        log_fatal("PMI_KVS_Get %d\n", rc);
-        free(tkey);
-        return rc;
-    }
-
-    log_info("tkey=%s tval=%s val=%s\n", tkey, tval, val);
-    free(tkey);
-    log_assert(!strcmp(tval, val), "value does not meet expectation");
-
-    return rc;
-}
-
-static int test_item7(void)
-{
-    int rc = 0;
-    char tkey[100];
-    char tval[100];
-    char val[100];
-    int i = 0, j;
-
-log_info("TEST7\n");
-
-    for (i = 0; i < size; i++) {
-        sprintf(tkey, "%d:KEY-%d", rank, i);
-        sprintf(tval, "VALUE-%d", i);
-        if (PMI_SUCCESS != (rc = PMI_KVS_Put(jobid, tkey, tval))) {
-            log_fatal("PMI_KVS_Put [%s=%s] %d\n", tkey, tval, rc);
-            return rc;
-        }
-    }
-
-    if (PMI_SUCCESS != (rc = PMI_KVS_Commit(jobid))) {
-        log_fatal("PMI_KVS_Commit %d\n", rc);
-        return rc;
-    }
-
-
-    log_info("BARRIER\n");
-    if (PMI_SUCCESS != (rc = PMI_Barrier())) {
-        log_fatal("PMI_Barrier %d\n", rc);
-        return rc;
-    }
-
-    for (i = 0; i < size; i++) {
-        for (j=0; j < size; j++) {
-            sprintf(tkey, "%d:KEY-%d", i, j);
-            sprintf(tval, "VALUE-%d", j);
-            log_info("Get key %s\n", tkey);
-            if (PMI_SUCCESS != (rc = PMI_KVS_Get(jobid, tkey, val, sizeof(val)))) {
-                log_fatal("PMI_KVS_Get [%s=?] %d\n", tkey, rc);
-                return rc;
-            }
-
-            log_info("tkey=%s tval=%s val=%s\n", tkey, tval, val);
-
-            log_assert(!strcmp(tval, val), "value does not meet expectation");
-        }
-    }
-
-    return rc;
-}
diff -Nru pmix-3.2.2~rc1/test/pmix_client.c pmix-4.0.0/test/pmix_client.c
--- pmix-3.2.2~rc1/test/pmix_client.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/pmix_client.c	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -22,8 +22,8 @@
  * $HEADER$
  *
  */
-#include <pmix_config.h>
-#include <pmix.h>
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -57,7 +57,7 @@
 
     /* handle early-fail test case */
     if (1 == params.early_fail && 0 == params.rank) {
-        exit(0);
+        exit(1);
     }
 
     /* init us */
@@ -70,19 +70,23 @@
         ninfo = 1;
     }
     if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, info, ninfo))) {
-        TEST_ERROR(("Client ns %s rank %d: PMIx_Init failed: %d", params.nspace, params.rank, rc));
+        TEST_ERROR(("Client ns %s rank %d: PMIx_Init failed: %s", params.nspace, params.rank, PMIx_Error_string(rc)));
         FREE_TEST_PARAMS(params);
         exit(rc);
     }
+    if (myproc.rank != params.rank) {
+        TEST_ERROR(("Client ns %s Rank returned in PMIx_Init %d does not match to rank from command line %d.", myproc.nspace, myproc.rank, params.rank));
+        FREE_TEST_PARAMS(params);
+        exit(1);
+    }
     if ( NULL != params.prefix && -1 != params.ns_id) {
         TEST_SET_FILE(params.prefix, params.ns_id, params.rank);
     }
     TEST_VERBOSE((" Client ns %s rank %d: PMIx_Init success", myproc.nspace, myproc.rank));
 
-    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
-    proc.rank = PMIX_RANK_WILDCARD;
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
     if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_UNIV_SIZE, NULL, 0, &val))) {
-        TEST_ERROR(("rank %d: PMIx_Get universe size failed: %d", myproc.rank, rc));
+        TEST_ERROR(("rank %d: PMIx_Get universe size failed: %s", myproc.rank, PMIx_Error_string(rc)));
         FREE_TEST_PARAMS(params);
         exit(rc);
     }
@@ -102,6 +106,40 @@
 
     TEST_VERBOSE(("rank %d: Universe size check: PASSED", myproc.rank));
 
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_HOSTNAME, NULL, 0, &val))) {
+        TEST_ERROR(("rank %d: PMIx_Get hostname failed: %s", myproc.rank, PMIx_Error_string(rc)));
+        FREE_TEST_PARAMS(params);
+        exit(rc);
+    }
+    if (NULL == val) {
+        TEST_ERROR(("rank %d: PMIx_Get hostname returned NULL value", myproc.rank));
+        FREE_TEST_PARAMS(params);
+        exit(1);
+    }
+    if (val->type != PMIX_STRING) {
+        TEST_ERROR(("rank %d: Hostname type mismatch: %s",
+                    myproc.rank, PMIx_Data_type_string(val->type)));
+        FREE_TEST_PARAMS(params);
+        exit(1);
+    }
+
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_NODEID, NULL, 0, &val))) {
+        TEST_ERROR(("rank %d: PMIx_Get nodeid failed: %s", myproc.rank, PMIx_Error_string(rc)));
+        FREE_TEST_PARAMS(params);
+        exit(rc);
+    }
+    if (NULL == val) {
+        TEST_ERROR(("rank %d: PMIx_Get nodeid returned NULL value", myproc.rank));
+        FREE_TEST_PARAMS(params);
+        exit(1);
+    }
+    if (val->type != PMIX_UINT32) {
+        TEST_ERROR(("rank %d: NodeID type mismatch: %s",
+                    myproc.rank, PMIx_Data_type_string(val->type)));
+        FREE_TEST_PARAMS(params);
+        exit(1);
+    }
+
     if( NULL != params.nspace && 0 != strcmp(myproc.nspace, params.nspace) ) {
         TEST_ERROR(("rank %d: Bad nspace!", myproc.rank));
         FREE_TEST_PARAMS(params);
@@ -112,7 +150,7 @@
         rc = test_fence(params, myproc.nspace, myproc.rank);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d Fence test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d Fence test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -121,7 +159,7 @@
         rc = test_job_fence(params, myproc.nspace, myproc.rank);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d Job fence test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d Job fence test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -130,7 +168,7 @@
         rc = test_publish_lookup(myproc.nspace, myproc.rank);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d Publish/Lookup test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d Publish/Lookup test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -139,7 +177,7 @@
         rc = test_spawn(myproc.nspace, myproc.rank);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d Spawn test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d Spawn test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -148,7 +186,7 @@
         rc = test_connect_disconnect(myproc.nspace, myproc.rank);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d Connect/Disconnect test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d Connect/Disconnect test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -157,7 +195,7 @@
         rc = test_resolve_peers(myproc.nspace, myproc.rank, params);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d Resolve peers test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d Resolve peers test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -166,7 +204,7 @@
         rc = test_error(myproc.nspace, myproc.rank, params);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d error registration and event handling test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d error registration and event handling test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -175,7 +213,7 @@
         rc = test_replace(myproc.nspace, myproc.rank, params);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d error key replace test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d error key replace test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -184,7 +222,7 @@
         rc = test_internal(myproc.nspace, myproc.rank, params);
         if (PMIX_SUCCESS != rc) {
             FREE_TEST_PARAMS(params);
-            TEST_ERROR(("%s:%d error key store internal test failed: %d", myproc.nspace, myproc.rank, rc));
+            TEST_ERROR(("%s:%d error key store internal test failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
             exit(rc);
         }
     }
@@ -199,7 +237,8 @@
     /* finalize us */
     TEST_VERBOSE(("Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank));
     if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
-        TEST_ERROR(("Client ns %s rank %d:PMIx_Finalize failed: %d", myproc.nspace, myproc.rank, rc));
+        TEST_ERROR(("Client ns %s rank %d:PMIx_Finalize failed: %s", myproc.nspace, myproc.rank, PMIx_Error_string(rc)));
+        exit(rc);
     } else {
         TEST_VERBOSE(("Client ns %s rank %d:PMIx_Finalize successfully completed", myproc.nspace, myproc.rank));
     }
@@ -208,5 +247,5 @@
     TEST_OUTPUT_CLEAR(("OK\n"));
     TEST_CLOSE_FILE();
     FREE_TEST_PARAMS(params);
-    exit(rc);
+    exit(0);
 }
diff -Nru pmix-3.2.2~rc1/test/pmix_regex.c pmix-4.0.0/test/pmix_regex.c
--- pmix-3.2.2~rc1/test/pmix_regex.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/pmix_regex.c	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
@@ -75,7 +75,6 @@
         free(regex);
     } else {
         fprintf(stderr, "Node reverse failed: %d\n\n\n", rc);
-        exit(rc);
     }
 
     fprintf(stderr, "PROCS: %s\n", TEST_PROCS);
@@ -92,7 +91,6 @@
         free(regex);
     } else {
         fprintf(stderr, "PPN reverse failed: %d\n", rc);
-        exit(rc);
     }
 
     fprintf(stderr, "NODES: %s\n", TEST_NODES2);
@@ -109,7 +107,6 @@
         free(regex);
     } else {
         fprintf(stderr, "Node reverse failed: %d\n\n\n", rc);
-        exit(rc);
     }
     return 0;
 }
diff -Nru pmix-3.2.2~rc1/test/pmix_test.c pmix-4.0.0/test/pmix_test.c
--- pmix-3.2.2~rc1/test/pmix_test.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/pmix_test.c	2021-01-02 08:56:17.000000000 +0000
@@ -48,6 +48,7 @@
     char **client_argv=NULL;
     int rc, i;
     struct stat stat_buf;
+    int test_fail = 0;
     char *tmp;
     int ns_nprocs;
     sigset_t unblock;
@@ -131,7 +132,7 @@
         while (NULL != pch) {
             ns_nprocs = (int)strtol(pch, NULL, 10);
             if (params.nprocs < (uint32_t)(launched+ns_nprocs)) {
-                TEST_ERROR(("Total number of processes doesn't correspond number specified by ns_dist parameter."));
+                TEST_ERROR(("srv #%d: Total number of processes doesn't correspond number specified by ns_dist parameter.", my_server_id));
                 FREE_TEST_PARAMS(params);
                 return PMIX_ERROR;
             }
@@ -143,7 +144,7 @@
         }
     }
     if (params.lsize != (uint32_t)launched) {
-        TEST_ERROR(("Total number of processes doesn't correspond number specified by ns_dist parameter."));
+        TEST_ERROR(("srv #%d: Total number of processes doesn't correspond number specified by ns_dist parameter.", my_server_id));
         cli_kill_all();
         test_fail = 1;
         goto done;
@@ -158,7 +159,7 @@
     }
 
     if( test_abort ){
-        TEST_ERROR(("Test was aborted!"));
+        TEST_ERROR(("srv #%d: Test was aborted!", my_server_id));
         /* do not simply kill the clients as that generates
          * event notifications which these tests then print
          * out, flooding the log */
@@ -176,12 +177,13 @@
     }
 
     /* deregister the errhandler */
-//    PMIx_Deregister_event_handler(0, op_callbk, NULL);
+    PMIx_Deregister_event_handler(0, op_callbk, NULL);
 
   done:
     TEST_VERBOSE(("srv #%d: call server_finalize!", my_server_id));
-    test_fail += server_finalize(&params);
+    test_fail += server_finalize(&params, test_fail);
 
+    TEST_VERBOSE(("srv #%d: exit sequence!", my_server_id));
     FREE_TEST_PARAMS(params);
     pmix_argv_free(client_argv);
     pmix_argv_free(client_env);
diff -Nru pmix-3.2.2~rc1/test/python/client.py pmix-4.0.0/test/python/client.py
--- pmix-3.2.2~rc1/test/python/client.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/python/client.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+from pmix import *
+import time
+import threading
+
+termEvent = threading.Event()
+
+def default_evhandler(evhdlr:int, status:int,
+                      source:dict, info:list, results:list):
+    print("DEFAULT HANDLER")
+    return PMIX_EVENT_ACTION_COMPLETE,None
+
+def model_evhandler(evhdlr:int, status:int,
+                    source:dict, info:list, results:list):
+    global termEvent
+
+    print("MODEL HANDLER")
+    termEvent.set()
+    return PMIX_EVENT_ACTION_COMPLETE,None
+
+def main():
+    global termEvent
+    
+    foo = PMIxClient()
+    print("Testing PMIx ", foo.get_version())
+    info = [{'key':PMIX_PROGRAMMING_MODEL, 'value':'TEST', 'val_type':PMIX_STRING},
+            {'key':PMIX_MODEL_LIBRARY_NAME, 'value':'PMIX', 'val_type':PMIX_STRING}]
+    my_result = foo.init(info)
+    print("Init result ", my_result)
+    if 0 != my_result[0]:
+        print("FAILED TO INIT")
+        exit(1)
+    # register an event handler
+    rc,myevhndlr = foo.register_event_handler(None, None, default_evhandler)
+    print("REGISTER DEFAULT", foo.error_string(rc))
+    # register the model handler
+    rc,mymodelhndlr = foo.register_event_handler([PMIX_MODEL_DECLARED], None, model_evhandler)
+    print("REGISTER MODEL", foo.error_string(rc))
+
+    # try putting something
+    print("PUT")
+    rc = foo.put(PMIX_GLOBAL, "mykey", {'value':1, 'val_type':PMIX_INT32})
+    print("Put result ",foo.error_string(rc));
+    # commit it
+    print("COMMIT")
+    rc = foo.commit()
+    print ("Commit result ", foo.error_string(rc))
+    # execute fence
+    print("FENCE")
+    procs = []
+    info = []
+    rc = foo.fence(procs, info)
+    print("Fence result ", foo.error_string(rc))
+    print("GET")
+    info = []
+    rc, get_val = foo.get({'nspace':"testnspace", 'rank': 0}, "mykey", info)
+    print("Get result: ", foo.error_string(rc))
+    print("Get value returned: ", get_val)
+    # test a fence that should return not_supported because
+    # we pass a required attribute that doesn't exist
+    procs = []
+    info = [{'key': 'ARBITRARY', 'flags': PMIX_INFO_REQD, 'value':10, 'val_type':PMIX_INT}]
+    rc = foo.fence(procs, info)
+    print("Fence should be not supported:", foo.error_string(rc))
+    # wait for model event
+    while not termEvent.is_set():
+        time.sleep(0.001)
+    # finalize
+    info = []
+    foo.finalize(info)
+    print("Client finalize complete")
+if __name__ == '__main__':
+    main()
diff -Nru pmix-3.2.2~rc1/test/python/Makefile.am pmix-4.0.0/test/python/Makefile.am
--- pmix-3.2.2~rc1/test/python/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/python/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2019      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+noinst_SCRIPTS = \
+	run_server.sh.in \
+	run_sched.sh.in \
+    server.py \
+    sched.py \
+    client.py
+
+#########################
+# Support for "make check"
+
+if WANT_PYTHON_BINDINGS
+
+TESTS = \
+    run_server.sh \
+    run_sched.sh
+
+endif
diff -Nru pmix-3.2.2~rc1/test/python/run_sched.sh.in pmix-4.0.0/test/python/run_sched.sh.in
--- pmix-3.2.2~rc1/test/python/run_sched.sh.in	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/python/run_sched.sh.in	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export PYTHONPATH=@PMIX_PYTHON_EGG_PATH@
+
+./sched.py
diff -Nru pmix-3.2.2~rc1/test/python/run_server.sh.in pmix-4.0.0/test/python/run_server.sh.in
--- pmix-3.2.2~rc1/test/python/run_server.sh.in	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/python/run_server.sh.in	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export PYTHONPATH=@PMIX_PYTHON_EGG_PATH@
+
+./server.py
diff -Nru pmix-3.2.2~rc1/test/python/sched.py pmix-4.0.0/test/python/sched.py
--- pmix-3.2.2~rc1/test/python/sched.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/python/sched.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+from pmix import *
+import signal, time
+import os
+import select
+import subprocess
+
+global killer
+
+class GracefulKiller:
+  kill_now = False
+  def __init__(self):
+    signal.signal(signal.SIGINT, self.exit_gracefully)
+    signal.signal(signal.SIGTERM, self.exit_gracefully)
+
+  def exit_gracefully(self,signum, frame):
+    self.kill_now = True
+
+def clientconnected(proc:tuple is not None):
+    print("CLIENT CONNECTED", proc)
+    return PMIX_SUCCESS
+
+def clientfinalized(proc:tuple is not None):
+    print("CLIENT FINALIZED", proc)
+    return PMIX_SUCCESS
+
+def clientfence(args:dict is not None):
+    print("SERVER FENCE", args)
+    return PMIX_SUCCESS
+
+def main():
+    try:
+        foo = PMIxServer()
+    except:
+        print("FAILED TO CREATE SERVER")
+        exit(1)
+    print("Testing server version ", foo.get_version())
+    args = [{'key':PMIX_SERVER_SCHEDULER, 'value':'T', 'val_type':PMIX_BOOL}]
+    map = {'clientconnected': clientconnected,
+           'clientfinalized': clientfinalized,
+           'fencenb': clientfence}
+    my_result = foo.init(args, map)
+    print("Testing PMIx_Initialized")
+    rc = foo.initialized()
+    print("Initialized: ", rc)
+    vers = foo.get_version()
+    print("Version: ", vers)
+
+    # Register a fabric
+    rc,fab = foo.fabric_register(None)
+    print("Fabric registered: ", foo.error_string(rc))
+
+    # setup the application
+    (rc, regex) = foo.generate_regex(["test000","test001","test002"])
+    print("Node regex, rc: ", regex, foo.error_string(rc))
+    (rc, ppn) = foo.generate_ppn(["0,1,2", "3,4,5", "6,7"])
+    print("PPN, rc: ", ppn, foo.error_string(rc))
+    darray = {'type':PMIX_INFO, 'array':[{'key':PMIX_ALLOC_FABRIC_ID,
+                            'value':'SIMPSCHED.net', 'val_type':PMIX_STRING},
+                           {'key':PMIX_ALLOC_FABRIC_SEC_KEY, 'value':'T',
+                            'val_type':PMIX_BOOL},
+                           {'key':PMIX_SETUP_APP_ENVARS, 'value':'T',
+                            'val_type':PMIX_BOOL}]}
+    kyvals = [{'key':PMIX_NODE_MAP, 'value':regex, 'val_type':PMIX_STRING},
+              {'key':PMIX_PROC_MAP, 'value':ppn, 'val_type':PMIX_STRING},
+              {'key':PMIX_ALLOC_FABRIC, 'value':darray, 'val_type':PMIX_DATA_ARRAY}]
+
+    appinfo = []
+    rc, appinfo = foo.setup_application("SIMPSCHED", kyvals)
+    print("SETUPAPP: ", appinfo)
+
+    rc = foo.setup_local_support("SIMPSCHED", appinfo)
+    print("SETUPLOCAL: ", foo.error_string(rc))
+
+    # get our environment as a base
+    env = os.environ.copy()
+
+    # register an nspace for the client app
+    kvals = [{'key':PMIX_NODE_MAP, 'value':regex, 'val_type':PMIX_STRING},
+             {'key':PMIX_PROC_MAP, 'value':ppn, 'val_type':PMIX_STRING},
+             {'key':PMIX_UNIV_SIZE, 'value':1, 'val_type':PMIX_UINT32},
+             {'key':PMIX_JOB_SIZE, 'value':1, 'val_type':PMIX_UINT32}]
+    print("REGISTERING NSPACE")
+    rc = foo.register_nspace("testnspace", 1, kvals)
+    print("RegNspace ", foo.error_string(rc))
+
+    # register a client
+    uid = os.getuid()
+    gid = os.getgid()
+    print("REGISTERING CLIENT")
+    rc = foo.register_client({'nspace':"testnspace", 'rank':0}, uid, gid)
+    print("RegClient ", foo.error_string(rc))
+
+    # setup the fork
+    rc = foo.setup_fork({'nspace':"testnspace", 'rank':0}, env)
+    print("SetupFrk", foo.error_string(rc))
+
+    # setup the client argv
+    args = ["./client.py"]
+    # open a subprocess with stdout and stderr
+    # as distinct pipes so we can capture their
+    # output as the process runs
+    p = subprocess.Popen(args, env=env,
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # define storage to catch the output
+    stdout = []
+    stderr = []
+    # loop until the pipes close
+    while True:
+        reads = [p.stdout.fileno(), p.stderr.fileno()]
+        ret = select.select(reads, [], [])
+
+        stdout_done = True
+        stderr_done = True
+
+        for fd in ret[0]:
+            # if the data
+            if fd == p.stdout.fileno():
+                read = p.stdout.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    print('stdout: ' + read)
+                    stdout_done = False
+            elif fd == p.stderr.fileno():
+                read = p.stderr.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    print('stderr: ' + read)
+                    stderr_done = False
+
+        if stdout_done and stderr_done:
+            break
+
+    print("FINALIZING")
+    foo.finalize()
+
+if __name__ == '__main__':
+    global killer
+    killer = GracefulKiller()
+    main()
diff -Nru pmix-3.2.2~rc1/test/python/server.py pmix-4.0.0/test/python/server.py
--- pmix-3.2.2~rc1/test/python/server.py	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/python/server.py	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+from pmix import *
+import signal, time
+import os
+import select
+import subprocess
+
+global killer
+
+class GracefulKiller:
+  kill_now = False
+  def __init__(self):
+    signal.signal(signal.SIGINT, self.exit_gracefully)
+    signal.signal(signal.SIGTERM, self.exit_gracefully)
+
+  def exit_gracefully(self,signum, frame):
+    self.kill_now = True
+
+def clientconnected(proc:dict is not None):
+    print("CLIENT CONNECTED", proc)
+    return PMIX_SUCCESS
+
+def clientfinalized(proc:dict is not None):
+    print("CLIENT FINALIZED", proc)
+    return PMIX_SUCCESS
+
+def clientfence(args:dict is not None):
+    # check directives
+    try:
+        if args['directives'] is not None:
+            for d in args['directives']:
+                # these are each an info dict
+                if "pmix" not in d['key']:
+                    # we do not support such directives - see if
+                    # it is required
+                    try:
+                        if d['flags'] & PMIX_INFO_REQD:
+                            # return an error
+                            return PMIX_ERR_NOT_SUPPORTED
+                    except:
+                        #it can be ignored
+                        pass
+    except:
+        pass
+    return PMIX_SUCCESS, None
+
+def main():
+    try:
+        foo = PMIxServer()
+    except:
+        print("FAILED TO CREATE SERVER")
+        exit(1)
+    print("Testing server version ", foo.get_version())
+    args = [{'key':'FOOBAR', 'value':'VAR', 'val_type':PMIX_STRING},
+            {'key':'BLAST', 'value':7, 'val_type':PMIX_INT32}]
+    map = {'clientconnected': clientconnected,
+           'clientfinalized': clientfinalized,
+           'fencenb': clientfence}
+    my_result = foo.init(args, map)
+    print("Testing PMIx_Initialized")
+    rc = foo.initialized()
+    print("Initialized: ", rc)
+    vers = foo.get_version()
+    print("Version: ", vers)
+
+    # get our environment as a base
+    env = os.environ.copy()
+    # register an nspace for the client app
+    (rc, regex) = foo.generate_regex(["test000","test001","test002"])
+    (rc, ppn) = foo.generate_ppn(["0,1,2", "3,4,5", "6,7"])
+    kvals = [{'key':PMIX_NODE_MAP, 'value':regex, 'val_type':PMIX_STRING},
+             {'key':PMIX_PROC_MAP, 'value':ppn, 'val_type':PMIX_STRING},
+             {'key':PMIX_UNIV_SIZE, 'value':1, 'val_type':PMIX_UINT32},
+             {'key':PMIX_JOB_SIZE, 'value':1, 'val_type':PMIX_UINT32}]
+    print("REGISTERING NSPACE")
+    rc = foo.register_nspace("testnspace", 1, kvals)
+    print("RegNspace ", foo.error_string(rc))
+
+    # register a client
+    uid = os.getuid()
+    gid = os.getgid()
+    rc = foo.register_client({'nspace':"testnspace", 'rank':0}, uid, gid)
+    print("RegClient ", foo.error_string(rc))
+    # setup the fork
+    rc = foo.setup_fork({'nspace':"testnspace", 'rank':0}, env)
+    print("SetupFrk", foo.error_string(rc))
+
+    # setup the client argv
+    args = ["./client.py"]
+    # open a subprocess with stdout and stderr
+    # as distinct pipes so we can capture their
+    # output as the process runs
+    p = subprocess.Popen(args, env=env,
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # define storage to catch the output
+    stdout = []
+    stderr = []
+    # loop until the pipes close
+    while True:
+        reads = [p.stdout.fileno(), p.stderr.fileno()]
+        ret = select.select(reads, [], [])
+
+        stdout_done = True
+        stderr_done = True
+
+        for fd in ret[0]:
+            # if the data
+            if fd == p.stdout.fileno():
+                read = p.stdout.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    print('stdout: ' + read)
+                    stdout_done = False
+            elif fd == p.stderr.fileno():
+                read = p.stderr.readline()
+                if read:
+                    read = read.decode('utf-8').rstrip()
+                    print('stderr: ' + read)
+                    stderr_done = False
+
+        if stdout_done and stderr_done:
+            break
+    print("FINALIZING")
+    foo.finalize()
+
+
+if __name__ == '__main__':
+    global killer
+    killer = GracefulKiller()
+    main()
diff -Nru pmix-3.2.2~rc1/test/run_tests00.pl.in pmix-4.0.0/test/run_tests00.pl.in
--- pmix-3.2.2~rc1/test/run_tests00.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests00.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests01.pl.in pmix-4.0.0/test/run_tests01.pl.in
--- pmix-3.2.2~rc1/test/run_tests01.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests01.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests02.pl.in pmix-4.0.0/test/run_tests02.pl.in
--- pmix-3.2.2~rc1/test/run_tests02.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests02.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests03.pl.in pmix-4.0.0/test/run_tests03.pl.in
--- pmix-3.2.2~rc1/test/run_tests03.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests03.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests04.pl.in pmix-4.0.0/test/run_tests04.pl.in
--- pmix-3.2.2~rc1/test/run_tests04.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests04.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests05.pl.in pmix-4.0.0/test/run_tests05.pl.in
--- pmix-3.2.2~rc1/test/run_tests05.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests05.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests06.pl.in pmix-4.0.0/test/run_tests06.pl.in
--- pmix-3.2.2~rc1/test/run_tests06.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests06.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests07.pl.in pmix-4.0.0/test/run_tests07.pl.in
--- pmix-3.2.2~rc1/test/run_tests07.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests07.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests08.pl.in pmix-4.0.0/test/run_tests08.pl.in
--- pmix-3.2.2~rc1/test/run_tests08.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests08.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests09.pl.in pmix-4.0.0/test/run_tests09.pl.in
--- pmix-3.2.2~rc1/test/run_tests09.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests09.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests10.pl.in pmix-4.0.0/test/run_tests10.pl.in
--- pmix-3.2.2~rc1/test/run_tests10.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests10.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests11.pl.in pmix-4.0.0/test/run_tests11.pl.in
--- pmix-3.2.2~rc1/test/run_tests11.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests11.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests12.pl.in pmix-4.0.0/test/run_tests12.pl.in
--- pmix-3.2.2~rc1/test/run_tests12.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests12.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests13.pl.in pmix-4.0.0/test/run_tests13.pl.in
--- pmix-3.2.2~rc1/test/run_tests13.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests13.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests14.pl.in pmix-4.0.0/test/run_tests14.pl.in
--- pmix-3.2.2~rc1/test/run_tests14.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests14.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests15.pl.in pmix-4.0.0/test/run_tests15.pl.in
--- pmix-3.2.2~rc1/test/run_tests15.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests15.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/run_tests.pl.in pmix-4.0.0/test/run_tests.pl.in
--- pmix-3.2.2~rc1/test/run_tests.pl.in	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/run_tests.pl.in	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/env perl
 #
-# Copyright (c) 2019      Intel, Inc.
+# Copyright (c) 2019-2020 Intel, Inc.  All rights reserved.
 #
 # Copyright (c) 2019      Cisco Systems, Inc.  All rights reserved
 # $COPYRIGHT$
@@ -25,6 +25,8 @@
              "-n 5 --test-internal 10",
              "-s 1 -n 2 --job-fence",
              "-s 1 -n 2 --job-fence -c");
+#             "-s 2 -n 2 --job-fence",
+#            "-s 2 -n 2 --job-fence -c");
 
 my $test;
 my $cmd;
diff -Nru pmix-3.2.2~rc1/test/server_callbacks.c pmix-4.0.0/test/server_callbacks.c
--- pmix-3.2.2~rc1/test/server_callbacks.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/server_callbacks.c	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
@@ -32,12 +32,9 @@
     .unpublish = unpublish_fn,
     .spawn = spawn_fn,
     .connect = connect_fn,
-    .disconnect = disconnect_fn
-#if 0
-,
+    .disconnect = disconnect_fn,
     .register_events = regevents_fn,
     .deregister_events = deregevents_fn
-#endif
 };
 
 typedef struct {
@@ -143,9 +140,19 @@
               const pmix_info_t info[], size_t ninfo,
               pmix_modex_cbfunc_t cbfunc, void *cbdata)
 {
+    size_t n;
+
     TEST_VERBOSE(("Getting data for %s:%d", proc->nspace, proc->rank));
 
-    /* return not_found fot single server mode */
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+                return PMIX_ERR_NOT_SUPPORTED;
+            }
+        }
+    }
+
+    /* return not_found for single server mode */
     if ((pmix_list_get_size(server_list) == 1) && (my_server_id == 0)) {
         return PMIX_ERR_NOT_FOUND;
     }
@@ -173,7 +180,7 @@
         }
         if (!found) {
             new_info = PMIX_NEW(pmix_test_info_t);
-            strncpy(new_info->data.key, info[i].key, strlen(info[i].key)+1);
+            PMIX_LOAD_KEY(new_info->data.key, info[i].key);
             pmix_value_xfer(&new_info->data.value, (pmix_value_t*)&info[i].value);
             new_info->namespace_published = strdup(proc->nspace);
             new_info->rank_published = proc->rank;
diff -Nru pmix-3.2.2~rc1/test/simple/doubleget.c pmix-4.0.0/test/simple/doubleget.c
--- pmix-3.2.2~rc1/test/simple/doubleget.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/doubleget.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,193 @@
+#include <stdio.h>
+#include "include/pmix.h"
+#include <sched.h>
+
+static pmix_proc_t allproc = {0};
+static pmix_proc_t myproc = {0};
+static bool mywait = false;
+static bool refresh = false;
+static bool timeout = false;
+
+#define ERR(msg, ...)							\
+    do {								\
+	time_t tm = time(NULL);						\
+	char *stm = ctime(&tm);						\
+	stm[strlen(stm)-1] = 0;						\
+	fprintf(stderr, "%s ERROR: %s:%d  " msg "\n", stm, __FILE__, __LINE__, ## __VA_ARGS__); \
+	exit(1);							\
+    } while(0);
+
+
+static int pmi_set_string(const char *key, void *data, size_t size)
+{
+    int rc;
+    pmix_value_t value;
+
+    PMIX_VALUE_CONSTRUCT(&value);
+    value.type = PMIX_BYTE_OBJECT;
+    value.data.bo.bytes = data;
+    value.data.bo.size  = size;
+    if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, key, &value))) {
+        ERR("Client ns %s rank %d: PMIx_Put failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    }
+
+    if (PMIX_SUCCESS != (rc = PMIx_Commit())) {
+        ERR("Client ns %s rank %d: PMIx_Commit failed: %s\n", myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    }
+
+    /* protect the data */
+    value.data.bo.bytes = NULL;
+    value.data.bo.size  = 0;
+    PMIX_VALUE_DESTRUCT(&value);
+    printf("%s:%d PMIx_Put on %s\n", myproc.nspace, myproc.rank, key);
+
+
+    return 0;
+}
+
+static int pmi_get_string(uint32_t peer_rank, const char *key, void **data_out, size_t *data_size_out)
+{
+    int rc;
+    pmix_proc_t proc;
+    pmix_value_t *pvalue;
+    pmix_info_t info;
+
+    PMIX_LOAD_PROCID(&proc, myproc.nspace, peer_rank);
+    if (refresh) {
+        PMIX_INFO_LOAD(&info, PMIX_GET_REFRESH_CACHE, &refresh, PMIX_BOOL);
+        rc = PMIx_Get(&proc, key, &info, 1, &pvalue);
+        PMIX_INFO_DESTRUCT(&info);
+    } else if (timeout) {
+        rc = 2;
+        PMIX_INFO_LOAD(&info, PMIX_TIMEOUT, &rc, PMIX_INT);
+        rc = PMIx_Get(&proc, key, &info, 1, &pvalue);
+        PMIX_INFO_DESTRUCT(&info);
+    } else {
+        rc = PMIx_Get(&proc, key, NULL, 0, &pvalue);
+    }
+    if (PMIX_SUCCESS != rc) {
+        ERR("Client ns %s rank %d: PMIx_Get on rank %u %s: %s\n", myproc.nspace, myproc.rank, peer_rank, key, PMIx_Error_string(rc));
+    }
+    if(pvalue->type != PMIX_BYTE_OBJECT){
+        ERR("Client ns %s rank %d: PMIx_Get %s: got wrong data type\n", myproc.nspace, myproc.rank, key);
+    }
+    *data_out = pvalue->data.bo.bytes;
+    *data_size_out = pvalue->data.bo.size;
+
+    /* protect the data */
+    pvalue->data.bo.bytes = NULL;
+    pvalue->data.bo.size = 0;
+    PMIX_VALUE_RELEASE(pvalue);
+    PMIX_PROC_DESTRUCT(&proc);
+
+    printf("%s:%d PMIx_get %s returned %zi bytes\n", myproc.nspace, myproc.rank, key, data_size_out[0]);
+
+    return 0;
+}
+
+static int pmix_exchange(bool flag)
+{
+    int rc;
+    pmix_info_t info;
+
+    fprintf(stderr, "Execute fence\n");
+    PMIX_INFO_CONSTRUCT(&info);
+    PMIX_INFO_LOAD(&info, PMIX_COLLECT_DATA, &flag, PMIX_BOOL);
+    rc = PMIx_Fence(&allproc, 1, &info, 1);
+    if (PMIX_SUCCESS != rc){
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Fence_nb failed: %d\n", myproc.nspace, myproc.rank, rc);
+        exit(1);
+    }
+    PMIX_INFO_DESTRUCT(&info);
+
+
+    return 0;
+}
+
+int main(int argc, char *argv[])
+{
+    char data[256] = {0};
+    char *data_out;
+    size_t size_out;
+    int rc;
+    pmix_value_t *pvalue;
+
+    /* check the args */
+    if (1 < argc) {
+        if (2 < argc || 0 == strncmp(argv[1], "-h", 2) || 0 == strncmp(argv[1], "--h", 3)) {
+            fprintf(stderr, "Usage:\n");
+            fprintf(stderr, "\t--wait       Test PMIX_GET_WAIT_FOR_KEY\n");
+            fprintf(stderr, "\t--refresh    Test PMIX_GET_REFRESH_CACHE\n");
+            fprintf(stderr, "\t--timeout    Test PMIX_GET_WAIT_FOR_KEY, but timeout\n");
+            exit(0);
+        }
+        if (0 == strncmp(argv[1], "--w", 3)) {
+            mywait = true;
+        } else if (0 == strncmp(argv[1], "--r", 3)) {
+            refresh = true;
+        } else if (0 == strncmp(argv[1], "--t", 3)) {
+            timeout = true;
+        } else {
+            fprintf(stderr, "Invalid cmd line option: %s\n", argv[1]);
+            fprintf(stderr, "Usage:\n");
+            fprintf(stderr, "\t--wait       Test PMIX_GET_WAIT_FOR_KEY\n");
+            fprintf(stderr, "\t--refresh    Test PMIX_GET_REFRESH_CACHE\n");
+            fprintf(stderr, "\t--timeout    Test PMIX_GET_WAIT_FOR_KEY, but timeout\n");
+            exit(1);
+        }
+    }
+
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+	   ERR("PMIx_Init failed");
+        exit(1);
+    }
+    if (myproc.rank == 0) {
+        printf("PMIx initialized\n");
+    }
+
+    /* job-related info is found in our nspace, assigned to the
+     * wildcard rank as it doesn't relate to a specific rank. Setup
+     * a name to retrieve such values */
+    PMIX_LOAD_PROCID(&allproc, myproc.nspace, PMIX_RANK_WILDCARD);
+
+    /* get the number of procs in our job */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&allproc, PMIX_JOB_SIZE, NULL, 0, &pvalue))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get job size failed: %d\n", myproc.nspace, myproc.rank, rc);
+        exit(1);
+    }
+    PMIX_VALUE_RELEASE(pvalue);
+
+    /* the below two lines break the subsequent PMIx_Get query on a key set later */
+    sprintf(data, "FIRST TIME rank %d", myproc.rank);
+    pmi_set_string("test-key-1", data, 256);
+    pmix_exchange(true);
+
+    if (1 == myproc.rank) {
+        if (timeout) {
+            sleep(10);
+        } else if (mywait) {
+            sleep(2);
+        }
+    }
+
+    sprintf(data, "SECOND TIME rank %d", myproc.rank);
+    if (0 == myproc.rank) {
+        pmi_set_string("test-key-2", data, 256);
+    } else {
+        pmi_set_string("test-key-3", data, 256);
+    }
+
+    if (0 == myproc.rank) {
+        pmi_get_string(1, "test-key-3", (void**)&data_out, &size_out);
+    } else {
+        pmi_get_string(0, "test-key-2", (void**)&data_out, &size_out);
+    }
+    printf("%d: obtained data \"%s\"\n", myproc.rank, data_out);
+
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        ERR("Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
+    }
+    if(myproc.rank == 0) printf("PMIx finalized\n");
+
+    exit(0);
+}
diff -Nru pmix-3.2.2~rc1/test/simple/get_put_example.c pmix-4.0.0/test/simple/get_put_example.c
--- pmix-3.2.2~rc1/test/simple/get_put_example.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/get_put_example.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,58 @@
+#include <stdbool.h>
+#include <sys/types.h>
+#include <pmix.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+
+int main(int argc, char **argv)
+{
+    int rc;
+    pmix_value_t value;
+    pmix_value_t *val = &value;
+    pmix_value_t pvalue;
+    pmix_proc_t myproc, rootproc;
+    pmix_info_t info;
+
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        fprintf(stderr, "Init failed.\n");
+        exit(1);
+    }
+    fprintf(stderr, "[%s:%u]: Running\n", myproc.nspace, myproc.rank);
+
+    PMIX_VALUE_LOAD(&pvalue, "hello", PMIX_STRING);
+    if (myproc.rank == 0) {
+      if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, "my.foo", &pvalue))) {
+        fprintf(stderr, "Put of my.foo failed\n");
+        exit(1);
+       }
+    }
+
+    PMIx_Commit();
+
+    bool tvalue = 1;
+    strcpy(info.key, PMIX_COLLECT_DATA);
+    PMIX_VALUE_LOAD(&(info.value), &tvalue, PMIX_BOOL);
+    PMIx_Fence(NULL, 0, &info, 1);
+
+    fprintf(stderr, "[%s:%u]: Fence complete\n", myproc.nspace, myproc.rank);
+
+    rootproc = myproc;
+    rootproc.rank = 0;
+
+
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&rootproc, "my.foo", NULL, 0, &val))) {
+       fprintf(stderr, "Get of root's my.foo failed\n");
+       exit(1);
+    }
+    fprintf(stderr, "[%s:%u]: Get complete and successful\n", myproc.nspace, myproc.rank);
+
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+       fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
+       exit(1);
+    }
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/test/simple/gwtest.c pmix-4.0.0/test/simple/gwtest.c
--- pmix-3.2.2~rc1/test/simple/gwtest.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/gwtest.c	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -26,7 +26,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix_server.h>
+#include "include/pmix_server.h"
 #include "src/include/types.h"
 #include "src/include/pmix_globals.h"
 
@@ -38,7 +38,6 @@
 #include <sys/wait.h>
 #include <errno.h>
 #include <signal.h>
-#include PMIX_EVENT_HEADER
 
 #include "src/class/pmix_list.h"
 #include "src/util/pmix_environ.h"
@@ -581,14 +580,10 @@
     x->info[3].value.data.string = strdup(ranks);
 
     PMIx_generate_regex(hostname, &regex);
-    (void)strncpy(x->info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN);
-    x->info[4].value.type = PMIX_STRING;
-    x->info[4].value.data.string = regex;
+    PMIX_INFO_LOAD(&x->info[4], PMIX_NODE_MAP, regex, PMIX_REGEX);
 
     PMIx_generate_ppn(ranks, &ppn);
-    (void)strncpy(x->info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN);
-    x->info[5].value.type = PMIX_STRING;
-    x->info[5].value.data.string = ppn;
+    PMIX_INFO_LOAD(&x->info[5], PMIX_PROC_MAP, ppn, PMIX_REGEX);
 
     (void)strncpy(x->info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN);
     x->info[6].value.type = PMIX_UINT32;
diff -Nru pmix-3.2.2~rc1/test/simple/makedata.pl pmix-4.0.0/test/simple/makedata.pl
--- pmix-3.2.2~rc1/test/simple/makedata.pl	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/makedata.pl	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,46 @@
+#!/usr/bin/env perl
+
+# Make a simple data file of a argv[0]-specified size (understand "k",
+# "m", and "g" suffixes) of a repeating pattern.
+
+use strict;
+
+my $size_arg = $ARGV[0];
+$size_arg =~ m/^(\d+)/;
+my $size = $1;
+$size_arg =~ m/([mkg])$/i;
+my $size_unit = lc($1);
+
+if ($size_unit eq "k") {
+    $size *= 1024;
+} elsif ($size_unit eq "m") {
+    $size *= 1048576;
+} elsif ($size_unit eq "g") {
+    $size *= 1073741824;
+}
+print "Generating size $size\n";
+
+my $file = "data-" . lc($size_arg);
+unlink($file);
+
+my $line = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^&*()_+=-;";
+my $line_len = length($line);
+$line .= $line;
+
+open(FILE, ">$file") || die "Can't open file $file";
+my $count = 0;
+my $line_count = 0;
+while ($count < $size) {
+    my $offset = $line_count % $line_len;
+    my $num_to_print =
+        ($size - $count < $line_len) ? $size - $count : $line_len;
+    if ($num_to_print > 0) {
+        my $printable = substr($line, $offset, $num_to_print - 1);
+        print FILE $printable . "\n";
+        $count += $num_to_print;
+    }
+    ++$line_count;
+}
+close(FILE);
+
+exit(0);
diff -Nru pmix-3.2.2~rc1/test/simple/Makefile.am pmix-4.0.0/test/simple/Makefile.am
--- pmix-3.2.2~rc1/test/simple/Makefile.am	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -24,9 +24,9 @@
 headers = simptest.h
 
 noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex \
-                  test_pmix simptool simpdie simplegacy simptimeout \
-                  gwtest gwclient stability quietclient simpjctrl \
-                  pmitest
+                  test_pmix simptool simpdie simptimeout \
+                  gwtest gwclient stability quietclient simpjctrl simpio simpsched \
+                  simpcoord simpcycle doubleget simpfabric get_put_example simpvni
 
 simptest_SOURCES = \
         simptest.c
@@ -82,12 +82,6 @@
 simpdie_LDADD = \
     $(top_builddir)/src/libpmix.la
 
-simplegacy_SOURCES = \
-        simplegacy.c
-simplegacy_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
-simplegacy_LDADD = \
-    $(top_builddir)/src/libpmi.la
-
 simptimeout_SOURCES = \
         simptimeout.c
 simptimeout_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
@@ -124,8 +118,50 @@
 simpjctrl_LDADD = \
     $(top_builddir)/src/libpmix.la
 
-pmitest_SOURCES = \
-        pmitest.c
-pmitest_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
-pmitest_LDADD = \
-    $(top_builddir)/src/libpmi.la
+simpio_SOURCES = \
+        simpio.c
+simpio_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simpio_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+simpsched_SOURCES = \
+        simpsched.c
+simpsched_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simpsched_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+simpcoord_SOURCES = \
+        simpcoord.c
+simpcoord_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simpcoord_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+simpcycle_SOURCES = \
+        simpcycle.c
+simpcycle_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simpcycle_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+doubleget_SOURCES = \
+        doubleget.c
+doubleget_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+doubleget_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+simpfabric_SOURCES = \
+        simpfabric.c
+simpfabric_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simpfabric_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+get_put_example_SOURCES = \
+        get_put_example.c
+get_put_example_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+get_put_example_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+simpvni_SOURCES = \
+        simpvni.c
+simpvni_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+simpvni_LDADD = \
+    $(top_builddir)/src/libpmix.la
diff -Nru pmix-3.2.2~rc1/test/simple/netconfig.txt pmix-4.0.0/test/simple/netconfig.txt
--- pmix-3.2.2~rc1/test/simple/netconfig.txt	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/netconfig.txt	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,10 @@
+# Example coordinate config file for the pnet/simptest component
+# Written as:
+# nodename  x  y  z  group
+
+test000  0  0  0  0
+test001  1  0  0  0
+test002  2  0  0  0
+test003  0  0  0  1
+test004  1  0  0  1
+test005  2  0  0  1
diff -Nru pmix-3.2.2~rc1/test/simple/pmitest.c pmix-4.0.0/test/simple/pmitest.c
--- pmix-3.2.2~rc1/test/simple/pmitest.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/pmitest.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,305 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
-/*
- *
- *  (C) 2001 by Argonne National Laboratory.
- *      See COPYRIGHT in top-level directory.
- */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "pmi.h"
-
-static const char * PMI_Err_str(int error)
-{
-    static char str[100];
-    switch (error)
-    {
-    case PMI_SUCCESS:
-	return "PMI_SUCCESS";
-    case PMI_FAIL:
-	return "PMI_FAIL";
-    case PMI_ERR_INVALID_ARG:
-	return "PMI_ERR_INVALID_ARG";
-    case PMI_ERR_INVALID_KEY:
-	return "PMI_ERR_INVALID_KEY";
-    case PMI_ERR_INVALID_KEY_LENGTH:
-	return "PMI_ERR_INVALID_KEY_LENGTH";
-    case PMI_ERR_INVALID_VAL:
-	return "PMI_ERR_INVALID_VAL";
-    case PMI_ERR_INVALID_VAL_LENGTH:
-	return "PMI_ERR_INVALID_VAL_LENGTH";
-    case PMI_ERR_INVALID_LENGTH:
-	return "PMI_ERR_INVALID_LENGTH";
-    case PMI_ERR_INIT:
-	return "PMI_ERR_INIT";
-    case PMI_ERR_NOMEM:
-	return "PMI_ERR_NOMEM";
-    }
-    sprintf(str, "PMI_ERR_UNKNOWN: %d", error);
-    return str;
-}
-
-#define PRINT_ERROR(error, fcname) if (error != PMI_SUCCESS) printf("%s failed: %s\n", fcname, PMI_Err_str(error)); else printf("%s unexpectedly succeeded\n", fcname); fflush(stdout);
-
-int main( int argc, char * argv[] )
-{
-    int rc, spawned, size, rank, name_max, id_maxlen, key_maxlen, val_maxlen;
-    char *kvsname, *id, *domain_id, *key, *val;
-
-    rc = PMI_Init( &spawned );
-    if ( rc != PMI_SUCCESS )
-    {
-       printf( "PMI_Init failed with rc = %s\n", PMI_Err_str(rc) );
-       return -1 ;
-   }
-   else
-   {
-       printf( "PMI_Init returned spawned = %d\n", spawned );
-   }
-
-   rc = PMI_Get_size( &size );
-   if ( rc == PMI_SUCCESS )
-   {
-       rc = PMI_Get_rank( &rank );
-       if ( rc == PMI_SUCCESS )
-           printf( "size = %d, rank = %d\n", size, rank );
-       else
-           printf( "PMI_Get_Rank failed with rc = %s\n", PMI_Err_str(rc) );
-   }
-   else
-       printf( "PMI_Get_size failed with rc = %s\n", PMI_Err_str(rc) );
-
-   rc = PMI_KVS_Get_name_length_max( &name_max );
-   if ( rc != PMI_SUCCESS )
-   {
-       printf( "PMI_KVS_Get_name_length_max failed with rc = %s\n", PMI_Err_str(rc) );
-       return -1;
-   }
-   else
-       printf( "PMI_KVS_Get_name_length_max got %d\n", name_max );
-
-   kvsname = (char *) malloc( name_max );
-   rc = PMI_KVS_Get_my_name( kvsname, name_max );
-   if ( rc != PMI_SUCCESS )
-   {
-       printf( "PMI_KVS_Get_my_name failed with rc = %s\n", PMI_Err_str(rc) );
-       return -1;
-   }
-   else
-       printf( "PMI_KVS_Get_my_name got %s\n", kvsname );
-
-   rc = PMI_Get_id_length_max( &id_maxlen );
-   if ( rc != PMI_SUCCESS )
-   {
-       printf("PMI_Get_id_length_max failed with rc = %s\n", PMI_Err_str(rc) );
-       return -1;
-   }
-   else
-       printf("PMI_Get_id_length_max got %d\n", id_maxlen);
-   id = (char *) malloc( id_maxlen );
-   rc = PMI_Get_id( id, id_maxlen );
-   if ( rc != PMI_SUCCESS )
-   {
-       printf("PMI_Get_id failed with rc = %s\n", PMI_Err_str(rc));
-   }
-   else
-       printf( "PMI_Get_id got %s\n", id );
-   domain_id = (char *) malloc( id_maxlen );
-   rc = PMI_Get_kvs_domain_id( domain_id, id_maxlen );
-   if ( rc != PMI_SUCCESS )
-   {
-       printf("PMI_Get_kvs_domain_id failed with rc = %s\n", PMI_Err_str(rc));
-   }
-   else
-       printf( "PMI_Get_kvs_domain_id got %s\n", domain_id );
-
-   rc = PMI_KVS_Get_key_length_max( &key_maxlen );
-   if (rc != PMI_SUCCESS )
-   {
-       printf("PMI_KVS_Get_key_length_max failed with rc = %s\n", PMI_Err_str(rc));
-       return -1;
-   }
-   else
-       printf( "PMI_Get_key_maxlen got %d\n", key_maxlen );
-   key = (char *) malloc( key_maxlen );
-   rc = PMI_KVS_Get_value_length_max( &val_maxlen );
-   if (rc != PMI_SUCCESS)
-   {
-       printf("PMI_KVS_Get_value_length_max failed with rc = %s\n", PMI_Err_str(rc));
-       return -1;
-   }
-   else
-       printf( "PMI_Get_val_maxlen got %d\n", val_maxlen );
-   val = (char *) malloc( val_maxlen );
-
-   sprintf(key, "test_key_%d", rank);
-   sprintf(val, "test_value_%d", rank);
-
-   rc = PMI_KVS_Put( kvsname, key, val );
-   if (rc != PMI_SUCCESS)
-   {
-       printf("PMI_KVS_Put failed with rc = %s\n", PMI_Err_str(rc));
-   }
-   rc = PMI_KVS_Commit( kvsname );
-   if (rc != PMI_SUCCESS)
-   {
-       printf("PMI_KVS_Commit failed with rc = %s\n", PMI_Err_str(rc));
-   }
-   rc = PMI_Barrier();
-   if (rc != PMI_SUCCESS)
-   {
-       printf("PMI_Barrier failed with rc = %s\n", PMI_Err_str(rc));
-   }
-
-   sprintf(key, "test_key_%d", (rank + 1) % size);
-   rc = PMI_KVS_Get( kvsname, key, val, val_maxlen );
-   if (rc != PMI_SUCCESS)
-   {
-       printf("PMI_KVS_Get(%s) failed with rc = %s\n", key, PMI_Err_str(rc));
-   }
-   else
-       printf("PMI_KVS_Get(%s) returned %s\n", key, val);
-
-    /* Test awkward character string put and get */
-   if (rank == 0)
-   {
-       sprintf(key, "foo");
-       sprintf(val, "foo=bar baz=bif name=\"Buzz Bee\" clink=~!@#$\\;':<>,. clank=a b c");
-
-       rc = PMI_KVS_Put( kvsname, key, val );
-       if (rc != PMI_SUCCESS)
-       {
-           printf("PMI_KVS_Put failed with rc = %s\n", PMI_Err_str(rc));
-       }
-       rc = PMI_KVS_Commit( kvsname );
-       if (rc != PMI_SUCCESS)
-       {
-           printf("PMI_KVS_Commit failed with rc = %s\n", PMI_Err_str(rc));
-       }
-   }
-
-   rc = PMI_Barrier();
-   if (rc != PMI_SUCCESS)
-   {
-       printf("PMI_Barrier failed with rc = %s\n", PMI_Err_str(rc));
-   }
-
-   if (rank == size - 1)
-   {
-       sprintf(key, "foo");
-       rc = PMI_KVS_Get( kvsname, key, val, val_maxlen );
-       if (rc != PMI_SUCCESS)
-       {
-           printf("PMI_KVS_Get(%s) failed with rc = %s\n", key, PMI_Err_str(rc));
-       }
-       else
-           printf("PMI_KVS_Get(%s) returned %s\n", key, val);
-   }
-
-   if ( rank == (size - 1) )
-   {
-       key[0] = '\0';
-       val[0] = '\0';
-       rc = PMI_KVS_Iter_first(kvsname, key, key_maxlen, val, val_maxlen);
-       if (rc == PMI_SUCCESS)
-       {
-           while (key[0] != '\0')
-           {
-              printf("PMI_KVS_Iter got key=%s val=%s\n",key,val);
-              rc = PMI_KVS_Iter_next(kvsname, key, key_maxlen, val, val_maxlen);
-              if (rc != PMI_SUCCESS)
-              {
-                  printf("PMK_KVS_Iter_next failed with rc = %s\n", PMI_Err_str(rc));
-                  break;
-              }
-          }
-      }
-      else
-      {
-       printf("PMI_KVS_Iter_first failed with rc = %s\n", PMI_Err_str(rc));
-   }
-}
-
-    /* error testing */
-if (rank != 0)
-{
-	printf("PMI error testing:\n");
-	strcpy(key, "test_key");
-	strcpy(val, "test_val");
-	rc = PMI_KVS_Put("baloney", key, val);
-	PRINT_ERROR(rc, "PMI_KVS_Put(baloney, key, val)");
-	rc = PMI_KVS_Put(NULL, key, val);
-	PRINT_ERROR(rc, "PMI_KVS_Put(NULL, key, val)");
-	rc = PMI_KVS_Put(kvsname, NULL, val);
-	PRINT_ERROR(rc, "PMI_KVS_Put(kvsname, NULL, val)");
-	rc = PMI_KVS_Put(kvsname, key, NULL);
-	PRINT_ERROR(rc, "PMI_KVS_Put(kvsname, key, NULL)");
-	rc = PMI_KVS_Get("baloney", key, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Get(baloney, key, val, val_maxlen)");
-	rc = PMI_KVS_Get(NULL, key, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Get(NULL, key, val, val_maxlen)");
-	rc = PMI_KVS_Get(kvsname, NULL, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Get(kvsname, NULL, val, val_maxlen)");
-	rc = PMI_KVS_Get(kvsname, key, NULL, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Get(kvsname, key, NULL, val_maxlen)");
-	rc = PMI_KVS_Get(kvsname, key, val, -1);
-	PRINT_ERROR(rc, "PMI_KVS_Get(kvsname, key, val, -1)");
-	rc = PMI_KVS_Commit(NULL);
-	PRINT_ERROR(rc, "PMI_KVS_Commit(NULL)");
-	rc = PMI_KVS_Commit("baloney");
-	PRINT_ERROR(rc, "PMI_KVS_Commit(baloney)");
-	rc = PMI_KVS_Get_my_name(NULL, name_max);
-	PRINT_ERROR(rc, "PMI_KVS_Get_my_name(NULL, name_max)");
-	rc = PMI_KVS_Get_my_name(kvsname, -1);
-	PRINT_ERROR(rc, "PMI_KVS_Get_my_name(kvsname, -1)");
-	rc = PMI_Get_id(NULL, id_maxlen);
-	PRINT_ERROR(rc, "PMI_Get_id(NULL, id_maxlen)");
-	rc = PMI_Get_id(id, -1);
-	PRINT_ERROR(rc, "PMI_Get_id(id, -1)");
-	rc = PMI_Get_kvs_domain_id(NULL, id_maxlen);
-	PRINT_ERROR(rc, "PMI_Get_domain_id(NULL, id_maxlen)");
-	rc = PMI_Get_kvs_domain_id(domain_id, -1);
-	PRINT_ERROR(rc, "PMI_Get_domain_id(domain_id, -1)");
-	rc = PMI_Init(NULL);
-	PRINT_ERROR(rc, "PMI_Init(NULL)");
-	rc = PMI_Get_rank(NULL);
-	PRINT_ERROR(rc, "PMI_Get_rank(NULL)");
-	rc = PMI_Get_size(NULL);
-	PRINT_ERROR(rc, "PMI_Get_size(NULL)");
-	rc = PMI_KVS_Get_name_length_max(NULL);
-	PRINT_ERROR(rc, "PMI_Get_name_length_max(NULL)");
-	rc = PMI_Get_id_length_max(NULL);
-	PRINT_ERROR(rc, "PMI_Get_id_length_max(NULL)");
-	rc = PMI_KVS_Get_key_length_max(NULL);
-	PRINT_ERROR(rc, "PMI_Get_key_length_max(NULL)");
-	rc = PMI_KVS_Get_value_length_max(NULL);
-	PRINT_ERROR(rc, "PMI_Get_value_length_max(NULL)");
-	rc = PMI_KVS_Iter_first("baloney", key, key_maxlen, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_first(baloney, key, key_maxlen, val, val_maxlen)");
-	rc = PMI_KVS_Iter_first(NULL, key, key_maxlen, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_first(NULL, key, key_maxlen, val, val_maxlen)");
-	rc = PMI_KVS_Iter_first(kvsname, NULL, key_maxlen, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_first(kvsname, NULL, key_maxlen, val, val_maxlen)");
-	rc = PMI_KVS_Iter_first(kvsname, key, -1, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_first(kvsname, key, -1, val, val_maxlen)");
-	rc = PMI_KVS_Iter_first(kvsname, key, key_maxlen, NULL, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_first(kvsname, key, key_maxlen, NULL, val_maxlen)");
-	rc = PMI_KVS_Iter_first(kvsname, key, key_maxlen, val, -1);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_first(kvsname, key, key_maxlen, val, -1)");
-	rc = PMI_KVS_Iter_next("baloney", key, key_maxlen, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_next(baloney, key, key_maxlen, val, val_maxlen)");
-	rc = PMI_KVS_Iter_next(NULL, key, key_maxlen, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_next(NULL, key, key_maxlen, val, val_maxlen)");
-	rc = PMI_KVS_Iter_next(kvsname, NULL, key_maxlen, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_next(kvsname, NULL, key_maxlen, val, val_maxlen)");
-	rc = PMI_KVS_Iter_next(kvsname, key, -1, val, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_next(kvsname, key, -1, val, val_maxlen)");
-	rc = PMI_KVS_Iter_next(kvsname, key, key_maxlen, NULL, val_maxlen);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_next(kvsname, key, key_maxlen, NULL, val_maxlen)");
-	rc = PMI_KVS_Iter_next(kvsname, key, key_maxlen, val, -1);
-	PRINT_ERROR(rc, "PMI_KVS_Iter_next(kvsname, key, key_maxlen, val, -1)");
-}
-
-rc = PMI_Finalize( );
-return 0;
-}
diff -Nru pmix-3.2.2~rc1/test/simple/simpclient.c pmix-4.0.0/test/simple/simpclient.c
--- pmix-3.2.2~rc1/test/simple/simpclient.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/simpclient.c	2021-01-02 08:56:17.000000000 +0000
@@ -161,7 +161,7 @@
                     myproc.nspace, myproc.rank, PMIx_Error_string(rc));
         exit(rc);
     }
-    pmix_output(0, "CLIENT LOCAL RANK: %u", val->data.uint32);
+    pmix_output(0, "CLIENT LOCAL RANK: %u", val->data.uint16);
     PMIX_VALUE_RELEASE(val);
 
     if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_HOSTNAME, NULL, 0, &val))) {
diff -Nru pmix-3.2.2~rc1/test/simple/simpcoord.c pmix-4.0.0/test/simple/simpcoord.c
--- pmix-3.2.2~rc1/test/simple/simpcoord.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/simpcoord.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "src/class/pmix_object.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+
+static pmix_proc_t myproc;
+
+int main(int argc, char **argv)
+{
+    int rc;
+    pmix_value_t value;
+    pmix_value_t *val = &value;
+    char *tmp;
+    pmix_proc_t proc;
+    uint32_t nprocs, n;
+    size_t ninfo, m;
+    pmix_coord_t *coords;
+    char *hostname;
+    pmix_byte_object_t *bptr;
+
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    pmix_output(0, "Client ns %s rank %d: Running", myproc.nspace, myproc.rank);
+
+    /* test something */
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    nprocs = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+    pmix_output(0, "Client %s:%d job size %d", myproc.nspace, myproc.rank, nprocs);
+
+    /* get our assumed hostname */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_HOSTNAME, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get hostname failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    hostname = strdup(val->data.string);
+    PMIX_VALUE_RELEASE(val);
+    pmix_output(0, "Client %s:%d hostname %s", myproc.nspace, myproc.rank, hostname);
+
+    /* get our assigned fabric endpts */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_FABRIC_ENDPT, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get fabric endpt failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto nextstep;
+    }
+    pmix_output(0, "Client %s:%d was assigned %lu endpts",
+                myproc.nspace, myproc.rank,
+                (unsigned long)val->data.darray->size);
+
+    bptr = (pmix_byte_object_t*)val->data.darray->array;
+    ninfo = val->data.darray->size;
+    /* print them out for diagnostics - someday we can figure
+     * out an automated way of testing the answer. We know that
+     * the simptest pnet component returns strings */
+    {
+        char **foo = NULL;
+        for (n=0; n < ninfo; n++) {
+            pmix_argv_append_nosize(&foo, bptr[0].bytes);
+        }
+        tmp = pmix_argv_join(foo, ',');
+        pmix_argv_free(foo);
+        pmix_output(0, "Rank %u[%s]: ASSIGNED ENDPTS: %s", myproc.rank, hostname, tmp);
+        free(tmp);
+    }
+
+  nextstep:
+    /* get our assigned fabric coordinates */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_FABRIC_COORDINATES, NULL, 0, &val)) ||
+        NULL == val) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get fabric coordinate failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    if (PMIX_DATA_ARRAY == val->type) {
+        coords = (pmix_coord_t*)val->data.darray->array;
+        ninfo = val->data.darray->size;
+        /* print them out for diagnostics - someday we can figure
+         * out an automated way of testing the answer */
+        for (m=0; m < ninfo; m++) {
+            char **foo = NULL;
+            char *view;
+            for (n=0; n < coords[m].dims; n++) {
+                asprintf(&tmp, "%d", coords[m].coord[n]);
+                pmix_argv_append_nosize(&foo, tmp);
+                free(tmp);
+            }
+            tmp = pmix_argv_join(foo, ',');
+            pmix_argv_free(foo);
+            if (PMIX_COORD_LOGICAL_VIEW == coords[m].view) {
+                view = "LOGICAL";
+            } else {
+                view = "PHYSICAL";
+            }
+            pmix_output(0, "Rank %u[%s]: COORD[%d] VIEW %s: %s", myproc.rank, hostname, (int)m, view, tmp);
+            free(tmp);
+        }
+    } else if (PMIX_COORD == val->type) {
+        char **foo = NULL;
+        char *view;
+        for (n=0; n < val->data.coord->dims; n++) {
+            asprintf(&tmp, "%d", val->data.coord->coord[n]);
+            pmix_argv_append_nosize(&foo, tmp);
+            free(tmp);
+        }
+        tmp = pmix_argv_join(foo, ',');
+        pmix_argv_free(foo);
+        if (PMIX_COORD_LOGICAL_VIEW == val->data.coord->view) {
+            view = "LOGICAL";
+        } else {
+            view = "PHYSICAL";
+        }
+        pmix_output(0, "Rank %u[%s]: COORD VIEW %s DIMS %lu: %s", myproc.rank, hostname, view, val->data.coord->dims, tmp);
+        free(tmp);
+    } else {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get fabric coordinate returned wrong type: %s",
+                    myproc.nspace, myproc.rank, PMIx_Data_type_string(val->type));
+    }
+
+ done:
+    /* finalize us */
+    pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n",
+                myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fflush(stderr);
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/test/simple/simpcycle.c pmix-4.0.0/test/simple/simpcycle.c
--- pmix-3.2.2~rc1/test/simple/simpcycle.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/simpcycle.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,191 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "src/class/pmix_object.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+#include "src/include/pmix_globals.h"
+
+#define MAXCNT 1
+
+static volatile bool completed = false;
+static pmix_proc_t myproc;
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    pmix_output(0, "Client %s:%d NOTIFIED with status %s", myproc.nspace, myproc.rank, PMIx_Error_string(status));
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+    }
+    completed = true;
+}
+
+/* this is an event notification function that we explicitly request
+ * be called when the PMIX_MODEL_DECLARED notification is issued.
+ * We could catch it in the general event notification function and test
+ * the status to see if the status matched, but it often is simpler
+ * to declare a use-specific notification callback point. In this case,
+ * we are asking to know whenever a model is declared as a means
+ * of testing server self-notification */
+static void model_callback(size_t evhdlr_registration_id,
+                           pmix_status_t status,
+                           const pmix_proc_t *source,
+                           pmix_info_t info[], size_t ninfo,
+                           pmix_info_t results[], size_t nresults,
+                           pmix_event_notification_cbfunc_fn_t cbfunc,
+                           void *cbdata)
+{
+    size_t n;
+
+    /* just let us know it was received */
+    fprintf(stderr, "%s:%d Model event handler called with status %d(%s)\n",
+            myproc.nspace, myproc.rank, status, PMIx_Error_string(status));
+    for (n=0; n < ninfo; n++) {
+        if (PMIX_STRING == info[n].value.type) {
+            fprintf(stderr, "%s:%d\t%s:\t%s\n",
+                    myproc.nspace, myproc.rank,
+                    info[n].key, info[n].value.data.string);
+        }
+    }
+
+    /* we must NOT tell the event handler state machine that we
+     * are the last step as that will prevent it from notifying
+     * anyone else that might be listening for declarations */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int rc, nprocs;
+    pmix_value_t value;
+    pmix_value_t *val = &value;
+    char *tmp;
+    pmix_proc_t proc;
+    pmix_info_t *iptr;
+    size_t ninfo;
+    pmix_status_t code;
+
+    /* init us and declare we are a test programming model */
+    PMIX_INFO_CREATE(iptr, 2);
+    PMIX_INFO_LOAD(&iptr[0], PMIX_PROGRAMMING_MODEL, "TEST", PMIX_STRING);
+    PMIX_INFO_LOAD(&iptr[1], PMIX_MODEL_LIBRARY_NAME, "PMIX", PMIX_STRING);
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, iptr, 2))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    PMIX_INFO_FREE(iptr, 2);
+    pmix_output(0, "Client ns %s rank %d: Running on node %s", myproc.nspace, myproc.rank, pmix_globals.hostname);
+
+    /* test something */
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_JOB_SIZE, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get job size failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    nprocs = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+    pmix_output(0, "Client %s:%d job size %d", myproc.nspace, myproc.rank, nprocs);
+
+   /* register a handler specifically for when models declare */
+    ninfo = 1;
+    PMIX_INFO_CREATE(iptr, ninfo);
+    PMIX_INFO_LOAD(&iptr[0], PMIX_EVENT_HDLR_NAME, "SIMPCLIENT-MODEL", PMIX_STRING);
+    code = PMIX_MODEL_DECLARED;
+    PMIx_Register_event_handler(&code, 1, iptr, ninfo,
+                                model_callback, NULL, NULL);
+    PMIX_INFO_FREE(iptr, ninfo);
+
+    /* register our errhandler */
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, NULL, NULL);
+
+    /* put a few values */
+    (void)asprintf(&tmp, "%s-%d-internal", myproc.nspace, myproc.rank);
+    value.type = PMIX_UINT32;
+    value.data.uint32 = 1234;
+    if (PMIX_SUCCESS != (rc = PMIx_Store_internal(&myproc, tmp, &value))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Store_internal failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+
+    /* get a list of our local peers */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get local peers failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    PMIX_VALUE_RELEASE(val);
+
+    /* finalize us */
+    pmix_output(0, "Client ns %s rank %d: Finalizing(1)", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n",
+                myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+
+    /* initialize us again */
+    fprintf(stderr, "Client Init(2)\n");
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    pmix_output(0, "Client ns %s rank %d: Finalizing(2)", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n",
+                myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+
+    fflush(stderr);
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/test/simple/simpfabric.c pmix-4.0.0/test/simple/simpfabric.c
--- pmix-3.2.2~rc1/test/simple/simpfabric.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/simpfabric.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+#include "include/pmix_server.h"
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <errno.h>
+#include <signal.h>
+
+#include "src/class/pmix_list.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+#include "src/util/argv.h"
+
+#include "simptest.h"
+
+static pmix_server_module_t mymodule = {0};
+
+typedef struct {
+    mylock_t lock;
+    pmix_status_t status;
+    pmix_info_t *info;
+    size_t ninfo;
+} mycaddy_t;
+
+static void local_cbfunc(pmix_status_t status, void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+static void setup_cbfunc(pmix_status_t status,
+                         pmix_info_t info[], size_t ninfo,
+                         void *provided_cbdata,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    mycaddy_t *mq = (mycaddy_t*)provided_cbdata;
+    size_t n;
+
+    /* transfer it to the caddy for return to the main thread */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+
+    DEBUG_WAKEUP_THREAD(&mq->lock);
+}
+
+int main(int argc, char **argv)
+{
+    pmix_info_t *info, *iptr;
+    pmix_status_t rc;
+    size_t ninfo;
+    int exit_code=0;
+    size_t n;
+    char *hosts, *procs;
+    char *regex, *ppn;
+    mylock_t lock;
+    mycaddy_t cd;
+    pmix_value_t *val;
+    pmix_topology_t *mytopo;
+    pmix_proc_t myproc;
+    pmix_cpuset_t mycpuset;
+    pmix_device_distance_t *distances;
+    size_t ndist;
+    pmix_device_type_t type = PMIX_DEVTYPE_OPENFABRICS | PMIX_DEVTYPE_NETWORK | PMIX_DEVTYPE_COPROC | PMIX_DEVTYPE_GPU;
+
+    /* smoke test */
+    if (PMIX_SUCCESS != 0) {
+        fprintf(stderr, "ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d\n", PMIX_SUCCESS);
+        exit(1);
+    }
+
+    fprintf(stderr, "PID: %lu Testing version %s\n", (unsigned long)getpid(), PMIx_Get_version());
+
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_SERVER_SCHEDULER, NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) {
+        fprintf(stderr, "Init failed with error %d\n", rc);
+        return rc;
+    }
+    PMIX_INFO_FREE(info, ninfo);
+
+    /* get my procID */
+    fprintf(stderr, "Getting procID\n");
+    rc = PMIx_Get(NULL, PMIX_PROCID, NULL, 0, &val);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "Get of my procID failed: %s\n", PMIx_Error_string(rc));
+        goto cleanup;
+    }
+    PMIX_LOAD_PROCID(&myproc, val->data.proc->nspace, val->data.proc->rank);
+    PMIX_VALUE_FREE(val, 1);
+
+    /* get my topology */
+    fprintf(stderr, "GETTING TOPOLOGY\n");
+    rc = PMIx_Get(&myproc, PMIX_TOPOLOGY2, NULL, 0, &val);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "Get of my topology failed: %s\n", PMIx_Error_string(rc));
+        goto cleanup;
+    }
+    mytopo = val->data.topo;
+    val->data.topo = NULL;
+    PMIX_VALUE_FREE(val, 1);
+    fprintf(stderr, "Got my topology: Source = %s\n", (NULL == mytopo->source) ? "NULL" : mytopo->source);
+
+    /* get my cpuset */
+    fprintf(stderr, "GETTING CPUSET\n");
+    PMIX_CPUSET_CONSTRUCT(&mycpuset);
+    rc = PMIx_Get_cpuset(&mycpuset, PMIX_CPUBIND_PROCESS);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "Get of my cpuset failed: %s\n", PMIx_Error_string(rc));
+        goto cleanup;
+    }
+    PMIx_server_generate_cpuset_string(&mycpuset, &ppn);
+    fprintf(stderr, "Got my cpuset: %s\n", ppn);
+    free(ppn);
+
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_DEVICE_TYPE, &type, PMIX_DEVTYPE);
+    rc = PMIx_Compute_distances(mytopo, &mycpuset,
+                                info, ninfo,
+                                &distances, &ndist);
+    PMIX_INFO_FREE(info, ninfo);
+    if (PMIX_SUCCESS != rc) {
+        fprintf(stderr, "Compute distances failed: %s\n", PMIx_Error_string(rc));
+    } else {
+        for (n=0; n < ndist; n++) {
+            fprintf(stderr, "Device[%d]: UUID %s OSname: %s Type %s MinDist %u MaxDist %u\n", (int)n, distances[n].uuid,
+                    distances[n].osname, PMIx_Device_type_string(distances[n].type), distances[n].mindist, distances[n].maxdist);
+        }
+    }
+
+    /* setup an application */
+    PMIX_INFO_CREATE(iptr, 4);
+    hosts = "test000,test001,test002";
+    PMIx_generate_regex(hosts, &regex);
+    PMIX_INFO_LOAD(&iptr[0], PMIX_NODE_MAP, regex, PMIX_REGEX);
+    free(regex);
+
+    procs = "0,1,2;3,4,5;6,7";
+    PMIx_generate_ppn(procs, &ppn);
+    PMIX_INFO_LOAD(&iptr[1], PMIX_PROC_MAP, ppn, PMIX_REGEX);
+    free(ppn);
+
+    PMIX_LOAD_KEY(iptr[2].key, PMIX_ALLOC_NETWORK);
+    iptr[2].value.type = PMIX_DATA_ARRAY;
+    PMIX_DATA_ARRAY_CREATE(iptr[2].value.data.darray, 2, PMIX_INFO);
+    info = (pmix_info_t*)iptr[2].value.data.darray->array;
+    PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NETWORK_ID, "SIMPSCHED.net", PMIX_STRING);
+    PMIX_INFO_LOAD(&info[1], PMIX_ALLOC_NETWORK_SEC_KEY, NULL, PMIX_BOOL);
+
+    PMIX_INFO_LOAD(&iptr[3], PMIX_SETUP_APP_ENVARS, NULL, PMIX_BOOL);
+
+    DEBUG_CONSTRUCT_LOCK(&cd.lock);
+    if (PMIX_SUCCESS != (rc = PMIx_server_setup_application("SIMPSCHED", iptr, 4,
+                                                             setup_cbfunc, &cd))) {
+        pmix_output(0, "[%s:%d] PMIx_server_setup_application failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+        DEBUG_DESTRUCT_LOCK(&cd.lock);
+        goto cleanup;
+    }
+    DEBUG_WAIT_THREAD(&cd.lock);
+    DEBUG_DESTRUCT_LOCK(&cd.lock);
+
+    /* setup the local subsystem */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    if (PMIX_SUCCESS != (rc = PMIx_server_setup_local_support("SIMPSCHED", cd.info, cd.ninfo,
+                                                                  local_cbfunc, &lock))) {
+        pmix_output(0, "[%s:%d] PMIx_server_setup_local_support failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+        DEBUG_DESTRUCT_LOCK(&lock);
+        goto cleanup;
+    }
+    DEBUG_WAIT_THREAD(&lock);
+    DEBUG_DESTRUCT_LOCK(&lock);
+
+  cleanup:
+    if (PMIX_SUCCESS != rc) {
+        exit_code = rc;
+    }
+    /* finalize the server library */
+    if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) {
+        fprintf(stderr, "Finalize failed with error %d\n", rc);
+        exit_code = rc;
+    }
+
+    if (0 == exit_code) {
+        fprintf(stderr, "Test finished OK!\n");
+    } else {
+        fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code);
+    }
+
+    return exit_code;
+}
diff -Nru pmix-3.2.2~rc1/test/simple/simpio.c pmix-4.0.0/test/simple/simpio.c
--- pmix-3.2.2~rc1/test/simple/simpio.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/simpio.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "src/class/pmix_object.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+
+#define MAXCNT 1
+
+static volatile bool completed = false;
+static pmix_proc_t myproc;
+
+static void notification_fn(size_t evhdlr_registration_id,
+                            pmix_status_t status,
+                            const pmix_proc_t *source,
+                            pmix_info_t info[], size_t ninfo,
+                            pmix_info_t results[], size_t nresults,
+                            pmix_event_notification_cbfunc_fn_t cbfunc,
+                            void *cbdata)
+{
+    pmix_output(0, "Client %s:%d NOTIFIED with status %s", myproc.nspace, myproc.rank, PMIx_Error_string(status));
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata);
+    }
+    completed = true;
+}
+
+static void errhandler_reg_callbk(pmix_status_t status,
+                                  size_t errhandler_ref,
+                                  void *cbdata)
+{
+    volatile bool *active = (volatile bool*)cbdata;
+
+    pmix_output(0, "Client: ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu",
+                status, (unsigned long)errhandler_ref);
+    *active = false;
+}
+
+#define SIMPIO_MSG_MAX   8192
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    volatile bool active;
+    int msgsize;
+    char msg[SIMPIO_MSG_MAX];
+    pmix_proc_t proc;
+    int cnt = 0;
+
+    /* init us */
+    if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+        exit(rc);
+    }
+    pmix_output(0, "Client ns %s rank %d: Running", myproc.nspace, myproc.rank);
+
+    /* register our errhandler */
+    active = true;
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                notification_fn, errhandler_reg_callbk, (void*)&active);
+    while (active) {
+        usleep(10);
+    }
+
+    /* if we are rank=0, read our stdin */
+    if (0 == myproc.rank) {
+        while (0 < (msgsize = read(0, msg, SIMPIO_MSG_MAX))) {
+            cnt += msgsize;
+            fprintf(stderr, "Rank %d: read %d bytes\n", myproc.rank, cnt);
+        }
+    }
+
+    /* call fence to sync */
+    PMIX_PROC_CONSTRUCT(&proc);
+    (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
+    proc.rank = PMIX_RANK_WILDCARD;
+    if (PMIX_SUCCESS != (rc = PMIx_Fence(&proc, 1, NULL, 0))) {
+        pmix_output(0, "Client ns %s rank %d PMIx_Fence failed: %s",
+                    myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    }
+
+    /* finalize us */
+    pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank);
+    if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n",
+                myproc.nspace, myproc.rank, PMIx_Error_string(rc));
+    } else {
+        fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
+    }
+    fflush(stderr);
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/test/simple/simplegacy.c pmix-4.0.0/test/simple/simplegacy.c
--- pmix-3.2.2~rc1/test/simple/simplegacy.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/simplegacy.c	1970-01-01 00:00:00.000000000 +0000
@@ -1,114 +0,0 @@
-/*
- * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2011 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
- *                         All rights reserved.
- * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
- * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- *
- */
-
-#include "src/include/pmix_config.h"
-#include <pmi.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <time.h>
-
-#define MAXCNT 3
-
-int main(int argc, char **argv)
-{
-    int rc, j, n;
-    char *tmp;
-    int spawned;
-    int rank;
-    int nprocs;
-    char value[1024];
-
-    fprintf(stderr, "Client calling init\n");
-    if (PMI_SUCCESS != (rc = PMI_Init(&spawned))) {
-        fprintf(stderr, "Client PMI_Init failed: %d\n", rc);
-        exit(rc);
-    }
-    fprintf(stderr, "Client Running\n");
-
-    /* test something */
-    if (PMI_SUCCESS != (rc = PMI_Get_rank(&rank))) {
-        fprintf(stderr, "Client PMI_Get_rank failed: %d\n", rc);
-        exit(rc);
-    }
-    if (PMI_SUCCESS != (rc = PMI_Get_universe_size(&nprocs))) {
-        fprintf(stderr, "Client %d: PMI_Get_universe_size failed: %d\n", rank, rc);
-        exit(rc);
-    }
-    fprintf(stderr, "Client %d job size %d\n", rank, nprocs);
-
-    for (j=0; j < 10; j++) {
-        (void)asprintf(&tmp, "%d-gasnet-0-%d", rank, j);
-        if (PMI_SUCCESS != (rc = PMI_KVS_Put("foobar", tmp, "myvalue"))) {
-            fprintf(stderr, "Client %d: j %d PMI_KVS_Put failed: %d\n",
-                    rank, j, rc);
-            goto done;
-        }
-        free(tmp);
-    }
-
-    if (PMIX_SUCCESS != (rc = PMI_KVS_Commit("foobar"))) {
-            fprintf(stderr, "Client %d: PMI_KVS_Commit failed: %d\n", rank, rc);
-            goto done;
-    }
-
-    fprintf(stderr, "Client rank %d: CALLING PMI_Barrier\n", rank);
-
-    /* call fence to ensure the data is received */
-    if (PMI_SUCCESS != (rc = PMI_Barrier())) {
-            fprintf(stderr, "Client %d: PMI_Barrier failed: %d\n", rank, rc);
-        goto done;
-    }
-
-    /* check the returned data */
-    for (j=0; j < 10; j++) {
-        for (n=0; n < nprocs; n++) {
-            (void)asprintf(&tmp, "%d-gasnet-0-%d", n, j);
-            fprintf(stderr, "Client %d: Calling get\n", rank);
-            if (PMI_SUCCESS != (rc = PMI_KVS_Get("foobar", tmp, value, 1024))) {
-             fprintf(stderr, "Client %d: PMI_Get failed: %d\n", rank, rc);
-                continue;
-            }
-            if (0 == strcmp(value, "myvalue")) {
-                fprintf(stderr, "Client %d: PMI_Get returned correct value\n", rank);
-            } else {
-                fprintf(stderr, "Client %d: PMI_Get returned incorrect value\n", rank);
-            }
-            free(tmp);
-        }
-    }
-
-  done:
-    /* finalize us */
-    fprintf(stderr, "Client rank %d: Finalizing\n", rank);
-    if (PMI_SUCCESS != (rc = PMI_Finalize())) {
-        fprintf(stderr, "Client rank %d: finalize failed %d\n", rank, rc);
-    } else {
-        fprintf(stderr, "Client %d:PMI_Finalize successfully completed\n", rank);
-    }
-    fflush(stderr);
-    return(rc);
-}
diff -Nru pmix-3.2.2~rc1/test/simple/simpsched.c pmix-4.0.0/test/simple/simpsched.c
--- pmix-3.2.2~rc1/test/simple/simpsched.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/simpsched.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+#include "include/pmix_server.h"
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <errno.h>
+#include <signal.h>
+
+#include "src/class/pmix_list.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+#include "src/util/argv.h"
+
+#include "simptest.h"
+
+static pmix_server_module_t mymodule = {0};
+
+typedef struct {
+    mylock_t lock;
+    pmix_status_t status;
+    pmix_info_t *info;
+    size_t ninfo;
+} mycaddy_t;
+
+static void local_cbfunc(pmix_status_t status, void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+static void setup_cbfunc(pmix_status_t status,
+                         pmix_info_t info[], size_t ninfo,
+                         void *provided_cbdata,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    mycaddy_t *mq = (mycaddy_t*)provided_cbdata;
+    size_t n;
+
+    /* transfer it to the caddy for return to the main thread */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+
+    DEBUG_WAKEUP_THREAD(&mq->lock);
+}
+
+int main(int argc, char **argv)
+{
+    pmix_info_t *info, *iptr;
+    pmix_status_t rc;
+    pmix_fabric_t myfabric;
+    size_t ninfo;
+    int exit_code=0;
+    size_t n;
+    char *hosts, *procs;
+    char *regex, *ppn;
+    mylock_t lock;
+    mycaddy_t cd;
+    pmix_value_t *val;
+
+    /* smoke test */
+    if (PMIX_SUCCESS != 0) {
+        fprintf(stderr, "ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d\n", PMIX_SUCCESS);
+        exit(1);
+    }
+
+    fprintf(stderr, "PID: %lu Testing version %s\n", (unsigned long)getpid(), PMIx_Get_version());
+
+    /* set a known network configuration for the pnet/test component */
+    putenv("PMIX_MCA_pnet_test_planes=plane:d:3:4,plane:s:2,plane:3");
+    putenv("PMIX_MCA_pnet_test_nodes=test000,test001,test002");
+    putenv("PMIX_MCA_pnet=test");
+
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_SERVER_SCHEDULER, NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) {
+        fprintf(stderr, "Init failed with error %d\n", rc);
+        return rc;
+    }
+    PMIX_INFO_FREE(info, ninfo);
+
+    /* register a fabric */
+    rc = PMIx_Fabric_register(&myfabric, NULL, 0);
+    if (PMIX_SUCCESS == rc) {
+        /* scan the returned info array for values */
+        for (n=0; n < myfabric.ninfo; n++) {
+            if (PMIX_CHECK_KEY(&myfabric.info[n], PMIX_FABRIC_VENDOR)) {
+                fprintf(stderr, "Fabric vendor: %s\n", myfabric.info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&myfabric.info[n], PMIX_FABRIC_IDENTIFIER)) {
+                fprintf(stderr, "Fabric ID: %s\n", myfabric.info[n].value.data.string);
+            } else if (PMIX_CHECK_KEY(&myfabric.info[n], PMIX_FABRIC_NUM_DEVICES)) {
+                fprintf(stderr, "Number of fabric vertices: %u\n", (unsigned)myfabric.info[n].value.data.size);
+            }
+        }
+
+        rc = PMIx_Get(NULL, PMIX_FABRIC_DEVICES, NULL, 0, &val);
+        if (PMIX_SUCCESS != rc) {
+            fprintf(stderr, "Fabric get devices failed with error: %s\n", PMIx_Error_string(rc));
+            goto cleanup;
+        }
+        fprintf(stderr, "Device info:\n");
+        info = (pmix_info_t*)val->data.darray->array;
+        ninfo = val->data.darray->size;
+        for (n=0; n < ninfo; n++) {
+            fprintf(stderr, "\t%s:\t%s\n", info[n].key, info[n].value.data.string);
+        }
+    } else {
+        fprintf(stderr, "Register fabric failed with error %s\n", PMIx_Error_string(rc));
+    }
+
+    /* setup an application */
+    PMIX_INFO_CREATE(iptr, 4);
+    hosts = "test000,test001,test002";
+    PMIx_generate_regex(hosts, &regex);
+    PMIX_INFO_LOAD(&iptr[0], PMIX_NODE_MAP, regex, PMIX_REGEX);
+    free(regex);
+
+    procs = "0,1,2;3,4,5;6,7";
+    PMIx_generate_ppn(procs, &ppn);
+    PMIX_INFO_LOAD(&iptr[1], PMIX_PROC_MAP, ppn, PMIX_REGEX);
+    free(ppn);
+
+    PMIX_LOAD_KEY(iptr[2].key, PMIX_ALLOC_NETWORK);
+    iptr[2].value.type = PMIX_DATA_ARRAY;
+    PMIX_DATA_ARRAY_CREATE(iptr[2].value.data.darray, 2, PMIX_INFO);
+    info = (pmix_info_t*)iptr[2].value.data.darray->array;
+    PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NETWORK_ID, "SIMPSCHED.net", PMIX_STRING);
+    PMIX_INFO_LOAD(&info[1], PMIX_ALLOC_NETWORK_SEC_KEY, NULL, PMIX_BOOL);
+
+    PMIX_INFO_LOAD(&iptr[3], PMIX_SETUP_APP_ENVARS, NULL, PMIX_BOOL);
+
+    DEBUG_CONSTRUCT_LOCK(&cd.lock);
+    if (PMIX_SUCCESS != (rc = PMIx_server_setup_application("SIMPSCHED", iptr, 4,
+                                                             setup_cbfunc, &cd))) {
+        pmix_output(0, "[%s:%d] PMIx_server_setup_application failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+        DEBUG_DESTRUCT_LOCK(&cd.lock);
+        goto cleanup;
+    }
+    DEBUG_WAIT_THREAD(&cd.lock);
+    DEBUG_DESTRUCT_LOCK(&cd.lock);
+
+    /* setup the local subsystem */
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    if (PMIX_SUCCESS != (rc = PMIx_server_setup_local_support("SIMPSCHED", cd.info, cd.ninfo,
+                                                                  local_cbfunc, &lock))) {
+        pmix_output(0, "[%s:%d] PMIx_server_setup_local_support failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+        DEBUG_DESTRUCT_LOCK(&lock);
+        goto cleanup;
+    }
+    DEBUG_WAIT_THREAD(&lock);
+    DEBUG_DESTRUCT_LOCK(&lock);
+
+  cleanup:
+    if (PMIX_SUCCESS != rc) {
+        exit_code = rc;
+    }
+    /* finalize the server library */
+    if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) {
+        fprintf(stderr, "Finalize failed with error %d\n", rc);
+        exit_code = rc;
+    }
+
+    if (0 == exit_code) {
+        fprintf(stderr, "Test finished OK!\n");
+    } else {
+        fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code);
+    }
+
+    return exit_code;
+}
diff -Nru pmix-3.2.2~rc1/test/simple/simptest.c pmix-4.0.0/test/simple/simptest.c
--- pmix-3.2.2~rc1/test/simple/simptest.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/simptest.c	2021-01-02 08:56:17.000000000 +0000
@@ -26,7 +26,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix_server.h>
+#include "include/pmix_server.h"
 #include "src/include/types.h"
 #include "src/include/pmix_globals.h"
 
@@ -39,15 +39,12 @@
 #include <errno.h>
 #include <signal.h>
 
-#if PMIX_HAVE_HWLOC
-#include "src/hwloc/hwloc-internal.h"
-#endif
-
 #include "src/class/pmix_list.h"
 #include "src/util/pmix_environ.h"
 #include "src/util/output.h"
 #include "src/util/printf.h"
 #include "src/util/argv.h"
+#include "src/runtime/pmix_progress_threads.h"
 
 #include "simptest.h"
 
@@ -198,15 +195,9 @@
 static pmix_list_t children;
 static bool istimeouttest = false;
 static mylock_t globallock;
-static bool nettest = false;
 static bool model = false;
 static bool xversion = false;
-static char *hostnames[] = {
-    "test000",
-    "test001",
-    "test002",
-    NULL
-};
+static pmix_event_base_t *simptest_evbase = NULL;
 
 static void set_namespace(int nprocs, char *nspace,
                           pmix_op_cbfunc_t cbfunc, myxfer_t *x);
@@ -264,7 +255,7 @@
     for (n=0; n < ninfo; n++) {
         PMIX_INFO_XFER(&x->info[n], &info[n]);
     }
-    PMIX_THREADSHIFT(x, dlcbfunc);
+    SIMPTEST_THREADSHIFT(x, dlcbfunc);
 
     if (NULL != release_fn) {
         release_fn(release_cbdata);
@@ -359,10 +350,6 @@
     wait_tracker_t *child;
     pmix_info_t *info;
     size_t ninfo;
-    bool hwloc = false;
-#if PMIX_HAVE_HWLOC
-    char *hwloc_file = NULL;
-#endif
     mylock_t mylock;
     pmix_status_t code;
     sigset_t unblock;
@@ -391,36 +378,14 @@
                 pmix_argv_append_nosize(&client_argv, argv[k]);
             }
             n += k;
-#if PMIX_HAVE_HWLOC
-        } else if (0 == strcmp("-hwloc", argv[n]) ||
-                   0 == strcmp("--hwloc", argv[n])) {
-            /* test hwloc support */
-            hwloc = true;
-        } else if (0 == strcmp("-hwloc-file", argv[n]) ||
-                   0 == strcmp("--hwloc-file", argv[n])) {
-            if (NULL == argv[n+1]) {
-                fprintf(stderr, "The --hwloc-file option requires an argument\n");
-                exit(1);
-            }
-            hwloc_file = strdup(argv[n+1]);
-            hwloc = true;
-            ++n;
-#endif
         } else if (0 == strcmp("-h", argv[n])) {
             /* print the options and exit */
             fprintf(stderr, "usage: simptest <options>\n");
             fprintf(stderr, "    -n N     Number of clients to run\n");
             fprintf(stderr, "    -e foo   Name of the client executable to run (default: simpclient\n");
             fprintf(stderr, "    -u       Enable legacy usock support\n");
-            fprintf(stderr, "    -hwloc   Test hwloc support\n");
-            fprintf(stderr, "    -hwloc-file FILE   Use file to import topology\n");
-            fprintf(stderr, "    -net-test  Test network endpt assignments\n");
             fprintf(stderr, "    -xversion  Cross-version test - simulate single node only\n");
             exit(0);
-        } else if (0 == strcmp("-net-test", argv[n]) ||
-                   0 == strcmp("--net-test", argv[n])) {
-            /* test network support */
-            nettest = true;
         }  else if (0 == strcmp("-model", argv[n]) ||
                    0 == strcmp("--model", argv[n])) {
             /* test network support */
@@ -432,11 +397,7 @@
         }
     }
     if (NULL == executable) {
-        if (nettest) {
-            executable = strdup("./simpcoord");
-        } else {
-            executable = strdup("./simpclient");
-        }
+        executable = strdup("./simpclient");
     }
     /* check for executable existence and permissions */
     if (0 != access(executable, X_OK)) {
@@ -444,13 +405,6 @@
         exit(1);
     }
 
-#if !PMIX_HAVE_HWLOC
-    if (hwloc) {
-        fprintf(stderr, "PMIx was not configured with HWLOC support - cannot continue\n");
-        exit(1);
-    }
-#endif
-
     fprintf(stderr, "Testing version %s\n", PMIx_Get_version());
 
     /* ensure that SIGCHLD is unblocked as we need to capture it */
@@ -469,48 +423,19 @@
 
 
     /* setup the server library and tell it to support tool connections */
-#if PMIX_HAVE_HWLOC
-    if (hwloc) {
-#if HWLOC_API_VERSION < 0x20000
-        ninfo = 2;
-#else
-        ninfo = 3;
-#endif
-    } else {
-        ninfo = 1;
-    }
-#else
-    ninfo = 1;
-#endif
-
+    ninfo = 2;
     PMIX_INFO_CREATE(info, ninfo);
     PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL);
-#if PMIX_HAVE_HWLOC
-    if (hwloc) {
-        if (NULL != hwloc_file) {
-            PMIX_INFO_LOAD(&info[1], PMIX_TOPOLOGY_FILE, hwloc_file, PMIX_STRING);
-        } else {
-            PMIX_INFO_LOAD(&info[1], PMIX_TOPOLOGY, NULL, PMIX_STRING);
-        }
-#if HWLOC_API_VERSION >= 0x20000
-        PMIX_INFO_LOAD(&info[2], PMIX_HWLOC_SHARE_TOPO, NULL, PMIX_BOOL);
-#endif
-    }
-#endif
-    if (nettest) {
-        /* set a known network configuration for the pnet/test component */
-        putenv("PMIX_MCA_pnet_test_planes=plane:d:3;plane:s:2;plane:d:5:2");
-        putenv("PMIX_MCA_pnet=test");
-    }
+    PMIX_INFO_LOAD(&info[1], PMIX_SERVER_SCHEDULER, NULL, PMIX_BOOL);
     if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) {
         fprintf(stderr, "Init failed with error %s\n", PMIx_Error_string(rc));
         return rc;
     }
     PMIX_INFO_FREE(info, ninfo);
-    if (nettest) {
-        unsetenv("PMIX_MCA_pnet");
-        unsetenv("PMIX_MCA_pnet_test_planes");
-    }
+
+    /* get our own event base */
+    simptest_evbase = pmix_progress_thread_init("simptest");
+    pmix_progress_thread_start("simptest");
 
     /* register the default errhandler */
     DEBUG_CONSTRUCT_LOCK(&mylock);
@@ -590,10 +515,6 @@
             PMIx_server_finalize();
             return rc;
         }
-        /* add the hostname we want them to use */
-        if (!xversion) {
-            PMIX_SETENV(rc, "PMIX_HOSTNAME", hostnames[n % 3], &client_env);
-        }
 
         x = PMIX_NEW(myxfer_t);
         if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid,
@@ -724,58 +645,20 @@
     pmix_info_t *isv1, *isv2;
     myxfer_t cd, lock;
     pmix_status_t rc;
-    char **map[3] = {NULL, NULL, NULL};
-    char *peers[3] = {NULL, NULL, NULL};
     char tmp[50] , **agg = NULL;
 
-    if (xversion) {
-        /* everything on one node */
-        PMIx_generate_regex(pmix_globals.hostname, &regex);
-        for (m=0; m < nprocs; m++) {
-            snprintf(tmp, 50, "%d", m);
-            pmix_argv_append_nosize(&agg, tmp);
-            memset(tmp, 0, 50);
-        }
-        rks = pmix_argv_join(agg, ',');
-        pmix_argv_free(agg);
-        PMIx_generate_ppn(rks, &ppn);
-        free(rks);
-        nnodes = 1;
-    } else {
-        if (nprocs < 3) {
-            /* take only the number of hostnames equal to
-             * the number of procs */
-            for (m=0; m < nprocs; m++) {
-                pmix_argv_append_nosize(&agg, hostnames[m]);
-            }
-            ppn = pmix_argv_join(agg, ',');
-            pmix_argv_free(agg);
-            agg = NULL;
-            nnodes = nprocs;
-        } else {
-            nnodes = 3;
-            ppn = pmix_argv_join(hostnames, ',');
-        }
-        PMIx_generate_regex(ppn, &regex);
-        free(ppn);
-        /* compute the placement of the procs */
-        for (m=0; m < nprocs; m++) {
-            snprintf(tmp, 50, "%d", m);
-            pmix_argv_append_nosize(&map[m%3], tmp);
-            memset(tmp, 0, 50);
-        }
-        for (m=0; m < 3; m++) {
-            if (NULL != map[m]) {
-                peers[m] = pmix_argv_join(map[m], ',');
-                pmix_argv_append_nosize(&agg, peers[m]);
-                pmix_argv_free(map[m]);
-            }
-        }
-        rks = pmix_argv_join(agg, ';');
-        pmix_argv_free(agg);
-        PMIx_generate_ppn(rks, &ppn);
-        free(rks);
-    }
+    /* everything on one node */
+    PMIx_generate_regex(pmix_globals.hostname, &regex);
+    for (m=0; m < nprocs; m++) {
+        snprintf(tmp, 50, "%d", m);
+        pmix_argv_append_nosize(&agg, tmp);
+        memset(tmp, 0, 50);
+    }
+    rks = pmix_argv_join(agg, ',');
+    pmix_argv_free(agg);
+    PMIx_generate_ppn(rks, &ppn);
+    free(rks);
+    nnodes = 1;
 
     x->ninfo = 1 + nprocs + nnodes;
     PMIX_INFO_CREATE(x->info, x->ninfo);
@@ -789,9 +672,9 @@
         PMIX_DATA_ARRAY_CREATE(x->info[n].value.data.darray, 9, PMIX_INFO);
     }
     iptr = (pmix_info_t*)x->info[n].value.data.darray->array;
-    PMIX_INFO_LOAD(&iptr[0], PMIX_NODE_MAP, regex, PMIX_STRING);
+    PMIX_INFO_LOAD(&iptr[0], PMIX_NODE_MAP, regex, PMIX_REGEX);
     isv1 = &iptr[0];
-    PMIX_INFO_LOAD(&iptr[1], PMIX_PROC_MAP, ppn, PMIX_STRING);
+    PMIX_INFO_LOAD(&iptr[1], PMIX_PROC_MAP, ppn, PMIX_REGEX);
     isv2 = &iptr[1];
     PMIX_INFO_LOAD(&iptr[2], PMIX_JOB_SIZE, &nprocs, PMIX_UINT32);
     PMIX_INFO_LOAD(&iptr[3], PMIX_JOBID, "1234", PMIX_STRING);
@@ -811,14 +694,14 @@
     PMIX_INFO_XFER(&iptr[0], isv1);
     PMIX_INFO_XFER(&iptr[1], isv2);
     PMIX_INFO_LOAD(&iptr[2], PMIX_SETUP_APP_ENVARS, NULL, PMIX_BOOL);
-    PMIX_LOAD_KEY(iptr[3].key, PMIX_ALLOC_NETWORK);
+    PMIX_LOAD_KEY(iptr[3].key, PMIX_ALLOC_FABRIC);
     iptr[3].value.type = PMIX_DATA_ARRAY;
     PMIX_DATA_ARRAY_CREATE(iptr[3].value.data.darray, 2, PMIX_INFO);
     ip = (pmix_info_t*)iptr[3].value.data.darray->array;
     asprintf(&rks, "%s.net", nspace);
-    PMIX_INFO_LOAD(&ip[0], PMIX_ALLOC_NETWORK_ID, rks, PMIX_STRING);
+    PMIX_INFO_LOAD(&ip[0], PMIX_ALLOC_FABRIC_ID, rks, PMIX_STRING);
     free(rks);
-    PMIX_INFO_LOAD(&ip[1], PMIX_ALLOC_NETWORK_SEC_KEY, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&ip[1], PMIX_ALLOC_FABRIC_SEC_KEY, NULL, PMIX_BOOL);
     PMIX_CONSTRUCT(&cd, myxfer_t);
     if (PMIX_SUCCESS != (rc = PMIx_server_setup_application(nspace, iptr, 4,
                                                              setup_cbfunc, &cd))) {
@@ -846,11 +729,7 @@
         x->info[n].value.type = PMIX_DATA_ARRAY;
         PMIX_DATA_ARRAY_CREATE(x->info[n].value.data.darray, 3, PMIX_INFO);
         iptr = (pmix_info_t*)x->info[n].value.data.darray->array;
-        if (xversion) {
-            PMIX_INFO_LOAD(&iptr[0], PMIX_HOSTNAME, pmix_globals.hostname, PMIX_STRING);
-        } else {
-            PMIX_INFO_LOAD(&iptr[0], PMIX_HOSTNAME, hostnames[m % 3], PMIX_STRING);
-        }
+        PMIX_INFO_LOAD(&iptr[0], PMIX_HOSTNAME, pmix_globals.hostname, PMIX_STRING);
         PMIX_INFO_LOAD(&iptr[1], PMIX_NODEID, &m, PMIX_UINT32);
         PMIX_INFO_LOAD(&iptr[2], PMIX_NODE_SIZE, &nprocs, PMIX_UINT32);
         ++n;
@@ -889,11 +768,7 @@
 
         (void)strncpy(info[k].key, PMIX_HOSTNAME, PMIX_MAX_KEYLEN);
         info[k].value.type = PMIX_STRING;
-        if (xversion) {
-            info[k].value.data.string = strdup(pmix_globals.hostname);
-        } else {
-            info[k].value.data.string = strdup(hostnames[m % 3]);
-        }
+        info[k].value.data.string = strdup(pmix_globals.hostname);
         ++k;
         /* move to next proc */
         ++n;
@@ -1019,11 +894,41 @@
     scd->ndata = ndata;
     scd->cbfunc.modexcbfunc = cbfunc;
     scd->cbdata = cbdata;
-    PMIX_THREADSHIFT(scd, fencbfn);
+    SIMPTEST_THREADSHIFT(scd, fencbfn);
     return PMIX_SUCCESS;
 }
 
 
+static void modex_resp(pmix_status_t status,
+                       char *data, size_t sz,
+                       void *cbdata)
+{
+    pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata;
+    scd->status = status;
+    scd->data = (char*)malloc(sz);
+    memcpy((char*)scd->data, data, sz);
+    scd->ndata = sz;
+    SIMPTEST_THREADSHIFT(scd, fencbfn);
+}
+
+static void dmdxfn(int sd, short args, void *cbdata)
+{
+    pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata;
+    pmix_proc_t proc;
+    pmix_status_t rc;
+
+    /* get the data */
+    PMIX_LOAD_PROCID(&proc, scd->pname.nspace, scd->pname.rank);
+    rc = PMIx_server_dmodex_request(&proc, modex_resp, scd);
+    if (PMIX_SUCCESS != rc) {
+        scd->status = rc;
+        scd->data = NULL;
+        scd->ndata = 0;
+        SIMPTEST_THREADSHIFT(scd, fencbfn);
+    }
+    return;
+}
+
 static pmix_status_t dmodex_fn(const pmix_proc_t *proc,
                      const pmix_info_t info[], size_t ninfo,
                      pmix_modex_cbfunc_t cbfunc, void *cbdata)
@@ -1036,10 +941,11 @@
     }
 
     scd = PMIX_NEW(pmix_shift_caddy_t);
-    scd->status = PMIX_ERR_NOT_FOUND;
+    scd->pname.nspace = strdup(proc->nspace);
+    scd->pname.rank = proc->rank;
     scd->cbfunc.modexcbfunc = cbfunc;
     scd->cbdata = cbdata;
-    PMIX_THREADSHIFT(scd, fencbfn);
+    SIMPTEST_THREADSHIFT(scd, dmdxfn);
 
     return PMIX_SUCCESS;
 }
@@ -1127,7 +1033,7 @@
         lk->n = n;
         lk->cbfunc = cbfunc;
         lk->cbdata = cbdata;
-        PMIX_THREADSHIFT(lk, lkcbfn);
+        SIMPTEST_THREADSHIFT(lk, lkcbfn);
     }
 
     return ret;
@@ -1285,7 +1191,7 @@
     qd.ndata = nqueries;
     qd.cbfunc = cbfunc;
     qd.cbdata = cbdata;
-    PMIX_THREADSHIFT(&qd, qfn);
+    SIMPTEST_THREADSHIFT(&qd, qfn);
     return PMIX_SUCCESS;
 }
 
@@ -1324,7 +1230,7 @@
 
     lg->cbfunc = cbfunc;
     lg->cbdata = cbdata;
-    PMIX_THREADSHIFT(lg, foobar);
+    SIMPTEST_THREADSHIFT(lg, foobar);
 }
 
 static pmix_status_t alloc_fn(const pmix_proc_t *client,
diff -Nru pmix-3.2.2~rc1/test/simple/simptool.c pmix-4.0.0/test/simple/simptool.c
--- pmix-3.2.2~rc1/test/simple/simptool.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/simptool.c	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -24,7 +24,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix_tool.h>
+#include "include/pmix_tool.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -38,51 +38,14 @@
 
 static pmix_proc_t myproc;
 
-static void cbfunc(pmix_status_t status,
-                   pmix_info_t *info, size_t ninfo,
-                   void *cbdata,
-                   pmix_release_cbfunc_t release_fn,
-                   void *release_cbdata)
-{
-    volatile bool *active = (volatile bool*)cbdata;
-
-    if (0 != strncmp(info[0].key, "foobar", PMIX_MAX_KEYLEN)) {
-        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info[0] key wrong: %s vs foobar",
-                    myproc.nspace, myproc.rank, info[0].key);
-    }
-    if (0 != strncmp(info[1].key, "spastic", PMIX_MAX_KEYLEN)) {
-        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info[1] key wrong: %s vs spastic",
-                    myproc.nspace, myproc.rank, info[1].key);
-    }
-    if (PMIX_STRING != info[0].value.type) {
-        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[0] wrong type: %d vs %d",
-                    myproc.nspace, myproc.rank, info[0].value.type, PMIX_STRING);
-    }
-    if (PMIX_STRING != info[1].value.type) {
-        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[1] wrong type: %d vs %d",
-                    myproc.nspace, myproc.rank, info[1].value.type, PMIX_STRING);
-    }
-    if (0 != strcmp(info[0].value.data.string, "0")) {
-        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[0] wrong value: %s vs 0",
-                    myproc.nspace, myproc.rank, info[1].value.data.string);
-    }
-    if (0 != strcmp(info[1].value.data.string, "1")) {
-        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[1] wrong value: %s vs 1",
-                    myproc.nspace, myproc.rank, info[1].value.data.string);
-    }
-
-    if (NULL != release_fn) {
-        release_fn(release_cbdata);
-    }
-    *active = false;
-}
-
 int main(int argc, char **argv)
 {
     pmix_status_t rc;
     pmix_query_t *query;
     size_t nq;
-    volatile bool active;
+    pmix_info_t *results = NULL;
+    size_t nresults = 0;
+
     /* init us */
     if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) {
         fprintf(stderr, "PMIx_tool_init failed: %d\n", rc);
@@ -96,14 +59,41 @@
     pmix_argv_append_nosize(&query[0].keys, "foobar");
     pmix_argv_append_nosize(&query[1].keys, "spastic");
     pmix_argv_append_nosize(&query[1].keys, PMIX_SERVER_URI);
-    active = true;
-    if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&active))) {
+    if (PMIX_SUCCESS != (rc = PMIx_Query_info(query, nq, &results, &nresults))) {
         pmix_output(0, "Client ns %s rank %d: PMIx_Query_info failed: %d", myproc.nspace, myproc.rank, rc);
         goto done;
     }
-    while(active) {
-        usleep(10);
+    if (2 != nresults || NULL == results) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info returned incorrect results: %d", myproc.nspace, myproc.rank, (int)nresults);
+        goto done;
+    }
+    if (0 != strncmp(results[0].key, "foobar", PMIX_MAX_KEYLEN)) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info[0] key wrong: %s vs foobar",
+                    myproc.nspace, myproc.rank, results[0].key);
+    }
+    if (0 != strncmp(results[1].key, "spastic", PMIX_MAX_KEYLEN)) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info[1] key wrong: %s vs spastic",
+                    myproc.nspace, myproc.rank, results[1].key);
+    }
+    if (PMIX_STRING != results[0].value.type) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[0] wrong type: %d vs %d",
+                    myproc.nspace, myproc.rank, results[0].value.type, PMIX_STRING);
+    }
+    if (PMIX_STRING != results[1].value.type) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[1] wrong type: %d vs %d",
+                    myproc.nspace, myproc.rank, results[1].value.type, PMIX_STRING);
+    }
+    if (0 != strcmp(results[0].value.data.string, "0")) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[0] wrong value: %s vs 0",
+                    myproc.nspace, myproc.rank, results[1].value.data.string);
+    }
+    if (0 != strcmp(results[1].value.data.string, "1")) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Query_info key[1] wrong value: %s vs 1",
+                    myproc.nspace, myproc.rank, results[1].value.data.string);
     }
+    pmix_output(0, "Client received result %s:%s", results[0].key, results[0].value.data.string);
+    pmix_output(0, "Client received result %s:%s", results[1].key, results[1].value.data.string);
+
  done:
     /* finalize us */
     pmix_output(0, "Client ns %s rank %d: Finalizing", myproc.nspace, myproc.rank);
diff -Nru pmix-3.2.2~rc1/test/simple/simpvni.c pmix-4.0.0/test/simple/simpvni.c
--- pmix-3.2.2~rc1/test/simple/simpvni.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/simple/simpvni.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,137 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <pthread.h>
+#include <pmix_tool.h>
+#include <pmix_server.h>
+
+typedef struct {
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    volatile bool active;
+    pmix_status_t status;
+} mylock_t;
+
+typedef struct {
+    mylock_t lock;
+    pmix_info_t *info;
+    size_t ninfo;
+} myxfer_t;
+
+#define DEBUG_CONSTRUCT_LOCK(l)                     \
+    do {                                            \
+        pthread_mutex_init(&(l)->mutex, NULL);      \
+        pthread_cond_init(&(l)->cond, NULL);        \
+        (l)->active = true;                         \
+        (l)->status = PMIX_SUCCESS;                 \
+    } while(0)
+
+#define DEBUG_DESTRUCT_LOCK(l)              \
+    do {                                    \
+        pthread_mutex_destroy(&(l)->mutex); \
+        pthread_cond_destroy(&(l)->cond);   \
+    } while(0)
+
+#define DEBUG_WAIT_THREAD(lck)                                      \
+    do {                                                            \
+        pthread_mutex_lock(&(lck)->mutex);                          \
+        while ((lck)->active) {                                     \
+            pthread_cond_wait(&(lck)->cond, &(lck)->mutex);         \
+        }                                                           \
+        pthread_mutex_unlock(&(lck)->mutex);                        \
+    } while(0)
+
+#define DEBUG_WAKEUP_THREAD(lck)                        \
+    do {                                                \
+        pthread_mutex_lock(&(lck)->mutex);              \
+        (lck)->active = false;                          \
+        pthread_cond_broadcast(&(lck)->cond);           \
+        pthread_mutex_unlock(&(lck)->mutex);            \
+    } while(0)
+
+static void setup_cbfunc(pmix_status_t status,
+                         pmix_info_t info[], size_t ninfo,
+                         void *provided_cbdata,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    myxfer_t *x = (myxfer_t*)provided_cbdata;
+    size_t n;
+
+    /* transfer it to the caddy for return to the main thread */
+    x->lock.status = status;
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(x->info, ninfo);
+        x->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&x->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+
+    DEBUG_WAKEUP_THREAD(&x->lock);
+}
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    myxfer_t x;
+    pmix_proc_t myproc;
+    pmix_info_t *info;
+    size_t n, ninfo;
+    char *myvni = NULL;
+
+    /* setup the PMIx tool library - don't need to connect
+     * to a PMIx server */
+    ninfo = 2;
+    PMIX_INFO_CREATE(info, ninfo);
+    n = 0;
+    PMIX_INFO_LOAD(&info[n], PMIX_TOOL_DO_NOT_CONNECT, NULL, PMIX_BOOL);
+    ++n;
+    PMIX_INFO_LOAD(&info[n], PMIX_LAUNCHER, NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, info, ninfo))) {
+        fprintf(stderr, "Init failed with error %s\n", PMIx_Error_string(rc));
+        return rc;
+    }
+    PMIX_INFO_FREE(info, ninfo);
+
+    /* ask for a security credential */
+    DEBUG_CONSTRUCT_LOCK(&x.lock);
+    x.info = NULL;
+    x.ninfo = 0;
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    n = 0;
+    PMIX_INFO_LOAD(&info[n], PMIX_ALLOC_FABRIC_SEC_KEY, NULL, PMIX_BOOL);
+    rc = PMIx_server_setup_application(myproc.nspace, info, ninfo, setup_cbfunc, &x);
+    if (PMIX_SUCCESS != rc) {
+        /* the thread won't be called back */
+        PMIX_INFO_FREE(info, ninfo);
+        goto done;
+    }
+    DEBUG_WAIT_THREAD(&x.lock);
+    PMIX_INFO_FREE(info, ninfo);
+    if (PMIX_SUCCESS != x.lock.status) {
+        /* couldn't allocate it */
+        rc = x.lock.status;
+        goto done;
+    }
+
+    /* find the VNI in the returned data - shouldn't be anything else */
+    for (n=0; n < x.ninfo; n++) {
+        if (PMIX_CHECK_KEY(&x.info[n], PMIX_CREDENTIAL)) {
+            myvni = strdup(x.info[n].value.data.string);
+            break;
+        }
+    }
+    fprintf(stderr, "VNI: %s\n", (NULL == myvni) ? "NULL" : myvni);
+
+  done:
+    rc = PMIx_tool_finalize();
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/test/simple/stability.c pmix-4.0.0/test/simple/stability.c
--- pmix-3.2.2~rc1/test/simple/stability.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/simple/stability.c	2021-01-02 08:56:17.000000000 +0000
@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2018 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -26,7 +26,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix_server.h>
+#include "include/pmix_server.h"
 #include "src/include/types.h"
 #include "src/include/pmix_globals.h"
 
@@ -38,11 +38,6 @@
 #include <sys/wait.h>
 #include <errno.h>
 #include <signal.h>
-#include PMIX_EVENT_HEADER
-
-#if PMIX_HAVE_HWLOC
-#include "src/hwloc/hwloc-internal.h"
-#endif
 
 #include "src/class/pmix_list.h"
 #include "src/util/pmix_environ.h"
@@ -253,10 +248,6 @@
     size_t ninfo;
     mylock_t mylock;
     int ncycles=1, m, delay=0;
-    bool hwloc = false;
-#if PMIX_HAVE_HWLOC
-    char *hwloc_file = NULL;
-#endif
     sigset_t unblock;
 
     /* smoke test */
@@ -293,29 +284,12 @@
                    0 == strcmp("--sleep", argv[n])) &&
                    NULL != argv[n+1]) {
             delay = strtol(argv[n+1], NULL, 10);
-#if PMIX_HAVE_HWLOC
-        } else if (0 == strcmp("-hwloc", argv[n]) ||
-                   0 == strcmp("--hwloc", argv[n])) {
-            /* test hwloc support */
-            hwloc = true;
-        } else if (0 == strcmp("-hwloc-file", argv[n]) ||
-                   0 == strcmp("--hwloc-file", argv[n])) {
-            if (NULL == argv[n+1]) {
-                fprintf(stderr, "The --hwloc-file option requires an argument\n");
-                exit(1);
-            }
-            hwloc_file = strdup(argv[n+1]);
-            hwloc = true;
-            ++n;
-#endif
         } else if (0 == strcmp("-h", argv[n])) {
             /* print the options and exit */
             fprintf(stderr, "usage: simptest <options>\n");
             fprintf(stderr, "    -n N     Number of clients to run\n");
             fprintf(stderr, "    -e foo   Name of the client executable to run (default: simpclient\n");
             fprintf(stderr, "    -reps N  Cycle for N repetitions");
-            fprintf(stderr, "    -hwloc   Test hwloc support\n");
-            fprintf(stderr, "    -hwloc-file FILE   Use file to import topology\n");
             fprintf(stderr, "    -net-test  Test network endpt assignments\n");
             fprintf(stderr, "    -arrays  Use the job session array to pass registration info\n");
             exit(0);
@@ -357,36 +331,12 @@
     }
 
     /* setup the server library and tell it to support tool connections */
-#if PMIX_HAVE_HWLOC
-    if (hwloc) {
-#if HWLOC_API_VERSION < 0x20000
-        ninfo = 4;
-#else
-        ninfo = 5;
-#endif
-    } else {
-        ninfo = 4;
-    }
-#else
     ninfo = 3;
-#endif
 
     PMIX_INFO_CREATE(info, ninfo);
     PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL);
     PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, NULL, PMIX_BOOL);
-    PMIX_INFO_LOAD(&info[2], PMIX_SERVER_GATEWAY, NULL, PMIX_BOOL);
-#if PMIX_HAVE_HWLOC
-    if (hwloc) {
-        if (NULL != hwloc_file) {
-            PMIX_INFO_LOAD(&info[3], PMIX_TOPOLOGY_FILE, hwloc_file, PMIX_STRING);
-        } else {
-            PMIX_INFO_LOAD(&info[3], PMIX_TOPOLOGY, NULL, PMIX_STRING);
-        }
-#if HWLOC_API_VERSION >= 0x20000
-        PMIX_INFO_LOAD(&info[4], PMIX_HWLOC_SHARE_TOPO, NULL, PMIX_BOOL);
-#endif
-    }
-#endif
+    PMIX_INFO_LOAD(&info[2], PMIX_SERVER_SCHEDULER, NULL, PMIX_BOOL);
     if (nettest) {
         /* set a known network configuration for the pnet/test component */
         putenv("PMIX_MCA_pnet_test_nverts=nodes:5;plane:d:3;plane:s:2;plane:d:5");
@@ -588,22 +538,13 @@
         PMIX_DATA_ARRAY_CREATE(x->info[n].value.data.darray, 2, PMIX_INFO);
         iptr = (pmix_info_t*)x->info[n].value.data.darray->array;
         (void)strncpy(iptr[0].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN);
-        iptr[0].value.type = PMIX_STRING;
-        iptr[0].value.data.string = regex;
-        (void)strncpy(iptr[1].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN);
-        iptr[1].value.type = PMIX_STRING;
-        iptr[1].value.data.string = ppn;
+        PMIX_INFO_LOAD(&iptr[0], PMIX_NODE_MAP, regex, PMIX_REGEX);
+        PMIX_INFO_LOAD(&iptr[1], PMIX_PROC_MAP, ppn, PMIX_REGEX);
         ++n;
     } else {
-        (void)strncpy(x->info[n].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN);
-        x->info[n].value.type = PMIX_STRING;
-        x->info[n].value.data.string = regex;
+        PMIX_INFO_LOAD(&x->info[n], PMIX_NODE_MAP, regex, PMIX_REGEX);
         ++n;
-
-        /* if we have some empty nodes, then fill their spots */
-        (void)strncpy(x->info[n].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN);
-        x->info[n].value.type = PMIX_STRING;
-        x->info[n].value.data.string = ppn;
+        PMIX_INFO_LOAD(&x->info[n], PMIX_PROC_MAP, ppn, PMIX_REGEX);
         ++n;
     }
 
diff -Nru pmix-3.2.2~rc1/test/sshot/daemon.c pmix-4.0.0/test/sshot/daemon.c
--- pmix-3.2.2~rc1/test/sshot/daemon.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/sshot/daemon.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix_server.h"
+#include "src/include/types.h"
+#include "src/include/pmix_globals.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <errno.h>
+#include <signal.h>
+
+#include "src/class/pmix_list.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+#include "src/util/argv.h"
+
+static pmix_server_module_t mymodule = {0};
+
+
+int main(int argc, char **argv)
+{
+    pmix_info_t *info;
+    size_t ninfo;
+    pmix_status_t rc;
+
+    fprintf(stderr, "PID: %lu Testing version %s\n", (unsigned long)getpid(), PMIx_Get_version());
+
+    /* set a known network configuration for the pnet/test component */
+    putenv("PMIX_MCA_pnet_sshot_scheduler_pipe=./myfifo");
+    putenv("PMIX_MCA_pnet_sshot_config_file=./mytopo.json");
+    putenv("PMIX_MCA_pnet=sshot");
+
+    ninfo = 1;
+    PMIX_INFO_CREATE(info, ninfo);
+    PMIX_INFO_LOAD(&info[0], PMIX_SERVER_SCHEDULER, NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) {
+        fprintf(stderr, "Init failed with error %d\n", rc);
+        return rc;
+    }
+    PMIX_INFO_FREE(info, ninfo);
+
+
+    /* finalize the server library */
+    if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) {
+        fprintf(stderr, "Finalize failed with error %d\n", rc);
+    }
+
+    return rc;
+}
diff -Nru pmix-3.2.2~rc1/test/sshot/generate.c pmix-4.0.0/test/sshot/generate.c
--- pmix-3.2.2~rc1/test/sshot/generate.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/sshot/generate.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,267 @@
+/*
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <getopt.h>
+#include <jansson.h>
+
+#define DEFAULT_NUM_SWITCHES     8
+#define DEFAULT_NUM_FABPORTS    10
+#define DEFAULT_NUM_EDGEPORTS   10
+#define DEFAULT_NUM_NODES        8
+#define DEFAULT_NUM_NICS         8
+#define DEFAULT_NUM_GROUPS       2
+
+static int iseven(int num)
+{
+    return !(num & 1);
+}
+
+static int help = 0;
+typedef struct {
+    int nswitches;
+    int *members;
+} grptrk_t;
+
+int main(int argc, char **argv)
+{
+    json_t *root, *switches, *fabricports, *edgeports;
+    json_t *sw, *port, *cport, *links;
+    json_t *nodes, *node, *nics;
+    char *s;
+    int idnum = 10;
+    int jnum = 0;
+    int pnum = 0;
+    int n, nn;
+    int nsw, nswitches = DEFAULT_NUM_SWITCHES;
+    int numfabports = DEFAULT_NUM_FABPORTS;
+    int numedgeports = DEFAULT_NUM_EDGEPORTS;
+    int nnics = DEFAULT_NUM_NICS;
+    int nnodes = DEFAULT_NUM_NODES;
+    int ngrps = DEFAULT_NUM_GROUPS;
+    char *tmp;
+    int opt;
+    grptrk_t map[100];
+    char *outfile = NULL;
+    static struct option myoptions[] = {
+        {"file", required_argument, NULL, 'f'},
+        {"nswitches", required_argument, NULL, 's'},
+        {"nfabports", required_argument, NULL, 'p'},
+        {"nedgeports", required_argument, NULL, 'e'},
+        {"nnodes", required_argument, NULL, 'c'},
+        {"nnics", required_argument, NULL, 'n'},
+        {"ngrps", required_argument, NULL, 'g'},
+        {"help", no_argument, &help, 1}
+    };
+    int option_index;
+
+    /* check for options */
+    while ((opt = getopt_long(argc, argv, "f:p:e:c:n",
+                              myoptions, &option_index)) != -1) {
+        switch (opt) {
+            case 'f':
+                outfile = optarg;
+                break;
+            case 's':
+                nswitches = strtol(optarg, NULL, 10);
+                break;
+            case 'p':
+                numfabports = strtol(optarg, NULL, 10);
+                break;
+            case 'e':
+                numedgeports = strtol(optarg, NULL, 10);
+                break;
+            case 'c':
+                nnodes = strtol(optarg, NULL, 10);
+                break;
+            case 'n':
+                nnics = strtol(optarg, NULL, 10);
+                break;
+            case 'g':
+                ngrps = strtol(optarg, NULL, 10);
+                break;
+            default:
+                fprintf(stderr, "Usage: %s\n    Options:\n"
+                        "        [-f] [output file - will add .json extension]\n"
+                        "        [-s] [number of switches]\n"
+                        "        [-g] [number of groups to organize the switches into (<100)\n"
+                        "        [-p] [number of fabricports per switch]\n"
+                        "        [-e] [number of edge ports per switch]\n"
+                        "        [-c] [number of nodes]\n"
+                        "        [-n] [number of NICs per node]\n", argv[0]);
+                exit(1);
+        }
+    }
+    if (help) {
+        fprintf(stdout, "Usage: %s\n    Options:\n"
+                "        [-f] [output file - will add .json extension]\n"
+                "        [-s] [number of switches]\n"
+                "        [-p] [number of fabricports per switch]\n"
+                "        [-e] [number of edge ports per switch]\n"
+                "        [-c] [number of nodes]\n"
+                "        [-n] [number of NICs per node]\n", argv[0]);
+        exit(0);
+    }
+
+    /* create a JSON object */
+    root = json_object();
+
+    /* we will be defining an array of switches */
+    switches = json_array();
+    json_object_set_new(root, "switches", switches);
+
+    for (n=0; n < ngrps; n++) {
+        map[n].nswitches = 0;
+        map[n].members = (int*)malloc(nswitches * sizeof(int));
+    }
+
+    for (nsw=0; nsw < nswitches; nsw++) {
+        /* each switch contains an object that starts with its identifiers */
+        sw = json_object();
+        (void)asprintf(&tmp, "0x30000c0r21b%d", nsw);
+        json_object_set_new(sw, "IP", json_string(tmp));
+        free(tmp);
+        nn = nsw % ngrps;
+        json_object_set_new(sw, "grpID", json_integer(nn));
+        map[nn].members[map[nn].nswitches] = nsw;
+        map[nn].nswitches++;
+        json_object_set_new(sw, "swcNum", json_integer(nsw));
+        /* each switch contains an array of fabric ports */
+        fabricports = json_array();
+        for (n=0; n < numfabports; n++) {
+            port = json_object();
+            /* each fabric port has an ID */
+            (void)asprintf(&tmp, "0x30000c0r21a01%d", idnum);
+            json_object_set_new(port, "id", json_string(tmp));
+            free(tmp);
+            ++idnum;
+            /* each fabric port has a "meta" object that contains the connection port */
+            cport = json_object();
+            (void)asprintf(&tmp, "0x30000c0r21j%dp%d", jnum, pnum);
+            json_object_set_new(cport, "conn_port", json_string(tmp));
+            json_object_set_new(port, "meta", cport);
+            free(tmp);
+            if (iseven(n)) {
+                pnum = 1;
+            } else {
+                pnum = 0;
+                ++jnum;
+            }
+            /* add the port to the array of fabricports */
+            json_array_append(fabricports, port);
+        }
+        /* add the fabricports to the switch */
+        json_object_set_new(sw, "fabricports", fabricports);
+
+        /* each switch also contains an array of edge ports */
+        edgeports = json_array();
+        for (n=0; n < numedgeports; n++) {
+            port = json_object();
+            /* each edge port has an ID */
+            (void)asprintf(&tmp, "0x30000c0r21a01%d", idnum);
+            json_object_set_new(port, "id", json_string(tmp));
+            free(tmp);
+            ++idnum;
+            /* each edge port has a "meta" object that contains the connection port */
+            cport = json_object();
+            (void)asprintf(&tmp, "0x30000c0r21j%dp%d", jnum, pnum);
+            json_object_set_new(cport, "conn_port", json_string(tmp));
+            json_object_set_new(port, "meta", cport);
+            free(tmp);
+            if (iseven(n)) {
+                pnum = 1;
+            } else {
+                pnum = 0;
+                ++jnum;
+            }
+            /* add the port to the array of edgeports */
+            json_array_append(edgeports, port);
+        }
+        /* add the edgeports to the switch */
+        json_object_set_new(sw, "edgeports", edgeports);
+
+        /* add the switch to the array */
+        json_array_append(switches, sw);
+    }
+
+    /* add the nmber of groups */
+    json_object_set_new(root, "numGroups", json_integer(ngrps));
+    /* set the max number of switches in a group */
+    nsw = (nswitches / ngrps) + (nswitches % ngrps);
+    json_object_set_new(root, "maxNumLocalSwitches", json_integer(nsw));
+
+    /* define an array of links */
+    links = json_array();
+    json_object_set_new(root, "links", links);
+    /* put some arbitrary endpoint info in it */
+    for (n=0; n < numfabports; n++) {
+        port = json_object();
+        (void)asprintf(&tmp, "0x30000c0r21a01%d", idnum);
+        json_object_set_new(port, "endpoint1", json_string(tmp));
+        free(tmp);
+        ++idnum;
+        (void)asprintf(&tmp, "0x30000c0r21a01%d", idnum);
+        json_object_set_new(port, "endpoint2", json_string(tmp));
+        free(tmp);
+        ++idnum;
+        /* add the port to the array of links */
+        json_array_append(links, port);
+    }
+
+    /* define an array of nodes */
+    nodes = json_array();
+    json_object_set_new(root, "nodes", nodes);
+    nsw = 0;
+
+    for (nn=0; nn < nnodes; nn++) {
+        /* each node contains an object that starts with its name */
+        node = json_object();
+        (void)asprintf(&tmp, "node%03d", nn);
+        json_object_set_new(node, "name", json_string(tmp));
+        free(tmp);
+        /* each node contains a set of NICs */
+        nics = json_array();
+        for (n=0; n < nnics; n++) {
+            port = json_object();
+            /* each NIC has an ID */
+            (void)asprintf(&tmp, "0x30000c0r21a01%d", idnum);
+            json_object_set_new(port, "id", json_string(tmp));
+            free(tmp);
+            ++idnum;
+            /* define a group for this node */
+            opt = nn % ngrps;
+            /* and a connection - each NIC needs to point to a
+             * switch in the same group */
+            nsw = n % map[opt].nswitches;
+            json_object_set_new(port, "switch", json_integer(map[opt].members[nsw]));
+            /* add the port to the NIC array */
+            json_array_append(nics, port);
+        }
+        /* add the NICs to the node */
+        json_object_set_new(node, "nics", nics);
+        /* add the node to the array */
+        json_array_append(nodes, node);
+    }
+
+    if (NULL == outfile) {
+        s = json_dumps(root, JSON_INDENT(4));
+        puts(s);
+    } else {
+        (void)asprintf(&tmp, "%s.json", outfile);
+        json_dump_file(root, tmp, JSON_INDENT(4));
+        free(tmp);
+    }
+    json_decref(root);
+
+    return 0;
+}
diff -Nru pmix-3.2.2~rc1/test/sshot/Makefile.am pmix-4.0.0/test/sshot/Makefile.am
--- pmix-3.2.2~rc1/test/sshot/Makefile.am	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/sshot/Makefile.am	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2020      Intel, Inc.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix $(pmix_check_jansson_CPPFLAGS)
+
+if HAVE_JANSSON
+noinst_PROGRAMS = generate server daemon testcoord
+
+generate_SOURCES = \
+        generate.c
+generate_LDFLAGS = $(pmix_check_jansson_LDFLAGS)
+generate_LDADD = $(pmix_check_jansson_LIBS)
+
+server_SOURCES = \
+        server.c
+server_LDFLAGS = $(pmix_check_jansson_LDFLAGS)
+server_LDADD = $(pmix_check_jansson_LIBS)
+
+daemon_SOURCES = \
+        daemon.c
+daemon_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+daemon_LDADD = \
+    $(top_builddir)/src/libpmix.la
+
+testcoord_SOURCES = \
+        testcoord.c
+testcoord_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS)
+testcoord_LDADD = \
+    $(top_builddir)/src/libpmix.la
+endif
diff -Nru pmix-3.2.2~rc1/test/sshot/server.c pmix-4.0.0/test/sshot/server.c
--- pmix-3.2.2~rc1/test/sshot/server.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/sshot/server.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <getopt.h>
+#include <fcntl.h>
+#include <jansson.h>
+
+int main(int argc, char **argv)
+{
+    int mypipe;
+    int len;
+    char *grps;
+    int rc;
+
+    /* open a named pipe */
+    mypipe = open("./myfifo", O_RDONLY | O_CREAT | O_NONBLOCK, S_IRWXG | S_IRWXU);
+
+    while (1) {
+        rc = read(mypipe, &len, sizeof(int));
+        if (0 == rc) {
+            continue;
+        }
+        if (0 > rc || sizeof(int) != rc) {
+            /* hit an error */
+            continue;
+        }
+        if (-1 == len) {
+            /* signal to exit */
+            break;
+        }
+        grps = (char*)malloc(len);
+        rc = read(mypipe, grps, len);
+        if (0 > rc || len != rc) {
+            /* hit an error */
+            continue;
+        }
+        fprintf(stderr, "GRPS: %s\n", grps);
+    }
+
+    unlink("./myfifo");
+    return 0;
+}
diff -Nru pmix-3.2.2~rc1/test/sshot/testcoord.c pmix-4.0.0/test/sshot/testcoord.c
--- pmix-3.2.2~rc1/test/sshot/testcoord.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/sshot/testcoord.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include "src/include/pmix_config.h"
+#include "include/pmix.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <pthread.h>
+#include <getopt.h>
+
+#include <pmix_server.h>
+
+#include "src/class/pmix_object.h"
+#include "src/util/output.h"
+#include "src/util/printf.h"
+
+typedef struct {
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    volatile bool active;
+    pmix_status_t status;
+} mylock_t;
+
+#define DEBUG_CONSTRUCT_LOCK(l)                     \
+    do {                                            \
+        pthread_mutex_init(&(l)->mutex, NULL);      \
+        pthread_cond_init(&(l)->cond, NULL);        \
+        (l)->active = true;                         \
+        (l)->status = PMIX_SUCCESS;                 \
+    } while(0)
+
+#define DEBUG_DESTRUCT_LOCK(l)              \
+    do {                                    \
+        pthread_mutex_destroy(&(l)->mutex); \
+        pthread_cond_destroy(&(l)->cond);   \
+    } while(0)
+
+#define DEBUG_WAIT_THREAD(lck)                                      \
+    do {                                                            \
+        pthread_mutex_lock(&(lck)->mutex);                          \
+        while ((lck)->active) {                                     \
+            pthread_cond_wait(&(lck)->cond, &(lck)->mutex);         \
+        }                                                           \
+        pthread_mutex_unlock(&(lck)->mutex);                        \
+    } while(0)
+
+#define DEBUG_WAKEUP_THREAD(lck)                        \
+    do {                                                \
+        pthread_mutex_lock(&(lck)->mutex);              \
+        (lck)->active = false;                          \
+        pthread_cond_broadcast(&(lck)->cond);           \
+        pthread_mutex_unlock(&(lck)->mutex);            \
+    } while(0)
+
+typedef struct {
+    mylock_t lock;
+    pmix_status_t status;
+    pmix_info_t *info;
+    size_t ninfo;
+} mycaddy_t;
+
+static void local_cbfunc(pmix_status_t status, void *cbdata)
+{
+    mylock_t *lock = (mylock_t*)cbdata;
+    lock->status = status;
+    DEBUG_WAKEUP_THREAD(lock);
+}
+
+static void setup_cbfunc(pmix_status_t status,
+                         pmix_info_t info[], size_t ninfo,
+                         void *provided_cbdata,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    mycaddy_t *mq = (mycaddy_t*)provided_cbdata;
+    size_t n;
+
+    /* transfer it to the caddy for return to the main thread */
+    if (0 < ninfo) {
+        PMIX_INFO_CREATE(mq->info, ninfo);
+        mq->ninfo = ninfo;
+        for (n=0; n < ninfo; n++) {
+            PMIX_INFO_XFER(&mq->info[n], &info[n]);
+        }
+    }
+
+    /* let the library release the data and cleanup from
+     * the operation */
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+
+    DEBUG_WAKEUP_THREAD(&mq->lock);
+}
+
+static int help = 0;
+
+int main(int argc, char **argv)
+{
+    pmix_status_t rc;
+    mycaddy_t cd;
+    mylock_t lock;
+    pmix_proc_t proc;
+    int tclass;
+    pmix_value_t *val;
+    pmix_geometry_t *geos;
+    size_t ngeos, n, m;
+    char *tmp;
+    pmix_info_t info[2];
+    static struct option myoptions[] = {
+        {"num_nodes", required_argument, NULL, 'n'},
+        {"num_devs", required_argument, NULL, 'd'},
+        {"ppn", required_argument, NULL, 'p'},
+        {"help", no_argument, &help, 1},
+        {0, 0, 0, 0}
+    };
+    int option_index;
+    int opt;
+    int nnodes = 2;
+    int ndevs = 8;
+    int ppn = 4;
+
+    while ((opt = getopt_long(argc, argv, "n:d:p:",
+                              myoptions, &option_index)) != -1) {
+        switch (opt) {
+            case 'n':
+                nnodes = strtol(optarg, NULL, 10);
+                break;
+            case 'd':
+                ndevs = strtol(optarg, NULL, 10);
+                break;
+            case 'p':
+                ppn = strtol(optarg, NULL, 10);
+                break;
+            default:
+                fprintf(stderr, "Usage: %s\n    Options:\n"
+                        "        [-n] [number of nodes]\n"
+                        "        [-d] [number of devices/node]\n"
+                        "        [-p] [number of procs/node]\n", argv[0]);
+                exit(1);
+        }
+    }
+    if (help) {
+        fprintf(stderr, "Usage: %s\n    Options:\n"
+                "        [-n] [number of nodes]\n"
+                "        [-d] [number of devices/node]\n"
+                "        [-p] [number of procs/node]\n", argv[0]);
+        exit(0);
+    }
+
+    pmix_asprintf(&tmp, "PMIX_MCA_pnet_sshot_num_nodes=%d", nnodes);
+    putenv(tmp);
+    pmix_asprintf(&tmp, "PMIX_MCA_pnet_sshot_devs_per_node=%d", ndevs);
+    putenv(tmp);
+    pmix_asprintf(&tmp, "PMIX_MCA_pnet_sshot_ppn=%d", ppn);
+    putenv(tmp);
+
+    if (PMIX_SUCCESS != (rc = PMIx_server_init(NULL, NULL, 0))) {
+        pmix_output(0, "Server init failed: %s", PMIx_Error_string(rc));
+        exit(rc);
+    }
+    pmix_output(0, "Server running");
+
+    DEBUG_CONSTRUCT_LOCK(&cd.lock);
+    if (PMIX_SUCCESS != (rc = PMIx_server_setup_application("SIMPSCHED", NULL, 0,
+                                                             setup_cbfunc, &cd))) {
+        pmix_output(0, "[%s:%d] PMIx_server_setup_application failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+        DEBUG_DESTRUCT_LOCK(&cd.lock);
+        goto done;
+    }
+    DEBUG_WAIT_THREAD(&cd.lock);
+    DEBUG_DESTRUCT_LOCK(&cd.lock);
+
+    DEBUG_CONSTRUCT_LOCK(&lock);
+    if (PMIX_SUCCESS != (rc = PMIx_server_setup_local_support("SIMPSCHED", cd.info, cd.ninfo,
+                                                              local_cbfunc, &lock))) {
+        pmix_output(0, "[%s:%d] PMIx_server_setup_local_support failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+        DEBUG_DESTRUCT_LOCK(&lock);
+        goto done;
+    }
+    DEBUG_WAIT_THREAD(&lock);
+    DEBUG_DESTRUCT_LOCK(&lock);
+
+    /* check a few things */
+    PMIX_LOAD_PROCID(&proc, "SIMPSCHED", PMIX_RANK_WILDCARD);
+    rc = PMIx_Get(&proc, "HPE_TRAFFIC_CLASS", NULL, 0, &val);
+    if (PMIX_SUCCESS != rc) {
+        pmix_output(0, "[%s:%d] Get of traffic class failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc));
+    }
+    PMIX_VALUE_GET_NUMBER(rc, val, tclass, int);
+    if (PMIX_SUCCESS != rc) {
+        pmix_output(0, "[%s:%d] Get of traffic class returned non-number", __FILE__, __LINE__);
+    }
+    pmix_output(0, "[%s:%d] Got traffic class %d", __FILE__, __LINE__, tclass);
+    PMIX_VALUE_RELEASE(val);
+
+    /* the test nodes are "nid000000" and "nid000001" */
+    PMIX_INFO_LOAD(&info[0], PMIX_NODE_INFO, NULL, PMIX_BOOL);
+    PMIX_INFO_LOAD(&info[1], PMIX_HOSTNAME, "nid000000", PMIX_STRING);
+
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_FABRIC_COORDINATES, info, 2, &val)) ||
+        NULL == val) {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get fabric coordinate failed: %s",
+                    proc.nspace, proc.rank, PMIx_Error_string(rc));
+        goto done;
+    }
+    if (PMIX_DATA_ARRAY == val->type) {
+        geos = (pmix_geometry_t*)val->data.darray->array;
+        ngeos = val->data.darray->size;
+        /* print them out for diagnostics - someday we can figure
+         * out an automated way of testing the answer */
+        for (m=0; m < ngeos; m++) {
+            char **foo = NULL;
+            char *view;
+            pmix_output(0, "Device[%d]: %s Osname: %s", (int)m, geos[m].uuid, geos[m].osname);
+            for (n=0; n < geos[m].coordinates[0].dims; n++) {
+                asprintf(&tmp, "%d", geos[m].coordinates[0].coord[n]);
+                pmix_argv_append_nosize(&foo, tmp);
+                free(tmp);
+            }
+            tmp = pmix_argv_join(foo, ',');
+            pmix_argv_free(foo);
+            if (PMIX_COORD_LOGICAL_VIEW == geos[m].coordinates[0].view) {
+                view = "LOGICAL";
+            } else {
+                view = "PHYSICAL";
+            }
+            pmix_output(0, "\tCOORD VIEW %s: %s", view, tmp);
+            free(tmp);
+        }
+    } else {
+        pmix_output(0, "Client ns %s rank %d: PMIx_Get fabric coordinate returned wrong type: %s",
+                    proc.nspace, proc.rank, PMIx_Data_type_string(val->type));
+    }
+
+ done:
+    /* finalize us */
+    pmix_output(0, "Server finalizing");
+    if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) {
+        fprintf(stderr, "Server finalize failed: %s\n", PMIx_Error_string(rc));
+    } else {
+        fprintf(stderr, "Server finalize successfully completed\n");
+    }
+    fflush(stderr);
+    return(rc);
+}
diff -Nru pmix-3.2.2~rc1/test/test_cd.h pmix-4.0.0/test/test_cd.h
--- pmix-3.2.2~rc1/test/test_cd.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_cd.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -9,7 +9,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_common.c pmix-4.0.0/test/test_common.c
--- pmix-3.2.2~rc1/test/test_common.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_common.c	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
@@ -13,7 +13,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix_common.h>
+#include "include/pmix_common.h"
 
 #include "test_common.h"
 #include <stdarg.h>
@@ -61,7 +61,8 @@
         } else if (0 == strcmp(argv[i], "--h") || 0 == strcmp(argv[i], "-h")) {
             /* print help */
             fprintf(stderr, "usage: pmix_test [-h] [-e foo] [-b] [-c] [-nb]\n");
-            fprintf(stderr, "\t-n       provides information about the job size (for checking purposes)\n");
+            fprintf(stderr, "\t-n       the job size (for checking purposes)\n");
+            fprintf(stderr, "\t-s       number of servers to emulate\n");
             fprintf(stderr, "\t-e foo   use foo as test client\n");
             fprintf(stderr, "\t-v       verbose output\n");
             fprintf(stderr, "\t-t <>    set timeout\n");
diff -Nru pmix-3.2.2~rc1/test/test_common.h pmix-4.0.0/test/test_common.h
--- pmix-3.2.2~rc1/test/test_common.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_common.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -18,13 +18,14 @@
 #define TEST_COMMON_H
 
 #include "src/include/pmix_config.h"
-#include <pmix_common.h>
+#include "include/pmix_common.h"
 
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <stdint.h>
+#include <sys/time.h>
 
 #include "src/include/pmix_globals.h"
 #include "src/class/pmix_list.h"
@@ -50,10 +51,15 @@
 
 #define STRIPPED_FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
 
-#define TEST_OUTPUT(x) { \
-    fprintf(file,"==%d== %s:%s: %s\n", getpid(), STRIPPED_FILE_NAME, __func__, \
-            pmix_test_output_prepare x ); \
-    fflush(file); \
+#define TEST_OUTPUT(x) {                            \
+    struct timeval tv;                              \
+    gettimeofday(&tv, NULL);                        \
+    double ts = tv.tv_sec + 1E-6*tv.tv_usec;        \
+    fprintf(file,"==%d== [%lf] %s:%s: %s\n",        \
+            getpid(), ts,STRIPPED_FILE_NAME,        \
+            __func__,                               \
+            pmix_test_output_prepare x );           \
+    fflush(file);                                   \
 }
 
 // Write output without adding anything to it.
@@ -65,9 +71,15 @@
 
 // Always write errors to the stderr
 #define TEST_ERROR(x) { \
-    fprintf(stderr,"==%d== ERROR [%s:%d:%s]: %s\n", getpid(), STRIPPED_FILE_NAME, __LINE__, __func__, \
-            pmix_test_output_prepare x ); \
-    fflush(stderr); \
+    struct timeval tv;                              \
+    gettimeofday(&tv, NULL);                        \
+    double ts = tv.tv_sec + 1E-6*tv.tv_usec;        \
+    fprintf(stderr,                                 \
+            "==%d== [%lf] ERROR [%s:%d:%s]: %s\n",  \
+            getpid(), ts,                           \
+            STRIPPED_FILE_NAME, __LINE__, __func__, \
+            pmix_test_output_prepare x );           \
+    fflush(stderr);                                 \
 }
 
 #define TEST_VERBOSE_ON() (pmix_test_verbose = 1)
@@ -245,8 +257,7 @@
     PMIX_VAL_SET(&value, dtype, data);                                                                              \
     TEST_VERBOSE(("%s:%d put key %s", my_nspace, my_rank, key));                                                  \
     if (PMIX_SUCCESS != (rc = PMIx_Put(flag, key, &value))) {                                                       \
-        TEST_ERROR(("%s:%d: PMIx_Put key %s failed: %d", my_nspace, my_rank, key, rc));                             \
-        rc = PMIX_ERROR;                                                                                            \
+        TEST_ERROR(("%s:%d: PMIx_Put key %s failed: %s", my_nspace, my_rank, key, PMIx_Error_string(rc)));                             \
     }                                                                                                               \
     PMIX_VALUE_DESTRUCT(&value);                                                                                    \
 } while (0)
@@ -258,15 +269,13 @@
     cbdata.status = PMIX_SUCCESS;                                                                                   \
     pmix_proc_t foobar; \
     SET_KEY(key, fence_num, ind, use_same_keys);                                                                    \
-    (void)strncpy(foobar.nspace, ns, PMIX_MAX_NSLEN); \
-    foobar.rank = r; \
+    PMIX_LOAD_PROCID(&foobar, ns, r); \
     TEST_VERBOSE(("%s:%d want to get from %s:%d key %s", my_nspace, my_rank, ns, r, key));                          \
     if (blocking) {                                                                                                 \
         if (PMIX_SUCCESS != (rc = PMIx_Get(&foobar, key, NULL, 0, &val))) {                                         \
             if( !( (rc == PMIX_ERR_NOT_FOUND || rc == PMIX_ERR_PROC_ENTRY_NOT_FOUND) && ok_notfnd ) ){                                                       \
-                TEST_ERROR(("%s:%d: PMIx_Get failed: %d from %s:%d, key %s", my_nspace, my_rank, rc, ns, r, key));  \
+                TEST_ERROR(("%s:%d: PMIx_Get failed: %s from %s:%d, key %s", my_nspace, my_rank, PMIx_Error_string(rc), ns, r, key));  \
             }                                                                                                       \
-            rc = PMIX_ERROR;                                                                                        \
         }                                                                                                           \
     } else {                                                                                                        \
         int count;                                                                                                  \
@@ -274,8 +283,7 @@
         PMIX_VALUE_CREATE(val, 1);                                                                                  \
         cbdata.kv = val;                                                                                            \
         if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&foobar, key, NULL, 0, get_cb, (void*)&cbdata))) {                    \
-            TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s", my_nspace, my_rank, rc, ns, r, key)); \
-            rc = PMIX_ERROR;                                                                                        \
+            TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %s from %s:%d, key=%s", my_nspace, my_rank, PMIx_Error_string(rc), ns, r, key)); \
         } else {                                                                                                    \
             count = 0;                                                                                              \
             while(cbdata.in_progress){                                                                              \
@@ -292,19 +300,16 @@
     if (PMIX_SUCCESS == rc) {                                                                                       \
         if( PMIX_SUCCESS != cbdata.status ){                                                                        \
             if( !( (cbdata.status == PMIX_ERR_NOT_FOUND || cbdata.status == PMIX_ERR_PROC_ENTRY_NOT_FOUND) && ok_notfnd ) ){ \
-                TEST_ERROR(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s",                                     \
-                            my_nspace, my_rank, rc, my_nspace, r, key));                                            \
+                TEST_ERROR(("%s:%d: PMIx_Get_nb failed: %s from %s:%d, key=%s",                                     \
+                            my_nspace, my_rank, PMIx_Error_string(rc), my_nspace, r, key));                                            \
             }                                                                                                       \
-            rc = PMIX_ERROR;                                                                                        \
         } else if (NULL == val) {                                                                                   \
             TEST_VERBOSE(("%s:%d: PMIx_Get returned NULL value", my_nspace, my_rank));                              \
-            rc = PMIX_ERROR;                                                                                        \
         }                                                                                                           \
         else if (val->type != PMIX_VAL_TYPE_ ## dtype || PMIX_VAL_CMP(dtype, PMIX_VAL_FIELD_ ## dtype((val)), data)) {  \
             TEST_ERROR(("%s:%u: from %s:%d Key %s value or type mismatch,"                                        \
                         " want type %d get type %d",                                                                \
                         my_nspace, my_rank, ns, r, key, PMIX_VAL_TYPE_ ## dtype, val->type));                    \
-            rc = PMIX_ERROR;                                                                                        \
         }                                                                                                           \
     }                                                                                                               \
     if (PMIX_SUCCESS == rc) {                                                                                       \
@@ -348,4 +353,67 @@
     }                                                                           \
 } while (0)
 
+/* Key-Value pair management macros */
+// TODO: add all possible types/fields here.
+
+#define PMIX_VAL_FIELD_int(x)       ((x)->data.integer)
+#define PMIX_VAL_FIELD_uint32_t(x)  ((x)->data.uint32)
+#define PMIX_VAL_FIELD_uint16_t(x)  ((x)->data.uint16)
+#define PMIX_VAL_FIELD_string(x)    ((x)->data.string)
+#define PMIX_VAL_FIELD_float(x)     ((x)->data.fval)
+#define PMIX_VAL_FIELD_byte(x)      ((x)->data.byte)
+#define PMIX_VAL_FIELD_flag(x)      ((x)->data.flag)
+
+#define PMIX_VAL_TYPE_int      PMIX_INT
+#define PMIX_VAL_TYPE_uint32_t PMIX_UINT32
+#define PMIX_VAL_TYPE_uint16_t PMIX_UINT16
+#define PMIX_VAL_TYPE_string   PMIX_STRING
+#define PMIX_VAL_TYPE_float    PMIX_FLOAT
+#define PMIX_VAL_TYPE_byte     PMIX_BYTE
+#define PMIX_VAL_TYPE_flag     PMIX_BOOL
+
+#define PMIX_VAL_set_assign(_v, _field, _val )   \
+    do {                                                            \
+        (_v)->type = PMIX_VAL_TYPE_ ## _field;                      \
+        PMIX_VAL_FIELD_ ## _field((_v)) = _val;                     \
+    } while (0)
+
+#define PMIX_VAL_set_strdup(_v, _field, _val )       \
+    do {                                                                \
+        (_v)->type = PMIX_VAL_TYPE_ ## _field;                          \
+        PMIX_VAL_FIELD_ ## _field((_v)) = strdup(_val);                 \
+    } while (0)
+
+#define PMIX_VAL_SET_int        PMIX_VAL_set_assign
+#define PMIX_VAL_SET_uint32_t   PMIX_VAL_set_assign
+#define PMIX_VAL_SET_uint16_t   PMIX_VAL_set_assign
+#define PMIX_VAL_SET_string     PMIX_VAL_set_strdup
+#define PMIX_VAL_SET_float      PMIX_VAL_set_assign
+#define PMIX_VAL_SET_byte       PMIX_VAL_set_assign
+#define PMIX_VAL_SET_flag       PMIX_VAL_set_assign
+
+#define PMIX_VAL_SET(_v, _field, _val )   \
+    PMIX_VAL_SET_ ## _field(_v, _field, _val)
+
+#define PMIX_VAL_cmp_val(_val1, _val2)      ((_val1) != (_val2))
+#define PMIX_VAL_cmp_float(_val1, _val2)    (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001))
+#define PMIX_VAL_cmp_ptr(_val1, _val2)      strncmp(_val1, _val2, strlen(_val1)+1)
+
+#define PMIX_VAL_CMP_int        PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_uint32_t   PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_uint16_t   PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_float      PMIX_VAL_cmp_float
+#define PMIX_VAL_CMP_string     PMIX_VAL_cmp_ptr
+#define PMIX_VAL_CMP_byte       PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_flag       PMIX_VAL_cmp_val
+
+#define PMIX_VAL_ASSIGN(_v, _field, _val) \
+    PMIX_VAL_set_assign(_v, _field, _val)
+
+#define PMIX_VAL_CMP(_field, _val1, _val2) \
+    PMIX_VAL_CMP_ ## _field(_val1, _val2)
+
+#define PMIX_VAL_FREE(_v) \
+     PMIx_free_value_data(_v)
+
 #endif // TEST_COMMON_H
diff -Nru pmix-3.2.2~rc1/test/test_error.c pmix-4.0.0/test/test_error.c
--- pmix-3.2.2~rc1/test/test_error.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_error.c	2021-01-02 08:56:17.000000000 +0000
@@ -69,9 +69,6 @@
     pmix_status_t status;
     pmix_proc_t source;
 
-    /* turn OFF event handler testing pending fix of timeout_errhandler */
-    return PMIX_SUCCESS;
-
     TEST_VERBOSE(("test-error: running  error handling test cases"));
     /* register specific client error handlers and test their invocation
      * by  trigerring events  from server side*/
diff -Nru pmix-3.2.2~rc1/test/test_error.h pmix-4.0.0/test/test_error.h
--- pmix-3.2.2~rc1/test/test_error.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_error.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -9,7 +9,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_fence.c pmix-4.0.0/test/test_fence.c
--- pmix-3.2.2~rc1/test/test_fence.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_fence.c	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2017 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -146,8 +146,7 @@
             PMIX_PROC_CREATE(pcs, npcs);
             i = 0;
             PMIX_LIST_FOREACH(p, desc->participants, participant_t) {
-                (void)strncpy(pcs[i].nspace, p->proc.nspace, PMIX_MAX_NSLEN);
-                pcs[i].rank = p->proc.rank;
+                PMIX_LOAD_PROCID(&pcs[i], p->proc.nspace, p->proc.rank);
                 i++;
             }
 
@@ -175,8 +174,7 @@
                     for (i = 0; i < nranks; i++) {
                         participant_t *prt;
                         prt = PMIX_NEW(participant_t);
-                        strncpy(prt->proc.nspace, ranks[i].nspace, strlen(ranks[i].nspace)+1);
-                        prt->proc.rank = ranks[i].rank;
+                        PMIX_LOAD_PROCID(&prt->proc, ranks[i].nspace, ranks[i].rank);
                         pmix_list_append(desc->participants, &prt->super);
                     }
                     PMIX_PROC_FREE(ranks, nranks);
@@ -189,7 +187,7 @@
                 snprintf(sval, 500, "%d:%s:%d", fence_num, p->proc.nspace, p->proc.rank);
                 GET(string, sval, p->proc.nspace, p->proc.rank, fence_num, put_ind++, params.use_same_keys, 1, 0);
                 if (PMIX_SUCCESS != rc) {
-                    TEST_ERROR(("%s:%d: PMIx_Get failed (%d) from %s:%d", my_nspace, my_rank, rc, p->proc.nspace, p->proc.rank));
+                    TEST_ERROR(("%s:%d: PMIx_Get failed (%s) from %s:%d", my_nspace, my_rank, PMIx_Error_string(rc), p->proc.nspace, p->proc.rank));
                     PMIX_PROC_FREE(pcs, npcs);
                     PMIX_LIST_DESTRUCT(&test_fences);
                     exit(rc);
@@ -327,6 +325,7 @@
     pmix_value_t value;
     pmix_value_t *val = &value;
     pmix_proc_t proc;
+    pmix_info_t info;
 
     (void)strncpy(proc.nspace, my_nspace, PMIX_MAX_NSLEN);
     proc.rank = my_rank;
@@ -415,13 +414,24 @@
             }
         }
 
-        /* ask for a non-existent key */
-        proc.rank = i+params.base_rank;
-        if (PMIX_SUCCESS == (rc = PMIx_Get(&proc, "foobar", NULL, 0, &val))) {
+      cleanout:
+        TEST_VERBOSE(("%s:%d: rank %d is OK", my_nspace, my_rank, i+params.base_rank));
+    }
+
+    /* ask for a non-existent key */
+    if (0 == my_rank) {
+        proc.rank = 1+params.base_rank;
+        j = 1;
+        PMIX_INFO_LOAD(&info, PMIX_TIMEOUT, &j, PMIX_INT);
+        PMIX_INFO_REQUIRED(&info);
+        if (PMIX_SUCCESS == (rc = PMIx_Get(&proc, "foobar", &info, 1, &val))) {
             TEST_ERROR(("%s:%d: PMIx_Get returned success instead of failure",
                         my_nspace, my_rank));
             exit(PMIX_ERROR);
         }
+        if (PMIX_ERR_NOT_SUPPORTED == rc) {
+            goto cleanout;
+        }
         if (PMIX_ERR_NOT_FOUND != rc && PMIX_ERR_PROC_ENTRY_NOT_FOUND != rc) {
             TEST_ERROR(("%s:%d [ERROR]: PMIx_Get returned %s instead of not_found",
                         my_nspace, my_rank, PMIx_Error_string(rc)));
@@ -431,7 +441,8 @@
             TEST_ERROR(("%s:%d [ERROR]: PMIx_Get did not return NULL value", my_nspace, my_rank));
             exit(PMIX_ERROR);
         }
-        TEST_VERBOSE(("%s:%d: rank %d is OK", my_nspace, my_rank, i+params.base_rank));
     }
+
+    free(peers);
     return PMIX_SUCCESS;
 }
diff -Nru pmix-3.2.2~rc1/test/test_fence.h pmix-4.0.0/test/test_fence.h
--- pmix-3.2.2~rc1/test/test_fence.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_fence.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -11,7 +11,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include <time.h>
 #include "test_common.h"
diff -Nru pmix-3.2.2~rc1/test/test_internal.h pmix-4.0.0/test/test_internal.h
--- pmix-3.2.2~rc1/test/test_internal.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_internal.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2017      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,7 +11,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_publish.c pmix-4.0.0/test/test_publish.c
--- pmix-3.2.2~rc1/test/test_publish.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_publish.c	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
diff -Nru pmix-3.2.2~rc1/test/test_publish.h pmix-4.0.0/test/test_publish.h
--- pmix-3.2.2~rc1/test/test_publish.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_publish.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -11,7 +11,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_replace.h pmix-4.0.0/test/test_replace.h
--- pmix-3.2.2~rc1/test/test_replace.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_replace.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2017      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2020      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -10,7 +11,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_resolve_peers.c pmix-4.0.0/test/test_resolve_peers.c
--- pmix-3.2.2~rc1/test/test_resolve_peers.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_resolve_peers.c	2021-01-02 08:56:17.000000000 +0000
@@ -25,29 +25,29 @@
     rc = PMIx_Resolve_peers(pmix_globals.hostname, nspace, &procs, &nprocs);
     if (PMIX_SUCCESS != rc) {
         TEST_ERROR(("%s:%d: Resolve peers test failed: rc = %d", my_nspace, my_rank, rc));
-        return rc;
+        exit(rc);
     }
     if (NULL == procs || 0 == nprocs) {
         TEST_ERROR(("%s:%d: Resolve peers didn't find any process from ns %s at this node\n", my_nspace, my_rank,my_nspace));
-        return PMIX_ERROR;
+        exit(PMIX_ERROR);
     }
     rc = get_all_ranks_from_namespace(params, nspace, &ranks, &nranks);
     if (PMIX_SUCCESS != rc) {
         TEST_ERROR(("%s:%d: get_all_ranks_from_namespace function failed", my_nspace, my_rank));
         PMIX_PROC_FREE(procs, nprocs);
-        return rc;
+        exit(rc);
     }
     if (nprocs != nranks) {
         TEST_ERROR(("%s:%d: Resolve peers returned incorect result: returned %lu processes, expected %lu", my_nspace, my_rank, nprocs, nranks));
         PMIX_PROC_FREE(procs, nprocs);
         PMIX_PROC_FREE(ranks, nranks);
-        return PMIX_ERROR;
+        exit(PMIX_ERROR);
     }
     for (i = 0; i < nprocs; i++) {
         if (procs[i].rank != ranks[i].rank) {
             TEST_ERROR(("%s:%d: Resolve peers returned incorrect result: returned value %s:%d, expected rank %d",
                         my_nspace, my_rank, procs[i].nspace, procs[i].rank, ranks[i].rank));
-            return PMIX_ERROR;
+            exit(PMIX_ERROR);
         }
     }
     PMIX_PROC_FREE(procs, nprocs);
@@ -114,7 +114,6 @@
             if (PMIX_SUCCESS == rc) {
                 TEST_VERBOSE(("%s:%d: Resolve peers succeeded for ns %s\n", my_nspace, my_rank, nspace));
             } else {
-                TEST_ERROR(("%s:%d: Resolve peers failed for different namespace\n", my_nspace, my_rank));
                 exit(rc);
             }
         }
@@ -122,7 +121,7 @@
         /* disconnect from the processes of this namespace. */
         rc = PMIx_Disconnect(procs, 2, NULL, 0);
         if (PMIX_SUCCESS == rc) {
-            TEST_VERBOSE(("%s:%d: Disconnect from %s succeeded %s.", my_nspace, my_rank, nspace));
+            TEST_VERBOSE(("%s:%d: Disconnect from %s succeeded.", my_nspace, my_rank, nspace));
         } else {
             TEST_ERROR(("%s:%d: Disconnect from %s failed.", my_nspace, my_rank, nspace));
             exit(rc);
diff -Nru pmix-3.2.2~rc1/test/test_resolve_peers.h pmix-4.0.0/test/test_resolve_peers.h
--- pmix-3.2.2~rc1/test/test_resolve_peers.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_resolve_peers.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -11,7 +11,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_server.c pmix-4.0.0/test/test_server.c
--- pmix-3.2.2~rc1/test/test_server.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_server.c	2021-01-02 08:56:17.000000000 +0000
@@ -2,8 +2,8 @@
  * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,7 +30,6 @@
 #include "server_callbacks.h"
 
 int my_server_id = 0;
-int test_fail = 0;
 
 server_info_t *my_server_info = NULL;
 pmix_list_t *server_list = NULL;
@@ -41,7 +40,7 @@
     close(s->rd_fd);
     close(s->wr_fd);
     if (s->evread) {
-        event_del(s->evread);
+        pmix_event_del(s->evread);
     }
     s->evread = NULL;
     if (NULL != s->hostname) {
@@ -188,7 +187,7 @@
             return;
         }
         free(tmp);
-        PMIX_INFO_LOAD(&info[4], PMIX_NODE_MAP, regex, PMIX_STRING);
+        PMIX_INFO_LOAD(&info[4], PMIX_NODE_MAP, regex, PMIX_REGEX);
     }
 
     /* generate the global proc map - if we have two
@@ -221,7 +220,7 @@
     }
     PMIx_generate_ppn(ranks, &ppn);
     free(ranks);
-    PMIX_INFO_LOAD(&info[5], PMIX_PROC_MAP, ppn, PMIX_STRING);
+    PMIX_INFO_LOAD(&info[5], PMIX_PROC_MAP, ppn, PMIX_REGEX);
     free(ppn);
 
     pmix_strncpy(info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN);
@@ -925,16 +924,14 @@
         server_info_t *server;
         PMIX_LIST_FOREACH(server, server_list, server_info_t) {
             server->evread = pmix_event_new(pmix_globals.evbase, server->rd_fd,
-                              EV_READ|EV_PERSIST, server_read_cb, server);
+                                            EV_READ|EV_PERSIST, server_read_cb, server);
             pmix_event_add(server->evread, NULL);
         }
     }
 
-#if 0
     /* register the errhandler */
     PMIx_Register_event_handler(NULL, 0, NULL, 0,
                                 errhandler, errhandler_reg_callbk, NULL);
-#endif
 
     /* setup to see sigchld on the forked tests */
     pmix_event_assign(&handler, pmix_globals.evbase, SIGCHLD,
@@ -953,12 +950,11 @@
     return rc;
 }
 
-int server_finalize(test_params *params)
+int server_finalize(test_params *params, int local_fail)
 {
     int rc = PMIX_SUCCESS;
-    int total_ret = 0;
+    int total_ret = local_fail;
 
-    total_ret = test_fail;
     if (0 != (rc = server_barrier())) {
         total_ret++;
         goto exit;
@@ -970,15 +966,16 @@
     }
 
     if (params->nservers && 0 == my_server_id) {
-        int ret;
         /* wait for all servers are finished */
-        ret = srv_wait_all(10.0);
-        if (!pmix_list_is_empty(server_list)) {
-            total_ret += ret;
-        }
+        total_ret += srv_wait_all(10.0);
         PMIX_LIST_RELEASE(server_list);
         TEST_VERBOSE(("SERVER %d FINALIZE PID:%d with status %d",
-                    my_server_id, getpid(), ret));
+                        my_server_id, getpid(), total_ret));
+        if (0 == total_ret) {
+            TEST_OUTPUT(("Test finished OK!"));
+        } else {
+            rc = PMIX_ERROR;
+        }
     }
     PMIX_LIST_RELEASE(server_nspace);
 
@@ -988,11 +985,6 @@
         total_ret += rc;
         goto exit;
     }
-    if (0 == total_ret) {
-        TEST_OUTPUT(("Test finished OK!"));
-    } else {
-        TEST_OUTPUT(("Test FAILED!"));
-    }
 
 exit:
     return total_ret;
diff -Nru pmix-3.2.2~rc1/test/test_server.h pmix-4.0.0/test/test_server.h
--- pmix-3.2.2~rc1/test/test_server.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_server.h	2021-01-02 08:56:17.000000000 +0000
@@ -2,7 +2,6 @@
  * Copyright (c) 2018      Mellanox Technologies, Inc.
  *                         All rights reserved.
  *
- * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -65,10 +64,9 @@
 extern pmix_list_t *server_list;
 extern server_info_t *my_server_info;
 extern pmix_list_t *server_nspace;
-extern int test_fail;
 
 int server_init(test_params *params);
-int server_finalize(test_params *params);
+int server_finalize(test_params *params, int local_fail);
 int server_barrier(void);
 int server_fence_contrib(char *data, size_t ndata,
                          pmix_modex_cbfunc_t cbfunc, void *cbdata);
diff -Nru pmix-3.2.2~rc1/test/test_spawn.c pmix-4.0.0/test/test_spawn.c
--- pmix-3.2.2~rc1/test/test_spawn.c	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_spawn.c	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,7 @@
 
 typedef struct {
     int in_progress;
-    char nspace[PMIX_MAX_NSLEN];
+    pmix_nspace_t nspace;
 } spawn_cbdata;
 
 static void spawn_cb(pmix_status_t status,
@@ -23,7 +23,7 @@
 {
     spawn_cbdata *cb = (spawn_cbdata*)cbdata;
 
-    strncpy(cb->nspace, nspace, strlen(nspace)+1);
+    PMIX_LOAD_NSPACE(cb->nspace, nspace);
     cb->in_progress = 0;
 }
 
@@ -32,7 +32,7 @@
     int rc;
     pmix_app_t *apps;
     size_t napps;
-    char nspace[PMIX_MAX_NSLEN+1];
+    pmix_nspace_t nspace;
     memset(nspace, 0, PMIX_MAX_NSLEN+1);
     napps = 1;
     PMIX_APP_CREATE(apps, napps);
@@ -44,17 +44,17 @@
     } else {
         spawn_cbdata cbdata;
         cbdata.in_progress = 1;
-        memset(cbdata.nspace, 0, PMIX_MAX_NSLEN);
+        memset(cbdata.nspace, 0, PMIX_MAX_NSLEN+1);
         rc = PMIx_Spawn_nb(NULL, 0, apps, napps, spawn_cb, (void*)&cbdata);
         if (PMIX_SUCCESS != rc) {
             PMIX_APP_FREE(apps, napps);
             exit(rc);
         }
         PMIX_WAIT_FOR_COMPLETION(cbdata.in_progress);
-        strncpy(nspace, cbdata.nspace, strlen(cbdata.nspace)+1);
+        PMIX_LOAD_NSPACE(nspace, cbdata.nspace);
     }
     PMIX_APP_FREE(apps, napps);
-    if (strncmp(nspace, "foobar", strlen(nspace)+1)) {
+    if (!PMIX_CHECK_NSPACE(nspace, "foobar")) {
         exit(PMIX_ERROR);
     }
     return rc;
diff -Nru pmix-3.2.2~rc1/test/test_spawn.h pmix-4.0.0/test/test_spawn.h
--- pmix-3.2.2~rc1/test/test_spawn.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/test_spawn.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
@@ -11,7 +11,7 @@
  */
 
 #include "src/include/pmix_config.h"
-#include <pmix.h>
+#include "include/pmix.h"
 
 #include "test_common.h"
 
diff -Nru pmix-3.2.2~rc1/test/test_v2/base64_enc_dec.c pmix-4.0.0/test/test_v2/base64_enc_dec.c
--- pmix-3.2.2~rc1/test/test_v2/base64_enc_dec.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/base64_enc_dec.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,129 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+/* This code was taken from Open MPI project, file
+   opal/mca/pmix/base/pmix_base_fns.c
+*/
+
+#include <assert.h>
+#include <string.h>
+#include "test_common.h"
+
+/* base64 encoding with illegal characters removed ('=' is replaced by ' ') */
+static inline unsigned char pmixt_base64_encsym (unsigned char value) {
+    assert (value < 64);
+
+    if (value < 26) {
+    return 'A' + value;
+    } else if (value < 52) {
+    return 'a' + (value - 26);
+    } else if (value < 62) {
+    return '0' + (value - 52);
+    }
+
+    return (62 == value) ? '+' : '/';
+}
+
+static inline unsigned char pmixt_base64_decsym (unsigned char value) {
+    if ('+' == value) {
+    return 62;
+    } else if ('/' == value) {
+    return 63;
+    } else if (' ' == value) {
+    return 64;
+    } else if (value <= '9') {
+    return (value - '0') + 52;
+    } else if (value <= 'Z') {
+    return (value - 'A');
+    } else if (value <= 'z') {
+    return (value - 'a') + 26;
+    }
+
+    return 64;
+}
+
+static inline void pmixt_base64_encode_block (const unsigned char in[3], char out[4], int len) {
+    out[0] = pmixt_base64_encsym (in[0] >> 2);
+    out[1] = pmixt_base64_encsym (((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4));
+    out[2] = 1 < len ? pmixt_base64_encsym(((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6)) : ' ';
+    out[3] = 2 < len ? pmixt_base64_encsym(in[2] & 0x3f) : ' ';
+}
+
+static inline int pmixt_base64_decode_block (const char in[4], unsigned char out[3]) {
+    char in_dec[4];
+
+    in_dec[0] = pmixt_base64_decsym (in[0]);
+    in_dec[1] = pmixt_base64_decsym (in[1]);
+    in_dec[2] = pmixt_base64_decsym (in[2]);
+    in_dec[3] = pmixt_base64_decsym (in[3]);
+
+    out[0] = in_dec[0] << 2 | in_dec[1] >> 4;
+    if (64 == in_dec[2]) {
+        return 1;
+    }
+
+    out[1] = in_dec[1] << 4 | in_dec[2] >> 2;
+    if (64 == in_dec[3]) {     
+        return 2;
+    }
+
+    out[2] = ((in_dec[2] << 6) & 0xc0) | in_dec[3];
+    return 3;
+}
+
+char *pmixt_encode(const void *val, size_t vallen)
+{
+    char *outdata, *tmp;
+    size_t i, outlen;
+
+    outlen = ((2 + vallen) * 4) / 3 + 2;
+    outdata = calloc (outlen, 1);
+    if (NULL == outdata) {
+        return NULL;
+    }
+
+    for (i = 0, tmp = outdata ; i < vallen ; i += 3, tmp += 4) {
+        pmixt_base64_encode_block((unsigned char *) val + i, tmp, vallen - i);
+    }
+
+    tmp[0] = (unsigned char)'\0';
+    TEST_VERBOSE(("vallen = %ld, outlen = %ld", vallen, outlen));
+    return outdata;
+}
+
+ssize_t pmixt_decode (const char *data, void *decdata, size_t buffsz)
+{
+    size_t input_len = strlen(data) / 4;
+    unsigned char *ret = decdata;
+    int out_len;
+    size_t i;
+
+    for (i = 0, out_len = 0 ; i < input_len ; i++, data += 4) {
+        // check against size of buffer to make sure we don't overrun
+        if (buffsz - out_len >= 1) {
+            out_len += pmixt_base64_decode_block(data, decdata + (3 * i));
+        }
+        else {
+            assert((buffsz - out_len) >= 1);
+            exit(1);
+        }
+    }
+    
+    return out_len;
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/cli_stages.c pmix-4.0.0/test/test_v2/cli_stages.c
--- pmix-3.2.2~rc1/test/test_v2/cli_stages.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/cli_stages.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,228 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "cli_stages.h"
+
+cli_info_t *cli_info = NULL;
+int cli_info_cnt = 0;
+bool test_abort = false;
+bool test_complete = false;
+
+int cli_rank(cli_info_t *cli)
+{
+    int i;
+    for(i=0; i < cli_info_cnt; i++){
+        if( cli == &cli_info[i] ){
+            return cli->rank;
+        }
+    }
+    return -1;
+}
+
+void cli_init(int nprocs)
+{
+    int n, i;
+    cli_state_t order[CLI_TERM+1];
+
+    cli_info = malloc( sizeof(cli_info_t) * nprocs);
+    cli_info_cnt = nprocs;
+
+    order[CLI_UNINIT] = CLI_FORKED;
+    order[CLI_FORKED] = CLI_FIN;
+    order[CLI_CONNECTED] = CLI_UNDEF;
+    order[CLI_FIN] = CLI_TERM;
+    order[CLI_DISCONN] = CLI_UNDEF;
+    order[CLI_TERM] = CLI_UNDEF;
+
+    for (n=0; n < nprocs; n++) {
+        cli_info[n].sd = -1;
+        cli_info[n].ev = NULL;
+        cli_info[n].pid = -1;
+        cli_info[n].state = CLI_UNINIT;
+        PMIX_CONSTRUCT(&(cli_info[n].modex), pmix_list_t);
+        for (i = 0; i < CLI_TERM+1; i++) {
+            cli_info[n].next_state[i] = order[i];
+        }
+        cli_info[n].rank = -1;
+        cli_info[n].ns = NULL;
+    }
+}
+
+void cli_connect(cli_info_t *cli, int sd, pmix_event_base_t * ebase, event_callback_fn callback)
+{
+    if( CLI_CONNECTED != cli->next_state[cli->state] ){
+        TEST_ERROR(("Rank %d has bad next state: expect %d have %d!",
+                     cli_rank(cli), CLI_CONNECTED, cli->next_state[cli->state]));
+        test_abort = true;
+        return;
+    }
+
+    cli->sd = sd;
+    cli->ev = pmix_event_new(ebase, sd,
+                      EV_READ|EV_PERSIST, callback, cli);
+    pmix_event_add(cli->ev,NULL);
+    pmix_ptl_base_set_nonblocking(sd);
+    TEST_VERBOSE(("Connection accepted from rank %d", cli_rank(cli) ));
+    cli->state = CLI_CONNECTED;
+}
+
+void cli_finalize(cli_info_t *cli)
+{
+    if( CLI_FIN != cli->next_state[cli->state] ){
+        TEST_ERROR(("rank %d: bad client next state: expect %d have %d!",
+                     cli_rank(cli), CLI_FIN, cli->next_state[cli->state]));
+        test_abort = true;
+    }
+
+    cli->state = CLI_FIN;
+}
+
+void cli_disconnect(cli_info_t *cli)
+{
+    if( CLI_DISCONN != cli->next_state[cli->state] ){
+        TEST_ERROR(("rank %d: bad client next state: expect %d have %d!",
+                     cli_rank(cli), CLI_DISCONN, cli->next_state[cli->state]));
+        test_abort = true;
+    }
+
+    if( 0 > cli->sd ){
+        TEST_ERROR(("Bad sd = %d of rank = %d ", cli->sd, cli_rank(cli)));
+        test_abort = true;
+    } else {
+        TEST_VERBOSE(("close sd = %d for rank = %d", cli->sd, cli_rank(cli)));
+        close(cli->sd);
+        cli->sd = -1;
+    }
+
+    if( NULL == cli->ev ){
+        TEST_ERROR(("Bad ev = NULL of rank = %d ", cli_rank(cli)));
+        test_abort = true;
+    } else {
+        TEST_VERBOSE(("remove event of rank %d from event queue", cli_rank(cli)));
+        pmix_event_del(cli->ev);
+        pmix_event_free(cli->ev);
+        cli->ev = NULL;
+    }
+
+    TEST_VERBOSE(("Destruct modex list for the rank %d", cli_rank(cli)));
+    PMIX_LIST_DESTRUCT(&(cli->modex));
+
+    cli->state = CLI_DISCONN;
+}
+
+void cli_terminate(cli_info_t *cli)
+{
+    if( CLI_TERM != cli->next_state[cli->state] ){
+        TEST_ERROR(("rank %d: bad client next state: expect %d have %d!",
+                     cli_rank(cli), CLI_TERM, cli->next_state[cli->state]));
+        test_abort = true;
+    }
+    cli->pid = -1;
+    TEST_VERBOSE(("Client rank = %d terminated", cli_rank(cli)));
+    cli->state = CLI_TERM;
+    if (NULL != cli->ns) {
+        free(cli->ns);
+    }
+}
+
+void cli_cleanup(cli_info_t *cli)
+{
+    if (CLI_TERM < cli->state) {
+        TEST_ERROR(("Bad rank %d state %d", cli_rank(cli), cli->state));
+        test_abort = true;
+        return;
+    }
+    switch( cli->next_state[cli->state] ){
+    case CLI_FORKED:
+        break;
+    case CLI_CONNECTED:
+        /* error - means that process terminated w/o calling finalize */
+        if (!test_abort) {
+            TEST_ERROR(("rank %d with state %d unexpectedly terminated.", cli_rank(cli), cli->state));
+        }
+        cli->state = CLI_TERM;
+        test_abort = true;
+        break;
+    case CLI_FIN:
+        /* error - means that process terminated w/o calling finalize */
+        if (!test_abort) {
+            TEST_ERROR(("rank %d with state %d unexpectedly terminated.", cli_rank(cli), cli->state));
+        }
+        cli_finalize(cli);
+        cli_cleanup(cli);
+        test_abort = true;
+        break;
+    case CLI_DISCONN:
+        cli_disconnect(cli);
+        cli_cleanup(cli);
+        break;
+    case CLI_TERM:
+        cli_terminate(cli);
+        break;
+    default:
+        TEST_ERROR(("Bad rank %d next state %d", cli_rank(cli), cli->next_state[cli->state]));
+        test_abort = true;
+        return;
+    }
+}
+
+
+void cli_kill_all(void)
+{
+    int i;
+    for(i = 0; i < cli_info_cnt; i++){
+        if( CLI_UNINIT == cli_info[i].state ){
+            TEST_ERROR(("Skip rank %d as it wasn't ever initialized (shouldn't happen)",
+                          i));
+            continue;
+        } else if( CLI_TERM <= cli_info[i].state ){
+            TEST_VERBOSE(("Skip rank %d as it was already terminated.", i));
+            continue;
+
+        }
+        TEST_VERBOSE(("Kill rank %d (pid = %d).", i, cli_info[i].pid));
+        kill(cli_info[i].pid, SIGKILL);
+        cli_cleanup(&cli_info[i]);
+    }
+}
+
+void errhandler(size_t evhdlr_registration_id,
+                pmix_status_t status,
+                const pmix_proc_t *source,
+                pmix_info_t info[], size_t ninfo,
+                pmix_info_t results[], size_t nresults,
+                pmix_event_notification_cbfunc_fn_t cbfunc,
+                void *cbdata)
+{
+    TEST_ERROR((" PMIX server event handler for %s:%d with status = %d", source->nspace, source->rank, status));
+    /* notify clients of error */
+    PMIx_Notify_event(status, source,
+                      PMIX_RANGE_NAMESPACE,
+                      NULL, 0,
+                      op_callbk, NULL);
+}
+
+void op_callbk(pmix_status_t status,
+                      void *cbdata)
+{
+    TEST_VERBOSE(( "OP CALLBACK CALLED WITH STATUS %d", status));
+}
+
+void errhandler_reg_callbk (pmix_status_t status,
+                            size_t errhandler_ref,
+                            void *cbdata)
+{
+    TEST_VERBOSE(("ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu",
+                status, (unsigned long)errhandler_ref));
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/cli_stages.h pmix-4.0.0/test/test_v2/cli_stages.h
--- pmix-3.2.2~rc1/test/test_v2/cli_stages.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/cli_stages.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,85 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef CLI_STAGES_H
+#define CLI_STAGES_H
+
+#include <src/include/pmix_config.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <sys/time.h>
+#include <time.h>
+#include <errno.h>
+#include "src/include/pmix_globals.h"
+#include "pmix_server.h"
+#include "src/class/pmix_list.h"
+#include "src/mca/ptl/base/base.h"
+
+#include "test_common.h"
+
+// In correct scenario each client has to sequentially pass all of this stages
+typedef enum {
+    CLI_UNINIT, CLI_FORKED, CLI_CONNECTED, CLI_FIN, CLI_DISCONN, CLI_TERM, CLI_UNDEF
+} cli_state_t;
+
+typedef struct {
+    pmix_list_t modex;
+    pid_t pid;
+    int sd;
+    pmix_event_t *ev;
+    cli_state_t state;
+    cli_state_t next_state[CLI_TERM+1];
+    pmix_rank_t rank;
+    char *ns;
+    int exit_code;
+    bool alive;
+} cli_info_t;
+
+extern cli_info_t *cli_info;
+extern int cli_info_cnt;
+extern bool test_abort;
+extern bool test_complete;
+
+int cli_rank(cli_info_t *cli);
+void cli_init(int nprocs);
+void cli_connect(cli_info_t *cli, int sd, struct event_base * ebase, event_callback_fn callback);
+void cli_finalize(cli_info_t *cli);
+void cli_disconnect(cli_info_t *cli);
+void cli_terminate(cli_info_t *cli);
+void cli_cleanup(cli_info_t *cli);
+void cli_kill_all(void);
+
+bool test_terminated(void);
+
+void errhandler(size_t evhdlr_registration_id,
+                pmix_status_t status,
+                const pmix_proc_t *source,
+                pmix_info_t info[], size_t ninfo,
+                pmix_info_t results[], size_t nresults,
+                pmix_event_notification_cbfunc_fn_t cbfunc,
+                void *cbdata);
+
+void op_callbk(pmix_status_t status,
+               void *cbdata);
+
+void errhandler_reg_callbk (pmix_status_t status,
+                            size_t errhandler_ref,
+                            void *cbdata);
+
+#endif // CLI_STAGES_H
diff -Nru pmix-3.2.2~rc1/test/test_v2/Makefile pmix-4.0.0/test/test_v2/Makefile
--- pmix-3.2.2~rc1/test/test_v2/Makefile	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/Makefile	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,47 @@
+INSTALLDIR = $(HOME)/pmix-install
+INCLUDES = -I. -I../../test -I../../include -I../.. -I../../src/include
+vpath %.h .:../../test:../../include:../..:../../src/include
+LDIR = $(INSTALLDIR)/lib
+
+CC = gcc
+CFLAGS = $(INCLUDES) -g -Wfatal-errors #-Wall -Wextra -Wpedantic -Wconversion -Wshadow
+LDFLAGS = -Wl,-rpath,$(LDIR)
+LIBS =-lpmix -lpthread -levent
+
+# Executables
+EXE = pmix_test test_init_fin test_helloworld test_get_basic
+
+# List of all .c source files.
+PCFILES = pmix_test.c test_common.c cli_stages.c test_server.c server_callbacks.c base64_enc_dec.c
+TC1FILES = test_init_fin.c test_common.c base64_enc_dec.c
+TC2FILES = test_helloworld.c test_common.c base64_enc_dec.c
+TC3FILES = test_get_basic.c test_common.c base64_enc_dec.c
+TCFILES = $(TC1FILES) $(TC2FILES) $(TC3FILES)
+CFILES = $(PCFILES) $(TCFILES)
+
+OBJ = $(CFILES:%.c=%.o)
+POBJ = $(PCFILES:%.c=%.o)
+T1OBJ = $(TC1FILES:%.c=%.o)
+T2OBJ = $(TC2FILES:%.c=%.o)
+T3OBJ = $(TC3FILES:%.c=%.o)
+
+all: pmix_test test_init_fin test_helloworld test_get_basic
+.PHONY: all
+
+clean:
+	rm -f $(EXE) $(OBJ) $(DEP)
+.PHONY: clean
+
+pmix_test: $(POBJ) 
+	$(CC) -o $@ $^ $(CFLAGS) -L$(LDIR) $(LDFLAGS) $(LIBS)
+
+test_init_fin: $(T1OBJ)
+	$(CC) -o $@ $^ $(CFLAGS) -L$(LDIR) $(LDFLAGS) $(LIBS)
+test_helloworld: $(T2OBJ)
+	$(CC) -o $@ $^ $(CFLAGS) -L$(LDIR) $(LDFLAGS) $(LIBS)
+test_get_basic: $(T3OBJ)
+	$(CC) -o $@ $^ $(CFLAGS) -L$(LDIR) $(LDFLAGS) $(LIBS)
+
+%.o : %.c
+	$(CC) -c -o $@ $< $(CFLAGS)
+
diff -Nru pmix-3.2.2~rc1/test/test_v2/pmix_regex.c pmix-4.0.0/test/test_v2/pmix_regex.c
--- pmix-3.2.2~rc1/test/test_v2/pmix_regex.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/pmix_regex.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2018      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "src/util/argv.h"
+#include "src/util/pmix_environ.h"
+#include "src/util/output.h"
+#include "src/server/pmix_server_ops.h"
+#include "src/mca/preg/preg.h"
+
+#include "server_callbacks.h"
+#include "utils.h"
+
+#define TEST_NODES "odin001,odin002,odin003,odin010,odin011,odin075"
+#define TEST_PROCS "1,2,3,4;5-8;9,11-12;17-20;21-24;100"
+#define TEST_NODES2 "c712f6n01,c712f6n02,c712f6n03"
+
+bool spawn_wait = false;
+
+int main(int argc, char **argv)
+{
+    char *regex;
+    char **nodes, **procs;
+    pmix_status_t rc;
+
+    /* smoke test */
+    if (PMIX_SUCCESS != 0) {
+        TEST_ERROR(("ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d", PMIX_SUCCESS));
+        exit(1);
+    }
+
+    TEST_VERBOSE(("Testing version %s", PMIx_Get_version()));
+
+    PMIx_server_init(&mymodule, NULL, 0);
+
+    TEST_VERBOSE(("Start PMIx regex smoke test"));
+
+    fprintf(stderr, "NODES: %s\n", TEST_NODES);
+    PMIx_generate_regex(TEST_NODES, &regex);
+    fprintf(stderr, "REGEX: %s\n\n", regex);
+    /* test reverse parsing */
+    rc = pmix_preg.parse_nodes(regex, &nodes);
+    free(regex);
+    if (PMIX_SUCCESS == rc) {
+        regex = pmix_argv_join(nodes, ',');
+        pmix_argv_free(nodes);
+        fprintf(stderr, "NODES: %s\n", TEST_NODES);
+        fprintf(stderr, "RSULT: %s\n\n\n", regex);
+        free(regex);
+    } else {
+        fprintf(stderr, "Node reverse failed: %d\n\n\n", rc);
+    }
+
+    fprintf(stderr, "PROCS: %s\n", TEST_PROCS);
+    PMIx_generate_ppn(TEST_PROCS, &regex);
+    fprintf(stderr, "PPN: %s\n\n", regex);
+    /* test reverse parsing */
+    rc = pmix_preg.parse_procs(regex, &procs);
+    free(regex);
+    if (PMIX_SUCCESS == rc) {
+        regex = pmix_argv_join(procs, ';');
+        pmix_argv_free(procs);
+        fprintf(stderr, "PROCS: %s\n", TEST_PROCS);
+        fprintf(stderr, "RSULT: %s\n", regex);
+        free(regex);
+    } else {
+        fprintf(stderr, "PPN reverse failed: %d\n", rc);
+    }
+
+    fprintf(stderr, "NODES: %s\n", TEST_NODES2);
+    PMIx_generate_regex(TEST_NODES2, &regex);
+    fprintf(stderr, "REGEX: %s\n\n", regex);
+    /* test reverse parsing */
+    rc = pmix_preg.parse_nodes(regex, &nodes);
+    free(regex);
+    if (PMIX_SUCCESS == rc) {
+        regex = pmix_argv_join(nodes, ',');
+        pmix_argv_free(nodes);
+        fprintf(stderr, "NODES: %s\n", TEST_NODES2);
+        fprintf(stderr, "RSULT: %s\n\n\n", regex);
+        free(regex);
+    } else {
+        fprintf(stderr, "Node reverse failed: %d\n\n\n", rc);
+    }
+    return 0;
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/pmix_test.c pmix-4.0.0/test/test_v2/pmix_test.c
--- pmix-3.2.2~rc1/test/test_v2/pmix_test.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/pmix_test.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC.
+ *                         All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include "src/util/pmix_environ.h"
+#include "src/util/output.h"
+
+#include "server_callbacks.h"
+#include "utils.h"
+#include "test_server.h"
+#include "test_common.h"
+
+bool spawn_wait = false;
+
+int main(int argc, char **argv)
+{
+    char **client_env=NULL;
+    char **client_argv=NULL;
+    int rc, i;
+    struct stat stat_buf;
+    int test_fail = 0;
+    char *tmp;
+    int ns_nprocs;
+    sigset_t unblock;
+
+    default_params(&params, &val_params);
+
+    /* smoke test */
+    if (PMIX_SUCCESS != 0) {
+        TEST_ERROR(("ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d", PMIX_SUCCESS));
+        exit(1);
+    }
+
+    TEST_VERBOSE(("Testing version %s", PMIx_Get_version()));
+
+    parse_cmd(argc, argv, &params, &val_params);
+    TEST_VERBOSE(("Start PMIx_lite smoke test (timeout is %d)", params.timeout));
+
+    /* set common argv and env */
+    client_env = pmix_argv_copy(environ);
+    set_client_argv(&params, &client_argv);
+
+    tmp = pmix_argv_join(client_argv, ' ');
+    TEST_VERBOSE(("Executing test: %s", tmp));
+    free(tmp);
+
+    /* verify executable */
+    if( 0 > ( rc = stat(params.binary, &stat_buf) ) ){
+        TEST_ERROR(("Cannot stat() executable \"%s\": %d: %s", params.binary, errno, strerror(errno)));
+        free_params(&params, &val_params);
+        return 0;
+    } else if( !S_ISREG(stat_buf.st_mode) ){
+        TEST_ERROR(("Client executable \"%s\": is not a regular file", params.binary));
+        free_params(&params, &val_params);
+        return 0;
+    }else if( !(stat_buf.st_mode & S_IXUSR) ){
+        TEST_ERROR(("Client executable \"%s\": has no executable flag", params.binary));
+        free_params(&params, &val_params);
+        return 0;
+    }
+
+    /* ensure that SIGCHLD is unblocked as we need to capture it */
+    if (0 != sigemptyset(&unblock)) {
+        fprintf(stderr, "SIGEMPTYSET FAILED\n");
+        exit(1);
+    }
+    if (0 != sigaddset(&unblock, SIGCHLD)) {
+        fprintf(stderr, "SIGADDSET FAILED\n");
+        exit(1);
+    }
+    if (0 != sigprocmask(SIG_UNBLOCK, &unblock, NULL)) {
+        fprintf(stderr, "SIG_UNBLOCK FAILED\n");
+        exit(1);
+    }
+
+    if (PMIX_SUCCESS != (rc = server_init(&params, &val_params))) {
+        free_params(&params, &val_params);
+        return rc;
+    }
+
+    cli_init(val_params.pmix_local_size);
+
+    int launched = 0;
+    /* set namespaces and fork clients */
+    {
+        uint32_t j;
+        int base_rank = 0;
+
+        /* compute my start counter */
+        for(j = 0; j < (uint32_t)my_server_id; j++) {
+            base_rank += (params.nprocs % params.nservers) > (uint32_t)j ?
+                        params.nprocs / params.nservers + 1 :
+                        params.nprocs / params.nservers;
+        }
+        /* we have a single namespace for all clients */
+        ns_nprocs = params.nprocs;
+        launched += server_launch_clients(val_params.pmix_local_size, params.nprocs, base_rank,
+                                   &params, &val_params, &client_env, &client_argv);
+    }
+    if (val_params.pmix_local_size != (uint32_t)launched) {
+        TEST_ERROR(("srv #%d: Total number of processes doesn't correspond to number specified by ns_dist parameter.", 
+                    my_server_id));
+        cli_kill_all();
+        test_fail = 1;
+        goto done;
+    }
+
+    /* hang around until the client(s) finalize */
+    while (!test_complete) {
+        struct timespec ts;
+        ts.tv_sec = 0;
+        ts.tv_nsec = 100000;
+        nanosleep(&ts, NULL);
+    }
+
+    if( test_abort ){
+        TEST_ERROR(("srv #%d: Test was aborted!", my_server_id));
+        /* do not simply kill the clients as that generates
+         * event notifications which these tests then print
+         * out, flooding the log */
+      //  cli_kill_all();
+        test_fail = 1;
+    }
+
+    for(i=0; i < cli_info_cnt; i++){
+        if (cli_info[i].exit_code != 0) {
+            ++test_fail;
+        }
+    }
+
+    /* deregister the errhandler */
+    PMIx_Deregister_event_handler(0, op_callbk, NULL);
+
+  done:
+    TEST_VERBOSE(("srv #%d: call server_finalize!", my_server_id));
+    test_fail += server_finalize(&params, test_fail);
+
+    TEST_VERBOSE(("srv #%d: exit sequence!", my_server_id));
+    free_params(&params, &val_params);
+    pmix_argv_free(client_argv);
+    pmix_argv_free(client_env);
+
+    if (0 == test_fail) {
+        TEST_OUTPUT(("Test SUCCEEDED!"));
+    }
+    return test_fail;
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/server_callbacks.c pmix-4.0.0/test/test_v2/server_callbacks.c
--- pmix-3.2.2~rc1/test/test_v2/server_callbacks.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/server_callbacks.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,353 @@
+/*
+ * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include "server_callbacks.h"
+#include "src/util/argv.h"
+#include "test_server.h"
+
+extern bool spawn_wait;
+
+pmix_server_module_t mymodule = {
+    .client_connected = connected,
+    .client_finalized = finalized,
+    .abort = abort_fn,
+    .fence_nb = fencenb_fn,
+    .direct_modex = dmodex_fn,
+    .publish = publish_fn,
+    .lookup = lookup_fn,
+    .unpublish = unpublish_fn,
+    .spawn = spawn_fn,
+    .connect = connect_fn,
+    .disconnect = disconnect_fn,
+    .register_events = regevents_fn,
+    .deregister_events = deregevents_fn
+};
+
+typedef struct {
+    pmix_list_item_t super;
+    pmix_info_t data;
+    char *namespace_published;
+    int rank_published;
+} pmix_test_info_t;
+
+static void tcon(pmix_test_info_t *p)
+{
+    PMIX_INFO_CONSTRUCT(&p->data);
+}
+
+static void tdes(pmix_test_info_t *p)
+{
+    PMIX_INFO_DESTRUCT(&p->data);
+}
+
+PMIX_CLASS_INSTANCE(pmix_test_info_t,
+                          pmix_list_item_t,
+                          tcon, tdes);
+
+pmix_list_t *pmix_test_published_list = NULL;
+
+static int finalized_count = 0;
+
+pmix_status_t connected(const pmix_proc_t *proc, void *server_object,
+                        pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t finalized(const pmix_proc_t *proc, void *server_object,
+              pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    cli_info_t *cli = NULL;
+    int i;
+    for (i = 0; i < cli_info_cnt; i++) {
+        if((proc->rank == cli_info[i].rank) &&
+                (0 == strcmp(proc->nspace, cli_info[i].ns))){
+            cli = &cli_info[i];
+            break;
+        }
+    }
+    if (NULL == cli) {
+        TEST_ERROR(("cannot found rank %d", proc->rank));
+        return PMIX_SUCCESS;
+    }
+    if( CLI_TERM <= cli->state ){
+        TEST_ERROR(("double termination of rank %d", proc->rank));
+        return PMIX_SUCCESS;
+    }
+    TEST_VERBOSE(("Rank %s:%d terminated", proc->nspace, proc->rank));
+    cli_finalize(cli);
+    finalized_count++;
+    if (finalized_count == cli_info_cnt) {
+        if (NULL != pmix_test_published_list) {
+            PMIX_LIST_RELEASE(pmix_test_published_list);
+        }
+    }
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t abort_fn(const pmix_proc_t *proc, void *server_object,
+             int status, const char msg[],
+             pmix_proc_t procs[], size_t nprocs,
+             pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    TEST_VERBOSE(("Abort is called with status = %d, msg = %s",
+                  status, msg));
+    test_abort = true;
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs,
+               const pmix_info_t info[], size_t ninfo,
+               char *data, size_t ndata,
+               pmix_modex_cbfunc_t cbfunc, void *cbdata)
+{
+    TEST_VERBOSE(("Getting data for %s:%d",
+                  procs[0].nspace, procs[0].rank));
+
+    if ((pmix_list_get_size(server_list) == 1) && (my_server_id == 0)) {
+        if (NULL != cbfunc) {
+            cbfunc(PMIX_SUCCESS, data, ndata, cbdata, NULL, NULL);
+        }
+        return PMIX_SUCCESS;
+    }
+    return server_fence_contrib(data, ndata, cbfunc, cbdata);
+}
+
+pmix_status_t dmodex_fn(const pmix_proc_t *proc,
+              const pmix_info_t info[], size_t ninfo,
+              pmix_modex_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t n;
+
+    TEST_VERBOSE(("Getting data for %s:%d", proc->nspace, proc->rank));
+
+    if (NULL != info) {
+        for (n=0; n < ninfo; n++) {
+            if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) {
+                return PMIX_ERR_NOT_SUPPORTED;
+            }
+        }
+    }
+
+    /* return not_found for single server mode */
+    if ((pmix_list_get_size(server_list) == 1) && (my_server_id == 0)) {
+        return PMIX_ERR_NOT_FOUND;
+    }
+    // TODO: add support tracker for dmodex requests
+    return server_dmdx_get(proc->nspace, proc->rank, cbfunc, cbdata);
+}
+
+pmix_status_t publish_fn(const pmix_proc_t *proc,
+               const pmix_info_t info[], size_t ninfo,
+               pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t i;
+    int found;
+    pmix_test_info_t *new_info, *old_info;
+    if (NULL == pmix_test_published_list) {
+        pmix_test_published_list = PMIX_NEW(pmix_list_t);
+    }
+    for (i = 0; i < ninfo; i++) {
+        found = 0;
+        PMIX_LIST_FOREACH(old_info, pmix_test_published_list, pmix_test_info_t) {
+            if (!strcmp(old_info->data.key, info[i].key)) {
+                found = 1;
+                break;
+            }
+        }
+        if (!found) {
+            new_info = PMIX_NEW(pmix_test_info_t);
+            PMIX_LOAD_KEY(new_info->data.key, info[i].key);
+            pmix_value_xfer(&new_info->data.value, (pmix_value_t*)&info[i].value);
+            new_info->namespace_published = strdup(proc->nspace);
+            new_info->rank_published = proc->rank;
+            pmix_list_append(pmix_test_published_list, &new_info->super);
+        }
+    }
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys,
+              const pmix_info_t info[], size_t ninfo,
+              pmix_lookup_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t i, ndata, ret;
+    pmix_status_t rc = PMIX_SUCCESS;
+    pmix_pdata_t *pdata;
+    pmix_test_info_t *tinfo;
+    if (NULL == pmix_test_published_list) {
+        return PMIX_ERR_NOT_FOUND;
+    }
+    ndata = pmix_argv_count(keys);
+    PMIX_PDATA_CREATE(pdata, ndata);
+    ret = 0;
+    for (i = 0; i < ndata; i++) {
+        PMIX_LIST_FOREACH(tinfo, pmix_test_published_list, pmix_test_info_t) {
+            if (0 == strcmp(tinfo->data.key, keys[i])) {
+                (void)strncpy(pdata[i].proc.nspace, tinfo->namespace_published, PMIX_MAX_NSLEN);
+                pdata[i].proc.rank = tinfo->rank_published;
+                memset(pdata[i].key, 0, PMIX_MAX_KEYLEN+1);
+                (void)strncpy(pdata[i].key, keys[i], PMIX_MAX_KEYLEN);
+                pmix_value_xfer(&pdata[i].value, &tinfo->data.value);
+                ret++;
+                break;
+            }
+        }
+    }
+    if (ret != ndata) {
+        rc = PMIX_ERR_NOT_FOUND;
+        goto error;
+    }
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, pdata, ndata, cbdata);
+    }
+error:
+    PMIX_PDATA_FREE(pdata, ndata);
+    return rc;
+}
+
+pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys,
+                 const pmix_info_t info[], size_t ninfo,
+                 pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    size_t i;
+    pmix_test_info_t *iptr, *next;
+    if (NULL == pmix_test_published_list) {
+        return PMIX_ERR_NOT_FOUND;
+    }
+    PMIX_LIST_FOREACH_SAFE(iptr, next, pmix_test_published_list, pmix_test_info_t) {
+        if (1) {  // if data posted by this process
+            if (NULL == keys) {
+                pmix_list_remove_item(pmix_test_published_list, &iptr->super);
+                PMIX_RELEASE(iptr);
+            } else {
+                ninfo = pmix_argv_count(keys);
+                for (i = 0; i < ninfo; i++) {
+                    if (!strcmp(iptr->data.key, keys[i])) {
+                        pmix_list_remove_item(pmix_test_published_list, &iptr->super);
+                        PMIX_RELEASE(iptr);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
+
+typedef struct {
+    pmix_status_t status;
+    pmix_spawn_cbfunc_t cbfunc;
+    void *cbdata;
+} release_cbdata;
+
+
+static void * _release_cb(void *arg)
+{
+    release_cbdata *cb = (release_cbdata*)arg;
+    if (NULL != cb->cbfunc) {
+        cb->cbfunc(cb->status, "foobar", cb->cbdata);
+    }
+    free(cb);
+    spawn_wait = false;
+    pthread_exit(NULL);
+}
+
+static void release_cb(pmix_status_t status, void *cbdata)
+{
+    pthread_t thread;
+
+    if (0 > pthread_create(&thread, NULL, _release_cb, cbdata)) {
+        spawn_wait = false;
+        return;
+    }
+    pthread_detach(thread);
+}
+
+pmix_status_t spawn_fn(const pmix_proc_t *proc,
+             const pmix_info_t job_info[], size_t ninfo,
+             const pmix_app_t apps[], size_t napps,
+             pmix_spawn_cbfunc_t cbfunc, void *cbdata)
+{
+    release_cbdata *cb = malloc(sizeof(release_cbdata));
+
+    cb->status = PMIX_SUCCESS;
+    cb->cbfunc = cbfunc;
+    cb->cbdata = cbdata;
+
+    spawn_wait = true;
+    PMIx_server_register_nspace("foobar", napps, NULL, 0, release_cb, (void*)cb);
+    return PMIX_SUCCESS;
+}
+static int numconnect = 0;
+
+pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs,
+                         const pmix_info_t info[], size_t ninfo,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    numconnect++;
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs,
+                            const pmix_info_t info[], size_t ninfo,
+                            pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t regevents_fn (pmix_status_t *codes, size_t ncodes,
+                           const pmix_info_t info[], size_t ninfo,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    TEST_VERBOSE ((" pmix host server regevents_fn called "));
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
+
+pmix_status_t deregevents_fn (pmix_status_t *codes, size_t ncodes,
+                             pmix_op_cbfunc_t cbfunc, void *cbdata)
+{
+    TEST_VERBOSE ((" pmix host server deregevents_fn called "));
+    if (NULL != cbfunc) {
+        cbfunc(PMIX_SUCCESS, cbdata);
+    }
+    return PMIX_SUCCESS;
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/server_callbacks.h pmix-4.0.0/test/test_v2/server_callbacks.h
--- pmix-3.2.2~rc1/test/test_v2/server_callbacks.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/server_callbacks.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#ifndef PMIX_SERVER_CALLBACK_H
+#define PMIX_SERVER_CALLBACK_H
+
+#include "cli_stages.h"
+
+pmix_status_t connected(const pmix_proc_t *proc, void *server_object,
+                        pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t finalized(const pmix_proc_t *proc, void *server_object,
+                        pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t abort_fn(const pmix_proc_t *proc,
+                       void *server_object,
+                       int status, const char msg[],
+                       pmix_proc_t procs[], size_t nprocs,
+                       pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs,
+                         const pmix_info_t info[], size_t ninfo,
+                         char *data, size_t ndata,
+                         pmix_modex_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t dmodex_fn(const pmix_proc_t *proc,
+                        const pmix_info_t info[], size_t ninfo,
+                        pmix_modex_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t publish_fn(const pmix_proc_t *proc,
+                         const pmix_info_t info[], size_t ninfo,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys,
+                        const pmix_info_t info[], size_t ninfo,
+                        pmix_lookup_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys,
+                           const pmix_info_t info[], size_t ninfo,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t spawn_fn(const pmix_proc_t *proc,
+                       const pmix_info_t job_info[], size_t ninfo,
+                       const pmix_app_t apps[], size_t napps,
+                       pmix_spawn_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs,
+                         const pmix_info_t info[], size_t ninfo,
+                         pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs,
+                            const pmix_info_t info[], size_t ninfo,
+                            pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t regevents_fn(pmix_status_t *codes, size_t ncodes,
+                           const pmix_info_t info[], size_t ninfo,
+                           pmix_op_cbfunc_t cbfunc, void *cbdata);
+pmix_status_t deregevents_fn(pmix_status_t *codes, size_t ncodes,
+                             pmix_op_cbfunc_t cbfunc, void *cbdata);
+extern pmix_server_module_t mymodule;
+
+#endif
diff -Nru pmix-3.2.2~rc1/test/test_v2/test_common.c pmix-4.0.0/test/test_v2/test_common.c
--- pmix-3.2.2~rc1/test/test_v2/test_common.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/test_common.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,853 @@
+/*
+ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include "test_common.h"
+#include <stdarg.h>
+#include <stdio.h>
+#include <ctype.h>
+
+int pmix_test_verbose = 0;
+test_params params;
+
+FILE *file;
+
+#define OUTPUT_MAX 1024
+char *pmix_test_output_prepare(const char *fmt, ... )
+{
+    static char output[OUTPUT_MAX];
+    va_list args;
+    va_start( args, fmt );
+    memset(output, 0, sizeof(output));
+    vsnprintf(output, OUTPUT_MAX - 1, fmt, args);
+    va_end(args);
+    return output;
+}
+
+// presently a placeholder - to be developed into a client-only command parser,
+// separating the processing logic for client command line options from that for
+// other callers (test app or server), unlike how parse_cmd() presently works.
+void parse_cmd_client(int argc, char **argv, test_params *params, validation_params *v_params)
+{
+    ;
+}
+
+void parse_cmd(int argc, char **argv, test_params *params, validation_params *v_params)
+{
+    int i;
+    uint32_t job_size;
+
+    /* set output to stderr by default */
+    file = stdout;
+    if( params->nspace != NULL ) {
+        params->nspace = NULL;
+    }
+
+    /* parse user options */
+    for (i=1; i < argc; i++) {
+        if (0 == strcmp(argv[i], "--n") || 0 == strcmp(argv[i], "-n")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->np = strdup(argv[i]);
+                job_size = strtol(argv[i], NULL, 10);
+                params->nprocs = job_size;
+                PMIXT_VAL_PARAM_SETNUM(v_params, pmix_job_size, job_size);
+                if (-1 == params->ns_size) {
+                    params->ns_size = job_size;
+                }
+            }
+        } else if (0 == strcmp(argv[i], "--h") || 0 == strcmp(argv[i], "-h")) {
+            /* print help */
+            fprintf(stderr, "usage: pmix_test [-h] [-e foo] [-b] [-nb]\n");
+            fprintf(stderr, "\t-n       the job size (for checking purposes)\n");
+            fprintf(stderr, "\t-s       number of servers to emulate\n");
+            fprintf(stderr, "\t-e foo   use foo as test client\n");
+            fprintf(stderr, "\t-v       verbose output\n");
+            fprintf(stderr, "\t-t <>    set timeout\n");
+            fprintf(stderr, "\t-o out   redirect clients logs to file out.<rank>\n");
+            fprintf(stderr, "\t-c       fence[_nb] callback shall include all collected data\n");
+            fprintf(stderr, "\t-nb      use non-blocking fence\n");
+            exit(0);
+        } else if (0 == strcmp(argv[i], "--exec") || 0 == strcmp(argv[i], "-e")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->binary = strdup(argv[i]);
+            }
+        } else if (0 == strcmp(argv[i], "--nservers") || 0 == strcmp(argv[i], "-s")){
+            i++;
+            if (NULL != argv[i]) {
+                params->nservers = atoi(argv[i]);
+            }
+            if (2 < params->nservers) {
+                fprintf(stderr, "Only support up to 2 servers\n");
+                exit(1);
+            }
+        } else if( 0 == strcmp(argv[i], "--verbose") || 0 == strcmp(argv[i],"-v") ){
+            PMIXT_VERBOSE_ON();
+            params->verbose = 1;
+        } else if (0 == strcmp(argv[i], "--timeout") || 0 == strcmp(argv[i], "-t")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->timeout = atoi(argv[i]);
+                if( params->timeout == 0 ){
+                    params->timeout = TEST_DEFAULT_TIMEOUT;
+                }
+            }
+        } else if( 0 == strcmp(argv[i], "-o")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->prefix = strdup(argv[i]);
+            }
+        /*
+        } else if( 0 == strcmp(argv[i], "-s")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->nspace = strdup(argv[i]);
+
+        */
+        } else if (0 == strcmp(argv[i], "--rank") || 0 == strcmp(argv[i], "-r")) {
+            i++;
+            if (NULL != argv[i]) {
+                pmix_rank_t rank = strtol(argv[i], NULL, 10);
+                PMIXT_VAL_PARAM_SETNUM(v_params, pmix_rank, rank);
+            }
+        } else if (0 == strcmp(argv[i], "--collect-corrupt")) {
+            params->collect_bad = 1;
+        } else if (0 == strcmp(argv[i], "--collect") || 0 == strcmp(argv[i], "-c")) {
+            params->collect = 1;
+        } else if (0 == strcmp(argv[i], "--non-blocking") || 0 == strcmp(argv[i], "-nb")) {
+            params->nonblocking = 1;
+        } else if (0 == strcmp(argv[i], "--ns-size")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->ns_size = strtol(argv[i], NULL, 10);
+            }
+        } else if (0 == strcmp(argv[i], "--ns-id")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->ns_id = strtol(argv[i], NULL, 10);
+            }
+        } else if (0 == strcmp(argv[i], "--base-rank")) {
+            i++;
+            if (NULL != argv[i]) {
+                params->base_rank = strtol(argv[i], NULL, 10);
+            }
+
+        } else if (0 == strcmp(argv[i], "--validate-params")) {
+            i++;
+            v_params->validate_params = 1;
+            v_params_ascii_str = strdup(argv[i]);
+	    }
+        else {
+            fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+            exit(1);
+        }
+    }
+    if (NULL == params->binary) {
+        char *basename = NULL;
+        basename = strrchr(argv[0], '/');
+        if (basename) {
+            *basename = '\0';
+            /* pmix_test and pmix_clients are the shell scripts that
+             * make sure that actual binary placed in "./.libs" directory
+             * is properly linked.
+             * after launch pmix_test you'll find the following process:
+             *      <pmix-root-dir>/test/.libs/lt-pmix_test
+             *
+             * To launch
+             *      <pmix-root-dir>/test/pmix_client
+             * instead of
+             *      <pmix-root-dir>/test/.libs/pmix_client
+             * we need to do a step back in directory tree.
+             */
+            if (0 > asprintf(&params->binary, "%s/../pmix_client", argv[0])) {
+                exit(1);
+            }
+            *basename = '/';
+        } else {
+            if (0 > asprintf(&params->binary, "pmix_client")) {
+                exit(1);
+            }
+        }
+    }
+
+    if( params->collect_bad ){
+        params->collect = params->rank % 2;
+    }
+}
+
+void default_params(test_params *params, validation_params *v_params) {
+    params->binary = NULL;
+    params->np = NULL;
+    params->prefix = NULL;
+    params->nspace = NULL;
+    params->nprocs = 1;
+    params->timeout = TEST_DEFAULT_TIMEOUT;
+    params->verbose = 0;
+    params->rank = PMIX_RANK_UNDEF;
+    params->collect_bad = 0;
+    params->collect = 0;
+    params->nonblocking = 0;
+    params->ns_size = -1;
+    params->ns_id = -1;
+    params->base_rank = 0;
+    params->nservers = 1;
+
+    v_params->version = PMIXT_VALIDATION_PARAMS_VER;
+    v_params->validate_params = false;
+    v_params->check_pmix_rank = false;
+    v_params->pmix_rank = PMIX_RANK_UNDEF;
+    v_params->check_pmix_nspace = false;
+    v_params->pmix_nspace[0] = '\0';
+    v_params->check_pmix_job_size = false;
+    v_params->pmix_job_size = 0;
+    v_params->check_pmix_univ_size = false;
+    v_params->pmix_univ_size = 0;
+    v_params->check_pmix_jobid = false;
+    v_params->pmix_jobid[0] = '\0';
+    v_params->check_pmix_local_size = false;
+    v_params->pmix_local_size = 0;
+    v_params->check_pmix_local_rank = false;
+    v_params->pmix_local_rank = 0;
+    v_params->check_pmix_nodeid = false;
+    v_params->pmix_nodeid = 0;
+    v_params->check_pmix_local_peers = false;
+    v_params->pmix_local_peers[0] = '\0';
+    v_params->check_pmix_hostname = false;
+    v_params->pmix_hostname[0] = '\0';
+}
+
+void free_params(test_params *params, validation_params *vparams) {
+    if (NULL != params->binary) {
+        free(params->binary);
+    }
+    if (NULL != params->np) {
+        free(params->np);
+    }
+    if (NULL != params->prefix) {
+        free(params->prefix);
+    }
+    if (NULL != params->nspace) {
+        free(params->nspace);
+    }
+    if (NULL != v_params_ascii_str) {
+        free(v_params_ascii_str);
+    }
+}
+
+void set_client_argv(test_params *params, char ***argv)
+{
+    pmix_argv_append_nosize(argv, params->binary);
+    pmix_argv_append_nosize(argv, "-n");
+    if (NULL == params->np) {
+        pmix_argv_append_nosize(argv, "1");
+    } else {
+        pmix_argv_append_nosize(argv, params->np);
+    }
+    if( params->verbose ){
+        pmix_argv_append_nosize(argv, "-v");
+    }
+    if (NULL != params->prefix) {
+        pmix_argv_append_nosize(argv, "-o");
+        pmix_argv_append_nosize(argv, params->prefix);
+    }
+    if (params->nonblocking) {
+        pmix_argv_append_nosize(argv, "-nb");
+    }
+    if (params->collect) {
+        pmix_argv_append_nosize(argv, "-c");
+    }
+    if (params->collect_bad) {
+        pmix_argv_append_nosize(argv, "--collect-corrupt");
+    }
+}
+
+void pmixt_pre_init(int argc, char **argv, test_params *params, validation_params *v_params) {
+
+    ssize_t v_size = -1;
+
+    default_params(params, v_params);
+    parse_cmd(argc, argv, params, v_params);
+    //parse_cmd_client(argc, argv, params, v_params);
+
+    if (v_params->validate_params) {
+        v_size = pmixt_decode(v_params_ascii_str, v_params, sizeof(*v_params));
+        if (v_size != sizeof(*v_params)) {
+            assert(v_size == sizeof(*v_params));
+            exit(1);
+        }
+    }
+    else {
+        // we're not doing any parameter validation - is this reasonable?
+        TEST_VERBOSE(("Parameter validation disabled\n"));
+    }
+
+    /* set filename if available in params */
+    if ( NULL != params->prefix && -1 != params->ns_id ) {
+	char *fname = malloc( strlen(params->prefix) + MAX_DIGIT_LEN + 2 );
+        sprintf(fname, "%s.%d.%d", params->prefix, params->ns_id, params->rank);
+        file = fopen(fname, "w");
+        free(fname);
+        if( NULL == file ){
+            fprintf(stderr, "Cannot open file %s for writing!", fname);
+            exit(1);
+        }
+    }
+    else {
+        file = stdout;
+    }
+}
+
+void pmixt_fix_rank_and_ns(pmix_proc_t *this_proc, test_params *params, validation_params *v_params) {
+    // Fix rank if running under RM
+    if( PMIX_RANK_UNDEF == v_params->pmix_rank ){
+        char *ranklist = getenv("SLURM_GTIDS");
+        char *rankno = getenv("SLURM_LOCALID");
+        // Fix rank if running under SLURM
+        if( NULL != ranklist && NULL != rankno ){
+            char **argv = pmix_argv_split(ranklist, ',');
+            int count = pmix_argv_count(argv);
+            int rankidx = strtoul(rankno, NULL, 10);
+            if( rankidx >= count ){
+                fprintf(stderr, "It feels like we are running under SLURM:\n\t"
+                        "SLURM_GTIDS=%s, SLURM_LOCALID=%s\nbut env vars are conflicting\n",
+                        ranklist, rankno);
+                if( (stdout != file) && (stderr != file) ) {
+                    fclose(file);
+                }
+                exit(1);
+            }
+            v_params->pmix_rank = strtoul(argv[rankidx], NULL, 10);
+            pmix_argv_free(argv);
+        }
+        else if ( NULL == getenv("PMIX_RANK") ) { /* we must not be running under SLURM */
+            // **Call separate function for your own resource manager here**
+            // It should likely be wrapped in condition that checks for existence of an RM env var
+            // Function name should be of form: fix_rank_and_nspace_rm_*
+            // and should check/fix both rank and nspace
+            // e.g: fix_rank_and_ns_rm_pbs(), fix_rank_and_ns_rm_torque(), etc.
+        }
+        else { /* unknown situation - PMIX_RANK is not null but SLURM env vars are */
+            fprintf(stderr, "It feels like we are running under SLURM:\n\t"
+                    "PMIX_RANK=%s\nbut SLURM env vars are null\n",
+                    getenv("PMIX_RANK"));
+            if( (stdout != file) && (stderr != file) ) {
+                  fclose(file);
+            }
+            exit(1);
+        }
+    }
+
+    // Fix namespace if running under RM
+    if( NULL == params->nspace ){
+        char *nspace = getenv("PMIX_NAMESPACE");
+        if( NULL != nspace ){
+            params->nspace = strdup(nspace);
+        }
+	else { /* If we aren't running under SLURM, you should have set nspace
+	          in your custom fix_rank_and_ns_rm_* function! */
+            fprintf(stderr, "nspace not set. Is the fix_rank_and_ns_rm_*"
+	            " function for this resource manager failing to set it?\n");
+            if( (stdout != file) && (stderr != file) ) {
+                fclose(file);
+            }
+            exit(1);
+        }
+    }
+
+    if (this_proc->rank != v_params->pmix_rank) {
+        TEST_ERROR(("Client ns %s Rank returned in PMIx_Init %d does not match rank from command line %d.",
+	    this_proc->nspace, this_proc->rank, v_params->pmix_rank));
+        if( (stdout != file) && (stderr != file) ) {
+            fclose(file);
+        }
+        exit(1);
+    }
+}
+
+void pmixt_post_init(pmix_proc_t *this_proc, test_params *params, validation_params *val_params) {
+    pmixt_fix_rank_and_ns(this_proc, params, val_params);
+    TEST_VERBOSE((" Client/PMIX ns %s rank %d: PMIx_Init success", this_proc->nspace, this_proc->rank));
+}
+
+void pmixt_post_finalize(pmix_proc_t *this_proc, test_params *params, validation_params *v_params) {
+    TEST_VERBOSE((" Client ns %s rank %d: PMIx_Finalize success", this_proc->nspace, this_proc->rank));
+    if( (stdout != file) && (stderr != file) ) {
+        fclose(file);
+    }
+    free_params(params, v_params);
+    exit(EXIT_SUCCESS);
+}
+
+/* This function is used to validate values returned from calls to Get against sidechannel validation data */
+void pmixt_validate_predefined(pmix_proc_t *myproc, const pmix_key_t key, pmix_value_t *value,
+                               const pmix_data_type_t expected_type, validation_params *val_params)
+{
+    pmix_status_t rc = PMIX_ERROR;
+    if (val_params->validate_params) {
+        if (expected_type != value->type) {
+            TEST_ERROR(("Type mismatch for key. Key: %s Type: %u Expected type: %u",
+                    key, value->type, expected_type));
+            exit(1);
+        }
+        switch (value->type) {
+            case PMIX_UINT16:
+                //empty statement to separate label and declaration
+                ;
+                uint16_t uint16data;
+                PMIX_VALUE_GET_NUMBER(rc, value, uint16data, uint16_t);
+                if (PMIX_SUCCESS != rc) {
+                    TEST_ERROR(("Failed to retrieve value correctly. Key: %s Type: %u",
+                    key, value->type));
+                    exit(1);
+                }
+                if (PMIXT_CHECK_KEY(key, PMIX_LOCAL_RANK) && val_params->check_pmix_local_rank) {
+                    if (val_params->pmix_local_rank != uint16data) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %d Validation value: %d",
+                                     key, uint16data, val_params->pmix_local_rank));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d: Data validated: Key: %s Value: %u",
+                        myproc->nspace, myproc->rank, key, uint16data));
+                } // other possibilities will be added here later
+                else {
+                   TEST_ERROR(("Check input for case PMIX_UINT16: key: %s check_pmix_local_rank %u",
+                       key, val_params->check_pmix_local_rank));
+                    exit(1);
+                }
+                break;
+            case PMIX_UINT32:
+                //empty statement to separate label and declaration
+                ;
+                uint32_t uint32data;
+                PMIX_VALUE_GET_NUMBER(rc, value, uint32data, uint32_t);
+                if (PMIX_SUCCESS != rc) {
+                    TEST_ERROR(("Failed to retrieve value correctly. Key: %s Type: %u",
+                    key, value->type));
+                    exit(1);
+                }
+                if (PMIXT_CHECK_KEY(key, PMIX_JOB_SIZE) && val_params->check_pmix_job_size) {
+                    if (val_params->pmix_job_size != uint32data) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %d Validation value: %d",
+                                     key, uint32data, val_params->pmix_job_size));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d: Data validated: Key: %s Value: %u",
+                        myproc->nspace, myproc->rank, key, uint32data));
+                }
+                else if (PMIXT_CHECK_KEY(key, PMIX_UNIV_SIZE) && val_params->check_pmix_univ_size) {
+                    if (val_params->pmix_univ_size != uint32data) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %d Validation value: %d",
+                                     key, uint32data, val_params->pmix_univ_size));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d: Data validated: Key: %s Value: %u",
+                        myproc->nspace, myproc->rank, key, uint32data));
+                }
+                else if (PMIXT_CHECK_KEY(key, PMIX_LOCAL_SIZE) && val_params->check_pmix_local_size) {
+                    if (val_params->pmix_local_size != uint32data) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %d Validation value: %d",
+                                     key, uint32data, val_params->pmix_local_size));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d: Data validated: Key: %s Value: %u",
+                        myproc->nspace, myproc->rank, key, uint32data));
+                }
+                else if (PMIXT_CHECK_KEY(key, PMIX_NODEID) && val_params->check_pmix_nodeid) {
+                    if (val_params->pmix_nodeid != uint32data) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %d Validation value: %d",
+                                     key, uint32data, val_params->pmix_nodeid));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d: Data validated: Key: %s Value: %u",
+                        myproc->nspace, myproc->rank, key, uint32data));
+                } // other possibilities will be added here later
+                else {
+                   TEST_ERROR(("Check input for case PMIX_UINT32: key: %s check_pmix_job_size: %u check_pmix_univ_size %u check_pmix_local_size %u",
+                        key, val_params->check_pmix_job_size, val_params->check_pmix_univ_size, val_params->check_pmix_local_size));
+                    exit(1);
+                }
+                break;
+            case PMIX_PROC_RANK:
+                //empty statement to separate label and declaration
+                ;
+                pmix_rank_t rankdata;
+                PMIX_VALUE_GET_NUMBER(rc, value, rankdata, pmix_rank_t);
+                if (PMIX_SUCCESS != rc) {
+                    TEST_ERROR(("Failed to retrieve value correctly. Key: %s Type: %u",
+                    key, value->type));
+                    exit(1);
+                }
+                if (PMIXT_CHECK_KEY(key, PMIX_RANK) && val_params->check_pmix_rank) {
+                    if (val_params->pmix_rank != rankdata) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %d Validation value: %d",
+                                     key, rankdata, val_params->pmix_rank));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d Data validated: Key: %s Value: %u",
+                        myproc->nspace, myproc->rank, key, rankdata));
+                } // other possibilities will be added here later
+                else {
+                    TEST_ERROR(("Check input for case PMIX_PROC_RANK: key: %s check_pmix_rank: %u",
+                        key, val_params->check_pmix_rank));
+                    exit(1);
+                }
+                break;
+            case PMIX_STRING:
+                //empty statement to separate label and declaration
+                ;
+                void *stringdata;
+                size_t lsize;
+                PMIX_VALUE_UNLOAD(rc, value, &stringdata, &lsize);
+                if (PMIX_SUCCESS != rc) {
+                    TEST_ERROR(("Failed to retrieve value correctly. Key: %s Type: %u",
+                    key, value->type));
+                    exit(1);
+                }
+                if (PMIXT_CHECK_KEY(key, PMIX_NSPACE) && val_params->check_pmix_nspace) {
+                    if (0 != strcmp(val_params->pmix_nspace, stringdata)) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %s Validation value: %s",
+                                     key, stringdata, val_params->pmix_nspace));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d Data validated: Key: %s Value: %s",
+                        myproc->nspace, myproc->rank, key, stringdata));
+                }
+                else if (PMIXT_CHECK_KEY(key, PMIX_JOBID) && val_params->check_pmix_jobid) {
+                    if (0 != strcmp(val_params->pmix_jobid, stringdata)) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %s Validation value: %s",
+                                     key, stringdata, val_params->pmix_jobid));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d Data validated: Key: %s Value: %s",
+                        myproc->nspace, myproc->rank, key, stringdata));
+                }
+                else if (PMIXT_CHECK_KEY(key, PMIX_LOCAL_PEERS) && val_params->check_pmix_local_peers) {
+                    if (0 != strcmp(val_params->pmix_local_peers, stringdata)) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %s Validation value: %s",
+                                     key, stringdata, val_params->pmix_local_peers));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d Data validated: Key: %s Value: %s",
+                        myproc->nspace, myproc->rank, key, stringdata));
+                }
+                else if (PMIXT_CHECK_KEY(key, PMIX_HOSTNAME) && val_params->check_pmix_hostname) {
+                    if (0 != strcmp(val_params->pmix_hostname, stringdata)) {
+                        TEST_ERROR(("Key %s failed validation. Get value: %s Validation value: %s",
+                                     key, stringdata, val_params->pmix_hostname));
+                        exit(1);
+                    }
+                    TEST_VERBOSE(("Namespace %s: Rank %d Data validated: Key: %s Value: %s",
+                        myproc->nspace, myproc->rank, key, stringdata));
+                } // other possibilities will be added here later
+                else {
+                    TEST_ERROR(("Check input for case PMIX_STRING: key: %s stringdata: %s",
+                        key, stringdata));
+                    exit(1);
+                }
+                free(stringdata);
+                break;
+            default:
+                TEST_ERROR(("No test logic for type: %d, key: %s", value->type, key));
+                exit(1);
+        }
+    }
+    else {
+        TEST_VERBOSE(("All validation disabled, will not validate: %s", key));
+    }
+}
+
+/* lifted from contrib/perf_tools/pmix.c, pmi_get_local_ranks() */
+// Not sure how to go about validating what we get here, which comes from PMIx_Get
+void pmixt_get_local_ranks(pmix_proc_t *this_proc, int **local_ranks, int *local_cnt) {
+    pmix_value_t value, *val = &value;
+    char *ptr;
+    int i, rc;
+    //pmix_proc_t job_proc = &this_proc;
+#if (PMIX_VERSION_MAJOR > 1 )
+    this_proc->rank = PMIX_RANK_WILDCARD;
+#endif
+
+    /* get our job size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(this_proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_SIZE failed: %d", this_proc->nspace, this_proc->rank, rc);
+        abort();
+    }
+    *local_cnt = val->data.uint32;
+    PMIX_VALUE_RELEASE(val);
+
+    *local_ranks = calloc(*local_cnt, sizeof(int));
+    /* get our job size */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(this_proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) {
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_PEERS failed: %d", this_proc->nspace, this_proc->rank, rc);
+        abort();
+    }
+    ptr = val->data.string;
+    for(i=0; NULL != ptr && i < *local_cnt; i++ ){
+         char *loc_rank = strsep(&ptr, ",");
+         (*local_ranks)[i] = atoi(loc_rank);
+    }
+    if( i != *local_cnt || NULL != ptr ){
+        fprintf(stderr, "Client ns %s rank %d: number of local peers doesn't match",
+                this_proc->nspace, this_proc->rank);
+        abort();
+    }
+}
+
+static void fcon(fence_desc_t *p)
+{
+    p->blocking = 0;
+    p->data_exchange = 0;
+    p->participants = PMIX_NEW(pmix_list_t);
+}
+
+static void fdes(fence_desc_t *p)
+{
+    PMIX_LIST_RELEASE(p->participants);
+}
+
+PMIX_CLASS_INSTANCE(fence_desc_t,
+                    pmix_list_item_t,
+                    fcon, fdes);
+
+PMIX_CLASS_INSTANCE(participant_t,
+                    pmix_list_item_t,
+                    NULL, NULL);
+
+PMIX_CLASS_INSTANCE(key_replace_t,
+                    pmix_list_item_t,
+                    NULL, NULL);
+
+static int ns_id = -1;
+static fence_desc_t *fdesc = NULL;
+pmix_list_t *participants = NULL;
+pmix_list_t test_fences;
+pmix_list_t *noise_range = NULL;
+pmix_list_t key_replace;
+
+#define CHECK_STRTOL_VAL(val, str, store) do {                  \
+    if (0 == val) {                                             \
+        if (0 != strncmp(str, "0", 1)) {                        \
+            if (!store) {                                       \
+                return 1;                                       \
+            }                                                   \
+        }                                                       \
+    }                                                           \
+} while (0)
+
+static int parse_token(char *str, int step, int store)
+{
+    char *pch;
+    int count = 0;
+    int remember = -1;
+    int i;
+    int rank;
+    participant_t *proc;
+
+    switch (step) {
+    case 0:
+        if (store) {
+            fdesc = PMIX_NEW(fence_desc_t);
+            participants = fdesc->participants;
+        }
+        pch = strchr(str, '|');
+        if (NULL != pch) {
+            while (pch != str) {
+                if ('d' == *str) {
+                    if (store && NULL != fdesc) {
+                        fdesc->data_exchange = 1;
+                    }
+                } else if ('b' == *str) {
+                    if (store && NULL != fdesc) {
+                        fdesc->blocking = 1;
+                    }
+                } else if (' ' != *str) {
+                    if (!store) {
+                        return 1;
+                    }
+                }
+                str++;
+            }
+            if (0 < parse_token(pch+1, 1, store)) {
+                if (!store) {
+                    return 1;
+                }
+            }
+        } else {
+            if (0 < parse_token(str, 1, store)) {
+                if (!store) {
+                    return 1;
+                }
+            }
+        }
+        if (store && NULL != fdesc) {
+            pmix_list_append(&test_fences, &fdesc->super);
+        }
+        break;
+    case 1:
+        if (store && NULL == participants) {
+            participants = PMIX_NEW(pmix_list_t);
+            noise_range = participants;
+        }
+        pch = strtok(str, ";");
+        while (NULL != pch) {
+            if (0 < parse_token(pch, 2, store)) {
+                if (!store) {
+                    return 1;
+                }
+            }
+            pch = strtok (NULL, ";");
+        }
+        break;
+    case 2:
+        pch = strchr(str, ':');
+        if (NULL != pch) {
+            *pch = '\0';
+            pch++;
+            while (' ' == *str) {
+                str++;
+            }
+            ns_id = (int)(strtol(str, NULL, 10));
+            CHECK_STRTOL_VAL(ns_id, str, store);
+            if (0 < parse_token(pch, 3, store)) {
+                if (!store) {
+                    return 1;
+                }
+            }
+        } else {
+            if (!store) {
+                return 1;
+            }
+        }
+        break;
+    case 3:
+        while (' ' == *str) {
+            str++;
+        }
+        if ('\0' == *str) {
+            /* all ranks from namespace participate */
+            if (store && NULL != participants) {
+                proc = PMIX_NEW(participant_t);
+                (void)snprintf(proc->proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, ns_id);
+                proc->proc.rank = PMIX_RANK_WILDCARD;
+                pmix_list_append(participants, &proc->super);
+            }
+        }
+        while ('\0' != *str) {
+            if (',' == *str && 0 != count) {
+                *str = '\0';
+                if (-1 != remember) {
+                    rank = (int)(strtol(str-count, NULL, 10));
+                    CHECK_STRTOL_VAL(rank, str-count, store);
+                    for (i = remember; i < rank; i++) {
+                        if (store && NULL != participants) {
+                            proc = PMIX_NEW(participant_t);
+                            (void)snprintf(proc->proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, ns_id);
+                            proc->proc.rank = i;
+                            pmix_list_append(participants, &proc->super);
+                        }
+                    }
+                    remember = -1;
+                }
+                rank = (int)(strtol(str-count, NULL, 10));
+                CHECK_STRTOL_VAL(rank, str-count, store);
+                if (store && NULL != participants) {
+                    proc = PMIX_NEW(participant_t);
+                    (void)snprintf(proc->proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, ns_id);
+                    proc->proc.rank = rank;
+                    pmix_list_append(participants, &proc->super);
+                }
+                count = -1;
+            } else if ('-' == *str && 0 != count) {
+                *str = '\0';
+                remember = (int)(strtol(str-count, NULL, 10));
+                CHECK_STRTOL_VAL(remember, str-count, store);
+                count = -1;
+            }
+            str++;
+            count++;
+        }
+        if (0 != count) {
+            if (-1 != remember) {
+                rank = (int)(strtol(str-count, NULL, 10));
+                CHECK_STRTOL_VAL(rank, str-count, store);
+                for (i = remember; i < rank; i++) {
+                    if (store && NULL != participants) {
+                        proc = PMIX_NEW(participant_t);
+                        (void)snprintf(proc->proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, ns_id);
+                        proc->proc.rank = i;
+                        pmix_list_append(participants, &proc->super);
+                    }
+                }
+                remember = -1;
+            }
+            rank = (int)(strtol(str-count, NULL, 10));
+            CHECK_STRTOL_VAL(rank, str-count, store);
+            if (store && NULL != participants) {
+                proc = PMIX_NEW(participant_t);
+                (void)snprintf(proc->proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, ns_id);
+                proc->proc.rank = rank;
+                pmix_list_append(participants, &proc->super);
+            }
+        }
+        break;
+    default:
+        fprintf(stderr, "Incorrect parsing step.\n");
+        return 1;
+    }
+    return 0;
+}
+
+int parse_fence(char *fence_param, int store)
+{
+    int ret = 0;
+    char *tmp = strdup(fence_param);
+    char * pch, *ech;
+
+    pch = strchr(tmp, '[');
+    while (NULL != pch) {
+        pch++;
+        ech = strchr(pch, ']');
+        if (NULL != ech) {
+            *ech = '\0';
+            ech++;
+            ret += parse_token(pch, 0, store);
+            pch = strchr(ech, '[');
+        } else {
+            ret = 1;
+            break;
+        }
+    }
+    free(tmp);
+    return ret;
+}
+
+
+static int is_digit(const char *str)
+{
+    if (NULL == str)
+        return 0;
+
+    while (0 != *str) {
+        if (!isdigit(*str)) {
+        return 0;
+        }
+        else {
+            str++;
+        }
+    }
+    return 1;
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/test_common.h pmix-4.0.0/test/test_v2/test_common.h
--- pmix-3.2.2~rc1/test/test_v2/test_common.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/test_common.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,444 @@
+/*
+ * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Artem Y. Polyakov <artpol84@gmail.com>.
+ *                         All rights reserved.
+ * Copyright (c) 2015      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2020      Triad National Security, LLC.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#ifndef TEST_COMMON_H
+#define TEST_COMMON_H
+
+#include <src/include/pmix_config.h>
+#include <pmix_common.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <sys/time.h>
+
+#include "src/include/pmix_globals.h"
+#include "src/class/pmix_list.h"
+#include "src/util/argv.h"
+
+#define TEST_NAMESPACE "smoky_nspace"
+
+#define TEST_CREDENTIAL "dummy"
+
+#define PMIXT_VALIDATION_PARAMS_VER 1
+
+#define PMIXT_CHECK_EXPECT(rc, expected_rc, params, vparams) \
+do {                                                   \
+   int pmix_rc = (rc);                               \
+   if (expected_rc != pmix_rc) {                      \
+       TEST_ERROR(("Client ns %s rank %d: PMIx call failed: %s", \
+           vparams.pmix_nspace, vparams.pmix_rank,           \
+	   PMIx_Error_string(pmix_rc)));                   \
+       exit(-1);                                       \
+   }                                                   \
+} while (0)
+
+#define PMIXT_CHECK(stmt, params, vparams) PMIXT_CHECK_EXPECT(stmt, PMIX_SUCCESS, params, vparams)
+
+#define PMIX_WAIT_FOR_COMPLETION(m) \
+    do {                            \
+        while ((m)) {               \
+            usleep(10);             \
+        }                           \
+    } while(0)
+
+
+/* WARNING: pmix_test_output_prepare is currently not threadsafe!
+ * fix it once needed!
+ */
+char *pmix_test_output_prepare(const char *fmt,... );
+extern int pmix_test_verbose;
+extern FILE *file;
+
+#define STRIPPED_FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
+
+#define TEST_OUTPUT(x) {                            \
+    struct timeval tv;                              \
+    gettimeofday(&tv, NULL);                        \
+    double ts = tv.tv_sec + 1E-6*tv.tv_usec;        \
+    fprintf(file,"==%d== [%lf] %s:%s: %s\n",        \
+            getpid(), ts,STRIPPED_FILE_NAME,        \
+            __func__,                               \
+            pmix_test_output_prepare x );           \
+    fflush(file);                                   \
+}
+
+// Write output without adding anything to it.
+// Need for automate tests to receive "OK" string
+#define TEST_OUTPUT_CLEAR(x) { \
+    fprintf(file, "==%d== %s", getpid(), pmix_test_output_prepare x ); \
+    fflush(file); \
+}
+
+// Always write errors to stderr
+#define TEST_ERROR(x) { \
+    struct timeval tv;                              \
+    gettimeofday(&tv, NULL);                        \
+    double ts = tv.tv_sec + 1E-6*tv.tv_usec;        \
+    fprintf(stderr,                                 \
+            "==%d== [%lf] ERROR [%s:%d:%s]: %s\n",  \
+            getpid(), ts,                           \
+            STRIPPED_FILE_NAME, __LINE__, __func__, \
+            pmix_test_output_prepare x );           \
+    fflush(stderr);                                 \
+}
+
+#define PMIXT_VERBOSE_ON() (pmix_test_verbose = 1)
+#define TEST_VERBOSE_GET() (pmix_test_verbose)
+
+#define TEST_VERBOSE(x) { \
+    if( pmix_test_verbose ){ \
+        TEST_OUTPUT(x); \
+    } \
+}
+
+#define TEST_DEFAULT_TIMEOUT 10
+#define MAX_DIGIT_LEN 10
+#define TEST_REPLACE_DEFAULT "3:1"
+
+#define TEST_SET_FILE(prefix, ns_id, rank) { \
+    char *fname = malloc( strlen(prefix) + MAX_DIGIT_LEN + 2 ); \
+    sprintf(fname, "%s.%d.%d", prefix, ns_id, rank); \
+    file = fopen(fname, "w"); \
+    free(fname); \
+    if( NULL == file ){ \
+        fprintf(stderr, "Cannot open file %s for writing!", fname); \
+        exit(1); \
+    } \
+}
+
+#define TEST_CLOSE_FILE() { \
+    if ( stderr != file ) { \
+        fclose(file); \
+    } \
+}
+
+#define PMIXT_VAL_PARAM_SETNUM(base, field, val) {\
+    base->field = val;               \
+    base->check_ ## field = true;    \
+}
+
+#define PMIXT_VAL_PARAM_SETSTR(base, field, val, len) {\
+    strncpy(base->field, val, len);               \
+    base->check_ ## field = true;    \
+}
+
+/* a convenience macro for checking keys (test suite-specific) */
+#define PMIXT_CHECK_KEY(a, b) \
+    (0 == strncmp((a), (b), PMIX_MAX_KEYLEN))
+
+// order of these fields should be in order that we introduce them
+typedef struct {
+    uint32_t version;
+    bool validate_params;
+    bool check_pmix_rank;
+    pmix_rank_t pmix_rank;
+    bool check_pmix_nspace;
+    pmix_nspace_t pmix_nspace;
+    bool check_pmix_job_size;
+    uint32_t pmix_job_size;
+    bool check_pmix_univ_size;
+    uint32_t pmix_univ_size;
+    bool check_pmix_jobid;
+    char pmix_jobid[PMIX_MAX_KEYLEN];
+    bool check_pmix_local_size;
+    uint32_t pmix_local_size;
+    bool check_pmix_local_rank;
+    uint16_t pmix_local_rank;
+    bool check_pmix_nodeid;
+    uint32_t pmix_nodeid;
+    bool check_pmix_local_peers;
+    char pmix_local_peers[PMIX_MAX_KEYLEN];
+    bool check_pmix_hostname;
+    char pmix_hostname[PMIX_MAX_KEYLEN];
+    /*
+    bool check_pmix_job_num_apps;
+    uint32_t pmix_job_num_apps;
+    bool check_pmix_app_size;
+    uint32_t pmix_app_size;
+    bool check_pmix_node_size;
+    uint32_t pmix_node_size;
+    bool check_pmix_max_procs;
+    uint32_t pmix_max_procs;
+    bool check_pmix_num_slots;
+    uint32_t pmix_num_slots;
+    bool check_pmix_num_nodes;
+    uint32_t pmix_num_nodes;
+    bool check_pmix_node_rank;
+    uint16_t pmix_node_rank;
+    */
+    // more as needed
+} validation_params;
+
+validation_params val_params;
+char *v_params_ascii_str;
+
+typedef struct {
+    char *binary;
+    char *np;
+    char *prefix;
+    char *nspace;
+    uint32_t nprocs;
+    int timeout;
+    int verbose;
+    pmix_rank_t rank;
+    int collect_bad;
+    int collect;
+    int nonblocking;
+    int ns_size;
+    int ns_id;
+    pmix_rank_t base_rank;
+    int nservers;
+} test_params;
+
+extern test_params params;
+
+void parse_cmd(int argc, char **argv, test_params *params, validation_params *v_params);
+int parse_fence(char *fence_param, int store);
+int parse_noise(char *noise_param, int store);
+int parse_replace(char *replace_param, int store, int *key_num);
+
+void default_params(test_params *params, validation_params *v_params);
+void free_params(test_params *params, validation_params *vparams);
+void set_client_argv(test_params *params, char ***argv);
+
+void pmixt_fix_rank_and_ns(pmix_proc_t *this_proc, test_params *params, validation_params *v_params);
+void pmixt_post_init(pmix_proc_t *this_proc, test_params *params, validation_params *val_params);
+void pmixt_post_finalize(pmix_proc_t *this_proc, test_params *params, validation_params *v_params);
+void pmixt_pre_init(int argc, char **argv, test_params *params, validation_params *v_params);
+void pmixt_validate_predefined(pmix_proc_t *myproc, const pmix_key_t key, pmix_value_t *value, const pmix_data_type_t expected_type, validation_params *val_params);
+
+char *pmixt_encode(const void *val, size_t vallen);
+ssize_t pmixt_decode (const char *data, void *decdata, size_t buffsz);
+
+typedef struct {
+    pmix_list_item_t super;
+    int blocking;
+    int data_exchange;
+    pmix_list_t *participants;  // list of participants
+} fence_desc_t;
+PMIX_CLASS_DECLARATION(fence_desc_t);
+
+typedef struct {
+    pmix_list_item_t super;
+    pmix_proc_t proc;
+} participant_t;
+PMIX_CLASS_DECLARATION(participant_t);
+
+typedef struct {
+    pmix_list_item_t super;
+    int key_idx;
+} key_replace_t;
+PMIX_CLASS_DECLARATION(key_replace_t);
+
+extern pmix_list_t test_fences;
+extern pmix_list_t *noise_range;
+extern pmix_list_t key_replace;
+
+int get_total_ns_number(test_params params);
+int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t **ranks, size_t *nranks);
+
+typedef struct {
+    int in_progress;
+    pmix_value_t *kv;
+    int status;
+} get_cbdata;
+
+#define SET_KEY(key, fence_num, ind, use_same_keys) do {                                                            \
+    if (use_same_keys) {                                                                                            \
+        (void)snprintf(key, sizeof(key)-1, "key-%d", ind);                                                            \
+    } else {                                                                                                        \
+        (void)snprintf(key, sizeof(key)-1, "key-f%d:%d", fence_num, ind);                                             \
+    }                                                                                                               \
+} while (0)
+
+#define PUT(dtype, data, flag, fence_num, ind, use_same_keys) do {                                                  \
+    char key[50];                                                                                                   \
+    pmix_value_t value;                                                                                             \
+    SET_KEY(key, fence_num, ind, use_same_keys);                                                                    \
+    PMIX_VAL_SET(&value, dtype, data);                                                                              \
+    TEST_VERBOSE(("%s:%d put key %s", my_nspace, my_rank, key));                                                  \
+    if (PMIX_SUCCESS != (rc = PMIx_Put(flag, key, &value))) {                                                       \
+        TEST_ERROR(("%s:%d: PMIx_Put key %s failed: %s", my_nspace, my_rank, key, PMIx_Error_string(rc)));                             \
+    }                                                                                                               \
+    PMIX_VALUE_DESTRUCT(&value);                                                                                    \
+} while (0)
+
+#define GET(dtype, data, ns, r, fence_num, ind, use_same_keys, blocking, ok_notfnd) do {                        \
+    char key[50];                                                                                                   \
+    pmix_value_t *val;                                                                                              \
+    get_cbdata cbdata;                                                                                              \
+    cbdata.status = PMIX_SUCCESS;                                                                                   \
+    pmix_proc_t foobar; \
+    SET_KEY(key, fence_num, ind, use_same_keys);                                                                    \
+    PMIX_LOAD_PROCID(&foobar, ns, r); \
+    TEST_VERBOSE(("%s:%d want to get from %s:%d key %s", my_nspace, my_rank, ns, r, key));                          \
+    if (blocking) {                                                                                                 \
+        if (PMIX_SUCCESS != (rc = PMIx_Get(&foobar, key, NULL, 0, &val))) {                                         \
+            if( !( (rc == PMIX_ERR_NOT_FOUND || rc == PMIX_ERR_PROC_ENTRY_NOT_FOUND) && ok_notfnd ) ){                                                       \
+                TEST_ERROR(("%s:%d: PMIx_Get failed: %s from %s:%d, key %s", my_nspace, my_rank, PMIx_Error_string(rc), ns, r, key));  \
+            }                                                                                                       \
+        }                                                                                                           \
+    } else {                                                                                                        \
+        int count;                                                                                                  \
+        cbdata.in_progress = 1;                                                                                     \
+        PMIX_VALUE_CREATE(val, 1);                                                                                  \
+        cbdata.kv = val;                                                                                            \
+        if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&foobar, key, NULL, 0, get_cb, (void*)&cbdata))) {                    \
+            TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %s from %s:%d, key=%s", my_nspace, my_rank, PMIx_Error_string(rc), ns, r, key)); \
+        } else {                                                                                                    \
+            count = 0;                                                                                              \
+            while(cbdata.in_progress){                                                                              \
+                struct timespec ts;                                                                                 \
+                ts.tv_sec = 0;                                                                                      \
+                ts.tv_nsec = 100;                                                                                   \
+                nanosleep(&ts,NULL);                                                                                \
+                count++;                                                                                            \
+            }                                                                                                       \
+            rc = cbdata.status;                                                                                     \
+            PMIX_ACQUIRE_OBJECT(&cbdata);                                                                           \
+        }                                                                                                           \
+    }                                                                                                               \
+    if (PMIX_SUCCESS == rc) {                                                                                       \
+        if( PMIX_SUCCESS != cbdata.status ){                                                                        \
+            if( !( (cbdata.status == PMIX_ERR_NOT_FOUND || cbdata.status == PMIX_ERR_PROC_ENTRY_NOT_FOUND) && ok_notfnd ) ){ \
+                TEST_ERROR(("%s:%d: PMIx_Get_nb failed: %s from %s:%d, key=%s",                                     \
+                            my_nspace, my_rank, PMIx_Error_string(rc), my_nspace, r, key));                                            \
+            }                                                                                                       \
+        } else if (NULL == val) {                                                                                   \
+            TEST_VERBOSE(("%s:%d: PMIx_Get returned NULL value", my_nspace, my_rank));                              \
+        }                                                                                                           \
+        else if (val->type != PMIX_VAL_TYPE_ ## dtype || PMIX_VAL_CMP(dtype, PMIX_VAL_FIELD_ ## dtype((val)), data)) {  \
+            TEST_ERROR(("%s:%u: from %s:%d Key %s value or type mismatch,"                                        \
+                        " want type %d get type %d",                                                                \
+                        my_nspace, my_rank, ns, r, key, PMIX_VAL_TYPE_ ## dtype, val->type));                    \
+        }                                                                                                           \
+    }                                                                                                               \
+    if (PMIX_SUCCESS == rc) {                                                                                       \
+        TEST_VERBOSE(("%s:%d: GET OF %s from %s:%d SUCCEEDED", my_nspace, my_rank, key, ns, r));                 \
+        PMIX_VALUE_RELEASE(val);                                                                                    \
+    }                                                                                                               \
+} while (0)
+
+#define PMIXT_OUTPUT_VALUE(this_proc, ns, r, key, datatype, value) do {         \
+    // we use the datatype of the value to extract the value                    \
+} while (0)
+
+
+#define FENCE(blocking, data_ex, pcs, nprocs) do {                              \
+    if( blocking ){                                                             \
+        pmix_info_t *info = NULL;                                               \
+        size_t ninfo = 0;                                                       \
+        if (data_ex) {                                                          \
+            bool value = 1;                                            \
+            PMIX_INFO_CREATE(info, 1);                                          \
+            (void)strncpy(info->key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN);       \
+            pmix_value_load(&info->value, &value, PMIX_BOOL);                   \
+            ninfo = 1;                                                          \
+        }                                                                       \
+        rc = PMIx_Fence(pcs, nprocs, info, ninfo);                              \
+        PMIX_INFO_FREE(info, ninfo);                                            \
+    } else {                                                                    \
+        int in_progress = 1, count;                                             \
+        rc = PMIx_Fence_nb(pcs, nprocs, NULL, 0, release_cb, &in_progress);     \
+        if ( PMIX_SUCCESS == rc ) {                                             \
+            count = 0;                                                          \
+            while( in_progress ){                                               \
+                struct timespec ts;                                             \
+                ts.tv_sec = 0;                                                  \
+                ts.tv_nsec = 100;                                               \
+                nanosleep(&ts,NULL);                                            \
+                count++;                                                        \
+            }                                                                   \
+            TEST_VERBOSE(("PMIx_Fence_nb(barrier,collect): free time: %lfs",    \
+                            count*100*1E-9));                                   \
+        }                                                                       \
+    }                                                                           \
+    if (PMIX_SUCCESS == rc) {                                                   \
+        TEST_VERBOSE(("%s:%d: Fence successfully completed",                    \
+                        my_nspace, my_rank));                                   \
+    }                                                                           \
+} while (0)
+
+/* Key-Value pair management macros */
+// TODO: add all possible types/fields here.
+
+#define PMIX_VAL_FIELD_int(x)       ((x)->data.integer)
+#define PMIX_VAL_FIELD_uint32_t(x)  ((x)->data.uint32)
+#define PMIX_VAL_FIELD_uint16_t(x)  ((x)->data.uint16)
+#define PMIX_VAL_FIELD_string(x)    ((x)->data.string)
+#define PMIX_VAL_FIELD_float(x)     ((x)->data.fval)
+#define PMIX_VAL_FIELD_byte(x)      ((x)->data.byte)
+#define PMIX_VAL_FIELD_flag(x)      ((x)->data.flag)
+
+#define PMIX_VAL_TYPE_int      PMIX_INT
+#define PMIX_VAL_TYPE_uint32_t PMIX_UINT32
+#define PMIX_VAL_TYPE_uint16_t PMIX_UINT16
+#define PMIX_VAL_TYPE_string   PMIX_STRING
+#define PMIX_VAL_TYPE_float    PMIX_FLOAT
+#define PMIX_VAL_TYPE_byte     PMIX_BYTE
+#define PMIX_VAL_TYPE_flag     PMIX_BOOL
+
+#define PMIX_VAL_set_assign(_v, _field, _val )   \
+    do {                                                            \
+        (_v)->type = PMIX_VAL_TYPE_ ## _field;                      \
+        PMIX_VAL_FIELD_ ## _field((_v)) = _val;                     \
+    } while (0)
+
+#define PMIX_VAL_set_strdup(_v, _field, _val )       \
+    do {                                                                \
+        (_v)->type = PMIX_VAL_TYPE_ ## _field;                          \
+        PMIX_VAL_FIELD_ ## _field((_v)) = strdup(_val);                 \
+    } while (0)
+
+#define PMIX_VAL_SET_int        PMIX_VAL_set_assign
+#define PMIX_VAL_SET_uint32_t   PMIX_VAL_set_assign
+#define PMIX_VAL_SET_uint16_t   PMIX_VAL_set_assign
+#define PMIX_VAL_SET_string     PMIX_VAL_set_strdup
+#define PMIX_VAL_SET_float      PMIX_VAL_set_assign
+#define PMIX_VAL_SET_byte       PMIX_VAL_set_assign
+#define PMIX_VAL_SET_flag       PMIX_VAL_set_assign
+
+#define PMIX_VAL_SET(_v, _field, _val )   \
+    PMIX_VAL_SET_ ## _field(_v, _field, _val)
+
+#define PMIX_VAL_cmp_val(_val1, _val2)      ((_val1) != (_val2))
+#define PMIX_VAL_cmp_float(_val1, _val2)    (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001))
+#define PMIX_VAL_cmp_ptr(_val1, _val2)      strncmp(_val1, _val2, strlen(_val1)+1)
+
+#define PMIX_VAL_CMP_int        PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_uint32_t   PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_uint16_t   PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_float      PMIX_VAL_cmp_float
+#define PMIX_VAL_CMP_string     PMIX_VAL_cmp_ptr
+#define PMIX_VAL_CMP_byte       PMIX_VAL_cmp_val
+#define PMIX_VAL_CMP_flag       PMIX_VAL_cmp_val
+
+#define PMIX_VAL_ASSIGN(_v, _field, _val) \
+    PMIX_VAL_set_assign(_v, _field, _val)
+
+#define PMIX_VAL_CMP(_field, _val1, _val2) \
+    PMIX_VAL_CMP_ ## _field(_val1, _val2)
+
+#define PMIX_VAL_FREE(_v) \
+     PMIx_free_value_data(_v)
+
+#endif // TEST_COMMON_H
diff -Nru pmix-3.2.2~rc1/test/test_v2/test_get_basic.c pmix-4.0.0/test/test_v2/test_get_basic.c
--- pmix-3.2.2~rc1/test/test_v2/test_get_basic.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/test_get_basic.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2020      Triad National Security, LLC.
+ *                         All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* Test of rank position identification inside group */
+
+#include "pmix.h"
+#include "test_common.h"
+#include <stdio.h>
+#include <stdlib.h>
+/*
+#include <assert.h>
+#include <sys/time.h>
+#include <stdarg.h>
+#include <unistd.h>
+*/
+
+extern FILE *file;
+
+pmix_proc_t this_proc;
+
+int main(int argc, char *argv[]) {
+
+    pmix_value_t *val;
+    size_t ninfo = 1;
+    test_params params;
+    validation_params v_params;
+    pmix_proc_t job_proc;
+
+    pmixt_pre_init(argc, argv, &params, &v_params);
+    /* initialization */
+    PMIXT_CHECK(PMIx_Init(&this_proc, NULL, ninfo), params, v_params);
+
+    /* Handles everything that needs to happen after PMIx_Init() */
+    pmixt_post_init(&this_proc, &params, &v_params);
+
+    job_proc = this_proc;
+    job_proc.rank = PMIX_RANK_WILDCARD; // Note that PMIX_RANK_WILDCARD == -2
+    PMIXT_CHECK(PMIx_Get(&job_proc, PMIX_JOB_SIZE, NULL, 0, &val), params, v_params);
+    /* After using PMIx_Get to get a value, we need to compare it our validation parameters
+       we've passed as an argument; this is the main purpose of pmixt_validate_predefined(). */
+
+    pmixt_validate_predefined(&job_proc, PMIX_JOB_SIZE, val, PMIX_UINT32, &v_params);
+
+    PMIXT_CHECK(PMIx_Get(&job_proc, PMIX_UNIV_SIZE, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&job_proc, PMIX_UNIV_SIZE, val, PMIX_UINT32, &v_params);
+
+    PMIXT_CHECK(PMIx_Get(&job_proc, PMIX_LOCAL_SIZE, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&job_proc, PMIX_LOCAL_SIZE, val, PMIX_UINT32, &v_params);
+
+    PMIXT_CHECK(PMIx_Get(&this_proc, PMIX_LOCAL_RANK, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&this_proc, PMIX_LOCAL_RANK, val, PMIX_UINT16, &v_params);
+
+    PMIXT_CHECK(PMIx_Get(&this_proc, PMIX_NODEID, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&this_proc, PMIX_NODEID, val, PMIX_UINT32, &v_params);
+
+    PMIXT_CHECK(PMIx_Get(&job_proc, PMIX_LOCAL_PEERS, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&job_proc, PMIX_LOCAL_PEERS, val, PMIX_STRING, &v_params);
+
+    PMIXT_CHECK(PMIx_Get(&this_proc, PMIX_HOSTNAME, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&this_proc, PMIX_HOSTNAME, val, PMIX_STRING, &v_params);
+
+    // Code hangs when PMIx_Get of PMIX_RANK is enabled
+    /*
+    job_proc.rank = PMIX_RANK_UNDEF;
+    PMIXT_CHECK(PMIx_Get(&job_proc, PMIX_RANK, NULL, 0, &val), params, v_params);
+    pmixt_validate_predefined(&job_proc, PMIX_RANK, val, PMIX_PROC_RANK, &v_params);
+    */
+
+    /* finalize */
+    PMIXT_CHECK(PMIx_Finalize(NULL, 0), params, v_params);
+
+    /* Handles cleanup */
+    pmixt_post_finalize(&this_proc, &params, &v_params);
+}
\ No newline at end of file
diff -Nru pmix-3.2.2~rc1/test/test_v2/test_helloworld.c pmix-4.0.0/test/test_v2/test_helloworld.c
--- pmix-3.2.2~rc1/test/test_v2/test_helloworld.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/test_helloworld.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2020      Triad National Security, LLC.
+ *                         All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* Test of namespace and rank validity */
+
+#include "pmix.h"
+#include "test_common.h"
+
+pmix_proc_t this_proc;
+
+int main(int argc, char *argv[]) {
+
+    size_t ninfo = 1;
+    test_params params;
+    validation_params v_params;
+
+    /* Handles all setup that's required prior to calling PMIx_Init() */
+    pmixt_pre_init(argc, argv, &params, &v_params);
+    /* initialization */
+    PMIXT_CHECK(PMIx_Init(&this_proc, NULL, ninfo), params, v_params);
+
+    /* Handles everything that needs to happen after PMIx_Init() */
+    pmixt_post_init(&this_proc, &params, &v_params);
+
+    /* Check that our validation side-channel (passed from server to client as argument)
+     * has the same values as this_proc */
+    if (!(0 == strncmp(v_params.pmix_nspace, this_proc.nspace, PMIX_MAX_NSLEN))) {
+        TEST_ERROR(("Client ns %s validation ns: %s rank %d: nspace validation failed",
+           this_proc.nspace, v_params.pmix_nspace, v_params.pmix_rank));
+        exit(1);
+    }
+    if (!(v_params.pmix_rank == this_proc.rank)) {
+        TEST_ERROR(("Client ns %s rank %d validation rank: %d: rank validation failed",
+           this_proc.nspace, this_proc.rank, v_params.pmix_rank));
+        exit(1);
+    }
+    TEST_VERBOSE(("nspace validated: %s, rank validated: %d", v_params.pmix_nspace, 
+                   v_params.pmix_rank));  
+
+    PMIXT_CHECK(PMIx_Finalize(NULL, 0), params, v_params);
+
+    /* Handles cleanup */
+    pmixt_post_finalize(&this_proc, &params, &v_params);
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/test_server.c pmix-4.0.0/test/test_v2/test_server.c
--- pmix-3.2.2~rc1/test/test_v2/test_server.c	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/test_server.c	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,1191 @@
+ /*
+ * Copyright (c) 2015-2019 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * Copyright (c) 2016-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "pmix_server.h"
+#include "src/include/pmix_globals.h"
+#include "src/util/error.h"
+
+#include "test_server.h"
+#include "test_common.h"
+#include "cli_stages.h"
+#include "server_callbacks.h"
+
+int my_server_id = 0;
+
+server_info_t *my_server_info = NULL;
+pmix_list_t *server_list = NULL;
+pmix_list_t *server_nspace = NULL;
+
+/* server destructor */
+static void sdes(server_info_t *s)
+{
+    close(s->rd_fd);
+    close(s->wr_fd);
+    if (s->evread) {
+        pmix_event_del(s->evread);
+    }
+    s->evread = NULL;
+    if (NULL != s->hostname) {
+        free(s->hostname);
+    }
+}
+
+/* server constructor */
+static void scon(server_info_t *s)
+{
+    s->hostname = NULL;
+    s->idx = 0;
+    s->pid = 0;
+    s->rd_fd = -1;
+    s->wr_fd = -1;
+    s->evread = NULL;
+    s->modex_cbfunc = NULL;
+    s->cbdata = NULL;
+}
+
+PMIX_CLASS_INSTANCE(server_info_t,
+                    pmix_list_item_t,
+                    scon, sdes);
+
+/* namespace destructor */
+static void nsdes(server_nspace_t *ns)
+{
+    if (ns->task_map) {
+        free(ns->task_map);
+    }
+}
+
+/* namespace constructor */
+static void nscon(server_nspace_t *ns)
+{
+    memset(ns->name, 0, PMIX_MAX_NSLEN);
+    ns->ntasks = 0;
+    ns->task_map = NULL;
+}
+
+PMIX_CLASS_INSTANCE(server_nspace_t,
+                    pmix_list_item_t,
+                    nscon, nsdes);
+
+static int server_send_procs(void);
+static void server_read_cb(int fd, short event, void *arg);
+static int srv_wait_all(double timeout);
+static int server_fwd_msg(msg_hdr_t *msg_hdr, char *buf, size_t size);
+static int server_send_msg(msg_hdr_t *msg_hdr, char *data, size_t size);
+static void remove_server_item(server_info_t *server);
+static void server_unpack_dmdx(char *buf, int *sender, pmix_proc_t *proc);
+static int server_pack_dmdx(int sender_id, const char *nspace, int rank,
+                            char **buf);
+static void _dmdx_cb(int status, char *data, size_t sz, void *cbdata);
+static void fill_global_validation_params(pmix_proc_t proc, int univ_size, test_params *params, validation_params *v_params);
+
+static void release_cb(pmix_status_t status, void *cbdata)
+{
+    int *ptr = (int*)cbdata;
+    *ptr = 0;
+}
+
+static void fill_seq_ranks_array(size_t nprocs, int base_rank, char **ranks)
+{
+    uint32_t i;
+    int len = 0, max_ranks_len;
+    if (0 >= nprocs) {
+        return;
+    }
+    max_ranks_len = nprocs * (MAX_DIGIT_LEN+1);
+    *ranks = (char*) malloc(max_ranks_len);
+    for (i = 0; i < nprocs; i++) {
+        len += snprintf(*ranks + len, max_ranks_len-len-1, "%d", i+base_rank);
+        if (i != nprocs-1) {
+            len += snprintf(*ranks + len, max_ranks_len-len-1, "%c", ',');
+        }
+    }
+    if (len >= max_ranks_len-1) {
+        free(*ranks);
+        *ranks = NULL;
+        TEST_ERROR(("Not enough allocated space for global ranks array."));
+    }
+}
+
+static int server_find_id(const char *nspace, int rank)
+{
+    server_nspace_t *tmp;
+
+    PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) {
+        if (0 == strcmp(tmp->name, nspace)) {
+            return tmp->task_map[rank];
+        }
+    }
+    return -1;
+}
+
+static void set_namespace(int local_size, int univ_size,
+                          int base_rank, char *name, validation_params *v_params)
+{
+    size_t ninfo;
+    pmix_info_t *info;
+    ninfo = 8;
+    char *regex, *ppn, *tmp;
+    char *ranks = NULL, **nodes = NULL;
+    char **rks=NULL;
+    int i;
+    int rc;
+
+    PMIX_INFO_CREATE(info, ninfo);
+    pmix_strncpy(info[0].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN);
+    info[0].value.type = PMIX_UINT32;
+    info[0].value.data.uint32 = univ_size;
+
+    pmix_strncpy(info[1].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN);
+    info[1].value.type = PMIX_UINT32;
+    info[1].value.data.uint32 = 0;
+
+    pmix_strncpy(info[2].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN);
+    info[2].value.type = PMIX_UINT32;
+    info[2].value.data.uint32 = local_size;
+
+    /* generate the array of local peers */
+    fill_seq_ranks_array(local_size, base_rank, &ranks);
+    if (NULL == ranks) {
+        return;
+    }
+    PMIXT_VAL_PARAM_SETSTR(v_params, pmix_local_peers, ranks, PMIX_MAX_KEYLEN);
+    pmix_strncpy(info[3].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN);
+    info[3].value.type = PMIX_STRING;
+    info[3].value.data.string = strdup(ranks);
+
+    /* assemble the node and proc map info */
+    if (1 == params.nservers) {
+        pmix_argv_append_nosize(&nodes, my_server_info->hostname);
+    } else {
+        char hostname[PMIX_MAXHOSTNAMELEN];
+        for (i = 0; i < params.nservers; i++) {
+            snprintf(hostname, PMIX_MAXHOSTNAMELEN, "node%d", i);
+            pmix_argv_append_nosize(&nodes, hostname);
+        }
+    }
+
+    if (NULL != nodes) {
+        tmp = pmix_argv_join(nodes, ',');
+        pmix_argv_free(nodes);
+        nodes = NULL;
+        if (PMIX_SUCCESS != (rc = PMIx_generate_regex(tmp, &regex) )) {
+            PMIX_ERROR_LOG(rc);
+            return;
+        }
+        free(tmp);
+        PMIX_INFO_LOAD(&info[4], PMIX_NODE_MAP, regex, PMIX_REGEX);
+    }
+
+    /* generate the global proc map - if we have two
+     * servers, then the procs not on this server must
+     * be on the other */
+    if (2 == params.nservers) {
+        pmix_argv_append_nosize(&rks, ranks);
+        free(ranks);
+        nodes = NULL;
+        if (0 == my_server_id) {
+            for (i=base_rank+local_size; i < univ_size; i++) {
+                asprintf(&ppn, "%d", i);
+                pmix_argv_append_nosize(&nodes, ppn);
+                free(ppn);
+            }
+            ppn = pmix_argv_join(nodes, ',');
+            pmix_argv_append_nosize(&rks, ppn);
+            free(ppn);
+        } else {
+            for (i=0; i < base_rank; i++) {
+                asprintf(&ppn, "%d", i);
+                pmix_argv_append_nosize(&nodes, ppn);
+                free(ppn);
+            }
+            ppn = pmix_argv_join(nodes, ',');
+            pmix_argv_prepend_nosize(&rks, ppn);
+            free(ppn);
+        }
+        ranks = pmix_argv_join(rks, ';');
+    }
+    PMIx_generate_ppn(ranks, &ppn);
+    free(ranks);
+    PMIX_INFO_LOAD(&info[5], PMIX_PROC_MAP, ppn, PMIX_REGEX);
+    free(ppn);
+
+    pmix_strncpy(info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN);
+    info[6].value.type = PMIX_UINT32;
+    info[6].value.data.uint32 = univ_size;
+
+    pmix_strncpy(info[7].key, PMIX_APPNUM, PMIX_MAX_KEYLEN);
+    info[7].value.type = PMIX_UINT32;
+    info[7].value.data.uint32 = getpid ();
+
+    int in_progress = 1;
+    if (PMIX_SUCCESS == (rc = PMIx_server_register_nspace(name, local_size,
+                                    info, ninfo, release_cb, &in_progress))) {
+        PMIX_WAIT_FOR_COMPLETION(in_progress);
+    }
+    PMIX_INFO_FREE(info, ninfo);
+}
+
+static void server_unpack_procs(char *buf, size_t size)
+{
+    char *ptr = buf;
+    size_t i;
+    size_t ns_count;
+    char *nspace;
+
+    while ((size_t)(ptr - buf) < size) {
+        memcpy (&ns_count, ptr, sizeof(size_t));
+        ptr += sizeof(size_t);
+
+        for (i = 0; i < ns_count; i++) {
+            server_nspace_t *tmp, *ns_item = NULL;
+            size_t ltasks, ntasks;
+            int server_id;
+
+            memcpy (&server_id, ptr, sizeof(int));
+            ptr += sizeof(int);
+
+            nspace = ptr;
+            ptr += PMIX_MAX_NSLEN+1;
+
+            memcpy (&ntasks, ptr, sizeof(size_t));
+            ptr += sizeof(size_t);
+
+            memcpy (&ltasks, ptr, sizeof(size_t));
+            ptr += sizeof(size_t);
+
+            PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) {
+                if (0 == strcmp(nspace, tmp->name)) {
+                    ns_item = tmp;
+                    break;
+                }
+            }
+            if (NULL == ns_item) {
+                ns_item = PMIX_NEW(server_nspace_t);
+                memcpy(ns_item->name, nspace, PMIX_MAX_NSLEN);
+                pmix_list_append(server_nspace, &ns_item->super);
+                ns_item->ltasks = ltasks;
+                ns_item->ntasks = ntasks;
+                ns_item->task_map = (int*)malloc(sizeof(int) * ntasks);
+                memset(ns_item->task_map, -1, sizeof(int) * ntasks);
+            } else {
+                assert(ns_item->ntasks == ntasks);
+            }
+            size_t i;
+            for (i = 0; i < ltasks; i++) {
+                int rank;
+                memcpy (&rank, ptr, sizeof(int));
+                ptr += sizeof(int);
+                if (ns_item->task_map[rank] >= 0) {
+                    continue;
+                }
+                ns_item->task_map[rank] = server_id;
+            }
+        }
+    }
+}
+
+static size_t server_pack_procs(int server_id, char **buf, size_t size)
+{
+    size_t ns_count = pmix_list_get_size(server_nspace);
+    size_t buf_size = sizeof(size_t) + (PMIX_MAX_NSLEN+1)*ns_count;
+    server_nspace_t *tmp;
+    char *ptr;
+
+    if (0 == ns_count) {
+        return 0;
+    }
+
+    buf_size += size;
+    /* compute size: server_id + total + local procs count + ranks */
+    PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) {
+        buf_size += sizeof(int) + sizeof(size_t) + sizeof(size_t) +
+                sizeof(int) * tmp->ltasks;
+    }
+    *buf = (char*)realloc(*buf, buf_size);
+    memset(*buf + size, 0, buf_size);
+    ptr = *buf + size;
+    /* pack ns count */
+    memcpy(ptr, &ns_count, sizeof(size_t));
+    ptr += sizeof(size_t);
+
+    assert(server_nspace->pmix_list_length);
+
+    PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) {
+        size_t i;
+        /* pack server_id */
+        memcpy(ptr, &server_id, sizeof(int));
+        ptr += sizeof(int);
+        /* pack ns name */
+        memcpy(ptr, tmp->name, PMIX_MAX_NSLEN+1);
+        ptr += PMIX_MAX_NSLEN+1;
+        /* pack ns total size */
+        memcpy(ptr, &tmp->ntasks, sizeof(size_t));
+        ptr += sizeof(size_t);
+        /* pack ns local size */
+        memcpy(ptr, &tmp->ltasks, sizeof(size_t));
+        ptr += sizeof(size_t);
+        /* pack ns ranks */
+        for(i = 0; i < tmp->ntasks; i++) {
+            if (tmp->task_map[i] == server_id) {
+                int rank = (int)i;
+                memcpy(ptr, &rank, sizeof(int));
+                ptr += sizeof(int);
+            }
+        }
+    }
+    assert((size_t)(ptr - *buf) == buf_size);
+    return buf_size;
+}
+
+static void remove_server_item(server_info_t *server)
+{
+    pmix_list_remove_item(server_list, &server->super);
+    PMIX_DESTRUCT_LOCK(&server->lock);
+    PMIX_RELEASE(server);
+}
+
+static int srv_wait_all(double timeout)
+{
+    server_info_t *server, *next;
+    pid_t pid;
+    int status;
+    struct timeval tv;
+    double start_time, cur_time;
+    int ret = 0;
+
+    gettimeofday(&tv, NULL);
+    start_time = tv.tv_sec + 1E-6*tv.tv_usec;
+    cur_time = start_time;
+
+    /* Remove this server from the list */
+    PMIX_LIST_FOREACH_SAFE(server, next, server_list, server_info_t) {
+        if (server->pid == getpid()) {
+            /* remove himself */
+            remove_server_item(server);
+            break;
+        }
+    }
+
+    while (!pmix_list_is_empty(server_list) &&
+                                (timeout >= (cur_time - start_time))) {
+        pid = waitpid(-1, &status, 0);
+        if (pid >= 0) {
+            PMIX_LIST_FOREACH_SAFE(server, next, server_list, server_info_t) {
+                if (server->pid == pid) {
+                    TEST_VERBOSE(("server %d finalize PID:%d with status %d", server->idx,
+                                server->pid, WEXITSTATUS(status)));
+                    ret += WEXITSTATUS(status);
+                    remove_server_item(server);
+                }
+            }
+        }
+        // calculate current timestamp
+        gettimeofday(&tv, NULL);
+        cur_time = tv.tv_sec + 1E-6*tv.tv_usec;
+    }
+
+    return ret;
+}
+
+static int server_fwd_msg(msg_hdr_t *msg_hdr, char *buf, size_t size)
+{
+    server_info_t *tmp_server, *server = NULL;
+    int rc = PMIX_SUCCESS;
+
+    PMIX_LIST_FOREACH(tmp_server, server_list, server_info_t) {
+        if (tmp_server->idx == msg_hdr->dst_id) {
+            server = tmp_server;
+            break;
+        }
+    }
+    if (NULL == server) {
+        return PMIX_ERROR;
+    }
+    rc = write(server->wr_fd, msg_hdr, sizeof(msg_hdr_t));
+    if (rc != sizeof(msg_hdr_t)) {
+        return PMIX_ERROR;
+    }
+    rc = write(server->wr_fd, buf, size);
+    if (rc != (ssize_t)size) {
+        return PMIX_ERROR;
+    }
+    return PMIX_SUCCESS;
+}
+
+static int server_send_msg(msg_hdr_t *msg_hdr, char *data, size_t size)
+{
+    size_t ret = 0;
+    server_info_t *server = NULL, *server_tmp;
+    if (0 == my_server_id) {
+        PMIX_LIST_FOREACH(server_tmp, server_list, server_info_t) {
+            if (server_tmp->idx == msg_hdr->dst_id) {
+                server = server_tmp;
+                break;
+            }
+        }
+        if (NULL == server) {
+            abort();
+        }
+    } else {
+        server = (server_info_t *)pmix_list_get_first(server_list);
+    }
+
+    ret += write(server->wr_fd, msg_hdr, sizeof(msg_hdr_t));
+    ret += write(server->wr_fd, data, size);
+    if (ret != (sizeof(*msg_hdr) + size)) {
+        return PMIX_ERROR;
+    }
+    return PMIX_SUCCESS;
+}
+
+static void _send_procs_cb(pmix_status_t status, const char *data,
+                           size_t ndata, void *cbdata,
+                           pmix_release_cbfunc_t relfn, void *relcbd)
+{
+    server_info_t *server = (server_info_t*)cbdata;
+
+    server_unpack_procs((char*)data, ndata);
+    free((char*)data);
+    PMIX_WAKEUP_THREAD(&server->lock);
+}
+
+static int server_send_procs(void)
+{
+    server_info_t *server;
+    msg_hdr_t msg_hdr;
+    int rc = PMIX_SUCCESS;
+    char *buf = NULL;
+
+    if (0 == my_server_id) {
+        server = my_server_info;
+    } else {
+        server = (server_info_t *)pmix_list_get_first(server_list);
+    }
+
+    msg_hdr.cmd = CMD_FENCE_CONTRIB;
+    msg_hdr.dst_id = 0;
+    msg_hdr.src_id = my_server_id;
+    msg_hdr.size = server_pack_procs(my_server_id, &buf, 0);
+    server->modex_cbfunc = _send_procs_cb;
+    server->cbdata = (void*)server;
+
+    server->lock.active = true;
+
+    if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, buf, msg_hdr.size))) {
+        if (buf) {
+            free(buf);
+        }
+        return PMIX_ERROR;
+    }
+    if (buf) {
+        free(buf);
+    }
+
+    PMIX_WAIT_THREAD(&server->lock);
+    return PMIX_SUCCESS;
+}
+
+int server_barrier(void)
+{
+    server_info_t *server;
+    msg_hdr_t msg_hdr;
+    int rc = PMIX_SUCCESS;
+
+    if (0 == my_server_id) {
+        server = my_server_info;
+    } else {
+        server = (server_info_t *)pmix_list_get_first(server_list);
+    }
+
+    msg_hdr.cmd = CMD_BARRIER_REQUEST;
+    msg_hdr.dst_id = 0;
+    msg_hdr.src_id = my_server_id;
+    msg_hdr.size = 0;
+
+    server->lock.active = true;
+
+    if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, NULL, 0))) {
+        return PMIX_ERROR;
+    }
+    PMIX_WAIT_THREAD(&server->lock);
+
+    return PMIX_SUCCESS;
+}
+
+static void _libpmix_cb(void *cbdata)
+{
+    char *ptr = (char*)cbdata;
+    if (ptr) {
+        free(ptr);
+    }
+}
+
+static void server_read_cb(int fd, short event, void *arg)
+{
+    server_info_t *server = (server_info_t*)arg;
+    msg_hdr_t msg_hdr;
+    char *msg_buf = NULL;
+    static char *fence_buf = NULL;
+    int rc;
+    static size_t barrier_cnt = 0;
+    static size_t contrib_cnt = 0;
+    static size_t fence_buf_offset = 0;
+
+    rc = read(server->rd_fd, &msg_hdr, sizeof(msg_hdr_t));
+    if (rc <= 0) {
+        return;
+    }
+    if (msg_hdr.size) {
+        msg_buf = (char*) malloc(sizeof(char) * msg_hdr.size);
+        rc += read(server->rd_fd, msg_buf, msg_hdr.size);
+    }
+    if (rc != (int)(sizeof(msg_hdr_t) + msg_hdr.size)) {
+        TEST_ERROR(("error read from %d", server->idx));
+    }
+
+    if (my_server_id != msg_hdr.dst_id) {
+        server_fwd_msg(&msg_hdr, msg_buf, msg_hdr.size);
+        free(msg_buf);
+        return;
+    }
+
+    switch(msg_hdr.cmd) {
+        case CMD_BARRIER_REQUEST:
+            barrier_cnt++;
+            TEST_VERBOSE(("CMD_BARRIER_REQ req from %d cnt %lu", msg_hdr.src_id,
+                          (unsigned long)barrier_cnt));
+            if (pmix_list_get_size(server_list) == barrier_cnt) {
+                barrier_cnt = 0; /* reset barrier counter */
+                server_info_t *tmp_server;
+                PMIX_LIST_FOREACH(tmp_server, server_list, server_info_t) {
+                    msg_hdr_t resp_hdr;
+                    resp_hdr.dst_id = tmp_server->idx;
+                    resp_hdr.src_id = my_server_id;
+                    resp_hdr.cmd = CMD_BARRIER_RESPONSE;
+                    resp_hdr.size = 0;
+                    server_send_msg(&resp_hdr, NULL, 0);
+                }
+            }
+            break;
+        case CMD_BARRIER_RESPONSE:
+            TEST_VERBOSE(("%d: CMD_BARRIER_RESP", my_server_id));
+            PMIX_WAKEUP_THREAD(&server->lock);
+            break;
+        case CMD_FENCE_CONTRIB:
+            contrib_cnt++;
+            if (msg_hdr.size > 0) {
+                fence_buf = (char*)realloc((void*)fence_buf,
+                                           fence_buf_offset + msg_hdr.size);
+                memcpy(fence_buf + fence_buf_offset, msg_buf, msg_hdr.size);
+                fence_buf_offset += msg_hdr.size;
+                free(msg_buf);
+                msg_buf = NULL;
+            }
+
+            TEST_VERBOSE(("CMD_FENCE_CONTRIB req from %d cnt %lu size %d",
+                        msg_hdr.src_id, (unsigned long)contrib_cnt, msg_hdr.size));
+            if (pmix_list_get_size(server_list) == contrib_cnt) {
+                server_info_t *tmp_server;
+                PMIX_LIST_FOREACH(tmp_server, server_list, server_info_t) {
+                    msg_hdr_t resp_hdr;
+                    resp_hdr.dst_id = tmp_server->idx;
+                    resp_hdr.src_id = my_server_id;
+                    resp_hdr.cmd = CMD_FENCE_COMPLETE;
+                    resp_hdr.size = fence_buf_offset;
+                    server_send_msg(&resp_hdr, fence_buf, fence_buf_offset);
+                }
+                TEST_VERBOSE(("CMD_FENCE_CONTRIB complete, size %lu",
+                              (unsigned long)fence_buf_offset));
+                if (fence_buf) {
+                    free(fence_buf);
+                    fence_buf = NULL;
+                    fence_buf_offset = 0;
+                }
+                contrib_cnt = 0;
+            }
+            break;
+        case CMD_FENCE_COMPLETE:
+            TEST_VERBOSE(("%d: CMD_FENCE_COMPLETE size %d", my_server_id,
+                        msg_hdr.size));
+            server->modex_cbfunc(PMIX_SUCCESS, msg_buf, msg_hdr.size,
+                                 server->cbdata, _libpmix_cb, msg_buf);
+            msg_buf = NULL;
+            break;
+        case CMD_DMDX_REQUEST: {
+            int *sender_id;
+            pmix_proc_t proc;
+            if (NULL == msg_buf) {
+                abort();
+            }
+            sender_id = (int*)malloc(sizeof(int));
+            server_unpack_dmdx(msg_buf, sender_id, &proc);
+            TEST_VERBOSE(("%d: CMD_DMDX_REQUEST from %d: %s:%d", my_server_id,
+                        *sender_id, proc.nspace, proc.rank));
+            rc = PMIx_server_dmodex_request(&proc, _dmdx_cb, (void*)sender_id);
+            break;
+        }
+        case CMD_DMDX_RESPONSE:
+            TEST_VERBOSE(("%d: CMD_DMDX_RESPONSE", my_server_id));
+            server->modex_cbfunc(PMIX_SUCCESS, msg_buf, msg_hdr.size,
+                                 server->cbdata, _libpmix_cb, msg_buf);
+            msg_buf = NULL;
+            break;
+    }
+    if (NULL != msg_buf) {
+        free(msg_buf);
+    }
+}
+
+int server_fence_contrib(char *data, size_t ndata,
+                         pmix_modex_cbfunc_t cbfunc, void *cbdata)
+{
+    server_info_t *server;
+    msg_hdr_t msg_hdr;
+    int rc = PMIX_SUCCESS;
+
+    if (0 == my_server_id) {
+        server = my_server_info;
+    } else {
+        server = (server_info_t *)pmix_list_get_first(server_list);
+    }
+    msg_hdr.cmd = CMD_FENCE_CONTRIB;
+    msg_hdr.dst_id = 0;
+    msg_hdr.src_id = my_server_id;
+    msg_hdr.size = ndata;
+    server->modex_cbfunc = cbfunc;
+    server->cbdata = cbdata;
+
+    if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, data, ndata))) {
+        return PMIX_ERROR;
+    }
+    return rc;
+}
+
+static int server_pack_dmdx(int sender_id, const char *nspace, int rank,
+                            char **buf)
+{
+    size_t buf_size = sizeof(int) + PMIX_MAX_NSLEN +1 + sizeof(int);
+    char *ptr;
+
+    *buf = (char*)malloc(buf_size);
+    ptr = *buf;
+
+    memcpy(ptr, &sender_id, sizeof(int));
+    ptr += sizeof(int);
+
+    memcpy(ptr, nspace, PMIX_MAX_NSLEN+1);
+    ptr += PMIX_MAX_NSLEN +1;
+
+    memcpy(ptr, &rank, sizeof(int));
+    ptr += sizeof(int);
+
+    return buf_size;
+}
+
+static void server_unpack_dmdx(char *buf, int *sender, pmix_proc_t *proc)
+{
+    char *ptr = buf;
+
+    *sender = *(int *)ptr;
+    ptr += sizeof(int);
+
+    memcpy(proc->nspace, ptr, PMIX_MAX_NSLEN +1);
+    ptr += PMIX_MAX_NSLEN +1;
+
+    proc->rank = *(int *)ptr;
+    ptr += sizeof(int);
+}
+
+
+static void _dmdx_cb(int status, char *data, size_t sz, void *cbdata)
+{
+    msg_hdr_t msg_hdr;
+    int *sender_id = (int*)cbdata;
+
+    msg_hdr.cmd = CMD_DMDX_RESPONSE;
+    msg_hdr.src_id = my_server_id;
+    msg_hdr.size = sz;
+    msg_hdr.dst_id = *sender_id;
+    TEST_VERBOSE(("srv #%d: DMDX RESPONSE: receiver=%d, size=%lu,",
+                  my_server_id, *sender_id, (unsigned long)sz));
+    free(sender_id);
+
+    server_send_msg(&msg_hdr, data, sz);
+}
+
+int server_dmdx_get(const char *nspace, int rank,
+                    pmix_modex_cbfunc_t cbfunc, void *cbdata)
+{
+    server_info_t *server = NULL, *tmp;
+    msg_hdr_t msg_hdr;
+    pmix_status_t rc = PMIX_SUCCESS;
+    char *buf = NULL;
+
+
+    if (0 > (msg_hdr.dst_id = server_find_id(nspace, rank))) {
+        TEST_ERROR(("%d: server cannot found for %s:%d", my_server_id, nspace, rank));
+        goto error;
+    }
+
+    if (0 == my_server_id) {
+        PMIX_LIST_FOREACH(tmp, server_list, server_info_t) {
+            if (tmp->idx == msg_hdr.dst_id) {
+                server = tmp;
+                break;
+            }
+        }
+    } else {
+        server = (server_info_t *)pmix_list_get_first(server_list);
+    }
+
+    if (server == NULL) {
+        goto error;
+    }
+
+    msg_hdr.cmd = CMD_DMDX_REQUEST;
+    msg_hdr.src_id = my_server_id;
+    msg_hdr.size = server_pack_dmdx(my_server_id, nspace, rank, &buf);
+    server->modex_cbfunc = cbfunc;
+    server->cbdata = cbdata;
+
+    if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, buf, msg_hdr.size))) {
+        rc = PMIX_ERROR;
+    }
+    free(buf);
+    return rc;
+
+error:
+    cbfunc(PMIX_ERROR, NULL, 0, cbdata, NULL, 0);
+    return PMIX_ERROR;
+}
+
+static void set_handler_default(int sig)
+{
+    struct sigaction act;
+
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    sigemptyset(&act.sa_mask);
+
+    sigaction(sig, &act, (struct sigaction *)0);
+}
+
+static pmix_event_t handler;
+static void wait_signal_callback(int fd, short event, void *arg)
+{
+    pmix_event_t *sig = (pmix_event_t*) arg;
+    int status;
+    pid_t pid;
+    int i;
+
+    if (SIGCHLD != pmix_event_get_signal(sig)) {
+        return;
+    }
+
+    /* we can have multiple children leave but only get one
+     * sigchild callback, so reap all the waitpids until we
+     * don't get anything valid back */
+    while (1) {
+        pid = waitpid(-1, &status, WNOHANG);
+        if (-1 == pid && EINTR == errno) {
+            /* try it again */
+            continue;
+        }
+        /* if we got garbage, then nothing we can do */
+        if (pid <= 0) {
+            goto done;
+        }
+        /* we are already in an event, so it is safe to access the list */
+        for(i=0; i < cli_info_cnt; i++){
+            if( cli_info[i].pid == pid ){
+                /* found it! */
+                if (WIFEXITED(status)) {
+                    cli_info[i].exit_code = WEXITSTATUS(status);
+                } else {
+                    if (WIFSIGNALED(status)) {
+                        cli_info[i].exit_code = WTERMSIG(status) + 128;
+                    }
+                }
+                cli_cleanup(&cli_info[i]);
+                cli_info[i].alive = false;
+                break;
+            }
+        }
+    }
+  done:
+    for(i=0; i < cli_info_cnt; i++){
+        if (cli_info[i].alive) {
+            /* someone is still alive */
+            return;
+        }
+    }
+    /* get here if nobody is still alive */
+    test_complete = true;
+}
+
+int server_init(test_params *params, validation_params *val_params)
+{
+    pmix_info_t info[2];
+    uint32_t local_size;
+    int rc = PMIX_SUCCESS;
+
+    /* fork/init servers procs */
+    if (params->nservers >= 1) {
+        int i;
+        server_info_t *server_info = NULL;
+        server_list = PMIX_NEW(pmix_list_t);
+
+        TEST_VERBOSE(("pmix server %d started PID:%d", my_server_id, getpid()));
+        for (i = params->nservers - 1; i >= 0; i--) {
+            pid_t pid;
+            server_info = PMIX_NEW(server_info_t);
+
+            int fd1[2];
+            int fd2[2];
+
+            pipe(fd1);
+            pipe(fd2);
+
+            if (0 != i) {
+                pid = fork();
+                if (pid < 0) {
+                    TEST_ERROR(("Fork failed"));
+                    return pid;
+                }
+                if (pid == 0) {
+                    server_list = PMIX_NEW(pmix_list_t);
+                    my_server_info = server_info;
+                    my_server_id = i;
+                    asprintf(&server_info->hostname, "node%d", i);
+                    server_info->idx = 0;
+                    TEST_VERBOSE(("my_server_id after fork: %d, server_info->idx = %d", my_server_id, server_info->idx));
+                    server_info->pid = getppid();
+                    server_info->rd_fd = fd1[0];
+                    server_info->wr_fd = fd2[1];
+                    close(fd1[1]);
+                    close(fd2[0]);
+                    PMIX_CONSTRUCT_LOCK(&server_info->lock);
+                    pmix_list_append(server_list, &server_info->super);
+                    break;
+                }
+                asprintf(&server_info->hostname, "node%d", i);
+                server_info->idx = i; // idx is id of server we are talking to (descriptor of remote peer)
+                server_info->pid = pid;
+                server_info->wr_fd = fd1[1];
+                server_info->rd_fd = fd2[0];
+                PMIX_CONSTRUCT_LOCK(&server_info->lock);
+                close(fd1[0]);
+                close(fd2[1]);
+            } else {
+                my_server_info = server_info;
+                server_info->hostname = strdup("node0");
+                server_info->pid = getpid();
+                server_info->idx  = 0;
+                server_info->rd_fd = fd1[0];
+                server_info->wr_fd = fd1[1];
+                PMIX_CONSTRUCT_LOCK(&server_info->lock);
+                close(fd2[0]);
+                close(fd2[1]);
+            }
+            pmix_list_append(server_list, &server_info->super);
+        }
+        // set validation params for server-specific (i.e., node-specific) info.
+        // putting this here even though we have a separate function to do this
+        // (fill_global_validation_params()) because server_info is currently a
+        // local variable
+        PMIXT_VAL_PARAM_SETSTR(val_params, pmix_hostname, server_info->hostname, PMIX_MAX_KEYLEN-1);
+    }
+    else {
+        PMIXT_VAL_PARAM_SETSTR(val_params, pmix_hostname, "node0", 6);
+    }
+    PMIXT_VAL_PARAM_SETNUM(val_params, pmix_nodeid, my_server_id);
+    TEST_VERBOSE(("my_server_id: %d, pmix_node_id: %d, pmix_hostname: %s",
+                   my_server_id, val_params->pmix_nodeid, val_params->pmix_hostname));
+    /* compute local proc size */
+    local_size = (params->nprocs % params->nservers) > (uint32_t)my_server_id ?
+                params->nprocs / params->nservers + 1 :
+                params->nprocs / params->nservers;
+    PMIXT_VAL_PARAM_SETNUM(val_params, pmix_local_size, local_size);
+    TEST_VERBOSE(("pmix_local_size = %d, params->nprocs = %d", local_size, params->nprocs));
+    /* setup the server library */
+    uint32_t u32 = 0666;
+    PMIX_INFO_LOAD(&info[0], PMIX_SOCKET_MODE, &u32, PMIX_UINT32);
+    PMIX_INFO_LOAD(&info[1], PMIX_HOSTNAME, my_server_info->hostname, PMIX_STRING);
+
+    server_nspace = PMIX_NEW(pmix_list_t);
+
+    if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 2))) {
+        TEST_ERROR(("Init failed with error %d", rc));
+        goto error;
+    }
+
+    /* register test server read thread */
+    if (params->nservers && pmix_list_get_size(server_list)) {
+        server_info_t *server;
+        PMIX_LIST_FOREACH(server, server_list, server_info_t) {
+            server->evread = pmix_event_new(pmix_globals.evbase, server->rd_fd,
+                                            EV_READ|EV_PERSIST, server_read_cb, server);
+            pmix_event_add(server->evread, NULL);
+        }
+    }
+
+    /* register the errhandler */
+    PMIx_Register_event_handler(NULL, 0, NULL, 0,
+                                errhandler, errhandler_reg_callbk, NULL);
+
+    /* setup to see sigchld on the forked tests */
+    pmix_event_assign(&handler, pmix_globals.evbase, SIGCHLD,
+                      EV_SIGNAL|EV_PERSIST, wait_signal_callback, &handler);
+    pmix_event_add(&handler, NULL);
+
+
+    if (0 != (rc = server_barrier())) {
+        goto error;
+    }
+
+    return PMIX_SUCCESS;
+
+error:
+    PMIX_DESTRUCT(server_nspace);
+    return rc;
+}
+
+int server_finalize(test_params *params, int local_fail)
+{
+    int rc = PMIX_SUCCESS;
+    int total_ret = local_fail;
+
+    if (0 != (rc = server_barrier())) {
+        total_ret++;
+        goto exit;
+    }
+
+    if (0 != my_server_id) {
+        server_info_t *server = (server_info_t*)pmix_list_get_first(server_list);
+        remove_server_item(server);
+    }
+
+    if (params->nservers && 0 == my_server_id) {
+        /* wait for all servers are finished */
+        total_ret += srv_wait_all(10.0);
+        PMIX_LIST_RELEASE(server_list);
+        TEST_VERBOSE(("SERVER %d FINALIZE PID:%d with status %d",
+                        my_server_id, getpid(), total_ret));
+        if (0 == total_ret) {
+            TEST_OUTPUT(("Test finished OK!"));
+        } else {
+            rc = PMIX_ERROR;
+        }
+    }
+    PMIX_LIST_RELEASE(server_nspace);
+
+    /* finalize the server library */
+    if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) {
+        TEST_ERROR(("Finalize failed with error %d", rc));
+        total_ret += rc;
+        goto exit;
+    }
+
+exit:
+    return total_ret;
+}
+
+static void fill_global_validation_params(pmix_proc_t proc, int univ_size,
+                                   test_params *params, validation_params *v_params)
+{
+    v_params->validate_params = true;
+
+    PMIXT_VAL_PARAM_SETNUM(v_params, pmix_univ_size, univ_size);
+    PMIXT_VAL_PARAM_SETNUM(v_params, pmix_job_size, params->nprocs);
+    PMIXT_VAL_PARAM_SETSTR(v_params, pmix_nspace, proc.nspace, PMIX_MAX_NSLEN);
+    // more will be added as validation_params struct grows
+    // some validation params set inside server_init
+}
+
+int fill_client_validation_params(pmix_proc_t proc, validation_params *val_params)
+{
+    int local_rank;
+
+    PMIXT_VAL_PARAM_SETNUM(val_params, pmix_rank, proc.rank);
+
+    if (val_params->pmix_nodeid == 0) {
+        PMIXT_VAL_PARAM_SETNUM(val_params, pmix_local_rank, proc.rank);
+    }
+    else {
+        local_rank = (int) proc.rank % (int) (val_params->pmix_nodeid * val_params->pmix_local_size);
+        PMIXT_VAL_PARAM_SETNUM(val_params, pmix_local_rank, local_rank);
+    }
+    // this function will be filled out as validation_params struct grows
+    // some validation params are set in other places out of necessity
+    return PMIX_SUCCESS;
+}
+
+int server_launch_clients(int local_size, int univ_size, int base_rank,
+                   test_params *params, validation_params *v_params, char *** client_env, char ***base_argv)
+{
+    int n;
+    uid_t myuid;
+    gid_t mygid;
+    char *ranks = NULL;
+    char digit[MAX_DIGIT_LEN];
+    int rc;
+    static int cli_counter = 0;
+    static int num_ns = 0;
+    pmix_proc_t proc;
+    int rank_counter = 0;
+    validation_params local_v_params;
+    char *vptr;
+    server_nspace_t *nspace_item = PMIX_NEW(server_nspace_t);
+
+    TEST_VERBOSE(("%d: lsize: %d, base rank %d, local_size %d, univ_size %d", my_server_id, v_params->pmix_local_size,
+                  base_rank, local_size, univ_size));
+
+    TEST_VERBOSE(("Setting job info"));
+    (void)snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, num_ns);
+    set_namespace(local_size, univ_size, base_rank, proc.nspace, v_params);
+    if (NULL != ranks) {
+        free(ranks);
+    }
+
+    /* add namespace entry */
+    nspace_item->ntasks = univ_size;
+    nspace_item->ltasks = local_size;
+    nspace_item->task_map = (int*)malloc(sizeof(int) * univ_size);
+    memset(nspace_item->task_map, -1, sizeof(int)*univ_size);
+    strcpy(nspace_item->name, proc.nspace);
+    pmix_list_append(server_nspace, &nspace_item->super);
+    /* populate our validation parameters with all non-client-specific information */
+    fill_global_validation_params(proc, univ_size, params, v_params);
+
+    for (n = 0; n < local_size; n++) {
+        proc.rank = base_rank + n;
+        nspace_item->task_map[proc.rank] = my_server_id;
+    }
+
+    server_send_procs();
+
+    myuid = getuid();
+    mygid = getgid();
+
+    /* fork/exec the test */
+    for (n = 0; n < local_size; n++) {
+        proc.rank = base_rank + rank_counter;
+        rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL);
+        if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) {
+            TEST_ERROR(("Server register client failed with error %d", rc));
+            PMIx_server_finalize();
+            cli_kill_all();
+            return 0;
+        }
+        if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, client_env))) {//n
+            TEST_ERROR(("Server fork setup failed with error %d", rc));
+            PMIx_server_finalize();
+            cli_kill_all();
+            return rc;
+        }
+        TEST_VERBOSE(("run namespace:%s rank:%d", proc.nspace, proc.rank));
+
+        cli_info[cli_counter].pid = fork();
+        //sleep for debugging purposes (to attach to clients)
+        //sleep(60);
+        if (cli_info[cli_counter].pid < 0) {
+            TEST_ERROR(("Fork failed"));
+            PMIx_server_finalize();
+            cli_kill_all();
+            return 0;
+        }
+        cli_info[cli_counter].rank = proc.rank;//n
+        cli_info[cli_counter].ns = strdup(proc.nspace);
+
+        char **client_argv = pmix_argv_copy(*base_argv);
+
+        // assign values of globally applicable params
+        local_v_params = *v_params; // is copying into a local truly necessary? we run the
+                                   // risk of messing up any pointers that might be in
+                                   // the struct later because this is a 'shallow' copy
+        /* client-specific params filled in by fill_client_validation_params() */
+        if (v_params->validate_params) {
+            if (PMIX_SUCCESS != fill_client_validation_params(proc, &local_v_params)) {
+                TEST_VERBOSE(("Unable to populate client-specific validation params."));
+                exit(1);
+            }
+            vptr = (char *) &local_v_params;
+            char *v_params_ascii = pmixt_encode(vptr, sizeof(local_v_params));
+            /* provide the validation data to the client */
+            pmix_argv_append_nosize(&client_argv, "--validate-params");
+            pmix_argv_append_nosize(&client_argv, v_params_ascii);
+            free(v_params_ascii);
+        }
+        /* add two last arguments: -r <rank> */
+        /*
+        sprintf(digit, "%d", proc.rank);
+        pmix_argv_append_nosize(&client_argv, "-r");
+        pmix_argv_append_nosize(&client_argv, digit);
+        pmix_argv_append_nosize(&client_argv, "-s");
+        pmix_argv_append_nosize(&client_argv, proc.nspace);
+        */
+        sprintf(digit, "%d", univ_size);
+        pmix_argv_append_nosize(&client_argv, "--ns-size");
+        pmix_argv_append_nosize(&client_argv, digit);
+
+        sprintf(digit, "%d", num_ns);
+        pmix_argv_append_nosize(&client_argv, "--ns-id");
+        pmix_argv_append_nosize(&client_argv, digit);
+
+        sprintf(digit, "%d", 0);
+        pmix_argv_append_nosize(&client_argv, "--base-rank");
+        pmix_argv_append_nosize(&client_argv, digit);
+
+
+        if (cli_info[cli_counter].pid == 0) {
+            sigset_t sigs;
+            set_handler_default(SIGTERM);
+            set_handler_default(SIGINT);
+            set_handler_default(SIGHUP);
+            set_handler_default(SIGPIPE);
+            set_handler_default(SIGCHLD);
+            sigprocmask(0, 0, &sigs);
+            sigprocmask(SIG_UNBLOCK, &sigs, 0);
+
+            if( !TEST_VERBOSE_GET() ){
+                // Hide clients stdout
+                if (NULL == freopen("/dev/null","w", stdout)) {
+                    return 0;
+                }
+            }
+            execve(params->binary, client_argv, *client_env);
+            /* Does not return */
+            TEST_ERROR(("execve() failed"));
+            return 0;
+        }
+        cli_info[cli_counter].alive = true;
+        cli_info[cli_counter].state = CLI_FORKED;
+
+        pmix_argv_free(client_argv);
+
+        cli_counter++;
+        rank_counter++;
+    }
+    num_ns++;
+    return rank_counter;
+}
diff -Nru pmix-3.2.2~rc1/test/test_v2/test_server.h pmix-4.0.0/test/test_v2/test_server.h
--- pmix-3.2.2~rc1/test/test_v2/test_server.h	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/test/test_v2/test_server.h	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2018      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ */
+
+#ifndef TEST_SERVER_C
+#define TEST_SERVER_C
+
+#include "pmix_server.h"
+#include "test_common.h"
+
+
+typedef enum {
+    CMD_BARRIER_REQUEST,
+    CMD_BARRIER_RESPONSE,
+    CMD_FENCE_CONTRIB,
+    CMD_FENCE_COMPLETE,
+    CMD_DMDX_REQUEST,
+    CMD_DMDX_RESPONSE
+} server_cmd_t;
+
+typedef struct {
+    int dst_id;
+    int src_id;
+    int cmd;
+    size_t size;
+} msg_hdr_t;
+
+struct server_info_t
+{
+    pmix_list_item_t super;
+    char *hostname;
+    pid_t pid;
+    int idx;
+    int rd_fd;
+    int wr_fd;
+    pmix_event_t *evread;
+    pmix_lock_t lock;
+    pmix_modex_cbfunc_t modex_cbfunc;
+    void *cbdata;
+};
+typedef struct server_info_t server_info_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(server_info_t);
+
+struct server_nspace_t
+{
+    pmix_list_item_t super;
+    char name[PMIX_MAX_NSLEN+1];
+    size_t ntasks; /* total number of tasks in this namespace */
+    size_t ltasks; /* local */
+    int *task_map;
+};
+typedef struct server_nspace_t server_nspace_t;
+PMIX_EXPORT PMIX_CLASS_DECLARATION(server_nspace_t);
+
+extern int my_server_id;
+extern pmix_list_t *server_list;
+extern server_info_t *my_server_info;
+extern pmix_list_t *server_nspace;
+
+int server_init(test_params *params, validation_params *val_params);
+int server_finalize(test_params *params, int local_fail);
+int server_barrier(void);
+int server_fence_contrib(char *data, size_t ndata,
+                         pmix_modex_cbfunc_t cbfunc, void *cbdata);
+int server_dmdx_get(const char *nspace, int rank,
+                    pmix_modex_cbfunc_t cbfunc, void *cbdata);
+int server_launch_clients(int local_size, int univ_size, int base_rank,
+                   test_params *params, validation_params *v_params, char *** client_env, char ***base_argv);
+
+
+#endif // TEST_SERVER_C
+
diff -Nru pmix-3.2.2~rc1/test/utils.h pmix-4.0.0/test/utils.h
--- pmix-3.2.2~rc1/test/utils.h	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/test/utils.h	2021-01-02 08:56:17.000000000 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2018 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2015      Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
diff -Nru pmix-3.2.2~rc1/.travis.yml pmix-4.0.0/.travis.yml
--- pmix-3.2.2~rc1/.travis.yml	1970-01-01 00:00:00.000000000 +0000
+++ pmix-4.0.0/.travis.yml	2021-01-02 08:56:17.000000000 +0000
@@ -0,0 +1,50 @@
+sudo: false
+dist: xenial
+
+cache:
+  apt: true
+
+language: python
+python:
+  - "3.5"
+
+addons:
+  apt:
+    packages:
+      - perl
+      - libevent-dev
+      - libhwloc-dev
+
+jobs:
+  fast_finish: true
+  include:
+    - compiler: gcc
+      env:
+        - CFLAGS="-Wall -Wextra -Wno-unused-function -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers -Wno-sign-compare -Wno-missing-braces"
+          ENABLE_WERROR="--enable-werror"
+    - compiler: gcc
+      env:
+        - CFLAGS="-fsanitize=undefined"  # TODO: add -fsanitize=address
+    - compiler: clang
+      env:
+        - CFLAGS="-Wall -Wextra -Wno-unused-function -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers -Wno-sign-compare -Wno-missing-braces"
+          ENABLE_WERROR="--enable-werror"
+    # TODO:
+    #- compiler: clang
+    #  env:
+    #    - CFLAGS="-fsanitize=address"
+    #      ASAN_OPTIONS=detect_stack_use_after_return=1:detect_leaks=1:check_initialization_order=true:strict_init_order=true:detect_stack_use_after_scope=1
+    - compiler: clang
+      env:
+        - CFLAGS="-fsanitize=undefined"
+
+script:
+        # prepare
+        # build
+        - ./autogen.pl
+        - pip install -q cython
+        - ./configure ${ENABLE_WERROR}
+        - if [ $? -ne 0 ]; then cat config.log; fi
+        - make all
+        # test
+        - make check
diff -Nru pmix-3.2.2~rc1/VERSION pmix-4.0.0/VERSION
--- pmix-3.2.2~rc1/VERSION	2020-12-05 13:21:10.000000000 +0000
+++ pmix-4.0.0/VERSION	2021-01-02 08:56:17.000000000 +0000
@@ -4,7 +4,6 @@
 # Copyright (c) 2013      Mellanox Technologies, Inc.
 #                         All rights reserved.
 # Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
-# Copyright (c) 2020      IBM Corporation.  All rights reserved.
 
 # This is the VERSION file for PMIx, describing the precise
 # version of PMIx in this distribution.  The various components of
@@ -14,9 +13,9 @@
 # major, minor, and release are generally combined in the form
 # <major>.<minor>.<release>.
 
-major=3
-minor=2
-release=2
+major=4
+minor=0
+release=0
 
 # greek is used for alpha or beta release tags.  If it is non-empty,
 # it will be appended to the version number.  It does not have to be
@@ -24,7 +23,7 @@
 # The only requirement is that it must be entirely printable ASCII
 # characters and have no white space.
 
-greek=rc1
+greek=rc3
 
 # If repo_rev is empty, then the repository version number will be
 # obtained during "make dist" via the "git describe --tags --always"
@@ -45,7 +44,7 @@
 
 # The date when this release was created
 
-date="Oct 30, 2020"
+date="Mar 10, 2018"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library
@@ -76,13 +75,12 @@
 # Version numbers are described in the Libtool current:revision:age
 # format.
 
-libpmix_so_version=4:32:2
-libpmi_so_version=1:1:0
-libpmi2_so_version=1:0:0
+libpmix_so_version=5:0:3
 
 # "Common" components install standalone libraries that are run-time
 # # linked by one or more components.  So they need to be versioned as
 # # well.  Yuck; this somewhat breaks the
 # # components-don't-affect-the-build-system abstraction.
 #
-libmca_common_dstore_so_version=1:2:0
+libmca_common_dstore_so_version=1:3:0
+libmca_common_sse_so_version=1:0:0