From c98f70f6f0703cacb38bfd5ebbdd98ca14405762 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:50:08 -0700
Subject: [PATCH 01/33] Cachepath:  parse_loc utilities.

---
 src/client/beboot/parseloc.h |   7 ++
 src/utils/parseloc.c         | 145 +++++++++++++++++++++++++++++------
 2 files changed, 127 insertions(+), 25 deletions(-)

diff --git a/src/client/beboot/parseloc.h b/src/client/beboot/parseloc.h
index c5362e2e..1731906a 100644
--- a/src/client/beboot/parseloc.h
+++ b/src/client/beboot/parseloc.h
@@ -24,6 +24,13 @@ extern "C" {
 #include "spindle_launch.h"
    
 char *parse_location(char *loc, number_t number);
+char *parse_location_noerr(char *loc, number_t number);
+char *realize(char *path);
+char **parse_colonsep_prefixes(char *colonsep_list, number_t number);
+int is_local_prefix(const char *path, char **local_prefixes);
+static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number );
+void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number );
+void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath );
 
 #if defined(__cplusplus)
 }
diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c
index 0d8dcc81..28c446dd 100644
--- a/src/utils/parseloc.c
+++ b/src/utils/parseloc.c
@@ -22,6 +22,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <assert.h>
+#include <errno.h>
 
 #if !defined(USE_PLUGIN_DEBUG)
 #include "spindle_debug.h"
@@ -34,13 +35,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "ccwarns.h"
 #include "spindle_launch.h"
 
-#if defined(__cplusplus)
-extern "C" {
-#endif
-   char *parse_location(char *loc, number_t number);
-#if defined(__cplusplus)
-}
-#endif
+extern int spindle_mkdir(char *orig_path);
 
 #if defined(CUSTOM_GETENV)
 extern char *custom_getenv();
@@ -166,38 +161,64 @@ char *parse_location_noerr(char *loc, number_t number)
  **/
 char *realize(char *path)
 {
+   int local_errno;
    char *result;
-   char *origpath, *cur_slash = NULL, *trailing;
-   struct stat buf;
+   char *origpath, *cur_slash = NULL, *prev_slash = NULL;
+   struct stat *buf = calloc( 1, sizeof( struct stat ) );
    char newpath[MAX_PATH_LEN+1];
    int lastpos;
    newpath[MAX_PATH_LEN] = '\0';
 
    origpath = strdup(path);
-   for (;;) {
-      if (stat(origpath, &buf) != -1)
-         break;
-      if (cur_slash)
-         *cur_slash = '/';
+   errno=0;
+   while( stat( origpath, buf ) == -1 ){
+      local_errno = errno;
+      debug_printf("Failed to stat '%s' (%s).\n", origpath, strerror(local_errno));
+      prev_slash = cur_slash;
       cur_slash = strrchr(origpath, '/');
-      if (!cur_slash)
-         break;
-      *cur_slash = '\0';
+      if( prev_slash )
+          *prev_slash = '/';
+      if( cur_slash )
+          *cur_slash = '\0';
+      else{
+          debug_printf("Nothing in the original path can be stat'ed.  (%s)\n", path);
+          return NULL;
+      }
    }
-   if (cur_slash)
-      trailing = cur_slash + 1;
-   else
-      trailing = "";
 
+   errno = 0;
    result = realpath(origpath, newpath);
    if (!result) {
+      local_errno = errno;
+      err_printf(
+          "Error:  realpath(3) failed to create canonical version of '%s' (%s).  Returning '%s'.\n",
+          origpath, strerror(local_errno), path );
+      errno = 0;
+      int rc = stat( origpath, buf );
+      local_errno = errno;
+      err_printf(
+          "        Statting that path results in rc=%d, errno=%d, error='%s'.\n",
+          rc, local_errno, strerror(local_errno));
       free(origpath);
-      return path;
+      return NULL;
    }
+   free(buf);
 
-   strncat(newpath, "/", MAX_PATH_LEN);
-   strncat(newpath, trailing, MAX_PATH_LEN);
-   newpath[MAX_PATH_LEN] = '\0';
+   if( cur_slash ){
+       if( strlen( newpath ) + strlen( cur_slash+1 ) > MAX_PATH_LEN ){
+            err_printf(
+                    "Error:  The realized path exceeds MAX_PATH_LEN (%d).\n"
+                    "  Original path:     '%s'\n"
+                    "  Statable part:     '%s'\n"
+                    "  Canonical version: '%s'\n"
+                    "  Returning original path.\n",
+                    MAX_PATH_LEN, path, origpath, newpath);
+            free(origpath);
+            return path;
+       }
+       strncat(newpath, "/",         2);
+       strncat(newpath, cur_slash+1, MAX_PATH_LEN - strlen( newpath ));
+   }
    free(origpath);
 
    lastpos = strlen(newpath)-1;
@@ -278,3 +299,77 @@ int is_local_prefix(const char *path, char **local_prefixes) {
    return 0;
 }
 
+/* validateCandidatePath determines if candidatePath passes parse_location(), realize(), and spindle_mkdir(), which is to say, can
+ * spindle create a directory from this path?
+ *
+ * If not NULL, then realizedPath, parsedPath, and/or symbolicPath will hold the respective intermediate/final results.
+ *
+ * Return 1 if the candidatePath is valid, otherwise 0.
+ */
+static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number ){
+    int rc;
+    char *parsedCandidatePath, *realizedCandidatePath;
+    parsedCandidatePath = parse_location( candidatePath, number );
+    if( parsedCandidatePath ){
+       realizedCandidatePath = realize( parsedCandidatePath );
+       if( realizedCandidatePath ){
+           rc = spindle_mkdir( parsedCandidatePath );
+           if( 0 == rc ){
+               if( symbolicPath) *symbolicPath = candidatePath;
+               if( parsedPath  ) *parsedPath   = parsedCandidatePath;
+               if( realizedPath) *realizedPath = realizedCandidatePath;
+               return 1;
+           }else{
+               debug_printf2("Unable to create directory %s, moving on to the next candidate.\n", realizedCandidatePath );
+           }
+        }else{
+            debug_printf2( "Unable to realize candidate %s, moving on to the next candidate.\n", parsedCandidatePath );
+        }
+    }else{
+        debug_printf2("Unable to parse candidate %s, moving on to the next candidate.\n", candidatePath );
+    }
+    return 0;
+}
+
+/**
+ * determineValidCachePaths()  works exclusively with the cachepaths parameter.  Because not all paths may be valid on all
+ * compute nodes, and because we want to have all nodes reach a consensus on which cache path to use, we
+ * determine the validity of all paths in the origPathList, save the intermediate results, and return a bit
+ * index to the user.  Via allReduce() all nodes reach a consensus on the set of valid paths, and retrieves
+ * that informatino via getValidPathByIndex().
+ */
+static char *realizedCachePaths[64], *parsedCachePaths[64], *symbolicCachePaths[64];
+
+void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number ){
+
+    char *saveptr, *candidatePath, *pathList = strdup( origPathList );
+    uint64_t bitoffset = 0;
+
+    *validBitIdx = 0;
+    debug_printf2("origPathList='%s', number='%lu'.\n", origPathList, number );
+
+    candidatePath = strtok_r( pathList, ":", &saveptr );
+    while( NULL != candidatePath && bitoffset < 64 ){
+        *validBitIdx |= validateCandidatePath(
+                            candidatePath,
+                            &realizedCachePaths[bitoffset],
+                            &parsedCachePaths[bitoffset],
+                            &symbolicCachePaths[bitoffset], number ) << bitoffset;
+        bitoffset++;
+        candidatePath = strtok_r( NULL, ":", &saveptr );
+    }
+    free( pathList );
+}
+
+void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath ){
+    uint64_t bitoffset = 0;
+    if (!validBitIdx){
+        return;
+    }
+    while( (bitoffset < 64) && (((1 << bitoffset) & validBitIdx) == 0) ){
+        bitoffset++;
+    }
+    if( realizedCachePath ) *realizedCachePath = realizedCachePaths[bitoffset];
+    if( parsedCachePath   ) *parsedCachePath   = parsedCachePaths[bitoffset];
+    if( symbolicCachePath ) *symbolicCachePath = symbolicCachePaths[bitoffset];
+}

From 77fd7d3cca429c5d39c7b0879dce426d79fe0379 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:50:45 -0700
Subject: [PATCH 02/33] Cachepath:  remove/rename [orig_]location.

---
 src/client/beboot/spindle_bootstrap.c  |  2 +-
 src/client/client/client.c             | 15 +++++++--------
 src/client/client/intercept.h          |  1 +
 src/client/client/intercept_exec.c     |  5 +----
 src/client/client/intercept_readlink.c | 16 +++++++++++-----
 src/client/client/should_intercept.c   | 25 +++++++++++++++----------
 src/client/client/should_intercept.h   |  1 +
 7 files changed, 37 insertions(+), 28 deletions(-)

diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index 7805a9db..98424d50 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -114,7 +114,6 @@ static void setup_environment()
 
    setenv("LD_AUDIT", client_lib, 1);
    setenv("LDCS_LOCATION", location, 1);
-   setenv("LDCS_ORIG_LOCATION", orig_location, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -161,6 +160,7 @@ static int parse_cmdline(int argc, char *argv[])
    }
 
    symbolic_location = argv[i++];
+   i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
    opts_s = argv[i++];
diff --git a/src/client/client/client.c b/src/client/client/client.c
index a50bb228..d7827d4b 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -41,6 +41,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "exec_util.h"
 #include "intercept.h"
 #include "fixlocale.h"
+#include "should_intercept.h"
 
 errno_location_t app_errno_location;
 
@@ -70,11 +71,8 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-/* location has the realize'd path to the local file cache. orig_location is not realized and
- * may contain symlinks
- */
-char *location;
-char *orig_location;
+static char *location;
+static char *chosen_realized_cachepath, *chosen_parsed_cachepath, *chosen_symbolic_cachepath;
 number_t number;
 static int have_stat_patches;
 
@@ -201,7 +199,6 @@ static int init_server_connection()
       return 0;
 
    location = getenv("LDCS_LOCATION");
-   orig_location = getenv("LDCS_ORIG_LOCATION");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -221,7 +218,6 @@ static int init_server_connection()
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
       unsetenv("LDCS_LOCATION");
-      unsetenv("LDCS_ORIG_LOCATION");
       unsetenv("LDCS_NUMBER");
       unsetenv("LDCS_CONNECTION");
       unsetenv("LDCS_RANKINFO");
@@ -265,6 +261,9 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, &chosen_symbolic_cachepath );
+   set_should_intercept_cachepath(   chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
+   set_intercept_readlink_cachepath( chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
    
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
@@ -472,7 +471,7 @@ char *client_library_load(const char *name)
 
    char *orig_file_name = (char *) name;
    if (is_in_spindle_cache(name)) {
-      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location);
+      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, chosen_realized_cachepath);
       memset(fixed_name, 0, MAX_PATH_LEN+1);
       send_orig_path_request(ldcsid, orig_file_name, fixed_name);
       orig_file_name = fixed_name;
diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h
index 4ace2328..aae968f7 100644
--- a/src/client/client/intercept.h
+++ b/src/client/client/intercept.h
@@ -89,6 +89,7 @@ int execvpe_wrapper(const char *path, char *const argv[], const char *envp[]);
 pid_t vfork_wrapper();
 char *dlerror_wrapper();
 
+void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 ssize_t readlink_wrapper(const char *path, char *buf, size_t bufsiz);
 ssize_t readlinkat_wrapper(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 14b555ed..8480b75e 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -142,7 +142,6 @@ static char **removeEnvironmentStrs(char **envp)
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
              strIsPrefix("LDCS_LOCATION=", envp[i]) ||
-             strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -177,7 +176,6 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
          unsetf("LDCS_LOCATION");
-         unsetf("LDCS_ORIG_LOCATION");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
@@ -198,13 +196,12 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
    if (envp) {
       debug_printf2("Propogating spindle environment by copying it to new envp list\n");
       for (cur = (char **) envp; *cur; cur++, orig_size++);
-      new_size = orig_size + 10;
+      new_size = orig_size + 20;
       newenv = (char **) malloc(new_size * sizeof(char*));
       
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   
       propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_ORIG_LOCATION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS");
diff --git a/src/client/client/intercept_readlink.c b/src/client/client/intercept_readlink.c
index 0a326330..2799baee 100644
--- a/src/client/client/intercept_readlink.c
+++ b/src/client/client/intercept_readlink.c
@@ -31,19 +31,25 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-extern char *location;
+static char *cachepath;
+
+void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
+    cachepath = chosen_realized_cachepath;
+    chosen_parsed_cachepath = chosen_parsed_cachepath;
+    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
+}
 
 static int fix_local_readlink(char *buf, size_t bufsiz)
 {
    char spindle_id[32];
-   int location_len, result;
+   int cachepath_len, result;
    char tmp[MAX_PATH_LEN+1];
 
-   location_len = strlen(location);   
+   cachepath_len = strlen(cachepath);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number);
-   if (strstr(buf, spindle_id) && strncmp(location, buf, location_len) == 0) {
+   if (strstr(buf, spindle_id) && strncmp(cachepath, buf, cachepath_len) == 0) {
       debug_printf2("readlink received spindle cache path %s. Translating\n", buf);
-      result = send_orig_path_request(ldcsid, buf+location_len+1, tmp);
+      result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp);
       if (result == -1)
          return -1;
       debug_printf2("readlink translated spindle local path %s to %s\n", buf, tmp);
diff --git a/src/client/client/should_intercept.c b/src/client/client/should_intercept.c
index bfabbbf8..3a348d3d 100644
--- a/src/client/client/should_intercept.c
+++ b/src/client/client/should_intercept.c
@@ -29,22 +29,27 @@
 #include "spindle_debug.h"
 
 extern int relocate_spindleapi();
+static char *cachepath, *orig_cachepath;
+
+void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
+    cachepath = chosen_realized_cachepath;
+    orig_cachepath = chosen_parsed_cachepath;
+    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
+}
 
-extern char *location;
-extern char *orig_location;
 
 int is_in_spindle_cache(const char *pathname)
 {
-   static int location_size = 0;
-   static int orig_location_size = 0;
-   if (!location_size) {
-      location_size = strlen(location);
+   static int cachepath_size = 0;
+   static int orig_cachepath_size = 0;
+   if (!cachepath_size) {
+      cachepath_size = strlen(cachepath);
    }
-   if (!orig_location_size) {
-      orig_location_size = strlen(orig_location);
+   if (!orig_cachepath_size) {
+      orig_cachepath_size = strlen(orig_cachepath);
    }
-   return ((strncmp(pathname, location, location_size) == 0) ||
-           (strncmp(pathname, orig_location, orig_location_size) == 0));
+   return ((strncmp(pathname, cachepath, cachepath_size) == 0) ||
+           (strncmp(pathname, orig_cachepath, orig_cachepath_size) == 0));
 }
 
 extern int is_local_prefix(const char *path, char **cached_local_prefixes);
diff --git a/src/client/client/should_intercept.h b/src/client/client/should_intercept.h
index f6a9b510..6a545913 100644
--- a/src/client/client/should_intercept.h
+++ b/src/client/client/should_intercept.h
@@ -27,6 +27,7 @@
 #define EXCL_OPEN 2
 #define ERR_CALL 3
 
+void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 int open_filter(const char *fname, int flags);
 int fopen_filter(const char *fname, const char *flags);
 int exec_filter(const char *fname);

From c3dba387739641d8f119cc25241c6283729cdb2d Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:51:51 -0700
Subject: [PATCH 03/33] Cachepath:  Configure-time support.

---
 config.h.in            |  3 +++
 configure              | 16 ++++++++++++++++
 configure.common.ac    |  5 +++++
 src/client/config.h.in |  3 +++
 src/client/configure   | 16 ++++++++++++++++
 src/fe/config.h.in     |  3 +++
 src/fe/configure       | 16 ++++++++++++++++
 src/server/config.h.in |  3 +++
 src/server/configure   | 16 ++++++++++++++++
 9 files changed, 81 insertions(+)

diff --git a/config.h.in b/config.h.in
index a346ae96..6be0335d 100644
--- a/config.h.in
+++ b/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/configure b/configure
index aaa2f552..c9e517c8 100755
--- a/configure
+++ b/configure
@@ -846,6 +846,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1589,6 +1590,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16615,6 +16618,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16651,6 +16662,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF
diff --git a/configure.common.ac b/configure.common.ac
index 780aef00..31a04d4d 100644
--- a/configure.common.ac
+++ b/configure.common.ac
@@ -17,6 +17,10 @@ AC_ARG_WITH(default-num-ports,
             [AS_HELP_STRING([--with-default-numports=NUM],[Number of TCP/IP ports to scan for Spindle server communication])],
             [NUM_COBO_PORTS=${withval}],
             [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS])
+AC_ARG_WITH(cachepaths,
+            [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
+            [CACHEPATHS=${withval}],
+            [CACHEPATHS=$DEFAULT_LOC])
 AC_ARG_WITH(localstorage,
             [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
             [SPINDLE_LOC=${withval}],
@@ -29,6 +33,7 @@ AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle]
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
 AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
 TESTRM=unknown
diff --git a/src/client/config.h.in b/src/client/config.h.in
index 6c5eae90..a42cc0ab 100644
--- a/src/client/config.h.in
+++ b/src/client/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/src/client/configure b/src/client/configure
index dbd69e1f..cbae8891 100755
--- a/src/client/configure
+++ b/src/client/configure
@@ -810,6 +810,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1532,6 +1533,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -12587,6 +12590,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -12623,6 +12634,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF
diff --git a/src/fe/config.h.in b/src/fe/config.h.in
index 357feea0..5057ef33 100644
--- a/src/fe/config.h.in
+++ b/src/fe/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/src/fe/configure b/src/fe/configure
index 95e99784..29db0f61 100755
--- a/src/fe/configure
+++ b/src/fe/configure
@@ -831,6 +831,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1570,6 +1571,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16437,6 +16440,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16473,6 +16484,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF
diff --git a/src/server/config.h.in b/src/server/config.h.in
index 6eea236a..dc9439b3 100644
--- a/src/server/config.h.in
+++ b/src/server/config.h.in
@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
diff --git a/src/server/configure b/src/server/configure
index e32c1cfe..55e9c6f5 100755
--- a/src/server/configure
+++ b/src/server/configure
@@ -837,6 +837,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1567,6 +1568,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16434,6 +16437,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16470,6 +16481,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF

From 7e1715d72c1a32a314c7096ee1c19637e5ab86c9 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:52:29 -0700
Subject: [PATCH 04/33] Cachepath:  Internal messaging for path resolution

---
 src/client/client_comlib/client_api.c         | 53 ++++++++++++
 src/client/client_comlib/client_api.h         |  1 +
 src/fe/startup/spindle_fe.cc                  | 18 +++++
 src/include/ldcs_api.h                        |  3 +
 .../auditserver/ldcs_audit_server_handlers.c  | 80 +++++++++++++++++++
 .../auditserver/ldcs_audit_server_md_cobo.c   |  6 ++
 src/server/comlib/ldcs_api_util.c             |  3 +
 7 files changed, 164 insertions(+)

diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 96390dca..267b8a93 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -37,6 +37,59 @@ static struct lock_t comm_lock;
 #define COMM_LOCK do { if (lock(&comm_lock) == -1) return -1; } while (0)
 #define COMM_UNLOCK unlock(&comm_lock)
    
+
+int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath, char **chosen_symbolic_cachepath ){
+   ldcs_message_t message;
+   char buffer[MAX_PATH_LEN+1];
+   buffer[MAX_PATH_LEN] = '\0';
+
+   message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST;
+   message.header.len = MAX_PATH_LEN;
+   message.data = buffer;
+
+   COMM_LOCK;
+
+   debug_printf3("sending message of type: request_location_path.\n" );
+   client_send_msg(fd, &message);
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      assert(0);
+   }
+   if( chosen_realized_cachepath ){
+       *chosen_realized_cachepath = strdup( buffer );
+   }
+
+   COMM_LOCK;
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      assert(0);
+   }
+   if( chosen_parsed_cachepath ){
+       *chosen_parsed_cachepath = strdup( buffer );
+   }
+
+   COMM_LOCK;
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      assert(0);
+   }
+   if( chosen_symbolic_cachepath ){
+       *chosen_symbolic_cachepath = strdup( buffer );
+   }
+
+   return 0;
+}
+
 int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) {
    ldcs_message_t message;
    char buffer[MAX_PATH_LEN+1+sizeof(int)];
diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h
index 74f82346..982c4b1c 100644
--- a/src/client/client_comlib/client_api.h
+++ b/src/client/client_comlib/client_api.h
@@ -42,6 +42,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath);
 int send_dirlists_request(int fd, char **local_result, char **exece_result, char **to_free);
 int send_procmaps_query(int fd, int pid, char *result);
 int send_pickone_query(int fd, char *key, int *result);
+int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath, char **chosen_realized_cachepath );
 
 int get_python_prefix(int fd, char **prefix);
 
diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc
index 31b29cf7..43e8c6ba 100644
--- a/src/fe/startup/spindle_fe.cc
+++ b/src/fe/startup/spindle_fe.cc
@@ -41,6 +41,7 @@ static const char *logging_file = NULL;
 #endif
 static const char spindle_bootstrap[] = LIBEXECDIR "/spindle_bootstrap";
 static bool sendAndWaitForAlive();
+static void determineCachepathConsensus();
 
 #define STARTUP_TIMEOUT 60
 
@@ -71,6 +72,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    buffer_size += sizeof(opt_t);
    buffer_size += sizeof(unique_id_t);
    buffer_size += args->location ? strlen(args->location) + 1 : 1;
+   buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1;
    buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1;
    buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1;
    buffer_size += args->numa_files ? strlen(args->numa_files) + 1 : 1;
@@ -91,6 +93,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    pack_param(args->startup_type, buf, pos);
    pack_param(args->shm_cache_size, buf, pos);
    pack_param(args->location, buf, pos);
+   pack_param(args->candidate_cachepaths, buf, pos);
    pack_param(args->pythonprefix, buf, pos);
    pack_param(args->preloadfile, buf, pos);
    pack_param(args->bundle_timeout_ms, buf, pos);
@@ -230,6 +233,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin
       (*spindle_argv)[n++] = strdup(uniqueid_s);
    }
    (*spindle_argv)[n++] = strdup(params->location);
+   (*spindle_argv)[n++] = strdup(params->candidate_cachepaths);
    (*spindle_argv)[n++] = strdup(number_s);
    (*spindle_argv)[n++] = strdup(opt_s);
    (*spindle_argv)[n++] = strdup(cachesize_s);
@@ -395,9 +399,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
    /* Start FE server */
    debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; "
                 "use_launcher = %u; startup_type = %u; shm_cache_size = %u; location = %s; "
+                "cachepaths = %s; "
                 "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n",
                 (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id,
                 params->use_launcher, params->startup_type, params->shm_cache_size, params->location,
+                params->candidate_cachepaths,
                 params->pythonprefix, params->preloadfile, params->bundle_timeout_ms,
                 params->bundle_cachesize_kb);
    printSpindleFlags(params->opts);
@@ -427,6 +433,7 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
 
    /* Wait for servers to indicate startup */
    sendAndWaitForAlive();
+   determineCachepathConsensus();
 
    return 0;   
 }
@@ -483,6 +490,17 @@ void markRSHPidReapedFE()
    clear_fe_rsh_pid();
 }
 
+static void determineCachepathConsensus( void ){
+   ldcs_message_t consensus_req_msg;
+   consensus_req_msg.header.type = LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS;
+   consensus_req_msg.header.len = 0;
+   consensus_req_msg.data = NULL;
+   int result = ldcs_audit_server_fe_broadcast(&consensus_req_msg, NULL);
+   if (result == -1) {
+      debug_printf("Failure sending cachepath consensus message\n");
+   }
+}
+
 static bool sendAndWaitForAlive()
 {
    int result;
diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h
index e8ffa43d..e6ccbafb 100644
--- a/src/include/ldcs_api.h
+++ b/src/include/ldcs_api.h
@@ -85,6 +85,9 @@ typedef enum {
    LDCS_MSG_PICKONE_RESP,
    LDCS_MSG_ALIVE_REQ,
    LDCS_MSG_ALIVE_RESP,
+   LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS,
+   LDCS_MSG_CHOSEN_CACHEPATH_REQUEST,
+   LDCS_MSG_CHOSEN_CACHEPATH,
    LDCS_MSG_UNKNOWN
 } ldcs_message_ids_t;
 
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 8a18de24..98c6807a 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -178,7 +178,10 @@ static int handle_setup_alias(ldcs_process_data_t *procdata, char *pathname, cha
 static int handle_client_dirlists_req(ldcs_process_data_t *procdata, int nc);
 static int handle_close_client_query(ldcs_process_data_t *procdata, int nc);
 static int handle_alive_msg(ldcs_process_data_t *procdata, ldcs_message_t *msg);
+static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg);
+static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc);
 
+extern void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath );
 /**
  * Query from client to server.  Returns info about client's rank in server data structures. 
  **/
@@ -1890,6 +1893,8 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t
          return handle_client_pickone_msg(procdata, nc, msg);
       case LDCS_MSG_END:
          return handle_client_end(procdata, nc);
+      case LDCS_MSG_CHOSEN_CACHEPATH_REQUEST:
+         return handle_chosen_cachepath_request(procdata, nc);
       default:
          err_printf("Received unexpected message from client %d: %d\n", nc, (int) msg->header.type);
          assert(0);
@@ -1987,6 +1992,8 @@ int handle_server_message(ldcs_process_data_t *procdata, node_peer_t peer, ldcs_
       case LDCS_MSG_ALIVE_REQ:
       case LDCS_MSG_ALIVE_RESP:
          return handle_alive_msg(procdata, msg);
+      case LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS:
+         return handle_cachepath_consensus(procdata, msg);
       default:
          err_printf("Received unexpected message from node: %d\n", (int) msg->header.type);
          assert(0);
@@ -2945,6 +2952,79 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs
    }
 }
 
+/**
+ * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which of the locations, commpaths, and cachepaths are
+ * available across all of the servers.
+ */
+
+static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_message_t *msg){
+
+    int num_children = ldcs_audit_server_md_get_num_children(procdata);
+
+    if (num_children) {
+        spindle_broadcast(procdata, msg);
+        msgbundle_force_flush(procdata);
+    }
+
+    ldcs_audit_server_md_consensus(procdata, msg);
+
+    if( procdata->cachepath_bitidx == 0 ){
+       err_printf("No valid cachepath path available.  Falling back to \"location\" path (%s).\n", procdata->location);
+       procdata->cachepath = procdata->location;
+    }else{
+        // ldcs_audit_server_filemngt_init() does it's own realize() pass.
+        getValidCachePathByIndex( procdata->cachepath_bitidx,
+                &procdata->cachepath,
+                &procdata->parsed_cachepath,
+                &procdata->symbolic_cachepath);
+    }
+
+    debug_printf3("Initializing file cache location %s\n", procdata->location);
+    ldcs_audit_server_filemngt_init(procdata->cachepath);
+
+    test_printf("<internal> cachepath=%s\n", procdata->cachepath);
+    return 0;
+}
+
+/**
+ * Handle LDCS_MSG_CHOSEN_CACHEPATH_REQUEST
+ */
+static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc){
+   ldcs_message_t msg;
+   int connid;
+   ldcs_client_t *client;
+
+   assert(nc != -1);
+   client = procdata->client_table + nc;
+   connid = client->connid;
+   if (client->state != LDCS_CLIENT_STATUS_ACTIVE || connid < 0)
+      return 0;
+
+
+   msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH;
+
+   msg.header.len = strlen(procdata->cachepath) + 1;
+   msg.data = procdata->cachepath;
+   ldcs_send_msg(connid, &msg);
+   procdata->server_stat.clientmsg.cnt++;
+   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
+
+   msg.header.len = strlen(procdata->parsed_cachepath) + 1;
+   msg.data = procdata->parsed_cachepath;
+   ldcs_send_msg(connid, &msg);
+   procdata->server_stat.clientmsg.cnt++;
+   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
+
+   msg.header.len = strlen(procdata->symbolic_cachepath) + 1;
+   msg.data = procdata->symbolic_cachepath;
+   ldcs_send_msg(connid, &msg);
+   procdata->server_stat.clientmsg.cnt++;
+   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
+
+   return 0;
+}
+
+
 /**
  * Handle alive message, which is a broadcast/response ping through all servers
  */
diff --git a/src/server/auditserver/ldcs_audit_server_md_cobo.c b/src/server/auditserver/ldcs_audit_server_md_cobo.c
index 08c9b952..d8b5442f 100644
--- a/src/server/auditserver/ldcs_audit_server_md_cobo.c
+++ b/src/server/auditserver/ldcs_audit_server_md_cobo.c
@@ -401,3 +401,9 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata)
    cobo_get_num_childs(&num_childs);
    return num_childs;
 }
+
+void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg){
+    if( msg->header.type == LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS ){
+        cobo_allreduce( &ldcs_process_data->cachepath_bitidx, COBO_OP_BITWISE_AND );
+    }
+}
diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c
index 2bc2455d..b6beb56d 100644
--- a/src/server/comlib/ldcs_api_util.c
+++ b/src/server/comlib/ldcs_api_util.c
@@ -91,6 +91,9 @@ char* _message_type_to_str (ldcs_message_ids_t type) {
       STR_CASE(LDCS_MSG_PICKONE_RESP);
       STR_CASE(LDCS_MSG_ALIVE_REQ);
       STR_CASE(LDCS_MSG_ALIVE_RESP);      
+      STR_CASE(LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS);
+      STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH_REQUEST);
+      STR_CASE(LDCS_MSG_CHOSEN_CACHEPATH);
       STR_CASE(LDCS_MSG_UNKNOWN);
    }
    return "unknown";

From 2020634e2943a5941767c6380b695f76e430db54 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:53:25 -0700
Subject: [PATCH 05/33] Cachepath:  Adds cobo_allreduce()

---
 src/cobo/cobo.c      | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/cobo/ldcs_cobo.h | 16 ++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index 785a5d03..ad2ea878 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -1434,6 +1434,48 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf)
     return COBO_SUCCESS;
 }
 
+int cobo_allreduce( int64_t *pval, cobo_op_t op ){
+
+    /* if i have any children, receive their data */
+    int64_t child_val;
+    for(int i=cobo_num_child-1; i>=0; i--) {
+        /* read int64_t from child */
+        if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) {
+            err_printf("Reducing data from child (rank %d) failed\n", cobo_child[i]);
+            exit(1);
+        }
+
+        /* compare child's val to our current val */
+        switch( op ){
+            case COBO_OP_MIN:           if( child_val < *pval ) *pval = child_val; break;
+            case COBO_OP_MAX:           if( child_val > *pval ) *pval = child_val; break;
+            case COBO_OP_BITWISE_AND:   *pval &= child_val; break;
+            case COBO_OP_BITWISE_OR:    *pval |= child_val; break;
+            case COBO_OP_LOGICAL_AND:   *pval = *pval && child_val; break;
+            case COBO_OP_LOGICAL_OR:    *pval = *pval || child_val; break;
+            case COBO_OP_SUM:           *pval += child_val; break;
+            case COBO_OP_NOOP:          break;
+            default:
+                err_printf("Illegal op (%d).  Ignoring.\n", op);
+                break;
+        }
+    }
+
+    /* forward data to parent if we're not rank 0, otherwise set the recvbuf */
+    if (cobo_me != 0) {
+        /* not the root, so forward our reduction result to our parent */
+        if (cobo_write_fd(cobo_parent_fd, pval, sizeof(*pval)) < 0) {
+            err_printf("Sending reduced data to parent failed\n");
+            exit(1);
+        }
+    }
+
+    /* broadcast result of reduction from rank 0 to all tasks */
+    cobo_bcast_tree(pval, sizeof(int64_t));
+
+    return COBO_SUCCESS;
+}
+
 /* provide list of ports and number of ports as input, get number of tasks and my rank as output */
 int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int* num_ranks)
 {
diff --git a/src/cobo/ldcs_cobo.h b/src/cobo/ldcs_cobo.h
index edacd4b1..30cc9673 100644
--- a/src/cobo/ldcs_cobo.h
+++ b/src/cobo/ldcs_cobo.h
@@ -55,6 +55,7 @@ extern "C" {
 #define cobo_allgather COMBINE(COBO_NAMESPACE, cobo_allgather)
 #define cobo_alltoall  COMBINE(COBO_NAMESPACE, cobo_alltoall )
 #define cobo_allgather_str COMBINE(COBO_NAMESPACE, cobo_allgather_str)
+#define cobo_allreduce COMBINE(COBO_NAMESPACE, cobo_allreduce)
 #define cobo_server_open COMBINE(COBO_NAMESPACE, cobo_server_open)
 #define cobo_server_close COMBINE(COBO_NAMESPACE, cobo_server_close)
 #define cobo_server_get_root_socket COMBINE(COBO_NAMESPACE, cobo_server_get_root_socket)
@@ -67,6 +68,19 @@ extern "C" {
 #define cobo_register_preconnect_cb COMBINE(COBO_NAMESPACE, cobo_register_preconnect_cb)
 #endif
 
+// Used for cobo_allreduce().
+typedef enum{
+    COBO_OP_MIN,
+    COBO_OP_MAX,
+    COBO_OP_BITWISE_AND,
+    COBO_OP_BITWISE_OR,
+    COBO_OP_LOGICAL_AND,
+    COBO_OP_LOGICAL_OR,
+    COBO_OP_SUM,
+    COBO_OP_NOOP,
+    NUM_COBO_OP
+} cobo_op_t;
+
 /*
  * ==========================================================================
  * ==========================================================================
@@ -128,6 +142,8 @@ int cobo_alltoall (void* sendbuf, int sendcount, void* recvbuf);
  */
 int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf);
 
+int cobo_allreduce(int64_t *pval, cobo_op_t op);
+
 /*
  * ==========================================================================
  * ==========================================================================

From dfdf1665aa0c2d6e8d56b904aac0182a4fc24ee2 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:54:01 -0700
Subject: [PATCH 06/33] Cachepath:  Adds parameters to config_mgr

---
 src/fe/startup/config_mgr.cc | 23 +++++++++++++++++++++++
 src/fe/startup/config_mgr.h  |  4 +++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index 36a30b20..eb710b36 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -56,6 +56,12 @@ using namespace std;
 #define SPINDLE_LOC_STR "$TMPDIR"
 #endif
 
+#if defined(CACHEPATHS)
+#define SPINDLE_CACHEPATHS_STR CACHEPATHS
+#else
+#define SPINDLE_CACHEPATHS_STR "$TMPDIR"
+#endif
+
 #if defined(SPINDLE_LOCAL_PREFIX)
 #define SPINDLE_LOCAL_PREFIX_STR SPINDLE_LOCAL_PREFIX
 #else
@@ -269,6 +275,8 @@ void initOptionsList()
      "Strip debug and symbol information from binaries before distributing them." },
    { confLocation, "location", shortLocation, groupMisc, cvString, {}, SPINDLE_LOC_STR,
      "Back-end directory for storing relocated files.  Should be a non-shared location such as a ramdisk." },
+   { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR,
+     "Colon-separated list of candidate paths for cached libraries."},
    { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false",
      "Don't remove local file cache after execution." },
    { confDisableLogging, "disable-logging", shortDisableLogging, groupMisc, cvBool, {}, DISABLE_LOGGING_STR,
@@ -740,6 +748,21 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
             args.location = strdup(loc.c_str());
             break;
          }
+         case confCachePaths:{
+            // Paramemter values are colon-separated lists of paths.
+            // Append "/spindle.$NUMBER" to each path in the list.
+            string paths = strresult;
+            size_t idx = paths.find(":");
+            string number_var_with_colon("/spindle.$NUMBER:");
+            string number_var_without_colon("/spindle.$NUMBER");
+            while( idx != string::npos ){
+               paths.replace(idx, 1, number_var_with_colon);
+               idx = paths.find(":", idx + number_var_with_colon.size());
+            };
+            paths += number_var_without_colon;
+            args.candidate_cachepaths = strdup(paths.c_str());
+            break;
+         }
          case confCachePrefix:
          case confPythonPrefix:
             if (args.pythonprefix)
diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index 8e70daa6..27be1ae8 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -30,6 +30,7 @@ enum SpindleConfigID {
    confPort,
    confNumPorts,
    confLocation,
+   confCachePaths,
    confCachePrefix,
    confPythonPrefix,
    confLocalPrefix,
@@ -125,7 +126,8 @@ enum CmdlineShortOptions {
    shortSpindleLevel = 296,
    shortLocalPrefix = 297,
    shortExecExcludes = 298,
-   shortPatchLdso
+   shortPatchLdso,
+   shortCachePaths,
 };
 
 enum CmdlineGroups {

From 38bacd3ae153f2a7b03f8614659fbcff0723d61d Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:55:39 -0700
Subject: [PATCH 07/33] Cachepath:  adds flux parameter support

---
 src/flux/flux-spindle.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c
index 6c99341a..65de5ed9 100644
--- a/src/flux/flux-spindle.c
+++ b/src/flux/flux-spindle.c
@@ -382,7 +382,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL;
     const char *followfork = NULL, *preload = NULL, *level = NULL;
     const char *pyprefix = NULL, *location = NULL;
-    char *numafiles = NULL;
+    char *numafiles = NULL, *cachepaths = NULL;
 
     if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0)
         return -1;
@@ -404,7 +404,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
      *  supplied by the user, but not unpacked (This handles typos, etc).
      */
     if (json_unpack_ex (opts, &error, JSON_STRICT,
-                        "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s}",
+                        "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s s?s}",
                         "noclean", &noclean,
                         "nostrip", &nostrip,
                         "push", &push,
@@ -419,7 +419,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
                         "numa", &numa,
                         "numa-files", &numafiles,
                         "preload", &preload,
-                        "level", &level) < 0)
+                        "level", &level,
+                        "cachepaths", &cachepaths) < 0)
        logerrno_printf_and_return(1, "Error in spindle option: %s\n", error.text);
 
     if (noclean)
@@ -462,6 +463,9 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
         free (ctx->params.pythonprefix);
         ctx->params.pythonprefix = tmp;
     }
+    if( cachepaths ){
+        ctx->params.candidate_cachepaths = cachepaths;
+    }
     if (location) {
        ctx->params.location = (char *) location;
     }

From 6d34e1848f9561e0aef7a896872c0d3db725f4cd Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:56:05 -0700
Subject: [PATCH 08/33] Cachepath:  Adds logging support.

---
 src/logging/spindle_logd.cc | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/logging/spindle_logd.cc b/src/logging/spindle_logd.cc
index 53eb5dc0..45d58a7d 100644
--- a/src/logging/spindle_logd.cc
+++ b/src/logging/spindle_logd.cc
@@ -202,7 +202,7 @@ class TestVerifier
    std::vector<std::string> err_strings;
    std::set<std::pair<int, std::string> > target_libs;
    std::set<std::pair<int, std::string> > libs_loaded;
-   char *location;
+   char *cachepath;
 
    void logerror(std::string s)
    {
@@ -249,7 +249,8 @@ class TestVerifier
          tmp_s = getenv("TEMPDIR");
       if (!tmp_s)
          tmp_s = "/tmp";
-      location = strdup(tmp_s);
+      // These are reasonable fallbacks that should be replaced via <internal> messages, below.
+      cachepath  = strdup(tmp_s);
    }
 
    ~TestVerifier()
@@ -268,7 +269,7 @@ class TestVerifier
           strstr(filename, "retzero") == NULL &&
           strstr(filename, ".py") == NULL)
          return true;
-      bool is_from_temp = (strstr(filename, location) != NULL) && (strncmp(filename, "/__not_exist", 12) != 0);
+      bool is_from_temp = (strstr(filename, cachepath) != NULL) && (strncmp(filename, "/__not_exist", 12) != 0);
       bool is_local_test = strstr(filename, "liblocal") != NULL;
 
       if (is_from_temp && !is_local_test && ret_code == -1) {
@@ -294,12 +295,12 @@ class TestVerifier
       char buffer[4096];
       int ret;
 
-      if (strstr(s, "<internal> location=" ) == s ){
-        free( location );
-        const char *loc_start = strstr( s, "=") + 1;
-        size_t loc_len = strlen( loc_start );
-        location = strdup( loc_start );
-        location[ loc_len - 1 ] = '\0'; // Remove trailing '\n'.
+      if (strstr(s, "<internal> cachepath=" ) == s ){
+        free( cachepath );
+        const char *cachepath_start = strstr( s, "=") + 1;
+        size_t cachepath_len = strlen( cachepath_start );
+        cachepath = strdup( cachepath_start );
+        cachepath[ cachepath_len - 1 ] = '\0'; // Remove trailing '\n'.
       }
       if (strstr(s, "open(") == s) {
          const char *first_quote, *last_quote, *equals;

From 69f03deeb031adc629bf2698327bebd71599704c Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:56:34 -0700
Subject: [PATCH 09/33] Cachepath:  Removes out-of-root cleanup checks.

---
 src/server/auditserver/cleanup_proc.cc |  4 ++--
 testsuite/test_driver.c                | 23 -----------------------
 2 files changed, 2 insertions(+), 25 deletions(-)

diff --git a/src/server/auditserver/cleanup_proc.cc b/src/server/auditserver/cleanup_proc.cc
index c23a9f7d..a3d3ddcc 100644
--- a/src/server/auditserver/cleanup_proc.cc
+++ b/src/server/auditserver/cleanup_proc.cc
@@ -72,7 +72,8 @@ static void rmDirSet(const set<string> &dirs, const char *prefix_dir)
             continue;
 
          if (strncmp(prefix_dir, componentpath.c_str(), prefix_size) != 0) {
-            err_printf("Tried to clean a file %s that wasn't in our prefix %s\n", componentpath.c_str(), prefix_dir);
+            // We have multiple directory roots.  Not a problem if the directory
+            // we're looking for isn't in this one.
             continue;
          }
          unlink(componentpath.c_str());
@@ -83,7 +84,6 @@ static void rmDirSet(const set<string> &dirs, const char *prefix_dir)
    sort(ordered_dirs.begin(), ordered_dirs.end(), longest_str_first);
    for (vector<string>::iterator i = ordered_dirs.begin(); i != ordered_dirs.end(); i++) {
       if (strncmp(prefix_dir, i->c_str(), prefix_size) != 0) {
-         err_printf("Tried to rmdir directory %s that wasn't in our prefix %s\n", i->c_str(), prefix_dir);
          continue;
       }      
       rmdir(i->c_str());
diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index 13f63fea..a0b46a75 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1169,27 +1169,6 @@ static char* getCacheLocation(char *env_var)
    return strdup(last_slash);
 }
 
-static int checkLinkForLeak(const char *path, const char *spindle_loc)
-{
-   char link_target[4096];
-   int result, error;
-   memset(link_target, 0, sizeof(link_target));
-
-   result = readlink(path, link_target, sizeof(link_target));
-   if (result == -1) {
-      error = errno;
-      err_printf("Failed to read link %s: %s\n", path, strerror(error));
-      return -1;
-   }
-
-   if (strstr(link_target, spindle_loc)) {
-      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc);
-      return -1;
-   }
-
-   return 0;
-}
-
 static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc)
 {
    if (strstr(path, spindle_loc)) {
@@ -1286,9 +1265,7 @@ void check_for_path_leaks()
          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
-      checkLinkForLeak(path, spindle_loc);
    }
-   checkLinkForLeak("/proc/self/exe", spindle_loc);
 
    /**
     * Check link_maps for leaked spindle paths

From 36540bec1006bef965fb72475357e7e15a79ce21 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 3 Oct 2025 09:57:06 -0700
Subject: [PATCH 10/33] Cachepath:  Set of small, miscellaneous patches.

---
 src/include/spindle_launch.h                       | 5 +++++
 src/server/auditserver/ldcs_audit_server_md.h      | 3 +++
 src/server/auditserver/ldcs_audit_server_process.c | 5 +++--
 src/server/auditserver/ldcs_audit_server_process.h | 5 +++++
 src/server/startup/spindle_be.cc                   | 4 +++-
 5 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 47a4d92e..5476734d 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -126,6 +126,11 @@ typedef struct {
    /* The local-disk location where Spindle will store its cache */
    char *location;
 
+   /* Path[s] for cached libraries. */
+   char *candidate_cachepaths;  /* Colon-separated list of candidate paths (max 64) */
+   char *chosen_cachepath;      /* The consensus path (same across all nodes). */
+   uint64_t cachepath_bitidx;   /* Bit index used by allReduce() to arrive at consensus. */
+
    /* Colon-seperated list of directories where Python is installed */
    char *pythonprefix;
 
diff --git a/src/server/auditserver/ldcs_audit_server_md.h b/src/server/auditserver/ldcs_audit_server_md.h
index eb5bf9f6..ba7943e2 100644
--- a/src/server/auditserver/ldcs_audit_server_md.h
+++ b/src/server/auditserver/ldcs_audit_server_md.h
@@ -107,6 +107,9 @@ int ldcs_audit_server_md_broadcast_noncontig(ldcs_process_data_t *ldcs_process_d
 int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata);
 
 int ldcs_audit_server_md_is_parent(node_peer_t peer);
+
+void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg);
+
    
 #if defined(__cplusplus)
 }
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index 312095be..dca91c20 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -142,6 +142,9 @@ int ldcs_audit_server_process(spindle_args_t *args)
 
    debug_printf3("Initializing server data structures\n");
    ldcs_process_data.location = args->location;
+   ldcs_process_data.cachepaths = args->candidate_cachepaths;
+   ldcs_process_data.cachepath = args->chosen_cachepath;
+   ldcs_process_data.cachepath_bitidx = args->cachepath_bitidx;
    ldcs_process_data.number = args->number;
    ldcs_process_data.pythonprefix = args->pythonprefix;
    ldcs_process_data.localprefix = args->local_prefixes;
@@ -191,8 +194,6 @@ int ldcs_audit_server_process(spindle_args_t *args)
    }
    ldcs_process_data.server_stat.hostname=ldcs_process_data.hostname;
 
-   debug_printf3("Initializing file cache location %s\n", ldcs_process_data.location);
-   ldcs_audit_server_filemngt_init(ldcs_process_data.location);
    if (ldcs_process_data.opts & OPT_PROCCLEAN)
       init_cleanup_proc(ldcs_process_data.location);
 
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 18b3320a..9ba1675e 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -126,6 +126,11 @@ struct ldcs_process_data_struct
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
   char *location;
+  char *cachepaths;
+  char *cachepath;
+  char *symbolic_cachepath;
+  char *parsed_cachepath;
+  int64_t cachepath_bitidx;
   char *hostname;
   char *pythonprefix;
   char *localprefix;
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 7493c020..4f583756 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -59,6 +59,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size)
    unpack_param(args->startup_type, buf, pos);
    unpack_param(args->shm_cache_size, buf, pos);
    unpack_param(args->location, buf, pos);
+   unpack_param(args->candidate_cachepaths, buf, pos);
    unpack_param(args->pythonprefix, buf, pos);
    unpack_param(args->preloadfile, buf, pos);
    unpack_param(args->bundle_timeout_ms, buf, pos);
@@ -152,7 +153,8 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    debug_printf("Translated location from %s to %s\n", args.location, new_location);
    free(args.location);
    args.location = new_location;
-   test_printf("<internal> location=%s\n", args.location);
+
+   determineValidCachePaths( &args.cachepath_bitidx, args.candidate_cachepaths, args.number);
 
    result = ldcs_audit_server_process(&args);
    if (result == -1) {

From 0eb779ad3fe1d2d8e4bbd96d1c157cc616412996 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Tue, 21 Oct 2025 12:21:37 -0700
Subject: [PATCH 11/33] Fixes per Matt's comments.

---
 src/client/client/intercept_exec.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 8480b75e..27928d35 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -196,7 +196,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
    if (envp) {
       debug_printf2("Propogating spindle environment by copying it to new envp list\n");
       for (cur = (char **) envp; *cur; cur++, orig_size++);
-      new_size = orig_size + 20;
+      new_size = orig_size + 9;
       newenv = (char **) malloc(new_size * sizeof(char*));
       
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   

From 61ba31cb937b6e55e11b723cbcf2f1f02a69c88f Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 22 Oct 2025 12:20:35 -0700
Subject: [PATCH 12/33] Single source of truth for client cachepath.

Previously,

chosen_realized_cachepath was copied into
set_intercept_readlink_cachepath()

chosen_realized_cachepath and chosen_parsed_cachepath were copied
into set_should_intercept_cachepath()

This PR removes both setter functions and makes the original
pointers global.
---
 src/client/client/client.c             |  7 ++-----
 src/client/client/intercept.h          |  1 -
 src/client/client/intercept_readlink.c | 12 +++---------
 src/client/client/should_intercept.c   | 17 +++++------------
 src/client/client/should_intercept.h   |  1 -
 5 files changed, 10 insertions(+), 28 deletions(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index d7827d4b..f803e75b 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -72,7 +72,7 @@ static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
 static char *location;
-static char *chosen_realized_cachepath, *chosen_parsed_cachepath, *chosen_symbolic_cachepath;
+char *chosen_realized_cachepath, *chosen_parsed_cachepath;
 number_t number;
 static int have_stat_patches;
 
@@ -261,10 +261,7 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
-   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, &chosen_symbolic_cachepath );
-   set_should_intercept_cachepath(   chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
-   set_intercept_readlink_cachepath( chosen_realized_cachepath, chosen_parsed_cachepath, chosen_symbolic_cachepath );
-   
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, NULL);
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
 
diff --git a/src/client/client/intercept.h b/src/client/client/intercept.h
index aae968f7..4ace2328 100644
--- a/src/client/client/intercept.h
+++ b/src/client/client/intercept.h
@@ -89,7 +89,6 @@ int execvpe_wrapper(const char *path, char *const argv[], const char *envp[]);
 pid_t vfork_wrapper();
 char *dlerror_wrapper();
 
-void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 ssize_t readlink_wrapper(const char *path, char *buf, size_t bufsiz);
 ssize_t readlinkat_wrapper(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
diff --git a/src/client/client/intercept_readlink.c b/src/client/client/intercept_readlink.c
index 2799baee..37d440e4 100644
--- a/src/client/client/intercept_readlink.c
+++ b/src/client/client/intercept_readlink.c
@@ -31,23 +31,17 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-static char *cachepath;
-
-void set_intercept_readlink_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
-    cachepath = chosen_realized_cachepath;
-    chosen_parsed_cachepath = chosen_parsed_cachepath;
-    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
-}
 
 static int fix_local_readlink(char *buf, size_t bufsiz)
 {
    char spindle_id[32];
    int cachepath_len, result;
    char tmp[MAX_PATH_LEN+1];
+   extern char *chosen_realized_cachepath;
 
-   cachepath_len = strlen(cachepath);   
+   cachepath_len = strlen(chosen_realized_cachepath);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number);
-   if (strstr(buf, spindle_id) && strncmp(cachepath, buf, cachepath_len) == 0) {
+   if (strstr(buf, spindle_id) && strncmp(chosen_realized_cachepath, buf, cachepath_len) == 0) {
       debug_printf2("readlink received spindle cache path %s. Translating\n", buf);
       result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp);
       if (result == -1)
diff --git a/src/client/client/should_intercept.c b/src/client/client/should_intercept.c
index 3a348d3d..cee4e43c 100644
--- a/src/client/client/should_intercept.c
+++ b/src/client/client/should_intercept.c
@@ -29,27 +29,20 @@
 #include "spindle_debug.h"
 
 extern int relocate_spindleapi();
-static char *cachepath, *orig_cachepath;
-
-void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath ){
-    cachepath = chosen_realized_cachepath;
-    orig_cachepath = chosen_parsed_cachepath;
-    chosen_symbolic_cachepath = chosen_symbolic_cachepath;
-}
-
 
 int is_in_spindle_cache(const char *pathname)
 {
    static int cachepath_size = 0;
    static int orig_cachepath_size = 0;
+   extern char *chosen_realized_cachepath, *chosen_parsed_cachepath;
    if (!cachepath_size) {
-      cachepath_size = strlen(cachepath);
+      cachepath_size = strlen(chosen_realized_cachepath);
    }
    if (!orig_cachepath_size) {
-      orig_cachepath_size = strlen(orig_cachepath);
+      orig_cachepath_size = strlen(chosen_parsed_cachepath);
    }
-   return ((strncmp(pathname, cachepath, cachepath_size) == 0) ||
-           (strncmp(pathname, orig_cachepath, orig_cachepath_size) == 0));
+   return ((strncmp(pathname, chosen_realized_cachepath, cachepath_size) == 0) ||
+           (strncmp(pathname, chosen_parsed_cachepath, orig_cachepath_size) == 0));
 }
 
 extern int is_local_prefix(const char *path, char **cached_local_prefixes);
diff --git a/src/client/client/should_intercept.h b/src/client/client/should_intercept.h
index 6a545913..f6a9b510 100644
--- a/src/client/client/should_intercept.h
+++ b/src/client/client/should_intercept.h
@@ -27,7 +27,6 @@
 #define EXCL_OPEN 2
 #define ERR_CALL 3
 
-void set_should_intercept_cachepath( char *chosen_realized_cachepath, char *chosen_parsed_cachepath, char *chosen_symbolic_cachepath );
 int open_filter(const char *fname, int flags);
 int fopen_filter(const char *fname, const char *flags);
 int exec_filter(const char *fname);

From 78ee5ff02f3f902b26a3655a78f2a89d8282c5d7 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 22 Oct 2025 14:39:23 -0700
Subject: [PATCH 13/33] Comments the cachepath variables.

---
 .../auditserver/ldcs_audit_server_process.h       | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 9ba1675e..f9f4c30c 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -125,12 +125,15 @@ struct ldcs_process_data_struct
   int exit_readys_recvd;
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
-  char *location;
-  char *cachepaths;
-  char *cachepath;
-  char *symbolic_cachepath;
-  char *parsed_cachepath;
-  int64_t cachepath_bitidx;
+  char *location;               /* Single user-specified path for fifo, daemons, etc. */
+                                /*   (Everything except the cachepath.) */
+  char *cachepaths;             /* Up to 64 colon-separated list of candidate cachepaths. */
+  char *cachepath;              /* The earliest path in the list available to all servers. */
+                                /*   (Environment variables replaced, symbolic links realized.) */
+  char *symbolic_cachepath;     /* The original representation of the cachepath. */
+  char *parsed_cachepath;       /* The cachepath with environment variables replaced. */
+                                /*   (Symbolic links, if any, remain.) */
+  int64_t cachepath_bitidx;     /* Bit index of valid cachepaths on a given server. */
   char *hostname;
   char *pythonprefix;
   char *localprefix;

From f9aa472a55bb411c796107043a26d8abca5997e2 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 13:07:21 -0700
Subject: [PATCH 14/33] Removes internal vars from spindle_launch.h

Removes chosen_cachepath and cachepath_bitindex from
  spindle_launch.h

Updates initialization of matching variables in ldcs_process_data.

determineValidCachePaths() moved from spindle_be.cc to
  ldcs_audit_server_process.c to get ldcs_process_data visibility.

Added #include "parseloc.h" to ldcs_audit_server_process.c to get
  declaration of determineValidCachePaths().

Relocated "parseloc.h" to src/util so ldcs_audit_server_process.c
  could find it.

Trued up signedness of types caused my making "parseloc.h" more
  visible, e.g., cachepath_bitidx is now uint64_t everywhere.
---
 src/cobo/cobo.c                                    | 2 +-
 src/cobo/ldcs_cobo.h                               | 2 +-
 src/include/spindle_launch.h                       | 2 --
 src/server/auditserver/ldcs_audit_server_process.c | 9 +++++++--
 src/server/auditserver/ldcs_audit_server_process.h | 2 +-
 src/server/startup/spindle_be.cc                   | 2 --
 src/{client/beboot => utils}/parseloc.h            | 2 +-
 7 files changed, 11 insertions(+), 10 deletions(-)
 rename src/{client/beboot => utils}/parseloc.h (91%)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index ad2ea878..59c2f809 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -1434,7 +1434,7 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf)
     return COBO_SUCCESS;
 }
 
-int cobo_allreduce( int64_t *pval, cobo_op_t op ){
+int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
 
     /* if i have any children, receive their data */
     int64_t child_val;
diff --git a/src/cobo/ldcs_cobo.h b/src/cobo/ldcs_cobo.h
index 30cc9673..fafbda6a 100644
--- a/src/cobo/ldcs_cobo.h
+++ b/src/cobo/ldcs_cobo.h
@@ -142,7 +142,7 @@ int cobo_alltoall (void* sendbuf, int sendcount, void* recvbuf);
  */
 int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf);
 
-int cobo_allreduce(int64_t *pval, cobo_op_t op);
+int cobo_allreduce(uint64_t *pval, cobo_op_t op);
 
 /*
  * ==========================================================================
diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 5476734d..81c0728e 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -128,8 +128,6 @@ typedef struct {
 
    /* Path[s] for cached libraries. */
    char *candidate_cachepaths;  /* Colon-separated list of candidate paths (max 64) */
-   char *chosen_cachepath;      /* The consensus path (same across all nodes). */
-   uint64_t cachepath_bitidx;   /* Bit index used by allReduce() to arrive at consensus. */
 
    /* Colon-seperated list of directories where Python is installed */
    char *pythonprefix;
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index dca91c20..566cad01 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -37,6 +37,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "msgbundle.h"
 #include "exitnote.h"
 #include "cleanup_proc.h"
+#include "parseloc.h"
 
 //#define GPERFTOOLS
 #if defined(GPERFTOOLS)
@@ -143,8 +144,8 @@ int ldcs_audit_server_process(spindle_args_t *args)
    debug_printf3("Initializing server data structures\n");
    ldcs_process_data.location = args->location;
    ldcs_process_data.cachepaths = args->candidate_cachepaths;
-   ldcs_process_data.cachepath = args->chosen_cachepath;
-   ldcs_process_data.cachepath_bitidx = args->cachepath_bitidx;
+   ldcs_process_data.cachepath = NULL;
+   ldcs_process_data.cachepath_bitidx = 0;
    ldcs_process_data.number = args->number;
    ldcs_process_data.pythonprefix = args->pythonprefix;
    ldcs_process_data.localprefix = args->local_prefixes;
@@ -230,6 +231,10 @@ int ldcs_audit_server_process(spindle_args_t *args)
    if (fd != -1) {
       ldcs_listen_register_fd(fd, serverid, forceExitCB, (void *) &ldcs_process_data);
    }
+   determineValidCachePaths(
+           &ldcs_process_data.cachepath_bitidx,
+           ldcs_process_data.cachepaths,
+           ldcs_process_data.number );
    return 0;
 }  
 
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index f9f4c30c..1495cebd 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -133,7 +133,7 @@ struct ldcs_process_data_struct
   char *symbolic_cachepath;     /* The original representation of the cachepath. */
   char *parsed_cachepath;       /* The cachepath with environment variables replaced. */
                                 /*   (Symbolic links, if any, remain.) */
-  int64_t cachepath_bitidx;     /* Bit index of valid cachepaths on a given server. */
+  uint64_t cachepath_bitidx;    /* Bit index of valid cachepaths on a given server. */
   char *hostname;
   char *pythonprefix;
   char *localprefix;
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 4f583756..733d3244 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -154,8 +154,6 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    free(args.location);
    args.location = new_location;
 
-   determineValidCachePaths( &args.cachepath_bitidx, args.candidate_cachepaths, args.number);
-
    result = ldcs_audit_server_process(&args);
    if (result == -1) {
       err_printf("Error in ldcs_audit_server_process\n");
diff --git a/src/client/beboot/parseloc.h b/src/utils/parseloc.h
similarity index 91%
rename from src/client/beboot/parseloc.h
rename to src/utils/parseloc.h
index 1731906a..a99409c3 100644
--- a/src/client/beboot/parseloc.h
+++ b/src/utils/parseloc.h
@@ -28,7 +28,7 @@ char *parse_location_noerr(char *loc, number_t number);
 char *realize(char *path);
 char **parse_colonsep_prefixes(char *colonsep_list, number_t number);
 int is_local_prefix(const char *path, char **local_prefixes);
-static int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number );
+int validateCandidatePath( char *candidatePath, char **realizedPath, char **parsedPath, char **symbolicPath, number_t number );
 void determineValidCachePaths( uint64_t *validBitIdx, char *origPathList, number_t number );
 void getValidCachePathByIndex( uint64_t validBitIdx, char **realizedCachePath, char **parsedCachePath, char **symbolicCachePath );
 

From ef75381f8ef4f51ef7d4547d7e35a7a43dba0e20 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 14:18:59 -0700
Subject: [PATCH 15/33] Client cachepath message now uses single response.

The three-message-reply response is now a single message with
two strings.  The symbolic version of the cachepath is no longer
communicated as it was not being used.
---
 src/client/client/client.c                    |  2 +-
 src/client/client_comlib/client_api.c         | 31 ++++---------------
 src/client/client_comlib/client_api.h         |  2 +-
 .../auditserver/ldcs_audit_server_handlers.c  | 19 +++---------
 4 files changed, 13 insertions(+), 41 deletions(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index f803e75b..ac8d4b63 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -261,7 +261,7 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
-   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath, NULL);
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath );
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
 
diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 267b8a93..cdff2bb5 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -36,9 +36,9 @@ static struct lock_t comm_lock;
 
 #define COMM_LOCK do { if (lock(&comm_lock) == -1) return -1; } while (0)
 #define COMM_UNLOCK unlock(&comm_lock)
-   
 
-int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath, char **chosen_symbolic_cachepath ){
+
+int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){
    ldcs_message_t message;
    char buffer[MAX_PATH_LEN+1];
    buffer[MAX_PATH_LEN] = '\0';
@@ -59,32 +59,13 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
       assert(0);
    }
+   char *local_crc = strdup( buffer );
+   char *local_cpc = strdup( &buffer[ strlen(local_crc) + 1 ] );
    if( chosen_realized_cachepath ){
-       *chosen_realized_cachepath = strdup( buffer );
-   }
-
-   COMM_LOCK;
-   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
-   COMM_UNLOCK;
-
-   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
-      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+       *chosen_realized_cachepath = local_crc;
    }
    if( chosen_parsed_cachepath ){
-       *chosen_parsed_cachepath = strdup( buffer );
-   }
-
-   COMM_LOCK;
-   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
-   COMM_UNLOCK;
-
-   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
-      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
-   }
-   if( chosen_symbolic_cachepath ){
-       *chosen_symbolic_cachepath = strdup( buffer );
+       *chosen_parsed_cachepath = local_cpc;
    }
 
    return 0;
diff --git a/src/client/client_comlib/client_api.h b/src/client/client_comlib/client_api.h
index 982c4b1c..3d7c41be 100644
--- a/src/client/client_comlib/client_api.h
+++ b/src/client/client_comlib/client_api.h
@@ -42,7 +42,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath);
 int send_dirlists_request(int fd, char **local_result, char **exece_result, char **to_free);
 int send_procmaps_query(int fd, int pid, char *result);
 int send_pickone_query(int fd, char *key, int *result);
-int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath, char **chosen_realized_cachepath );
+int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath);
 
 int get_python_prefix(int fd, char **prefix);
 
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 98c6807a..c8e76d54 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -3003,21 +3003,12 @@ static int handle_chosen_cachepath_request(ldcs_process_data_t *procdata, int nc
 
    msg.header.type = LDCS_MSG_CHOSEN_CACHEPATH;
 
-   msg.header.len = strlen(procdata->cachepath) + 1;
-   msg.data = procdata->cachepath;
-   ldcs_send_msg(connid, &msg);
-   procdata->server_stat.clientmsg.cnt++;
-   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
-
-   msg.header.len = strlen(procdata->parsed_cachepath) + 1;
-   msg.data = procdata->parsed_cachepath;
-   ldcs_send_msg(connid, &msg);
-   procdata->server_stat.clientmsg.cnt++;
-   procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
-
-   msg.header.len = strlen(procdata->symbolic_cachepath) + 1;
-   msg.data = procdata->symbolic_cachepath;
+   msg.header.len = strlen(procdata->cachepath) + 1 + strlen(procdata->parsed_cachepath) + 1;
+   msg.data = calloc( 1, msg.header.len );
+   strcpy( msg.data, procdata->cachepath );
+   strcpy( &msg.data[ strlen(procdata->cachepath)+1 ], procdata->parsed_cachepath );
    ldcs_send_msg(connid, &msg);
+   free( msg.data );
    procdata->server_stat.clientmsg.cnt++;
    procdata->server_stat.clientmsg.time += ldcs_get_time() - client->query_arrival_time;
 

From 3c83f6c4af9caa00c73efaffd256dc3294d336ee Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 14:33:07 -0700
Subject: [PATCH 16/33] Removes assert(0) in network error paths.

---
 src/client/client_comlib/client_api.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index cdff2bb5..4b0f9ded 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -57,7 +57,7 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
 
    if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    char *local_crc = strdup( buffer );
    char *local_cpc = strdup( &buffer[ strlen(local_crc) + 1 ] );
@@ -102,7 +102,7 @@ int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) {
 
    if (message.header.type != LDCS_MSG_FILE_QUERY_ANSWER) {
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    
    if (message.header.len > sizeof(int)) {
@@ -195,7 +195,7 @@ int send_existance_test(int fd, char *path, int *exists)
 
    if (message.header.type != LDCS_MSG_EXISTS_ANSWER || message.header.len != sizeof(uint32_t)) {
       err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
 
    memcpy(exists, buffer, sizeof(*exists));
@@ -232,7 +232,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath)
 
    if (message.header.type != LDCS_MSG_ORIGPATH_ANSWER || message.header.len > MAX_PATH_LEN) {
       err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    strncpy(newpath, buffer, MAX_PATH_LEN+1);
 
@@ -380,7 +380,7 @@ int send_ldso_info_request(int fd, const char *ldso_path, char *result_path)
 
    if (message.header.type != LDCS_MSG_LOADER_DATA_RESP) {
       err_printf("Got unexpected message after ldso req: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    return 0;
 }
@@ -422,7 +422,7 @@ int send_rankinfo_query(int fd, int *mylrank, int *mylsize, int *mymdrank, int *
    if (message.header.type != LDCS_MSG_MYRANKINFO_QUERY_ANSWER || message.header.len != 4*sizeof(int)) {
       err_printf("Received incorrect response to rankinfo query %d\n", message.header.type);
       *mylrank = *mylsize = *mymdrank = *mymdsize = -1;
-      assert(0);
+      return -1;
    }
 
    p = (int *) message.data;
@@ -457,7 +457,7 @@ int send_procmaps_query(int fd, int pid, char *result)
 
    if (message.header.type != LDCS_MSG_PROCMAPS_RESP) {
       err_printf("Received incorrect response to procmaps query %d\n", message.header.type);
-      assert(0);
+      return -1;
    }
 
    memcpy(result, buffer, MAX_PATH_LEN);
@@ -488,7 +488,7 @@ int send_pickone_query(int fd, char *key, int *result)
 
    if (message.header.type != LDCS_MSG_PICKONE_RESP) {
       err_printf("Received incorrect response to procmaps query %d\n", message.header.type);
-      assert(0);
+      return -1;
    }
 
    *result = *((int *) message.data);

From 1134a1c47072b13cf19e036c0d80323b674898e9 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 15:18:40 -0700
Subject: [PATCH 17/33] Renames ldcs_audit_server_md_consensus().

New name is ldcs_audit_server_md_allreduce_AND().

If we get to the point where we're using other allreduce operations
we can solve the problem of duplicating the op list in md-land and
cobo-land.  For now, we're only using one op in md-land, so the
op can go into the function name.
---
 src/server/auditserver/ldcs_audit_server_handlers.c | 2 +-
 src/server/auditserver/ldcs_audit_server_md.h       | 2 +-
 src/server/auditserver/ldcs_audit_server_md_cobo.c  | 6 ++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index c8e76d54..29107bad 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -2966,7 +2966,7 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag
         msgbundle_force_flush(procdata);
     }
 
-    ldcs_audit_server_md_consensus(procdata, msg);
+    ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx );
 
     if( procdata->cachepath_bitidx == 0 ){
        err_printf("No valid cachepath path available.  Falling back to \"location\" path (%s).\n", procdata->location);
diff --git a/src/server/auditserver/ldcs_audit_server_md.h b/src/server/auditserver/ldcs_audit_server_md.h
index ba7943e2..a4640370 100644
--- a/src/server/auditserver/ldcs_audit_server_md.h
+++ b/src/server/auditserver/ldcs_audit_server_md.h
@@ -108,7 +108,7 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata);
 
 int ldcs_audit_server_md_is_parent(node_peer_t peer);
 
-void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg);
+void ldcs_audit_server_md_allreduce_AND( uint64_t *val );
 
    
 #if defined(__cplusplus)
diff --git a/src/server/auditserver/ldcs_audit_server_md_cobo.c b/src/server/auditserver/ldcs_audit_server_md_cobo.c
index d8b5442f..27393a55 100644
--- a/src/server/auditserver/ldcs_audit_server_md_cobo.c
+++ b/src/server/auditserver/ldcs_audit_server_md_cobo.c
@@ -402,8 +402,6 @@ int ldcs_audit_server_md_get_num_children(ldcs_process_data_t *procdata)
    return num_childs;
 }
 
-void ldcs_audit_server_md_consensus(ldcs_process_data_t *ldcs_process_data, ldcs_message_t *msg){
-    if( msg->header.type == LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS ){
-        cobo_allreduce( &ldcs_process_data->cachepath_bitidx, COBO_OP_BITWISE_AND );
-    }
+void ldcs_audit_server_md_allreduce_AND( uint64_t *val ){
+    cobo_allreduce( val, COBO_OP_BITWISE_AND );
 }

From 57eb3fe752797f39a1f7da3721385eee210dd1d5 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 15:27:27 -0700
Subject: [PATCH 18/33] Adds explicit enum values to CmdlineShortOptions.

---
 src/fe/startup/config_mgr.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index 27be1ae8..c2d3cd7e 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -126,8 +126,8 @@ enum CmdlineShortOptions {
    shortSpindleLevel = 296,
    shortLocalPrefix = 297,
    shortExecExcludes = 298,
-   shortPatchLdso,
-   shortCachePaths,
+   shortPatchLdso = 299,
+   shortCachePaths = 300,
 };
 
 enum CmdlineGroups {

From 4a438758f4d8e51c0e4089829547f4ad95f39da3 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 23 Oct 2025 15:38:19 -0700
Subject: [PATCH 19/33] Return instead of exit on network errors.

---
 src/cobo/cobo.c | 58 ++++++++++++++++++++++++-------------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index 59c2f809..65741008 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -160,7 +160,7 @@ static char* cobo_getenv(char* envvar, int type)
     char* str = getenv(envvar);
     if (str == NULL && type == ENV_REQUIRED) {
         err_printf("Missing required environment variable: %s\n", envvar);
-        exit(1);
+        return NULL;
     }
     return str;
 }
@@ -171,7 +171,7 @@ static void* cobo_malloc(size_t n, char* msg)
     void* p = malloc(n);
     if (!p) {
         err_printf("Call to malloc(%lu) failed: %s (%m errno %d)\n", n, msg, errno);
-        exit(1);
+        return NULL;
     }
     return p;
 }
@@ -513,7 +513,7 @@ static int cobo_connect_hostname(char* hostname, int rank)
                       break;
                    case HSHAKE_INTERNAL_ERROR:
                       err_printf("Internal error doing handshake: %s", spindle_handshake_last_error_str());
-                      exit(-1);
+                      return -1;
                       break;
                    case HSHAKE_DROP_CONNECTION:
                       debug_printf3("Handshake said to drop connection\n");
@@ -768,7 +768,7 @@ static int cobo_open_tree()
     if (sockfd < 0) {
         err_printf("Creating parent socket (socket() %m errno=%d)\n",
                    errno);
-        exit(1);
+        return -1;
     }
     setCloseOnExec(sockfd);
 
@@ -817,7 +817,7 @@ static int cobo_open_tree()
     if (!port_is_bound) {
         /* TODO: would like to send an abort back to server */
         err_printf("Failed to open socket on any port\n");
-        exit(1);
+        return -1;
     }
 
     /* accept a connection from parent and receive socket table */
@@ -837,7 +837,7 @@ static int cobo_open_tree()
               break;
            case HSHAKE_INTERNAL_ERROR:
               err_printf("Internal error doing handshake: %s", spindle_handshake_last_error_str());
-              exit(-1);
+              return -1;
               break;
            case HSHAKE_DROP_CONNECTION:
               debug_printf3("Handshake said to drop connection\n");
@@ -907,26 +907,26 @@ static int cobo_open_tree()
     /* read our rank number */
     if (cobo_read_fd(cobo_parent_fd, &cobo_me, sizeof(int)) < 0) {
         err_printf("Receiving my rank from parent failed\n");
-        exit(1);
+        return -1;
     }
 
     /* discover how many ranks are in our world */
     if (cobo_read_fd(cobo_parent_fd, &cobo_nprocs, sizeof(int)) < 0) {
         err_printf("Receiving number of tasks from parent failed\n");
-        exit(1);
+        return -1;
     }
 
     /* read the size of the hostlist (in bytes) */
     if (cobo_read_fd(cobo_parent_fd, &cobo_hostlist_size, sizeof(int)) < 0) {
         err_printf("Receiving size of hostname table from parent failed\n");
-        exit(1);
+        return -1;
     }
 
     /* allocate space for the hostlist and read it in */
     cobo_hostlist = (void*) cobo_malloc(cobo_hostlist_size, "Hostlist data buffer");
     if (cobo_read_fd(cobo_parent_fd, cobo_hostlist, cobo_hostlist_size) < 0) {
         err_printf("Receiving hostname table from parent failed\n");
-        exit(1);
+        return -1;
     }
 
 /*
@@ -969,7 +969,7 @@ static int cobo_open_tree()
         if (cobo_child_fd[i] == -1) {
             err_printf("Failed to connect to child (rank %d) on %s failed\n",
                        c, child_hostname);
-            exit(1);
+            return -1;
         }
 
         /* tell child what rank he is and forward the hostname table to him */
@@ -978,7 +978,7 @@ static int cobo_open_tree()
         if (forward != COBO_SUCCESS) {
             err_printf("Failed to forward hostname table to child (rank %d) on %s failed\n",
                        c, child_hostname);
-            exit(1);
+            return -1;
         }
 
         /* free the child hostname string */
@@ -1033,7 +1033,7 @@ static int cobo_bcast_tree(void* buf, int size)
     if (cobo_me != 0) {
         if (cobo_read_fd(cobo_parent_fd, buf, size) < 0) {
             err_printf("Receiving broadcast data from parent failed\n");
-            exit(1);
+            return -1;
         }
     }
 
@@ -1042,7 +1042,7 @@ static int cobo_bcast_tree(void* buf, int size)
         if (cobo_write_fd(cobo_child_fd[i], buf, size) < 0) {
             err_printf("Broadcasting data to child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
     }
 
@@ -1058,7 +1058,7 @@ int cobo_bcast_down(void* buf, int size)
       if (cobo_write_fd(cobo_child_fd[i], buf, size) < 0) {
          err_printf("Broadcasting data to child (rank %d) failed\n",
                     cobo_child[i]);
-         exit(1);
+         return -1;
       }
    }   
    return rc;
@@ -1080,7 +1080,7 @@ static int cobo_allreduce_max_int_tree(int* sendbuf, int* recvbuf)
         if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(child_val)) < 0) {
             err_printf("Reducing data from child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
 
         /* compare child's max to our current max */
@@ -1094,7 +1094,7 @@ static int cobo_allreduce_max_int_tree(int* sendbuf, int* recvbuf)
         /* not the root, so forward our reduction result to our parent */
         if (cobo_write_fd(cobo_parent_fd, &max_val, sizeof(max_val)) < 0) {
             err_printf("Sending reduced data to parent failed\n");
-            exit(1);
+            return -1;
         }
     } else {
         /* we're the root, got the result, set the recvbuf */
@@ -1129,7 +1129,7 @@ static int cobo_gather_tree(void* sendbuf, int sendcount, void* recvbuf)
         if (cobo_read_fd(cobo_child_fd[i], (char*)bigbuf + offset, sendcount * cobo_child_incl[i]) < 0) {
             err_printf("Gathering data from child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
         offset += sendcount * cobo_child_incl[i];
     }
@@ -1138,7 +1138,7 @@ static int cobo_gather_tree(void* sendbuf, int sendcount, void* recvbuf)
     if (cobo_me != 0) {
         if (cobo_write_fd(cobo_parent_fd, bigbuf, bigcount) < 0) {
             err_printf("Sending gathered data to parent failed\n");
-            exit(1);
+            return -1;
         }
         cobo_free(bigbuf);
     }
@@ -1158,7 +1158,7 @@ static int cobo_scatter_tree(void* sendbuf, int sendcount, void* recvbuf)
         bigbuf = (void*) cobo_malloc(bigcount, "Temporary scatter buffer in cobo_scatter_tree");
         if (cobo_read_fd(cobo_parent_fd, bigbuf, bigcount) < 0) {
             err_printf("Receiving scatter data from parent failed\n");
-            exit(1);
+            return -1;
         }
     }
 
@@ -1169,7 +1169,7 @@ static int cobo_scatter_tree(void* sendbuf, int sendcount, void* recvbuf)
         if (cobo_write_fd(cobo_child_fd[i], (char*)bigbuf + offset, sendcount * cobo_child_incl[i]) < 0) {
             err_printf("Scattering data to child (rank %d) failed\n",
                        cobo_child[i]);
-            exit(1);
+            return -1;
         }
         offset += sendcount * cobo_child_incl[i];
     }
@@ -1257,7 +1257,7 @@ int cobo_bcast(void* buf, int sendcount, int root)
         rc = cobo_bcast_tree(buf, sendcount);
     } else {
         err_printf("Cannot execute bcast from non-zero root\n");
-        exit(1);
+        return -1;
     }
 
     cobo_gettimeofday(&end);
@@ -1283,7 +1283,7 @@ int cobo_gather(void* sendbuf, int sendcount, void* recvbuf, int root)
         rc = cobo_gather_tree(sendbuf, sendcount, recvbuf);
     } else {
         err_printf("Cannot execute gather to non-zero root\n");
-        exit(1);
+        return -1;
     }
 
     cobo_gettimeofday(&end);
@@ -1309,7 +1309,7 @@ int cobo_scatter(void* sendbuf, int sendcount, void* recvbuf, int root)
         rc = cobo_scatter_tree(sendbuf, sendcount, recvbuf);
     } else {
         err_printf("Cannot execute scatter from non-zero root\n");
-        exit(1);
+        return -1;
     }
 
     cobo_gettimeofday(&end);
@@ -1354,7 +1354,7 @@ int cobo_alltoall(void* sendbuf, int sendcount, void* recvbuf)
     int rc = COBO_SUCCESS;
 
     err_printf("Cannot execute alltoall\n");
-    exit(1);
+    return -1;
 
     cobo_gettimeofday(&end);
     debug_printf3("Exiting cobo_alltoall(), took %f seconds for %d procs\n", cobo_getsecs(&end,&start), cobo_nprocs);
@@ -1442,7 +1442,7 @@ int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
         /* read int64_t from child */
         if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) {
             err_printf("Reducing data from child (rank %d) failed\n", cobo_child[i]);
-            exit(1);
+            return -1;
         }
 
         /* compare child's val to our current val */
@@ -1466,7 +1466,7 @@ int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
         /* not the root, so forward our reduction result to our parent */
         if (cobo_write_fd(cobo_parent_fd, pval, sizeof(*pval)) < 0) {
             err_printf("Sending reduced data to parent failed\n");
-            exit(1);
+            return -1;
         }
     }
 
@@ -1524,7 +1524,7 @@ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int*
     cobo_ports = cobo_int_dup(portlist, num_ports);
     if (cobo_ports == NULL) {
         err_printf("Failed to copy port list\n");
-        exit(1);
+        return -1;
     }
 
     /* open the tree */
@@ -1533,7 +1533,7 @@ int cobo_open(uint64_t sessionid, int* portlist, int num_ports, int* rank, int*
     /* need to check that tree opened successfully before returning, so do a barrier */
     if (cobo_barrier() != COBO_SUCCESS) {
         err_printf("Failed to open tree\n");
-        exit(1);
+        return -1;
     }
 
     if (cobo_me == 0) {

From 97ae725c4c204865ab5055b5deef4d50206f0945 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 24 Oct 2025 06:54:47 -0700
Subject: [PATCH 20/33] Apply rename to configuration and parameters.

---
 config.h.in                  |  6 +++---
 configure                    | 17 +++++++++--------
 configure.common.ac          | 12 ++++++------
 src/client/config.h.in       |  6 +++---
 src/client/configure         | 17 +++++++++--------
 src/fe/config.h.in           |  6 +++---
 src/fe/configure             | 17 +++++++++--------
 src/fe/startup/config_mgr.cc | 16 ++++++++--------
 src/fe/startup/config_mgr.h  |  4 ++--
 src/flux/sessionmgr.c        |  8 ++++----
 src/server/config.h.in       |  6 +++---
 src/server/configure         | 17 +++++++++--------
 12 files changed, 68 insertions(+), 64 deletions(-)

diff --git a/config.h.in b/config.h.in
index 6be0335d..f85aef75 100644
--- a/config.h.in
+++ b/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -130,9 +133,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/configure b/configure
index c9e517c8..897d9d5f 100755
--- a/configure
+++ b/configure
@@ -847,7 +847,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1592,7 +1592,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16626,11 +16627,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -16638,7 +16639,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16658,7 +16659,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 
diff --git a/configure.common.ac b/configure.common.ac
index 31a04d4d..b25bc3af 100644
--- a/configure.common.ac
+++ b/configure.common.ac
@@ -21,18 +21,18 @@ AC_ARG_WITH(cachepaths,
             [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
             [CACHEPATHS=${withval}],
             [CACHEPATHS=$DEFAULT_LOC])
-AC_ARG_WITH(localstorage,
-            [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
-            [SPINDLE_LOC=${withval}],
-            [SPINDLE_LOC=$DEFAULT_LOC])
+AC_ARG_WITH(commpath,
+            [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])],
+            [COMMPATH=${withval}],
+            [COMMPATH=$DEFAULT_LOC])
 AC_ARG_WITH(default-local-prefix,
             [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])],
             [SPINDLE_LOCAL_PREFIX=${withval}],
-            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"])
+            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"])
 AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle])
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
-AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping])
 AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
diff --git a/src/client/config.h.in b/src/client/config.h.in
index a42cc0ab..d78a3e7f 100644
--- a/src/client/config.h.in
+++ b/src/client/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -121,9 +124,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/src/client/configure b/src/client/configure
index cbae8891..b7dbffc0 100755
--- a/src/client/configure
+++ b/src/client/configure
@@ -811,7 +811,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1535,7 +1535,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -12598,11 +12599,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -12610,7 +12611,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -12630,7 +12631,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 
diff --git a/src/fe/config.h.in b/src/fe/config.h.in
index 5057ef33..3ac96e42 100644
--- a/src/fe/config.h.in
+++ b/src/fe/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -166,9 +169,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/src/fe/configure b/src/fe/configure
index 29db0f61..d983778a 100755
--- a/src/fe/configure
+++ b/src/fe/configure
@@ -832,7 +832,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1573,7 +1573,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16448,11 +16449,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -16460,7 +16461,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16480,7 +16481,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 
diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index eb710b36..16c4a0f8 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -50,10 +50,10 @@ using namespace std;
 #define SPINDLE_NUM_PORTS_STR "250"
 #endif
 
-#if defined(SPINDLE_LOC)
-#define SPINDLE_LOC_STR SPINDLE_LOC
+#if defined(COMMPATH)
+#define SPINDLE_COMMPATH_STR COMMPATH
 #else
-#define SPINDLE_LOC_STR "$TMPDIR"
+#define SPINDLE_COMMPATH_STR "$TMPDIR"
 #endif
 
 #if defined(CACHEPATHS)
@@ -273,8 +273,8 @@ void initOptionsList()
      "Provides a text file containing a white-space separated list of files that should be relocated to each node before execution begins" },
    { confStrip, "strip", shortStrip, groupMisc, cvBool, {}, "true", 
      "Strip debug and symbol information from binaries before distributing them." },
-   { confLocation, "location", shortLocation, groupMisc, cvString, {}, SPINDLE_LOC_STR,
-     "Back-end directory for storing relocated files.  Should be a non-shared location such as a ramdisk." },
+   { confCommPath, "commpath", shortCommPath, groupMisc, cvString, {}, SPINDLE_COMMPATH_STR,
+     "Back-end directory communication and housekeeping.  Should be a non-shared location such as a ramdisk." },
    { confCachePaths, "cachepaths", shortCachePaths, groupMisc, cvString, {}, SPINDLE_CACHEPATHS_STR,
      "Colon-separated list of candidate paths for cached libraries."},
    { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false",
@@ -743,9 +743,9 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
          case confNumPorts:
             args.num_ports = numresult;
             break;
-         case confLocation: {
-            string loc = strresult + "/spindle.$NUMBER";
-            args.location = strdup(loc.c_str());
+         case confCommPath: {
+            string path = strresult + "/spindle.$NUMBER";
+            args.location = strdup(path.c_str());
             break;
          }
          case confCachePaths:{
diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index c2d3cd7e..ec3c8135 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -29,7 +29,7 @@ enum SpindleConfigID {
    confCmdlineNewgroup,
    confPort,
    confNumPorts,
-   confLocation,
+   confCommPath,
    confCachePaths,
    confCachePrefix,
    confPythonPrefix,
@@ -83,7 +83,7 @@ enum CmdlineShortOptions {
    shortAuditType = 'k',
    shortRelocSO = 'l',
    shortNoClean = 'n',
-   shortLocation = 'o',
+   shortCommPath = 'o',
    shortPush = 'p',
    shortPull = 'q',
    shortPythonPrefix = 'r',
diff --git a/src/flux/sessionmgr.c b/src/flux/sessionmgr.c
index 49324a2f..17027163 100644
--- a/src/flux/sessionmgr.c
+++ b/src/flux/sessionmgr.c
@@ -109,16 +109,16 @@ char **strip_start_from_argv(int argc, char **argv)
 extern char *parse_location(char *loc, int number);
 extern int spindle_mkdir(char *orig_path);
 
-#if !defined(SPINDLE_LOC)
-#error SPINDLE_LOC must be defined in config.h
+#if !defined(COMMPATH)
+#error COMMPATH must be defined in config.h
 #endif
 const char *get_session_dir()
 {
    int result;
    char *dir;
-   dir = parse_location((char *) (SPINDLE_LOC "/spindle_session"), 0);
+   dir = parse_location((char *) (COMMPATH "/spindle_session"), 0);
    if (!dir) {
-      spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", SPINDLE_LOC);
+      spindle_debug_printf(1, "ERROR: Could not parse directory for spindle session location from %s/spindle_session\n", COMMPATH);
       return NULL;
    }
       
diff --git a/src/server/config.h.in b/src/server/config.h.in
index dc9439b3..87fbf990 100644
--- a/src/server/config.h.in
+++ b/src/server/config.h.in
@@ -9,6 +9,9 @@
 /* Colon-separated list of potential back-end cache directories */
 #undef CACHEPATHS
 
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -148,9 +151,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX
diff --git a/src/server/configure b/src/server/configure
index 55e9c6f5..b71bdd82 100755
--- a/src/server/configure
+++ b/src/server/configure
@@ -838,7 +838,7 @@ enable_maintainer_mode
 with_default_port
 with_default_num_ports
 with_cachepaths
-with_localstorage
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1570,7 +1570,8 @@ Optional Packages:
                           communication
   --with-cachepaths=DIR   Colon-separated list of potential back-end cache
                           directories
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16445,11 +16446,11 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
@@ -16457,7 +16458,7 @@ fi
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16477,7 +16478,7 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
 _ACEOF
 
 

From 97566886e9f374a0c3b7922cf473e4590923ae31 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 24 Oct 2025 12:29:41 -0700
Subject: [PATCH 21/33] Renaming location variables/fields to commpath.

---
 src/fe/startup/config_mgr.cc                     |  2 +-
 src/fe/startup/parse_launcher.cc                 |  4 ++--
 src/fe/startup/spindle_fe.cc                     | 10 +++++-----
 src/flux/flux-spindle.c                          |  8 ++++----
 src/include/spindle_launch.h                     |  4 ++--
 .../auditserver/ldcs_audit_server_handlers.c     | 12 ++++++------
 .../auditserver/ldcs_audit_server_process.c      | 14 +++++++-------
 .../auditserver/ldcs_audit_server_process.h      |  2 +-
 src/server/startup/spindle_be.cc                 | 16 ++++++++--------
 9 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index 16c4a0f8..1b9d9ade 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -745,7 +745,7 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
             break;
          case confCommPath: {
             string path = strresult + "/spindle.$NUMBER";
-            args.location = strdup(path.c_str());
+            args.commpath = strdup(path.c_str());
             break;
          }
          case confCachePaths:{
diff --git a/src/fe/startup/parse_launcher.cc b/src/fe/startup/parse_launcher.cc
index f05ba18a..4484e998 100644
--- a/src/fe/startup/parse_launcher.cc
+++ b/src/fe/startup/parse_launcher.cc
@@ -292,7 +292,7 @@ void ModifyArgv::modifyCmdLine()
    snprintf(options_str, 32, "%lu", (unsigned long) params->opts);
    string options(options_str);
    
-   string location(params->location);
+   string commpath(params->commpath);
    
    char number_str[32];
    snprintf(number_str, 32, "%lu", (unsigned long) params->number);
@@ -319,7 +319,7 @@ void ModifyArgv::modifyCmdLine()
       if (p == parser->appExecutableAt()) {
 #if defined(os_bluegene)
          string bg_env_str = parser->getParser()->getBGString();
-         parser->getParser()->addBGEnvStr(n, new_argv, bg_env_str, default_libstr, intercept_libstr, location, number, options, shmcache_size);
+         parser->getParser()->addBGEnvStr(n, new_argv, bg_env_str, default_libstr, intercept_libstr, commpath, number, options, shmcache_size);
 #else
          char **a_argv;
          int a_argc;
diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc
index 43e8c6ba..95bf970b 100644
--- a/src/fe/startup/spindle_fe.cc
+++ b/src/fe/startup/spindle_fe.cc
@@ -71,7 +71,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    buffer_size += sizeof(number_t);
    buffer_size += sizeof(opt_t);
    buffer_size += sizeof(unique_id_t);
-   buffer_size += args->location ? strlen(args->location) + 1 : 1;
+   buffer_size += args->commpath ? strlen(args->commpath) + 1 : 1;
    buffer_size += args->candidate_cachepaths ? strlen(args->candidate_cachepaths) + 1 : 1;
    buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1;
    buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1;
@@ -92,7 +92,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    pack_param(args->use_launcher, buf, pos);
    pack_param(args->startup_type, buf, pos);
    pack_param(args->shm_cache_size, buf, pos);
-   pack_param(args->location, buf, pos);
+   pack_param(args->commpath, buf, pos);
    pack_param(args->candidate_cachepaths, buf, pos);
    pack_param(args->pythonprefix, buf, pos);
    pack_param(args->preloadfile, buf, pos);
@@ -232,7 +232,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin
       (*spindle_argv)[n++] = strdup(numports_s);
       (*spindle_argv)[n++] = strdup(uniqueid_s);
    }
-   (*spindle_argv)[n++] = strdup(params->location);
+   (*spindle_argv)[n++] = strdup(params->commpath);
    (*spindle_argv)[n++] = strdup(params->candidate_cachepaths);
    (*spindle_argv)[n++] = strdup(number_s);
    (*spindle_argv)[n++] = strdup(opt_s);
@@ -398,11 +398,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
    
    /* Start FE server */
    debug_printf("spindle_args_t { number = %lu; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; "
-                "use_launcher = %u; startup_type = %u; shm_cache_size = %u; location = %s; "
+                "use_launcher = %u; startup_type = %u; shm_cache_size = %u; commpath = %s; "
                 "cachepaths = %s; "
                 "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n",
                 (unsigned long) params->number, params->port, params->num_ports, params->opts, params->unique_id,
-                params->use_launcher, params->startup_type, params->shm_cache_size, params->location,
+                params->use_launcher, params->startup_type, params->shm_cache_size, params->commpath,
                 params->candidate_cachepaths,
                 params->pythonprefix, params->preloadfile, params->bundle_timeout_ms,
                 params->bundle_cachesize_kb);
diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c
index 65de5ed9..101bb6ac 100644
--- a/src/flux/flux-spindle.c
+++ b/src/flux/flux-spindle.c
@@ -381,7 +381,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     int numa = 0;
     const char *relocaout = NULL, *reloclibs = NULL, *relocexec = NULL, *relocpython = NULL;
     const char *followfork = NULL, *preload = NULL, *level = NULL;
-    const char *pyprefix = NULL, *location = NULL;
+    const char *pyprefix = NULL, *commpath = NULL;
     char *numafiles = NULL, *cachepaths = NULL;
 
     if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0)
@@ -415,7 +415,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
                         "reloc-exec", &relocexec,
                         "reloc-python", &relocpython,
                         "python-prefix", &pyprefix,
-                        "location", &location,
+                        "commpath", &commpath,
                         "numa", &numa,
                         "numa-files", &numafiles,
                         "preload", &preload,
@@ -466,8 +466,8 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     if( cachepaths ){
         ctx->params.candidate_cachepaths = cachepaths;
     }
-    if (location) {
-       ctx->params.location = (char *) location;
+    if (commpath) {
+       ctx->params.commpath = (char *) commpath;
     }
     if (level) {
        if (strcmp(level, "high") == 0) {
diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 81c0728e..ca7b8d3d 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -123,8 +123,8 @@ typedef struct {
    /* Size of client shared memory cache */
    unsigned int shm_cache_size;
 
-   /* The local-disk location where Spindle will store its cache */
-   char *location;
+   /* The local-disk location for communication and housekeeping. */
+   char *commpath;
 
    /* Path[s] for cached libraries. */
    char *candidate_cachepaths;  /* Colon-separated list of candidate paths (max 64) */
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 29107bad..91d8779c 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -202,7 +202,7 @@ static int handle_client_info_msg(ldcs_process_data_t *procdata, int nc, ldcs_me
    else if(msg->header.type == LDCS_MSG_LOCATION) {
       strncpy(client->remote_location, msg->data, sizeof(client->remote_location)-1);
       client->remote_location[sizeof(client->remote_location)-1] = '\0';
-      debug_printf2("Server recvd location %s from %d\n", msg->data, nc);
+      debug_printf2("Server recvd remote_location %s from %d\n", msg->data, nc);
    }
    else if (msg->header.type == LDCS_MSG_CPU) {
       int clientcpu;
@@ -2953,7 +2953,7 @@ static int handle_client_pickone_msg(ldcs_process_data_t *procdata, int nc, ldcs
 }
 
 /**
- * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which of the locations, commpaths, and cachepaths are
+ * Handle LDCS_MSG_REQUEST_CACHEPATH_CONSENSUS to determine which cachepaths are
  * available across all of the servers.
  */
 
@@ -2969,8 +2969,8 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag
     ldcs_audit_server_md_allreduce_AND( &procdata->cachepath_bitidx );
 
     if( procdata->cachepath_bitidx == 0 ){
-       err_printf("No valid cachepath path available.  Falling back to \"location\" path (%s).\n", procdata->location);
-       procdata->cachepath = procdata->location;
+       err_printf("No valid cachepath path available.  Falling back to \"commpath\" path (%s).\n", procdata->commpath);
+       procdata->cachepath = procdata->commpath;
     }else{
         // ldcs_audit_server_filemngt_init() does it's own realize() pass.
         getValidCachePathByIndex( procdata->cachepath_bitidx,
@@ -2979,7 +2979,7 @@ static int handle_cachepath_consensus(ldcs_process_data_t *procdata, ldcs_messag
                 &procdata->symbolic_cachepath);
     }
 
-    debug_printf3("Initializing file cache location %s\n", procdata->location);
+    debug_printf3("Initializing file cache cachepath %s\n", procdata->cachepath);
     ldcs_audit_server_filemngt_init(procdata->cachepath);
 
     test_printf("<internal> cachepath=%s\n", procdata->cachepath);
@@ -3224,7 +3224,7 @@ int exit_note_cb(int fd, int serverid, void *data)
       eresult = -1;
    }
 
-   result = handleExitNote(fd, procdata->location);
+   result = handleExitNote(fd, procdata->commpath);
    if (result == -1) {
       debug_printf("handleExitNote failed\n");
       eresult = -1;
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index 566cad01..a73a7b7f 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -113,7 +113,7 @@ void startprofile(spindle_args_t *args)
    char hostname[257];
    char *home = getenv("HOME");
    if (!home || !*home)
-      home = ldcs_process_data.location;
+      home = ldcs_process_data.commpath;
    gethostname(hostname, sizeof(hostname));
    snprintf(filename, 4096, "%s/spindled.%lu.%s.%d.prof", home, (unsigned long) args->number, hostname, getpid());
    ProfilerStart(filename);
@@ -142,7 +142,7 @@ int ldcs_audit_server_process(spindle_args_t *args)
    startprofile(args);
 
    debug_printf3("Initializing server data structures\n");
-   ldcs_process_data.location = args->location;
+   ldcs_process_data.commpath = args->commpath;
    ldcs_process_data.cachepaths = args->candidate_cachepaths;
    ldcs_process_data.cachepath = NULL;
    ldcs_process_data.cachepath_bitidx = 0;
@@ -196,11 +196,11 @@ int ldcs_audit_server_process(spindle_args_t *args)
    ldcs_process_data.server_stat.hostname=ldcs_process_data.hostname;
 
    if (ldcs_process_data.opts & OPT_PROCCLEAN)
-      init_cleanup_proc(ldcs_process_data.location);
+      init_cleanup_proc(ldcs_process_data.commpath);
 
    debug_printf3("Initializing connections for clients at %s and %lu\n",
-                 ldcs_process_data.location, (unsigned long) ldcs_process_data.number);
-   serverid = ldcs_create_server(ldcs_process_data.location, ldcs_process_data.number);
+                 ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number);
+   serverid = ldcs_create_server(ldcs_process_data.commpath, ldcs_process_data.number);
    if (serverid == -1) {
       err_printf("Unable to setup area for client connections\n");
       return -1;
@@ -216,7 +216,7 @@ int ldcs_audit_server_process(spindle_args_t *args)
       ldcs_listen_register_fd(fd, serverid, &_ldcs_server_CB, (void *) &ldcs_process_data);
 
    if (args->opts & OPT_BEEXIT) {
-      fd = createExitNote(args->location);
+      fd = createExitNote(args->commpath);
       if (fd != -1) {
          ldcs_listen_register_fd(fd, serverid, exit_note_cb, (void *) &ldcs_process_data);
       }
@@ -254,7 +254,7 @@ int ldcs_audit_server_run()
 
    _ldcs_server_stat_print(&ldcs_process_data.server_stat);
   
-   debug_printf("destroy server (%s,%lu)\n", ldcs_process_data.location, (unsigned long) ldcs_process_data.number);
+   debug_printf("destroy server (%s,%lu)\n", ldcs_process_data.commpath, (unsigned long) ldcs_process_data.number);
    ldcs_destroy_server(ldcs_process_data.serverid);
   
    /* destroy md support (multi-daemon) */
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 1495cebd..82b60023 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -125,7 +125,7 @@ struct ldcs_process_data_struct
   int exit_readys_recvd;
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
-  char *location;               /* Single user-specified path for fifo, daemons, etc. */
+  char *commpath;               /* Single user-specified path for fifo, daemons, etc. */
                                 /*   (Everything except the cachepath.) */
   char *cachepaths;             /* Up to 64 colon-separated list of candidate cachepaths. */
   char *cachepath;              /* The earliest path in the list available to all servers. */
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 733d3244..fa0eccb3 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -58,7 +58,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size)
    unpack_param(args->use_launcher, buf, pos);
    unpack_param(args->startup_type, buf, pos);
    unpack_param(args->shm_cache_size, buf, pos);
-   unpack_param(args->location, buf, pos);
+   unpack_param(args->commpath, buf, pos);
    unpack_param(args->candidate_cachepaths, buf, pos);
    unpack_param(args->pythonprefix, buf, pos);
    unpack_param(args->preloadfile, buf, pos);
@@ -144,15 +144,15 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    assert(args.port == port);
    
    
-   /* Expand environment variables in location. */
-   char *new_location = parse_location(args.location, args.number);
-   if (!new_location) {
-      err_printf("Failed to convert location %s\n", args.location);
+   /* Expand environment variables in commpath. */
+   char *new_commpath = parse_location(args.commpath, args.number);
+   if (!new_commpath) {
+      err_printf("Failed to convert commpath %s\n", args.commpath);
       return -1;
    }
-   debug_printf("Translated location from %s to %s\n", args.location, new_location);
-   free(args.location);
-   args.location = new_location;
+   debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath);
+   free(args.commpath);
+   args.commpath = new_commpath;
 
    result = ldcs_audit_server_process(&args);
    if (result == -1) {

From 2b0acc013ef84ef2f0da1ae10a447fd19aeebe20 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Fri, 24 Oct 2025 12:45:10 -0700
Subject: [PATCH 22/33] Renames enums.

---
 src/client/client/client.c                           |  4 ++--
 src/client/client/intercept_exec.c                   |  6 +++---
 src/client/client_comlib/client_api.c                |  2 +-
 src/fe/startup/parse_launcher_args.cc                | 12 ++++++------
 src/include/ldcs_api.h                               |  2 +-
 src/server/auditserver/ldcs_audit_server_handlers.c  |  4 ++--
 .../auditserver/ldcs_audit_server_md_msocket.c       |  2 +-
 src/server/comlib/ldcs_api_util.c                    |  2 +-
 8 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index ac8d4b63..1635e060 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -198,7 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
+   location = getenv("LDCS_COMMPATH");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -217,7 +217,7 @@ static int init_server_connection()
    if (!(opts & OPT_FOLLOWFORK)) {
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
-      unsetenv("LDCS_LOCATION");
+      unsetenv("LDCS_COMMPATH");
       unsetenv("LDCS_NUMBER");
       unsetenv("LDCS_CONNECTION");
       unsetenv("LDCS_RANKINFO");
diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 27928d35..bae9eba3 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -141,7 +141,7 @@ static char **removeEnvironmentStrs(char **envp)
          continue;
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
-             strIsPrefix("LDCS_LOCATION=", envp[i]) ||
+             strIsPrefix("LDCS_COMMPATH=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -175,7 +175,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf = orig_unsetenv ? orig_unsetenv : unsetenv;
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
-         unsetf("LDCS_LOCATION");
+         unsetf("LDCS_COMMPATH");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
@@ -201,7 +201,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
       
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   
       propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION");
+      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_COMMPATH");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS");
diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 4b0f9ded..f5e07cc0 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -349,7 +349,7 @@ int send_cpu(int fd, int cpu) {
 int send_location(int fd, char *location) {
    ldcs_message_t message;
 
-   message.header.type = LDCS_MSG_LOCATION;
+   message.header.type = LDCS_MSG_COMMPATH;
    message.header.len = strlen(location)+1;
    message.data = location;
 
diff --git a/src/fe/startup/parse_launcher_args.cc b/src/fe/startup/parse_launcher_args.cc
index 84cf7f23..710b009d 100644
--- a/src/fe/startup/parse_launcher_args.cc
+++ b/src/fe/startup/parse_launcher_args.cc
@@ -127,7 +127,7 @@ static cmdoption_t openmpi_options[] = {
 };
 
 
-static const char *openmpi_bg_env_str = "-x LD_AUDIT=%s -x LDCS_LOCATION=%s -x LDCS_NUMBER=%s -x LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *openmpi_bg_env_str = "-x LD_AUDIT=%s -x LDCS_COMMPATH=%s -x LDCS_NUMBER=%s -x LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
 static const unsigned int openmpi_size = (sizeof(openmpi_options) / sizeof(cmdoption_t));
 
 static cmdoption_t srun_options[] = {
@@ -218,11 +218,11 @@ static cmdoption_t srun_options[] = {
    { NULL,   "--usage",              0 },
    { "-V",   "--version",            0 }
 };
-static const char *srun_bg_env_str = "--runjob-opts=--envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_str_nopreload = "--runjob-opts=--envs LD_AUDIT=%s%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_bare_str = "%s --envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_bare_str_preload = "%sLD_PRELOAD=%s:%s LD_AUDIT=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
-static const char *srun_bg_env_bare_str_nopreload = "%s LD_AUDIT=%s LDCS_LOCATION=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_str = "--runjob-opts=--envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_str_nopreload = "--runjob-opts=--envs LD_AUDIT=%s%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_bare_str = "%s --envs LD_AUDIT=%s LD_PRELOAD=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_bare_str_preload = "%sLD_PRELOAD=%s:%s LD_AUDIT=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
+static const char *srun_bg_env_bare_str_nopreload = "%s LD_AUDIT=%s LDCS_COMMPATH=%s LDCS_NUMBER=%s LDCS_OPTIONS=%s LDCS_CACHESIZE=%s";
 
 static const unsigned int srun_size (sizeof(srun_options) / sizeof(cmdoption_t));
 
diff --git a/src/include/ldcs_api.h b/src/include/ldcs_api.h
index e6ccbafb..49ba45e3 100644
--- a/src/include/ldcs_api.h
+++ b/src/include/ldcs_api.h
@@ -45,7 +45,7 @@ typedef enum {
    LDCS_MSG_END,
    LDCS_MSG_CWD,
    LDCS_MSG_PID,
-   LDCS_MSG_LOCATION,
+   LDCS_MSG_COMMPATH,
    LDCS_MSG_CPU,
    LDCS_MSG_MYRANKINFO_QUERY,
    LDCS_MSG_MYRANKINFO_QUERY_ANSWER,
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 91d8779c..0129c25d 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -199,7 +199,7 @@ static int handle_client_info_msg(ldcs_process_data_t *procdata, int nc, ldcs_me
       client->remote_pid=mypid;
       debug_printf2("Server recvd pid %d from %d\n", mypid, nc);
    } 
-   else if(msg->header.type == LDCS_MSG_LOCATION) {
+   else if(msg->header.type == LDCS_MSG_COMMPATH) {
       strncpy(client->remote_location, msg->data, sizeof(client->remote_location)-1);
       client->remote_location[sizeof(client->remote_location)-1] = '\0';
       debug_printf2("Server recvd remote_location %s from %d\n", msg->data, nc);
@@ -1866,7 +1866,7 @@ int handle_client_message(ldcs_process_data_t *procdata, int nc, ldcs_message_t
    switch (msg->header.type) {
       case LDCS_MSG_CWD:
       case LDCS_MSG_PID:
-      case LDCS_MSG_LOCATION:
+      case LDCS_MSG_COMMPATH:
       case LDCS_MSG_CPU:
          return handle_client_info_msg(procdata, nc, msg);
       case LDCS_MSG_PYTHONPREFIX_REQ:
diff --git a/src/server/auditserver/ldcs_audit_server_md_msocket.c b/src/server/auditserver/ldcs_audit_server_md_msocket.c
index c9d616c2..6db18bd4 100644
--- a/src/server/auditserver/ldcs_audit_server_md_msocket.c
+++ b/src/server/auditserver/ldcs_audit_server_md_msocket.c
@@ -62,7 +62,7 @@ int ldcs_audit_server_md_init ( ldcs_process_data_t *ldcs_process_data ) {
   int rc=0;
 
   char* ldcs_nportsstr=getenv("LDCS_NPORTS");
-  char* ldcs_locmodstr=getenv("LDCS_LOCATION_MOD");
+  char* ldcs_locmodstr=getenv("LDCS_COMMPATH_MOD");
 
   int usedport;
   int serverfd, serverid, i;
diff --git a/src/server/comlib/ldcs_api_util.c b/src/server/comlib/ldcs_api_util.c
index b6beb56d..5ea51ab8 100644
--- a/src/server/comlib/ldcs_api_util.c
+++ b/src/server/comlib/ldcs_api_util.c
@@ -51,7 +51,7 @@ char* _message_type_to_str (ldcs_message_ids_t type) {
       STR_CASE(LDCS_MSG_END);
       STR_CASE(LDCS_MSG_CWD);
       STR_CASE(LDCS_MSG_PID);
-      STR_CASE(LDCS_MSG_LOCATION);
+      STR_CASE(LDCS_MSG_COMMPATH);
       STR_CASE(LDCS_MSG_CPU);      
       STR_CASE(LDCS_MSG_MYRANKINFO_QUERY);
       STR_CASE(LDCS_MSG_MYRANKINFO_QUERY_ANSWER);

From fa47cf1c9f963f0ab4e9d8534d2f86d50aa99470 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 5 Nov 2025 15:28:52 -0800
Subject: [PATCH 23/33] Use strdup() for commpath instead of stack var.

Unlikely it would ever make a difference, but this is much more
correct.
---
 src/server/startup/spindle_be.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index fa0eccb3..c3e7bbd9 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -152,7 +152,7 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    }
    debug_printf("Translated commpath from %s to %s\n", args.commpath, new_commpath);
    free(args.commpath);
-   args.commpath = new_commpath;
+   args.commpath = strdup(new_commpath);
 
    result = ldcs_audit_server_process(&args);
    if (result == -1) {

From e025e83ca707c08fce92c4b07666b4200ef70d85 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Wed, 12 Nov 2025 10:13:59 -0800
Subject: [PATCH 24/33] Adds LDCS_COMMPATH

---
 src/client/client/client.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/client/client/client.c b/src/client/client/client.c
index 1635e060..69488b10 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -198,7 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_COMMPATH");
+   location = getenv("LDCS_LOCATION");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");

From a2c25ac39656a3abe53902dab6b6dd99147a7b99 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 13 Nov 2025 12:56:57 -0800
Subject: [PATCH 25/33] Restores checkLinkForLeak() to test_driver.c

---
 testsuite/test_driver.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index a0b46a75..544bfd20 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1169,6 +1169,28 @@ static char* getCacheLocation(char *env_var)
    return strdup(last_slash);
 }
 
+static int checkLinkForLeak(const char *path, const char *spindle_loc)
+{
+   char link_target[4096];
+   int result, error;
+   memset(link_target, 0, sizeof(link_target));
+
+   result = readlink(path, link_target, sizeof(link_target));
+   if (result == -1) {
+      error = errno;
+      err_printf("Failed to read link %s: %s\n", path, strerror(error));
+      return -1;
+   }
+
+   if (strstr(link_target, spindle_loc)) {
+      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc);
+      return -1;
+   }
+
+   return 0;
+}
+
+
 static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc)
 {
    if (strstr(path, spindle_loc)) {
@@ -1265,7 +1287,9 @@ void check_for_path_leaks()
          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
+      checkLinkForLeak(path, spindle_loc);
    }
+   checkLinkForLeak("/proc/self/exe", spindle_loc);
 
    /**
     * Check link_maps for leaked spindle paths

From 77b7375569ffbd1e27bdf887c205efd362a7ecf1 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 13 Nov 2025 15:24:58 -0800
Subject: [PATCH 26/33] Replacing "location" with "commpath" as needed.

src/client/beboot/spindle_bootstrap.c
    Moved orig_location from static global to local
    Renamed symbolic_location to symbolic_commpath
    Renamed orig_location to orig_commpath
    Renamed location to commpath
    Renamed LDCS_LOCATION to LDCS_COMMPATH

src/client/client/client.c
    Renamed LDCS_LOCATION to LDCS_COMMPATH
    Renamed location to commpath
---
 src/client/beboot/spindle_bootstrap.c | 24 ++++++++++++------------
 src/client/client/client.c            | 22 +++++++++++-----------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index 98424d50..d5f8a197 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -53,7 +53,7 @@ static int rankinfo[4]={-1,-1,-1,-1};
 number_t number;
 static int use_cache;
 static unsigned int cachesize;
-static char *location, *number_s, *orig_location, *symbolic_location;
+static char *commpath, *number_s, *symbolic_commpath;
 static char **cmdline;
 static char *executable;
 static char *client_lib;
@@ -91,7 +91,7 @@ extern char *realize(char *path);
 static int establish_connection()
 {
    debug_printf2("Opening connection to server\n");
-   ldcsid = client_open_connection(location, number);
+   ldcsid = client_open_connection(commpath, number);
    if (ldcsid == -1) 
       return -1;
 
@@ -113,7 +113,7 @@ static void setup_environment()
       connection_str = client_get_connection_string(ldcsid);
 
    setenv("LD_AUDIT", client_lib, 1);
-   setenv("LDCS_LOCATION", location, 1);
+   setenv("LDCS_COMMPATH", commpath, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -159,7 +159,7 @@ static int parse_cmdline(int argc, char *argv[])
       daemon_args[i - 3] = NULL;
    }
 
-   symbolic_location = argv[i++];
+   symbolic_commpath = argv[i++];
    i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
@@ -173,7 +173,7 @@ static int parse_cmdline(int argc, char *argv[])
    return 0;
 }
 
-static void launch_daemon(char *location)
+static void launch_daemon(char *commpath)
 {
    /*grand-child fork, then execv daemon.  By grand-child forking we ensure that
      the app won't get confused by seeing an unknown process as a child. */
@@ -183,12 +183,12 @@ static void launch_daemon(char *location)
    char unique_file[MAX_PATH_LEN+1];
    char buffer[32];
 
-   result = spindle_mkdir(location);
+   result = spindle_mkdir(commpath);
    if (result == -1) {
       debug_printf("Exiting due to spindle_mkdir error\n");
       exit(-1);
    }
-   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location);
+   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath);
    unique_file[MAX_PATH_LEN] = '\0';
    fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600);
    if (fd == -1) {
@@ -343,14 +343,14 @@ int main(int argc, char *argv[])
       }
    }
 
-   orig_location = parse_location(symbolic_location, number);
-   if (!orig_location) {
+   char *orig_commpath = parse_location(symbolic_commpath, number);
+   if (!orig_commpath) {
       return -1;
    }
-   location = realize(orig_location);
+   commpath = realize(orig_commpath);
 
    if (daemon_args) {
-      launch_daemon(location);
+      launch_daemon(commpath);
    }
    
    result = establish_connection();
@@ -374,7 +374,7 @@ int main(int argc, char *argv[])
 #else
       shm_cache_limit = cachesize;
 #endif
-      shmcache_init(location, number, cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, cachesize, shm_cache_limit);
       use_cache = 1;
    }      
    
diff --git a/src/client/client/client.c b/src/client/client/client.c
index 69488b10..0f0b6047 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -71,7 +71,7 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-static char *location;
+static char *commpath;
 char *chosen_realized_cachepath, *chosen_parsed_cachepath;
 number_t number;
 static int have_stat_patches;
@@ -198,7 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
+   commpath = getenv("LDCS_COMMPATH");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -207,9 +207,9 @@ static int init_server_connection()
    opts = strtoul(opts_s, NULL, 10);
    shm_cachesize = atoi(cachesize_s) * 1024;
 
-   if (strchr(location, '$')) {
-      location = parse_location(location, number);
-      if (!location) {
+   if (strchr(commpath, '$')) {
+      commpath = parse_location(commpath, number);
+      if (!commpath) {
          exit(-1);
       }
    }
@@ -231,14 +231,14 @@ static int init_server_connection()
 #else
       shm_cache_limit = shm_cachesize;
 #endif
-      shmcache_init(location, number, shm_cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, shm_cachesize, shm_cache_limit);
    }
 
    if (connection) {
       /* boostrapper established the connection for us.  Reuse it. */
       debug_printf("Recreating existing connection to server\n");
-      debug_printf3("location = %s, number = %lu, connection = %s, rankinfo = %s\n",
-                    location, (unsigned long) number, connection, rankinfo_s);
+      debug_printf3("commpath = %s, number = %lu, connection = %s, rankinfo = %s\n",
+                    commpath, (unsigned long) number, connection, rankinfo_s);
       ldcsid  = client_register_connection(connection);
       if (ldcsid == -1)
          return -1;
@@ -248,13 +248,13 @@ static int init_server_connection()
    }
    else {
       /* Establish a new connection */
-      debug_printf("open connection to ldcs %s %lu\n", location, (unsigned long) number);
-      ldcsid = client_open_connection(location, number);
+      debug_printf("open connection to ldcs %s %lu\n", commpath, (unsigned long) number);
+      ldcsid = client_open_connection(commpath, number);
       if (ldcsid == -1)
          return -1;
 
       send_pid(ldcsid);
-      send_location(ldcsid, location);
+      send_location(ldcsid, commpath);
       send_rankinfo_query(ldcsid, rankinfo+0, rankinfo+1, rankinfo+2, rankinfo+3);
 #if defined(LIBNUMA)      
       if (opts & OPT_NUMA)

From 332e67fa8813e70e8b5016c0c57a5d295c823f3b Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 13 Nov 2025 16:06:35 -0800
Subject: [PATCH 27/33] Continues location rename.

src/client/client_comlib/client_api.c
    Added setenv("LDCS_CHOSEN_PARSED_CACHEPATH", local_cpc);

testsuite/test_driver.c
    Replaced LDCS_LOCATION and LDCS_ORIG_LOCATION checks for cachepath with
        LDCS_CHOSEN_PARSED_CACHEPATH
    Replaced spindle_loc with cachepath
---
 src/client/client_comlib/client_api.c |  2 ++
 testsuite/test_driver.c               | 48 +++++++++++++--------------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index f5e07cc0..827acbf0 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -67,6 +67,8 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
    if( chosen_parsed_cachepath ){
        *chosen_parsed_cachepath = local_cpc;
    }
+   // Required by testsuite/test_driver.c
+   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", local_cpc, 1);
 
    return 0;
 }
diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index 544bfd20..01bf3238 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1169,7 +1169,7 @@ static char* getCacheLocation(char *env_var)
    return strdup(last_slash);
 }
 
-static int checkLinkForLeak(const char *path, const char *spindle_loc)
+static int checkLinkForLeak(const char *path, const char *cachepath)
 {
    char link_target[4096];
    int result, error;
@@ -1182,8 +1182,8 @@ static int checkLinkForLeak(const char *path, const char *spindle_loc)
       return -1;
    }
 
-   if (strstr(link_target, spindle_loc)) {
-      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, spindle_loc);
+   if (strstr(link_target, cachepath)) {
+      err_printf("Link at '%s' has path '%s', which leaks spindle path with '%s'\n", path, link_target, cachepath);
       return -1;
    }
 
@@ -1191,10 +1191,10 @@ static int checkLinkForLeak(const char *path, const char *spindle_loc)
 }
 
 
-static int checkPathForLeak(const char *what, const char *path, const char *spindle_loc)
+static int checkPathForLeak(const char *what, const char *path, const char *cachepath)
 {
-   if (strstr(path, spindle_loc)) {
-      err_printf("%s: Path '%s' leaks spindle path with '%s'\n", what, path, spindle_loc);
+   if (strstr(path, cachepath)) {
+      err_printf("%s: Path '%s' leaks spindle path with '%s'\n", what, path, cachepath);
       return -1;
    }
    return 0;
@@ -1202,14 +1202,14 @@ static int checkPathForLeak(const char *what, const char *path, const char *spin
 
 static int leak_check_cb(struct dl_phdr_info *p, size_t psize, void *opaque)
 {
-   char *spindle_loc = (char *) opaque;
+   char *cachepath = (char *) opaque;
    if (!p->dlpi_name || p->dlpi_name[0] == '\0')
       return 0;
-   checkPathForLeak("dl_iterate_phdr", p->dlpi_name, spindle_loc);
+   checkPathForLeak("dl_iterate_phdr", p->dlpi_name, cachepath);
    return 0;
 }
 
-static int check_proc_maps(char *path, char *spindle_loc)
+static int check_proc_maps(char *path, char *cachepath)
 {
    int fd, error, result;
    struct stat statbuf;
@@ -1248,8 +1248,8 @@ static int check_proc_maps(char *path, char *spindle_loc)
    maps[filesize] = '\0';
    close(fd);
 
-   if (strstr(maps, spindle_loc)) {
-      err_printf("Found leaked spindle path '%s' in maps '%s'\n", spindle_loc, path);
+   if (strstr(maps, cachepath)) {
+      err_printf("Found leaked spindle path '%s' in maps '%s'\n", cachepath, path);
       return -1;
    }
 
@@ -1259,17 +1259,15 @@ static int check_proc_maps(char *path, char *spindle_loc)
 
 void check_for_path_leaks()
 {
-   char *spindle_loc = NULL;
+   char *cachepath = NULL;
    DIR *proc_fds = NULL;
    struct dirent *d;
    char path[4096];
    struct link_map *lm;
    char *dlerr_msg = NULL;
 
-   spindle_loc = getCacheLocation("LDCS_LOCATION");
-   if (!spindle_loc)
-      spindle_loc = getCacheLocation("LDCS_ORIG_LOCATION");
-   if (!spindle_loc) {
+   cachepath = getCacheLocation("LDCS_CHOSEN_PARSED_CACHEPATH");
+   if (!cachepath) {
       err_printf("Failed to calculate cache location");
       goto done;
    }
@@ -1287,9 +1285,9 @@ void check_for_path_leaks()
          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
-      checkLinkForLeak(path, spindle_loc);
+      checkLinkForLeak(path, cachepath);
    }
-   checkLinkForLeak("/proc/self/exe", spindle_loc);
+   checkLinkForLeak("/proc/self/exe", cachepath);
 
    /**
     * Check link_maps for leaked spindle paths
@@ -1297,22 +1295,22 @@ void check_for_path_leaks()
    for (lm = _r_debug.r_map; lm != NULL; lm = lm->l_next) {
       if (!lm->l_name || lm->l_name[0] == '\0')
          continue;
-      checkPathForLeak("link_map", lm->l_name, spindle_loc);
+      checkPathForLeak("link_map", lm->l_name, cachepath);
    }
 
    /**
     * Check libraries in dl_iterate_phdr for leaked paths
     **/
-   dl_iterate_phdr(leak_check_cb, spindle_loc);
+   dl_iterate_phdr(leak_check_cb, cachepath);
 
    /**
     * Check /proc/pid/maps under various aliases for leaked names
     **/
-   check_proc_maps("/proc/self/maps", spindle_loc);
+   check_proc_maps("/proc/self/maps", cachepath);
    snprintf(path, sizeof(path), "/proc/self/task/%d/maps", getpid());
-   check_proc_maps(path, spindle_loc);
+   check_proc_maps(path, cachepath);
    snprintf(path, sizeof(path), "/proc/%d/maps", getpid());
-   check_proc_maps(path, spindle_loc);
+   check_proc_maps(path, cachepath);
 
    /**
     * Check that dlerror doesn't leak the /__not_exists/ prefix
@@ -1324,8 +1322,8 @@ void check_for_path_leaks()
    }
    
   done:
-   if (spindle_loc)
-      free(spindle_loc);
+   if (cachepath)
+      free(cachepath);
    if (proc_fds)
       closedir(proc_fds);
 }

From 193cba287bf67c22acf853af3fa7132beeb38716 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Sun, 7 Dec 2025 11:04:26 -0800
Subject: [PATCH 28/33] Fixes -Wsign-compare warning in new code.

---
 src/cobo/cobo.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cobo/cobo.c b/src/cobo/cobo.c
index 65741008..2d1a53a9 100644
--- a/src/cobo/cobo.c
+++ b/src/cobo/cobo.c
@@ -1437,7 +1437,7 @@ int cobo_allgather_str(char* sendstr, char*** recvstr, char** recvbuf)
 int cobo_allreduce( uint64_t *pval, cobo_op_t op ){
 
     /* if i have any children, receive their data */
-    int64_t child_val;
+    uint64_t child_val;
     for(int i=cobo_num_child-1; i>=0; i--) {
         /* read int64_t from child */
         if (cobo_read_fd(cobo_child_fd[i], &child_val, sizeof(int64_t)) < 0) {

From 5f3a1df0f5cdae657aff65a4addeea49dbd51ff4 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Sun, 7 Dec 2025 11:33:43 -0800
Subject: [PATCH 29/33] LDCS_CHOSEN_PARSED_CACHEPATH set in bootstrap.

---
 src/client/beboot/spindle_bootstrap.c | 6 +++++-
 src/client/client_comlib/client_api.c | 2 --
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index d5f8a197..d0e06439 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -107,13 +107,17 @@ static void setup_environment()
 {
    char rankinfo_str[256];
    snprintf(rankinfo_str, 256, "%d %d %d %d %d", ldcsid, rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3]);
-   
+
    char *connection_str = NULL;
    if (opts & OPT_RELOCAOUT) 
       connection_str = client_get_connection_string(ldcsid);
 
+   char *chosen_parsed_cachepath;
+   send_cachepath_query( ldcsid , NULL, &chosen_parsed_cachepath);
+
    setenv("LD_AUDIT", client_lib, 1);
    setenv("LDCS_COMMPATH", commpath, 1);
+   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", chosen_parsed_cachepath, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
diff --git a/src/client/client_comlib/client_api.c b/src/client/client_comlib/client_api.c
index 827acbf0..f5e07cc0 100644
--- a/src/client/client_comlib/client_api.c
+++ b/src/client/client_comlib/client_api.c
@@ -67,8 +67,6 @@ int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chose
    if( chosen_parsed_cachepath ){
        *chosen_parsed_cachepath = local_cpc;
    }
-   // Required by testsuite/test_driver.c
-   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", local_cpc, 1);
 
    return 0;
 }

From 857f9e733b7085a15a835f14d4fb82c15e151263 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Thu, 18 Dec 2025 19:46:36 -0800
Subject: [PATCH 30/33] Updates test_driver.c to ignore FIFO files.

All tests pass with both distinct and identical
commpaths/cachepaths.
---
 testsuite/test_driver.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/testsuite/test_driver.c b/testsuite/test_driver.c
index 01bf3238..aae6c5c8 100644
--- a/testsuite/test_driver.c
+++ b/testsuite/test_driver.c
@@ -1283,6 +1283,11 @@ void check_for_path_leaks()
    for (d = readdir(proc_fds); d != NULL; d = readdir(proc_fds)) {
       if (d->d_name[0] == '.')
          continue;
+      // Ignore Spindle fifo files for now.
+      if ( strncmp( "315", d->d_name, 3 ) == 0 )
+          continue;
+      if ( strncmp( "316", d->d_name, 3 ) == 0 )
+          continue;
       strncpy(path, "/proc/self/fd/", sizeof(path));
       strncat(path, d->d_name, sizeof(path)-1);
       checkLinkForLeak(path, cachepath);

From 5e508a4ca95bd3c355e0931587b4b93139f928aa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 1 Jan 2026 07:19:57 +0000
Subject: [PATCH 31/33] Bump docker/setup-buildx-action in /.github/workflows

Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.11.1 to 3.12.0.
- [Release notes](https://github.com/docker/setup-buildx-action/releases)
- [Commits](https://github.com/docker/setup-buildx-action/compare/e468171a9de216ec08956ac3ada2f0791b6bd435...8d2750c68a42422c14e847fe6c8ac0403b4cbd6f)

---
updated-dependencies:
- dependency-name: docker/setup-buildx-action
  dependency-version: 3.12.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/container.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
index d741453a..59657d83 100644
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef

From 7ce216c216bab30e10a87630505a62a63d1546e0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 1 Jan 2026 07:19:53 +0000
Subject: [PATCH 32/33] Bump actions/checkout from 6.0.0 to 6.0.1 in
 /.github/workflows

Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.0 to 6.0.1.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/1af3b93b6815bc44a9784bd300feb67ff0d1eeb3...8e8c483db84b4bee98b60c0593521ed34d9990e8)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: 6.0.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml        | 6 +++---
 .github/workflows/container.yml | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dd68a5a2..f465e09e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Setup Docker Compose
         uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
@@ -68,7 +68,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Setup Docker Compose
         uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
@@ -112,7 +112,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Setup Docker Compose
         uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
index 59657d83..4795d9ab 100644
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -11,7 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f

From 8de2636e9123b43440830445104709439730f2b0 Mon Sep 17 00:00:00 2001
From: Barry <rountree4@llnl.gov>
Date: Mon, 12 Jan 2026 01:38:42 -0800
Subject: [PATCH 33/33] Sets TMPDIR=/tmp in each Dockerfile

Additionally populates /etc/environment just in case ssh is used
to set up the servers.
---
 containers/spindle-flux-ubuntu/Dockerfile          | 2 ++
 containers/spindle-serial-ubuntu/Dockerfile        | 2 ++
 containers/spindle-slurm-ubuntu/base/Dockerfile    | 2 ++
 containers/spindle-slurm-ubuntu/testing/Dockerfile | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile
index 3af607a8..57badbfc 100644
--- a/containers/spindle-flux-ubuntu/Dockerfile
+++ b/containers/spindle-flux-ubuntu/Dockerfile
@@ -5,6 +5,8 @@ FROM fluxrm/flux-sched:${flux_sched_version} AS builder
 ARG replicas=4
 ENV workers=${replicas}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
     && apt-get -qq install -y --no-install-recommends \
diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile
index 3070596e..62c0cbf5 100644
--- a/containers/spindle-serial-ubuntu/Dockerfile
+++ b/containers/spindle-serial-ubuntu/Dockerfile
@@ -1,6 +1,8 @@
 ARG ubuntu_version=noble
 FROM ubuntu:${ubuntu_version}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
 # install latest pkg utils:
diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile b/containers/spindle-slurm-ubuntu/base/Dockerfile
index d4724276..d6eee9f2 100644
--- a/containers/spindle-slurm-ubuntu/base/Dockerfile
+++ b/containers/spindle-slurm-ubuntu/base/Dockerfile
@@ -1,6 +1,8 @@
 ARG UBUNTU_VERSION=noble
 FROM ubuntu:${UBUNTU_VERSION}
 USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 RUN apt-get update \
  && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile b/containers/spindle-slurm-ubuntu/testing/Dockerfile
index 91d6901c..99768535 100644
--- a/containers/spindle-slurm-ubuntu/testing/Dockerfile
+++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile
@@ -2,6 +2,8 @@ ARG BASE_VERSION=latest
 FROM ghcr.io/llnl/spindle-slurm-base:${BASE_VERSION}
 ARG replicas=4
 ENV workers=${replicas}
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
 
 ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing