Commpath

config.h.in

@@ -6,6 +6,12 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
+/* Back-end directory for communication and housekeeping */
+#undef COMMPATH
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 
@@ -127,9 +133,6 @@
 /* Default mode for slurm launch */
 #undef SLURMLAUNCH_ENABLED
 
-/* The default local directory for Spindle */
-#undef SPINDLE_LOC
-
 /* The default colon-separated list of directories that Spindle will not cache
    files out of */
 #undef SPINDLE_LOCAL_PREFIX

configure

@@ -846,7 +846,8 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
-with_localstorage
+with_cachepaths
+with_commpath
 with_default_local_prefix
 with_testrm
 with_rm
@@ -1589,7 +1590,10 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
-  --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
+  --with-compath=DIR      Back-end directory for communication and
+                          housekeeping
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
                           will not cache files out of
@@ -16615,19 +16619,27 @@ else
 fi
 
 
-# Check whether --with-localstorage was given.
-if test "${with_localstorage+set}" = set; then :
-  withval=$with_localstorage; SPINDLE_LOC=${withval}
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
 else
-  SPINDLE_LOC=$DEFAULT_LOC
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
+# Check whether --with-commpath was given.
+if test "${with_commpath+set}" = set; then :
+  withval=$with_commpath; COMMPATH=${withval}
+else
+  COMMPATH=$DEFAULT_LOC
 fi
 
 
 # Check whether --with-default-local-prefix was given.
 if test "${with_default_local_prefix+set}" = set; then :
   withval=$with_default_local_prefix; SPINDLE_LOCAL_PREFIX=${withval}
 else
-  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"
+  SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"
 fi
 
 
@@ -16647,7 +16659,12 @@ _ACEOF
 
 
 cat >>confdefs.h <<_ACEOF
-#define SPINDLE_LOC "$SPINDLE_LOC"
+#define COMMPATH "$COMMPATH"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
 _ACEOF
 
 

configure.common.ac

@@ -17,18 +17,23 @@ AC_ARG_WITH(default-num-ports,
             [AS_HELP_STRING([--with-default-numports=NUM],[Number of TCP/IP ports to scan for Spindle server communication])],
             [NUM_COBO_PORTS=${withval}],
             [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS])
-AC_ARG_WITH(localstorage,
-            [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
-            [SPINDLE_LOC=${withval}],
-            [SPINDLE_LOC=$DEFAULT_LOC])
+AC_ARG_WITH(cachepaths,
+            [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
+            [CACHEPATHS=${withval}],
+            [CACHEPATHS=$DEFAULT_LOC])
+AC_ARG_WITH(commpath,
+            [AS_HELP_STRING([--with-compath=DIR],[Back-end directory for communication and housekeeping])],
+            [COMMPATH=${withval}],
+            [COMMPATH=$DEFAULT_LOC])
 AC_ARG_WITH(default-local-prefix,
             [AS_HELP_STRING([--with-default-local-prefix=DIRS],[Colon-seperated list of directories that Spindle will not cache files out of])],
             [SPINDLE_LOCAL_PREFIX=${withval}],
-            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$SPINDLE_LOC"])
+            [SPINDLE_LOCAL_PREFIX="$DEFAULT_LOCAL_PREFIX:$COMMPATH"])
 AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle])
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
-AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([COMMPATH],"[$COMMPATH]",[Back-end directory for communication and housekeeping])
+AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
 TESTRM=unknown

src/client/beboot/spindle_bootstrap.c

@@ -53,7 +53,7 @@ static int rankinfo[4]={-1,-1,-1,-1};
 number_t number;
 static int use_cache;
 static unsigned int cachesize;
-static char *location, *number_s, *orig_location, *symbolic_location;
+static char *commpath, *number_s, *symbolic_commpath;
 static char **cmdline;
 static char *executable;
 static char *client_lib;
@@ -91,7 +91,7 @@ extern char *realize(char *path);
 static int establish_connection()
 {
    debug_printf2("Opening connection to server\n");
-   ldcsid = client_open_connection(location, number);
+   ldcsid = client_open_connection(commpath, number);
    if (ldcsid == -1) 
       return -1;
 
@@ -107,14 +107,17 @@ static void setup_environment()
 {
    char rankinfo_str[256];
    snprintf(rankinfo_str, 256, "%d %d %d %d %d", ldcsid, rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3]);
-   
+
    char *connection_str = NULL;
    if (opts & OPT_RELOCAOUT) 
       connection_str = client_get_connection_string(ldcsid);
 
+   char *chosen_parsed_cachepath;
+   send_cachepath_query( ldcsid , NULL, &chosen_parsed_cachepath);
+
    setenv("LD_AUDIT", client_lib, 1);
-   setenv("LDCS_LOCATION", location, 1);
-   setenv("LDCS_ORIG_LOCATION", orig_location, 1);
+   setenv("LDCS_COMMPATH", commpath, 1);
+   setenv("LDCS_CHOSEN_PARSED_CACHEPATH", chosen_parsed_cachepath, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -160,7 +163,8 @@ static int parse_cmdline(int argc, char *argv[])
       daemon_args[i - 3] = NULL;
    }
 
-   symbolic_location = argv[i++];
+   symbolic_commpath = argv[i++];
+   i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
    opts_s = argv[i++];
@@ -173,7 +177,7 @@ static int parse_cmdline(int argc, char *argv[])
    return 0;
 }
 
-static void launch_daemon(char *location)
+static void launch_daemon(char *commpath)
 {
    /*grand-child fork, then execv daemon.  By grand-child forking we ensure that
      the app won't get confused by seeing an unknown process as a child. */
@@ -183,12 +187,12 @@ static void launch_daemon(char *location)
    char unique_file[MAX_PATH_LEN+1];
    char buffer[32];
 
-   result = spindle_mkdir(location);
+   result = spindle_mkdir(commpath);
    if (result == -1) {
       debug_printf("Exiting due to spindle_mkdir error\n");
       exit(-1);
    }
-   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location);
+   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", commpath);
    unique_file[MAX_PATH_LEN] = '\0';
    fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600);
    if (fd == -1) {
@@ -343,14 +347,14 @@ int main(int argc, char *argv[])
       }
    }
 
-   orig_location = parse_location(symbolic_location, number);
-   if (!orig_location) {
+   char *orig_commpath = parse_location(symbolic_commpath, number);
+   if (!orig_commpath) {
       return -1;
    }
-   location = realize(orig_location);
+   commpath = realize(orig_commpath);
 
    if (daemon_args) {
-      launch_daemon(location);
+      launch_daemon(commpath);
    }
 
    result = establish_connection();
@@ -374,7 +378,7 @@ int main(int argc, char *argv[])
 #else
       shm_cache_limit = cachesize;
 #endif
-      shmcache_init(location, number, cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, cachesize, shm_cache_limit);
       use_cache = 1;
    }      
 

src/client/client/client.c

@@ -41,6 +41,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "exec_util.h"
 #include "intercept.h"
 #include "fixlocale.h"
+#include "should_intercept.h"
 
 errno_location_t app_errno_location;
 
@@ -70,11 +71,8 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-/* location has the realize'd path to the local file cache. orig_location is not realized and
- * may contain symlinks
- */
-char *location;
-char *orig_location;
+static char *commpath;
+char *chosen_realized_cachepath, *chosen_parsed_cachepath;
 number_t number;
 static int have_stat_patches;
 
@@ -200,8 +198,7 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
-   orig_location = getenv("LDCS_ORIG_LOCATION");
+   commpath = getenv("LDCS_COMMPATH");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -210,18 +207,17 @@ static int init_server_connection()
    opts = strtoul(opts_s, NULL, 10);
    shm_cachesize = atoi(cachesize_s) * 1024;
 
-   if (strchr(location, '$')) {
-      location = parse_location(location, number);
-      if (!location) {
+   if (strchr(commpath, '$')) {
+      commpath = parse_location(commpath, number);
+      if (!commpath) {
          exit(-1);
       }
    }
 
    if (!(opts & OPT_FOLLOWFORK)) {
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
-      unsetenv("LDCS_LOCATION");
-      unsetenv("LDCS_ORIG_LOCATION");
+      unsetenv("LDCS_COMMPATH");
       unsetenv("LDCS_NUMBER");
       unsetenv("LDCS_CONNECTION");
       unsetenv("LDCS_RANKINFO");
@@ -235,14 +231,14 @@ static int init_server_connection()
 #else
       shm_cache_limit = shm_cachesize;
 #endif
-      shmcache_init(location, number, shm_cachesize, shm_cache_limit);
+      shmcache_init(commpath, number, shm_cachesize, shm_cache_limit);
    }
 
    if (connection) {
       /* boostrapper established the connection for us.  Reuse it. */
       debug_printf("Recreating existing connection to server\n");
-      debug_printf3("location = %s, number = %lu, connection = %s, rankinfo = %s\n",
-                    location, (unsigned long) number, connection, rankinfo_s);
+      debug_printf3("commpath = %s, number = %lu, connection = %s, rankinfo = %s\n",
+                    commpath, (unsigned long) number, connection, rankinfo_s);
       ldcsid  = client_register_connection(connection);
       if (ldcsid == -1)
          return -1;
@@ -252,20 +248,20 @@ static int init_server_connection()
    }
    else {
       /* Establish a new connection */
-      debug_printf("open connection to ldcs %s %lu\n", location, (unsigned long) number);
-      ldcsid = client_open_connection(location, number);
+      debug_printf("open connection to ldcs %s %lu\n", commpath, (unsigned long) number);
+      ldcsid = client_open_connection(commpath, number);
       if (ldcsid == -1)
          return -1;
 
       send_pid(ldcsid);
-      send_location(ldcsid, location);
+      send_location(ldcsid, commpath);
       send_rankinfo_query(ldcsid, rankinfo+0, rankinfo+1, rankinfo+2, rankinfo+3);
 #if defined(LIBNUMA)      
       if (opts & OPT_NUMA)
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
-
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath );
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
 
@@ -472,7 +468,7 @@ char *client_library_load(const char *name)
 
    char *orig_file_name = (char *) name;
    if (is_in_spindle_cache(name)) {
-      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location);
+      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, chosen_realized_cachepath);
       memset(fixed_name, 0, MAX_PATH_LEN+1);
       send_orig_path_request(ldcsid, orig_file_name, fixed_name);
       orig_file_name = fixed_name;

src/client/client/intercept_exec.c

@@ -141,8 +141,7 @@ static char **removeEnvironmentStrs(char **envp)
          continue;
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
-             strIsPrefix("LDCS_LOCATION=", envp[i]) ||
-             strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) ||
+             strIsPrefix("LDCS_COMMPATH=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -176,8 +175,7 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf = orig_unsetenv ? orig_unsetenv : unsetenv;
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
-         unsetf("LDCS_LOCATION");
-         unsetf("LDCS_ORIG_LOCATION");
+         unsetf("LDCS_COMMPATH");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
@@ -198,13 +196,12 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
    if (envp) {
       debug_printf2("Propogating spindle environment by copying it to new envp list\n");
       for (cur = (char **) envp; *cur; cur++, orig_size++);
-      new_size = orig_size + 10;
+      new_size = orig_size + 9;
       newenv = (char **) malloc(new_size * sizeof(char*));
 
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   
       propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_ORIG_LOCATION");
+      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_COMMPATH");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS");

src/client/client/intercept_readlink.c

@@ -31,19 +31,19 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-extern char *location;
 
 static int fix_local_readlink(char *buf, size_t bufsiz)
 {
    char spindle_id[32];
-   int location_len, result;
+   int cachepath_len, result;
    char tmp[MAX_PATH_LEN+1];
+   extern char *chosen_realized_cachepath;
 
-   location_len = strlen(location);   
+   cachepath_len = strlen(chosen_realized_cachepath);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number);
-   if (strstr(buf, spindle_id) && strncmp(location, buf, location_len) == 0) {
+   if (strstr(buf, spindle_id) && strncmp(chosen_realized_cachepath, buf, cachepath_len) == 0) {
       debug_printf2("readlink received spindle cache path %s. Translating\n", buf);
-      result = send_orig_path_request(ldcsid, buf+location_len+1, tmp);
+      result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp);
       if (result == -1)
          return -1;
       debug_printf2("readlink translated spindle local path %s to %s\n", buf, tmp);