Watchdog: dump /dev/shm + log FI_EFA_ENABLE_SHM_TRANSFER env var

When the 60s watchdog fires, list /dev/shm contents to verify
whether SHM regions exist or not. Also log FI_EFA_ENABLE_SHM_TRANSFER
at rank 0 startup to confirm the env var was received.

Author: Sbozzolo

palace/main.cpp

@@ -210,6 +210,14 @@ int main(int argc, char *argv[])
         FILE *f = fopen(path, "a");
         if (!f) return;
         fprintf(f, "=== WATCHDOG FIRED (60s timeout) ===\n");
+        // Dump /dev/shm contents to see if SHM regions exist
+        fprintf(f, "--- /dev/shm listing ---\n");
+        FILE *shm_ls = popen("ls -la /dev/shm/ 2>&1 | head -30", "r");
+        if (shm_ls) {
+          char line[512];
+          while (fgets(line, sizeof(line), shm_ls)) fputs(line, f);
+          pclose(shm_ls);
+        }
         // Dump kernel stack
         char stack_path[64];
         snprintf(stack_path, sizeof(stack_path), "/proc/%d/stack", getpid());
@@ -250,6 +258,8 @@ int main(int argc, char *argv[])
     };
     log_env("FI_PROVIDER");
     log_env("FI_LOG_LEVEL");
+    log_env("FI_EFA_ENABLE_SHM_TRANSFER");
+    log_env("FI_EFA_RNR_RETRY");
     log_env("FI_OFI_RXM_USE_SRX");
     log_env("FI_UNIVERSE_SIZE");
     log_env("OMPI_MCA_pml");