Sandia-OpenSHMEM · markbrown314 · Jul 18, 2025 · Jul 17, 2025 · Jul 17, 2025
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -172,6 +172,7 @@ jobs:
           submodules: 'true'
       - name: Install dependencies
         run: |
+          sudo apt-get -y update
           sudo apt-get install -y gfortran rpm mpich libmpich-dev libhwloc-dev gdb
           sudo sysctl -w kernel.yama.ptrace_scope=0
           sudo sysctl -w kernel.randomize_va_space=0
@@ -219,7 +220,7 @@ jobs:
         if: ${{ matrix.xpmem_version }}
         run: |
           cd repos/xpmem
-          sudo apt-get install linux-headers-`uname -r`
+          sudo apt-get -y install linux-headers-`uname -r`
           ./autogen.sh
           ./configure --prefix=${XPMEM_INSTALL_DIR}
           make -j
@@ -332,6 +333,7 @@ jobs:
 #      - uses: actions/checkout@v2
 #      - name: Install dependencies
 #        run: |
+#          sudo apt-get -y update
 #          sudo apt-get install -y gfortran libhwloc-dev libev-dev libev-libevent-dev
 #
 #      # LIBFABRIC
@@ -531,6 +533,7 @@ jobs:
           submodules: 'true'
       - name: Install dependencies
         run: |
+          sudo apt-get -y update
           sudo apt-get install -y gfortran mpich libmpich-dev gdb
           sudo sysctl -w kernel.yama.ptrace_scope=0
           sudo sysctl -w kernel.randomize_va_space=0
@@ -545,7 +548,7 @@ jobs:
       - name: Build XPMEM
         run: |
           cd repos/xpmem
-          sudo apt-get install linux-headers-`uname -r`
+          sudo apt-get -y install linux-headers-`uname -r`
           ./autogen.sh
           ./configure --prefix=/usr
           make -j
@@ -648,6 +651,7 @@ jobs:
           submodules: 'true'
       - name: Install dependencies
         run: |
+          sudo apt-get -y update
           sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb
           sudo sysctl -w kernel.yama.ptrace_scope=0
           sudo sysctl -w kernel.randomize_va_space=0
@@ -662,7 +666,7 @@ jobs:
       - name: Build XPMEM
         run: |
           cd repos/xpmem
-          sudo apt-get install linux-headers-`uname -r`
+          sudo apt-get -y install linux-headers-`uname -r`
           ./autogen.sh
           ./configure --prefix=/usr
           make -j
@@ -744,6 +748,7 @@ jobs:
           submodules: 'true'
       - name: Install dependencies
         run: |
+          sudo apt-get -y update
           sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb
           sudo sysctl -w kernel.yama.ptrace_scope=0
           sudo sysctl -w kernel.randomize_va_space=0
@@ -758,7 +763,7 @@ jobs:
       - name: Build XPMEM
         run: |
           cd repos/xpmem
-          sudo apt-get install linux-headers-`uname -r`
+          sudo apt-get -y install linux-headers-`uname -r`
           ./autogen.sh
           ./configure --prefix=/usr
           make -j

diff --git a/configure.ac b/configure.ac
@@ -340,6 +340,12 @@ AC_ARG_ENABLE([ofi-hmem],
 AS_IF([test "$enable_ofi_hmem" = "yes"],
       [AC_DEFINE([USE_FI_HMEM], [1], [If defined, the OFI transport will enable FI_HMEM.])])
 
+AC_ARG_ENABLE([multi-init],
+    [AS_HELP_STRING([--disable-multi-init],
+    [Disable multiple initialization and finalization feature. (default: enabled)])],
+    [enable_multi_init=$enableval], [enable_multi_init=yes])
+
+
 PKG_INSTALLDIR()
 
 dnl check for programs
@@ -1002,7 +1008,13 @@ AC_CONFIG_FILES([Makefile
   modules/tests-sos/test/performance/tests/Makefile
   modules/tests-sos/test/apps/Makefile])
 
-AC_OUTPUT
+AS_IF([test "$enable_pmi_mpi" = "yes" -a "$enable_multi_init" = "yes"],
+      [AC_MSG_WARN([Disabled multiple initilization and finalization. Feature is not supported with PMI MPI])])
+
+AM_CONDITIONAL([ENABLE_MULTI_INIT], [test "$enable_multi_init" = "yes" -a "$enable_pmi_mpi" != "yes"])
+
+AS_IF([test "$enable_pmi_mpi" != "yes" -a "$enable_multi_init" = "yes"],
+      [AC_DEFINE([USE_MULTI_INIT], [1], [If defined, multiple init will be supported.])])
 
 AS_IF([test "$enable_pmi_mpi" != "yes" -a "$enable_pmi_simple" != "yes" -a "$opal_external_pmix_version_found" != 1 -a "$ompi_check_pmi_happy" != "yes" -a -z "$pmi_type"],
          [AC_MSG_ERROR([No PMI client interface was configured, consider --enable-pmi-simple or --with-pmi])])
@@ -1025,6 +1037,8 @@ AS_IF([test "$enable_fortran" = "yes" -a "$FC" = ""],
 AS_IF([test "$enable_fortran" = "yes"],
       [AC_MSG_WARN([The Fortran API was deprecated in OpenSHMEM v1.4 and removed in OpenSHMEM v1.5])])
 
+AC_OUTPUT
+
 FORT="$FC"
 if test "$FORT" = "" ; then
   FORT="none"

diff --git a/mpp/shmemx_c_func.h4 b/mpp/shmemx_c_func.h4
@@ -105,3 +105,6 @@ SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_create(void *base, size_t size
 SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_preinit(void);
 SHMEM_FUNCTION_ATTRIBUTES int SHPRE()shmemx_heap_preinit_thread(int requested, int *provided);
 SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_postinit(void);
+
+/* Support querying intialization status */
+SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_query_initialized(int *initialized);
diff --git a/src/init.c b/src/init.c
@@ -28,6 +28,7 @@
 #include <stdio.h>
 #include <limits.h>
 #include <string.h>
+#include <pthread.h>
 
 #define SHMEM_INTERNAL_INCLUDE
 #include "shmem.h"
@@ -37,6 +38,7 @@
 #include "runtime.h"
 #include "build_info.h"
 #include "shmem_team.h"
+#include "shmem_atomic.h"
 
 #if defined(ENABLE_REMOTE_VIRTUAL_ADDRESSING) && defined(__linux__)
 #include <sys/personality.h>
@@ -89,10 +91,10 @@ int shmem_external_heap_device = -1;
 
 int shmem_internal_my_pe = -1;
 int shmem_internal_num_pes = -1;
-int shmem_internal_initialized = 0;
-int shmem_internal_finalized = 0;
+shmem_internal_cntr_t shmem_internal_init_cntr = 0;
 int shmem_internal_initialized_with_start_pes = 0;
 int shmem_internal_global_exit_called = 0;
+int shmem_internal_inited_once = 0;
 
 int shmem_internal_thread_level;
 
@@ -134,44 +136,58 @@ shmem_internal_randr_fini(void)
     return;
 }
 
-
 static void
-shmem_internal_shutdown(void)
+shmem_release_resources(void)
 {
-    if (!shmem_internal_initialized ||
-        shmem_internal_finalized) {
-        return;
+    if (shmem_internal_params.DEBUG) {
+        DEBUG_MSG("releasing resources\n");
     }
 
-    shmem_internal_barrier_all();
-
-    shmem_internal_finalized = 1;
-
     shmem_internal_team_fini();
-
     shmem_transport_fini();
-
     shmem_shr_transport_fini();
-
     SHMEM_MUTEX_DESTROY(shmem_internal_mutex_alloc);
-
     shmem_internal_randr_fini();
-
     shmem_internal_symmetric_fini();
     shmem_runtime_fini();
 }
 
 
+static void
+shmem_internal_shutdown(void)
+{
+    if (!shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
+        RAISE_WARN_STR("too many calls to shmem_finalize");
+        return;
+    }
+
+    shmem_internal_barrier_all();
+
+#ifndef USE_MULTI_INIT
+    shmem_release_resources();
+#else
+    shmem_internal_cntr_dec(&shmem_internal_init_cntr);
+#endif
+
+    return;
+}
+
+
 static void
 shmem_internal_shutdown_atexit(void)
 {
-    if ( shmem_internal_initialized && !shmem_internal_finalized &&
-         !shmem_internal_initialized_with_start_pes && !shmem_internal_global_exit_called &&
-         shmem_internal_my_pe == 0) {
-        RAISE_WARN_STR("shutting down without a call to shmem_finalize");
+    if (shmem_internal_cntr_read(&shmem_internal_init_cntr) &&
+        !shmem_internal_initialized_with_start_pes &&
+        !shmem_internal_global_exit_called &&
+        shmem_internal_my_pe == 0) {
+
+        RAISE_WARN_STR("too few calls to shmem_finalize");
     }
 
-    shmem_internal_shutdown();
+#ifdef USE_MULTI_INIT
+    shmem_release_resources();
+#endif
+
 }
 
 
@@ -518,7 +534,6 @@ shmem_internal_heap_postinit(void)
     randr_initialized = 1;
 
     atexit(shmem_internal_shutdown_atexit);
-    shmem_internal_initialized = 1;
 
     /* finish up */
 #ifndef USE_PMIX
@@ -554,11 +569,25 @@ shmem_internal_init(int tl_requested, int *tl_provided)
 {
     int ret;
 
-    ret = shmem_internal_heap_preinit(tl_requested, tl_provided);
-    if (ret) goto cleanup;
+#ifndef USE_MULTI_INIT
+    if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
+      RAISE_ERROR_STR("attempt to reinitialize library");
+    }
+#endif
 
-    ret = shmem_internal_heap_postinit();
-    if (ret) goto cleanup;
+    if (shmem_internal_cntr_fadd(&shmem_internal_init_cntr, 1) == 0
+        && !shmem_internal_inited_once) {
+
+        shmem_internal_inited_once = 1;
+
+        ret = shmem_internal_heap_preinit(tl_requested, tl_provided);
+        if (ret) goto cleanup;
+
+        ret = shmem_internal_heap_postinit();
+        if (ret) goto cleanup;
+    } else {
+      *tl_provided = shmem_internal_thread_level;
+    }
 
     return 0;
 

diff --git a/src/init_c.c b/src/init_c.c
@@ -20,6 +20,7 @@
 #include "shmem.h"
 #include "shmem_internal.h"
 #include "shmemx.h"
+#include "shmem_atomic.h"
 
 #ifdef ENABLE_PROFILING
 #include "pshmem.h"
@@ -59,10 +60,13 @@
 
 #endif /* ENABLE_PROFILING */
 
+extern shmem_internal_cntr_t shmem_internal_init_cntr;
+
+
 void SHMEM_FUNCTION_ATTRIBUTES
 start_pes(int npes)
 {
-    if (!shmem_internal_initialized) {
+    if (!shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
         shmem_internal_start_pes(npes);
     }
 }
@@ -73,10 +77,6 @@ shmem_init(void)
 {
     int tl_provided, ret;
 
-    if (shmem_internal_initialized) {
-        RAISE_ERROR_STR("attempt to reinitialize library");
-    }
-
     ret = shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided);
     if (ret) abort();
 }
@@ -87,7 +87,7 @@ shmemx_heap_preinit(void)
 {
     int tl_provided, ret;
 
-    if (shmem_internal_initialized) {
+    if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
         RAISE_ERROR_STR("attempt to reinitialize library");
     }
 
@@ -108,9 +108,6 @@ int SHMEM_FUNCTION_ATTRIBUTES
 shmem_init_thread(int tl_requested, int *tl_provided)
 {
     int ret;
-    if (shmem_internal_initialized) {
-        RAISE_ERROR_STR("attempt to reinitialize library");
-    }
 
     ret = shmem_internal_init(tl_requested, tl_provided);
     return ret;
@@ -121,7 +118,7 @@ int SHMEM_FUNCTION_ATTRIBUTES
 shmemx_heap_preinit_thread(int tl_requested, int *tl_provided)
 {
     int ret;
-    if (shmem_internal_initialized) {
+    if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
         RAISE_ERROR_STR("attempt to reinitialize library");
     }
 
@@ -138,6 +135,11 @@ shmem_query_thread(int *provided)
     *provided = shmem_internal_thread_level;
 }
 
+void SHMEM_FUNCTION_ATTRIBUTES
+shmemx_query_initialized(int *initialized)
+{
+    *initialized = shmem_internal_cntr_read(&shmem_internal_init_cntr);
+}
 
 void SHMEM_FUNCTION_ATTRIBUTES
 shmem_global_exit(int status)

diff --git a/src/init_f.c b/src/init_f.c
@@ -26,10 +26,6 @@ void SHMEM_FUNCTION_ATTRIBUTES FC_START_PES(fortran_integer_t *npes);
 void
 FC_START_PES(fortran_integer_t *npes)
 {
-    if (shmem_internal_initialized) {
-        RAISE_ERROR_STR("attempt to reinitialize library");
-    }
-
     shmem_internal_start_pes(*npes);
 }
 
@@ -40,11 +36,6 @@ void
 FC_SHMEM_INIT(void)
 {
     int tl_provided;
-
-    if (shmem_internal_initialized) {
-        RAISE_ERROR_STR("attempt to reinitialize library");
-    }
-
     shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided);
 }