diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7aebd0f59..ff4795a7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -172,6 +172,7 @@ jobs: submodules: 'true' - name: Install dependencies run: | + sudo apt-get -y update sudo apt-get install -y gfortran rpm mpich libmpich-dev libhwloc-dev gdb sudo sysctl -w kernel.yama.ptrace_scope=0 sudo sysctl -w kernel.randomize_va_space=0 @@ -219,7 +220,7 @@ jobs: if: ${{ matrix.xpmem_version }} run: | cd repos/xpmem - sudo apt-get install linux-headers-`uname -r` + sudo apt-get -y install linux-headers-`uname -r` ./autogen.sh ./configure --prefix=${XPMEM_INSTALL_DIR} make -j @@ -332,6 +333,7 @@ jobs: # - uses: actions/checkout@v2 # - name: Install dependencies # run: | +# sudo apt-get -y update # sudo apt-get install -y gfortran libhwloc-dev libev-dev libev-libevent-dev # # # LIBFABRIC @@ -531,6 +533,7 @@ jobs: submodules: 'true' - name: Install dependencies run: | + sudo apt-get -y update sudo apt-get install -y gfortran mpich libmpich-dev gdb sudo sysctl -w kernel.yama.ptrace_scope=0 sudo sysctl -w kernel.randomize_va_space=0 @@ -545,7 +548,7 @@ jobs: - name: Build XPMEM run: | cd repos/xpmem - sudo apt-get install linux-headers-`uname -r` + sudo apt-get -y install linux-headers-`uname -r` ./autogen.sh ./configure --prefix=/usr make -j @@ -648,6 +651,7 @@ jobs: submodules: 'true' - name: Install dependencies run: | + sudo apt-get -y update sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb sudo sysctl -w kernel.yama.ptrace_scope=0 sudo sysctl -w kernel.randomize_va_space=0 @@ -662,7 +666,7 @@ jobs: - name: Build XPMEM run: | cd repos/xpmem - sudo apt-get install linux-headers-`uname -r` + sudo apt-get -y install linux-headers-`uname -r` ./autogen.sh ./configure --prefix=/usr make -j @@ -744,6 +748,7 @@ jobs: submodules: 'true' - name: Install dependencies run: | + sudo apt-get -y update sudo apt-get install -y gfortran mpich libmpich-dev libev-dev libev-libevent-dev libhwloc-dev gdb sudo sysctl -w kernel.yama.ptrace_scope=0 sudo sysctl -w kernel.randomize_va_space=0 @@ -758,7 +763,7 @@ jobs: - name: Build XPMEM run: | cd repos/xpmem - sudo apt-get install linux-headers-`uname -r` + sudo apt-get -y install linux-headers-`uname -r` ./autogen.sh ./configure --prefix=/usr make -j diff --git a/configure.ac b/configure.ac index 0145c53f5..6104d60e2 100644 --- a/configure.ac +++ b/configure.ac @@ -340,6 +340,12 @@ AC_ARG_ENABLE([ofi-hmem], AS_IF([test "$enable_ofi_hmem" = "yes"], [AC_DEFINE([USE_FI_HMEM], [1], [If defined, the OFI transport will enable FI_HMEM.])]) +AC_ARG_ENABLE([multi-init], + [AS_HELP_STRING([--disable-multi-init], + [Disable multiple initialization and finalization feature. (default: enabled)])], + [enable_multi_init=$enableval], [enable_multi_init=yes]) + + PKG_INSTALLDIR() dnl check for programs @@ -1002,7 +1008,13 @@ AC_CONFIG_FILES([Makefile modules/tests-sos/test/performance/tests/Makefile modules/tests-sos/test/apps/Makefile]) -AC_OUTPUT +AS_IF([test "$enable_pmi_mpi" = "yes" -a "$enable_multi_init" = "yes"], + [AC_MSG_WARN([Disabled multiple initilization and finalization. Feature is not supported with PMI MPI])]) + +AM_CONDITIONAL([ENABLE_MULTI_INIT], [test "$enable_multi_init" = "yes" -a "$enable_pmi_mpi" != "yes"]) + +AS_IF([test "$enable_pmi_mpi" != "yes" -a "$enable_multi_init" = "yes"], + [AC_DEFINE([USE_MULTI_INIT], [1], [If defined, multiple init will be supported.])]) AS_IF([test "$enable_pmi_mpi" != "yes" -a "$enable_pmi_simple" != "yes" -a "$opal_external_pmix_version_found" != 1 -a "$ompi_check_pmi_happy" != "yes" -a -z "$pmi_type"], [AC_MSG_ERROR([No PMI client interface was configured, consider --enable-pmi-simple or --with-pmi])]) @@ -1025,6 +1037,8 @@ AS_IF([test "$enable_fortran" = "yes" -a "$FC" = ""], AS_IF([test "$enable_fortran" = "yes"], [AC_MSG_WARN([The Fortran API was deprecated in OpenSHMEM v1.4 and removed in OpenSHMEM v1.5])]) +AC_OUTPUT + FORT="$FC" if test "$FORT" = "" ; then FORT="none" diff --git a/mpp/shmemx_c_func.h4 b/mpp/shmemx_c_func.h4 index 948146156..697288fbe 100644 --- a/mpp/shmemx_c_func.h4 +++ b/mpp/shmemx_c_func.h4 @@ -105,3 +105,6 @@ SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_create(void *base, size_t size SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_preinit(void); SHMEM_FUNCTION_ATTRIBUTES int SHPRE()shmemx_heap_preinit_thread(int requested, int *provided); SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_postinit(void); + +/* Support querying intialization status */ +SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_query_initialized(int *initialized); diff --git a/src/init.c b/src/init.c index 01ca23dfd..782da6aca 100644 --- a/src/init.c +++ b/src/init.c @@ -28,6 +28,7 @@ #include #include #include +#include #define SHMEM_INTERNAL_INCLUDE #include "shmem.h" @@ -37,6 +38,7 @@ #include "runtime.h" #include "build_info.h" #include "shmem_team.h" +#include "shmem_atomic.h" #if defined(ENABLE_REMOTE_VIRTUAL_ADDRESSING) && defined(__linux__) #include @@ -89,10 +91,10 @@ int shmem_external_heap_device = -1; int shmem_internal_my_pe = -1; int shmem_internal_num_pes = -1; -int shmem_internal_initialized = 0; -int shmem_internal_finalized = 0; +shmem_internal_cntr_t shmem_internal_init_cntr = 0; int shmem_internal_initialized_with_start_pes = 0; int shmem_internal_global_exit_called = 0; +int shmem_internal_inited_once = 0; int shmem_internal_thread_level; @@ -134,44 +136,58 @@ shmem_internal_randr_fini(void) return; } - static void -shmem_internal_shutdown(void) +shmem_release_resources(void) { - if (!shmem_internal_initialized || - shmem_internal_finalized) { - return; + if (shmem_internal_params.DEBUG) { + DEBUG_MSG("releasing resources\n"); } - shmem_internal_barrier_all(); - - shmem_internal_finalized = 1; - shmem_internal_team_fini(); - shmem_transport_fini(); - shmem_shr_transport_fini(); - SHMEM_MUTEX_DESTROY(shmem_internal_mutex_alloc); - shmem_internal_randr_fini(); - shmem_internal_symmetric_fini(); shmem_runtime_fini(); } +static void +shmem_internal_shutdown(void) +{ + if (!shmem_internal_cntr_read(&shmem_internal_init_cntr)) { + RAISE_WARN_STR("too many calls to shmem_finalize"); + return; + } + + shmem_internal_barrier_all(); + +#ifndef USE_MULTI_INIT + shmem_release_resources(); +#else + shmem_internal_cntr_dec(&shmem_internal_init_cntr); +#endif + + return; +} + + static void shmem_internal_shutdown_atexit(void) { - if ( shmem_internal_initialized && !shmem_internal_finalized && - !shmem_internal_initialized_with_start_pes && !shmem_internal_global_exit_called && - shmem_internal_my_pe == 0) { - RAISE_WARN_STR("shutting down without a call to shmem_finalize"); + if (shmem_internal_cntr_read(&shmem_internal_init_cntr) && + !shmem_internal_initialized_with_start_pes && + !shmem_internal_global_exit_called && + shmem_internal_my_pe == 0) { + + RAISE_WARN_STR("too few calls to shmem_finalize"); } - shmem_internal_shutdown(); +#ifdef USE_MULTI_INIT + shmem_release_resources(); +#endif + } @@ -518,7 +534,6 @@ shmem_internal_heap_postinit(void) randr_initialized = 1; atexit(shmem_internal_shutdown_atexit); - shmem_internal_initialized = 1; /* finish up */ #ifndef USE_PMIX @@ -554,11 +569,25 @@ shmem_internal_init(int tl_requested, int *tl_provided) { int ret; - ret = shmem_internal_heap_preinit(tl_requested, tl_provided); - if (ret) goto cleanup; +#ifndef USE_MULTI_INIT + if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) { + RAISE_ERROR_STR("attempt to reinitialize library"); + } +#endif - ret = shmem_internal_heap_postinit(); - if (ret) goto cleanup; + if (shmem_internal_cntr_fadd(&shmem_internal_init_cntr, 1) == 0 + && !shmem_internal_inited_once) { + + shmem_internal_inited_once = 1; + + ret = shmem_internal_heap_preinit(tl_requested, tl_provided); + if (ret) goto cleanup; + + ret = shmem_internal_heap_postinit(); + if (ret) goto cleanup; + } else { + *tl_provided = shmem_internal_thread_level; + } return 0; diff --git a/src/init_c.c b/src/init_c.c index 80ba3afbc..130cbba6c 100644 --- a/src/init_c.c +++ b/src/init_c.c @@ -20,6 +20,7 @@ #include "shmem.h" #include "shmem_internal.h" #include "shmemx.h" +#include "shmem_atomic.h" #ifdef ENABLE_PROFILING #include "pshmem.h" @@ -59,10 +60,13 @@ #endif /* ENABLE_PROFILING */ +extern shmem_internal_cntr_t shmem_internal_init_cntr; + + void SHMEM_FUNCTION_ATTRIBUTES start_pes(int npes) { - if (!shmem_internal_initialized) { + if (!shmem_internal_cntr_read(&shmem_internal_init_cntr)) { shmem_internal_start_pes(npes); } } @@ -73,10 +77,6 @@ shmem_init(void) { int tl_provided, ret; - if (shmem_internal_initialized) { - RAISE_ERROR_STR("attempt to reinitialize library"); - } - ret = shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided); if (ret) abort(); } @@ -87,7 +87,7 @@ shmemx_heap_preinit(void) { int tl_provided, ret; - if (shmem_internal_initialized) { + if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) { RAISE_ERROR_STR("attempt to reinitialize library"); } @@ -108,9 +108,6 @@ int SHMEM_FUNCTION_ATTRIBUTES shmem_init_thread(int tl_requested, int *tl_provided) { int ret; - if (shmem_internal_initialized) { - RAISE_ERROR_STR("attempt to reinitialize library"); - } ret = shmem_internal_init(tl_requested, tl_provided); return ret; @@ -121,7 +118,7 @@ int SHMEM_FUNCTION_ATTRIBUTES shmemx_heap_preinit_thread(int tl_requested, int *tl_provided) { int ret; - if (shmem_internal_initialized) { + if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) { RAISE_ERROR_STR("attempt to reinitialize library"); } @@ -138,6 +135,11 @@ shmem_query_thread(int *provided) *provided = shmem_internal_thread_level; } +void SHMEM_FUNCTION_ATTRIBUTES +shmemx_query_initialized(int *initialized) +{ + *initialized = shmem_internal_cntr_read(&shmem_internal_init_cntr); +} void SHMEM_FUNCTION_ATTRIBUTES shmem_global_exit(int status) diff --git a/src/init_f.c b/src/init_f.c index 9822549b0..c1b95abb7 100644 --- a/src/init_f.c +++ b/src/init_f.c @@ -26,10 +26,6 @@ void SHMEM_FUNCTION_ATTRIBUTES FC_START_PES(fortran_integer_t *npes); void FC_START_PES(fortran_integer_t *npes) { - if (shmem_internal_initialized) { - RAISE_ERROR_STR("attempt to reinitialize library"); - } - shmem_internal_start_pes(*npes); } @@ -40,11 +36,6 @@ void FC_SHMEM_INIT(void) { int tl_provided; - - if (shmem_internal_initialized) { - RAISE_ERROR_STR("attempt to reinitialize library"); - } - shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided); } diff --git a/src/shmem_atomic.h b/src/shmem_atomic.h index 7fc35cccc..12a34599f 100644 --- a/src/shmem_atomic.h +++ b/src/shmem_atomic.h @@ -164,6 +164,11 @@ shmem_internal_cntr_dec(shmem_internal_cntr_t *val) { return; } +static inline +uint64_t +shmem_internal_cntr_fadd(shmem_internal_cntr_t *ptr, uint64_t value) { + return __atomic_fetch_add(ptr, value, __ATOMIC_RELEASE); +} # else /* HAVE_STDATOMIC_H */ #include @@ -197,6 +202,11 @@ shmem_internal_cntr_dec(shmem_internal_cntr_t *val) { return; } +static inline +uint64_t +shmem_internal_cntr_fadd(shmem_internal_cntr_t *ptr, uint64_t value) { + return atomic_fetch_add(ptr, value); +} # endif # else /* !define( ENABLE_THREADS ) */ @@ -228,6 +238,14 @@ shmem_internal_cntr_dec(shmem_internal_cntr_t *val) { *val = *val-1; return; } + +static inline +uint64_t +shmem_internal_cntr_fadd(shmem_internal_cntr_t *ptr, uint64_t value) { + uint64_t orig_value = *ptr; + *ptr = *ptr + value; + return orig_value; +} # endif /* ENABLE_THREADS */ #endif diff --git a/src/shmem_internal.h b/src/shmem_internal.h index 465dc736a..be95aaa08 100644 --- a/src/shmem_internal.h +++ b/src/shmem_internal.h @@ -33,7 +33,6 @@ extern int shmem_internal_my_pe; extern int shmem_internal_num_pes; -extern int shmem_internal_initialized; extern int shmem_internal_finalized; extern int shmem_internal_thread_level; @@ -189,7 +188,9 @@ extern hwloc_topology_t shmem_internal_topology; #ifdef ENABLE_ERROR_CHECKING #define SHMEM_ERR_CHECK_INITIALIZED() \ do { \ - if (!shmem_internal_initialized) { \ + int initialized = 0; \ + shmemx_query_initialized(&initialized); \ + if (!initialized) { \ RETURN_ERROR_STR(PACKAGE_NAME " library not initialized\n"); \ abort(); \ } \ diff --git a/src/symmetric_heap_c.c b/src/symmetric_heap_c.c index 30b319ea9..bb3e2e117 100644 --- a/src/symmetric_heap_c.c +++ b/src/symmetric_heap_c.c @@ -34,6 +34,7 @@ #include "shmem_comm.h" #include "shmem_collectives.h" #include "shmemx.h" +#include "shmem_atomic.h" #ifdef ENABLE_PROFILING #include "pshmem.h" @@ -73,6 +74,8 @@ #endif /* ENABLE_PROFILING */ +extern shmem_internal_cntr_t shmem_internal_init_cntr; + static char *shmem_internal_heap_curr = NULL; void* dlmalloc(size_t); @@ -439,7 +442,7 @@ shmem_malloc_with_hints(size_t size, long hints) void SHMEM_FUNCTION_ATTRIBUTES shmemx_heap_create(void *base, size_t size, int device_type, int device_index) { - if (shmem_internal_initialized) { + if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) { RAISE_WARN_MSG("Ignoring pre-setup. Heap already initialized\n"); return; }