|
| 1 | +diff --git a/CMakeLists.txt b/CMakeLists.txt |
| 2 | +index dd204aab01..541dbaeb30 100644 |
| 3 | +--- a/CMakeLists.txt |
| 4 | ++++ b/CMakeLists.txt |
| 5 | +@@ -320,6 +320,37 @@ endif() |
| 6 | + if (MFEM_USE_MPI) |
| 7 | + find_package(MPI REQUIRED) |
| 8 | + set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH}) |
| 9 | ++ # Detect the MPI implementation (vendor) once, for downstream code that needs |
| 10 | ++ # implementation-specific handling (e.g. cuDSS ships a prebuilt communication |
| 11 | ++ # layer for Open MPI only). Open MPI defines OMPI_MAJOR_VERSION and the MPICH |
| 12 | ++ # family (MPICH, Intel MPI, MVAPICH, Cray MPICH) defines MPICH_VERSION in |
| 13 | ++ # mpi.h. Results are cached in MFEM_MPI_IS_OPENMPI and MFEM_MPI_IS_MPICH for |
| 14 | ++ # reuse by any Find module or feature below. |
| 15 | ++ include(CheckCXXSourceCompiles) |
| 16 | ++ set(_mfem_mpi_req_incl_save "${CMAKE_REQUIRED_INCLUDES}") |
| 17 | ++ list(APPEND CMAKE_REQUIRED_INCLUDES ${MPI_CXX_INCLUDE_DIRS}) |
| 18 | ++ check_cxx_source_compiles(" |
| 19 | ++#include <mpi.h> |
| 20 | ++#ifndef OMPI_MAJOR_VERSION |
| 21 | ++#error MPI implementation is not Open MPI. |
| 22 | ++#endif |
| 23 | ++int main() { return 0; } |
| 24 | ++" MFEM_MPI_IS_OPENMPI) |
| 25 | ++ check_cxx_source_compiles(" |
| 26 | ++#include <mpi.h> |
| 27 | ++#ifndef MPICH_VERSION |
| 28 | ++#error MPI implementation is not MPICH. |
| 29 | ++#endif |
| 30 | ++int main() { return 0; } |
| 31 | ++" MFEM_MPI_IS_MPICH) |
| 32 | ++ set(CMAKE_REQUIRED_INCLUDES "${_mfem_mpi_req_incl_save}") |
| 33 | ++ if (MFEM_MPI_IS_OPENMPI) |
| 34 | ++ message(STATUS "MPI implementation: Open MPI") |
| 35 | ++ elseif (MFEM_MPI_IS_MPICH) |
| 36 | ++ message(STATUS "MPI implementation: MPICH-family") |
| 37 | ++ else() |
| 38 | ++ message(STATUS "MPI implementation: unrecognized") |
| 39 | ++ endif() |
| 40 | + if (MFEM_MPIEXEC) |
| 41 | + string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC}) |
| 42 | + endif() |
| 43 | +diff --git a/config/cmake/modules/FindCUDSS.cmake b/config/cmake/modules/FindCUDSS.cmake |
| 44 | +index 689151305d..6e27faa249 100644 |
| 45 | +--- a/config/cmake/modules/FindCUDSS.cmake |
| 46 | ++++ b/config/cmake/modules/FindCUDSS.cmake |
| 47 | +@@ -49,20 +49,53 @@ if (MFEM_USE_OPENMP) |
| 48 | + message(STATUS "CUDSS threading layer library: ${MFEM_CUDSS_THREADING_LIB}") |
| 49 | + endif() |
| 50 | + |
| 51 | +-# Set the full name of the cuDSS communication library if MFEM use OpenMPI. |
| 52 | +-# The communication layer library (libcudss_commlayer_mpi.so) is located under the |
| 53 | +-# cuDSS library directory by default. |
| 54 | +-# The communication layer library is used pre-built communication layers for OpenMPI |
| 55 | +-# by default. |
| 56 | ++# Set the full name of the cuDSS communication layer library when MFEM is built |
| 57 | ++# with MPI. cuDSS requires a communication layer that matches the MPI |
| 58 | ++# implementation MFEM is linked against, so we must not bake in a layer for the |
| 59 | ++# wrong implementation. NVIDIA ships a prebuilt layer for Open MPI |
| 60 | ++# (libcudss_commlayer_openmpi.so), which we locate automatically. For other |
| 61 | ++# implementations (e.g. MPICH) no prebuilt layer is shipped: a matching layer |
| 62 | ++# must be built (see cuDSS's cudss_build_commlayer.sh) and its path supplied via |
| 63 | ++# the CUDSS_COMM_LIB environment variable at runtime. These layers live under |
| 64 | ++# the cuDSS library directory by default. |
| 65 | + if (MFEM_USE_MPI) |
| 66 | +- find_file( |
| 67 | +- CUDSS_COMM_LIB |
| 68 | +- NAMES libcudss_commlayer_openmpi.so |
| 69 | +- PATHS ${CUDSS_LIBRARY_DIR} |
| 70 | +- NO_DEFAULT_PATH |
| 71 | +- ) |
| 72 | ++ # cuDSS requires a communication layer matching the MPI implementation. Use |
| 73 | ++ # the vendor detected during MPI setup (MFEM_MPI_IS_MPICH). We only deviate |
| 74 | ++ # from the prebuilt Open MPI layer when the MPI is positively identified as |
| 75 | ++ # MPICH-family, so the previous behavior is preserved when the vendor cannot |
| 76 | ++ # be determined. |
| 77 | ++ if (MFEM_MPI_IS_MPICH) |
| 78 | ++ # MPICH-family MPI: NVIDIA does not ship a prebuilt layer. Use a user-built |
| 79 | ++ # layer if one has been installed alongside cuDSS; otherwise leave |
| 80 | ++ # MFEM_CUDSS_COMM_LIB unset and rely on the CUDSS_COMM_LIB environment |
| 81 | ++ # variable at runtime. |
| 82 | ++ find_file( |
| 83 | ++ CUDSS_COMM_LIB |
| 84 | ++ NAMES libcudss_commlayer_mpich.so libcudss_commlayer_mpi.so |
| 85 | ++ PATHS ${CUDSS_LIBRARY_DIR} |
| 86 | ++ NO_DEFAULT_PATH |
| 87 | ++ ) |
| 88 | ++ else() |
| 89 | ++ # Open MPI (or an undetermined implementation): use the prebuilt Open MPI |
| 90 | ++ # communication layer shipped with cuDSS. |
| 91 | ++ find_file( |
| 92 | ++ CUDSS_COMM_LIB |
| 93 | ++ NAMES libcudss_commlayer_openmpi.so |
| 94 | ++ PATHS ${CUDSS_LIBRARY_DIR} |
| 95 | ++ NO_DEFAULT_PATH |
| 96 | ++ ) |
| 97 | ++ endif() |
| 98 | ++ |
| 99 | + if (NOT DEFINED MFEM_CUDSS_COMM_LIB AND CUDSS_COMM_LIB) |
| 100 | + set(MFEM_CUDSS_COMM_LIB "${CUDSS_COMM_LIB}") |
| 101 | + endif() |
| 102 | +- message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}") |
| 103 | ++ |
| 104 | ++ if (MFEM_CUDSS_COMM_LIB) |
| 105 | ++ message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}") |
| 106 | ++ else() |
| 107 | ++ message(STATUS |
| 108 | ++ "CUDSS communication layer library not found for the detected MPI " |
| 109 | ++ "implementation; set the CUDSS_COMM_LIB environment variable at runtime to " |
| 110 | ++ "a layer matching your MPI (see cuDSS's cudss_build_commlayer.sh).") |
| 111 | ++ endif() |
| 112 | + endif() |
| 113 | +diff --git a/config/defaults.mk b/config/defaults.mk |
| 114 | +index 3fa1c2e20c..ab9766d198 100644 |
| 115 | +--- a/config/defaults.mk |
| 116 | ++++ b/config/defaults.mk |
| 117 | +@@ -376,8 +376,17 @@ CUDSS_OPT = -I$(CUDSS_INCLUDE_DIR) |
| 118 | + CUDSS_LIB = \ |
| 119 | + $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss |
| 120 | + # The cuDSS communication and threading libraries. |
| 121 | ++# NVIDIA ships a prebuilt cuDSS communication layer for Open MPI only, and cuDSS |
| 122 | ++# requires a layer matching the MPI implementation. Detect MPICH-family MPI |
| 123 | ++# (MPICH, Intel MPI, MVAPICH, Cray MPICH all define MPICH_VERSION in mpi.h) so |
| 124 | ++# we do not bake in the Open MPI layer for a non-Open MPI build. This mirrors |
| 125 | ++# the CMake MFEM_MPI_IS_MPICH detection and is evaluated lazily, i.e. only when |
| 126 | ++# MFEM_CUDSS_COMM_LIB is expanded (cuDSS builds). For MPICH-family MPI, build a |
| 127 | ++# matching layer (see cuDSS's cudss_build_commlayer.sh) and set CUDSS_COMM_LIB. |
| 128 | ++MFEM_MPI_IS_MPICH = $(shell $(MPICXX) -E -dM -include mpi.h -x c++ /dev/null 2>/dev/null | grep -c 'define MPICH_VERSION') |
| 129 | + MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\ |
| 130 | +- $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so)))) |
| 131 | ++ $(if $(filter-out 0,$(MFEM_MPI_IS_MPICH)),,\ |
| 132 | ++ $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))) |
| 133 | + MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\ |
| 134 | + $(subst @MFEM_DIR@,$(MFEM_DIR),$(CUDSS_LIBRARY_DIR)/libcudss_mtlayer_gomp.so)))) |
| 135 | + |
0 commit comments