Fix Cabana integration (#5076)

Resolve compatibility issues between external libraries.

Author: kodiakhq[bot]

.gitlab-ci.yml

@@ -149,6 +149,7 @@ fedora:41:
      myconfig: 'maxset'
      make_check_python: 'true'
      with_stokesian_dynamics: 'true'
+     with_fftw: 'false'
      cmake_params: '-D CMAKE_INCLUDE_PATH=/usr/include/mpich-x86_64 -D CMAKE_PREFIX_PATH=/usr/lib64/mpich/lib/'
   script:
     - bash maintainer/CI/build_cmake.sh
@@ -219,7 +220,7 @@ cuda12-coverage:
      with_scafacos: 'true'
      with_walberla: 'true'
      with_stokesian_dynamics: 'true'
-     with_cabana: 'true'
+     with_shared_memory_parallelism: 'true'
   script:
     - bash maintainer/CI/build_cmake.sh
   timeout: 90m
@@ -246,7 +247,7 @@ cuda12-maxset:
      with_walberla_avx: 'true'
      with_stokesian_dynamics: 'true'
      with_caliper: 'true'
-     with_cabana: 'true'
+     with_shared_memory_parallelism: 'true'
   script:
     - bash maintainer/CI/build_cmake.sh
   artifacts:

CMakeLists.txt

@@ -116,12 +116,13 @@ FetchContent_Declare(
   heffte
   GIT_REPOSITORY https://github.com/icl-utk-edu/heffte.git
   GIT_TAG        v2.4.1
+  PATCH_COMMAND  patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/heffte.patch
 )
 FetchContent_Declare(
   cabana
   GIT_REPOSITORY https://github.com/ECP-copa/Cabana.git
   GIT_TAG        7914d28708b1c67b40d5743c1fbc09e221144dd3
-  PATCH_COMMAND  patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch_cabana_doxygen.diff
+  PATCH_COMMAND  patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cabana.patch
 )
 # cmake-format: on
 
@@ -143,7 +144,8 @@ option(ESPRESSO_BUILD_WITH_WALBERLA
 option(ESPRESSO_BUILD_WITH_WALBERLA_AVX
        "Build waLBerla lattice-Boltzmann with AVX vectorization" OFF)
 option(ESPRESSO_BUILD_WITH_WALBERLA_FFT "Build waLBerla with FFT support" OFF)
-option(ESPRESSO_BUILD_WITH_CABANA "Build with Cabana and Kokkos packages" OFF)
+option(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM
+       "Build with shared memory parallelism support" OFF)
 option(ESPRESSO_BUILD_BENCHMARKS "Enable benchmarks" OFF)
 option(ESPRESSO_BUILD_WITH_VALGRIND "Build with Valgrind instrumentation" OFF)
 option(ESPRESSO_BUILD_WITH_CALIPER "Build with Caliper instrumentation" OFF)
@@ -482,19 +484,13 @@ endif()
 # Libraries
 #
 
-if(ESPRESSO_BUILD_WITH_CABANA)
-  # Find kokkos
-  if(NOT EXISTS Kokkos_DIR)
-    message(
-      FATAL_ERROR
-        "Kokkos_DIR not set. Please set it to the directory containing KokkosConfig.cmake (typically the Kokkos build folder)"
-    )
-  endif()
-
-  # Fetch Cabana 0.7.0 Cabana already uses find_package(Kokkos) internally
+if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM)
+  FetchContent_GetProperties(cabana)
+  set(Cabana_REQUIRE_HEFFTE OFF CACHE BOOL "")
   if(NOT cabana_POPULATED)
     FetchContent_MakeAvailable(cabana)
   endif()
+  find_package(Kokkos REQUIRED)
 endif()
 
 if(ESPRESSO_BUILD_WITH_FFTW)

cmake/patch_cabana_doxygen.diff renamed to cmake/cabana.patch

@@ -1,8 +1,7 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index c228d50a..00693256 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -216,12 +216,6 @@ if(Cabana_ENABLE_TESTING)
+https://github.com/ECP-copa/Cabana/pull/815
+--- CMakeLists.txt
++++ CMakeLists.txt
+@@ -216,12 +216,6 @@
    enable_testing()
  endif()
 

cmake/espresso_cmake_config.cmakein

@@ -23,7 +23,7 @@
 
 #cmakedefine ESPRESSO_BUILD_WITH_FPE
 
-#cmakedefine ESPRESSO_BUILD_WITH_CABANA
+#cmakedefine ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM
 
 #define PACKAGE_NAME "${PROJECT_NAME}"
 

cmake/espresso_unit_test.cmake

@@ -71,6 +71,7 @@ function(ESPRESSO_UNIT_TEST)
   if(NOT TEST_NUM_PROC AND ESPRESSO_MPIEXEC_GUARD_SINGLETON_NUMA AND "${TEST_DEPENDS}" MATCHES "(^|;)([Bb]oost::mpi|MPI::MPI_CXX)($|;)")
     list(APPEND TEST_ENV_VARIABLES "OMPI_MCA_hwloc_base_binding_policy=none")
   endif()
+  list(APPEND TEST_ENV_VARIABLES "OMP_PROC_BIND=false")
   set_tests_properties(
     ${TEST_NAME} PROPERTIES ENVIRONMENT "${TEST_ENV_VARIABLES}")
 

cmake/heffte.patch

@@ -0,0 +1,10 @@
+https://github.com/icl-utk-edu/heffte/issues/83
+--- cmake/FindHeffteFFTW.cmake
++++ cmake/FindHeffteFFTW.cmake
+@@ -129,4 +129,6 @@
+ # create imported target
++if(NOT TARGET Heffte::FFTW)
+ add_library(Heffte::FFTW INTERFACE IMPORTED GLOBAL)
+ target_link_libraries(Heffte::FFTW INTERFACE ${FFTW_LIBRARIES})
+ set_target_properties(Heffte::FFTW PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${FFTW_INCLUDES})
++endif()

doc/sphinx/installation.rst

@@ -642,6 +642,8 @@ using a CMake flag (see :ref:`Options and Variables`).
 - ``STOKESIAN_DYNAMICS`` Enables the Stokesian Dynamics feature
   (see :ref:`Stokesian Dynamics`). Requires BLAS and LAPACK.
 
+- ``SHARED_MEMORY_PARALLELISM`` Enables shared-memory parallelism.
+
 
 
 .. _Configuring:
@@ -804,6 +806,7 @@ The following options control features from external libraries:
 * ``ESPRESSO_BUILD_WITH_SCAFACOS``: Build with ScaFaCoS support.
 * ``ESPRESSO_BUILD_WITH_GSL``: Build with GSL support.
 * ``ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS`` Build with Stokesian Dynamics support.
+* ``ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM``: Build with shared-memory parallelism support (OpenMP, Cabana, Kokkos, etc.)
 * ``ESPRESSO_BUILD_WITH_WALBERLA``: Build with waLBerla support.
 * ``ESPRESSO_BUILD_WITH_WALBERLA_FFT``: Build waLBerla with FFT and PFFT support, used in FFT-based electrokinetics.
 * ``ESPRESSO_BUILD_WITH_WALBERLA_AVX``: Build waLBerla with AVX kernels instead of regular kernels.

maintainer/CI/build_cmake.sh

@@ -101,7 +101,7 @@ set_default_value with_asan false
 set_default_value with_static_analysis false
 set_default_value with_caliper false
 set_default_value with_fpe false
-set_default_value with_cabana false
+set_default_value with_shared_memory_parallelism false
 set_default_value myconfig "default"
 set_default_value build_procs ${ci_procs}
 set_default_value check_procs ${build_procs}
@@ -150,6 +150,7 @@ cmake_params="${cmake_params} -D ESPRESSO_BUILD_BENCHMARKS=${make_check_benchmar
 cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_CCACHE=${with_ccache}"
 cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_CALIPER=${with_caliper}"
 cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_FPE=${with_fpe}"
+cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM=${with_shared_memory_parallelism}"
 cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_HDF5=${with_hdf5}"
 cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_FFTW=${with_fftw}"
 cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_GSL=${with_gsl}"

src/config/features.def

@@ -116,4 +116,4 @@ WALBERLA_FFT external
 VALGRIND external
 CALIPER external
 FPE external
-CABANA external
+SHARED_MEMORY_PARALLELISM external

src/core/CMakeLists.txt

@@ -82,8 +82,11 @@ install(TARGETS espresso_core
         LIBRARY DESTINATION ${ESPRESSO_INSTALL_PYTHON}/espressomd)
 
 target_link_libraries(
-  espresso_core PRIVATE espresso::config espresso::utils::mpi espresso::shapes
-                        espresso::cpp_flags
+  espresso_core
+  PRIVATE
+    espresso::config espresso::utils::mpi espresso::shapes espresso::cpp_flags
+    $<$<BOOL:${ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM}>:Kokkos::kokkos>
+    $<$<BOOL:${ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM}>:Cabana::Core>
   PUBLIC espresso::utils MPI::MPI_CXX Random123 espresso::particle_observables
          Boost::serialization Boost::mpi espresso::instrumentation)
 
@@ -101,10 +104,6 @@ if(ESPRESSO_BUILD_WITH_FFTW)
   add_subdirectory(p3m)
 endif()
 
-if(ESPRESSO_BUILD_WITH_CABANA)
-  target_link_libraries(espresso_core PRIVATE Cabana::Core)
-endif()
-
 add_subdirectory(accumulators)
 add_subdirectory(analysis)
 add_subdirectory(bond_breakage)

src/core/communication.cpp

@@ -30,7 +30,7 @@
 #include <walberla_bridge/walberla_init.hpp>
 #endif
 
-#ifdef CABANA
+#ifdef SHARED_MEMORY_PARALLELISM
 #include <Cabana_Core.hpp>
 #include <Kokkos_Core.hpp>
 #endif
@@ -90,15 +90,15 @@ void init(std::shared_ptr<boost::mpi::environment> mpi_env) {
   cuda_on_program_start();
 #endif
 
-#ifdef CABANA
+#ifdef SHARED_MEMORY_PARALLELISM
   Kokkos::initialize();
 #endif
 }
 
 void deinit() {
   Communication::m_callbacks.reset();
 
-#ifdef CABANA
+#ifdef SHARED_MEMORY_PARALLELISM
   Kokkos::finalize();
 #endif
 }

src/core/p3m/CMakeLists.txt

@@ -38,8 +38,11 @@ if(ESPRESSO_BUILD_WITH_FFTW)
   )
   target_link_libraries(
     espresso_p3m
-    PRIVATE Heffte espresso::cpp_flags espresso::p3m::cpp_flags espresso::utils
-            espresso::config espresso::instrumentation MPI::MPI_CXX Boost::mpi)
+    PRIVATE
+      Heffte espresso::cpp_flags espresso::p3m::cpp_flags espresso::utils
+      espresso::config espresso::instrumentation MPI::MPI_CXX Boost::mpi
+      $<$<BOOL:${ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM}>:Kokkos::kokkos>
+      $<$<BOOL:${ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM}>:Cabana::Core>)
   set_target_properties(espresso_p3m PROPERTIES POSITION_INDEPENDENT_CODE ON)
   target_link_libraries(espresso_core PUBLIC espresso_p3m)
   install(TARGETS espresso_p3m

src/python/pypresso.cmakein

@@ -13,6 +13,9 @@ else
   PYTHONPATH="@ESPRESSO_PYTHON_DIR@"
 fi
 export PYTHONPATH
+if test -z "$OMP_PROC_BIND"; then
+  export OMP_PROC_BIND=false
+fi
 
 espresso_parse_cmake_boolean_value() {
   local varname="${2}"

testsuite/python/CMakeLists.txt

@@ -117,9 +117,12 @@ function(python_test)
     list(APPEND TEST_LABELS "parallel_odd")
   endif()
 
+  # cmake-format: off
   set_tests_properties(
     ${TEST_NAME} PROPERTIES PROCESSORS ${TEST_NUM_PROC} SKIP_RETURN_CODE 5
+                            ENVIRONMENT "OMP_PROC_BIND=false"
                             DEPENDS "${TEST_DEPENDS}" LABELS "${TEST_LABELS}")
+  # cmake-format: on
 
   set(python_tests ${python_tests} ${TEST_FILE_CONFIGURED} PARENT_SCOPE)
 endfunction(python_test)