improve cmake support for charm on top of reconverse (#3903)

Author: JiakunYan

.github/workflows/ci.yaml

@@ -443,3 +443,32 @@ jobs:
       run: |
         make -C netlrts-win-x86_64/tests test TESTOPTS="++local"
 
+  multicore-linux-x86_64_reconverse:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y cmake libfabric-bin libfabric-dev
+    - name: build
+      run: ./build charm++ multicore-linux-x86_64 -j3 --with-production 
+    - name: test
+      run: |
+        multicore-linux-x86_64/_deps/lci-src/lcrun -n 2 multicore-linux-x86_64/bin/ckhello +pe 4
+
+  multicore-darwin-x86_64_reconverse:
+    timeout-minutes: 60
+    runs-on: macos-13
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Dependencies
+      run: |
+        brew update
+        brew install cmake libfabric
+    - name: build
+      run: ./build charm++ multicore-darwin-x86_64 -j3 --with-production 
+    - name: test
+      run: |
+        multicore-darwin-x86_64/_deps/lci-src/lcrun -n 2 multicore-darwin-x86_64/bin/ckhello +pe 4

CMakeLists.txt

@@ -187,7 +187,6 @@ option(DRONE_MODE          "Enable drone mode" OFF)
 option(TASK_QUEUE          "Enable task queue" OFF)
 option(RECONVERSE          "Use the Reconverse communication layer" OFF)
 
-
 if(TRACING STREQUAL "")
   if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(TRACING 1)
@@ -273,7 +272,6 @@ else()
   set(CMK_RECONVERSE 0)
 endif()
 
-
 if(${AMPI_MPICH_TESTS})
   add_definitions(-DAMPI_ERRHANDLER_RETURN=1)
   set(BUILD_MPICH_TESTS true)
@@ -457,7 +455,11 @@ file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/bin/)
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/include/cklibs)
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib_so)
+if(BUILD_SHARED)
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib_so)
+else()
+  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+endif()
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
 if (NOT CMAKE_INSTALL_PREFIX)
@@ -469,6 +471,12 @@ if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release")
 endif()
 
+if(RECONVERSE)
+  # This needs to be include before we set charmc as the linker
+  # but after CMAKE_LIBRARY_OUTPUT_DIRECTORY has been set
+  add_subdirectory(cmake/fetch_reconverse)
+endif()
+
 if (NOT CMK_MEMPOOL_CUTOFFNUM)
   set(CMK_MEMPOOL_CUTOFFNUM 28)
 endif()
@@ -1017,6 +1025,9 @@ if(${TARGET} STREQUAL "charm4py")
 else()
   # Check that we are able to build and link an executable
   add_executable(ckhello ${CMAKE_SOURCE_DIR}/tests/charm++/simplearrayhello/hello.C)
+  if(RECONVERSE)
+    target_link_libraries(ckhello PRIVATE reconverse)
+  endif()
   add_dependencies(ckhello ck ckqt conv-static
     converse ckmain
     moduleNDMeshStreamer modulecompletion)

buildcmake

@@ -131,7 +131,10 @@ opt_shmem=0
 opt_xpmem=0
 opt_qlogic=0
 opt_randomized_msgq=0
-opt_reconverse=0
+opt_reconverse=1 # fake argument for now
+opt_fetch_reconverse_tag=""
+opt_fetch_reconverse_dir=""
+opt_reconverse_fetch_lci=1
 opt_refnum_type="unsigned short"
 opt_replay=0
 opt_shrinkexpand=0
@@ -144,6 +147,7 @@ opt_tcp=0
 opt_tracing= #undef
 opt_tracing_commthread=0
 opt_zlib=1
+declare -a opt_cmake_extra_args=()
 
 # default to not building ROMIO on AMPI due to GCC 14 cascade failures
 case "$actual_triplet" in
@@ -352,6 +356,15 @@ function processArgs() {
       --enable-reconverse)
         opt_reconverse=1
         ;;
+      --with-fetch-reconverse-tag=*)
+        opt_fetch_reconverse_tag=${arg#*=}
+        ;;
+      --with-fetch-reconverse-dir=*)
+        opt_fetch_reconverse_dir=${arg#*=}
+        ;;
+      --without-reconverse-fetch-lci)
+        opt_reconverse_fetch_lci=0
+        ;;
       --enable-controlpoint)
         opt_controlpoint=1
         ;;
@@ -466,6 +479,11 @@ function processArgs() {
       --incdir=*)
         opt_incdir+=("-I${arg#*=}")
         ;;
+      --with-cmake-args=*)
+        # Split the value into words (quotes preserved by the shell before this point)
+        read -r -a _vals <<< "${arg#*=}"
+        opt_cmake_extra_args+=("${_vals[@]}")
+        ;;
       *)
         echo "*** Note: Adding unknown option '$arg' to compiler flags."
         opt_extra_opts+=("$arg")
@@ -638,6 +656,17 @@ fi
 
 # Run configure step
 
+CMAKE_EXTRA_ARGS=()
+if [[ -n "$opt_fetch_reconverse_tag" ]]; then
+  CMAKE_EXTRA_ARGS+=(-DAUTOFETCH_RECONVERSE_TAG="$opt_fetch_reconverse_tag")
+fi
+if [[ -n "$opt_fetch_reconverse_dir" ]]; then
+  CMAKE_EXTRA_ARGS+=(-DFETCHCONTENT_SOURCE_DIR_RECONVERSE="$opt_fetch_reconverse_dir")
+fi
+if (( ${#opt_cmake_extra_args[@]} > 0 )); then
+  CMAKE_EXTRA_ARGS+=("${opt_cmake_extra_args[@]}")
+fi
+
 CC=$opt_CC CXX=$opt_CXX FC=$opt_FC cmake "$my_srcdir" \
   -G "Unix Makefiles" \
   -DARCH="$opt_arch" \
@@ -675,6 +704,7 @@ CC=$opt_CC CXX=$opt_CXX FC=$opt_FC cmake "$my_srcdir" \
   -DQLOGIC="$opt_qlogic" \
   -DRANDOMIZED_MSGQ="$opt_randomized_msgq" \
   -DRECONVERSE="$opt_reconverse" \
+  -DRECONVERSE_AUTOFETCH_LCI2="$opt_reconverse_fetch_lci" \
   -DREFNUM_TYPE="$opt_refnum_type" \
   -DREPLAY="$opt_replay" \
   -DSHRINKEXPAND="$opt_shrinkexpand" \
@@ -688,7 +718,8 @@ CC=$opt_CC CXX=$opt_CXX FC=$opt_FC cmake "$my_srcdir" \
   -DTRACING_COMMTHREAD="$opt_tracing_commthread" \
   -DCXI="$opt_cxi" \
   -DCMK_BUILD_OFI="$opt_build_ofi" \
-  -DZLIB="$opt_zlib"
+  -DZLIB="$opt_zlib" \
+  "${CMAKE_EXTRA_ARGS[@]}"
 
 
 # Run build step

cmake/converse.cmake

@@ -59,9 +59,6 @@ set(conv-core-cxx-sources
     src/conv-core/hrctimer.C
 )
 
-set(reconverse-h-sources
-    reconverse/include/converse.h)
-
 #set(reconverse-comm-backend-sources
 #    reconverse/comm_backend/comm_backend_internal.h
 #    reconverse/comm_backend/comm_backend.h)
@@ -237,13 +234,9 @@ file(WRITE ${CMAKE_BINARY_DIR}/include/topomanager_config.h "// empty\n" )
 # )
 # add_dependencies(converse hwloc)
 
-add_subdirectory(reconverse)
-# add_dependencies(converse reconverse)
-
 add_library(charm_cxx_utils STATIC
     ${conv-util-cxx-sources})
 
-add_library(converse INTERFACE)
 
 add_library(topomanager STATIC
     ${tmgr-cxx-sources}
@@ -253,12 +246,14 @@ target_include_directories(topomanager PUBLIC
     src/util/topomanager
     ${CMAKE_BINARY_DIR}/include)
 
-target_link_libraries(converse INTERFACE
-    reconverse
-    topomanager
-    charm_cxx_utils
-    hwloc
-)
+# add_library(converse INTERFACE)
+# target_link_libraries(converse INTERFACE
+#     reconverse
+#     topomanager
+#     charm_cxx_utils
+# )
+add_custom_target(converse)
+add_dependencies(converse reconverse topomanager charm_cxx_utils)
 
 #file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/include/comm_backend)
 
@@ -276,13 +271,10 @@ foreach(filename
     ${conv-util-h-sources}
     ${conv-ldb-h-sources}
     ${tmgr-h-sources}
-    ${reconverse-h-sources}
 )
     configure_file(${filename} ${CMAKE_BINARY_DIR}/include/ COPYONLY)
 endforeach()
 
-
-
 # target_include_directories(converse PRIVATE src/arch/util) # for machine*.*
 # target_include_directories(converse PRIVATE src/util) # for sockRoutines.C
 # target_include_directories(converse PRIVATE src/conv-core src/util/topomanager src/ck-ldb src/ck-perf src/ck-cp)

cmake/fetch_reconverse/CMakeLists.txt

@@ -0,0 +1,29 @@
+# This directory is solely for fetching reconverse.
+
+set_directory_properties(PROPERTIES
+  RULE_LAUNCH_COMPILE ""
+  COMPILE_OPTIONS ""
+  COMPILE_DEFINITIONS ""
+  INCLUDE_DIRECTORIES ""
+)
+
+set(AUTOFETCH_RECONVERSE_TAG
+    main
+    CACHE STRING "The tag to fetch for reconverse")
+
+include(FetchContent)
+FetchContent_Declare(
+    reconverse
+    GIT_REPOSITORY https://github.com/charmplusplus/reconverse.git
+    GIT_TAG ${AUTOFETCH_RECONVERSE_TAG}
+)
+
+# Set cmake variables for reconverse
+set(_save_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+set(BUILD_SHARED_LIBS ON CACHE INTERNAL "")
+
+FetchContent_MakeAvailable(reconverse)
+
+set(BUILD_SHARED_LIBS ${_save_BUILD_SHARED_LIBS} CACHE INTERNAL "")
+
+configure_file(${reconverse_SOURCE_DIR}/include/converse.h ${CMAKE_BINARY_DIR}/include/ COPYONLY)