YAKL and SAMXX CIME and CMake Integration Notes

The E3SM CMake files are located in E3SM/components and E3SM/components/cmake.

Adding the USE_CUDA CMake variable

This variable needs to be visible at the highest level CMakeLists.txt, so it has to be specified through CIME using the Macros.cmake file in E3SM/cime_config/machines/config_compilers.xml.

To make this variable available to add to that file, however, it had to first be added to the XML schema in cime/config/xml_schemas/config_compilers_v2.xsd. It was added in the <xs:group name="compilerVars"> group as a choice as follows:

      <xs:element name="CUDA_FLAGS" type="flagsVar"/>
      <xs:element name="USE_CUDA" type="upperBoolean"/>

Then they are added to appropriate machine, compiler configurations in config_compilers.xml as:

  <CUDA_FLAGS>
    <append> -O3 -arch sm_70 --use_fast_math </append>
  </CUDA_FLAGS>
  <USE_CUDA>TRUE</USE_CUDA>

Finally, when CMake starts its top-level CMakeLists.txt for E3SM, it sees these things set in the local Macros.cmake files in the case build directory.

Creating USE_SAMXX and USE_YAKL CMake variables

Go to E3SM/components/cmake/common_setup.cmake, and add new variables there. Use existing code to get the feel for where to test for adding a given variable. CAM often hooks into CIME through the ${CAM_CONFIG_OPTS} variable that comes from the CAM configure command. USE_YAKL and USE_SAMXX are created with the code below in cmake/build_model.cmake

# Look for -crm samxx in the CAM_CONFIG_OPTS CIME variable
# If it's found, then enable USE_SAMXX
string(FIND "${CAM_CONFIG_OPTS}" "-crm samxx" HAS_SAMXX)
if (NOT HAS_SAMXX EQUAL -1)
  # The following is for the SAMXX code:
  set(USE_SAMXX TRUE)
endif()

# If samxx is being used, then YAKL must be used as well
set(USE_YAKL ${USE_SAMXX})

# If YAKL is being used, then we need to enable USE_CXX
if (${USE_YAKL})
  set(USE_CXX TRUE)
endif()

We then add the YAKL and SAMXX builds into the cmake/build_model.cmake file inside if (COMP_NAME STREQUAL "eam") with the following code:

    # If YAKL is needed, then set YAKL CMake vars
    if (USE_YAKL)
      # YAKL_ARCH can be CUDA, HIP, SYCL, OPENMP45, or empty
      if (USE_CUDA)
        set(YAKL_ARCH "CUDA")
        # CUDA_FLAGS is set through Macros.cmake / config_compilers.xml
        # We can't have duplicate flags with nvcc, so we only specify CPPDEFS,
        # and the rest is up to CUDAFLAGS
        set(YAKL_CUDA_FLAGS "${CPPDEFS}")
      else()
        # For normal C++ compilers duplicate flags are fine, the last ones win typically
        set(YAKL_CXX_FLAGS "${CPPDEFS} ${CXXFLAGS}")
        set(YAKL_ARCH "")
      endif()
      message(STATUS "Building YAKL")
      # Build YAKL as a static library
      # YAKL_HOME is YAKL's source directlry
      set(YAKL_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../externals/YAKL)
      # YAKL_BIN is where we're placing the YAKL library
      set(YAKL_BIN  ${CMAKE_CURRENT_BINARY_DIR}/yakl)
      # Build the YAKL static library
      add_subdirectory(${YAKL_HOME} ${YAKL_BIN})
      # Add both YAKL source and YAKL binary directories to the include paths
      include_directories(${YAKL_HOME} ${YAKL_BIN})
    endif()

    # if samxx is needed, build samxx as a static library
    if (USE_SAMXX)
      message(STATUS "Building SAMXX")
      # SAMXX_HOME is where the samxx source code lives
      set(SAMXX_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/crm/samxx)
      # SAMXX_BIN is where the samxx library will live
      set(SAMXX_BIN  ${CMAKE_CURRENT_BINARY_DIR}/samxx)
      # Build the static samxx library
      add_subdirectory(${SAMXX_HOME} ${SAMXX_BIN})
      # Add samxx F90 files to the main E3SM build
      set(SOURCES ${SOURCES} cmake/atm/../../eam/src/physics/crm/samxx/cpp_interface_mod.F90
                             cmake/atm/../../eam/src/physics/crm/samxx/params.F90
                             cmake/atm/../../eam/src/physics/crm/crm_ecpp_output_module.F90 )
    endif()

From here, each component (YAKL and SAMXX) can fend for itself with its own CMakeLists.txt, which will build a library. Finally, we link the library into the eam component's library with the following code near the end of cmake/build_model.cmake:

    if (COMP_NAME STREQUAL "eam")
      if (USE_YAKL)
        target_link_libraries(${TARGET_NAME} PRIVATE yakl)
      endif()
      if (USE_SAMXX)
        target_link_libraries(${TARGET_NAME} PRIVATE samxx)
      endif()
    endif()