Merge pull request #489 from eagles-project/mjs/generalize_4_HIP

Modify CMake config to work with HIP

Author: mjschmidt271

CMakeLists.txt

@@ -170,14 +170,26 @@ if (NOT HAERO_ENABLE_GPU AND NOT APPLE)
   endif()
 endif()
 
+set(HAERO_AMD_GPU FALSE)
+# NOTE: this is hacky, but done as a result of not knowing what the
+# Kokkos_ARCH_$DEVICE_ARCH variable will be named
+if ("${HAERO_DEVICE_ARCH}" MATCHES "AMD")
+  set(HAERO_AMD_GPU 1)
+endif()
+
 # Other third-party libraries.
 add_subdirectory(ext)
 
 # If we're building on a GPU, set the C++ compiler appropriately.
 if (HAERO_ENABLE_GPU)
-  set(CMAKE_CXX_COMPILER ${EKAT_NVCC_WRAPPER})
-  # the esa_on_defaulted_function_ignored supresses a warning about KOKKOS markups of defaulted constructors/destructors
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr -Xcudafe --diag_suppress=esa_on_defaulted_function_ignored")
+  if (HAERO_AMD_GPU)
+    set(CMAKE_CXX_COMPILER "hipcc")
+    set(CMAKE_CXX_FAGS "${CMAKE_CXX_FLAGS} --nocudalib --nocudainc")
+  else()
+    set(CMAKE_CXX_COMPILER ${EKAT_NVCC_WRAPPER})
+    # the esa_on_defaulted_function_ignored supresses a warning about KOKKOS markups of defaulted constructors/destructors
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr -Xcudafe --diag_suppress=esa_on_defaulted_function_ignored")
+  endif()
   # Avoid buggy experimental Kokkos CUDA stuff.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DKokkos_CXX_STANDARD=C++17")
 endif()

cmake/haero.cmake.in

@@ -16,11 +16,6 @@ set(HAERO_ENABLE_GPU  @HAERO_ENABLE_GPU@)
 
 # C++ compiler settings
 set(HAERO_CXX_STANDARD @CMAKE_CXX_STANDARD@)
-if (HAERO_ENABLE_GPU)
-  set(HAERO_CXX_COMPILER @CMAKE_INSTALL_PREFIX@/bin/nvcc_wrapper)
-else()
-  set(HAERO_CXX_COMPILER @CMAKE_CXX_COMPILER@)
-endif()
 set(HAERO_CXX_FLAGS    "@CMAKE_CXX_FLAGS@ -DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX")
 
 # C compiler settings
@@ -45,8 +40,12 @@ add_library(kokkos INTERFACE)
 target_link_libraries(kokkos INTERFACE kokkoscontainers kokkoscore)
 
 if (HAERO_ENABLE_GPU)
-  # FIXME: this assumes an NVIDIA GPU!
-  set(HAERO_LIBRARIES @HAERO_LIBRARIES@;kokkos;cuda;m)
+  if (HAERO_AMD_GPU)
+    set(HAERO_LIBRARIES @HAERO_LIBRARIES@;kokkos;rocm;m)
+  else()
+    set(HAERO_LIBRARIES @HAERO_LIBRARIES@;kokkos;cuda;m)
+  endif()
 else()
+  set(HAERO_CXX_COMPILER @CMAKE_CXX_COMPILER@)
   set(HAERO_LIBRARIES @HAERO_LIBRARIES@;kokkos;m)
 endif()

ext/CMakeLists.txt

@@ -20,9 +20,9 @@ if (HAERO_BUILDS_EKAT)
   include(EkatBuildEkat)
   set(EKAT_ENABLE_MPI ${HAERO_ENABLE_MPI} CACHE BOOL "Enable MPI")
   set(EKAT_ENABLE_FORTRAN OFF CACHE BOOL "Enable EKAT Fortran support")
-  set(EKAT_ENABLE_TESTS OFF CACHE BOOL "Disable EKAT tests")
-  set(Kokkos_ENABLE_LIBDL OFF CACHE BOOL "Disable broken Kokkos libdl support")
-  set(Kokkos_ENABLE_DEPRECATED_CODE_3 OFF CACHE BOOL "Disable deprecated code to avoid warnings of using deprecated functions.")
+  set(EKAT_ENABLE_TESTS OFF CACHE BOOL "Enable EKAT tests")
+  set(Kokkos_ENABLE_LIBDL OFF CACHE BOOL "Enable broken Kokkos libdl support")
+  set(Kokkos_ENABLE_DEPRECATED_CODE_3 OFF CACHE BOOL "Enable deprecated code to avoid warnings of using deprecated functions.")
   if (HAERO_SKIP_FIND_YAML_CPP)
     set(EKAT_SKIP_FIND_YAML_CPP ON CACHE BOOL "EKAT will build yaml-cpp ")
     message(STATUS "SKIP FIND YAML CPP")
@@ -43,18 +43,27 @@ if (HAERO_BUILDS_EKAT)
     set(EKAT_ENABLE_FPE OFF CACHE BOOL "")
   endif()
   if (HAERO_ENABLE_GPU)
+    if (HAERO_AMD_GPU)
+      set(Kokkos_ENABLE_HIP ON CACHE BOOL "Enable HIP Kokkos backend")
+    else()
+      set(Kokkos_ENABLE_CUDA ON CACHE BOOL "Enable CUDA Kokkos backend")
+      set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Enable CUDA lambdas")
+      set(EKAT_NVCC_WRAPPER ${CMAKE_BINARY_DIR}/bin/nvcc_wrapper)
+      set(EKAT_NVCC_WRAPPER ${EKAT_NVCC_WRAPPER} PARENT_SCOPE)
+      file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/ekat/extern/kokkos/bin/nvcc_wrapper
+           DESTINATION ${CMAKE_BINARY_DIR}/bin)
+    endif()
     set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "Enable serial Kokkos backend")
-    set(Kokkos_ENABLE_CUDA ON CACHE BOOL "Enable CUDA Kokkos backend")
-    set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Enable CUDA lambdas")
   else()
     set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "Enable serial Kokkos backend")
-    set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "Disable CUDA Kokkos backend")
+    set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "Enable CUDA Kokkos backend")
+    set(Kokkos_ENABLE_HIP OFF CACHE BOOL "Enable HIP Kokkos backend")
     if (OPENMP_FOUND)
       message(STATUS "OpenMP is enabled")
       set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "Enable OpenMP Kokkos backend")
     else()
       message(STATUS "OpenMP is not supported. Using threads backend instead.")
-      set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Disable OpenMP Kokkos backend")
+      set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Enable OpenMP Kokkos backend")
       set(Kokkos_ENABLE_PTHREAD ON CACHE BOOL "Enable pthreads Kokkos backend")
     endif()
   endif()
@@ -68,8 +77,6 @@ if (HAERO_BUILDS_EKAT)
        ${PROJECT_BINARY_DIR}/externals/kokkos
        ${CMAKE_CURRENT_SOURCE_DIR}/ekat/extern/spdlog/include
        ${CMAKE_CURRENT_SOURCE_DIR}/ekat/extern/yaml-cpp/include)
-  file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/ekat/extern/kokkos/bin/nvcc_wrapper
-       DESTINATION ${CMAKE_BINARY_DIR}/bin)
   if (NOT TARGET yaml-cpp)
     add_library(yaml-cpp STATIC IMPORTED GLOBAL)
     if (CMAKE_BUILD_TYPE MATCHES "Debug")
@@ -91,8 +98,6 @@ if (HAERO_BUILDS_EKAT)
     endif()
   endif()
 endif()
-set(EKAT_NVCC_WRAPPER ${CMAKE_BINARY_DIR}/bin/nvcc_wrapper)
-set(EKAT_NVCC_WRAPPER ${EKAT_NVCC_WRAPPER} PARENT_SCOPE)
 
 add_library(ext_libraries INTERFACE)
 target_link_libraries (ext_libraries INTERFACE ekat;${HAERO_EXT_LIBRARIES})
@@ -108,4 +113,3 @@ set(HAERO_EXT_INCLUDE_DIRS ${HAERO_EXT_INCLUDE_DIRS} PARENT_SCOPE)
 
 install(DIRECTORY ${PROJECT_BINARY_DIR}/include/ DESTINATION include)
 install(DIRECTORY ${PROJECT_BINARY_DIR}/bin/ DESTINATION bin)
-

haero/CMakeLists.txt

@@ -44,10 +44,16 @@ target_include_directories(haero PUBLIC
   $<INSTALL_INTERFACE:include>
 )
 target_link_libraries(haero PUBLIC ext_libraries)
-target_compile_options(haero PRIVATE  $<$<COMPILE_LANGUAGE:CXX>:${KOKKOS_CUDA_OPTIONS}>)
-if(CMAKE_COMPILER_IS_GNUCXX AND HAERO_STANDALONE)
-  MESSAGE(STATUS "Using GNU compiler, turning on warnings are errors....")
-  target_compile_options(haero PRIVATE -Werror)
+if (HAERO_ENABLE_GPU)
+  if (HAERO_AMD_GPU)
+    # Nothing... yet
+  else()
+    target_compile_options(haero PRIVATE  $<$<COMPILE_LANGUAGE:CXX>:${KOKKOS_CUDA_OPTIONS}>)
+  endif()
+  if(CMAKE_COMPILER_IS_GNUCXX AND HAERO_STANDALONE)
+    MESSAGE(STATUS "Using GNU compiler, turning on warnings are errors....")
+    target_compile_options(haero PRIVATE -Werror)
+  endif()
 endif()
 set(HAERO_LIBRARIES haero;${HAERO_LIBRARIES})
 set(HAERO_LIBRARIES haero;${HAERO_LIBRARIES} PARENT_SCOPE)
@@ -73,4 +79,3 @@ install(FILES aero_process.hpp
               utils.hpp
               root_finders.hpp
         DESTINATION include/haero)
-

setup

@@ -69,6 +69,8 @@ ENABLE_GPU=OFF
 
 # OPTIONAL: Select one of the following device architectures based on the above.
 # (These options are taken from kokkos_arch.cmake)
+# Source in docs (incomplete list here):
+# https://kokkos.org/kokkos-core-wiki/get-started/configuration-guide.html
 #DEVICE_ARCH=AMDAVX          # AMD CPU
 #DEVICE_ARCH=ARMV80          # ARMv8.0 Compatible CPU
 #DEVICE_ARCH=ARMV81          # ARMv8.1 Compatible CPU
@@ -106,10 +108,15 @@ ENABLE_GPU=OFF
 #DEVICE_ARCH=TURING75        # NVIDIA Turing generation CC 7.5 (CUDA)
 #DEVICE_ARCH=AMPERE80        # NVIDIA Ampere generation CC 8.0 (CUDA)
 #DEVICE_ARCH=AMPERE86        # NVIDIA Ampere generation CC 8.6 (CUDA)
-#DEVICE_ARCH=VEGA900         # AMD GPU MI25 GFX900 (HIP)
-#DEVICE_ARCH=VEGA906         # AMD GPU MI50/MI60 GFX906 (HIP)
-#DEVICE_ARCH=VEGA908         # AMD GPU (HIP)
-#DEVICE_ARCH=VEGA90A         # AMD GPU MI200 GFX90A (HIP)
+#DEVICE_ARCH=AMD_GFX942_APU  # MI300A
+#DEVICE_ARCH=AMD_GFX942      # MI300A, MI300X
+#DEVICE_ARCH=AMD_GFX940      # MI300A (pre-production--not sure on meaning)
+#DEVICE_ARCH=AMD_GFX90A      # MI200 series
+#DEVICE_ARCH=AMD_GFX908      # MI100
+#DEVICE_ARCH=AMD_GFX906      # MI50, MI60
+#DEVICE_ARCH=AMD_GFX1103     # Ryzen 8000G Phoenix series APU
+#DEVICE_ARCH=AMD_GFX1100     # 7900xt
+#DEVICE_ARCH=AMD_GFX1030     # V620, W6800
 #DEVICE_ARCH=INTEL_GEN       # Intel GPUs (SPIR64-based devices)
 #DEVICE_ARCH=INTEL_DG1       # Intel Iris XeMAX GPU
 #DEVICE_ARCH=INTEL_GEN9      # Intel GPU Gen9
@@ -142,6 +149,14 @@ PRECISION=double
 CXX=c++
 CC=cc
 
+# NVIDIA GPU + gcc
+# CXX=[PATH]/nvcc_wrapper
+# CC=gcc
+
+# AMD GPU
+# CXX=hipcc
+# CC=amdclang
+
 # Override compilers here (ONLY if you know what you're doing!).
 
 # C++ compiler.
@@ -159,7 +174,7 @@ pushd "\$SOURCE_DIR"/machines >& /dev/null
 for MACHINE_FILE in \$(ls)
 do
   MACHINE=\${MACHINE_FILE/\.sh/}
-  echo \`hostname\` | grep -q "\$MACHINE" 
+  echo \`hostname\` | grep -q "\$MACHINE"
   host_match=\$?
   echo \$SYSTEM_NAME | grep -q "\$MACHINE"
   sys_match=\$?
@@ -193,6 +208,7 @@ fi
 # Set the device architecture if needed.
 if [ ! "\$DEVICE_ARCH" = "" ]; then
   OPTIONS="\$OPTIONS -DKokkos_ARCH_\$DEVICE_ARCH:BOOL=ON"
+  OPTIONS="\$OPTIONS -DHAERO_DEVICE_ARCH=\$DEVICE_ARCH"
 fi
 
 # Configure ekat if needed.
@@ -241,4 +257,3 @@ echo "  1. cd $1"
 echo "  2. Edit config.sh"
 echo "  3. source config.sh"
 echo "  4. Build using 'make -j'."
-