Merge branch 'oksanaguba/eamxx/spot-clean' (PR #6916)

EAMxx: Kokkos 4.5 update, fixes and workarounds for Kokkos update, Aurora support

* Update externals/ekat/extern/kokkos to 4.5.
* Update EKAT itself to fix deprecated usage and add Aurora cmake files.
* Fix and work around various issues on Frontier due to the Kokkos update.
* Fix deprecated usage in Hommexx.

[BFB]

Author: ambrad

.github/workflows/eamxx-sa-sanitizer.yml

@@ -94,4 +94,4 @@ jobs:
           machine: ghci-snl-cuda
           generate: false
           submit: ${{ env.submit }}
-          cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}
+          cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }};SCREAM_SMALL_KERNELS=ON

.github/workflows/eamxx-sa-testing.yml

@@ -106,8 +106,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build_type: [sp, dbg, opt]
-    name: gcc-cuda / ${{ matrix.build_type }}
+        test:
+          - build_type: sp
+            SK: OFF
+          - build_type: dbg
+            SK: ON
+          - build_type: opt
+            SK: OFF
+    name: gcc-cuda / ${{ matrix.test.build_type }}
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -149,9 +155,9 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/test-all-eamxx
         with:
-          build_type: ${{ matrix.build_type }}
+          build_type: ${{ matrix.test.build_type }}
           machine: ghci-snl-cuda
           generate: ${{ env.generate }}
           submit: ${{ env.submit }}
-          cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}
+          cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }};SCREAM_SMALL_KERNELS=${{ matrix.test.SK }}
           ekat: ${{ env.ekat }}

cime_config/machines/cmake_macros/craycray-mphipcc_frontier.cmake

@@ -35,3 +35,7 @@ set(PIO_FILESYSTEM_HINTS "lustre")
 
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=OFF")
 set(USE_HIP "TRUE")
+
+# Work around a compiler-side issue (register allocation) in
+# eamxx_mam_microphysics_process_interface.cpp
+set(SCREAM_ENABLE_MAM OFF CACHE STRING "")

cime_config/machines/cmake_macros/oneapi-ifx.cmake

@@ -4,15 +4,15 @@ if (compile_threaded)
   string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
   string(APPEND CMAKE_EXE_LINKER_FLAGS  " -qopenmp")
 endif()
-string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2 -gline-tables-only -g")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2 -gline-tables-only -g")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -gline-tables-only -g")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
-string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")
-string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
-string(APPEND CMAKE_Fortran_FLAGS   " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
+string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99 -gline-tables-only -g")
+string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g")
+string(APPEND CMAKE_Fortran_FLAGS   " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g")
 string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
@@ -23,4 +23,6 @@ set(MPICXX "mpicxx")
 set(SCC "icx")
 set(SCXX "icpx")
 set(SFC "ifx")
-set(E3SM_LINK_WITH_FORTRAN "TRUE")
+
+
+#set(E3SM_LINK_WITH_FORTRAN "TRUE")

cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake

@@ -4,15 +4,17 @@ if (compile_threaded)
   string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
   string(APPEND CMAKE_EXE_LINKER_FLAGS  " -qopenmp")
 endif()
-string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
-string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
-string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
+
+# 'just' -g may lead to linker internal errors and/or huge builds out of quotas
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2 -g -gline-tables-only")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2 -fpscomp logicals -g -gline-tables-only")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -g -gline-tables-only")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g -fpscomp logicals -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
 string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
 string(APPEND CMAKE_C_FLAGS   " -fp-model precise -std=gnu99")
 string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
-string(APPEND CMAKE_Fortran_FLAGS   " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
+string(APPEND CMAKE_Fortran_FLAGS   " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
 string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
 string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
 string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")

cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake

@@ -1,8 +1,12 @@
 
-string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax")
 if (compile_threaded)
   string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
 endif()
-string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ")
+
+string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
+string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
 string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ")
 string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")
+
+set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")

cime_config/machines/config_machines.xml

@@ -3434,17 +3434,17 @@
     <BATCH_SYSTEM>pbspro</BATCH_SYSTEM>
     <SUPPORTED_BY>e3sm</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>208</MAX_TASKS_PER_NODE>
-    <MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">96</MAX_TASKS_PER_NODE>
+    <MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>104</MAX_MPITASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">48</MAX_MPITASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
        <executable>mpiexec</executable>
        <!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
        <arguments>
           <arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
           <arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
-          <arg name="ranks_bind">-envall</arg>
+          <arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND}</arg>
           <arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
           <arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
        </arguments>
@@ -3462,9 +3462,9 @@
         <modules compiler="!gnu">
           <command name="load">oneapi/eng-compiler/2024.07.30.002</command>
         </modules>
-        <modules compiler="oneapi-ifxgpu">
-          <command name="load">kokkos/4.4.01-omp-sycl</command>
-        </modules>
+	<!--<modules compiler="oneapi-ifxgpu"> -->
+	<!--          <command name="load">kokkos/4.4.01-omp-sycl</command> -->
+        <!--</modules> -->
         <modules compiler="gnu">
            <command name="unload">spack-pe-gcc cmake</command>
            <command name="load">gcc/10.3.0</command>
@@ -3478,25 +3478,40 @@
         <env name="PNETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002</env>
         <env name="LD_LIBRARY_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:$ENV{LD_LIBRARY_PATH}</env>
         <env name="PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:$ENV{PATH}</env>
+        <env name="RANKS_BIND">list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203</env>
      </environment_variables>
      <environment_variables DEBUG="TRUE">
         <env name="HYDRA_TOPO_DEBUG">1</env>
      </environment_variables>
      <environment_variables compiler="oneapi-ifxgpu">
         <env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
-        <env name="ONEAPI_MPICH_GPU">NO_GPU</env>
-        <env name="MPIR_CVAR_ENABLE_GPU">0</env>
+
+	<!--        <env name="ONEAPI_MPICH_GPU">NO_GPU</env> -->
+
+	<env name="MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE"></env>
+        <env name="MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE"></env>
+        <env name="MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE"></env>
+        <env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
+        <env name="UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS">1</env>
+        <env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>
+	<!--<env name="FI_PROVIDER">cxi</env>-->
+	<env name="FI_MR_CACHE_MONITOR">disabled</env>
+        <env name="FI_CXI_OVFLOW_BUF_SIZE">8388608</env>
+        <env name="PALS_PING_PERIOD">240</env>
+        <env name="PALS_RPC_TIMEOUT">240</env>
+
+	<env name="MPIR_CVAR_ENABLE_GPU">0</env>
         <env name="romio_cb_read">disable</env>
         <env name="romio_cb_write">disable</env>
         <env name="SYCL_CACHE_PERSISTENT">1</env>
         <env name="GATOR_INITIAL_MB">4000MB</env>
         <env name="GATOR_DISABLE">0</env>
-        <env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
+        <env name="GPU_TILE_COMPACT">/lus/flare/projects/E3SM_Dec/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
         <env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
         <env name="FI_CXI_CQ_FILL_PERCENT">20</env>
-        <env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
+	<!--<env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>-->
         <env name="ZES_ENABLE_SYSMAN">1</env>
-        <env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>
+	<!--<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>-->
     </environment_variables>
     <environment_variables compiler="oneapi-ifx">
         <env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->

components/eamxx/CMakeLists.txt

@@ -97,6 +97,7 @@ option (Kokkos_ENABLE_SERIAL "" ON)
 set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "")
 set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
 set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
+set (SYCL_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
 
 # Determine if this is a Cuda build.
 if (Kokkos_ENABLE_CUDA)
@@ -106,14 +107,21 @@ if (Kokkos_ENABLE_CUDA)
   set (CUDA_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
 endif ()
 
-# Determine if this is a Cuda build.
+# Determine if this is a HIP build.
 if (Kokkos_ENABLE_HIP)
   # Add CUDA as a language for CUDA builds
   enable_language(HIP)
   set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
   set (HIP_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
 endif ()
 
+# Determine if this is a sycl build.
+if (Kokkos_ENABLE_SYCL)
+  #enable_language(SYCL)
+  set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
+  set (SYCL_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
+endif ()
+
 if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" )
   set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp")
 endif()
@@ -178,7 +186,7 @@ elseif(MACH)
 endif()
 
 set(DEFAULT_SMALL_KERNELS FALSE)
-if (Kokkos_ENABLE_HIP)
+if (Kokkos_ENABLE_HIP OR Kokkos_ENABLE_SYCL)
   set(DEFAULT_SMALL_KERNELS TRUE)
 endif()
 
@@ -197,7 +205,13 @@ set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to n
 set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation")
 set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine")
 option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON)
-option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON)
+
+if (Kokkos_ENABLE_SYCL)
+  option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF)
+else()
+  option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON)
+endif()
+
 set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them")
 set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only")
 set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only")

components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/aurora_run/shell_commands

@@ -0,0 +1,6 @@
+
+$CIMEROOT/../components/eamxx/scripts/atmchange transport_alg=0 -b
+$CIMEROOT/../components/eamxx/scripts/atmchange hypervis_subcycle_q=1 -b
+$CIMEROOT/../components/eamxx/scripts/atmchange dt_tracer_factor=1 -b
+$CIMEROOT/../components/eamxx/scripts/atmchange tstep_type=9 -b
+$CIMEROOT/../components/eamxx/scripts/atmchange theta_hydrostatic_mode=False -b

components/eamxx/cmake/machine-files/aurora.cmake

@@ -1,14 +1,25 @@
 include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
 common_setup()
 
-include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
+include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake)
 include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)
-
 set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
 set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
 set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
 set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")
 
-set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}")
-set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}")
-set(PNETCDF_PATH "$ENV{PNETCDF_PATH}")
+SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
+SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel  -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"")
+
+set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE)
+
+#this is needed for cime builds!
+set(NETCDF_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
+set(NETCDF_DIR  "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
+set(NETCDF_C_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
+set(NETCDF_C      "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
+
+