Skip to content

EAMxx: Kokkos 4.5 update, fixes and workarounds for Kokkos update, Aurora support #6916

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 40 commits into from
Mar 26, 2025
Merged

EAMxx: Kokkos 4.5 update, fixes and workarounds for Kokkos update, Aurora support #6916

Show file tree
Hide file tree
Changes from 34 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
d3ccf36
cmake macros files
oksanaguba Jan 18, 2025
b803734
revert to master version of xml
oksanaguba Jan 18, 2025
ce03bcd
update cmake file for eamxx
oksanaguba Jan 18, 2025
98bdad0
update eamxx cmake machine files
oksanaguba Jan 18, 2025
5e8e8fe
explictly set enum type to int
oksanaguba Jan 18, 2025
fe8f4be
disable compose for now
oksanaguba Jan 18, 2025
311fee4
set team size to 4 for SYCL
oksanaguba Jan 18, 2025
69ff997
add comment
oksanaguba Jan 18, 2025
0e7f948
switch to kokkos::printf in p3 test
oksanaguba Jan 18, 2025
ff8a396
yakl fixes for rrtmgp, unsorted fixes for io
oksanaguba Jan 18, 2025
608e68d
compile template fix
oksanaguba Jan 18, 2025
2c1e15f
switch to kokkos::printf in shoc
oksanaguba Jan 18, 2025
0292883
remove need for c_bool
oksanaguba Jan 19, 2025
1f54616
shell comm for aurora ers test
oksanaguba Jan 19, 2025
e84ee19
switch to ekat branch for sycl
oksanaguba Jan 19, 2025
013eb67
using team_size as for hip in remapper
oksanaguba Jan 20, 2025
b7ec987
cleaning out IO cmake vars for rrtmgp
oksanaguba Jan 20, 2025
9da1b24
fix comment
oksanaguba Jan 21, 2025
1604057
fix cmake vars for mam, compose
oksanaguba Jan 21, 2025
cbde44f
Hommexx/SL: Code updates to support Kokkos update.
ambrad Jan 27, 2025
e5213b9
fix for compose cmake var
oksanaguba Jan 28, 2025
acd9910
remove cmake comments
oksanaguba Jan 28, 2025
db10855
revert batch file to master
oksanaguba Jan 29, 2025
c5c5df1
replace old io paths in eamxx
oksanaguba Jan 30, 2025
921d223
Homme/eamxx: fix cmake syntax to link libraries
bartgol Jan 30, 2025
341a7b1
EAMxx: Fix logical-vs-integer issue.
ambrad Jan 30, 2025
0cf3ced
fix MAM variable
oksanaguba Feb 2, 2025
480dd95
Hommexx: Remove use of a quantity Kokkos no longer provides.
ambrad Feb 4, 2025
db441ba
Hommexx: remove deprecated (and unused by Kokkkos) set_num_device usage
bartgol Feb 6, 2025
6a21177
aurora entry for gpu runs, perf restored
oksanaguba Feb 6, 2025
f077eac
EAMxx: Turn MAM off in old Frontier s/w stack due to compiler-side is…
ambrad Feb 12, 2025
c8384ac
Workflows: hard-code SCREAM_SMALL_KERNELS=ON for CUDA dbg eamxx builds
bartgol Feb 15, 2025
eb67265
make homme standalone build on aurora again
oksanaguba Mar 2, 2025
8762ee9
EAMxx: Fix find_library hints in RRTMGP cmake.
ambrad Mar 8, 2025
d13f65a
fix kokkos rank/gpu warnings
oksanaguba Mar 13, 2025
14dcb03
delete eamxx machine files not in use
oksanaguba Mar 13, 2025
ae3e033
EKAT: Update submodule.
ambrad Mar 13, 2025
2a0b296
fix ranks
oksanaguba Mar 14, 2025
140b3a3
delete not used cmake files
oksanaguba Mar 17, 2025
d528c20
remove mpiexec params, no deterioration of perf on 450 nodes
oksanaguba Mar 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/eamxx-sa-sanitizer.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,4 +94,4 @@ jobs:
machine: ghci-snl-cuda
generate: false
submit: ${{ env.submit }}
cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}
cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }};SCREAM_SMALL_KERNELS=ON
14 changes: 10 additions & 4 deletions .github/workflows/eamxx-sa-testing.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,14 @@ jobs:
strategy:
fail-fast: false
matrix:
build_type: [sp, dbg, opt]
name: gcc-cuda / ${{ matrix.build_type }}
test:
- build_type: sp
SK: OFF
- build_type: dbg
SK: ON
- build_type: opt
SK: OFF
name: gcc-cuda / ${{ matrix.test.build_type }}
steps:
- name: Check out the repository
uses: actions/checkout@v4
Expand Down Expand Up @@ -148,9 +154,9 @@ jobs:
- name: Run tests
uses: ./.github/actions/test-all-eamxx
with:
build_type: ${{ matrix.build_type }}
build_type: ${{ matrix.test.build_type }}
machine: ghci-snl-cuda
generate: ${{ env.generate }}
submit: ${{ env.submit }}
cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}
cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }};SCREAM_SMALL_KERNELS=${{ matrix.test.SK }}
ekat: ${{ env.ekat }}
4 changes: 4 additions & 0 deletions cime_config/machines/cmake_macros/craycray-mphipcc_frontier.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,7 @@ set(PIO_FILESYSTEM_HINTS "lustre")

string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=OFF")
set(USE_HIP "TRUE")

# Work around a compiler-side issue (register allocation) in
# eamxx_mam_microphysics_process_interface.cpp
set(SCREAM_ENABLE_MAM OFF CACHE STRING "")
18 changes: 10 additions & 8 deletions cime_config/machines/cmake_macros/oneapi-ifx.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ if (compile_threaded)
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -gline-tables-only -g")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -gline-tables-only -g")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -gline-tables-only -g")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99 -gline-tables-only -g")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g")
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
Expand All @@ -23,4 +23,6 @@ set(MPICXX "mpicxx")
set(SCC "icx")
set(SCXX "icpx")
set(SFC "ifx")
set(E3SM_LINK_WITH_FORTRAN "TRUE")


#set(E3SM_LINK_WITH_FORTRAN "TRUE")
19 changes: 19 additions & 0 deletions cime_config/machines/cmake_macros/oneapi-ifx_auroracpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()

string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")

#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")










19 changes: 19 additions & 0 deletions cime_config/machines/cmake_macros/oneapi-ifx_sunspotcpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()

string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")

#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")










12 changes: 7 additions & 5 deletions cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,17 @@ if (compile_threaded)
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g")

# 'just' -g may lead to linker internal errors and/or huge builds out of quotas
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g -gline-tables-only")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -fpscomp logicals -g -gline-tables-only")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -g -gline-tables-only")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpscomp logicals -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
Expand Down
8 changes: 6 additions & 2 deletions cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@

string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax")
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ")

string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ")
string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")

set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")
9 changes: 9 additions & 0 deletions cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot-pvc.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@

string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax")
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()
string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")

set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "")
31 changes: 23 additions & 8 deletions cime_config/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3441,7 +3441,7 @@
<arguments>
<arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
<arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
<arg name="ranks_bind">-envall</arg>
<arg name="ranks_bind">--no-vni --cpu-bind $ENV{RANKS_BIND} -envall</arg>
Copy link
Contributor

@mahf708 mahf708 Mar 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any comment or docs on this?

Copy link
Contributor

@mahf708 mahf708 Mar 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

trying to understand why these "errors" happen at the end of normal runs:

x4313c6s1b0n0.hostmgmt2313.cm.aurora.alcf.anl.gov 38: Used 0 of out of 46655998, high_water was 41084528
x4003c5s7b0n0.hostmgmt2003.cm.aurora.alcf.anl.gov: rank 95 exited with code 64
x4003c5s7b0n0.hostmgmt2003.cm.aurora.alcf.anl.gov: rank 84 died from signal 15

Copy link
Member

@amametjanov amametjanov Mar 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think --no-vni is not needed anymore: can be removed.
Also, default is -genvall now: -envall can be removed.

<arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
<arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
</arguments>
Expand All @@ -3459,9 +3459,9 @@
<modules compiler="!gnu">
<command name="load">oneapi/eng-compiler/2024.07.30.002</command>
</modules>
<modules compiler="oneapi-ifxgpu">
<command name="load">kokkos/4.4.01-omp-sycl</command>
</modules>
<!--<modules compiler="oneapi-ifxgpu"> -->
<!-- <command name="load">kokkos/4.4.01-omp-sycl</command> -->
<!--</modules> -->
<modules compiler="gnu">
<command name="unload">spack-pe-gcc cmake</command>
<command name="load">gcc/10.3.0</command>
Expand All @@ -3474,14 +3474,29 @@
<env name="PNETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002</env>
<env name="LD_LIBRARY_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:$ENV{LD_LIBRARY_PATH}</env>
<env name="PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:$ENV{PATH}</env>
<env name="RANKS_BIND">list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203</env>
</environment_variables>
<environment_variables DEBUG="TRUE">
<env name="HYDRA_TOPO_DEBUG">1</env>
</environment_variables>
<environment_variables compiler="oneapi-ifxgpu">
<env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
<env name="ONEAPI_MPICH_GPU">NO_GPU</env>
<env name="MPIR_CVAR_ENABLE_GPU">0</env>

<!-- <env name="ONEAPI_MPICH_GPU">NO_GPU</env> -->

<env name="MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE"></env>
<env name="MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE"></env>
<env name="MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE"></env>
<env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
<env name="UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS">1</env>
<env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>
<env name="FI_PROVIDER">cxi</env>
<env name="FI_MR_CACHE_MONITOR">disabled</env>
<env name="FI_CXI_OVFLOW_BUF_SIZE">8388608</env>
<env name="PALS_PING_PERIOD">240</env>
<env name="PALS_RPC_TIMEOUT">240</env>

<env name="MPIR_CVAR_ENABLE_GPU">0</env>
<env name="romio_cb_read">disable</env>
<env name="romio_cb_write">disable</env>
<env name="SYCL_CACHE_PERSISTENT">1</env>
Expand All @@ -3490,9 +3505,9 @@
<env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
<env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
<!--<env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>-->
<env name="ZES_ENABLE_SYSMAN">1</env>
<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>
<!--<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>-->
</environment_variables>
<environment_variables compiler="oneapi-ifx">
<env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->
Expand Down
20 changes: 17 additions & 3 deletions components/eamxx/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ option (Kokkos_ENABLE_SERIAL "" ON)
set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "")
set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?
set (SYCL_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there?

# Determine if this is a Cuda build.
if (Kokkos_ENABLE_CUDA)
Expand All @@ -106,14 +107,21 @@ if (Kokkos_ENABLE_CUDA)
set (CUDA_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
endif ()

# Determine if this is a Cuda build.
# Determine if this is a HIP build.
if (Kokkos_ENABLE_HIP)
# Add CUDA as a language for CUDA builds
enable_language(HIP)
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
set (HIP_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
endif ()

# Determine if this is a sycl build.
if (Kokkos_ENABLE_SYCL)
#enable_language(SYCL)
set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE)
set (SYCL_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there?
Copy link
Member

@ambrad ambrad Jan 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You could turn off COMPOSE here, I believe. (I'll fix the COMPOSE issues once I'm on the machine.)

endif ()

if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" )
set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp")
endif()
Expand Down Expand Up @@ -178,7 +186,7 @@ elseif(MACH)
endif()

set(DEFAULT_SMALL_KERNELS FALSE)
if (Kokkos_ENABLE_HIP)
if (Kokkos_ENABLE_HIP OR Kokkos_ENABLE_SYCL)
set(DEFAULT_SMALL_KERNELS TRUE)
endif()

Expand All @@ -197,7 +205,13 @@ set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to n
set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation")
set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine")
option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON)
option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON)

if (Kokkos_ENABLE_SYCL)
option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF)
Copy link
Contributor

@bartgol bartgol Jan 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure this will work as you intend. You may have to simply do set(SCREAM_ENABLE_MAM OFF)

Copy link
Contributor Author

@oksanaguba oksanaguba Jan 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, but that one unlike SET worked.

//Whether to enable MAM aerosol support
SCREAM_ENABLE_MAM:BOOL=OFF

if you have a preference i will change it.

Copy link
Contributor

@bartgol bartgol Jan 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm puzzled. This

option(BLAH "" ON) 
message ("BLAH: ${BLAH}")
option(BLAH "" OFF)
message ("BLAH: ${BLAH}")

prints

BLAH: ON
BLAH: ON

Are you sure you did not have another line before those that set SCREAM_ENABLE_MAM to OFF? Maybe in the mach file? Also, was it a clean build?

My preference is for using set(... CACHE ..) (or option(...)) only once.

Copy link
Contributor

@bartgol bartgol Jan 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I found this in Professional CMake:

An important difference between normal and cache variables is that the set() command will only
overwrite a cache variable if the FORCE keyword is present, unlike normal variables where the set()
command will always overwrite a pre-existing value. The set() command acts more like set-if-not-
set when used to define cache variables, as does the option() command (which has no FORCE
capability). The main reason for this is that cache variables are primarily intended as a
customization point for developers.

which makes me think something else may have caused SCREAM_ENABLE_MAM=OFF in your cache file...

else()
option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON)
endif()

set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for ALL components that support them")
set(SCREAM_P3_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for P3 only")
set(SCREAM_SHOC_SMALL_KERNELS ${SCREAM_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels for SHOC only")
Expand Down
6 changes: 6 additions & 0 deletions components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/aurora_run/shell_commands
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@

$CIMEROOT/../components/eamxx/scripts/atmchange transport_alg=0 -b
$CIMEROOT/../components/eamxx/scripts/atmchange hypervis_subcycle_q=1 -b
$CIMEROOT/../components/eamxx/scripts/atmchange dt_tracer_factor=1 -b
$CIMEROOT/../components/eamxx/scripts/atmchange tstep_type=9 -b
$CIMEROOT/../components/eamxx/scripts/atmchange theta_hydrostatic_mode=False -b
21 changes: 16 additions & 5 deletions components/eamxx/cmake/machine-files/aurora.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,25 @@
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()

include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake)
include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)

set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
Copy link
Member

@amametjanov amametjanov Mar 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For reproducibility, please replace --cpu-bind depth -envall with

--cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203

set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")

set(NETCDF_C_PATH "$ENV{NETCDF_C_PATH}")
set(NETCDF_FORTRAN_PATH "$ENV{NETCDF_FORTRAN_PATH}")
set(PNETCDF_PATH "$ENV{PNETCDF_PATH}")
SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"")

set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE)
set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE)
set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE)

#this is needed for cime builds!
set(NETCDF_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_DIR "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_C_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_C "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
Comment on lines +20 to +23
Copy link
Contributor

@mahf708 mahf708 Feb 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Feels like this belongs to the machine entry, not here, e.g.,

<environment_variables>
        <env name="NETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002</env>

Copy link
Contributor

@bartgol bartgol Feb 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you want to run test-all-scream, I think you do need this machine file. Which speaks to a broader issue: how to make test-all-scream use config_machines for this stuff?

Copy link
Contributor

@mahf708 mahf708 Feb 6, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am talking about the path only, i.e., this change for the first line:

Suggested change
set(NETCDF_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_DIR "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_C_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_C "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_PATH ""$ENV{NETCDF_PATH}"")
set(NETCDF_DIR "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_C_PATH "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")
set(NETCDF_C "/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002")

and then repeat for others by setting the paths in the xml and just get the env var here.

What's with NetCDF_C_PATH --> NETCDF_C_PATH though?

Copy link
Contributor

@bartgol bartgol Feb 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, it's fine to use these cmake files for test-all-scream. But our CIME runs still end up using these files, which is something I'd like to stop.



27 changes: 27 additions & 0 deletions components/eamxx/cmake/machine-files/auroracpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()

include (${EKAT_MACH_FILES_PATH}/kokkos/serial.cmake)
# kokkos sycl is on in the above file
#include (${EKAT_MACH_FILES_PATH}/kokkos/sycl.cmake)
include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake)

#AB flags from ekat
# -fsycl -fsycl-unnamed-lambda -sycl-std=2020 -qopenmp-simd -Wsycl-strict -fsycl-device-code-split=per_kernel

#SET(MPICH_DIR "/soft/restricted/CNDA/updates/mpich/52.2/mpich-ofi-all-icc-default-pmix-gpu-drop52/" CACHE STRING "")

set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG" CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE)
set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE)
set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -mlong-double-64 -DNDEBUG -fortlib" CACHE STRING "" FORCE)
#set(CMAKE_EXE_LINKER_FLAGS " -Wl,-\-defsym,main=MAIN_\_ -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib -L${MPICH_DIR}/lib" CACHE STRING "" FORCE)

#this is needed for cime builds!
set(NETCDF_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007")
set(NETCDF_DIR "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2023.05.15.007")
set(NETCDF_C_PATH "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007")
set(NETCDF_C "/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2023.05.15.007")



24 changes: 24 additions & 0 deletions components/eamxx/cmake/machine-files/sunspot-pvc.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
include(${CMAKE_CURRENT_LIST_DIR}/common.cmake)
common_setup()

include (${EKAT_MACH_FILES_PATH}/kokkos/intel-pvc.cmake)
include (${EKAT_MACH_FILES_PATH}/mpi/other.cmake)
set(EKAT_MPIRUN_EXE "mpiexec" CACHE STRING "" FORCE)
set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "" FORCE)
set(EKAT_MPI_EXTRA_ARGS "--label --cpu-bind depth -envall" CACHE STRING "")
set(EKAT_MPI_THREAD_FLAG "-d" CACHE STRING "")

SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"")

set(CMAKE_CXX_FLAGS " -\-intel -Xclang -fsycl-allow-virtual-functions -mlong-double-64 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "" FORCE)
set(CMAKE_C_FLAGS "-O3 -DNDEBUG" CACHE STRING "" FORCE)
set(CMAKE_EXE_LINKER_FLAGS " -lifcore -\-intel -Xclang -fsycl-allow-virtual-functions -lsycl -mlong-double-64 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "" FORCE)

set(NETCDF_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "")
set(NETCDF_DIR "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "")
set(NETCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "")
set(NETCDF_C "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "")


Loading
Loading