Skip to content

Commit e254080

Browse files
jedwards4bjedwards4b
authored
Merge pull request #253 from sjsprecious/add_cirrus_mach
Add CISL CIRRUS cloud machine configuration
2 parents 6006939 + f3f301b commit e254080

File tree

5 files changed

+143
-0
lines changed

5 files changed

+143
-0
lines changed

‎machines/cirrus/cirrus.cmake‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
if (COMP_NAME STREQUAL gptl)
2+
string(APPEND CPPDEFS " -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
3+
endif()
4+
set(MPI_PATH "$ENV{MPI_ROOT}")
5+
set(PNETCDF_PATH "$ENV{PNETCDF}")
6+
set(PIO_LIBDIR "$ENV{PIO}/lib")
7+
set(PIO_INCDIR "$ENV{PIO}/include")
8+
if(EXISTS "$ENV{LAPACK}/lib")
9+
set(LAPACK_LIBDIR "$ENV{LAPACK}/lib")
10+
elseif(EXISTS "$ENV{LAPACK}/lib64")
11+
set(LAPACK_LIBDIR "$ENV{LAPACK}/lib64")
12+
else()
13+
message(FATAL_ERROR "Neither lib nor lib64 directory found in $ENV{LAPACK}")
14+
endif()
15+
if (GPU_TYPE STREQUAL a10 OR GPU_TYPE STREQUAL a2)
16+
set(CUDA_PATH "$ENV{CUDA_ROOT}")
17+
set(CUBLAS_PATH "$ENV{CUBLAS_ROOT}")
18+
endif()
19+
string(APPEND CPPDEFS " -DHAVE_GETTID")

‎machines/cirrus/config_machines.xml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<machine MACH="cirrus">
2+
<DESC>NCAR AMD EPYC 9354P (CPU nodes) and NVIDIA A10/A2 (GPU nodes) cloud service </DESC>
3+
<OS>LINUX</OS>
4+
<COMPILERS>gnu,intel,nvhpc</COMPILERS>
5+
<MPILIBS>openmpi</MPILIBS>
6+
<CIME_OUTPUT_ROOT>/tmp</CIME_OUTPUT_ROOT>
7+
<DIN_LOC_ROOT>/glade/inputdata</DIN_LOC_ROOT>
8+
<DIN_LOC_ROOT_CLMFORC>/glade/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
9+
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
10+
<BASELINE_ROOT>$CIME_OUTPUT_ROOT/cesm_baselines</BASELINE_ROOT>
11+
<CCSM_CPRNC>/glade/cprnc/cprnc</CCSM_CPRNC>
12+
<GMAKE_J>16</GMAKE_J>
13+
<BATCH_SYSTEM>none</BATCH_SYSTEM>
14+
<SUPPORTED_BY>cisl</SUPPORTED_BY>
15+
<MAX_TASKS_PER_NODE>32</MAX_TASKS_PER_NODE> <!-- this may vary depending on your CIRRUS runner configuration -->
16+
<MEM_PER_TASK>10</MEM_PER_TASK>
17+
<MAX_MEM_PER_NODE>128</MAX_MEM_PER_NODE> <!-- this may vary depending on your CIRRUS runner configuration -->
18+
<MAX_GPUS_PER_NODE>1</MAX_GPUS_PER_NODE>
19+
<MAX_MPITASKS_PER_NODE>32</MAX_MPITASKS_PER_NODE> <!-- this may vary depending on your CIRRUS runner configuration -->
20+
<MAX_CPUTASKS_PER_GPU_NODE>32</MAX_CPUTASKS_PER_GPU_NODE> <!-- this may vary depending on your CIRRUS runner configuration -->
21+
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
22+
<mpirun mpilib="openmpi">
23+
<executable>mpiexec</executable>
24+
<arguments>
25+
<arg name="anum_tasks"> -n {{ total_tasks }}</arg>
26+
<arg name="allow_run_as_root"> --allow-run-as-root</arg>
27+
</arguments>
28+
</mpirun>
29+
<module_system type="none">
30+
</module_system>
31+
<resource_limits>
32+
<resource name="RLIMIT_STACK">-1</resource>
33+
</resource_limits>
34+
</machine>

‎machines/cirrus/gnu_cirrus.cmake‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
set(MPICC "mpicc")
2+
set(MPICXX "mpicxx")
3+
set(MPIFC "mpif90")
4+
set(SCC "gcc")
5+
set(SCXX "g++")
6+
set(SFC "gfortran")
7+
8+
if (USE_KOKKOS)
9+
# Generic setting that are used regardless of Architecture or Kokkos backend
10+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_DEPRECATED_CODE=OFF -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=OFF")
11+
if (KOKKOS_GPU_OFFLOAD)
12+
string(APPEND CPPDEFS " -DGPU -DTHRUST_IGNORE_CUB_VERSION_CHECK -DHOMMEXX_ENABLE_GPU")
13+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE86=ON -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF")
14+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=OFF -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=OFF")
15+
string(APPEND CXXFLAGS " -extended-lambda -Wext-lambda-captures-this -std=c++17 -arch=sm_86")
16+
if (CUBLAS_PATH)
17+
string(APPEND SLIBS " -L${CUDA_PATH}/lib64 -lcudart -L${CUBLAS_PATH} -lcublas -L${CUDA_PATH}/lib64/stubs -lcuda")
18+
else()
19+
string(APPEND SLIBS " -L${CUDA_PATH}/lib64 -lcudart -lcublas -L${CUDA_PATH}/lib64/stubs -lcuda")
20+
endif()
21+
else()
22+
# Enable EPYC arch in kokkos
23+
if (compile_threaded)
24+
# Settings used when OpenMP is the Kokkos backend
25+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN4=ON -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=ON -DKokkos_ENABLE_SERIAL=OFF -DKokkos_ENABLE_OPENMP=ON")
26+
else()
27+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN4=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=OFF")
28+
endif()
29+
string(APPEND CXXFLAGS " -march=znver4 -mtune=znver4")
30+
endif()
31+
if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
32+
set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) # only works with gnu v10 and above
33+
endif()
34+
string(APPEND LDFLAGS " -lstdc++ -lkokkoscontainers -lkokkoscore -lkokkossimd ")
35+
endif()

‎machines/cirrus/intel_cirrus.cmake‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
set(SCC icx)
2+
set(SCXX icpx)
3+
set(SFC ifort)
4+
set(MPICC "mpicc")
5+
set(MPICXX "mpicxx")
6+
set(MPIFC "mpif90")
7+
string(APPEND CFLAGS " -qopt-report -march=core-avx2")
8+
string(APPEND CXXFLAGS " -qopt-report -march=core-avx2")
9+
string(APPEND FFLAGS " -qopt-report -march=core-avx2")
10+
string(APPEND SLIBS " -qmkl=cluster")
11+
12+
if (USE_KOKKOS)
13+
# Generic setting that are used regardless of Architecture or Kokkos backend
14+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_DEPRECATED_CODE=OFF -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=OFF")
15+
# Enable EPYC arch in kokkos
16+
if (compile_threaded)
17+
# Settings used when OpenMP is the Kokkos backend
18+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN4=ON -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=ON -DKokkos_ENABLE_SERIAL=OFF -DKokkos_ENABLE_OPENMP=ON")
19+
else()
20+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN4=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=OFF")
21+
endif()
22+
set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE)
23+
string(APPEND LDFLAGS " -lstdc++ -lkokkoscontainers -lkokkoscore -lkokkossimd ")
24+
endif()

‎machines/cirrus/nvhpc_cirrus.cmake‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
set(MPICC "mpicc")
2+
set(MPICXX "mpicxx")
3+
set(MPIFC "mpif90")
4+
set(SCC "nvc")
5+
set(SCXX "nvc++")
6+
set(SFC "nvfortran")
7+
8+
string(APPEND CPPDEFS " -DHAVE_IEEE_ARITHMETIC")
9+
string(APPEND FFLAGS " -tp=zen4 -Mstack_arrays -Mallocatable=03")
10+
string(APPEND CXXFLAGS " -tp=zen4")
11+
string(APPEND LDFLAGS " -tp=zen4 -Mnofma")
12+
13+
if (USE_KOKKOS)
14+
# Generic setting that are used regardless of Architecture or Kokkos backend
15+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_DEPRECATED_CODE=OFF -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=OFF")
16+
if (KOKKOS_GPU_OFFLOAD)
17+
string(APPEND CPPDEFS " -DGPU -DTHRUST_IGNORE_CUB_VERSION_CHECK -DHOMMEXX_ENABLE_GPU")
18+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE86=ON -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON -DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF")
19+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=OFF -DKokkos_ENABLE_AGGRESSIVE_VECTORIZATION=OFF")
20+
string(APPEND CXXFLAGS " -extended-lambda -Wext-lambda-captures-this -std=c++17 -arch=sm_86")
21+
if (CUBLAS_PATH)
22+
string(APPEND SLIBS " -L${CUDA_PATH}/lib64 -lcudart -L${CUBLAS_PATH}/lib64 -lcublas -L${CUDA_PATH}/lib64/stubs -lcuda")
23+
else()
24+
string(APPEND SLIBS " -L${CUDA_PATH}/lib64 -lcudart -lcublas -L${CUDA_PATH}/lib64/stubs -lcuda")
25+
endif()
26+
else()
27+
# Enable EPYC arch in kokkos
28+
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_ZEN4=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=OFF") # work-around for nvidia as kokkos is not passing "-mp" for threaded build
29+
endif()
30+
string(APPEND LDFLAGS " -lstdc++ -lkokkoscontainers -lkokkoscore -lkokkossimd ")
31+
endif()

0 commit comments

Comments
 (0)