Skip to content

Commit 717eab6

Browse files
jedwards4bjedwards4b
authored
Merge pull request #188 from jedwards4b/add/tacc_vista
add tacc vista system and update stampede
2 parents 4b5e4eb + 5fe6901 commit 717eab6

File tree

11 files changed

+194
-5
lines changed

11 files changed

+194
-5
lines changed

‎machines/cmake_macros/intel.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ endif()
2626
set(MPICC "mpicc")
2727
set(MPICXX "mpicxx")
2828
set(MPIFC "mpif90")
29-
set(SCC "icc")
30-
set(SCXX "icpc")
29+
set(SCC "icx")
30+
set(SCXX "icpx")
3131
set(SFC "ifort")
3232
if (MPILIB STREQUAL mpi-serial)
3333
string(APPEND SLIBS " -mkl")

‎machines/cmake_macros/nvhpc.cmake‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
string(APPEND CFLAGS " -gopt -time")
2-
string(APPEND CXXFLAGS " -gopt -time")
2+
if (compile_threaded)
3+
string(APPEND CFLAGS " -mp")
4+
endif()
5+
36
if (NOT DEBUG)
47
string(APPEND CFLAGS " -O -Mnofma")
58
string(APPEND CXXFLAGS " -O -Mnofma")
@@ -9,7 +12,12 @@ else()
912
string(APPEND CXXFLAGS " -O0 -Mnofma -g -Wall -Kieee -traceback")
1013
string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee")
1114
endif()
12-
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRNVIDIA -DNO_QUAD_PRECISION")
15+
16+
string(APPEND CFLAGS " -Mnofma")
17+
string(APPEND FFLAGS " -Mnofma")
18+
19+
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRNVIDIA -DNO_QUAD_PRECISION -DCPRPGI")
20+
1321
set(CXX_LINKER "CXX")
1422
set(FC_AUTO_R8 "-r8")
1523
string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee")
@@ -72,4 +80,4 @@ if (OPENACC_GPU_FLAGS)
7280
endif()
7381
if (OPENMP_GPU_FLAGS)
7482
string(APPEND LDFLAGS " ${OPENMP_GPU_FLAGS}")
75-
endif()
83+
endif()

‎machines/config_machines.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,13 @@ This allows using a different mpirun command to launch unit tests
6868
<value MACH="sandia-srn-sems">(s999964|climate|penn)</value>
6969
<value MACH="sandiatoss3">(skybridge|chama)-login</value>
7070
<value MACH="stampede2-skx">.*stampede2</value>
71+
<value MACH="stampede3-spr">.*.stampede3.tacc.utexas.edu</value>
7172
<value MACH="swan">swan.*</value>
7273
<value MACH="theia">tfe</value>
7374
<value MACH="theta">theta.*</value>
7475
<value MACH="thunder">.*.thunder.ucar.edu</value>
7576
<value MACH="ubuntu-latest">$ENV{CIME_TEST_PLATFORM}:ubuntu-latest</value>
77+
<value MACH="vista">.*\.vista.tacc.utexas.edu</value>
7678
<value MACH="zeus">(login[1,2]-ib|n[0-9][0-9][0-9]-ib)</value>
7779
</NODENAME_REGEX>
7880

‎machines/stampede3-spr/config_batch.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<batch_system MACH="stampede3-spr" type="slurm" >
2+
<batch_submit>ssh login1 cd $CASEROOT ; sbatch</batch_submit>
3+
<submit_args>
4+
<argument> --time $JOB_WALLCLOCK_TIME </argument>
5+
<argument> -p $JOB_QUEUE </argument>
6+
<argument> --account $PROJECT </argument>
7+
</submit_args>
8+
<queues>
9+
<queue walltimemax="24:00:00" nodemin="1" nodemax="16" default="true">spr</queue>
10+
</queues>
11+
</batch_system>

‎machines/stampede3-spr/config_machines.xml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
<machine MACH="stampede3-spr">
2+
<DESC>Intel Xeon CPU MAX 9480 ("Sapphire Rapids HBM") 112 cores on two sockets (2x 56 cores), batch system is SLURM</DESC>
3+
<OS>LINUX</OS>
4+
<COMPILERS>intel</COMPILERS>
5+
<MPILIBS>impi,mvapich</MPILIBS>
6+
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
7+
<DIN_LOC_ROOT>/work/02503/edwardsj/CESM/inputdata</DIN_LOC_ROOT>
8+
<DIN_LOC_ROOT_CLMFORC>/work/02503/edwardsj/CESM/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
9+
<DOUT_S_ROOT>$ENV{WORK}/archive/$CASE</DOUT_S_ROOT>
10+
<BASELINE_ROOT>/work/02503/edwardsj/CESM/cesm_baselines</BASELINE_ROOT>
11+
<CCSM_CPRNC>/work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc.sp3</CCSM_CPRNC>
12+
<GMAKE_J>4</GMAKE_J>
13+
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
14+
<SUPPORTED_BY>cseg</SUPPORTED_BY>
15+
<MAX_TASKS_PER_NODE>112</MAX_TASKS_PER_NODE>
16+
<MAX_MPITASKS_PER_NODE>112</MAX_MPITASKS_PER_NODE>
17+
<mpirun mpilib="impi">
18+
<executable>ibrun</executable>
19+
<arguments>
20+
<arg name="ntasks"> -n {{ total_tasks }} </arg>
21+
</arguments>
22+
</mpirun>
23+
<mpirun mpilib="mvapich2">
24+
<executable>ibrun</executable>
25+
<arguments>
26+
<arg name="ntasks"> -n {{ total_tasks }} </arg>
27+
</arguments>
28+
</mpirun>
29+
<module_system type="module">
30+
<init_path lang="perl">/opt/apps/lmod/lmod/init/perl</init_path>
31+
<init_path lang="python">/opt/apps/lmod/lmod/init/env_modules_python.py</init_path>
32+
<init_path lang="sh">/opt/apps/lmod/lmod/init/sh</init_path>
33+
<init_path lang="csh">/opt/apps/lmod/lmod/init/csh</init_path>
34+
<cmd_path lang="perl">/opt/apps/lmod/lmod/libexec/lmod perl</cmd_path>
35+
<cmd_path lang="python">/opt/apps/lmod/lmod/libexec/lmod python</cmd_path>
36+
<cmd_path lang="sh">module</cmd_path>
37+
<cmd_path lang="csh">module</cmd_path>
38+
<modules>
39+
<command name="purge"></command>
40+
<command name="load">TACC</command>
41+
<command name="load">intel/24.0</command>
42+
<command name="load">cmake/3.28.1</command>
43+
</modules>
44+
<modules mpilib="mvapich2">
45+
<command name="load">mvapich/3.0</command>
46+
</modules>
47+
<modules mpilib="impi">
48+
<command name="load">impi</command>
49+
</modules>
50+
<modules mpilib="!mpi-serial">
51+
<command name="load">pnetcdf/1.12.3</command>
52+
<command name="load">parallel-netcdf/4.9.2</command>
53+
<command name="load">phdf5/1.14.3</command>
54+
</modules>
55+
<modules mpilib="mpi-serial">
56+
<command name="load">netcdf/4.9.2</command>
57+
</modules>
58+
</module_system>
59+
<environment_variables>
60+
<env name="OMP_STACKSIZE">256M</env>
61+
<env name="ESMFMKFILE">/work2/02503/edwardsj/stampede3/intel24.0/esmf/v8.6.1/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk</env>
62+
<env name="PIO_VERSION_MAJOR">2</env>
63+
<env name="PIO_LIBDIR">/work2/02503/edwardsj/stampede3/intel24.0/parallelio2.6.3rc1/lib</env>
64+
<env name="PIO_INCDIR">/work2/02503/edwardsj/stampede3/intel24.0/parallelio2.6.3rc1/include</env>
65+
<env name="PIO_TYPENAME_VALID_VALUES">netcdf,pnetcdf,netcdf4p,netcdf4c</env>
66+
</environment_variables>
67+
<environment_variables mpilib="impi">
68+
<env name="I_MPI_F90">ifort</env>
69+
</environment_variables>
70+
</machine>

‎machines/stampede3-spr/intel_stampede3-spr.cmake‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
if (MPILIB STREQUAL mpi-serial)
2+
string(APPEND FFLAGS " -mcmodel medium")
3+
endif()
4+
string(APPEND CFLAGS " -xCORE-AVX512")
5+
string(APPEND FFLAGS " -xCORE-AVX512")
6+
7+
string(APPEND LDFLAGS " -L$ENV{TACC_HDF5_LIB} -lhdf5 $(MKL)")
8+
execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs OUTPUT_VARIABLE SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0 OUTPUT_STRIP_TRAILING_WHITESPACE)
9+
string(APPEND SLIBS " ${SHELL_CMD_OUTPUT_BUILD_INTERNAL_IGNORE0} -L$ENV{TACC_HDF5_LIB} -lhdf5")
10+
set(TRILINOS_PATH "$ENV{TRILINOS_PATH}")
11+
set(HAS_F2008_CONTIGUOUS "FALSE")

‎machines/stampede3-spr/stampede3-spr.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
string(APPEND CPPDEFS " -DHAVE_NANOTIME")
2+
set(NETCDF_PATH "$ENV{TACC_NETCDF_DIR}")
3+
set(PIO_FILESYSTEM_HINTS "lustre")
4+
set(PNETCDF_PATH "$ENV{TACC_PNETCDF_DIR}")
5+
string(APPEND SLIBS " -Wl,-rpath,$ENV{PIO_LIBDIR}")

‎machines/vista/config_batch.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<batch_system MACH="vista" type="slurm" >
2+
<batch_submit>sbatch</batch_submit>
3+
<submit_args>
4+
<argument> -A $PROJECT </argument>
5+
<argument> --time $JOB_WALLCLOCK_TIME </argument>
6+
<argument> -p $JOB_QUEUE </argument>
7+
</submit_args>
8+
<queues>
9+
<queue walltimemax="48:00:00" nodemin="1" nodemax="256" default="true">gg</queue>
10+
<queue walltimemax="02:00:00" nodemin="1" nodemax="4" >gg-4k</queue>
11+
</queues>
12+
</batch_system>

‎machines/vista/config_machines.xml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
<machine MACH="vista">
2+
<DESC>Grace Hopper ARM TACC , batch system is SLURM</DESC>
3+
<OS>LINUX</OS>
4+
<COMPILERS>nvhpc,gnu</COMPILERS>
5+
<MPILIBS>openmpi</MPILIBS>
6+
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
7+
<DIN_LOC_ROOT>/work2/02503/edwardsj/CESM/inputdata</DIN_LOC_ROOT>
8+
<DIN_LOC_ROOT_CLMFORC>/work2/02503/edwardsj/CESM/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
9+
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
10+
<BASELINE_ROOT>/work2/02503/edwardsj/CESM/cesm_baselines</BASELINE_ROOT>
11+
<CCSM_CPRNC>/work2/02503/edwardsj/vista/cprnc/cprnc</CCSM_CPRNC>
12+
<GMAKE_J>4</GMAKE_J>
13+
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
14+
<SUPPORTED_BY></SUPPORTED_BY>
15+
<MAX_TASKS_PER_NODE>144</MAX_TASKS_PER_NODE>
16+
<MAX_MPITASKS_PER_NODE>144</MAX_MPITASKS_PER_NODE>
17+
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
18+
<mpirun mpilib="openmpi">
19+
<executable>mpirun</executable>
20+
<arguments>
21+
<arg name="num_tasks"> -np {{ total_tasks }}</arg>
22+
</arguments>
23+
</mpirun>
24+
<module_system type="module" allow_error="true">
25+
<init_path lang="perl">/opt/apps/lmod/lmod/init/perl</init_path>
26+
<init_path lang="python">/opt/apps/lmod/lmod/init/env_modules_python.py</init_path>
27+
<init_path lang="sh">/opt/apps/lmod/lmod/init/sh</init_path>
28+
<init_path lang="csh">/opt/apps/lmod/lmod/init/csh</init_path>
29+
<cmd_path lang="perl">/opt/apps/lmod/lmod/libexec/lmod perl</cmd_path>
30+
<cmd_path lang="python">/opt/apps/lmod/lmod/libexec/lmod python</cmd_path>
31+
<cmd_path lang="sh">module</cmd_path>
32+
<cmd_path lang="csh">module</cmd_path>
33+
<modules>
34+
<command name="purge"/>
35+
<command name="load">nvidia/24.9</command>
36+
<command name="load">ucx/1.17.0</command>
37+
<command name="load">openmpi/5.0.5_nvc249</command>
38+
<command name="load">pnetcdf/1.13.0</command>
39+
<command name="load">hdf5/1.14.4</command>
40+
<command name="load">netcdf/4.9.2</command>
41+
<command name="use">/work/02503/edwardsj/vista/modulefiles/nvhpc/24.9.0/openmpi/5.0.5_nvc249</command>
42+
<command name="load">parallelio/2.6.5</command>
43+
</modules>
44+
</module_system>
45+
<environment_variables>
46+
<env name="OMP_STACKSIZE">256M</env>
47+
<env name="ESMF_RUNTIME_PROFILE">ON</env>
48+
<env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
49+
</environment_variables>
50+
<environment_variables compiler="nvhpc">
51+
<env name="ESMFMKFILE">/work/02503/edwardsj/vista/esmf/v8.8.0_nvhpc249_openmpi/lib/libO/Linux.nvhpc.64.openmpi.default/esmf.mk</env>
52+
<env name="NETCDF_PATH">$ENV{TACC_NETCDF_DIR}</env>
53+
<env name="PNETCDF_PATH">$ENV{TACC_PNETCDF_DIR}</env>
54+
</environment_variables>
55+
56+
<environment_variables compiler="gnu">
57+
<env name="PIO_LIBDIR">/work/02503/edwardsj/vista/pio2.6.3_gnu_openmpi/lib</env>
58+
<env name="PIO_INCDIR">/work/02503/edwardsj/vista/pio2.6.3_gnu_openmpi/include</env>
59+
<env name="NETCDF_PATH">/scratch/00422/cazes/netcdf_installs/netcdf_4.7.4_gcc_14.2</env>
60+
<env name="PNETCDF_PATH">/scratch/00422/cazes/pnetcdf_1.13.0_gcc_14.2/</env>
61+
<env name="ESMFMKFILE">/work/02503/edwardsj/vista/esmf/v8.6.2b00_gnu_openmpi/lib/libO/Linux.gfortran.64.openmpi.default/esmf.mk</env>
62+
<env name="LD_LIBRARY_PATH">/home1/apps/nvidia/Linux_aarch64/24.7/compilers/lib:/scratch/projects/compilers/gcc11/openmpi/5.0.3_cpu/lib:/scratch/projects/compilers/ucx/1.17/lib/ucx:/scratch/projects/compilers/ucx/1.17/lib:/opt/apps/gcc/14.2.0/lib64:/opt/apps/gcc/14.2.0/lib</env>
63+
</environment_variables>
64+
</machine>

‎machines/vista/nvhpc_vista.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
set(SUPPORTS_CXX "TRUE")
2+
string(APPEND SLIBS " -L$ENV{TACC_NETCDF_DIR}/lib -lnetcdff -lnetcdf ")
3+
if (MPILIB STREQUAL openmpi)
4+
string(APPEND SLIBS " -L$ENV{TACC_PNETCDF_DIR}/lib -lpnetcdf ")
5+
endif()

0 commit comments

Comments
 (0)