Skip to content

Updates LC machines #7767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+43 −140
Open

Updates LC machines #7767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1dab6a8
Removes decomissioned ruby machine
jasonb5 Sep 11, 2025
ca0b689
Updates dane configuration
jasonb5 Sep 23, 2025
75e9329
Adds missing modules and removes hard coded paths
jasonb5 Oct 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 0 additions & 22 deletions cime_config/allactive/config_pesall.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2376,28 +2376,6 @@
</nthrds>
</pes>
</mach>
<mach name="ruby">
<pes compset="any" pesize="any">
<ntasks>
<ntasks_atm>-4</ntasks_atm>
<ntasks_lnd>-4</ntasks_lnd>
<ntasks_rof>-4</ntasks_rof>
<ntasks_ice>-4</ntasks_ice>
<ntasks_ocn>-4</ntasks_ocn>
<ntasks_glc>-4</ntasks_glc>
<ntasks_wav>-4</ntasks_wav>
<ntasks_cpl>-4</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
</pes>
</mach>
<mach name="quartz">
<pes compset="any" pesize="any">
<ntasks>
Expand Down
5 changes: 5 additions & 0 deletions cime_config/machines/Depends.dane.oneapi-ifx.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# compile mpas_seaice_core_interface.f90 with ifort, not ifx
if (NOT MPILIB STREQUAL "openmpi")
e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
e3sm_add_flags("${CMAKE_BINARY_DIR}/core_landice/mode_forward/mpas_li_core_interface.f90" "-fc=ifort")
endif()
10 changes: 0 additions & 10 deletions cime_config/machines/cmake_macros/intel_dane.cmake
View file
Open in desktop

This file was deleted.

10 changes: 0 additions & 10 deletions cime_config/machines/cmake_macros/intel_ruby.cmake
View file
Open in desktop

This file was deleted.

9 changes: 9 additions & 0 deletions cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
list(APPEND CMAKE_BUILD_RPATH "$ENV{HDF5_ROOT}/lib" )
list(APPEND CMAKE_BUILD_RPATH "$ENV{NETCDF_C_PATH}/lib" )
list(APPEND CMAKE_BUILD_RPATH "$ENV{NETCDF_FORTRAN_PATH}/lib" )
list(APPEND CMAKE_BUILD_RPATH "$ENV{PNETCDF_PATH}/lib" )

# get_cmake_property(_variableNames VARIABLES)
# foreach (_variableName ${_variableNames})
# message("${_variableName}=${${_variableName}}")
# endforeach()
7 changes: 0 additions & 7 deletions cime_config/machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,13 +223,6 @@
</batch_system>

<!-- for lawrence livermore computing -->
<batch_system MACH="ruby" type="slurm">
<queues>
<queue walltimemax="24:00:00" nodemax="520" default="true">pbatch</queue>
<queue walltimemax="01:00:00" nodemax="12">pdebug</queue>
</queues>
</batch_system>

<batch_system MACH="dane" type="slurm">
<queues>
<queue walltimemax="24:00:00" nodemax="520" default="true">pbatch</queue>
Expand Down
120 changes: 29 additions & 91 deletions cime_config/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3032,101 +3032,32 @@
</environment_variables>
</machine>

<machine MACH="ruby">
<DESC>LLNL Linux Cluster, Linux (pgi), 56 pes/node, batch system is Slurm</DESC>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<PROJECT>cbronze</PROJECT>
<CIME_OUTPUT_ROOT>/p/lustre2/$USER/e3sm_scratch/ruby</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/usr/workspace/e3sm/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/usr/workspace/e3sm/apps/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>boutte3 -at- llnl.gov</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>56</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>56</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="mpi-serial">
<executable/>
</mpirun>
<mpirun mpilib="default">
<executable>srun</executable>
<arguments>
<arg name="pmi_layer">--mpi=pmi2</arg>
<arg name="export">--export=ALL</arg>
<arg name="num_tasks">-n {{ total_tasks }} -N {{ num_nodes }}</arg>
<arg name="thread_count">-c 1</arg>
<arg name="binding">--cpu_bind=cores</arg>
<arg name="placement">-m plane={{ tasks_per_node }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
<init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
<cmd_path lang="csh">module</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
<modules compiler="intel">
<command name="load">python/3.9.12</command>
<command name="load">git</command>
<command name="load">subversion</command>
<command name="load">cmake/3.19.2</command>
<command name="load">mkl/2022.1.0</command>
<command name="load">intel-classic/2021.6.0-magic</command>
<command name="use --append">/usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core</command>
<command name="load">mvapich2/2.3.7-ll7cmqm</command>
<command name="load">hdf5/1.10.7-ewjpbjd</command>
<command name="load">netcdf-c/4.4.1.1-vaxofek</command>
<command name="load">netcdf-fortran/4.4.4-3pzbx2u</command>
<command name="load">parallel-netcdf/1.11.0-tzgdala</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables compiler="intel">
<env name="OMP_STACKSIZE">128M</env>
<env name="HDF5_USE_FILE_LOCKING">FALSE</env>
<env name="HDF5_ROOT">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg</env>
<env name="NETCDF_C_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn</env>
<env name="NETCDF_FORTRAN_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2</env>
<env name="PNETCDF_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5</env>
</environment_variables>
</machine>

<machine MACH="dane">
<DESC>LLNL Linux Cluster, 112 pes/node, batch system is Slurm</DESC>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<COMPILERS>oneapi-ifx</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<PROJECT>cbronze</PROJECT>
<PROJECT>e3smtest</PROJECT>
<CIME_OUTPUT_ROOT>/p/lustre2/$USER/e3sm_scratch/dane</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/usr/workspace/e3sm/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DIN_LOC_ROOT>/p/vast1/e3sm/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/p/vast1/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/usr/workspace/e3sm/apps/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>boutte3 -at- llnl.gov</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>224</MAX_TASKS_PER_NODE>
<MAX_TASKS_PER_NODE>112</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>112</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="mpi-serial">
<executable/>
</mpirun>
<mpirun mpilib="default">
<executable>srun</executable>
<arguments>
<arg name="pmi_layer">--mpi=pmi2</arg>
<arg name="export">--export=ALL</arg>
<arg name="num_tasks">-n {{ total_tasks }} -N {{ num_nodes }}</arg>
<arg name="thread_count">-c 1</arg>
<arg name="num_tasks"> -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit</arg>
<arg name="binding">--cpu_bind=cores</arg>
<arg name="thread_count">-c $ENV{OMP_NUM_THREADS}</arg>
<arg name="placement">-m plane={{ tasks_per_node }}</arg>
</arguments>
</mpirun>
Expand All @@ -3139,30 +3070,37 @@
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
<modules compiler="intel">
<modules>
<command name="load">python/3.9.12</command>
<command name="load">git</command>
<command name="load">subversion</command>
<command name="load">cmake/3.26.3</command>
<command name="load">mkl/2022.1.0</command>
<command name="load">intel-classic/2021.6.0-magic</command>
<command name="load">cmake/3.19.2</command>
<command name="use --append">/usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core</command>
<command name="load">mvapich2/2.3.7-27jao34</command>
<command name="load">hdf5/1.10.7-766kapa</command>
<command name="load">netcdf-c/4.4.1.1-2uznnlw</command>
<command name="load">netcdf-fortran/4.4.4-itpstyo</command>
<command name="load">parallel-netcdf/1.11.0-26sxm4m</command>
</modules>
<modules compiler="oneapi-ifx">
<command name="load">intel/2023.2.1-magic</command>
<command name="use">/usr/workspace/e3sm/spack/dane/intel/modules/Core</command>
<command name="load">intel-oneapi-runtime/2023.2.0</command>
<command name="load">gcc-runtime/12.1.1</command>
<command name="load">glibc/2.28</command>
<command name="load">mvapich2/2.3.7</command>
<command name="load">hdf5/1.14.5</command>
<command name="load">netcdf-c/4.9.2</command>
<command name="load">netcdf-fortran/4.6.1</command>
<command name="load">parallel-netcdf/1.14.0</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables compiler="intel">
<environment_variables>
<env name="OMP_STACKSIZE">128M</env>
<env name="HDF5_USE_FILE_LOCKING">FALSE</env>
<env name="HDF5_ROOT">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv</env>
<env name="NETCDF_C_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib</env>
<env name="NETCDF_FORTRAN_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp</env>
<env name="PNETCDF_PATH">/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck</env>
</environment_variables>
<environment_variables compiler="oneapi-ifx">
<env name="HDF5_ROOT">$ENV{HDF5_ROOT}</env>
<env name="NETCDF_C_PATH">$ENV{NETCDF_C_PATH}</env>
<env name="NETCDF_FORTRAN_PATH">$ENV{NETCDF_FORTRAN_PATH}</env>
<env name="PNETCDF_PATH">$ENV{PNETCDF_PATH}</env>
<env name="PATH">/usr/tce/packages/intel/intel-2023.2.1/compiler/2023.2.1/linux/bin/intel64:$ENV{PATH}"</env>
</environment_variables>
</machine>

Expand Down