From 1dab6a80c2c5e466b103bb364f25984a573e95cc Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 11 Sep 2025 12:13:31 -0700 Subject: [PATCH 1/3] Removes decomissioned ruby machine --- cime_config/allactive/config_pesall.xml | 22 ------ .../machines/cmake_macros/intel_ruby.cmake | 10 --- cime_config/machines/config_batch.xml | 7 -- cime_config/machines/config_machines.xml | 67 ------------------- 4 files changed, 106 deletions(-) delete mode 100644 cime_config/machines/cmake_macros/intel_ruby.cmake diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 3ad22b7b5a80..d5dcee16bfb0 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -2376,28 +2376,6 @@ - - - - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 - - - 1 - 1 - 1 - 1 - 1 - 1 - - - diff --git a/cime_config/machines/cmake_macros/intel_ruby.cmake b/cime_config/machines/cmake_macros/intel_ruby.cmake deleted file mode 100644 index e874bfb7eaf6..000000000000 --- a/cime_config/machines/cmake_macros/intel_ruby.cmake +++ /dev/null @@ -1,10 +0,0 @@ -string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") - -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg/lib") -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn/lib") -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2/lib") -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5/lib") - -set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 37ccafc4bb8d..3b61a20fdc0f 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -223,13 +223,6 @@ - - - pbatch - pdebug - - - pbatch diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 85cf8e5a600e..df201a62cb64 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3032,73 +3032,6 @@ - - LLNL Linux Cluster, Linux (pgi), 56 pes/node, batch system is Slurm - LINUX - intel - mpich - cbronze - /p/lustre2/$USER/e3sm_scratch/ruby - /usr/workspace/e3sm/ccsm3data/inputdata - /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 - /p/lustre2/$USER/archive/$CASE - /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/apps/cprnc - 8 - slurm - boutte3 -at- llnl.gov - 56 - 56 - - - - - srun - - --mpi=pmi2 - --export=ALL - -n {{ total_tasks }} -N {{ num_nodes }} - -c 1 - --cpu_bind=cores - -m plane={{ tasks_per_node }} - - - - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - module - module - /usr/share/lmod/lmod/libexec/lmod python - /usr/share/lmod/lmod/libexec/lmod perl - - python/3.9.12 - git - subversion - cmake/3.19.2 - mkl/2022.1.0 - intel-classic/2021.6.0-magic - /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core - mvapich2/2.3.7-ll7cmqm - hdf5/1.10.7-ewjpbjd - netcdf-c/4.4.1.1-vaxofek - netcdf-fortran/4.4.4-3pzbx2u - parallel-netcdf/1.11.0-tzgdala - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - - 128M - FALSE - /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg - /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn - /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 - /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 - - - LLNL Linux Cluster, 112 pes/node, batch system is Slurm LINUX From ca0b689d7106a00b719f1121509996e5a4afeb92 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 23 Sep 2025 15:05:11 -0700 Subject: [PATCH 2/3] Updates dane configuration --- .../machines/Depends.dane.oneapi-ifx.cmake | 5 ++ .../machines/cmake_macros/intel_dane.cmake | 10 ---- .../cmake_macros/oneapi-ifx_dane.cmake | 9 ++++ cime_config/machines/config_machines.xml | 51 ++++++++++--------- 4 files changed, 41 insertions(+), 34 deletions(-) create mode 100644 cime_config/machines/Depends.dane.oneapi-ifx.cmake delete mode 100644 cime_config/machines/cmake_macros/intel_dane.cmake create mode 100644 cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake diff --git a/cime_config/machines/Depends.dane.oneapi-ifx.cmake b/cime_config/machines/Depends.dane.oneapi-ifx.cmake new file mode 100644 index 000000000000..ce517471ef82 --- /dev/null +++ b/cime_config/machines/Depends.dane.oneapi-ifx.cmake @@ -0,0 +1,5 @@ +# compile mpas_seaice_core_interface.f90 with ifort, not ifx +if (NOT MPILIB STREQUAL "openmpi") + e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") + e3sm_add_flags("${CMAKE_BINARY_DIR}/core_landice/mode_forward/mpas_li_core_interface.f90" "-fc=ifort") +endif() diff --git a/cime_config/machines/cmake_macros/intel_dane.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake deleted file mode 100644 index ef25a97b300e..000000000000 --- a/cime_config/machines/cmake_macros/intel_dane.cmake +++ /dev/null @@ -1,10 +0,0 @@ -string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") - -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv/lib") -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib/lib") -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp/lib") -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck/lib") - -set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake b/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake new file mode 100644 index 000000000000..faad4f8c89a0 --- /dev/null +++ b/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake @@ -0,0 +1,9 @@ +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/hdf5-1.14.5-oq4xvuvma5emdmqlrhhugf5f2fbl2miq/lib" ) +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-c-4.9.2-3a4qa33n36d4ryrne2des3653ka4rsfa/lib" ) +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-fortran-4.6.1-7mybbfylyzezbv2sc2rjhnkewhnyx3ge/lib" ) +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/parallel-netcdf-1.14.0-d6xfqf4wccp4eiohquw3idjppqff7mtj/lib" ) + +# get_cmake_property(_variableNames VARIABLES) +# foreach (_variableName ${_variableNames}) +# message("${_variableName}=${${_variableName}}") +# endforeach() diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index df201a62cb64..b7af6b6089cc 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3035,19 +3035,19 @@ LLNL Linux Cluster, 112 pes/node, batch system is Slurm LINUX - intel + oneapi-ifx mpich - cbronze + e3smtest /p/lustre2/$USER/e3sm_scratch/dane - /usr/workspace/e3sm/ccsm3data/inputdata - /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 + /p/vast1/e3sm/ccsm3data/inputdata + /p/vast1/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER /usr/workspace/e3sm/apps/cprnc 8 slurm boutte3 -at- llnl.gov - 224 + 112 112 @@ -3055,11 +3055,9 @@ srun - --mpi=pmi2 - --export=ALL - -n {{ total_tasks }} -N {{ num_nodes }} - -c 1 + -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit --cpu_bind=cores + -c $ENV{OMP_NUM_THREADS} -m plane={{ tasks_per_node }} @@ -3072,30 +3070,35 @@ module /usr/share/lmod/lmod/libexec/lmod python /usr/share/lmod/lmod/libexec/lmod perl - + python/3.9.12 git subversion + cmake/3.26.3 mkl/2022.1.0 - intel-classic/2021.6.0-magic - cmake/3.19.2 - /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core - mvapich2/2.3.7-27jao34 - hdf5/1.10.7-766kapa - netcdf-c/4.4.1.1-2uznnlw - netcdf-fortran/4.4.4-itpstyo - parallel-netcdf/1.11.0-26sxm4m + + + intel/2023.2.1-magic + /usr/workspace/e3sm/spack/dane/intel/modules/Core + mvapich2/2.3.7 + hdf5/1.14.5 + netcdf-c/4.9.2 + netcdf-fortran/4.6.1 + parallel-netcdf/1.14.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - + 128M - FALSE - /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv - /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib - /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp - /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck + + + /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/hdf5-1.14.5-oq4xvuvma5emdmqlrhhugf5f2fbl2miq + /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-c-4.9.2-3a4qa33n36d4ryrne2des3653ka4rsfa + /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-fortran-4.6.1-7mybbfylyzezbv2sc2rjhnkewhnyx3ge + /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/parallel-netcdf-1.14.0-d6xfqf4wccp4eiohquw3idjppqff7mtj + /usr/tce/packages/intel/intel-2023.2.1-magic/lib:$ENV{LD_LIBRARY_PATH} + /usr/tce/packages/intel/intel-2023.2.1/compiler/2023.2.1/linux/bin/intel64:$ENV{PATH}" From 75e9329725a226571c3b1102530ebe002d690b7e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 7 Oct 2025 15:47:38 -0700 Subject: [PATCH 3/3] Adds missing modules and removes hard coded paths --- .../machines/cmake_macros/oneapi-ifx_dane.cmake | 8 ++++---- cime_config/machines/config_machines.xml | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake b/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake index faad4f8c89a0..6fc9370fd4bd 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifx_dane.cmake @@ -1,7 +1,7 @@ -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/hdf5-1.14.5-oq4xvuvma5emdmqlrhhugf5f2fbl2miq/lib" ) -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-c-4.9.2-3a4qa33n36d4ryrne2des3653ka4rsfa/lib" ) -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-fortran-4.6.1-7mybbfylyzezbv2sc2rjhnkewhnyx3ge/lib" ) -list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/parallel-netcdf-1.14.0-d6xfqf4wccp4eiohquw3idjppqff7mtj/lib" ) +list(APPEND CMAKE_BUILD_RPATH "$ENV{HDF5_ROOT}/lib" ) +list(APPEND CMAKE_BUILD_RPATH "$ENV{NETCDF_C_PATH}/lib" ) +list(APPEND CMAKE_BUILD_RPATH "$ENV{NETCDF_FORTRAN_PATH}/lib" ) +list(APPEND CMAKE_BUILD_RPATH "$ENV{PNETCDF_PATH}/lib" ) # get_cmake_property(_variableNames VARIABLES) # foreach (_variableName ${_variableNames}) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index b7af6b6089cc..2cc7431bc23b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3080,6 +3080,9 @@ intel/2023.2.1-magic /usr/workspace/e3sm/spack/dane/intel/modules/Core + intel-oneapi-runtime/2023.2.0 + gcc-runtime/12.1.1 + glibc/2.28 mvapich2/2.3.7 hdf5/1.14.5 netcdf-c/4.9.2 @@ -3093,11 +3096,10 @@ 128M - /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/hdf5-1.14.5-oq4xvuvma5emdmqlrhhugf5f2fbl2miq - /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-c-4.9.2-3a4qa33n36d4ryrne2des3653ka4rsfa - /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/netcdf-fortran-4.6.1-7mybbfylyzezbv2sc2rjhnkewhnyx3ge - /usr/workspace/e3sm/spack/dane/intel/libs/linux-sapphirerapids/parallel-netcdf-1.14.0-d6xfqf4wccp4eiohquw3idjppqff7mtj - /usr/tce/packages/intel/intel-2023.2.1-magic/lib:$ENV{LD_LIBRARY_PATH} + $ENV{HDF5_ROOT} + $ENV{NETCDF_C_PATH} + $ENV{NETCDF_FORTRAN_PATH} + $ENV{PNETCDF_PATH} /usr/tce/packages/intel/intel-2023.2.1/compiler/2023.2.1/linux/bin/intel64:$ENV{PATH}"