diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index e1fab7c53fc1..35f5840b40bf 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -1023,9 +1023,9 @@ - + - summit|ascent: any compset on ne30np4 grid + summit|ascent|aurora: any compset on ne30np4 grid -2 -2 @@ -1036,16 +1036,6 @@ -2 -2 - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - summit|ascent: BGC compset on ne30np4 grid @@ -1732,16 +1722,16 @@ - allactive+sunspot|aurora: default, 96 mpi x 1 omp @ root 0 + allactive+sunspot|aurora: default, 1 node x 1 omp @ root 0 - 96 - 96 - 96 - 96 - 96 - 96 - 96 - 96 + -1 + 12 + -1 + -1 + -1 + -1 + -1 + -1 diff --git a/cime_config/machines/Depends.oneapi-ifx.cmake b/cime_config/machines/Depends.oneapi-ifx.cmake deleted file mode 100644 index 5a958df26eba..000000000000 --- a/cime_config/machines/Depends.oneapi-ifx.cmake +++ /dev/null @@ -1,5 +0,0 @@ - -# compile mpas_seaice_core_interface.f90 with ifort, not ifx -if (NOT MPILIB STREQUAL "openmpi") - e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") -endif() diff --git a/cime_config/machines/Depends.oneapi-ifxgpu.cmake b/cime_config/machines/Depends.oneapi-ifxgpu.cmake deleted file mode 100644 index 5a958df26eba..000000000000 --- a/cime_config/machines/Depends.oneapi-ifxgpu.cmake +++ /dev/null @@ -1,5 +0,0 @@ - -# compile mpas_seaice_core_interface.f90 with ifort, not ifx -if (NOT MPILIB STREQUAL "openmpi") - e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort") -endif() diff --git a/cime_config/machines/cmake_macros/oneapi-ifx.cmake b/cime_config/machines/cmake_macros/oneapi-ifx.cmake index 9ab0cdda7d51..45ed57aa2daf 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifx.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifx.cmake @@ -4,15 +4,16 @@ if (compile_threaded) string(APPEND CMAKE_CXX_FLAGS " -qopenmp") string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp") endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0") +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -gline-tables-only -g") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -gline-tables-only -g") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99") -string(APPEND CMAKE_CXX_FLAGS " -fp-model precise") -string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise") +string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99 -gline-tables-only -g") +string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g") +# -mllvm -disable-hir-temp-cleanup for oneapi v2025.2.0 https://github.com/argonne-lcf/AuroraBugTracking/issues/64 +string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g -mllvm -disable-hir-temp-cleanup") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI") string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free") diff --git a/cime_config/machines/cmake_macros/oneapi-ifx_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifx_aurora.cmake index 886ac25cd092..9748a53fa577 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifx_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifx_aurora.cmake @@ -3,3 +3,4 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() +set(Kokkos_ENABLE_SYCL FALSE CACHE BOOL "") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake index d7dfae002192..b97552c8a711 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake @@ -4,15 +4,19 @@ if (compile_threaded) string(APPEND CMAKE_CXX_FLAGS " -qopenmp") string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp") endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0") + +# 'just' -g may lead to linker internal errors and/or huge builds out of quotas +# EAMxx ignores generic CMAKE_CXX_FLAGS +string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g -gline-tables-only") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -fpscomp logicals -g") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -g -gline-tables-only") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpscomp logicals") string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99") +string(APPEND CMAKE_C_FLAGS " -fp-model precise") string(APPEND CMAKE_CXX_FLAGS " -fp-model precise") -string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise") +# -mllvm -disable-hir-temp-cleanup for oneapi v2025.2.0 https://github.com/argonne-lcf/AuroraBugTracking/issues/64 +string(APPEND CMAKE_Fortran_FLAGS " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -mllvm -disable-hir-temp-cleanup") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI") string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index 6835515164f0..bf256e41aa17 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -1,7 +1,14 @@ -string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax") if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") -string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") + +string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") + +# Let's start with the best case: using device buffers in MPI calls by default. +# This is paired with MPIR_CVAR_ENABLE_GPU=1 in config_machines.xml. If this +# ends up causing instability, we can switch to OFF. +set(SCREAM_MPI_ON_DEVICE ON CACHE STRING "") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index c05c40b01988..6ef569154c20 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -575,11 +575,16 @@ - /lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle + /lus/flare/projects/E3SMinput/tools/qsub/throttle + (\d+)\.aurora-pbs + + -l filesystems=home:flare + - EarlyAppAccess - workq-route - workq + debug + debug-scaling + prod + prod-large diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index c3752156e1a7..162cff84d6d6 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3623,103 +3623,109 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss ALCF Aurora, 10624 nodes, 2x52c SPR, 6x2s PVC, 2x512GB DDR5, 2x64GB CPU-HBM, 6x128GB GPU-HBM, Slingshot 11, PBSPro aurora-uan-.* LINUX - oneapi-ifx,oneapi-ifxgpu,gnu - mpich - CSC249ADSE15_CNDA - /lus/flare/projects/CSC249ADSE15_CNDA/performance_archive + oneapi-ifxgpu,oneapi-ifx + mpich,mpich1024 + E3SM_Dec + /lus/flare/projects/E3SMinput/baselines .* - /lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch - /lus/flare/projects/CSC249ADSE15_CNDA/inputdata - /lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + /lus/flare/projects/$PROJECT/$USER/scratch + /lus/flare/projects/E3SMinput/data + /lus/flare/projects/E3SMinput/data/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE - /lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER - /lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + /lus/flare/projects/E3SMinput/baselines/$COMPILER + /lus/flare/projects/E3SMinput/tools/cprnc/cprnc 16 - e3sm_developer + e3sm_integration 4 pbspro e3sm - 208 - 104 - 104 - 48 - FALSE + 102 + 96 + 102 + 12 + TRUE - mpiexec - - - -np {{ total_tasks }} --label - -ppn {{ tasks_per_node }} - -envall - -d $ENV{OMP_NUM_THREADS} - $ENV{GPU_TILE_COMPACT} - + mpiexec + + + -np {{ total_tasks }} --label + -ppn {{ tasks_per_node }} + --cpu-bind $ENV{RANKS_BIND} + -d $ENV{OMP_NUM_THREADS} $ENV{RLIMITS} + $ENV{GPU_TILE_COMPACT} + - /lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh - /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh - /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py - module - module - /soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python - - cmake - - - oneapi/eng-compiler/2024.04.15.002 - - - kokkos/git.7ff87a5-omp-sycl - - - spack-pe-gcc cmake - gcc/10.3.0 + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/env_modules_python.py + module + module + /usr/share/lmod/lmod/libexec/lmod python + + /lus/flare/projects/E3SMinput/soft/modulefiles + cmake/3.31.8 + oneapi/release/2025.2.0 + netcdf/4.9.3c-4.6.2f + pnetcdf/1.14.0 - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - - /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH} - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH} - - - 1 + + mpich-config/collective-tuning/1024 + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0 + + 131072 + 20 + level_zero:gpu - NO_GPU - 0 + 1 + 1 + 1 + disabled + 8388608 + 240 + 240 + 1 + 1 + 0x001 + 1 + 1 + + 1 disable disable - 1 4000MB 0 - /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh - 131072 - 20 - $ENV{KOKKOS_ROOT} + /lus/flare/projects/E3SMinput/tools/mpi_wrapper_utils/gpu_tile_compact.sh + list:1-8:9-16:17-24:25-32:33-40:41-48:53-60:61-68:69-76:77-84:85-92:93-100 --gpu-bind list:0.0:0.1:1.0:1.1:2.0:2.1:3.0:3.1:4.0:4.1:5.0:5.1 --mem-bind list:0:0:0:0:0:0:1:1:1:1:1:1 1 - 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 + + 0 DISABLED - 131072 - 20 0 + core - - verbose,granularity=thread,balanced + + granularity=core,balanced 128M - - threads - 128M + + --rlimits CORE + + -1 + -1 diff --git a/externals/scorpio b/externals/scorpio index 804f87c48fc9..ff1d22fced71 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 804f87c48fc9013009307b806d1aff2afa3a0d7c +Subproject commit ff1d22fced718e216557eba2069607abf6d73efc