Skip to content

Update Aurora config in maint-3.0 #7882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: maint-3.0
Choose a base branch
from
+125 −121
Open

Update Aurora config in maint-3.0 #7882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
06a1227
Update Aurora config in maint-3.0
amametjanov Oct 29, 2025
7bb04ea
Update scorpio submodule
amametjanov Nov 12, 2025
165dc72
Cleanup
amametjanov Nov 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 11 additions & 21 deletions cime_config/allactive/config_pesall.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1023,9 +1023,9 @@
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="summit|ascent">
<mach name="summit|ascent|aurora">
<pes compset="any" pesize="any">
<comment>summit|ascent: any compset on ne30np4 grid</comment>
<comment>summit|ascent|aurora: any compset on ne30np4 grid</comment>
<ntasks>
<ntasks_atm>-2</ntasks_atm>
<ntasks_lnd>-2</ntasks_lnd>
Expand All @@ -1036,16 +1036,6 @@
<ntasks_glc>-2</ntasks_glc>
<ntasks_wav>-2</ntasks_wav>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
</pes>
<pes compset=".*EAM.+ELM.+MPASSI.+SGLC.+SWAV.+BGC.*" pesize="any">
<comment>summit|ascent: BGC compset on ne30np4 grid</comment>
Expand Down Expand Up @@ -1732,16 +1722,16 @@
</mach>
<mach name="sunspot|aurora">
<pes compset="any" pesize="any">
<comment>allactive+sunspot|aurora: default, 96 mpi x 1 omp @ root 0</comment>
<comment>allactive+sunspot|aurora: default, 1 node x 1 omp @ root 0</comment>
<ntasks>
<ntasks_atm>96</ntasks_atm>
<ntasks_lnd>96</ntasks_lnd>
<ntasks_rof>96</ntasks_rof>
<ntasks_ice>96</ntasks_ice>
<ntasks_ocn>96</ntasks_ocn>
<ntasks_glc>96</ntasks_glc>
<ntasks_wav>96</ntasks_wav>
<ntasks_cpl>96</ntasks_cpl>
<ntasks_atm>-1</ntasks_atm>
<ntasks_lnd>12</ntasks_lnd>
<ntasks_rof>-1</ntasks_rof>
<ntasks_ice>-1</ntasks_ice>
<ntasks_ocn>-1</ntasks_ocn>
<ntasks_glc>-1</ntasks_glc>
<ntasks_wav>-1</ntasks_wav>
<ntasks_cpl>-1</ntasks_cpl>
</ntasks>
</pes>
</mach>
Expand Down
5 changes: 0 additions & 5 deletions cime_config/machines/Depends.oneapi-ifx.cmake
View file
Open in desktop

This file was deleted.

5 changes: 0 additions & 5 deletions cime_config/machines/Depends.oneapi-ifxgpu.cmake
View file
Open in desktop

This file was deleted.

15 changes: 8 additions & 7 deletions cime_config/machines/cmake_macros/oneapi-ifx.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@ if (compile_threaded)
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0")
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -gline-tables-only -g")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -gline-tables-only -g")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99 -gline-tables-only -g")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g")
# -mllvm -disable-hir-temp-cleanup for oneapi v2025.2.0 https://github.com/argonne-lcf/AuroraBugTracking/issues/64
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g -mllvm -disable-hir-temp-cleanup")
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
Expand Down
1 change: 1 addition & 0 deletions cime_config/machines/cmake_macros/oneapi-ifx_aurora.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()
set(Kokkos_ENABLE_SYCL FALSE CACHE BOOL "")
16 changes: 10 additions & 6 deletions cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,19 @@ if (compile_threaded)
string(APPEND CMAKE_CXX_FLAGS " -qopenmp")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -qopenmp")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpe0")

# 'just' -g may lead to linker internal errors and/or huge builds out of quotas
# EAMxx ignores generic CMAKE_CXX_FLAGS
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g -gline-tables-only")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -fpscomp logicals -g")
string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fp-model precise -O2 -g -gline-tables-only")
string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -fpscomp logicals")
string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99")
string(APPEND CMAKE_C_FLAGS " -fp-model precise")
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
# -mllvm -disable-hir-temp-cleanup for oneapi v2025.2.0 https://github.com/argonne-lcf/AuroraBugTracking/issues/64
string(APPEND CMAKE_Fortran_FLAGS " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -mllvm -disable-hir-temp-cleanup")
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")
Expand Down
13 changes: 10 additions & 3 deletions cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@

string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core")
string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=16 -Wl,--no-relax")
if (compile_threaded)
string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64")
endif()
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"")
string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions")

string(APPEND KOKKOS_OPTIONS " -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_SERIAL=On -DKokkos_ARCH_INTEL_PVC=On -DKokkos_ENABLE_SYCL=On -DKokkos_ENABLE_EXPLICIT_INSTANTIATION=Off")
string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ")
string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ")

# Let's start with the best case: using device buffers in MPI calls by default.
# This is paired with MPIR_CVAR_ENABLE_GPU=1 in config_machines.xml. If this
# ends up causing instability, we can switch to OFF.
set(SCREAM_MPI_ON_DEVICE ON CACHE STRING "")
13 changes: 9 additions & 4 deletions cime_config/machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,11 +575,16 @@
</batch_system>

<batch_system MACH="aurora" type="pbspro">
<batch_submit>/lus/flare/projects/CSC249ADSE15_CNDA/tools/qsub/throttle</batch_submit>
<batch_submit>/lus/flare/projects/E3SMinput/tools/qsub/throttle</batch_submit>
<jobid_pattern>(\d+)\.aurora-pbs</jobid_pattern>
<directives>
<directive> -l filesystems=home:flare </directive>
</directives>
<queues>
<queue walltimemax="00:59:00" jobmin="1" jobmax="2560" default="true">EarlyAppAccess</queue>
<queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq-route</queue>
<queue walltimemax="00:59:00" jobmin="1" jobmax="10624">workq</queue>
<queue walltimemax="01:00:00" nodemin="1" nodemax="2" default="true">debug</queue>
<queue walltimemax="01:00:00" nodemin="3" nodemax="31" >debug-scaling</queue>
<queue walltimemax="01:00:00" nodemin="32" nodemax="1919" >prod</queue>
<queue walltimemax="01:00:00" nodemin="1920" nodemax="10624">prod-large</queue>
</queues>
</batch_system>

Expand Down
144 changes: 75 additions & 69 deletions cime_config/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3623,103 +3623,109 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss
<DESC>ALCF Aurora, 10624 nodes, 2x52c SPR, 6x2s PVC, 2x512GB DDR5, 2x64GB CPU-HBM, 6x128GB GPU-HBM, Slingshot 11, PBSPro</DESC>
<NODENAME_REGEX>aurora-uan-.*</NODENAME_REGEX>
<OS>LINUX</OS>
<COMPILERS>oneapi-ifx,oneapi-ifxgpu,gnu</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<CHARGE_ACCOUNT>CSC249ADSE15_CNDA</CHARGE_ACCOUNT>
<SAVE_TIMING_DIR>/lus/flare/projects/CSC249ADSE15_CNDA/performance_archive</SAVE_TIMING_DIR>
<COMPILERS>oneapi-ifxgpu,oneapi-ifx</COMPILERS>
<MPILIBS>mpich,mpich1024</MPILIBS>
<PROJECT>E3SM_Dec</PROJECT>
<SAVE_TIMING_DIR>/lus/flare/projects/E3SMinput/baselines</SAVE_TIMING_DIR>
<SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
<CIME_OUTPUT_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<CIME_OUTPUT_ROOT>/lus/flare/projects/$PROJECT/$USER/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/lus/flare/projects/E3SMinput/data</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/lus/flare/projects/E3SMinput/data/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc</CCSM_CPRNC>
<BASELINE_ROOT>/lus/flare/projects/E3SMinput/baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/lus/flare/projects/E3SMinput/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>16</GMAKE_J>
<TESTS>e3sm_developer</TESTS>
<TESTS>e3sm_integration</TESTS>
<NTEST_PARALLEL_JOBS>4</NTEST_PARALLEL_JOBS>
<BATCH_SYSTEM>pbspro</BATCH_SYSTEM>
<SUPPORTED_BY>e3sm</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>208</MAX_TASKS_PER_NODE>
<MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">104</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>104</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">48</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<MAX_TASKS_PER_NODE>102</MAX_TASKS_PER_NODE>
<MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">96</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>102</MAX_MPITASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
<!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
<arguments>
<arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
<arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
<arg name="ranks_bind">-envall</arg>
<arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS}</arg>
<arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
</arguments>
<executable>mpiexec</executable>
<!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
<arguments>
<arg name="total_num_tasks">-np {{ total_tasks }} --label</arg>
<arg name="ranks_per_node">-ppn {{ tasks_per_node }}</arg>
<arg name="ranks_bind">--cpu-bind $ENV{RANKS_BIND}</arg>
<arg name="threads_per_rank">-d $ENV{OMP_NUM_THREADS} $ENV{RLIMITS}</arg>
<arg name="gpu_maps">$ENV{GPU_TILE_COMPACT}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="sh">/lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh</init_path>
<init_path lang="csh">/soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh</init_path>
<init_path lang="python">/soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py</init_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<cmd_path lang="python">/soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python</cmd_path>
<modules>
<command name="load">cmake</command>
</modules>
<modules compiler="!gnu">
<command name="load">oneapi/eng-compiler/2024.04.15.002</command>
</modules>
<modules compiler="oneapi-ifxgpu">
<command name="load">kokkos/git.7ff87a5-omp-sycl</command>
</modules>
<modules compiler="gnu">
<command name="unload">spack-pe-gcc cmake</command>
<command name="load">gcc/10.3.0</command>
<init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
<init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
<modules>
<command name="use">/lus/flare/projects/E3SMinput/soft/modulefiles</command>
<command name="load">cmake/3.31.8</command>
<command name="load">oneapi/release/2025.2.0</command>
<command name="load">netcdf/4.9.3c-4.6.2f</command>
<command name="load">pnetcdf/1.14.0</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables>
<env name="NETCDF_C_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002</env>
<env name="NETCDF_FORTRAN_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002</env>
<env name="PNETCDF_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002</env>
<env name="LD_LIBRARY_PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH}</env>
<env name="PATH">/lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH}</env>
</environment_variables>
<environment_variables DEBUG="TRUE">
<env name="HYDRA_TOPO_DEBUG">1</env>
<modules mpilib="mpich1024">
<command name="load">mpich-config/collective-tuning/1024</command>
</modules>
</module_system>
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
<environment_variables>
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
<env name="RLIMITS"> </env>
</environment_variables>
<environment_variables compiler="oneapi-ifxgpu">
<env name="ONEAPI_DEVICE_SELECTOR">level_zero:gpu</env>
<env name="ONEAPI_MPICH_GPU">NO_GPU</env>
<env name="MPIR_CVAR_ENABLE_GPU">0</env>
<env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
<env name="UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS">1</env>
<env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>
<env name="FI_MR_CACHE_MONITOR">disabled</env>
<env name="FI_CXI_OVFLOW_BUF_SIZE">8388608</env>
<env name="PALS_PING_PERIOD">240</env>
<env name="PALS_RPC_TIMEOUT">240</env>
<env name="SYCL_PI_LEVEL_ZERO_SINGLE_THREAD_MODE">1</env>
<env name="SYCL_PI_LEVEL_ZERO_DISABLE_USM_ALLOCATOR">1</env>
<env name="SYCL_PI_LEVEL_ZERO_USM_RESIDENT">0x001</env>
<env name="UR_L0_USE_DRIVER_INORDER_LISTS">1</env>
<env name="UR_L0_USE_COPY_ENGINE_FOR_IN_ORDER_QUEUE">1</env>

<env name="MPIR_CVAR_ENABLE_GPU">1</env>
<env name="romio_cb_read">disable</env>
<env name="romio_cb_write">disable</env>
<env name="SYCL_CACHE_PERSISTENT">1</env>
<env name="GATOR_INITIAL_MB">4000MB</env>
<env name="GATOR_DISABLE">0</env>
<env name="GPU_TILE_COMPACT">/soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
<env name="Kokkos_ROOT">$ENV{KOKKOS_ROOT}</env>
<env name="GPU_TILE_COMPACT">/lus/flare/projects/E3SMinput/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
<env name="RANKS_BIND">list:1-8:9-16:17-24:25-32:33-40:41-48:53-60:61-68:69-76:77-84:85-92:93-100 --gpu-bind list:0.0:0.1:1.0:1.1:2.0:2.1:3.0:3.1:4.0:4.1:5.0:5.1 --mem-bind list:0:0:0:0:0:0:1:1:1:1:1:1</env>
<env name="ZES_ENABLE_SYSMAN">1</env>
<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4</env>
<!-- default is ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE: enable this to run 4 MPI/tile or 48 MPI/node
<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4</env>-->
<!-- <env name="ZE_FLAT_DEVICE_HIERARCHY">FLAT</env>
<env name="ZEX_NUMBER_OF_CCS">0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4</env>-->
</environment_variables>
<environment_variables compiler="oneapi-ifx">
<env name="LIBOMPTARGET_DEBUG">0</env><!--default 0, max 5 -->
<env name="OMP_TARGET_OFFLOAD">DISABLED</env><!--default OMP_TARGET_OFFLOAD=MANDATORY-->
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
<env name="MPIR_CVAR_ENABLE_GPU">0</env>
<env name="GPU_TILE_COMPACT"> </env>
<env name="RANKS_BIND">core</env>
</environment_variables>
<environment_variables BUILD_THREADED="TRUE" compiler="!gnu">
<env name="KMP_AFFINITY">verbose,granularity=thread,balanced</env>
<environment_variables BUILD_THREADED="TRUE">
<env name="KMP_AFFINITY">granularity=core,balanced</env>
<env name="OMP_STACKSIZE">128M</env>
</environment_variables>
<environment_variables BUILD_THREADED="TRUE" compiler="gnu">
<env name="OMP_PLACES">threads</env>
<env name="OMP_STACKSIZE">128M</env>
<environment_variables DEBUG="TRUE">
<env name="RLIMITS">--rlimits CORE</env>
</environment_variables>
<resource_limits DEBUG="TRUE">
<resource name="RLIMIT_CORE">-1</resource>
</resource_limits>
<resource_limits>
<resource name="RLIMIT_STACK">-1</resource>
</resource_limits>
Expand Down
2 changes: 1 addition & 1 deletion externals/scorpio
Open in desktop
Submodule scorpio updated 1 files
+4 −2 src/clib/pio_nc.c