Skip to content

Commit 33a593d

Browse files
amametjanovamametjanov
committed
Merge branch 'azamat/aurora/oneapi-202520' (PR #7693)
Update oneapi compiler on Aurora: 1. oneapi/release/2025.0.5 -> 2025.2.0 2. update to world-shared E3SMinput paths 3. cleanup: put e3sm-vars into e3sm-modules [BFB]
2 parents 9d82f7f + 77777bf commit 33a593d

File tree

4 files changed

+22
-27
lines changed

4 files changed

+22
-27
lines changed

‎cime_config/machines/cmake_macros/oneapi-ifx.cmake‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
1212
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
1313
string(APPEND CMAKE_C_FLAGS " -fp-model precise -std=gnu99 -gline-tables-only -g")
1414
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise -gline-tables-only -g")
15-
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g")
15+
# -mllvm -disable-hir-temp-cleanup for oneapi v2025.2.0 https://github.com/argonne-lcf/AuroraBugTracking/issues/64
16+
string(APPEND CMAKE_Fortran_FLAGS " -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -gline-tables-only -g -mllvm -disable-hir-temp-cleanup")
1617
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
1718
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
1819
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")

‎cime_config/machines/cmake_macros/oneapi-ifxgpu.cmake‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g")
1515
string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
1616
string(APPEND CMAKE_C_FLAGS " -fp-model precise")
1717
string(APPEND CMAKE_CXX_FLAGS " -fp-model precise")
18-
string(APPEND CMAKE_Fortran_FLAGS " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise")
18+
# -mllvm -disable-hir-temp-cleanup for oneapi v2025.2.0 https://github.com/argonne-lcf/AuroraBugTracking/issues/64
19+
string(APPEND CMAKE_Fortran_FLAGS " -fpscomp logicals -traceback -convert big_endian -assume byterecl -assume realloc_lhs -fp-model precise -mllvm -disable-hir-temp-cleanup")
1920
string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL -DHAVE_SLASHPROC -DHIDE_MPI")
2021
string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -fixed -132")
2122
string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -free")

‎cime_config/machines/config_batch.xml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,6 @@
203203
<directive> -l select={{ num_nodes }} </directive>
204204
<directive default="n"> -r {{ rerunnable }} </directive>
205205
<directive> -j oe </directive>
206-
<directive> -V </directive>
207206
</directives>
208207
</batch_system>
209208

@@ -561,16 +560,16 @@
561560
</batch_system>
562561

563562
<batch_system MACH="aurora" type="pbspro">
564-
<batch_submit>/lus/flare/projects/E3SM_Dec/tools/qsub/throttle</batch_submit>
563+
<batch_submit>/lus/flare/projects/E3SMinput/tools/qsub/throttle</batch_submit>
565564
<jobid_pattern>(\d+)\.aurora-pbs</jobid_pattern>
566565
<directives>
567566
<directive> -l filesystems=home:flare </directive>
568567
</directives>
569568
<queues>
570569
<queue walltimemax="01:00:00" nodemin="1" nodemax="2" default="true">debug</queue>
571570
<queue walltimemax="01:00:00" nodemin="3" nodemax="31" >debug-scaling</queue>
572-
<queue walltimemax="01:00:00" nodemin="32" nodemax="2048" >prod</queue>
573-
<queue walltimemax="01:00:00" nodemin="2049" nodemax="10624">prod-large</queue>
571+
<queue walltimemax="01:00:00" nodemin="32" nodemax="1919" >prod</queue>
572+
<queue walltimemax="01:00:00" nodemin="1920" nodemax="10624">prod-large</queue>
574573
</queues>
575574
</batch_system>
576575

‎cime_config/machines/config_machines.xml‎

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3492,24 +3492,24 @@
34923492
<COMPILERS>oneapi-ifxgpu,oneapi-ifx</COMPILERS>
34933493
<MPILIBS>mpich,mpich1024</MPILIBS>
34943494
<PROJECT>E3SM_Dec</PROJECT>
3495-
<SAVE_TIMING_DIR>/lus/flare/projects/E3SM_Dec/performance_archive</SAVE_TIMING_DIR>
3495+
<SAVE_TIMING_DIR>/lus/flare/projects/E3SMinput/baselines</SAVE_TIMING_DIR>
34963496
<SAVE_TIMING_DIR_PROJECTS>.*</SAVE_TIMING_DIR_PROJECTS>
34973497
<CIME_OUTPUT_ROOT>/lus/flare/projects/$PROJECT/$USER/scratch</CIME_OUTPUT_ROOT>
3498-
<DIN_LOC_ROOT>/lus/flare/projects/E3SM_Dec/inputdata</DIN_LOC_ROOT>
3499-
<DIN_LOC_ROOT_CLMFORC>/lus/flare/projects/E3SM_Dec/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
3498+
<DIN_LOC_ROOT>/lus/flare/projects/E3SMinput/data</DIN_LOC_ROOT>
3499+
<DIN_LOC_ROOT_CLMFORC>/lus/flare/projects/E3SMinput/data/atm/datm7</DIN_LOC_ROOT_CLMFORC>
35003500
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
3501-
<BASELINE_ROOT>/lus/flare/projects/E3SM_Dec/baselines/$COMPILER</BASELINE_ROOT>
3502-
<CCSM_CPRNC>/lus/flare/projects/E3SM_Dec/tools/cprnc/cprnc</CCSM_CPRNC>
3501+
<BASELINE_ROOT>/lus/flare/projects/E3SMinput/baselines/$COMPILER</BASELINE_ROOT>
3502+
<CCSM_CPRNC>/lus/flare/projects/E3SMinput/tools/cprnc/cprnc</CCSM_CPRNC>
35033503
<GMAKE_J>16</GMAKE_J>
3504-
<TESTS>e3sm_developer</TESTS>
3504+
<TESTS>e3sm_integration</TESTS>
35053505
<NTEST_PARALLEL_JOBS>4</NTEST_PARALLEL_JOBS>
35063506
<BATCH_SYSTEM>pbspro</BATCH_SYSTEM>
35073507
<SUPPORTED_BY>e3sm</SUPPORTED_BY>
35083508
<MAX_TASKS_PER_NODE>102</MAX_TASKS_PER_NODE>
35093509
<MAX_TASKS_PER_NODE compiler="oneapi-ifxgpu">96</MAX_TASKS_PER_NODE>
35103510
<MAX_MPITASKS_PER_NODE>102</MAX_MPITASKS_PER_NODE>
35113511
<MAX_MPITASKS_PER_NODE compiler="oneapi-ifxgpu">12</MAX_MPITASKS_PER_NODE>
3512-
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
3512+
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
35133513
<mpirun mpilib="default">
35143514
<executable>mpiexec</executable>
35153515
<!--executable>numactl -m 2-3 mpiexec</executable--><!--for HBM runs-->
@@ -3529,9 +3529,13 @@
35293529
<cmd_path lang="csh">module</cmd_path>
35303530
<cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
35313531
<modules>
3532-
<command name="load">cmake/3.30.5</command>
3533-
<command name="load">oneapi/release/2025.0.5</command>
3534-
</modules>
3532+
<command name="use">/lus/flare/projects/E3SMinput/soft/modulefiles</command>
3533+
<command name="load">cmake/3.31.8</command>
3534+
<command name="load">oneapi/release/2025.2.0</command>
3535+
<command name="load">netcdf/4.9.3c-4.6.2f</command>
3536+
<command name="load">pnetcdf/1.14.0</command>
3537+
<command name="load">adios2/2.10.2</command>
3538+
</modules>
35353539
<modules mpilib="mpich1024">
35363540
<command name="load">mpich-config/collective-tuning/1024</command>
35373541
</modules>
@@ -3540,16 +3544,6 @@
35403544
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
35413545
<MAX_GB_OLD_TEST_DATA>0</MAX_GB_OLD_TEST_DATA>
35423546
<environment_variables>
3543-
<env name="NETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002</env>
3544-
<env name="PNETCDF_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002</env>
3545-
<env name="LD_LIBRARY_PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:$ENV{LD_LIBRARY_PATH}</env>
3546-
<env name="PATH">/lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:$ENV{PATH}</env>
3547-
<env name="PKG_CONFIG_PATH">/lus/flare/projects/E3SM_Dec/soft/protobuf/21.6/oneapi.release.2025.0.5/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}</env>
3548-
<env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lus/flare/projects/E3SM_Dec/soft/adios2/2.10.2/oneapi.release.2025.0.5; else echo "$ADIOS2_ROOT"; fi}</env>
3549-
<env name="BLOSC2_ROOT">$SHELL{if [ -z "$BLOSC2_ROOT" ]; then echo /lus/flare/projects/E3SM_Dec/soft/c-blosc2/2.15.2/oneapi.release.2025.0.5; else echo "$BLOSC2_ROOT"; fi}</env>
3550-
<env name="MGARD_ROOT">$SHELL{if [ -z "$MGARD_ROOT" ]; then echo /lus/flare/projects/E3SM_Dec/soft/mgard/1.5.2/oneapi.release.2025.0.5; else echo "$MGARD_ROOT"; fi}</env>
3551-
<env name="SZ_ROOT">$SHELL{if [ -z "$SZ_ROOT" ]; then echo /lus/flare/projects/E3SM_Dec/soft/sz/2.1.12.5/oneapi.release.2025.0.5; else echo "$SZ_ROOT"; fi}</env>
3552-
<env name="ZFP_ROOT">$SHELL{if [ -z "$ZFP_ROOT" ]; then echo /lus/flare/projects/E3SM_Dec/soft/zfp/1.0.1/oneapi.release.2025.0.5; else echo "$ZFP_ROOT"; fi}</env>
35533547
<env name="FI_CXI_DEFAULT_CQ_SIZE">131072</env>
35543548
<env name="FI_CXI_CQ_FILL_PERCENT">20</env>
35553549
<env name="RLIMITS"> </env>
@@ -3574,7 +3568,7 @@
35743568
<env name="romio_cb_write">disable</env>
35753569
<env name="GATOR_INITIAL_MB">4000MB</env>
35763570
<env name="GATOR_DISABLE">0</env>
3577-
<env name="GPU_TILE_COMPACT">/lus/flare/projects/E3SM_Dec/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
3571+
<env name="GPU_TILE_COMPACT">/lus/flare/projects/E3SMinput/tools/mpi_wrapper_utils/gpu_tile_compact.sh</env>
35783572
<env name="RANKS_BIND">list:1-8:9-16:17-24:25-32:33-40:41-48:53-60:61-68:69-76:77-84:85-92:93-100 --gpu-bind list:0.0:0.1:1.0:1.1:2.0:2.1:3.0:3.1:4.0:4.1:5.0:5.1 --mem-bind list:0:0:0:0:0:0:1:1:1:1:1:1</env>
35793573
<env name="ZES_ENABLE_SYSMAN">1</env>
35803574
<!-- default is ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE: enable this to run 4 MPI/tile or 48 MPI/node

0 commit comments

Comments
 (0)