Skip to content

Commit ab11bdc

Browse files
xylarxylar
authored
Merge pull request #204 from xylar/update-pm-cpu-gpu
Update Perlmutter CPU and GPU
2 parents 4f0b8ee + f94d434 commit ab11bdc

16 files changed

+643
-935
lines changed

‎mache/cime_machine_config/config_machines.xml‎

Lines changed: 175 additions & 392 deletions
Large diffs are not rendered by default.

‎mache/spack/pm-cpu_gnu_mpich.csh‎

Lines changed: 34 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,38 @@
1-
module rm cray-hdf5-parallel &> /dev/null
2-
module rm cray-netcdf-hdf5parallel &> /dev/null
3-
module rm cray-parallel-netcdf &> /dev/null
4-
module rm PrgEnv-gnu &> /dev/null
5-
module rm PrgEnv-intel &> /dev/null
6-
module rm PrgEnv-nvidia &> /dev/null
7-
module rm PrgEnv-cray &> /dev/null
8-
module rm PrgEnv-aocc &> /dev/null
9-
module rm gcc-native &> /dev/null
10-
module rm intel &> /dev/null
11-
module rm intel-oneapi &> /dev/null
12-
module rm cudatoolkit &> /dev/null
13-
module rm climate-utils &> /dev/null
14-
module rm cray-libsci &> /dev/null
15-
module rm matlab &> /dev/null
16-
module rm craype-accel-nvidia80 &> /dev/null
17-
module rm craype-accel-host &> /dev/null
18-
module rm perftools-base &> /dev/null
19-
module rm perftools &> /dev/null
20-
module rm darshan &> /dev/null
1+
module rm cpe \
2+
cray-hdf5-parallel \
3+
cray-netcdf-hdf5parallel \
4+
cray-parallel-netcdf \
5+
PrgEnv-gnu \
6+
PrgEnv-intel \
7+
PrgEnv-nvidia \
8+
PrgEnv-cray \
9+
PrgEnv-aocc \
10+
gcc-native \
11+
intel \
12+
intel-oneapi \
13+
cudatoolkit \
14+
climate-utils \
15+
cray-libsci \
16+
matlab \
17+
craype-accel-nvidia80 \
18+
craype-accel-host \
19+
perftools-base \
20+
perftools \
21+
darshan \
22+
cray-mpich &> /dev/null
2123

22-
module load PrgEnv-gnu/8.5.0
23-
module load gcc/12.2.0
24-
module load craype-accel-host
25-
{% if e3sm_lapack %}
26-
module load cray-libsci/23.02.1.1
27-
{% endif %}
28-
module load craype/2.7.20
29-
module rm cray-mpich &> /dev/null
30-
module load libfabric/1.15.2.0
31-
module load cray-mpich/8.1.25
24+
module load PrgEnv-gnu/8.5.0 \
25+
gcc-native/12.3 \
26+
cray-libsci/23.12.5 \
27+
craype-accel-host \
28+
craype/2.7.30 \
29+
libfabric/1.20.1 \
30+
cray-mpich/8.1.28 \
31+
cmake/3.24.3
3232
{% if e3sm_hdf5_netcdf %}
33-
module rm cray-hdf5-parallel &> /dev/null
34-
module rm cray-netcdf-hdf5parallel &> /dev/null
35-
module rm cray-parallel-netcdf &> /dev/null
36-
module load cray-hdf5-parallel/1.12.2.3
37-
module load cray-netcdf-hdf5parallel/4.9.0.3
38-
module load cray-parallel-netcdf/1.12.3.3
33+
module load cray-hdf5-parallel/1.12.2.9 \
34+
cray-netcdf-hdf5parallel/4.9.0.9 \
35+
cray-parallel-netcdf/1.12.3.9
3936
{% endif %}
4037

4138
{% if e3sm_hdf5_netcdf %}
@@ -55,6 +52,7 @@ setenv HDF5_USE_FILE_LOCKING FALSE
5552
## Not needed
5653
# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
5754
setenv FI_CXI_RX_MATCH_MODE software
55+
setenv FI_MR_CACHE_MONITOR kdreg2
5856
setenv MPICH_COLL_SYNC MPI_Bcast
5957
setenv GATOR_INITIAL_MB 4000MB
6058
setenv BLA_VENDOR Generic

‎mache/spack/pm-cpu_gnu_mpich.sh‎

Lines changed: 34 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,38 @@
1-
module rm cray-hdf5-parallel &> /dev/null
2-
module rm cray-netcdf-hdf5parallel &> /dev/null
3-
module rm cray-parallel-netcdf &> /dev/null
4-
module rm PrgEnv-gnu &> /dev/null
5-
module rm PrgEnv-intel &> /dev/null
6-
module rm PrgEnv-nvidia &> /dev/null
7-
module rm PrgEnv-cray &> /dev/null
8-
module rm PrgEnv-aocc &> /dev/null
9-
module rm gcc-native &> /dev/null
10-
module rm intel &> /dev/null
11-
module rm intel-oneapi &> /dev/null
12-
module rm cudatoolkit &> /dev/null
13-
module rm climate-utils &> /dev/null
14-
module rm cray-libsci &> /dev/null
15-
module rm matlab &> /dev/null
16-
module rm craype-accel-nvidia80 &> /dev/null
17-
module rm craype-accel-host &> /dev/null
18-
module rm perftools-base &> /dev/null
19-
module rm perftools &> /dev/null
20-
module rm darshan &> /dev/null
1+
module rm cpe \
2+
cray-hdf5-parallel \
3+
cray-netcdf-hdf5parallel \
4+
cray-parallel-netcdf \
5+
PrgEnv-gnu \
6+
PrgEnv-intel \
7+
PrgEnv-nvidia \
8+
PrgEnv-cray \
9+
PrgEnv-aocc \
10+
gcc-native \
11+
intel \
12+
intel-oneapi \
13+
cudatoolkit \
14+
climate-utils \
15+
cray-libsci \
16+
matlab \
17+
craype-accel-nvidia80 \
18+
craype-accel-host \
19+
perftools-base \
20+
perftools \
21+
darshan \
22+
cray-mpich &> /dev/null
2123

22-
module load PrgEnv-gnu/8.5.0
23-
module load gcc/12.2.0
24-
module load craype-accel-host
25-
{% if e3sm_lapack %}
26-
module load cray-libsci/23.02.1.1
27-
{% endif %}
28-
module load craype/2.7.20
29-
module rm cray-mpich &> /dev/null
30-
module load libfabric/1.15.2.0
31-
module load cray-mpich/8.1.25
24+
module load PrgEnv-gnu/8.5.0 \
25+
gcc-native/12.3 \
26+
cray-libsci/23.12.5 \
27+
craype-accel-host \
28+
craype/2.7.30 \
29+
libfabric/1.20.1 \
30+
cray-mpich/8.1.28 \
31+
cmake/3.24.3
3232
{% if e3sm_hdf5_netcdf %}
33-
module rm cray-hdf5-parallel &> /dev/null
34-
module rm cray-netcdf-hdf5parallel &> /dev/null
35-
module rm cray-parallel-netcdf &> /dev/null
36-
module load cray-hdf5-parallel/1.12.2.3
37-
module load cray-netcdf-hdf5parallel/4.9.0.3
38-
module load cray-parallel-netcdf/1.12.3.3
33+
module load cray-hdf5-parallel/1.12.2.9 \
34+
cray-netcdf-hdf5parallel/4.9.0.9 \
35+
cray-parallel-netcdf/1.12.3.9
3936
{% endif %}
4037

4138
{% if e3sm_hdf5_netcdf %}
@@ -55,6 +52,7 @@ export HDF5_USE_FILE_LOCKING=FALSE
5552
## Not needed
5653
# export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
5754
export FI_CXI_RX_MATCH_MODE=software
55+
export FI_MR_CACHE_MONITOR=kdreg2
5856

5957
if [ -z "${NERSC_HOST:-}" ]; then
6058
# happens when building spack environment

‎mache/spack/pm-cpu_gnu_mpich.yaml‎

Lines changed: 25 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ spack:
22
specs:
33
- gcc
44
- cray-mpich
5-
{% if e3sm_lapack %}
65
- cray-libsci
7-
{% endif %}
86
{% if e3sm_hdf5_netcdf %}
97
- hdf5
108
- netcdf-c
@@ -16,12 +14,10 @@ spack:
1614
unify: when_possible
1715
packages:
1816
all:
19-
compiler: [gcc@12.2.0]
17+
compiler: [gcc@12.3]
2018
providers:
21-
mpi: [[email protected]]
22-
{% if e3sm_lapack %}
23-
lapack: [[email protected]]
24-
{% endif %}
19+
mpi: [[email protected]]
20+
lapack: [[email protected]]
2521
bzip2:
2622
externals:
2723
- spec: [email protected]
@@ -101,65 +97,55 @@ spack:
10197
buildable: false
10298
gcc:
10399
externals:
104-
- spec: gcc@12.2.0
100+
- spec: gcc@12.3
105101
modules:
106102
- PrgEnv-gnu/8.5.0
107-
- gcc/12.2.0
103+
- gcc-native/12.3
104+
- cray-libsci/23.12.5
108105
- craype-accel-host
109-
- craype/2.7.20
110-
- libfabric/1.15.2.0
106+
- craype/2.7.30
107+
- libfabric/1.20.1
111108
buildable: false
112109
cray-mpich:
113110
externals:
114-
- spec: [email protected]
115-
prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1
116-
modules:
117-
- libfabric/1.15.2.0
118-
- cray-mpich/8.1.25
119-
buildable: false
120-
libfabric:
121-
externals:
122-
- spec: [email protected]
123-
prefix: /opt/cray/libfabric/1.15.2.0
111+
- spec: [email protected]
124112
modules:
125-
- libfabric/1.15.2.0
113+
- libfabric/1.20.1
114+
- cray-mpich/8.1.28
126115
buildable: false
127-
{% if e3sm_lapack %}
128116
cray-libsci:
129117
externals:
130-
- spec: [email protected]
131-
prefix: /opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64
118+
- spec: [email protected]
132119
modules:
133-
- cray-libsci/23.02.1.1
120+
- cray-libsci/23.12.5
134121
buildable: false
135-
{% endif %}
136122
{% if e3sm_hdf5_netcdf %}
137123
hdf5:
138124
externals:
139-
- spec: [email protected].3~cxx+fortran+hl~java+mpi+shared
140-
prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/GNU/9.1
125+
- spec: [email protected].9~cxx+fortran+hl~java+mpi+shared
126+
prefix: /opt/cray/pe/hdf5-parallel/1.12.2.9/gnu/12.3
141127
buildable: false
142128
parallel-netcdf:
143129
externals:
144-
- spec: [email protected].3+cxx+fortran+pic+shared
145-
prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/GNU/9.1/
130+
- spec: [email protected].9+cxx+fortran+pic+shared
131+
prefix: /opt/cray/pe/parallel-netcdf/1.12.3.9/gnu/12.3
146132
buildable: false
147133
netcdf-c:
148134
externals:
149-
- spec: [email protected].3+mpi~parallel-netcdf
150-
prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
135+
- spec: [email protected].9+mpi~parallel-netcdf
136+
prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
151137
buildable: false
152138
netcdf-fortran:
153139
externals:
154140
- spec: [email protected]
155-
prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/GNU/9.1
141+
prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.9/gnu/12.3
156142
buildable: false
157143
{% endif %}
158144
config:
159145
install_missing_compilers: false
160146
compilers:
161147
- compiler:
162-
spec: gcc@12.2.0
148+
spec: gcc@12.3
163149
paths:
164150
cc: cc
165151
cxx: CC
@@ -170,10 +156,11 @@ spack:
170156
target: x86_64
171157
modules:
172158
- PrgEnv-gnu/8.5.0
173-
- gcc/12.2.0
159+
- gcc-native/12.3
160+
- cray-libsci/23.12.5
174161
- craype-accel-host
175-
- craype/2.7.20
176-
- libfabric/1.15.2.0
162+
- craype/2.7.30
163+
- libfabric/1.20.1
177164
environment:
178165
prepend_path:
179166
PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.33__gd067c3f.shasta/lib64/pkgconfig"

‎mache/spack/pm-cpu_intel_mpich.csh‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,36 @@
1-
module rm cray-hdf5-parallel &> /dev/null
2-
module rm cray-netcdf-hdf5parallel &> /dev/null
3-
module rm cray-parallel-netcdf &> /dev/null
4-
module rm PrgEnv-gnu &> /dev/null
5-
module rm PrgEnv-intel &> /dev/null
6-
module rm PrgEnv-nvidia &> /dev/null
7-
module rm PrgEnv-cray &> /dev/null
8-
module rm PrgEnv-aocc &> /dev/null
9-
module rm gcc-native &> /dev/null
10-
module rm intel &> /dev/null
11-
module rm intel-oneapi &> /dev/null
12-
module rm cudatoolkit &> /dev/null
13-
module rm climate-utils &> /dev/null
14-
module rm cray-libsci &> /dev/null
15-
module rm matlab &> /dev/null
16-
module rm craype-accel-nvidia80 &> /dev/null
17-
module rm craype-accel-host &> /dev/null
18-
module rm perftools-base &> /dev/null
19-
module rm perftools &> /dev/null
20-
module rm darshan &> /dev/null
1+
module rm cpe \
2+
cray-hdf5-parallel \
3+
cray-netcdf-hdf5parallel \
4+
cray-parallel-netcdf \
5+
PrgEnv-gnu \
6+
PrgEnv-intel \
7+
PrgEnv-nvidia \
8+
PrgEnv-cray \
9+
PrgEnv-aocc \
10+
gcc-native \
11+
intel \
12+
intel-oneapi \
13+
cudatoolkit \
14+
climate-utils \
15+
cray-libsci \
16+
matlab \
17+
craype-accel-nvidia80 \
18+
craype-accel-host \
19+
perftools-base \
20+
perftools \
21+
darshan \
22+
cray-mpich &> /dev/null
2123

22-
module load PrgEnv-intel/8.5.0
23-
module load intel/2023.2.0
24-
module load craype-accel-host
25-
module load craype/2.7.30
26-
module load libfabric/1.15.2.0
27-
module rm cray-mpich &> /dev/null
28-
module load cray-mpich/8.1.28
24+
module load PrgEnv-intel/8.5.0 \
25+
intel/2023.2.0 \
26+
craype-accel-host \
27+
craype/2.7.30 \
28+
libfabric/1.20.1 \
29+
cray-mpich/8.1.28
2930
{% if e3sm_hdf5_netcdf %}
30-
module rm cray-hdf5-parallel &> /dev/null
31-
module rm cray-netcdf-hdf5parallel &> /dev/null
32-
module rm cray-parallel-netcdf &> /dev/null
33-
module load cray-hdf5-parallel/1.12.2.9
34-
module load cray-netcdf-hdf5parallel/4.9.0.9
35-
module load cray-parallel-netcdf/1.12.3.9
31+
module load cray-hdf5-parallel/1.12.2.9 \
32+
cray-netcdf-hdf5parallel/4.9.0.9 \
33+
cray-parallel-netcdf/1.12.3.9
3634
{% endif %}
3735

3836
{% if e3sm_hdf5_netcdf %}
@@ -52,6 +50,7 @@ setenv HDF5_USE_FILE_LOCKING FALSE
5250
## Not needed
5351
# setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
5452
setenv FI_CXI_RX_MATCH_MODE software
53+
setenv FI_MR_CACHE_MONITOR kdreg2
5554
setenv MPICH_COLL_SYNC MPI_Bcast
5655
setenv GATOR_INITIAL_MB 4000MB
5756
setenv BLA_VENDOR Intel10_64_dyn

0 commit comments

Comments
 (0)