Perlmutter (NERSC): Update Modules & Jobscript (#3493)

* Perlmutter (NERSC): Update Modules

Update after major system upgrade.

* Perlmutter (NERSC): Update Jobscript

* Perlmutter (NERSC): Remove Early Access Warning

* Perlmutter (NERSC): Process Affinity

Co-authored-by: Kevin Gott <[email protected]>

Co-authored-by: Kevin Gott <[email protected]>

Author: ax3l

Docs/source/install/hpc/perlmutter.rst

@@ -3,11 +3,6 @@
 Perlmutter (NERSC)
 ==================
 
-.. warning::
-
-   Perlmutter is still in acceptance testing and environments change often.
-   Please reach visit this page often for updates and reach out to us if something needs an update.
-
 The `Perlmutter cluster <https://docs.nersc.gov/systems/perlmutter/>`_ is located at NERSC.
 
 

Tools/machines/perlmutter-nersc/perlmutter.sbatch

@@ -1,6 +1,6 @@
 #!/bin/bash -l
 
-# Copyright 2021 Axel Huebl, Kevin Gott
+# Copyright 2021-2022 Axel Huebl, Kevin Gott
 #
 # This file is part of WarpX.
 #
@@ -13,22 +13,28 @@
 #SBATCH -A <proj>
 #SBATCH -q regular
 #SBATCH -C gpu
-#SBATCH -c 32
+#SBATCH --exclusive
 #SBATCH --ntasks-per-gpu=1
 #SBATCH --gpus-per-node=4
 #SBATCH -o WarpX.o%j
 #SBATCH -e WarpX.e%j
 
 # GPU-aware MPI
 export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_OFI_NIC_POLICY=GPU
 
-# expose one GPU per MPI rank
-export CUDA_VISIBLE_DEVICES=$SLURM_LOCALID
+# threads for OpenMP and threaded compressors per MPI rank
+export SRUN_CPUS_PER_TASK=32
 
 EXE=./warpx
 #EXE=../WarpX/build/bin/warpx.3d.MPI.CUDA.DP.OPMD.QED
 #EXE=./main3d.gnu.TPROF.MPI.CUDA.ex
 INPUTS=inputs_small
 
-srun ${EXE} ${INPUTS} \
+# CUDA visible devices are ordered inverse to local task IDs
+srun /bin/bash -l -c "  \
+    export CUDA_VISIBLE_DEVICES=$((3-SLURM_LOCALID));
+    ${EXE} ${INPUTS} \
+      amrex.the_arena_is_managed=0 \
+      amrex.use_gpu_aware_mpi=1"   \
   > output.txt

Tools/machines/perlmutter-nersc/perlmutter_warpx.profile.example

@@ -1,19 +1,14 @@
 # please set your project account
-#export proj=<yourProject>  # LBNL/AMP: m3906_g
+#export proj="<yourProject>_g"  # change me
 
 # required dependencies
 module load cmake/3.22.0
-module load PrgEnv-gnu
-module load cudatoolkit
-
-# optional: just an additional text editor
-# module load nano  # TODO: request from support
 
 # optional: for QED support with detailed tables
 module load boost/1.78.0-gnu
 
 # optional: for openPMD and PSATD+RZ support
-module load cray-hdf5-parallel/1.12.1.1
+module load cray-hdf5-parallel/1.12.1.5
 export CMAKE_PREFIX_PATH=$HOME/sw/perlmutter/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=$HOME/sw/perlmutter/adios2-2.7.1:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=$HOME/sw/perlmutter/blaspp-master:$CMAKE_PREFIX_PATH
@@ -25,7 +20,7 @@ export LD_LIBRARY_PATH=$HOME/sw/perlmutter/blaspp-master/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$HOME/sw/perlmutter/lapackpp-master/lib64:$LD_LIBRARY_PATH
 
 # optional: for Python bindings or libEnsemble
-module load cray-python/3.9.7.1
+module load cray-python/3.9.12.1
 
 if [ -d "$HOME/sw/perlmutter/venvs/warpx" ]
 then