Doc: WarpX no-MPI Perlmutter Container

A container for WarpX on GPU on Perlmutter without
MPI support.

Author: ax3l

Tools/machines/perlmutter-nersc/Containerfile

@@ -6,11 +6,11 @@
 # WarpX Python bindings are installed in /opt/venv
 #
 # On Perlmutter, run WarpX like this:
-#   podman-hpc run --rm --gpu --cuda-mpi --nccl -v $PWD:/opt/pwd -it registry.nersc.gov/m558/superfacility/warpx-perlmutter:latest warpx.2d /opt/pwd/inputs_base_2d
+#   podman-hpc run --rm --gpu -v $PWD:/opt/pwd -it registry.nersc.gov/m558/superfacility/warpx-perlmutter-nompi:latest warpx.2d /opt/pwd/inputs_base_2d
 # Prefix this line with "srun ..." in job scripts, as usual:
 #   srun --cpu-bind=cores bash -c "
 #     export CUDA_VISIBLE_DEVICES=\$((3-SLURM_LOCALID));
-#     podman run ... ${INPUTS} ${GPU_AWARE_MPI}" \
+#     podman run ... ${INPUTS}" \
 #   > output.txt
 #
 
@@ -49,8 +49,6 @@ ENV LD_LIBRARY_PATH="/opt/warpx/lib:${LD_LIBRARY_PATH}"
 ENV CMAKE_PREFIX_PATH="/opt/warpx:${CMAKE_PREFIX_PATH}"
 
 # Install essential system dependencies (development)
-# Perlmutter MPICH: cray-mpich/8.1.30 based on MPICH 3.4a2
-# Ubuntu 22.04 ships MPICH 4.0, so we build from source
 RUN apt-get update           && \
     apt-get install             \
         -y                      \
@@ -78,35 +76,6 @@ RUN apt-get update           && \
         zlib1g-dev           && \
     rm -rf /var/lib/apt/lists/*
 
-# Install MPICH 3.4 support from source
-#   after https://docs.nersc.gov/development/containers/shifter/how-to-use/#using-mpi-in-shifter
-# Note:
-#   w/o GPUdirect here, because we will swap out the MPI binaries on Perlmutter using a Podman-HPC
-#   plugin (--mpi/--cuda-mpi). This only works if we do NOT build GPUdirect here, this we skip:
-#     --with-ch4-shmmods=posix,gpudirect
-# Perlmutter MPICH: cray-mpich/8.1.30 based on MPICH 3.4a2
-# TODO: install libfabric from source and depend on it (or expose where embedded libfabric gets installed)
-ARG mpich=3.4.3
-ARG mpich_prefix=mpich-$mpich
-
-RUN \
-    curl -Lo $mpich_prefix.tar.gz https://www.mpich.org/static/downloads/$mpich/$mpich_prefix.tar.gz && \
-    tar xzf $mpich_prefix.tar.gz                                            && \
-    cd $mpich_prefix                                                        && \
-    FFLAGS=-fallow-argument-mismatch FCFLAGS=-fallow-argument-mismatch         \
-      ./configure                                                              \
-        --disable-fortran                                                      \
-        --prefix=/opt/warpx                                                    \
-        --with-device=ch4:ofi                                               && \
-    make -j ${NJOBS}                                                        && \
-    make install                                                            && \
-    make clean                                                              && \
-    cd ..                                                                   && \
-    rm -rf ${mpich_prefix}*
-
-RUN /sbin/ldconfig
-# ENV MPICH_GPU_SUPPORT_ENABLED=1
-
 # Install c-blosc from source
 RUN git clone                               \
       -b v2.14.4                            \
@@ -128,7 +97,6 @@ RUN git clone                               \
     rm -rf /tmp/c-blosc2*
 
 # Install ADIOS2 from source
-# TODO: install libfabric from source and depend on it (or expose where MPICH-embedded libfabric gets installed)
 # TODO: -DADIOS2_USE_SST=ON
 RUN git clone                                   \
         -b v2.10.2                              \
@@ -143,6 +111,7 @@ RUN git clone                                   \
           -DADIOS2_USE_SST=OFF     \
           -DADIOS2_USE_SZ=OFF      \
           -DADIOS2_USE_MGARD=OFF   \
+          -DADIOS2_USE_MPI=OFF     \
           -DADIOS2_USE_PNG=ON      \
           -DADIOS2_USE_Python=OFF  \
           -DADIOS2_USE_ZeroMQ=OFF  \
@@ -241,8 +210,9 @@ RUN git clone                       \
         -B /tmp/warpx-build         \
         -DCMAKE_INSTALL_PREFIX=/opt/warpx \
         -DWarpX_COMPUTE=CUDA        \
-        -DWarpX_DIMS="2;RZ"         \
+        -DWarpX_DIMS="1;2;RZ;3"     \
         -DWarpX_FFT=ON              \
+        -DWarpX_MPI=OFF             \
         -DWarpX_PYTHON=ON           \
         -DWarpX_QED_TABLE_GEN=OFF   \
                                  && \
@@ -255,8 +225,10 @@ RUN git clone                       \
           -j${NJOBS}                \
                                  && \
     rm -rf /tmp/warpx*           && \
+    ln -s /opt/warpx/bin/warpx.1d* /opt/warpx/bin/warpx.1d  && \
     ln -s /opt/warpx/bin/warpx.2d* /opt/warpx/bin/warpx.2d  && \
-    ln -s /opt/warpx/bin/warpx.rz* /opt/warpx/bin/warpx.rz
+    ln -s /opt/warpx/bin/warpx.rz* /opt/warpx/bin/warpx.rz  && \
+    ln -s /opt/warpx/bin/warpx.3d* /opt/warpx/bin/warpx.3d
 
 # Reduce to runtime
 FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS warpx

Tools/machines/perlmutter-nersc/entrypoint.sh

@@ -9,7 +9,5 @@ export LD_LIBRARY_PATH=${WARPX_ROOT}/lib:${LD_LIBRARY_PATH}
 # Activate python venv
 source /opt/venv/bin/activate
 
-# export MPICH_GPU_SUPPORT_ENABLED=1
-
 # Execute the provided command
 exec "$@"