Skip to content

Commit fc341b7

Browse files
ax3lax3l
committed
Doc: WarpX no-MPI Perlmutter Container
A container for WarpX on GPU on Perlmutter without MPI support. Add HDF5.
1 parent a51b0f5 commit fc341b7

File tree

4 files changed

+282
-0
lines changed

4 files changed

+282
-0
lines changed

‎Tools/machines/perlmutter-nersc/Containerfile‎ renamed to ‎Tools/machines/perlmutter-nersc/container/gpu/mpi/Containerfile‎

File renamed without changes.

‎Tools/machines/perlmutter-nersc/entrypoint.sh‎ renamed to ‎Tools/machines/perlmutter-nersc/container/gpu/mpi/entrypoint.sh‎

File renamed without changes.

‎Tools/machines/perlmutter-nersc/container/gpu/nompi/Containerfile‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
# Build this container from its current directory:
2+
# podman build --build-arg NJOBS=6 -t warpx-perlmutter-nompi .
3+
# Adjust NJOBS to the number of processes to use for parallel compilation.
4+
#
5+
# WarpX executables are installed in /opt/warpx/bin
6+
# WarpX Python bindings are installed in /opt/venv
7+
#
8+
# On Perlmutter, run WarpX like this:
9+
# podman-hpc run --rm --gpu -v $PWD:/opt/pwd -it registry.nersc.gov/m558/superfacility/warpx-perlmutter-nompi:latest warpx.2d /opt/pwd/inputs_base_2d
10+
# Prefix this line with "srun ..." in job scripts, as usual:
11+
# srun --cpu-bind=cores bash -c "
12+
# export CUDA_VISIBLE_DEVICES=\$((3-SLURM_LOCALID));
13+
# podman run ... ${INPUTS}" \
14+
# > output.txt
15+
#
16+
17+
# Base System and Essential Tools Installation
18+
FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS warpx-dev
19+
20+
# WarpX version
21+
ARG warpx_version=25.11
22+
23+
# Build parallelism
24+
ARG NJOBS
25+
ENV NJOBS=$NJOBS
26+
27+
# Perlmutter GPU: A100
28+
# Perlmutter CPU: Zen v3
29+
ENV AMREX_CUDA_ARCH=8.0 \
30+
CUDAARCHS=80 \
31+
CXXFLAGS="-march=znver3" \
32+
CFLAGS="-march=znver3"
33+
34+
# Ubuntu apt global options
35+
ENV DEBIAN_FRONTEND=noninteractive
36+
37+
# CMake global options
38+
ENV CMAKE_GENERATOR="Ninja"
39+
40+
# Autotools global options
41+
ENV FORCE_UNSAFE_CONFIGURE=1
42+
43+
# Pip global options
44+
ENV PYTHONDONTWRITEBYTECODE=1
45+
46+
# Custom install location
47+
ENV PATH="/opt/warpx/bin:${PATH}"
48+
ENV LD_LIBRARY_PATH="/opt/warpx/lib:${LD_LIBRARY_PATH}"
49+
ENV CMAKE_PREFIX_PATH="/opt/warpx:${CMAKE_PREFIX_PATH}"
50+
51+
# Install essential system dependencies (development)
52+
RUN apt-get update && \
53+
apt-get install \
54+
-y \
55+
--no-install-recommends \
56+
autoconf \
57+
build-essential \
58+
ca-certificates \
59+
cmake \
60+
coreutils \
61+
curl \
62+
g++ \
63+
git \
64+
pkg-config \
65+
python3 \
66+
python3-pip \
67+
python3-dev \
68+
python3-venv \
69+
unzip \
70+
libblas-dev \
71+
libbz2-dev \
72+
libhdf5-dev \
73+
libpng-dev \
74+
liblapack-dev \
75+
libzstd-dev \
76+
ninja-build \
77+
zlib1g-dev && \
78+
rm -rf /var/lib/apt/lists/*
79+
80+
# Install c-blosc from source
81+
RUN git clone \
82+
-b v2.14.4 \
83+
https://github.com/Blosc/c-blosc2.git \
84+
/tmp/c-blosc2 && \
85+
cmake \
86+
-S /tmp/c-blosc2 \
87+
-B /tmp/c-blosc2-build \
88+
-DCMAKE_INSTALL_PREFIX=/opt/warpx \
89+
-DBUILD_TESTS=OFF \
90+
-DBUILD_BENCHMARKS=OFF \
91+
-DBUILD_EXAMPLES=OFF \
92+
-DBUILD_FUZZERS=OFF \
93+
-DDEACTIVATE_AVX2=OFF \
94+
&& \
95+
cmake --build /tmp/c-blosc2-build \
96+
--target install \
97+
-j${NJOBS} && \
98+
rm -rf /tmp/c-blosc2*
99+
100+
# Install ADIOS2 from source
101+
# TODO: -DADIOS2_USE_SST=ON
102+
RUN git clone \
103+
-b v2.10.2 \
104+
https://github.com/ornladios/ADIOS2.git \
105+
/tmp/adios2 \
106+
&& \
107+
cmake -S /tmp/adios2 \
108+
-B /tmp/adios2-build \
109+
-DADIOS2_USE_Blosc2=ON \
110+
-DADIOS2_USE_BZip2=ON \
111+
-DADIOS2_USE_Fortran=OFF \
112+
-DADIOS2_USE_SST=OFF \
113+
-DADIOS2_USE_SZ=OFF \
114+
-DADIOS2_USE_MGARD=OFF \
115+
-DADIOS2_USE_MPI=OFF \
116+
-DADIOS2_USE_PNG=ON \
117+
-DADIOS2_USE_Python=OFF \
118+
-DADIOS2_USE_ZeroMQ=OFF \
119+
-DADIOS2_USE_ZFP=OFF \
120+
-DCMAKE_INSTALL_PREFIX=/opt/warpx \
121+
&& \
122+
cmake --build /tmp/adios2-build \
123+
--target install -j${NJOBS} && \
124+
rm -rf /tmp/adios2*
125+
126+
# Install BLAS++ and LAPACK++
127+
RUN git clone \
128+
-b v2024.05.31 \
129+
https://github.com/icl-utk-edu/blaspp.git \
130+
/tmp/blaspp && \
131+
cd /tmp/blaspp && \
132+
cmake -S /tmp/blaspp \
133+
-B /tmp/blaspp-build \
134+
-Duse_openmp=OFF \
135+
-Dgpu_backend=cuda \
136+
-DCMAKE_CXX_STANDARD=17 \
137+
-DCMAKE_INSTALL_PREFIX=/opt/warpx \
138+
-DBLAS_LIBRARIES=/usr/lib/x86_64-linux-gnu/libblas.so \
139+
-DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/liblapack.so \
140+
&& \
141+
cmake --build /tmp/blaspp-build \
142+
--target install \
143+
-j${NJOBS} && \
144+
rm -rf /tmp/blaspp*
145+
146+
RUN git clone \
147+
-b v2024.05.31 \
148+
https://github.com/icl-utk-edu/lapackpp.git \
149+
/tmp/lapackpp \
150+
&& \
151+
cmake -S /tmp/lapackpp \
152+
-B /tmp/lapackpp-build \
153+
-DCMAKE_CXX_STANDARD=17 \
154+
-Dbuild_tests=OFF \
155+
-DCMAKE_INSTALL_PREFIX=/opt/warpx \
156+
-DLAPACK_LIBRARIES=/usr/lib/x86_64-linux-gnu/liblapack.so \
157+
&& \
158+
cmake --build /tmp/lapackpp-build \
159+
--target install \
160+
-j${NJOBS} && \
161+
rm -rf /tmp/lapackpp*
162+
163+
# Create and activate Python virtual environment
164+
ENV PATH="/opt/venv/bin:${PATH}"
165+
RUN python3 -m venv /opt/venv && \
166+
pip install \
167+
--no-cache-dir \
168+
--upgrade \
169+
pip \
170+
uv \
171+
&& \
172+
uv pip install \
173+
--no-cache-dir \
174+
--upgrade \
175+
build \
176+
cmake \
177+
cupy-cuda12x \
178+
cython \
179+
h5py \
180+
matplotlib \
181+
numpy \
182+
openpmd-api \
183+
openpmd-viewer \
184+
pandas \
185+
scipy \
186+
packaging \
187+
setuptools[core] \
188+
wheel
189+
190+
# TODO: if we build cupy ourselves, we can likely shrink
191+
# the container size
192+
193+
# Optional: PyTorch (can be added on top as well in a warpx-ml container)
194+
# \
195+
# && \
196+
# uv pip install \
197+
# --no-cache-dir \
198+
# --upgrade \
199+
# --index-url https://download.pytorch.org/whl/cu126 \
200+
# torch
201+
# optimas[all]
202+
203+
# Build WarpX
204+
# TODO: -DWarpX_DIMS="1;2;RCYLINDER;RSPHERE;RZ;3"
205+
# TODO: -DWarpX_QED_TABLE_GEN=ON (needs boost math)
206+
RUN git clone \
207+
-b ${warpx_version} \
208+
https://github.com/BLAST-WarpX/warpx.git \
209+
/tmp/warpx-src && \
210+
cmake \
211+
-S /tmp/warpx-src \
212+
-B /tmp/warpx-build \
213+
-DCMAKE_INSTALL_PREFIX=/opt/warpx \
214+
-DWarpX_COMPUTE=CUDA \
215+
-DWarpX_DIMS="1;2;RZ;3" \
216+
-DWarpX_FFT=ON \
217+
-DWarpX_MPI=OFF \
218+
-DWarpX_PYTHON=ON \
219+
-DWarpX_QED_TABLE_GEN=OFF \
220+
&& \
221+
cmake --build /tmp/warpx-build \
222+
--target install \
223+
-j${NJOBS} \
224+
&& \
225+
cmake --build /tmp/warpx-build \
226+
--target pip_install \
227+
-j${NJOBS} \
228+
&& \
229+
rm -rf /tmp/warpx* && \
230+
ln -s /opt/warpx/bin/warpx.1d* /opt/warpx/bin/warpx.1d && \
231+
ln -s /opt/warpx/bin/warpx.2d* /opt/warpx/bin/warpx.2d && \
232+
ln -s /opt/warpx/bin/warpx.rz* /opt/warpx/bin/warpx.rz && \
233+
ln -s /opt/warpx/bin/warpx.3d* /opt/warpx/bin/warpx.3d
234+
235+
# Reduce to runtime
236+
FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS warpx
237+
238+
# Install essential system dependencies (runtime)
239+
RUN apt-get update && \
240+
apt-get install \
241+
-y \
242+
--no-install-recommends \
243+
ca-certificates \
244+
coreutils \
245+
python3 \
246+
python3-pip \
247+
python3-venv \
248+
libblas3 \
249+
libbz2-1.0 \
250+
libhdf5-103-1 \
251+
libpng16-16 \
252+
liblapack3 \
253+
libzstd1 \
254+
zlib1g && \
255+
rm -rf /var/lib/apt/lists/*
256+
257+
# Copy installed software from previous stage
258+
# TODO: Check BLAS still works
259+
# TODO: Check this does not copy CUDA devel packages
260+
COPY --from=warpx-dev /opt/warpx /opt/warpx
261+
COPY --from=warpx-dev /opt/venv /opt/venv
262+
263+
# Set up entrypoint
264+
COPY entrypoint.sh /opt/entrypoint.sh
265+
RUN chmod +x /opt/entrypoint.sh
266+
ENTRYPOINT ["/opt/entrypoint.sh"]
267+
268+
# Default command
269+
CMD ["/bin/bash", "-l"]

‎Tools/machines/perlmutter-nersc/container/gpu/nompi/entrypoint.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash --login
2+
3+
# Hint custom install location
4+
export WARPX_ROOT=/opt/warpx
5+
export CMAKE_PREFIX_PATH=${WARPX_ROOT}:${CMAKE_PREFIX_PATH}
6+
export PATH=${WARPX_ROOT}/bin:${PATH}
7+
export LD_LIBRARY_PATH=${WARPX_ROOT}/lib:${LD_LIBRARY_PATH}
8+
9+
# Activate python venv
10+
source /opt/venv/bin/activate
11+
12+
# Execute the provided command
13+
exec "$@"

0 commit comments

Comments
 (0)