Skip to content

Commit d4ba9dd

Browse files
BenWibkingBenWibking
committed
trimmed build
1 parent 41397a0 commit d4ba9dd

4 files changed

Lines changed: 128 additions & 22 deletions

File tree

‎.github/workflows/dockerfiles/ROCm-DiskGalaxy/Dockerfile‎

Lines changed: 107 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,127 @@
1-
ARG BASE_IMAGE=ghcr.io/quokka-astro/quokka-linux-amd64-rocm:development
2-
FROM --platform=linux/amd64 ${BASE_IMAGE} AS build
1+
FROM --platform=linux/amd64 registry.opensuse.org/opensuse/leap:15.6 AS build
32

4-
ARG QUOKKA_REPO=https://github.com/quokka-astro/quokka.git
5-
ARG QUOKKA_REF=development
3+
ARG ROCM_ZYPP_REPO_VERSION=6.4.4
4+
ARG ROCM_ZYPP_META_PKG=rocm6.4.4
5+
ARG MPICH_VERSION=4.1.2
6+
ARG MPICH_URL=https://www.mpich.org/static/downloads/4.1.2/mpich-4.1.2.tar.gz
7+
ARG AMREX_SPACEDIM=3
8+
ARG AMREX_MPI=ON
9+
ARG AMREX_AMD_ARCH=gfx90a
610

11+
RUN set -euo pipefail && \
12+
echo "rpm.install.excludedocs = yes" >> /etc/zypp/zypp.conf && \
13+
zypper --non-interactive refresh && \
14+
zypper --non-interactive update && \
15+
zypper clean --all
16+
17+
COPY . /tmp/quokka-src
718
WORKDIR /tmp/quokka-src
8-
COPY . .
919

20+
COPY .github/workflows/dockerfiles/ROCm-DiskGalaxy/rocm_so.conf /tmp/rocm_so.conf
1021
RUN set -euo pipefail && \
11-
if [[ ! -f CMakeLists.txt ]]; then \
12-
echo "No CMakeLists.txt in build context; cloning ${QUOKKA_REPO}@${QUOKKA_REF}..." && \
13-
find . -mindepth 1 -maxdepth 1 -exec rm -rf {} + && \
14-
git clone --depth 1 --recurse-submodules --shallow-submodules --branch "${QUOKKA_REF}" "${QUOKKA_REPO}" . ; \
22+
if ! zypper repos -E | grep -q devel_languages_perl; then \
23+
zypper --non-interactive addrepo https://download.opensuse.org/repositories/devel:/languages:/perl/15.6/devel:languages:perl.repo; \
1524
fi && \
25+
printf "[ROCm-%s]\nname=ROCm%s\nbaseurl=https://repo.radeon.com/rocm/zyp/%s/main\nenabled=1\ngpgcheck=1\ngpgkey=https://repo.radeon.com/rocm/rocm.gpg.key\n" \
26+
"${ROCM_ZYPP_REPO_VERSION}" "${ROCM_ZYPP_REPO_VERSION}" "${ROCM_ZYPP_REPO_VERSION}" > /etc/zypp/repos.d/rocm.repo && \
27+
zypper --non-interactive --gpg-auto-import-keys refresh && \
28+
zypper --non-interactive install --no-recommends \
29+
${ROCM_ZYPP_META_PKG} \
30+
cmake \
31+
git \
32+
gcc13 gcc13-c++ libstdc++6-devel-gcc13 \
33+
libfabric-devel \
34+
make tar gzip wget \
35+
patch \
36+
python3-devel && \
37+
zypper clean --all && \
38+
cat /tmp/rocm_so.conf >> /etc/ld.so.conf.d/rocm.conf && \
39+
rm /tmp/rocm_so.conf && \
40+
ldconfig
41+
42+
ENV PATH="/opt/rocm/bin:${PATH}" \
43+
LD_LIBRARY_PATH="/opt/rocm/lib:${LD_LIBRARY_PATH}" \
44+
ROCM_PATH="/opt/rocm" \
45+
AMREX_AMD_ARCH="${AMREX_AMD_ARCH}" \
46+
CC="amdclang" \
47+
CXX="amdclang++"
48+
49+
COPY .github/workflows/dockerfiles/ROCm-DiskGalaxy/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch /tmp/mpich-rocm.patch
50+
RUN set -euo pipefail && \
51+
mkdir -p /tmp/build-mpich && \
52+
cd /tmp/build-mpich && \
53+
wget -q "${MPICH_URL}" -O "mpich-${MPICH_VERSION}.tar.gz" && \
54+
tar xzf "mpich-${MPICH_VERSION}.tar.gz" && \
55+
cd "mpich-${MPICH_VERSION}" && \
56+
patch -p1 < /tmp/mpich-rocm.patch && \
57+
./configure --with-device=ch4:ofi --disable-fortran --disable-doc --prefix="/opt/mpich/${MPICH_VERSION}" && \
58+
make -j"$(nproc)" && \
59+
make install && \
60+
cd / && \
61+
rm -rf /tmp/build-mpich /tmp/mpich-rocm.patch
62+
63+
ENV MPICH_VERSION="${MPICH_VERSION}" \
64+
MPICH_DIR="/opt/mpich/${MPICH_VERSION}" \
65+
PATH="/opt/mpich/${MPICH_VERSION}/bin:${PATH}" \
66+
LD_LIBRARY_PATH="/opt/mpich/${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}"
67+
68+
RUN set -euo pipefail && \
1669
cmake -S . -B build \
1770
-DCMAKE_BUILD_TYPE=Release \
18-
-DAMReX_SPACEDIM=3 \
71+
-DAMReX_SPACEDIM="${AMREX_SPACEDIM}" \
1972
-DAMReX_GPU_BACKEND=HIP \
20-
-DAMReX_MPI=ON \
21-
-DAMReX_GPU_ARCH="${AMREX_AMD_ARCH:-gfx90a}" && \
73+
-DAMReX_MPI="${AMREX_MPI}" \
74+
-DAMReX_GPU_ARCH="${AMREX_AMD_ARCH}" \
75+
-DCMAKE_C_COMPILER="${CC}" \
76+
-DCMAKE_CXX_COMPILER="${CXX}" && \
2277
cmake --build build -j"$(nproc)" --target DiskGalaxy && \
2378
test -x build/src/problems/DiskGalaxy/DiskGalaxy
2479

25-
FROM --platform=linux/amd64 ${BASE_IMAGE}
80+
FROM --platform=linux/amd64 registry.opensuse.org/opensuse/leap:15.6
81+
82+
ARG MPICH_VERSION=4.1.2
2683

2784
USER root
85+
RUN set -euo pipefail && \
86+
echo "rpm.install.excludedocs = yes" >> /etc/zypp/zypp.conf && \
87+
zypper --non-interactive refresh && \
88+
zypper --non-interactive update && \
89+
zypper --non-interactive install --no-recommends libfabric && \
90+
zypper clean --all
91+
92+
COPY --from=build /opt/rocm /opt/rocm
93+
COPY --from=build /opt/mpich /opt/mpich
94+
COPY .github/workflows/dockerfiles/ROCm-DiskGalaxy/rocm_so.conf /tmp/rocm_so.conf
95+
RUN set -euo pipefail && \
96+
cat /tmp/rocm_so.conf >> /etc/ld.so.conf.d/rocm.conf && \
97+
rm /tmp/rocm_so.conf && \
98+
rm -rf \
99+
/opt/rocm/bin \
100+
/opt/rocm/include \
101+
/opt/rocm/share \
102+
/opt/rocm/lib/cmake \
103+
/opt/rocm/llvm/bin \
104+
/opt/rocm/llvm/include \
105+
/opt/rocm/llvm/share \
106+
/opt/rocm/llvm/lib/cmake && \
107+
find /opt/rocm -type f -name "*.a" -delete && \
108+
find /opt/rocm -type f -name "*.o" -delete && \
109+
find /opt/mpich -type f -name "*.a" -delete && \
110+
find /opt/mpich -type f -name "*.o" -delete && \
111+
ldconfig
112+
28113
RUN mkdir -p /opt/quokka/bin /opt/quokka/share/quokka/inputs /opt/quokka/share/quokka/extern/agora_data
29114
COPY --from=build /tmp/quokka-src/build/src/problems/DiskGalaxy/DiskGalaxy /opt/quokka/bin/DiskGalaxy
30115
COPY --from=build /tmp/quokka-src/inputs/DiskGalaxy.in /opt/quokka/share/quokka/inputs/DiskGalaxy.in
31116
COPY --from=build /tmp/quokka-src/extern/agora_data/ /opt/quokka/share/quokka/extern/agora_data/
32-
RUN chmod 0755 /opt/quokka/bin/DiskGalaxy && chown -R leap /opt/quokka
33-
ENV PATH="/opt/quokka/bin:${PATH}"
117+
RUN chmod 0755 /opt/quokka/bin/DiskGalaxy
118+
119+
RUN useradd -ms /bin/bash leap && \
120+
usermod -a -G render,video leap && \
121+
chown -R leap /opt/quokka
122+
123+
ENV MPICH_VERSION="${MPICH_VERSION}" \
124+
PATH="/opt/quokka/bin:/opt/mpich/${MPICH_VERSION}/bin:${PATH}" \
125+
LD_LIBRARY_PATH="/opt/mpich/${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}"
34126
USER leap
35127
WORKDIR /home/leap

‎.github/workflows/dockerfiles/ROCm-DiskGalaxy/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
diff --git a/src/mpl/src/gpu/mpl_gpu_hip.c b/src/mpl/src/gpu/mpl_gpu_hip.c
2+
index f6a2df3..b2c5954 100644
3+
--- a/src/mpl/src/gpu/mpl_gpu_hip.c
4+
+++ b/src/mpl/src/gpu/mpl_gpu_hip.c
5+
@@ -90,7 +90,13 @@ int MPL_gpu_query_pointer_attr(const void *ptr, MPL_pointer_attr_t * attr)
6+
hipError_t ret;
7+
ret = hipPointerGetAttributes(&attr->device_attr, ptr);
8+
if (ret == hipSuccess) {
9+
- switch (attr->device_attr.memoryType) {
10+
+#if defined(HIP_VERSION_MAJOR) && (HIP_VERSION_MAJOR >= 6)
11+
+ /* ROCm 6+ uses hipPointerAttribute_t::type */
12+
+ enum hipMemoryType memory_type = attr->device_attr.type;
13+
+#else
14+
+ enum hipMemoryType memory_type = attr->device_attr.memoryType;
15+
+#endif
16+
+ switch (memory_type) {
17+
case hipMemoryTypeHost:
18+
attr->type = MPL_GPU_POINTER_REGISTERED_HOST;
19+
attr->device = attr->device_attr.device;

‎.github/workflows/dockerfiles/ROCm-DiskGalaxy/rocm_so.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/opt/rocm/lib
2+
/opt/rocm/lib64

‎.github/workflows/rocm-diskgalaxy-container.yml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ concurrency:
1919
env:
2020
REGISTRY: ghcr.io
2121
IMAGE_NAME: ${{ github.repository }}
22-
BASE_IMAGE: ghcr.io/quokka-astro/quokka-linux-amd64-rocm:development
2322
IMAGE_TAG_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
2423
IS_INTERNAL_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
2524

@@ -51,8 +50,6 @@ jobs:
5150
with:
5251
context: .
5352
file: .github/workflows/dockerfiles/ROCm-DiskGalaxy/Dockerfile
54-
build-args: |
55-
BASE_IMAGE=${{ env.BASE_IMAGE }}
5653
push: true
5754
tags: |
5855
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:rocm-diskgalaxy
@@ -66,8 +63,6 @@ jobs:
6663
with:
6764
context: .
6865
file: .github/workflows/dockerfiles/ROCm-DiskGalaxy/Dockerfile
69-
build-args: |
70-
BASE_IMAGE=${{ env.BASE_IMAGE }}
7166
push: true
7267
tags: |
7368
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:rocm-diskgalaxy-pr-${{ github.event.pull_request.number }}
@@ -81,8 +76,6 @@ jobs:
8176
with:
8277
context: .
8378
file: .github/workflows/dockerfiles/ROCm-DiskGalaxy/Dockerfile
84-
build-args: |
85-
BASE_IMAGE=${{ env.BASE_IMAGE }}
8679
push: false
8780
tags: rocm-diskgalaxy-pr-${{ github.event.pull_request.number }}:${{ env.IMAGE_TAG_SHA }}
8881
cache-from: type=gha

0 commit comments

Comments
 (0)