Merge branch 'development' into HaloIC

Author: BenWibking

.ci/azure-pipelines-amdgpu.yml

@@ -48,6 +48,7 @@ pr:
     - extern/yaml-cpp/**
     - extern/openPMD-api/**
     - extern/AMReX-Hydro/**
+    - extern/turbulence/**
     exclude:
     - docs/**
     - paper/**

.ci/azure-pipelines.yml

@@ -47,6 +47,7 @@ pr:
     - extern/yaml-cpp/**
     - extern/openPMD-api/**
     - extern/AMReX-Hydro/**
+    - extern/turbulence/**
     exclude:
     - docs/**
     - paper/**

.devcontainer/rocm-container/Dockerfile

@@ -20,30 +20,33 @@ RUN if ! zypper repos -E | grep devel_languages_perl; then \
     zypper clean --all && \
     cat rocm_so.conf >> /etc/ld.so.conf.d/rocm.conf && \
     rm rocm_so.conf && \
-    ldconfig
+	    ldconfig
 
 ENV PATH="/opt/rocm/bin:${PATH}" \
     LD_LIBRARY_PATH="/opt/rocm/lib:${LD_LIBRARY_PATH}"
 
-RUN zypper --non-interactive install wget tar gzip gcc gcc-fortran file gcc-c++ make && zypper clean --all && \
+COPY mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch /tmp/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch
+
+RUN zypper --non-interactive install wget tar gzip gcc gcc-fortran file gcc-c++ make patch && zypper clean --all && \
     if [ ! -z "$(gcc --version | grep -Eo -m 1 [0-9][0-9]+\.[0-9]+\.[0-9]+ | grep -m 1 [0-9])" ]; then \
     export FFLAGS=-fallow-argument-mismatch; \
     fi && \
-    wget --progress=bar:force https://www.mpich.org/static/downloads/3.4.3/mpich-3.4.3.tar.gz && \
-    tar xzf mpich-3.4.3.tar.gz && \
-    cd mpich-3.4.3 && \
-    ./configure --with-device=ch4:ofi --prefix=/opt/mpich/3.4.3 --with-hip=/opt/rocm/hip && \
+    wget --progress=bar:force https://www.mpich.org/static/downloads/4.1.2/mpich-4.1.2.tar.gz && \
+    tar xzf mpich-4.1.2.tar.gz && \
+    cd mpich-4.1.2 && \
+    patch -p1 < /tmp/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch && \
+    ./configure --with-device=ch4:ofi --prefix=/opt/mpich/4.1.2 && \
     make install -j7 && \
     cd / && \
-    rm -rf mpich-3.4.3/ mpich-3.4.3.tar.gz
-
-ENV MPICH_VERSION="3.4.3" \
-    MPICH_URL="https://www.mpich.org/static/downloads/3.4.3/mpich-3.4.3.tar.gz" \
-    MPICH_DIR="/opt/mpich/3.4.3" \
-    PATH="/opt/mpich/3.4.3/bin:$PATH" \
-    LIBRARY_PATH="/opt/mpich/3.4.3/lib:$LIBRARY_PATH" \
-    LD_LIBRARY_PATH="/opt/mpich/3.4.3/lib:$LD_LIBRARY_PATH" \
-    MANPATH="/opt/mpich/3.4.3/share/man:$MANPATH"
+    rm -rf mpich-4.1.2/ mpich-4.1.2.tar.gz
+
+ENV MPICH_VERSION="4.1.2" \
+    MPICH_URL="https://www.mpich.org/static/downloads/4.1.2/mpich-4.1.2.tar.gz" \
+    MPICH_DIR="/opt/mpich/4.1.2" \
+    PATH="/opt/mpich/4.1.2/bin:$PATH" \
+    LIBRARY_PATH="/opt/mpich/4.1.2/lib:$LIBRARY_PATH" \
+    LD_LIBRARY_PATH="/opt/mpich/4.1.2/lib:$LD_LIBRARY_PATH" \
+    MANPATH="/opt/mpich/4.1.2/share/man:$MANPATH"
 
 # Install CMake
 RUN zypper --non-interactive install cmake && zypper clean --all

.devcontainer/rocm-container/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch

@@ -0,0 +1,19 @@
+diff --git a/src/mpl/src/gpu/mpl_gpu_hip.c b/src/mpl/src/gpu/mpl_gpu_hip.c
+index f6a2df3..b2c5954 100644
+--- a/src/mpl/src/gpu/mpl_gpu_hip.c
++++ b/src/mpl/src/gpu/mpl_gpu_hip.c
+@@ -90,7 +90,13 @@ int MPL_gpu_query_pointer_attr(const void *ptr, MPL_pointer_attr_t * attr)
+     hipError_t ret;
+     ret = hipPointerGetAttributes(&attr->device_attr, ptr);
+     if (ret == hipSuccess) {
+-        switch (attr->device_attr.memoryType) {
++#if defined(HIP_VERSION_MAJOR) && (HIP_VERSION_MAJOR >= 6)
++        /* ROCm 6+ uses hipPointerAttribute_t::type */
++        enum hipMemoryType memory_type = attr->device_attr.type;
++#else
++        enum hipMemoryType memory_type = attr->device_attr.memoryType;
++#endif
++        switch (memory_type) {
+             case hipMemoryTypeHost:
+                 attr->type = MPL_GPU_POINTER_REGISTERED_HOST;
+                 attr->device = attr->device_attr.device;

.dockerignore

@@ -0,0 +1,19 @@
+.git
+.gitmodules
+.github/workflows/**/node_modules
+.vscode
+
+# Local build artifacts
+build/
+cmake-build-*/
+
+# Large CI/regression assets (not needed for container builds)
+tests/
+
+# Developer-facing content (not needed for container builds)
+docs/
+paper/
+scripts/
+
+# OS/editor noise
+**/.DS_Store

.github/workflows/check_changes.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
     inputs:
       workflow_file:
-        description: 'Specific workflow file to check for changes (optional)'
+        description: "Specific workflow file to check for changes (optional)"
         required: false
         type: string
     outputs:
@@ -35,7 +35,7 @@ jobs:
       has_workflow_changes: ${{ steps.set-output.outputs.has_workflow_changes }}
       has_specific_workflow_changes: ${{ steps.set-output.outputs.has_specific_workflow_changes }}
     steps:
-      - uses: actions/checkout@v6.0.1
+      - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
         id: changes
         with:
@@ -52,6 +52,7 @@ jobs:
               - 'extern/grackle_data_files/**'
               - 'extern/openPMD-api/**'
               - 'extern/yaml-cpp/**'
+              - 'extern/turbulence/**'
               - 'inputs/**'
               - 'CMakeLists.txt'
               - '**/*.cpp'

.github/workflows/codeql.yml

@@ -21,7 +21,7 @@ jobs:
     outputs:
       code-changed: ${{ steps.changes.outputs.code }}
     steps:
-      - uses: actions/checkout@v6.0.1
+      - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
         id: changes
         with:
@@ -34,6 +34,7 @@ jobs:
               - 'extern/grackle_data_files/**'
               - 'extern/openPMD-api/**'
               - 'extern/yaml-cpp/**'
+              - 'extern/turbulence/**'
               - 'inputs/**'
               - 'CMakeLists.txt'
               - '**/*.cpp'

.github/workflows/dockerfiles/ROCm-DiskGalaxy/Dockerfile

@@ -0,0 +1,133 @@
+FROM --platform=linux/amd64 opensuse/leap:15.6 AS build
+
+ARG ROCM_ZYPP_REPO_VERSION=6.4.2
+ARG ROCM_ZYPP_PACKAGES=
+ARG MPICH_VERSION=4.1.2
+ARG MPICH_URL=https://www.mpich.org/static/downloads/4.1.2/mpich-4.1.2.tar.gz
+ARG AMREX_SPACEDIM=3
+ARG AMREX_MPI=ON
+ARG AMREX_AMD_ARCH=gfx90a
+ARG DOCKERFILES_DIR=.github/workflows/dockerfiles/ROCm-DiskGalaxy
+
+# Combine system updates, repo setup, and package installation
+RUN set -euo pipefail && \
+    echo "rpm.install.excludedocs = yes" >> /etc/zypp/zypp.conf && \
+    zypper --non-interactive refresh && \
+    zypper --non-interactive update && \
+    if ! zypper repos -E | grep -q devel_languages_perl; then \
+      zypper --non-interactive addrepo https://download.opensuse.org/repositories/devel:/languages:/perl/15.6/devel:languages:perl.repo; \
+    fi && \
+    printf "[ROCm-%s]\nname=ROCm%s\nbaseurl=https://repo.radeon.com/rocm/zyp/%s/main\nenabled=1\ngpgcheck=1\ngpgkey=https://repo.radeon.com/rocm/rocm.gpg.key\n" \
+      "${ROCM_ZYPP_REPO_VERSION}" "${ROCM_ZYPP_REPO_VERSION}" "${ROCM_ZYPP_REPO_VERSION}" > /etc/zypp/repos.d/rocm.repo && \
+    zypper --non-interactive --gpg-auto-import-keys refresh && \
+    if [ -n "${ROCM_ZYPP_PACKAGES}" ]; then \
+      zypper --non-interactive install --no-recommends ${ROCM_ZYPP_PACKAGES}; \
+    else \
+      zypper --non-interactive install --no-recommends \
+        rocm-llvm hipcc hip-runtime-amd hip-devel hsa-rocr comgr \
+        rocm-device-libs rocrand-devel hiprand-devel rocprim-devel rocsparse-devel; \
+    fi && \
+    zypper --non-interactive install --no-recommends \
+      cmake git gcc13 gcc13-c++ libstdc++6-devel-gcc13 \
+      make tar gzip wget patch hdf5-devel python3-devel && \
+    ln -sf /usr/lib64/libefa.so.1 /usr/lib64/libefa.so && \
+    zypper clean --all && \
+    printf "%s\n" "/opt/rocm/lib" "/opt/rocm/lib64" >> /etc/ld.so.conf.d/rocm.conf && \
+    ldconfig
+
+ENV PATH="/opt/rocm/bin:${PATH}" \
+    LD_LIBRARY_PATH="/opt/rocm/lib:${LD_LIBRARY_PATH}" \
+    ROCM_PATH="/opt/rocm" \
+    AMREX_AMD_ARCH="${AMREX_AMD_ARCH}" \
+    CC="amdclang" \
+    CXX="amdclang++"
+
+COPY . /tmp/quokka-src
+WORKDIR /tmp/quokka-src
+
+# Combine MPICH download, patch, and build
+RUN set -euo pipefail && \
+    test -f CMakeLists.txt || { \
+      echo "ERROR: Quokka source not found in the Docker build context." >&2; \
+      echo "Build from the repo root, e.g.: docker build -f .github/workflows/dockerfiles/ROCm-DiskGalaxy/Dockerfile ." >&2; \
+      exit 1; \
+    } && \
+    if [ -f "/tmp/quokka-src/${DOCKERFILES_DIR}/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch" ]; then \
+        PATCH_PATH="/tmp/quokka-src/${DOCKERFILES_DIR}/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch"; \
+    elif [ -f "/tmp/quokka-src/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch" ]; then \
+        PATCH_PATH="/tmp/quokka-src/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch"; \
+    else \
+        echo "ERROR: MPICH patch not found." >&2; exit 1; \
+    fi && \
+    mkdir -p /tmp/build-mpich && \
+    cd /tmp/build-mpich && \
+    wget -q "${MPICH_URL}" -O "mpich-${MPICH_VERSION}.tar.gz" && \
+    tar xzf "mpich-${MPICH_VERSION}.tar.gz" && \
+    cd "mpich-${MPICH_VERSION}" && \
+    patch -p1 < "$PATCH_PATH" && \
+    ./configure --with-device=ch4:ofi --disable-fortran --prefix="/opt/mpich/${MPICH_VERSION}" && \
+    make -j"$(nproc)" && \
+    make install && \
+    cd .. && \
+    rm -rf mpich-4.1.2/ mpich-4.1.2.tar.gz /tmp/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch && \
+    cd / && \
+    rm -rf /tmp/build-mpich
+
+
+ENV MPICH_VERSION="${MPICH_VERSION}" \
+    MPICH_DIR="/opt/mpich/${MPICH_VERSION}" \
+    PATH="/opt/mpich/${MPICH_VERSION}/bin:${PATH}" \
+    LD_LIBRARY_PATH="/opt/mpich/${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}"
+
+# Build Quokka
+RUN set -euo pipefail && \
+    cmake -S . -B build \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DAMReX_SPACEDIM="${AMREX_SPACEDIM}" \
+      -DAMReX_GPU_BACKEND=HIP \
+      -DAMReX_MPI="${AMREX_MPI}" \
+      -DAMReX_GPU_ARCH="${AMREX_AMD_ARCH}" \
+      -DCMAKE_C_COMPILER="${CC}" \
+      -DCMAKE_CXX_COMPILER="${CXX}" \
+      -DQUOKKA_PYTHON=OFF && \
+    cmake --build build -j"$(nproc)" --target DiskGalaxy && \
+    test -x build/src/problems/DiskGalaxy/DiskGalaxy
+
+# Runtime stage (reduced image size)
+FROM --platform=linux/amd64 opensuse/leap:15.6
+
+ARG MPICH_VERSION=4.1.2
+
+COPY --from=build /opt/rocm /opt/rocm
+COPY --from=build /opt/mpich /opt/mpich
+
+# Install runtime deps, cleanup, and setup user
+RUN set -euo pipefail && \
+    zypper --non-interactive refresh && \
+    zypper --non-interactive install --no-recommends libhdf5 libpciaccess0 libnuma1 libdrm2 libdrm_amdgpu1 && \
+    zypper clean --all && \
+    printf "%s\n" "/opt/rocm/lib" "/opt/rocm/lib64" >> /etc/ld.so.conf.d/rocm.conf && \
+    rm -rf /opt/rocm/bin /opt/rocm/include /opt/rocm/share /opt/rocm/lib/cmake \
+           /opt/rocm/llvm/bin /opt/rocm/llvm/include /opt/rocm/llvm/share /opt/rocm/llvm/lib/cmake && \
+    find /opt/rocm -type f -name "*.a" -delete && \
+    find /opt/rocm -type f -name "*.o" -delete && \
+    find /opt/mpich -type f -name "*.a" -delete && \
+    find /opt/mpich -type f -name "*.o" -delete && \
+    ldconfig && \
+    mkdir -p /opt/quokka/bin /opt/quokka/share/quokka/inputs /opt/quokka/share/quokka/extern/agora_data && \
+    useradd -ms /bin/bash leap && \
+    usermod -a -G render,video leap
+
+COPY --from=build /tmp/quokka-src/build/src/problems/DiskGalaxy/DiskGalaxy /opt/quokka/bin/DiskGalaxy
+COPY --from=build /tmp/quokka-src/inputs/DiskGalaxy.in /opt/quokka/share/quokka/inputs/DiskGalaxy.in
+COPY --from=build /tmp/quokka-src/extern/agora_data/ /opt/quokka/share/quokka/extern/agora_data/
+
+RUN chmod 0755 /opt/quokka/bin/DiskGalaxy && \
+    chown -R leap /opt/quokka
+
+ENV MPICH_VERSION="${MPICH_VERSION}" \
+    PATH="/opt/quokka/bin:/opt/mpich/${MPICH_VERSION}/bin:${PATH}" \
+    LD_LIBRARY_PATH="/opt/mpich/${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}"
+
+USER leap
+WORKDIR /home/leap

.github/workflows/dockerfiles/ROCm-DiskGalaxy/mpich-4.1.2-rocm-6.4-hip-pointer-attr.patch

@@ -0,0 +1,19 @@
+diff --git a/src/mpl/src/gpu/mpl_gpu_hip.c b/src/mpl/src/gpu/mpl_gpu_hip.c
+index f6a2df3..b2c5954 100644
+--- a/src/mpl/src/gpu/mpl_gpu_hip.c
++++ b/src/mpl/src/gpu/mpl_gpu_hip.c
+@@ -90,7 +90,13 @@ int MPL_gpu_query_pointer_attr(const void *ptr, MPL_pointer_attr_t * attr)
+     hipError_t ret;
+     ret = hipPointerGetAttributes(&attr->device_attr, ptr);
+     if (ret == hipSuccess) {
+-        switch (attr->device_attr.memoryType) {
++#if defined(HIP_VERSION_MAJOR) && (HIP_VERSION_MAJOR >= 6)
++        /* ROCm 6+ uses hipPointerAttribute_t::type */
++        enum hipMemoryType memory_type = attr->device_attr.type;
++#else
++        enum hipMemoryType memory_type = attr->device_attr.memoryType;
++#endif
++        switch (memory_type) {
+             case hipMemoryTypeHost:
+                 attr->type = MPL_GPU_POINTER_REGISTERED_HOST;
+                 attr->device = attr->device_attr.device;

.github/workflows/dockerfiles/ROCm-DiskGalaxy/rocm_so.conf

@@ -0,0 +1,2 @@
+/opt/rocm/lib
+/opt/rocm/lib64