AMD GPU Support via HIP/ROCm

.github/scripts/install_hip_ubuntu.sh

@@ -0,0 +1,172 @@
+#! /usr/bin/env bash
+# Note: this is a very initial HIP installation script for CI. This will need expanding / improving as support for a wider range of HIPs is added.
+# Todo: Full version of this as a script, including:
+# Todo: Respect input env vars
+# Todo: better subpackage management etc
+# Todo: 7.x is different thatn 6.x etc
+# Todo: Ubuntu version differences
+
+# Define the hip packages to install, in a version agnostic way, targetting as few as required for CI to succeed
+HIP_PACKAGES_IN=(
+    "rocm-hip-runtime-dev"
+    "rocthrust-dev"
+    "hipcub-dev"
+    "rocrand-dev"
+    "hiprand-dev"
+    "rocprofiler-sdk-roctx"
+)
+
+## -------------------
+## Bash functions
+## -------------------
+# returns 0 (true) if a >= b
+function version_ge() {
+    [ "$#" != "2" ] && echo "${FUNCNAME[0]} requires exactly 2 arguments." && exit 1
+    [ "$(printf '%s\n' "$@" | sort -V | head -n 1)" == "$2" ]
+}
+# returns 0 (true) if a > b
+function version_gt() {
+    [ "$#" != "2" ] && echo "${FUNCNAME[0]} requires exactly 2 arguments." && exit 1
+    [ "$1" = "$2" ] && return 1 || version_ge $1 $2
+}
+# returns 0 (true) if a <= b
+function version_le() {
+    [ "$#" != "2" ] && echo "${FUNCNAME[0]} requires exactly 2 arguments." && exit 1
+    [ "$(printf '%s\n' "$@" | sort -V | head -n 1)" == "$1" ]
+}
+# returns 0 (true) if a < b
+function version_lt() {
+    [ "$#" != "2" ] && echo "${FUNCNAME[0]} requires exactly 2 arguments." && exit 1
+    [ "$1" = "$2" ] && return 1 || version_le $1 $2
+}
+
+## -----------------
+## Check for root/sudo
+## -----------------
+
+# Detect if the script is being run as root, storing true/false in is_root.
+is_root=false
+if (( $EUID == 0)); then
+   is_root=true
+fi
+# Find if sudo is available
+has_sudo=false
+if command -v sudo &> /dev/null ; then
+    has_sudo=true
+fi
+# Decide if we can proceed or not (root or sudo is required) and if so store whether sudo should be used or not. 
+if [ "$is_root" = false ] && [ "$has_sudo" = false ]; then 
+    echo "Root or sudo is required. Aborting."
+    exit 1
+elif [ "$is_root" = false ] ; then
+    USE_SUDO=sudo
+else
+    USE_SUDO=
+fi
+
+
+## ------------------
+## Select HIP version
+## ------------------
+
+# Get the hip version from the environment as $hip.
+HIP_VERSION_STR=${hip}
+
+# Split the version.
+# We (might/probably) don't know PATCH at this point - it depends which version gets installed.
+HIP_MAJOR=$(echo "${HIP_VERSION_STR}" | cut -d. -f1)
+HIP_MINOR=$(echo "${HIP_VERSION_STR}" | cut -d. -f2)
+HIP_PATCH=$(echo "${HIP_VERSION_STR}" | cut -d. -f3)
+# default patch to 0 if not provided
+HIP_PATCH=${HIP_PATCH:-0}
+# build a 3-part version string for package names
+HIP_MAJOR_MINOR_PATCH="${HIP_MAJOR}.${HIP_MINOR}.${HIP_PATCH}"
+# use lsb_release to find the OS.
+UBUNTU_NAME=$(lsb_release -sc)
+UBUNTU_VERSION=$(lsb_release -sr)
+UBUNTU_VERSION="${UBUNTU_VERSION//.}"
+
+echo "HIP_MAJOR: ${HIP_MAJOR}"
+echo "HIP_MINOR: ${HIP_MINOR}"
+echo "HIP_PATCH: ${HIP_PATCH}"
+echo "UBUNTU_NAME: ${UBUNTU_NAME}"
+echo "UBUNTU_VERSION: ${UBUNTU_VERSION}"
+
+
+# If we don't know the HIP_MAJOR or MINOR, error.
+if [ -z "${HIP_MAJOR}" ] ; then
+    echo "Error: Unknown HIP Major version. Aborting."
+    exit 1
+fi
+if [ -z "${HIP_MINOR}" ] ; then
+    echo "Error: Unknown HIP Minor version. Aborting."
+    exit 1
+fi
+# If we don't know the Ubuntu version, error.
+if [ -z ${UBUNTU_VERSION} ]; then
+    echo "Error: Unknown Ubuntu version. Aborting."
+    exit 1
+fi
+
+## ------------------------------
+## Select HIP packages to install
+## ------------------------------
+# build  a space-separated list of packages to install via apt
+HIP_PACKAGES=""
+for package in "${HIP_PACKAGES_IN[@]}"
+do : 
+    # Append the hip version to the pacakge name
+    HIP_PACKAGES+=" ${package}${HIP_MAJOR_MINOR_PATCH}"
+done
+echo "HIP_PACKAGES ${HIP_PACKAGES}"
+
+# enable command printing for CI debugging
+set -x
+
+# following ubuntu instructions from:
+# https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html
+# https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/install-methods/package-manager/package-manager-ubuntu.html
+
+# install deps
+$USE_SUDO apt-get install -y python3-setuptools python3-wheel
+
+# Signing key
+$USE_SUDO mkdir --parents --mode=0755 /etc/apt/keyrings
+wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | $USE_SUDO tee /etc/apt/keyrings/rocm.gpg > /dev/null
+
+# register packages
+$USE_SUDO tee /etc/apt/sources.list.d/rocm.list << EOF
+deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${HIP_VERSION_STR} ${UBUNTU_NAME} main
+deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/${HIP_VERSION_STR}/ubuntu ${UBUNTU_NAME} main
+EOF
+$USE_SUDO tee /etc/apt/preferences.d/rocm-pin-600 << EOF
+Package: *
+Pin: release o=repo.radeon.com
+Pin-Priority: 600
+EOF
+$USE_SUDO apt update
+
+# Install packages.
+$USE_SUDO apt install -y ${HIP_PACKAGES}
+
+## -----------------
+## Set environment vars / vars to be propagated
+## -----------------
+
+ROCM_PATH=/opt/rocm-${HIP_MAJOR_MINOR_PATCH}
+echo "ROCM_PATH=${ROCM_PATH}"
+export ROCM_PATH=${ROCM_PATH}
+export PATH="$ROCM_PATH/bin:$PATH"
+export LD_LIBRARY_PATH="$ROCM_PATH/lib:$LD_LIBRARY_PATH"
+
+# Check hipcc is now available.
+hipcc -V
+
+# If executed on github actions, make the appropriate echo statements to update the environment
+if [[ $GITHUB_ACTIONS ]]; then
+    # Set paths for subsequent steps, using ${ROCM_PATH}
+    echo "Adding HIP ${HIP_MAJOR_MINOR_PATCH} to ROCM_PATH, PATH and LD_LIBRARY_PATH"
+    echo "ROCM_PATH=${ROCM_PATH}" >> $GITHUB_ENV
+    echo "${ROCM_PATH}/bin" >> $GITHUB_PATH
+    echo "LD_LIBRARY_PATH=${ROCM_PATH}/bin/:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+fi

.github/workflows/Docs.yml

@@ -48,6 +48,7 @@ jobs:
         -DCMAKE_WARN_DEPRECATED="OFF"
         -DFLAMEGPU_WARNINGS_AS_ERRORS="ON"
         -DFLAMEGPU_BUILD_API_DOCUMENTATION="ON"
+        -DFLAMEGPU_GPU=OFF
 
     - name: Docs
       working-directory: ${{ env.BUILD_DIR }}

.github/workflows/Draft-Release.yml

@@ -7,9 +7,9 @@ on:
     tags:
       - 'v*.*.*'
   # Or on pull_requests which mutate this CI workflow, using pr rather than push:branch to ensure that the merged state will be OK, as that is what is important here.
-  pull_request:
-    paths:
-      - ".github/workflows/Draft-Release.yml"
+  # pull_request:
+  #   paths:
+  #     - ".github/workflows/Draft-Release.yml"
   # Or trigger on manual dispatch. This will not produce a release, but will perform the thorough build.
   workflow_dispatch:
 
@@ -51,9 +51,9 @@ jobs:
             cuda_arch: "50-real;120-real;120-virtual;"
             hostcxx: gcc-12
             os: ubuntu-24.04
-          - cuda: "12.0"
+          - cuda: "12.4"
             cuda_arch: "50-real;90-real;90-virtual;"
-            hostcxx: gcc-10
+            hostcxx: gcc-11
             os: ubuntu-22.04
         python:
           - "3.10"
@@ -320,7 +320,7 @@ jobs:
             hostcxx: gcc-toolset-13
             os: ubuntu-24.04
           # CUDA 12
-          - cuda: "12.0"
+          - cuda: "12.4"
             cuda_arch: "50-real;60-real;70-real;80-real;90-real;90-virtual"
             hostcxx: gcc-toolset-12
             os: ubuntu-24.04

.github/workflows/Lint.yml

@@ -30,7 +30,7 @@ jobs:
       # optional exclude: can be partial, include: must be specific
       matrix:
         cudacxx:
-          - cuda: "12.0"
+          - cuda: "12.4"
             os: ubuntu-22.04
     env:
       # Define constants

.github/workflows/MPI.yml

@@ -32,7 +32,7 @@ jobs:
       matrix:
         # CUDA_ARCH values are reduced compared to wheels due to CI memory issues while compiling the test suite.
         cudacxx:
-          - cuda: "12.0"
+          - cuda: "12.4"
             cuda_arch: "50-real;"
             hostcxx: gcc-11
             os: ubuntu-22.04

.github/workflows/Manylinux_2_28.yml

@@ -39,7 +39,7 @@ jobs:
             hostcxx: gcc-toolset-13
             os: ubuntu-24.04
           # CUDA 12
-          - cuda: "12.0"
+          - cuda: "12.4"
             cuda_arch: "50"
             hostcxx: gcc-toolset-12
             os: ubuntu-24.04

.github/workflows/Ubuntu-HIP.yml

@@ -0,0 +1,141 @@
+# Compile project on Ubuntu using HIP. 
+# This is currently separate to keep thigns simple (CI refactoring is required at some point anyway),
+# And HIP builds only support a subset of features currently
+name: Ubuntu-HIP
+
+on:
+  # Branch pushes that do not only modify other workflow files
+  push:
+    branches:
+      - '**'
+    paths:
+      - "**"
+      - "!.github/**"
+      - ".github/scripts/install_hip_ubuntu.sh"
+      - ".github/workflows/Ubuntu-HIP.yml"
+  # Disabled for now. See https://github.com/FLAMEGPU/FLAMEGPU2/pull/644
+  # pull_request:
+  # Allow manual invocation.
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    runs-on: ${{ matrix.compilers.os }}
+    strategy:
+      fail-fast: false
+      # Multiplicative build matrix
+      # optional exclude: can be partial, include: must be specific
+      matrix:
+        compilers:
+          # see https://llvm.org/docs/AMDGPUUsage.html#amdgpu-generic-processor-table for arch flags
+          # HIP 7.2 with any RDNA3 GPU (recent consumer, that we can actually test)
+          - hip: "7.2"
+            hip_arch: "gfx11-generic"
+            hostcxx: hipcc
+            os: ubuntu-24.04
+          # HIP 7.0 with a Vega/CDNA Datacentre GPU (oldest we will support?)
+          - hip: "7.0"
+            hip_arch: "gfx9-generic"
+            hostcxx: hipcc
+            os: ubuntu-24.04
+        config:
+          - name: "Release"
+            config: "Release"
+            SEATBELTS: "ON"
+          # Temp disabled to keep CI smaller
+          # - name: "Beltsoff"
+          #   config: "Release"
+          #   SEATBELTS: "OFF"
+
+    # Name the job based on matrix/env options
+    name: "build (${{ matrix.compilers.hip }}, ${{ matrix.config.name }}, ${{ matrix.compilers.os }})"
+
+    # Define job-wide env constants, and promote matrix elements to env constants for portable steps.
+    env:
+      # Workflow specific constants for building a specific example
+      # Note this assumes the example exists in cpp, rather than cpp_rtc subdirectory
+      INDIVIDUAL_EXAMPLE: ${{ fromJSON('{true:"game_of_life",false:""}')[matrix.config.name == 'Release' && matrix.VISUALISATION == 'OFF'] }}
+      # Define constants
+      BUILD_DIR: "build"
+      FLAMEGPU_BUILD_TESTS: "ON"
+      # Port matrix options to environment, for more portability.
+      HIP: ${{ matrix.compilers.hip }}
+      HIP_ARCH: ${{ matrix.compilers.hip_arch }}
+      HOSTCXX: ${{ matrix.compilers.hostcxx }}
+      OS: ${{ matrix.compilers.os }}
+      CONFIG: ${{ matrix.config.config }}
+      FLAMEGPU_SEATBELTS: ${{ matrix.config.SEATBELTS }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install HIP
+      if: ${{ startswith(env.OS, 'ubuntu') && env.HIP != '' }}
+      env:
+        hip: ${{ env.HIP }}
+      run: .github/scripts/install_hip_ubuntu.sh
+
+    - name: Install/Select host compiler (HIPCC only)
+      run: |
+        echo "CC=hipcc" >> $GITHUB_ENV
+        echo "CXX=hipcc" >> $GITHUB_ENV
+
+    - name: Add custom problem matchers for annotations
+      run: echo "::add-matcher::.github/problem-matchers.json"
+
+    # This pre-emptively patches a bug from ManyLinux where git dir is owned by diff user, blocking buildnumber generation
+    - name: Enable git safe-directory
+      run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+    - name: Configure cmake
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+        -DCMAKE_BUILD_TYPE="${{ env.CONFIG }}"
+        -Werror=dev
+        -DCMAKE_WARN_DEPRECATED="OFF"
+        -DFLAMEGPU_WARNINGS_AS_ERRORS="ON"
+        -DFLAMEGPU_GPU="HIP"
+        -DCMAKE_HIP_ARCHITECTURES="${{ env.HIP_ARCH }}"
+        -DFLAMEGPU_SEATBELTS="${{ env.FLAMEGPU_SEATBELTS }}"
+        -DFLAMEGPU_BUILD_TESTS="${{ env.FLAMEGPU_BUILD_TESTS }}"
+
+    # Check for bugs when cmake is reconfigured, i.e. fetch content patching
+    - name: Re-configure cmake
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+
+    - name: Build static library
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target flamegpu --verbose -j `nproc`
+
+    - name: Build tests
+      if: ${{ env.FLAMEGPU_BUILD_TESTS == 'ON' }}
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target tests --verbose -j `nproc`
+
+    - name: Build all remaining targets
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target all --verbose -j `nproc`
+
+    - name: Configure Individual example
+      if: ${{ env.INDIVIDUAL_EXAMPLE != '' }}
+      working-directory: examples/cpp/${{ env.INDIVIDUAL_EXAMPLE }}
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+        -DCMAKE_BUILD_TYPE="${{ env.CONFIG }}"
+        -Werror=dev
+        -DCMAKE_WARN_DEPRECATED="OFF"
+        -DFLAMEGPU_WARNINGS_AS_ERRORS="ON"
+        -DFLAMEGPU_SEATBELTS="${{ env.FLAMEGPU_SEATBELTS }}"
+        -DFLAMEGPU_GPU="HIP"
+        -DCMAKE_HIP_ARCHITECTURES="${{ env.HIP_ARCH }}"
+
+    - name: Build Individual example
+      if: ${{ env.INDIVIDUAL_EXAMPLE != '' }}
+      working-directory: examples/cpp/${{ env.INDIVIDUAL_EXAMPLE }}/${{ env.BUILD_DIR }}
+      run: cmake --build . --target ${{ env.INDIVIDUAL_EXAMPLE }} --verbose -j `nproc`
+

.github/workflows/Ubuntu.yml

@@ -39,9 +39,9 @@ jobs:
             cuda_arch: "120"
             hostcxx: gcc-12
             os: ubuntu-24.04
-          - cuda: "12.0"
+          - cuda: "12.4"
             cuda_arch: "50"
-            hostcxx: gcc-10
+            hostcxx: gcc-11
             os: ubuntu-22.04
         python:
           - "3.14"