ci.yml

name: Main
on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  # Pre-flight job to activate the cuda infrastructure
  prepare-dell:
    name: Activate self-host infrastructure
    runs-on: [self-hosted, spotter]
    steps:
      - uses: compiler-research/ci-workflows/actions/wake-on-lan@main
        with:
          mac: a4:bb:6d:51:d5:d2
          target-host: 192.168.100.30

  build:
    timeout-minutes: 600 # If some self-host machine is not available
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:

        include:
          # --- LINUX: Last 11 Runtimes (12-22) ---
          # Using Ubuntu 22.04 for older runtimes, 24.04 for newer ones
          - { name: ubu22-gcc11-runtime12, os: ubuntu-22.04, compiler: gcc-11, clang-runtime: '12' }
          - { name: ubu22-clang13-runtime13, os: ubuntu-22.04, compiler: clang-13, clang-runtime: '13' }
          - { name: ubu24-gcc13-runtime14, os: ubuntu-24.04, compiler: gcc-13, clang-runtime: '14' }
          - { name: ubu24-clang15-runtime15, os: ubuntu-24.04, compiler: clang-15, clang-runtime: '15' }
          - { name: ubu24-gcc14-runtime16, os: ubuntu-24.04, compiler: gcc-14, clang-runtime: '16' }
          - { name: ubu24-clang16-runtime17, os: ubuntu-24.04, compiler: clang-16, clang-runtime: '17' }
          - { name: ubu24-gcc14-runtime18, os: ubuntu-24.04, compiler: gcc-14, clang-runtime: '18' }
          - { name: ubu24-clang17-runtime19, os: ubuntu-24.04, compiler: clang-17, clang-runtime: '19' }
          - { name: ubu24-clang19-runtime20, os: ubuntu-24.04, compiler: clang-19, clang-runtime: '20' }
          - { name: ubu24-clang20-runtime21, os: ubuntu-24.04, compiler: clang-20, clang-runtime: '21' }
          - { name: ubu24-clang20-runtime22, os: ubuntu-24.04, compiler: clang-20, clang-runtime: '22' }
          - { name: ubu24-clang22-runtime22-asan, os: ubuntu-24.04, compiler: clang-22, clang-runtime: '22', flavor: asan, extra_cmake_options: '-DLLVM_USE_SANITIZER=Address' }
          # --- MACOS: Last 5 Runtimes (17-21) ---
          # Using macOS ARM (standard) for all; Runtime 17 on Intel for x86 sanity
          - { name: osx-intel-runtime17, os: macos-15-intel, compiler: clang, clang-runtime: '17' }
          - { name: osx-arm-runtime18, os: macos-26, compiler: clang, clang-runtime: '18' }
          - { name: osx-arm-runtime19, os: macos-26, compiler: clang, clang-runtime: '19' }
          - { name: osx-arm-runtime20, os: macos-26, compiler: clang, clang-runtime: '20' }
          - { name: osx-arm-runtime21, os: macos-26, compiler: clang, clang-runtime: '21' }

          # --- WINDOWS: Last 5 Runtimes (17-21) ---
          - { name: win-msvc-runtime17, os: windows-2022, compiler: msvc, clang-runtime: '17' }
          - { name: win-msvc-runtime18, os: windows-2022, compiler: msvc, clang-runtime: '18' }
          - { name: win-msvc-runtime19, os: windows-2022, compiler: msvc, clang-runtime: '19' }
          - { name: win-msvc-runtime20, os: windows-2022, compiler: msvc, clang-runtime: '20' }
          - { name: win-msvc-runtime21, os: windows-2022, compiler: msvc, clang-runtime: '21' }

          # --- CUSTOM: Debug, coverage, benchmarks, static analysis ---
          - name: ubu24-clang20-runtime22-debug
            os: ubuntu-24.04
            compiler: clang-20
            clang-runtime: '22'
            debug_build: true
            flavor: debug

          - name: ubu24-gcc13-runtime20-benchmarks
            os: ubuntu-24.04
            compiler: gcc-13
            clang-runtime: '20'
            extra_cmake_options: '-DCLAD_ENABLE_BENCHMARKS=On -DCLAD_ENABLE_ENZYME_BACKEND=On'
            extra_packages: "libomp-20-dev libopenblas-dev"
            benchmark: true

          - name: ubu22-clang13-runtime13-cuda
            os: ubuntu-22.04
            compiler: clang-13
            clang-runtime: '13'
            cuda: true

          - name: ubuntu-clang15-runtime16-doc
            os: ubuntu-latest
            compiler: clang-15
            clang-runtime: '16'
            doc_build: true

          - name: ubu24-clang16-runtime17-shared-libs
            os: ubuntu-24.04
            compiler: clang-16
            extra_cmake_options: '-DBUILD_SHARED_LIBS=On'
            clang-runtime: '17'

          - name: ubu22-clang16-runtime22-kokkos
            os: ubuntu-22.04
            compiler: clang-16
            clang-runtime: '22'
            extra_packages: 'libtrilinos-kokkos-dev ninja-build'
            extra_cmake_options: '-G Ninja'

          - name: ubu24-clang20-runtime22-vg
            os: ubuntu-24.04
            compiler: clang-20
            extra_packages: 'valgrind'
            extra_cmake_options: '-DCLAD_TEST_USE_VG=On'
            clang-runtime: '22'

          - name: ubu24-arm-clang16-runtime17-shared-libs
            os: ubuntu-24.04-arm
            compiler: clang-16
            extra_cmake_options: '-DBUILD_SHARED_LIBS=On'
            clang-runtime: '17'

          - name: ubu24-arm-clang16-runtime22-kokkos
            os: ubuntu-24.04-arm
            compiler: clang-16
            clang-runtime: '22'
            extra_packages: 'libtrilinos-kokkos-dev ninja-build'
            extra_cmake_options: '-G Ninja'

          - name: ubu24-arm-clang16-runtime18
            os: ubuntu-24.04-arm
            compiler: clang-16
            clang-runtime: '18'

    steps: &build_steps
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.11
    - name: Save PR Info
      if: ${{ matrix.coverage == true }}
      run: |
        mkdir -p ./pr
        echo ${{ github.event.number }} > ./pr/NR
        echo ${{ github.repository }} > ./pr/REPO
    - uses: actions/upload-artifact@v4
      if: ${{ matrix.coverage == true }}
      with:
        name: pr
        path: pr/
        overwrite: true
    - uses: nelonoel/branch-name@v1.0.1
    - name: Setup default Build Type on *nux
      if: ${{ (matrix.debug_build != true) && (runner.os != 'windows') }}
      run: |
        echo "BUILD_TYPE=Release" >> $GITHUB_ENV
        echo "CLAD_CODE_COVERAGE=0" >> $GITHUB_ENV
    - name: Setup default Build Type on *nux (build_debug & asserts)
      if: ${{ (matrix.debug_build == true) && (runner.os != 'windows') }}
      run: |
        echo "BUILD_TYPE=Debug" >> $GITHUB_ENV
        # Cleanup some space: https://github.com/orgs/community/discussions/25678
        # Under nektos/act this path is a bind-mount, so the mountpoint itself
        # cannot be removed ("Device or resource busy"); tolerate that so the
        # debug row is reproducible locally.
        rm -fr /opt/hostedtoolcache || true
    - name: Setup default Build Type on Windows
      if: runner.os == 'windows'
      run: |
        $env:BUILD_TYPE="Release"
        $env:CLAD_CODE_COVERAGE="0"
        echo "BUILD_TYPE=Release" >> $env:GITHUB_ENV
        echo "CLAD_CODE_COVERAGE=0" >> $env:GITHUB_ENV
    - name: Compute ncpus (cgroup-aware)
      if: runner.os != 'windows'
      run: |
        if [[ "${{ runner.os }}" == "macOS" ]]; then
          ncpus=$(sysctl -n hw.ncpu)
        else
          ncpus=$(nproc)
          if [[ -r /sys/fs/cgroup/cpu.max ]]; then
            read -r quota period < /sys/fs/cgroup/cpu.max || true
            if [[ "${quota:-max}" != "max" && "${period:-0}" -gt 0 ]]; then
              capped=$(( (quota + period - 1) / period ))
              if [[ "$capped" -ge 1 && "$capped" -lt "$ncpus" ]]; then
                ncpus="$capped"
              fi
            fi
          fi
        fi
        echo "ncpus=$ncpus" >> $GITHUB_ENV
    - name: Setup LLVM ${{ matrix.clang-runtime }}${{ matrix.flavor && format(' [{0}]', matrix.flavor) || '' }}
      if: runner.os != 'Windows'
      uses: compiler-research/ci-workflows/actions/setup-llvm@main
      with:
        version: ${{ matrix.clang-runtime }}
        os:      ${{ matrix.self-hosted-os || matrix.os }}
        flavor:  ${{ matrix.flavor || 'system' }}

    - name: Install extra Linux deps
      if: runner.os == 'Linux'
      run: |
        # cmake is preinstalled on github-hosted ubuntu-24.04 but absent
        # from catthehacker/ubuntu:act-24.04; install it so bin/repro works.
        # The debug row needs it too (to build clad), so this step must not
        # be gated on debug_build.
        sudo apt install -y cmake ${{ matrix.extra_packages }}
        # libomp-N-dev: best-effort. apt-llvm.org lags on the latest
        # major; recipe-flavor rows get libomp from the recipe build,
        # not apt. When missing, OpenMP lit tests skip via REQUIRES.
        if apt-cache show "libomp-${{ matrix.clang-runtime }}-dev" >/dev/null 2>&1; then
          sudo apt install -y "libomp-${{ matrix.clang-runtime }}-dev"
        fi

    - name: Setup compiler on Linux
      if: runner.os == 'Linux'
      run: |
        # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
        vers="${compiler#*-}"
        os_codename="`cat /etc/os-release | grep UBUNTU_CODENAME | cut -d = -f 2`"
        sudo apt update
        if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then
          sudo apt install -y gcc-${vers} g++-${vers}
          echo "CC=gcc-${vers}" >> $GITHUB_ENV
          echo "CXX=g++-${vers}" >> $GITHUB_ENV
        else
          if ! sudo apt install -y clang-${vers}; then
            curl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
            echo "deb https://apt.llvm.org/${os_codename}/ llvm-toolchain-${os_codename}-${vers} main" | sudo tee -a /etc/apt/sources.list
            sudo apt update
            sudo apt install -y clang-${vers}
          fi
          echo "CC=clang-${vers}" >> $GITHUB_ENV
          echo "CXX=clang++-${vers}" >> $GITHUB_ENV
        fi
      env:
        compiler: ${{ matrix.compiler }}
    - name: Setup compiler on macOS
      if: runner.os == 'macOS'
      run: |
        if [[ "${{ matrix.compiler }}" == *"clang"* ]]; then
          echo "CC=clang" >> $GITHUB_ENV
          echo "CXX=clang++" >> $GITHUB_ENV
        else
          echo "Unsupported compiler - fix YAML file"
        fi
    - name: Setup compiler on Windows
      if: runner.os == 'windows'
      run: |
        if ( "${{ matrix.compiler }}" -imatch "clang" )
        {
          $ver="${{ matrix.compiler }}".split("-")[1]
          choco install llvm --version=$ver --no-progress -my
          $env:PATH_TO_LLVM_BUILD="$env:ProgramFiles\LLVM"
          echo "PATH_TO_LLVM_BUILD=$env:PATH_TO_LLVM_BUILD"
          echo "PATH_TO_LLVM_BUILD=$env:PATH_TO_LLVM_BUILD" >> $env:GITHUB_ENV
          clang --version
          #
          $env:CC="clang"
          $env:CXX="clang++"
          echo "CC=clang" >> $env:GITHUB_ENV
          echo "CXX=clang++" >> $env:GITHUB_ENV
        }
        elseif ( "${{ matrix.compiler }}" -imatch "msvc" )
        {
          # MSVC is builtin in container image
        }
        else
        {
          echo "Unsupported compiler - fix YAML file"
        }

    - name: Setup LLVM extras on macOS
      if: runner.os == 'macOS'
      run: |
        # setup-llvm ran `brew install llvm@N` above; pick the prefix
        # and copy SDK headers into /usr/local/include/ (ln fails on
        # macOS-arm runners, so we copy).
        PATH_TO_LLVM_BUILD=$(brew --prefix llvm@${{ matrix.clang-runtime }})
        pip3 install lit
        sudo cp -r -n $(xcrun --show-sdk-path)/usr/include/ /usr/local/include/
        echo "PATH_TO_LLVM_BUILD=$PATH_TO_LLVM_BUILD" >> $GITHUB_ENV

    - name: Setup LLVM/Clang on Linux
      if: ${{ runner.os == 'Linux' }}
      run: |
        # setup-llvm always lands the install at $GITHUB_WORKSPACE/install
        # (a symlink to /usr/lib/llvm-N for flavor=system, a real install
        # tree for the recipe flavors). Point clad's build prefix there so
        # the cmake configs and their .a references resolve regardless of
        # flavor.
        PATH_TO_LLVM_BUILD="$GITHUB_WORKSPACE/install"
        pip3 install lit # LLVM lit is not part of the llvm releases...
        echo "PATH_TO_LLVM_BUILD=$PATH_TO_LLVM_BUILD" >> $GITHUB_ENV
    - name: Setup LLVM/Clang on Windows
      if: ${{ runner.os == 'windows' }}
      run: |
        C:\Miniconda\condabin\conda.bat install -y -c conda-forge "clangdev=${{ matrix.clang-runtime }}"
        $env:PATH_TO_LLVM_BUILD="$env:CONDA\Library"
        pip3 install lit # LLVM lit is not part of the llvm releases...
        echo "Runtime LLVM/Clang: $env:PATH_TO_LLVM_BUILD"
        echo "PATH_TO_LLVM_BUILD=$env:PATH_TO_LLVM_BUILD"
        echo "PATH_TO_LLVM_BUILD=$env:PATH_TO_LLVM_BUILD" >> $env:GITHUB_ENV
    - name: Setup CUDA 8 on Linux
      if: ${{ matrix.cuda == true && !matrix.os == 'self-hosted'}}
      run: |
        wget --no-verbose https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda_8.0.61_375.26_linux-run
        wget --no-verbose https://developer.nvidia.com/compute/cuda/8.0/Prod2/patches/2/cuda_8.0.61.2_linux-run
        sh ./cuda_8.0.61_375.26_linux-run --tar mxvf
        sudo cp InstallUtils.pm /usr/lib/x86_64-linux-gnu/perl-base
        export $PERL5LIB
        sudo sh cuda_8.0.61_375.26_linux-run --override --no-opengl-lib --silent --toolkit --kernel-source-path=/lib/modules/4.15.0-1113-azure/build
        sudo sh cuda_8.0.61.2_linux-run --silent --accept-eula
        export PATH=/usr/local/cuda-8.0/bin:${PATH}
        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda-8.0/lib64
        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
        echo "PATH=$PATH" >> $GITHUB_ENV
    - name: Setup code coverage
      if: ${{ (matrix.coverage == true) }}
      run: |
        sudo apt install -y lcov
        # Workaround ubuntu lcov-1.15-1 package bug
        if sudo apt list lcov --installed | grep "1.15-1" ; then
          wget https://launchpad.net/ubuntu/+source/lcov/1.15-2/+build/23784466/+files/lcov_1.15-2_all.deb
          sudo apt install -y ./lcov_1.15-2_all.deb
        fi
        echo "CLAD_CODE_COVERAGE=1" >> $GITHUB_ENV
        echo "BUILD_TYPE=Debug" >> $GITHUB_ENV
    - name: Display config *nix
      if: ${{ runner.os != 'windows' }}
      run: |
        cmake --version
        $CC --version
        $CXX --version
        xz --version
        tar --version
        echo "Use Clang/LLVM in $PATH_TO_LLVM_BUILD"
        echo "Building clad in `[[ -z "$BUILD_TYPE" ]] && echo RelWithDebInfo || echo $BUILD_TYPE`"
        python3 --version
    - name: Display config Windows
      if: ${{ runner.os == 'windows' }}
      run: |
        cmake --version
        echo "CC=$env:CC"
        if ( "$env:CC" -ne "" )
        {
          Start-Process -Wait -FilePath "$env:CC" -Verb RunAs -ArgumentList '--version'
        }
        echo "CXX=$env:CXX"
        if ( "$env:CXX" -ne "" )
        {
          Start-Process -Wait -FilePath "$env:CXX" -Verb RunAs -ArgumentList '--version'
        }
        xz --version
        tar --version
        echo "Use Clang/LLVM in $env:PATH_TO_LLVM_BUILD"
        echo "Building clad in $env:BUILD_TYPE"
        python3 --version
    - name: Build Clad on *nix
      if: ${{ runner.os != 'windows' }}
      run: |
        rm -rf obj && mkdir obj && cd obj
        cmake -DClang_DIR=${{ env.PATH_TO_LLVM_BUILD }}  \
          -DLLVM_DIR=${{ env.PATH_TO_LLVM_BUILD }}       \
          -DCMAKE_BUILD_TYPE=$([[ -z "$BUILD_TYPE" ]] && echo RelWithDebInfo || echo $BUILD_TYPE) \
          -DCLAD_CODE_COVERAGE=${{ env.CLAD_CODE_COVERAGE }}   \
          -DLLVM_EXTERNAL_LIT="`which lit`" \
          -DLLVM_ENABLE_WERROR=On           \
          $GITHUB_WORKSPACE                 \
          ${{ matrix.extra_cmake_options }}

        cmake --build . -- -j${{ env.ncpus }}
    - name: Build Clad on Windows
      if: ${{ runner.os == 'windows' }}
      run: |
        mkdir obj
        cd obj
        if ( "$env:BUILD_TYPE" -eq "" )
        {
          $env:BUILD_TYPE="RelWithDebInfo"
        }
        if ( "$env:CLAD_CODE_COVERAGE" -eq "" )
        {
          $env:CLAD_CODE_COVERAGE="0"
        }
        ((Get-Content -path C:/Miniconda/Library/lib/cmake/llvm/LLVMConfig.cmake -Raw) -replace 'LLVM_ENABLE_DIA_SDK ON','LLVM_ENABLE_DIA_SDK OFF') | Set-Content -Path C:/Miniconda/Library/lib/cmake/llvm/LLVMConfig.cmake
        $lit = @(which lit)
        cmake -DClang_DIR="$env:PATH_TO_LLVM_BUILD" -DLLVM_DIR="$env:PATH_TO_LLVM_BUILD" -DCMAKE_BUILD_TYPE="$env:BUILD_TYPE" -DCLAD_CODE_COVERAGE="$env:CLAD_CODE_COVERAGE" -DCLAD_BUILD_STATIC_ONLY="ON" -DLLVM_EXTERNAL_LIT="$lit" "$env:GITHUB_WORKSPACE" ${{ matrix.extra_cmake_options }}
        cmake --build . --config Release
    - name: Test Clad on *nix
      if: ${{ runner.os != 'windows' }}
      env:
        # lit reads LIT_OPTS at startup; without it, lit defaults to
        # multiprocessing.cpu_count() which sees the host count and
        # ignores cgroup CFS quotas, over-provisioning workers on
        # constrained containers (cf. the LLVM build step's cap).
        LIT_OPTS: -j${{ env.ncpus }}
      run: |
        cd obj
        # Packages downloaded and unzipped require setting the SDKROOT.
        if [[ "runner.os" == "macOS" ]]; then
          export SDKROOT=`xcrun --show-sdk-path`
        fi
        cmake --build . --target check-clad -- -j${{ env.ncpus }}
    - name: Test Clad on Windows
      if: ${{ runner.os == 'windows' }}
      run: |
        cd obj
        #cmake --build . --target check-clad
    - name: Benchmark Clad
      if: ${{ matrix.benchmark }}
      run: |
        baseline_hash=$([[ -z "${{ github.event.pull_request }}" ]] && echo ${{ github.event.before }} || echo ${{ github.event.pull_request.base.sha }})
        current_hash=${{ github.sha }}
        cd benchmark
        python3 benchmark_compare.py ${baseline_hash} ${current_hash} obj
        cd ..
    - name: Test build sphinx & doxygen documentation
      if: ${{ (matrix.doc_build == true) }}
      run: |
        sudo apt-get install -y doxygen graphviz
        pip install --upgrade pip
        pip install -r requirements.txt
        mkdir -p obj && cd obj
        cmake \
          -DClang_DIR="$PATH_TO_LLVM_BUILD" \
          -DLLVM_DIR="$PATH_TO_LLVM_BUILD"  \
          -DCLAD_INCLUDE_DOCS=ON            \
          -DCLAD_ENABLE_DOXYGEN=ON          \
          -DCLAD_ENABLE_SPHINX=ON           \
          -DCMAKE_INSTALL_PREFIX=../inst    \
          -DCMAKE_BUILD_TYPE=Debug          \
          -DLLVM_EXTERNAL_LIT="`which lit`" \
          ..
        cmake --build . --target sphinx-clad doxygen-clad -- -j${{ env.ncpus }}
    - name: Show reproduce-locally hint
      if: ${{ failure() }}
      shell: bash
      env:
        REPO: ${{ github.repository }}
        REPO_NAME: ${{ github.event.repository.name }}
        SHA: ${{ github.sha }}
        ROW: ${{ matrix.name }}
      run: |
        echo "::group::Reproduce this failure locally (requires Docker + nektos/act)"
        cat <<EOF
            git clone https://github.com/compiler-research/ci-workflows.git
            git clone https://github.com/$REPO.git
            cd $REPO_NAME
            git checkout $SHA
            ../ci-workflows/bin/repro $ROW
        EOF
        echo "::endgroup::"
    - name: Failed job config
      if: ${{ failure() && runner.debug && runner.os != 'windows' }}
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
      run: |
        echo "$GITHUB_CONTEXT"
        export
        if [[ "runner.os" == "Linux" ]]; then
          apt-mark showhold
          dpkg-query -L libclang-${{ matrix.clang-runtime }}-dev
          dpkg-query -L clang-${{ matrix.clang-runtime }}
          dpkg-query -L llvm-${{ matrix.clang-runtime }}-dev
          find /usr/lib/llvm-${{ matrix.clang-runtime }}/
        fi
        if [[ "runner.os" == "macOS" ]]; then
          brew search llvm
          find /usr/local/opt/llvm@*/
        fi
        pip show lit
        cat obj/CMakeCache.txt
        cat obj/CMakeFiles/*.log
    - name: Setup tmate session
      if: ${{ !cancelled() && runner.debug }}
      uses: mxschmitt/action-tmate@v3
      # When debugging increase to a suitable value!
      timeout-minutes: 30
    - name: Prepare code coverage report
      if: ${{ success() && (matrix.coverage == true) }}
      run: |
        # Create lcov report
        # capture coverage info
        vers="${CC#*-}"
        lcov --directory . --capture --output-file coverage.info --gcov-tool /usr/bin/gcov-${vers} --ignore-errors gcov
        # filter out system and extra files.
        # To also not include test code in coverage add them with full path to the patterns: '*/tests/*'
        lcov --remove coverage.info '/usr/*' "${HOME}"'/.cache/*' \
          "$GITHUB_WORKSPACE"'/test/*' "$GITHUB_WORKSPACE"'/unittests/*' \
          --output-file coverage.info
        # output coverage data for debugging (optional)
        lcov --list coverage.info

    - name: Upload to codecov.io
      if: ${{ success() && (matrix.coverage == true) }}
      uses: codecov/codecov-action@v5
      with:
        files: ./coverage.info
        fail_ci_if_error: true
        verbose: true
        token: ${{ secrets.CODECOV_TOKEN }}

  # Self-hosted CUDA rows. Separate job so `prepare-dell` gates only
  # what depends on the dell; the hosted-runner matrix in `build` runs
  # without waiting. Step list shared via the &build_steps anchor.
  build-selfhost:
    needs: prepare-dell
    timeout-minutes: 600 # If the dell is slow to come back up.
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: selfh-ubu22-gcc12-runtime19-analyzers
            os: [self-hosted, cuda, heavy]
            self-hosted-os: ubuntu-22.04
            compiler: gcc-12
            clang-runtime: '19'
            coverage: true
            cuda: true
            extra_cmake_options: '-DCLAD_ENABLE_ENZYME_BACKEND=On -DCLAD_CUDA_TEST_USE_SANITIZER=On'
            extra_packages: ' libzstd-dev '

          - name: selfh-ubu22-clang16-runtime18-cuda
            os: [self-hosted, cuda, heavy]
            self-hosted-os: ubuntu-22.04
            compiler: clang-16
            extra_cmake_options: '-DCLAD_CUDA_TEST_USE_SANITIZER=On'
            clang-runtime: '18'
            cuda: true

    steps: *build_steps