nightly-publish.yml

# Copyright Contributors to the OpenVDB Project
# SPDX-License-Identifier: Apache-2.0

name: fVDB Nightly Wheel Publish

on:
  schedule:
    - cron: "0 8 * * *"  # daily at 8am UTC
  workflow_dispatch:

concurrency:
  group: nightly-publish
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  ##############################################################################
  # GATE: SKIP BUILD IF NO NEW COMMITS ON MAIN (bypass for manual dispatch)
  ##############################################################################
  check-new-commits:
    name: Check for new commits on main
    if: github.repository == 'openvdb/fvdb-core'
    runs-on: ubuntu-latest
    outputs:
      should_build: ${{ steps.decide.outputs.should_build }}
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: main
          fetch-depth: 0
          persist-credentials: false
      - name: Decide whether to build
        id: decide
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "Manual dispatch -- always build"
            echo "should_build=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          SINCE=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ')
          COUNT=$(git log --oneline --since="$SINCE" | wc -l)
          echo "Commits on main since $SINCE: $COUNT"
          if [ "$COUNT" -gt 0 ]; then
            echo "should_build=true" >> "$GITHUB_OUTPUT"
          else
            echo "No new commits -- skipping nightly build"
            echo "should_build=false" >> "$GITHUB_OUTPUT"
          fi

  versions:
    uses: ./.github/workflows/load-versions.yml

  ##############################################################################
  # START EC2 BUILD RUNNERS
  ##############################################################################
  start-build-runner:
    name: Start CPU-only EC2 runner for nightly build
    needs: [check-new-commits, versions]
    if: needs.check-new-commits.outputs.should_build == 'true'
    runs-on: ubuntu-latest
    permissions:
      id-token: write  # Required for AWS OIDC
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }}
    steps:
      - name: Stagger job starts to avoid API rate limits
        run: |
          DELAY=$((RANDOM % 16))
          echo "Delaying start by ${DELAY} seconds for Python ${MATRIX_PYTHON_VERSION}"
          sleep $DELAY
        env:
          MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
        with:
          role-to-assume: ${{ needs.versions.outputs.aws-role }}
          aws-region: ${{ needs.versions.outputs.aws-region }}
      - name: Start EC2 runner
        id: start-build-runner
        uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
        with:
          mode: start
          github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
          ec2-instance-type: m6a.xlarge
          availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }}
          label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}
          aws-resource-tags: >
            [
              {"Key": "RunnerLabel", "Value": "nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"},
              {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"},
              {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"},
              {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"},
              {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"}
            ]

  ##############################################################################
  # BUILD NIGHTLY WHEELS
  ##############################################################################
  nightly-build:
    name: fVDB Nightly Build
    needs: [start-build-runner, check-new-commits, versions]
    if: needs.check-new-commits.outputs.should_build == 'true'
    runs-on: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}
    container:
      image: nvidia/cuda:${{ matrix.cuda-image-tag }}-cudnn-devel-rockylinux8
      env:
        PYTHONPATH: ""
        CPM_SOURCE_CACHE: "/__w/cpm_cache"
        FORCE_CUDA: 1
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }}
    steps:
      - name: Install prerequisites
        run: |
          dnf install -y git wget procps-ng findutils gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}
          echo "source /opt/rh/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}/enable" > /etc/profile.d/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}.sh
        env:
          NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET: ${{ needs.versions.outputs.gcc-toolset }}

      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: main
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
        with:
          version: "${{ needs.versions.outputs.uv-version }}"

      - name: Set up Python
        run: |
          uv python install ${MATRIX_PYTHON_VERSION}
          uv venv
        env:
          MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}

      - name: Install CMake
        run: >
          wget -nv https://github.com/Kitware/CMake/releases/download/v${NEEDS_VERSIONS_OUTPUTS_CMAKE_VERSION}/cmake-${NEEDS_VERSIONS_OUTPUTS_CMAKE_VERSION}-linux-x86_64.sh &&
          mkdir /opt/cmake &&
          sh cmake-${NEEDS_VERSIONS_OUTPUTS_CMAKE_VERSION}-linux-x86_64.sh --prefix=/usr/local --skip-license &&
          cmake --version
        env:
          NEEDS_VERSIONS_OUTPUTS_CMAKE_VERSION: ${{ needs.versions.outputs.cmake-version }}

      - name: Install system dependencies
        run: dnf install -y zlib-devel libpng-devel pkgconfig unzip procps-ng findutils

      - name: Install pip dependencies
        run: |
          CUDA_TAG="cu$(echo "${MATRIX_CUDA_VERSION}" | tr -d '.')"
          TORCH_VERSION="${MATRIX_TORCH_VERSION}"
          echo "Installing PyTorch ${TORCH_VERSION} for ${CUDA_TAG}"
          # Pin the tested PyTorch via a constraints file so the unbounded torch in
          # build_requirements.txt resolves to the version recorded in versions.json.
          echo "torch==${NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION}" > "${RUNNER_TEMP}/torch-constraints.txt"
          uv pip install --no-cache -c "${RUNNER_TEMP}/torch-constraints.txt" -r env/build_requirements.txt --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG}
        env:
          MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }}
          MATRIX_TORCH_VERSION: ${{ matrix.torch-version }}
          NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION: ${{ needs.versions.outputs.torch-full-version }}

      - name: Set nightly version
        run: |
          TORCH_TAG="$(echo "${MATRIX_TORCH_VERSION}" | tr -d '.')"
          CUDA_TAG="$(echo "${MATRIX_CUDA_VERSION}" | tr -d '.')"
          # Anchor the nightly at the upcoming release recorded in pyproject.toml
          # (e.g. 0.5.0.dev0 -> 0.5.0) so PEP 440 ordering puts nightlies between
          # the previous final release and the next one.
          CURRENT_VERSION=$(grep -E '^version *=' pyproject.toml | head -n1 | sed -E 's/^version *= *"([^"]+)".*/\1/')
          BASE_VERSION=$(echo "${CURRENT_VERSION%%+*}" | sed -E 's/(\.dev[0-9]+|\.post[0-9]+|(a|b|c|rc)[0-9]+)+$//')
          if [ -z "$BASE_VERSION" ]; then
            echo "Failed to parse base version from pyproject.toml (got '$CURRENT_VERSION')" >&2
            exit 1
          fi
          NIGHTLY_VERSION="${BASE_VERSION}.dev$(date -u '+%Y%m%d')+pt${TORCH_TAG}.cu${CUDA_TAG}"
          sed -i -E "s/^version *= *\"[^\"]+\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
          echo "Nightly version: $NIGHTLY_VERSION"
          grep '^version' pyproject.toml
        env:
          MATRIX_TORCH_VERSION: ${{ matrix.torch-version }}
          MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }}

      - name: Build fvdb
        run: |
          source .venv/bin/activate
          ./build.sh wheel strip_symbols verbose --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PUBLISH}"
        env:
          NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PUBLISH: ${{ needs.versions.outputs.cuda-arch-publish }}

      - name: Repair wheel for manylinux
        run: |
          source .venv/bin/activate
          pip install auditwheel patchelf
          ldd --version | head -n1
          CUDA_MAJOR=$(echo "${MATRIX_CUDA_VERSION}" | cut -d. -f1)
          auditwheel repair dist/fvdb_core-*.whl -w dist/ \
            --exclude libcuda.so.1 \
            --exclude libcudart.so.${CUDA_MAJOR} \
            --exclude libcufile.so.1 \
            --exclude libnvrtc.so.${CUDA_MAJOR} \
            --exclude libnvToolsExt.so.1 \
            --exclude libcublas.so.${CUDA_MAJOR} \
            --exclude libcublasLt.so.${CUDA_MAJOR} \
            --exclude libcudnn.so.9 \
            --exclude libcufft.so.11 \
            --exclude libcurand.so.10 \
            --exclude libcusolver.so.11 \
            --exclude libcusparse.so.${CUDA_MAJOR} \
            --exclude libnvJitLink.so.${CUDA_MAJOR} \
            --exclude libtorch.so \
            --exclude libtorch_cpu.so \
            --exclude libtorch_cuda.so \
            --exclude libtorch_python.so \
            --exclude libc10.so \
            --exclude libc10_cuda.so \
            --exclude libmkl_core.so.2 \
            --exclude libmkl_intel_lp64.so.2 \
            --exclude libmkl_intel_thread.so.2 \
            --exclude libomp.so \
            --exclude libprotobuf.so.31 \
            || echo "auditwheel failed, keeping original wheel"
          if ls dist/fvdb_core-*-manylinux*.whl 1> /dev/null 2>&1; then
            rm dist/fvdb_core-*-linux_x86_64.whl
          fi
          ls -lh dist/
        env:
          MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }}

      - name: Upload wheel
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: fvdb-nightly-${{ matrix.python-version }}-torch${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}
          path: dist/fvdb_core-*.whl
          retention-days: 7

  ##############################################################################
  # PUBLISH NIGHTLY WHEELS TO S3 SIMPLE INDEX
  ##############################################################################
  publish-to-s3:
    name: Publish nightly wheels to S3
    needs: [nightly-build, versions]
    if: ${{ !cancelled() && needs.nightly-build.result == 'success' }}
    runs-on: ubuntu-latest
    permissions:
      id-token: write  # Required for AWS OIDC
    env:
      S3_BUCKET: fvdb-packages
      SIMPLE_PREFIX: simple-nightly
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
        with:
          role-to-assume: ${{ needs.versions.outputs.aws-role }}
          aws-region: ${{ needs.versions.outputs.aws-region }}

      - name: Download all built wheels
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: fvdb-nightly-*
          path: dist
          merge-multiple: true

      - name: Determine normalized project name from wheel
        id: proj
        run: |
          set -euo pipefail
          shopt -s nullglob
          FIRST_WHL=$(ls dist/*.whl | head -n1)
          if [ -z "${FIRST_WHL:-}" ]; then
            echo "No wheels found in dist/" >&2; exit 1
          fi
          BASE=$(basename "$FIRST_WHL")
          PROJECT=${BASE%%-*}
          NORM=$(echo "$PROJECT" | tr '[:upper:]' '[:lower:]' | sed -E 's/[-_.]+/-/g')
          echo "project_name=$PROJECT" >> $GITHUB_OUTPUT
          echo "normalized=$NORM" >> $GITHUB_OUTPUT
          echo "Project: $PROJECT, Normalized: $NORM"

      - name: Prune nightly wheels older than 30 days
        run: |
          set -euo pipefail
          CUTOFF=$(date -u -d '30 days ago' '+%Y-%m-%d')
          echo "Pruning nightly wheels uploaded before $CUTOFF"
          LISTING=$(aws s3 ls "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/" 2>/dev/null || true)
          if [ -z "$LISTING" ]; then
            echo "No existing wheels found (first run or empty prefix) -- nothing to prune"
            exit 0
          fi
          DELETED=0
          while read -r line; do
            DATE_PART=$(echo "$line" | awk '{print $1}')
            FILE_PART=$(echo "$line" | awk '{print $4}')
            if [[ "$FILE_PART" == *.whl ]] && [[ "$DATE_PART" < "$CUTOFF" ]]; then
              echo "Removing: $FILE_PART (uploaded $DATE_PART)"
              aws s3 rm "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/$FILE_PART" --only-show-errors
              DELETED=$((DELETED + 1))
            fi
          done <<< "$LISTING"
          echo "Pruned $DELETED old nightly wheel(s)"
        env:
          STEPS_PROJ_OUTPUTS_NORMALIZED: ${{ steps.proj.outputs.normalized }}

      - name: Upload wheels to S3 project directory
        run: |
          set -euo pipefail
          DEST="s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/"
          echo "Uploading wheels to $DEST"
          shopt -s nullglob
          upload_failed=0
          for whl in dist/*.whl; do
            echo "Uploading $whl"
            if aws s3 cp "$whl" "$DEST"; then
              echo "Successfully uploaded $whl"
            else
              echo "FAILED to upload $whl"
              upload_failed=1
            fi
          done
          if [ $upload_failed -eq 1 ]; then
            echo "ERROR: One or more wheel uploads failed"
            exit 1
          fi
          echo "All wheels uploaded successfully"
          sleep 2
        env:
          STEPS_PROJ_OUTPUTS_NORMALIZED: ${{ steps.proj.outputs.normalized }}

      - name: Generate project index.html
        run: |
          set -euo pipefail
          OUTDIR="dist/index-gen/${STEPS_PROJ_OUTPUTS_NORMALIZED}"
          mkdir -p "$OUTDIR"
          INDEX_FILE="$OUTDIR/index.html"
          TITLE="Links for ${STEPS_PROJ_OUTPUTS_PROJECT_NAME}"
          wheels=$(aws s3 ls "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/" | { grep -E '\.whl$' || true; } | awk '{print $4}')

          echo "Found wheels in S3:"
          echo "$wheels"

          if [ -z "$wheels" ]; then
            echo "WARNING: No wheels found in S3 at s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/"
            echo "This may indicate an upload failure or timing issue"
          fi

          {
            echo "<!DOCTYPE html>";
            echo "<html><head><meta charset=\"utf-8\"><title>${TITLE}</title></head><body>";
            echo "<h1>${TITLE}</h1>";
            wheel_count=0
            for whl in $wheels; do
              wheel_count=$((wheel_count + 1))
              echo "Processing wheel ${wheel_count}: $whl" >&2
              if [ -f "dist/$whl" ]; then
                sha=$(sha256sum "dist/$whl" | awk '{print $1}')
                echo "  Computed SHA256 from local file: ${sha:0:16}..." >&2
              else
                echo "  Downloading $whl to compute hash..." >&2
                if aws s3 cp "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/$whl" "/tmp/$whl" --only-show-errors; then
                  sha=$(sha256sum "/tmp/$whl" | awk '{print $1}')
                  rm "/tmp/$whl"
                  echo "  Computed SHA256 from S3 file: ${sha:0:16}..." >&2
                else
                  echo "  ERROR: Failed to download $whl from S3" >&2
                  exit 1
                fi
              fi
              echo "<a href=\"${whl}#sha256=${sha}\">${whl}</a><br/>";
            done;
            echo "</body></html>";
          } > "$INDEX_FILE"

          echo "Total wheels processed: $wheel_count"
          echo "Generated index.html:"
          cat "$INDEX_FILE"

          aws s3 cp "$INDEX_FILE" "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/${STEPS_PROJ_OUTPUTS_NORMALIZED}/index.html" \
            --content-type text/html \
            --cache-control "public, max-age=60, must-revalidate" \
            --only-show-errors
          echo "Index uploaded successfully"
        env:
          STEPS_PROJ_OUTPUTS_NORMALIZED: ${{ steps.proj.outputs.normalized }}
          STEPS_PROJ_OUTPUTS_PROJECT_NAME: ${{ steps.proj.outputs.project_name }}

      - name: Generate root simple index.html
        run: |
          set -euo pipefail
          TMPDIR=$(mktemp -d)
          ROOT_INDEX="$TMPDIR/index.html"
          echo "Building root simple index from s3://${S3_BUCKET}/${SIMPLE_PREFIX}/"
          projects=$(aws s3 ls "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/" | awk '{print $2}' | sed 's:/$::')
          TITLE="Simple index"
          {
            echo "<!DOCTYPE html>";
            echo "<html><head><meta charset=\"utf-8\"><title>${TITLE}</title></head><body>";
            echo "<h1>${TITLE}</h1>";
            for p in $projects; do
              echo "<a href=\"${p}/\">${p}</a><br/>";
            done;
            echo "</body></html>";
          } > "$ROOT_INDEX"
          aws s3 cp "$ROOT_INDEX" "s3://${S3_BUCKET}/${SIMPLE_PREFIX}/index.html" \
            --content-type text/html \
            --cache-control "public, max-age=60, must-revalidate" \
            --only-show-errors

      - name: Invalidate CloudFront cache for simple-index paths
        run: |
          set -euo pipefail
          DOMAIN="${NEEDS_VERSIONS_OUTPUTS_AWS_CLOUDFRONT_DISTRIBUTION_DOMAIN}"
          if [ -z "$DOMAIN" ] || [ "$DOMAIN" = "null" ]; then
            echo "No CloudFront distribution domain configured (aws.cloudfront_distribution_domain in versions.json); skipping invalidation"
            exit 0
          fi
          DIST_ID=$(aws cloudfront list-distributions \
            --query "DistributionList.Items[?DomainName=='$DOMAIN'].Id | [0]" \
            --output text 2>/dev/null || echo "")
          if [ -z "$DIST_ID" ] || [ "$DIST_ID" = "None" ]; then
            echo "WARNING: Could not find CloudFront distribution for domain $DOMAIN; skipping invalidation"
            echo "(IAM role may need cloudfront:ListDistributions permission)"
            exit 0
          fi
          NORM="${STEPS_PROJ_OUTPUTS_NORMALIZED}"
          # Invalidate both the trailing-slash directory paths (what pip and
          # browsers actually request) and the underlying index.html object
          # paths. CloudFront caches each request URI independently, so
          # invalidating only one form leaves the other stale.
          PATHS=(
            "/${SIMPLE_PREFIX}/"
            "/${SIMPLE_PREFIX}/index.html"
            "/${SIMPLE_PREFIX}/${NORM}/"
            "/${SIMPLE_PREFIX}/${NORM}/index.html"
          )
          echo "Invalidating distribution $DIST_ID paths: ${PATHS[*]}"
          if ! aws cloudfront create-invalidation \
            --distribution-id "$DIST_ID" \
            --paths "${PATHS[@]}" \
            --query 'Invalidation.{Id:Id,Status:Status,CreateTime:CreateTime}' \
            --output table; then
            echo "WARNING: Failed to create CloudFront invalidation for distribution $DIST_ID; continuing without failing publish"
            echo "(IAM role may need cloudfront:CreateInvalidation permission)"
            exit 0
          fi
        env:
          NEEDS_VERSIONS_OUTPUTS_AWS_CLOUDFRONT_DISTRIBUTION_DOMAIN: ${{ needs.versions.outputs.aws-cloudfront-distribution-domain }}
          STEPS_PROJ_OUTPUTS_NORMALIZED: ${{ steps.proj.outputs.normalized }}

  ##############################################################################
  # STOP EC2 BUILD RUNNERS
  ##############################################################################
  stop-build-runner:
    name: Stop CPU-only EC2 runner for nightly build
    needs:
      - start-build-runner
      - nightly-build
      - versions
    runs-on: ubuntu-latest
    if: ${{ always() && needs.start-build-runner.result != 'skipped' }}
    permissions:
      id-token: write  # Required for AWS OIDC
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
        with:
          role-to-assume: ${{ needs.versions.outputs.aws-role }}
          aws-region: ${{ needs.versions.outputs.aws-region }}
      - name: Find EC2 instance ID by label
        id: find-instance
        run: |
          LABEL="nightly-${MATRIX_PYTHON_VERSION}-pt${MATRIX_TORCH_VERSION}-cu${MATRIX_CUDA_VERSION}-${{ github.run_id }}"
          echo "Looking for instance with RunnerLabel: $LABEL"

          INSTANCE_ID=$(aws ec2 describe-instances \
            --filters "Name=tag:RunnerLabel,Values=$LABEL" "Name=instance-state-name,Values=running,pending,stopping,stopped" \
            --query 'Reservations[0].Instances[0].InstanceId' \
            --output text)

          if [ "$INSTANCE_ID" == "None" ] || [ -z "$INSTANCE_ID" ]; then
            echo "ERROR: No instance found with RunnerLabel=$LABEL"
            echo "instance-id=" >> $GITHUB_OUTPUT
            exit 1
          else
            echo "Found instance: $INSTANCE_ID"
            echo "instance-id=$INSTANCE_ID" >> $GITHUB_OUTPUT
          fi
        env:
          MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}
          MATRIX_TORCH_VERSION: ${{ matrix.torch-version }}
          MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }}
      - name: Stop EC2 runner
        if: steps.find-instance.outputs.instance-id != ''
        uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
        with:
          mode: stop
          github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
          label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}
          ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }}

  ##############################################################################
  # START NIGHTLY TESTS GPU RUNNER
  #
  # The PR workflows build with NANOVDB_USE_SYNC_CUDA_MALLOC because they test on
  # fractional-GPU (vGPU) runners, which cannot use cudaMallocAsync. This whole
  # GPU instance is therefore the only place the default asynchronous NanoVDB
  # allocation path gets exercised, so it runs once a night against the published
  # nightly wheel instead of on every pull request.
  ##############################################################################
  start-tests-gpu-runner:
    name: Start EC2 GPU runner for nightly tests
    needs: [nightly-build, versions]
    if: ${{ !cancelled() && needs.nightly-build.result == 'success' }}
    runs-on: ubuntu-latest
    permissions:
      id-token: write  # Required for AWS OIDC
    outputs:
      label: ${{ steps.start-tests-gpu-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
        with:
          role-to-assume: ${{ needs.versions.outputs.aws-role }}
          aws-region: ${{ needs.versions.outputs.aws-region }}
      - name: Start EC2 GPU runner
        id: start-tests-gpu-runner
        uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
        with:
          mode: start
          github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
          ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU
          availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }}

  ##############################################################################
  # RUN NIGHTLY UNIT TESTS
  ##############################################################################
  nightly-unit-tests:
    name: fVDB Nightly Unit Tests
    needs: [start-tests-gpu-runner, versions]
    runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }}
    container:
      image: nvidia/cuda:${{ needs.versions.outputs.cuda-default-patch }}-cudnn-devel-${{ needs.versions.outputs.docker-os-rocky }}
      env:
        PYTHONPATH: ""
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: Install prerequisites
        run: |
          dnf install -y git wget procps-ng findutils gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}
          echo "source /opt/rh/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}/enable" > /etc/profile.d/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}.sh
        env:
          NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET: ${{ needs.versions.outputs.gcc-toolset }}

      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: main
          persist-credentials: false

      - name: Install uv
        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
        with:
          version: "${{ needs.versions.outputs.uv-version }}"

      - name: Set up Python
        run: |
          uv python install ${NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT}
          uv venv
        env:
          NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT: ${{ needs.versions.outputs.python-default }}

      - name: Install system dependencies
        run: dnf install -y zlib-devel libpng-devel

      - name: Install pip dependencies
        run: |
          CUDA_TAG="cu$(echo "${NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT}" | tr -d '.')"
          echo "torch==${NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION}" > "${RUNNER_TEMP}/torch-constraints.txt"
          uv pip install --no-cache -c "${RUNNER_TEMP}/torch-constraints.txt" -r env/test_requirements.txt --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} --index-strategy unsafe-best-match
          uv pip install --no-cache setuptools
        env:
          NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT: ${{ needs.versions.outputs.cuda-default }}
          NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION: ${{ needs.versions.outputs.torch-full-version }}

      # Test the default Python/CUDA cell of the publish matrix; the other cells
      # differ only in interpreter and CUDA minor version.
      - name: Download package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: fvdb-nightly-${{ needs.versions.outputs.python-default }}-torch${{ needs.versions.outputs.torch-version }}-cu${{ needs.versions.outputs.cuda-default }}
          path: ./dist

      - name: Install package
        run: |
          source .venv/bin/activate
          pip install ./dist/*.whl

      - name: Run unit tests
        run: |
          source .venv/bin/activate
          NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
          pip install "$NANOVDB_EDITOR_SPEC"
          cd tests;
          pytest -v unit

      - name: Cleanup
        if: always()
        run: |
          echo "Cleaning up /__w/_temp directory"
          rm -rf /__w/_temp/*
          echo "Cleanup completed"

  ##############################################################################
  # STOP NIGHTLY TESTS GPU RUNNER
  ##############################################################################
  stop-tests-gpu-runner:
    name: Stop GPU EC2 runner for nightly tests
    needs:
      - start-tests-gpu-runner # required to get output from the start-tests-gpu-runner job
      - nightly-unit-tests # required to wait when the main job is done
      - versions
    runs-on: ubuntu-latest
    permissions:
      id-token: write  # Required for AWS OIDC
    # required to stop the runner even if the error happened in the previous jobs,
    # but only if the start-tests-gpu-runner job was not skipped
    if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
        with:
          role-to-assume: ${{ needs.versions.outputs.aws-role }}
          aws-region: ${{ needs.versions.outputs.aws-region }}
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
        with:
          mode: stop
          github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
          label: ${{ needs.start-tests-gpu-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }}