nightly.yml

name: fVDB Nightly Build and Tests

on:
  schedule:
    - cron: "01 15 * * *"  # Runs every day at 4am pacific/11am UTC
  workflow_dispatch:


permissions:
  contents: read
  deployments: read
  pull-requests: read
  issues: read
  # Need ID token write permission to use OIDC
  id-token: write

jobs:
  ##############################################################################
  # BUILD FVDB
  ##############################################################################
  start-build-runner:
    name: Start CPU-only EC2 runner for build
    runs-on: ubuntu-latest
    if: github.repository == 'openvdb/fvdb-core'  # Prevent running in forks
    outputs:
      label: ${{ steps.start-build-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-build-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Start EC2 runner
        id: start-build-runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ami-0e14a711dad782a70
          ec2-instance-type: m6a.8xlarge
          subnet-id: subnet-03f2320d6e6e0005b
          security-group-id: sg-0cd08bd89d6212223

  fvdb-build:
    name: fVDB Build
    needs: start-build-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner
    container:
      image: aswf/ci-openvdb:2024-clang17.2
      env:
        PYTHONPATH: ""
        CPM_SOURCE_CACHE: "/__w/cpm_cache"
        CONDA_OVERRIDE_CUDA: "12.9"  # this is to build an environment on machines that lack a CUDA device and needs to be >= the CUDA version in the build_environment.yml file
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches
          ref: 'main'


      - name: Set up fvdb_build Conda env
        uses: mamba-org/setup-micromamba@v2
        with:
          post-cleanup: 'all'
          environment-name: fvdb_build
          environment-file: env/build_environment.yml

      - name: Build fvdb
        run: |
          micromamba activate fvdb_build
          ./build.sh wheel verbose gtests benchmarks --cuda-arch-list '8.9+PTX'

      - name: Upload wheel
        uses: actions/upload-artifact@v4
        with:
            name: fvdb-test-package
            path: dist/*.whl
            retention-days: 2

      #NOTE: This tar step is here because directly uploading the build directory
      #      wasn't working due to losing executable permissions on the files.
      - name: Tar build directory
        run: tar -cvf fvdb-gtest.tar build/

      - name: Upload gtests
        uses: actions/upload-artifact@v4
        with:
            name: fvdb-gtest
            path: fvdb-gtest.tar
            retention-days: 2

  fvdb-build-stop-runner:
    name: Stop CPU-only EC2 runner for build
    needs:
      - start-build-runner # required to get output from the start-build-runner job
      - fvdb-build # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-build-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }}


  ##############################################################################
  # START FVDB BENCHMARKS GPU RUNNER
  ##############################################################################
  start-benchmarks-gpu-runner:
    name: Start EC2 GPU runner for benchmarks
    needs: fvdb-build
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-benchmarks-gpu-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-benchmarks-gpu-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Start EC2 GPU runner
        id: start-benchmarks-gpu-runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ami-0f5c0b65fde08ae43
          ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU
          subnet-id: subnet-03f2320d6e6e0005b
          security-group-id: sg-0cd08bd89d6212223

  ##############################################################################
  # RUN FVDB BENCHMARKS
  ##############################################################################
  fvdb-benchmarks:
    needs: start-benchmarks-gpu-runner # required to start the main job when the runner is ready
    name: fVDB Benchmarks
    runs-on: ${{ needs.start-benchmarks-gpu-runner.outputs.label }} # run the job on the newly created runner
    container:
      image: aswf/ci-openvdb:2024-clang17.2
      env:
        PYTHONPATH: ""
        CPM_SOURCE_CACHE: "/__w/cpm_cache"
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      ### Get the PR branch and apply changes to the target branch
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches
          ref: 'main'

      - name: Set up fvdb_test Conda env
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          conda-remove-defaults: "true"
          activate-environment: fvdb_test
          environment-file: fvdb/env/test_environment.yml

      - name: Download package
        uses: actions/download-artifact@v4
        with:
            name: fvdb-test-package
            path: ./dist

      - name: Install package
        run: |
            conda activate fvdb_test
            pip install ./dist/*.whl

      - name: Disable git ownership verification
        run: |
          git config --global --add safe.directory "$(pwd)"

      - name: Run benchmarks
        run: |
          cd fvdb/tests;
          pytest benchmark --benchmark-json benchmark/output.json

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
            name: Python Benchmark with pytest-benchmark
            tool: 'pytest'
            output-file-path: tests/benchmark/output.json
            # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
            github-token: ${{ secrets.GITHUB_TOKEN }}
            auto-push: true
            # Show alert with commit comment on detecting possible performance regression
            alert-threshold: '200%'
            comment-on-alert: true
            fail-on-alert: true
            alert-comment-cc-users: '@openvdb/fvdb-dev'

      - name: Cleanup
        if: always()
        run: |
          echo "Cleaning up /__w/_temp directory"
          sudo rm -rf /__w/_temp/*
          echo "Cleanup completed"

  ##############################################################################
  # STOP FVDB BENCHMARKS GPU RUNNER
  ##############################################################################
  fvdb-benchmarks-stop-gpu-runner:
    name: Stop GPU EC2 runner for benchmarks
    needs:
      - start-benchmarks-gpu-runner # required to get output from the start-benchmarks-gpu-runner job
      - fvdb-benchmarks # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-benchmarks-gpu-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-benchmarks-gpu-runner.outputs.ec2-instance-id }}