Skip to content
fVDB Conda Build and Test

CI: g6f instances #1802

Sign in to view logs

CI: g6f instances

CI: g6f instances #1802

Workflow file for this run

.github/workflows/tests.yml at 49dd48e
name: fVDB Conda Build and Test
on:
# zizmor: ignore[dangerous-triggers] pull_request_target is required to provision AWS runners; privileged credentials are scoped to runner lifecycle jobs.
pull_request_target:
branches:
- '**'
workflow_dispatch:
inputs:
branch:
description: "Branch to build"
required: true
default: "main"
# Allow subsequent pushes to the same PR or REF to cancel any previous jobs.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
contents: read
deployments: read
pull-requests: read
issues: read
jobs:
##############################################################################
# CHECK FOR RELEVANT CHANGES
##############################################################################
check-changes:
uses: ./.github/workflows/check-changes.yml
versions:
uses: ./.github/workflows/load-versions.yml
##############################################################################
# BUILD FVDB
##############################################################################
start-build-runner:
name: Start CPU-only EC2 runner for build
needs: [check-changes, versions]
runs-on: ubuntu-latest
permissions:
id-token: write # Required for AWS OIDC
if: >-
needs.check-changes.outputs.should_test == 'true'
&& (github.event.pull_request.draft == false || github.event_name != 'pull_request_target')
outputs:
label: ${{ steps.start-build-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-build-runner.outputs.ec2-instance-id }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
with:
role-to-assume: ${{ needs.versions.outputs.aws-role }}
aws-region: ${{ needs.versions.outputs.aws-region }}
- name: Start EC2 runner
id: start-build-runner
uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
ec2-instance-type: m6a.8xlarge
availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }}
fvdb-build:
name: fVDB Build (Conda)
needs: [start-build-runner, versions] # required to start the main job when the runner is ready
runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner
container:
image: rockylinux/rockylinux:8
env:
PYTHONPATH: ""
CPM_SOURCE_CACHE: "/__w/cpm_cache"
CONDA_OVERRIDE_CUDA: ${{ needs.versions.outputs.cuda-conda-override }}
options: --rm
defaults:
run:
shell: bash -el {0}
steps:
- name: Install system dependencies
run: dnf install -y git procps-ng findutils
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 0 # Fetch all history for all branches
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
persist-credentials: false
- name: Fetch PR branch
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git config --global --add safe.directory "$(pwd)"
git fetch origin ${{ github.event.pull_request.head.sha }}
git branch pr_branch FETCH_HEAD
- name: Merge PR branch into base
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git merge pr_branch
# - name: Simulate merge for push events
# if: github.event_name == 'push'
# run: |
# echo "Push event detected - simulating merge with target branch"
# git config --global user.name "github-actions[bot]"
# git config --global user.email "github-actions[bot]@users.noreply.github.com"
# git config --global --add safe.directory "$(pwd)"
# # Get the target branch (main in this case, since push is only on main)
# TARGET_BRANCH="main"
# CURRENT_COMMIT="${{ github.sha }}"
# echo "Creating merge simulation: merging $CURRENT_COMMIT into $TARGET_BRANCH"
# git fetch origin $TARGET_BRANCH
# git checkout $TARGET_BRANCH
# git merge --no-ff $CURRENT_COMMIT --message "CI: Simulating merge of $CURRENT_COMMIT into $TARGET_BRANCH"
- name: Set up fvdb_build Conda env
uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
with:
post-cleanup: 'all'
environment-name: fvdb_build
environment-file: env/build_environment.yml
- name: Build fvdb
run: |
micromamba activate fvdb_build
NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
pip install "$NANOVDB_EDITOR_SPEC"
# sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances
# where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc.
./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list '8.9+PTX'
- name: Upload wheel
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: fvdb-test-package
path: dist/*.whl
retention-days: 2
#NOTE: This tar step is here because directly uploading the build directory
# wasn't working due to losing executable permissions on the files.
- name: Tar build directory
run: tar -cvf fvdb-gtest.tar build/
- name: Upload gtests
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: fvdb-gtest
path: fvdb-gtest.tar
retention-days: 2
fvdb-build-stop-runner:
name: Stop CPU-only EC2 runner for build
needs:
- start-build-runner # required to get output from the start-build-runner job
- fvdb-build # required to wait when the main job is done
- versions
runs-on: ubuntu-latest
permissions:
id-token: write # Required for AWS OIDC
# required to stop the runner even if the error happened in the previous jobs, but only if the
# start-build-runner job was not skipped
if: ${{ always() && needs.start-build-runner.result != 'skipped' }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
with:
role-to-assume: ${{ needs.versions.outputs.aws-role }}
aws-region: ${{ needs.versions.outputs.aws-region }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
with:
mode: stop
github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
label: ${{ needs.start-build-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }}
##############################################################################
# START FVDB TESTS GPU RUNNER
##############################################################################
start-tests-gpu-runner:
name: Start EC2 GPU runner for tests
needs: [fvdb-build, versions]
runs-on: ubuntu-latest
permissions:
id-token: write # Required for AWS OIDC
outputs:
label: ${{ steps.start-tests-gpu-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
with:
role-to-assume: ${{ needs.versions.outputs.aws-role }}
aws-region: ${{ needs.versions.outputs.aws-region }}
- name: Start EC2 GPU runner
id: start-tests-gpu-runner
uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU
availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }}
##############################################################################
# RUN FVDB GTESTS
##############################################################################
fvdb-gtests:
needs: [start-tests-gpu-runner, versions] # required to start the main job when the runner is ready
name: fVDB GTests (Conda)
runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} # run the job on the newly created runner
container:
image: nvidia/cuda:${{ needs.versions.outputs.cuda-130-patch }}-base-${{ needs.versions.outputs.docker-os-rocky }}
env:
PYTHONPATH: ""
CPM_SOURCE_CACHE: "/__w/cpm_cache"
CONDA_PKGS_DIRS: "/opt/conda-pkgs-cache"
options: --rm -v /tmp/conda-pkgs-cache:/opt/conda-pkgs-cache
defaults:
run:
shell: bash -el {0}
steps:
- name: Install system dependencies
run: dnf install -y git procps-ng findutils
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 0 # Fetch all history for all branches
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
persist-credentials: false
- name: Fetch PR branch
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git config --global --add safe.directory "$(pwd)"
git fetch origin ${{ github.event.pull_request.head.sha }}
git branch pr_branch FETCH_HEAD
- name: Merge PR branch into base
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git merge pr_branch
- name: Set up fvdb_test Conda env
uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
with:
cache-environment: false
post-cleanup: 'all'
environment-name: fvdb_test
environment-file: env/test_environment.yml
- name: Download gtests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: fvdb-gtest
path: .
- name: Extract tar
run: |
tar -xvf fvdb-gtest.tar
- name: Run tests
run: |
micromamba activate fvdb_test
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib
./build.sh ctest
##############################################################################
# RUN FVDB UNIT TESTS
##############################################################################
fvdb-unit-tests:
needs: [start-tests-gpu-runner, versions] # required to start the main job when the runner is ready
name: fVDB Unit Tests (Conda)
runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} # run the job on the newly created runner
container:
image: nvidia/cuda:${{ needs.versions.outputs.cuda-130-patch }}-base-${{ needs.versions.outputs.docker-os-rocky }}
env:
PYTHONPATH: ""
CPM_SOURCE_CACHE: "/__w/cpm_cache"
CONDA_PKGS_DIRS: "/opt/conda-pkgs-cache"
options: --rm -v /tmp/conda-pkgs-cache:/opt/conda-pkgs-cache
defaults:
run:
shell: bash -el {0}
steps:
- name: Install system dependencies
run: dnf install -y git procps-ng findutils
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 0 # Fetch all history for all branches
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
persist-credentials: false
- name: Fetch PR branch
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git config --global --add safe.directory "$(pwd)"
git fetch origin ${{ github.event.pull_request.head.sha }}
git branch pr_branch FETCH_HEAD
- name: Merge PR branch into base
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git merge pr_branch
- name: Set up fvdb_test Conda env
uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
with:
cache-environment: false
post-cleanup: 'all'
environment-name: fvdb_test
environment-file: env/test_environment.yml
- name: Download package
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: fvdb-test-package
path: ./dist
- name: Install package
run: |
micromamba activate fvdb_test
pip install ./dist/*.whl
- name: Run tests
env:
# The whole parametrized matrix shares one process, so the caching allocator's
# reserved pool only ever grows and its high-water mark ends up far above any
# single test's live footprint. Freeing straight back to the driver keeps the
# suite within the runner's VRAM, at the cost of a cudaMalloc/cudaFree per
# allocation. See
# https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-alloc-conf
PYTORCH_NO_CUDA_MEMORY_CACHING: 1
run: |
micromamba activate fvdb_test
NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
pip install "$NANOVDB_EDITOR_SPEC"
cd tests;
pytest -v unit
##############################################################################
# RUN FVDB DOCUMENTATION TESTS
##############################################################################
fvdb-docs-test:
needs: [start-tests-gpu-runner, versions] # required to start the main job when the runner is ready
name: fVDB Doc Tests (Conda)
runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} # run the job on the newly created runner
container:
image: nvidia/cuda:${{ needs.versions.outputs.cuda-130-patch }}-base-${{ needs.versions.outputs.docker-os-rocky }}
env:
PYTHONPATH: ""
CONDA_PKGS_DIRS: "/opt/conda-pkgs-cache"
options: --rm -v /tmp/conda-pkgs-cache:/opt/conda-pkgs-cache
defaults:
run:
shell: bash -el {0}
steps:
- name: Install system dependencies
run: dnf install -y git procps-ng findutils
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
fetch-depth: 0 # Fetch all history for all branches
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.sha }}
persist-credentials: false
- name: Fetch PR branch
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git config --global --add safe.directory "$(pwd)"
git fetch origin ${{ github.event.pull_request.head.sha }}
git branch pr_branch FETCH_HEAD
- name: Merge PR branch into base
if: github.event_name == 'pull_request_target'
run: |
cd $GITHUB_WORKSPACE
git merge pr_branch
- name: Set up fvdb_test Conda env
uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0
with:
cache-environment: false
post-cleanup: 'all'
environment-name: fvdb_test
environment-file: env/test_environment.yml
- name: Download package
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: fvdb-test-package
path: ./dist
- name: Install package
run: |
micromamba activate fvdb_test
pip install ./dist/*.whl
- name: Run markdown docs tests
run: |
micromamba activate fvdb_test
pytest --markdown-docs docs
- name: Run API reference validation
run: |
micromamba activate fvdb_test
pytest tests/test_teachme_api_refs.py -v
- name: Run notebook tests
run: |
micromamba activate fvdb_test
pytest --nbmake notebooks/
- name: Run example tests
run: |
micromamba activate fvdb_test
pytest tests/test_examples.py -v
##############################################################################
# STOP FVDB TESTS GPU RUNNER
##############################################################################
fvdb-tests-stop-gpu-runner:
name: Stop GPU EC2 runner for tests
needs:
- start-tests-gpu-runner # required to get output from the start-tests-gpu-runner job
- fvdb-gtests # required to wait when the main job is done
- fvdb-unit-tests # required to wait when the main job is done
- fvdb-docs-test # required to wait when the main job is done
- versions
runs-on: ubuntu-latest
permissions:
id-token: write # Required for AWS OIDC
# required to stop the runner even if the error happened in the previous jobs
# but only if the start-tests-gpu-runner job was not skipped
if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
with:
role-to-assume: ${{ needs.versions.outputs.aws-role }}
aws-region: ${{ needs.versions.outputs.aws-region }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
with:
mode: stop
github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
label: ${{ needs.start-tests-gpu-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }}