From 1c5d2c372b3a4db9f42d7a7c8f8b3fb66008c59d Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Tue, 21 Jul 2026 16:55:19 +1200 Subject: [PATCH 1/9] g6f instance setup Signed-off-by: Jonathan Swartz --- .github/versions.json | 1 + .github/workflows/tests.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/versions.json b/.github/versions.json index 19d0ff41..28eeea65 100644 --- a/.github/versions.json +++ b/.github/versions.json @@ -35,6 +35,7 @@ "region": "us-east-2", "cpu_ami": "ami-0e14a711dad782a70", "gpu_ami": "ami-0f5c0b65fde08ae43", + "fgpu_ami": "ami-0b5dca767eafff9d8", "subnet": "subnet-03f2320d6e6e0005b", "security_group": "sg-0cd08bd89d6212223", "cloudfront_distribution_domain": "d36m13axqqhiit.cloudfront.net" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a97bb300..e3f7e4be 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} ec2-image-id: ${{ needs.versions.outputs.aws-cpu-ami }} - ec2-instance-type: m6a.8xlarge + ec2-instance-type: m6a.16xlarge subnet-id: ${{ needs.versions.outputs.aws-subnet }} security-group-id: ${{ needs.versions.outputs.aws-security-group }} fvdb-build: @@ -207,7 +207,7 @@ jobs: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} ec2-image-id: ${{ needs.versions.outputs.aws-gpu-ami }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU + ec2-instance-type: g6f.xlarge # 4 CPU-core, 1/8 L4 GPU subnet-id: ${{ needs.versions.outputs.aws-subnet }} security-group-id: ${{ needs.versions.outputs.aws-security-group }} From 2c667b2ea225a64f29c3b333e8fadf776e114389 Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Tue, 21 Jul 2026 17:04:05 +1200 Subject: [PATCH 2/9] Fix fgpu load-versions Signed-off-by: Jonathan Swartz --- .github/workflows/load-versions.yml | 4 ++++ .github/workflows/tests.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/load-versions.yml b/.github/workflows/load-versions.yml index 1c03b8aa..42e13a5b 100644 --- a/.github/workflows/load-versions.yml +++ b/.github/workflows/load-versions.yml @@ -50,6 +50,8 @@ on: value: ${{ jobs.load.outputs.aws-cpu-ami }} aws-gpu-ami: value: ${{ jobs.load.outputs.aws-gpu-ami }} + aws-fgpu-ami: + value: ${{ jobs.load.outputs.aws-fgpu-ami }} aws-subnet: value: ${{ jobs.load.outputs.aws-subnet }} aws-security-group: @@ -87,6 +89,7 @@ jobs: aws-region: ${{ steps.parse.outputs.aws-region }} aws-cpu-ami: ${{ steps.parse.outputs.aws-cpu-ami }} aws-gpu-ami: ${{ steps.parse.outputs.aws-gpu-ami }} + aws-fgpu-ami: ${{ steps.parse.outputs.aws-fgpu-ami }} aws-subnet: ${{ steps.parse.outputs.aws-subnet }} aws-security-group: ${{ steps.parse.outputs.aws-security-group }} aws-cloudfront-distribution-domain: ${{ steps.parse.outputs.aws-cloudfront-distribution-domain }} @@ -125,6 +128,7 @@ jobs: echo "aws-region=$(jq -r '.aws.region' $CFG)" echo "aws-cpu-ami=$(jq -r '.aws.cpu_ami' $CFG)" echo "aws-gpu-ami=$(jq -r '.aws.gpu_ami' $CFG)" + echo "aws-fgpu-ami=$(jq -r '.aws.fgpu_ami' $CFG)" echo "aws-subnet=$(jq -r '.aws.subnet' $CFG)" echo "aws-security-group=$(jq -r '.aws.security_group' $CFG)" echo "aws-cloudfront-distribution-domain=$(jq -r '.aws.cloudfront_distribution_domain // ""' $CFG)" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3f7e4be..44915390 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -206,7 +206,7 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-image-id: ${{ needs.versions.outputs.aws-gpu-ami }} + ec2-image-id: ${{ needs.versions.outputs.aws-fgpu-ami }} ec2-instance-type: g6f.xlarge # 4 CPU-core, 1/8 L4 GPU subnet-id: ${{ needs.versions.outputs.aws-subnet }} security-group-id: ${{ needs.versions.outputs.aws-security-group }} From 5b0d76c53c4870e563d791d747ee1500955b4f65 Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Tue, 21 Jul 2026 17:38:19 +1200 Subject: [PATCH 3/9] Insert an 'nvidia-smi' Signed-off-by: Jonathan Swartz --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44915390..a8ee30a1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -272,6 +272,9 @@ jobs: run: | tar -xvf fvdb-gtest.tar + - name: Show GPU info + run: nvidia-smi + - name: Run tests run: | micromamba activate fvdb_test @@ -341,6 +344,9 @@ jobs: micromamba activate fvdb_test pip install ./dist/*.whl + - name: Show GPU info + run: nvidia-smi + - name: Run tests run: | micromamba activate fvdb_test @@ -411,6 +417,9 @@ jobs: micromamba activate fvdb_test pip install ./dist/*.whl + - name: Show GPU info + run: nvidia-smi + - name: Run markdown docs tests run: | micromamba activate fvdb_test From 75b8a61ef941ea137c176729efb42161dd1a5100 Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Wed, 29 Jul 2026 15:08:54 +1200 Subject: [PATCH 4/9] Bump NanoVDB commit Signed-off-by: Jonathan Swartz --- src/cmake/get_nanovdb.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmake/get_nanovdb.cmake b/src/cmake/get_nanovdb.cmake index d8ce343a..23683730 100644 --- a/src/cmake/get_nanovdb.cmake +++ b/src/cmake/get_nanovdb.cmake @@ -4,7 +4,7 @@ CPMAddPackage( NAME nanovdb GITHUB_REPOSITORY AcademySoftwareFoundation/openvdb - GIT_TAG f9754140ba6031813b37d8e1b239ed0253ebd96d + GIT_TAG b7fc4fc7af73e84071b5f625482cfad4c50eb247 SOURCE_SUBDIR nanovdb/nanovdb DOWNLOAD_ONLY YES ) From 6e98a3ffc08687641ac47c34ce846edca67ffe6a Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Wed, 29 Jul 2026 15:15:21 +1200 Subject: [PATCH 5/9] Added sync_cuda_malloc to build.sh to compile with NANOVDB_USE_SYNC_CUDA_MALLOC and setup tests.yml to use this Signed-off-by: Jonathan Swartz --- .github/workflows/tests.yml | 15 ++++----------- README.md | 3 +++ build.sh | 6 ++++++ src/cmake/get_nanovdb.cmake | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8ee30a1..2e26a88a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} ec2-image-id: ${{ needs.versions.outputs.aws-cpu-ami }} - ec2-instance-type: m6a.16xlarge + ec2-instance-type: m6a.8xlarge subnet-id: ${{ needs.versions.outputs.aws-subnet }} security-group-id: ${{ needs.versions.outputs.aws-security-group }} fvdb-build: @@ -134,7 +134,9 @@ jobs: micromamba activate fvdb_build NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) pip install "$NANOVDB_EDITOR_SPEC" - ./build.sh wheel verbose gtests benchmarks --cuda-arch-list '8.9+PTX' + # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances + # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc. + ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list '8.9+PTX' - name: Upload wheel uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -272,9 +274,6 @@ jobs: run: | tar -xvf fvdb-gtest.tar - - name: Show GPU info - run: nvidia-smi - - name: Run tests run: | micromamba activate fvdb_test @@ -344,9 +343,6 @@ jobs: micromamba activate fvdb_test pip install ./dist/*.whl - - name: Show GPU info - run: nvidia-smi - - name: Run tests run: | micromamba activate fvdb_test @@ -417,9 +413,6 @@ jobs: micromamba activate fvdb_test pip install ./dist/*.whl - - name: Show GPU info - run: nvidia-smi - - name: Run markdown docs tests run: | micromamba activate fvdb_test diff --git a/README.md b/README.md index 3cc68acd..692a3391 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,9 @@ The build script supports the following build modifiers: - `debug`: Build in debug mode with full debug symbols and no optimizations. - `strip_symbols`: Strip symbols from the build (will be ignored if debug is enabled). - `verbose`: Enable verbose build output for pip and CMake. +- `sync_cuda_malloc`: Compile with `-DNANOVDB_USE_SYNC_CUDA_MALLOC` so NanoVDB allocates with + `cudaMalloc`/`cudaFree` instead of the async variants. Required on vGPU slices, where async + allocation is unavailable because it depends on GPU unified memory. ### Running Tests diff --git a/build.sh b/build.sh index 1c441fb1..aa780cda 100755 --- a/build.sh +++ b/build.sh @@ -26,6 +26,8 @@ usage() { echo " debug Build in debug mode with full debug symbols and no optimizations." echo " lineinfo Enable CUDA lineinfo (sets FVDB_LINEINFO=ON)." echo " strip_symbols Strip symbols from the build (will be ignored if debug is enabled)." + echo " sync_cuda_malloc Compile with -DNANOVDB_USE_SYNC_CUDA_MALLOC so NanoVDB uses" + echo " synchronous cudaMalloc/cudaFree (required on vGPU slices)." echo " verbose Enable verbose build output for pip and CMake." echo " trace Enable CMake trace output for debugging configuration." echo "" @@ -311,6 +313,10 @@ while (( "$#" )); do echo "Enabling strip symbols build" CONFIG_SETTINGS+=" --config-settings=cmake.define.FVDB_STRIP_SYMBOLS=ON" is_config_arg_handled=true + elif [[ "$1" == "sync_cuda_malloc" ]]; then + echo "Enabling synchronous NanoVDB CUDA allocation (-DNANOVDB_USE_SYNC_CUDA_MALLOC)" + CONFIG_SETTINGS+=" --config-settings=cmake.define.FVDB_USE_SYNC_CUDA_MALLOC=ON" + is_config_arg_handled=true fi fi diff --git a/src/cmake/get_nanovdb.cmake b/src/cmake/get_nanovdb.cmake index 23683730..731a34f2 100644 --- a/src/cmake/get_nanovdb.cmake +++ b/src/cmake/get_nanovdb.cmake @@ -15,3 +15,19 @@ if(nanovdb_ADDED) add_library(nanovdb INTERFACE) target_include_directories(nanovdb INTERFACE ${nanovdb_SOURCE_DIR}/nanovdb) endif() + +# nanovdb::util::cuda::mallocAsync resolves to either cudaMallocAsync or plain +# cudaMalloc depending on whether NANOVDB_USE_SYNC_CUDA_MALLOC is defined. The +# choice is made in a header, so the macro has to be visible to every C++/CUDA +# translation unit that includes NanoVDB or an inline function ends up with two +# different bodies across the build. Hence a directory-wide compile definition +# rather than a per-target one. +# +# Async allocation depends on GPU unified memory, which is unavailable on vGPU +# slices (e.g. fractional-GPU cloud instances), so builds targeting those need +# the synchronous path. +option(FVDB_USE_SYNC_CUDA_MALLOC "Define NANOVDB_USE_SYNC_CUDA_MALLOC to force synchronous NanoVDB CUDA allocation" OFF) +if(FVDB_USE_SYNC_CUDA_MALLOC) + message(STATUS "FVDB: compiling with NANOVDB_USE_SYNC_CUDA_MALLOC") + add_compile_definitions(NANOVDB_USE_SYNC_CUDA_MALLOC) +endif() From 4abb8966f708b15eb1b4da48ed3bbbe1178d703f Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Wed, 29 Jul 2026 15:22:02 +1200 Subject: [PATCH 6/9] Adopt availability zones format from main for g6f AMI setup Signed-off-by: Jonathan Swartz --- .github/workflows/load-versions.yml | 4 ++++ .github/workflows/tests.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/load-versions.yml b/.github/workflows/load-versions.yml index a8ca42e8..562b868a 100644 --- a/.github/workflows/load-versions.yml +++ b/.github/workflows/load-versions.yml @@ -56,6 +56,8 @@ on: value: ${{ jobs.load.outputs.aws-cpu-az-config }} aws-gpu-az-config: value: ${{ jobs.load.outputs.aws-gpu-az-config }} + aws-fgpu-az-config: + value: ${{ jobs.load.outputs.aws-fgpu-az-config }} aws-security-group: value: ${{ jobs.load.outputs.aws-security-group }} aws-cloudfront-distribution-domain: @@ -94,6 +96,7 @@ jobs: aws-fgpu-ami: ${{ steps.parse.outputs.aws-fgpu-ami }} aws-cpu-az-config: ${{ steps.parse.outputs.aws-cpu-az-config }} aws-gpu-az-config: ${{ steps.parse.outputs.aws-gpu-az-config }} + aws-fgpu-az-config: ${{ steps.parse.outputs.aws-fgpu-az-config }} aws-security-group: ${{ steps.parse.outputs.aws-security-group }} aws-cloudfront-distribution-domain: ${{ steps.parse.outputs.aws-cloudfront-distribution-domain }} publish-matrix: ${{ steps.parse.outputs.publish-matrix }} @@ -134,6 +137,7 @@ jobs: echo "aws-fgpu-ami=$(jq -r '.aws.fgpu_ami' $CFG)" echo "aws-cpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.cpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)" echo "aws-gpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.gpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)" + echo "aws-fgpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.fgpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)" echo "aws-security-group=$(jq -r '.aws.security_group' $CFG)" echo "aws-cloudfront-distribution-domain=$(jq -r '.aws.cloudfront_distribution_domain // ""' $CFG)" echo "publish-matrix=$(jq -c '.publish_matrix' $CFG)" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 39e58ab5..291a5f37 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -206,8 +206,8 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + ec2-instance-type: g6f.xlarge # 4 CPU-core, 1/8 L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## # RUN FVDB GTESTS From 52cc1e2ddec26a84370fc4f9c1bfa7f2ee7b2df4 Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Wed, 29 Jul 2026 15:26:20 +1200 Subject: [PATCH 7/9] Setup cu128 and cu130 actions to use the g6f setup Signed-off-by: Jonathan Swartz --- .github/workflows/cu128.yml | 8 +++++--- .github/workflows/cu130.yml | 8 +++++--- .github/workflows/tests.yml | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cu128.yml b/.github/workflows/cu128.yml index 381559bb..a6d19e19 100644 --- a/.github/workflows/cu128.yml +++ b/.github/workflows/cu128.yml @@ -166,7 +166,9 @@ jobs: source .venv/bin/activate NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) pip install "$NANOVDB_EDITOR_SPEC" - ./build.sh wheel verbose gtests benchmarks --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" + # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances + # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc. + ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" env: NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR: ${{ needs.versions.outputs.cuda-arch-pr }} @@ -249,8 +251,8 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## # RUN FVDB GTESTS diff --git a/.github/workflows/cu130.yml b/.github/workflows/cu130.yml index 9de66e29..8241defb 100644 --- a/.github/workflows/cu130.yml +++ b/.github/workflows/cu130.yml @@ -166,7 +166,9 @@ jobs: source .venv/bin/activate NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) pip install "$NANOVDB_EDITOR_SPEC" - ./build.sh wheel verbose gtests benchmarks --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" + # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances + # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc. + ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" env: NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR: ${{ needs.versions.outputs.cuda-arch-pr }} @@ -249,8 +251,8 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## # RUN FVDB GTESTS diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 291a5f37..6bed5be1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -206,7 +206,7 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6f.xlarge # 4 CPU-core, 1/8 L4 GPU + ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## From 545ffb0c409d6834d908e9f41f3128ef94a07833 Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Wed, 29 Jul 2026 16:03:16 +1200 Subject: [PATCH 8/9] Added a GPU unit test stage to the nightly-publish to catch any errors that we might not catch in the PR CI that disables async malloc Signed-off-by: Jonathan Swartz --- .github/workflows/load-versions.yml | 4 + .github/workflows/nightly-publish.yml | 144 ++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/.github/workflows/load-versions.yml b/.github/workflows/load-versions.yml index 562b868a..fccdaa5d 100644 --- a/.github/workflows/load-versions.yml +++ b/.github/workflows/load-versions.yml @@ -30,6 +30,8 @@ on: value: ${{ jobs.load.outputs.cuda-128-patch }} cuda-130-patch: value: ${{ jobs.load.outputs.cuda-130-patch }} + cuda-default-patch: + value: ${{ jobs.load.outputs.cuda-default-patch }} cuda-conda-override: value: ${{ jobs.load.outputs.cuda-conda-override }} cuda-arch-pr: @@ -83,6 +85,7 @@ jobs: cuda-default: ${{ steps.parse.outputs.cuda-default }} cuda-128-patch: ${{ steps.parse.outputs.cuda-128-patch }} cuda-130-patch: ${{ steps.parse.outputs.cuda-130-patch }} + cuda-default-patch: ${{ steps.parse.outputs.cuda-default-patch }} cuda-conda-override: ${{ steps.parse.outputs.cuda-conda-override }} cuda-arch-pr: ${{ steps.parse.outputs.cuda-arch-pr }} cuda-arch-publish: ${{ steps.parse.outputs.cuda-arch-publish }} @@ -124,6 +127,7 @@ jobs: echo "cuda-default=$(jq -r '.cuda.default' $CFG)" echo "cuda-128-patch=$(jq -r '.cuda.versions["12.8"].patch' $CFG)" echo "cuda-130-patch=$(jq -r '.cuda.versions["13.0"].patch' $CFG)" + echo "cuda-default-patch=$(jq -r '.cuda.versions[.cuda.default].patch' $CFG)" echo "cuda-conda-override=$(jq -r '.cuda.conda_override' $CFG)" echo "cuda-arch-pr=$(jq -r '.cuda.arch_list_pr' $CFG)" echo "cuda-arch-publish=$(jq -r '.cuda.arch_list_publish' $CFG)" diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index 21382af9..7e6b4fc8 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -507,3 +507,147 @@ jobs: github-token: ${{ secrets.EC2_RUNNER_TOKEN }} label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }} + + ############################################################################## + # START NIGHTLY TESTS GPU RUNNER + # + # The PR workflows build with NANOVDB_USE_SYNC_CUDA_MALLOC because they test on + # fractional-GPU (vGPU) runners, which cannot use cudaMallocAsync. This whole + # GPU instance is therefore the only place the default asynchronous NanoVDB + # allocation path gets exercised, so it runs once a night against the published + # nightly wheel instead of on every pull request. + ############################################################################## + start-tests-gpu-runner: + name: Start EC2 GPU runner for nightly tests + needs: [nightly-build, versions] + if: ${{ !cancelled() && needs.nightly-build.result == 'success' }} + runs-on: ubuntu-latest + permissions: + id-token: write # Required for AWS OIDC + outputs: + label: ${{ steps.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 + with: + role-to-assume: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + - name: Start EC2 GPU runner + id: start-tests-gpu-runner + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: start + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + + ############################################################################## + # RUN NIGHTLY UNIT TESTS + ############################################################################## + nightly-unit-tests: + name: fVDB Nightly Unit Tests + needs: [start-tests-gpu-runner, versions] + runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} + container: + image: nvidia/cuda:${{ needs.versions.outputs.cuda-default-patch }}-cudnn-devel-${{ needs.versions.outputs.docker-os-rocky }} + env: + PYTHONPATH: "" + options: --rm + defaults: + run: + shell: bash -el {0} + steps: + - name: Install prerequisites + run: | + dnf install -y git wget procps-ng findutils gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET} + echo "source /opt/rh/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}/enable" > /etc/profile.d/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}.sh + env: + NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET: ${{ needs.versions.outputs.gcc-toolset }} + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: main + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 + with: + version: "${{ needs.versions.outputs.uv-version }}" + + - name: Set up Python + run: | + uv python install ${NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT} + uv venv + env: + NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT: ${{ needs.versions.outputs.python-default }} + + - name: Install system dependencies + run: dnf install -y zlib-devel libpng-devel + + - name: Install pip dependencies + run: | + CUDA_TAG="cu$(echo "${NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT}" | tr -d '.')" + echo "torch==${NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION}" > "${RUNNER_TEMP}/torch-constraints.txt" + uv pip install --no-cache -c "${RUNNER_TEMP}/torch-constraints.txt" -r env/test_requirements.txt --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} --index-strategy unsafe-best-match + uv pip install --no-cache setuptools + env: + NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT: ${{ needs.versions.outputs.cuda-default }} + NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION: ${{ needs.versions.outputs.torch-full-version }} + + # Test the default Python/CUDA cell of the publish matrix; the other cells + # differ only in interpreter and CUDA minor version. + - name: Download package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: fvdb-nightly-${{ needs.versions.outputs.python-default }}-torch${{ needs.versions.outputs.torch-version }}-cu${{ needs.versions.outputs.cuda-default }} + path: ./dist + + - name: Install package + run: | + source .venv/bin/activate + pip install ./dist/*.whl + + - name: Run unit tests + run: | + source .venv/bin/activate + NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) + pip install "$NANOVDB_EDITOR_SPEC" + cd tests; + pytest -v unit + + - name: Cleanup + if: always() + run: | + echo "Cleaning up /__w/_temp directory" + rm -rf /__w/_temp/* + echo "Cleanup completed" + + ############################################################################## + # STOP NIGHTLY TESTS GPU RUNNER + ############################################################################## + stop-tests-gpu-runner: + name: Stop GPU EC2 runner for nightly tests + needs: + - start-tests-gpu-runner # required to get output from the start-tests-gpu-runner job + - nightly-unit-tests # required to wait when the main job is done + - versions + runs-on: ubuntu-latest + permissions: + id-token: write # Required for AWS OIDC + # required to stop the runner even if the error happened in the previous jobs, + # but only if the start-tests-gpu-runner job was not skipped + if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 + with: + role-to-assume: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: stop + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + label: ${{ needs.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} From 49dd48ea5f7697d09f2c7653399ae16e5de7932b Mon Sep 17 00:00:00 2001 From: Jonathan Swartz Date: Wed, 29 Jul 2026 16:31:01 +1200 Subject: [PATCH 9/9] Fix build.sh ctest picking a dependency's CMake build directory build.sh ctest located the build root with find build -name CMakeCache.txt -type f -print -quit which stops at the first match. find walks directories in filesystem order, so the match was whichever cache the filesystem happened to hand back first, and every dependency CPM/FetchContent configures leaves a CMakeCache.txt of its own under _deps: nanovdb-subbuild, benchmark-build, blosc-build and googletest-build each have one alongside the real cache. When a dependency won, /src was appended to its path and the gtest job aborted before running a single test: Error: No CTestTestfile.cmake found in build/cp312-cp312-linux_x86_64 -Release/_deps/nanovdb-subbuild/src Prune _deps from the search and accept a candidate only if it actually contains src/CTestTestfile.cmake, so the result no longer depends on directory ordering. A tree with both Debug and Release configured now also selects whichever was built with tests enabled, rather than an arbitrary one. The ordering dependence dates back to #418 and went unnoticed because the real cache usually came back first. Signed-off-by: Jonathan Swartz --- build.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/build.sh b/build.sh index 9b18b90b..90baa562 100755 --- a/build.sh +++ b/build.sh @@ -428,22 +428,32 @@ elif [ "$BUILD_TYPE" == "ctest" ]; then # --- Find and Run Tests --- echo "Searching for test build directory..." - # Find CMakeCache.txt to locate the build root - CMAKE_CACHE=$(find build -name CMakeCache.txt -type f -print -quit 2>/dev/null) + # Find CMakeCache.txt to locate the build root. Every vendored dependency that + # CPM/FetchContent configures leaves a CMakeCache.txt of its own under _deps, so + # that subtree is pruned; otherwise find can return a dependency's cache first, + # since it walks directories in filesystem order rather than a defined one. + FOUND_CMAKE_CACHE=false + BUILD_DIR="" + while IFS= read -r cmake_cache; do + FOUND_CMAKE_CACHE=true + # Construct the test directory path (where CTestTestfile.cmake is generated) + # This discovers all tests from both src/tests/ and src/dispatch/ + candidate_dir="$(dirname "$cmake_cache")/src" + if [ -f "$candidate_dir/CTestTestfile.cmake" ]; then + BUILD_DIR="$candidate_dir" + break + fi + done < <(find build -type d -name _deps -prune -o -name CMakeCache.txt -type f -print 2>/dev/null) - if [ -z "$CMAKE_CACHE" ]; then + if [ "$FOUND_CMAKE_CACHE" = false ]; then echo "Error: Could not find CMakeCache.txt in build directory" echo "Please build the project first with tests enabled:" echo "pip install . -C cmake.define.FVDB_BUILD_TESTS=ON" exit 1 fi - # Construct the test directory path (where CTestTestfile.cmake is generated) - # This discovers all tests from both src/tests/ and src/dispatch/ - BUILD_DIR="$(dirname "$CMAKE_CACHE")/src" - - if [ ! -f "$BUILD_DIR/CTestTestfile.cmake" ]; then - echo "Error: No CTestTestfile.cmake found in $BUILD_DIR" + if [ -z "$BUILD_DIR" ]; then + echo "Error: No CTestTestfile.cmake found under build/*/src" echo "Please enable tests by building with:" echo "pip install . -C cmake.define.FVDB_BUILD_TESTS=ON" exit 1