CI: g6f instances

.github/versions.json

@@ -35,6 +35,7 @@
     "region": "us-east-2",
     "cpu_ami": "ami-0e14a711dad782a70",
     "gpu_ami": "ami-0f5c0b65fde08ae43",
+    "fgpu_ami": "ami-0b5dca767eafff9d8",
     "subnets": [
       { "az": "us-east-2a", "id": "subnet-0b75bb13b6048a2f1" },
       { "az": "us-east-2b", "id": "subnet-03f2320d6e6e0005b" },

.github/workflows/cu128.yml

@@ -166,7 +166,9 @@ jobs:
           source .venv/bin/activate
           NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
           pip install "$NANOVDB_EDITOR_SPEC"
-          ./build.sh wheel verbose gtests benchmarks --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}"
+          # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances
+          # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc.
+          ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}"
         env:
           NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR: ${{ needs.versions.outputs.cuda-arch-pr }}
 
@@ -249,8 +251,8 @@ jobs:
         with:
           mode: start
           github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
-          ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU
-          availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }}
+          ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU
+          availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }}
 
   ##############################################################################
   # RUN FVDB GTESTS

.github/workflows/cu130.yml

@@ -166,7 +166,9 @@ jobs:
           source .venv/bin/activate
           NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
           pip install "$NANOVDB_EDITOR_SPEC"
-          ./build.sh wheel verbose gtests benchmarks --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}"
+          # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances
+          # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc.
+          ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}"
         env:
           NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR: ${{ needs.versions.outputs.cuda-arch-pr }}
 
@@ -249,8 +251,8 @@ jobs:
         with:
           mode: start
           github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
-          ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU
-          availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }}
+          ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU
+          availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }}
 
   ##############################################################################
   # RUN FVDB GTESTS

.github/workflows/load-versions.yml

@@ -30,6 +30,8 @@ on:
         value: ${{ jobs.load.outputs.cuda-128-patch }}
       cuda-130-patch:
         value: ${{ jobs.load.outputs.cuda-130-patch }}
+      cuda-default-patch:
+        value: ${{ jobs.load.outputs.cuda-default-patch }}
       cuda-conda-override:
         value: ${{ jobs.load.outputs.cuda-conda-override }}
       cuda-arch-pr:
@@ -50,10 +52,14 @@ on:
         value: ${{ jobs.load.outputs.aws-cpu-ami }}
       aws-gpu-ami:
         value: ${{ jobs.load.outputs.aws-gpu-ami }}
+      aws-fgpu-ami:
+        value: ${{ jobs.load.outputs.aws-fgpu-ami }}
       aws-cpu-az-config:
         value: ${{ jobs.load.outputs.aws-cpu-az-config }}
       aws-gpu-az-config:
         value: ${{ jobs.load.outputs.aws-gpu-az-config }}
+      aws-fgpu-az-config:
+        value: ${{ jobs.load.outputs.aws-fgpu-az-config }}
       aws-security-group:
         value: ${{ jobs.load.outputs.aws-security-group }}
       aws-cloudfront-distribution-domain:
@@ -79,6 +85,7 @@ jobs:
       cuda-default: ${{ steps.parse.outputs.cuda-default }}
       cuda-128-patch: ${{ steps.parse.outputs.cuda-128-patch }}
       cuda-130-patch: ${{ steps.parse.outputs.cuda-130-patch }}
+      cuda-default-patch: ${{ steps.parse.outputs.cuda-default-patch }}
       cuda-conda-override: ${{ steps.parse.outputs.cuda-conda-override }}
       cuda-arch-pr: ${{ steps.parse.outputs.cuda-arch-pr }}
       cuda-arch-publish: ${{ steps.parse.outputs.cuda-arch-publish }}
@@ -89,8 +96,10 @@ jobs:
       aws-region: ${{ steps.parse.outputs.aws-region }}
       aws-cpu-ami: ${{ steps.parse.outputs.aws-cpu-ami }}
       aws-gpu-ami: ${{ steps.parse.outputs.aws-gpu-ami }}
+      aws-fgpu-ami: ${{ steps.parse.outputs.aws-fgpu-ami }}
       aws-cpu-az-config: ${{ steps.parse.outputs.aws-cpu-az-config }}
       aws-gpu-az-config: ${{ steps.parse.outputs.aws-gpu-az-config }}
+      aws-fgpu-az-config: ${{ steps.parse.outputs.aws-fgpu-az-config }}
       aws-security-group: ${{ steps.parse.outputs.aws-security-group }}
       aws-cloudfront-distribution-domain: ${{ steps.parse.outputs.aws-cloudfront-distribution-domain }}
       publish-matrix: ${{ steps.parse.outputs.publish-matrix }}
@@ -118,6 +127,7 @@ jobs:
             echo "cuda-default=$(jq -r '.cuda.default' $CFG)"
             echo "cuda-128-patch=$(jq -r '.cuda.versions["12.8"].patch' $CFG)"
             echo "cuda-130-patch=$(jq -r '.cuda.versions["13.0"].patch' $CFG)"
+            echo "cuda-default-patch=$(jq -r '.cuda.versions[.cuda.default].patch' $CFG)"
             echo "cuda-conda-override=$(jq -r '.cuda.conda_override' $CFG)"
             echo "cuda-arch-pr=$(jq -r '.cuda.arch_list_pr' $CFG)"
             echo "cuda-arch-publish=$(jq -r '.cuda.arch_list_publish' $CFG)"
@@ -128,8 +138,10 @@ jobs:
             echo "aws-region=$(jq -r '.aws.region' $CFG)"
             echo "aws-cpu-ami=$(jq -r '.aws.cpu_ami' $CFG)"
             echo "aws-gpu-ami=$(jq -r '.aws.gpu_ami' $CFG)"
+            echo "aws-fgpu-ami=$(jq -r '.aws.fgpu_ami' $CFG)"
             echo "aws-cpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.cpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)"
             echo "aws-gpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.gpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)"
+            echo "aws-fgpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.fgpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)"
             echo "aws-security-group=$(jq -r '.aws.security_group' $CFG)"
             echo "aws-cloudfront-distribution-domain=$(jq -r '.aws.cloudfront_distribution_domain // ""' $CFG)"
             echo "publish-matrix=$(jq -c '.publish_matrix' $CFG)"

.github/workflows/nightly-publish.yml

@@ -507,3 +507,147 @@ jobs:
           github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
           label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}
           ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }}
+
+  ##############################################################################
+  # START NIGHTLY TESTS GPU RUNNER
+  #
+  # The PR workflows build with NANOVDB_USE_SYNC_CUDA_MALLOC because they test on
+  # fractional-GPU (vGPU) runners, which cannot use cudaMallocAsync. This whole
+  # GPU instance is therefore the only place the default asynchronous NanoVDB
+  # allocation path gets exercised, so it runs once a night against the published
+  # nightly wheel instead of on every pull request.
+  ##############################################################################
+  start-tests-gpu-runner:
+    name: Start EC2 GPU runner for nightly tests
+    needs: [nightly-build, versions]
+    if: ${{ !cancelled() && needs.nightly-build.result == 'success' }}
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # Required for AWS OIDC
+    outputs:
+      label: ${{ steps.start-tests-gpu-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
+        with:
+          role-to-assume: ${{ needs.versions.outputs.aws-role }}
+          aws-region: ${{ needs.versions.outputs.aws-region }}
+      - name: Start EC2 GPU runner
+        id: start-tests-gpu-runner
+        uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
+        with:
+          mode: start
+          github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
+          ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU
+          availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }}
+
+  ##############################################################################
+  # RUN NIGHTLY UNIT TESTS
+  ##############################################################################
+  nightly-unit-tests:
+    name: fVDB Nightly Unit Tests
+    needs: [start-tests-gpu-runner, versions]
+    runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }}
+    container:
+      image: nvidia/cuda:${{ needs.versions.outputs.cuda-default-patch }}-cudnn-devel-${{ needs.versions.outputs.docker-os-rocky }}
+      env:
+        PYTHONPATH: ""
+      options: --rm
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - name: Install prerequisites
+        run: |
+          dnf install -y git wget procps-ng findutils gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}
+          echo "source /opt/rh/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}/enable" > /etc/profile.d/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}.sh
+        env:
+          NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET: ${{ needs.versions.outputs.gcc-toolset }}
+
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: main
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
+        with:
+          version: "${{ needs.versions.outputs.uv-version }}"
+
+      - name: Set up Python
+        run: |
+          uv python install ${NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT}
+          uv venv
+        env:
+          NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT: ${{ needs.versions.outputs.python-default }}
+
+      - name: Install system dependencies
+        run: dnf install -y zlib-devel libpng-devel
+
+      - name: Install pip dependencies
+        run: |
+          CUDA_TAG="cu$(echo "${NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT}" | tr -d '.')"
+          echo "torch==${NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION}" > "${RUNNER_TEMP}/torch-constraints.txt"
+          uv pip install --no-cache -c "${RUNNER_TEMP}/torch-constraints.txt" -r env/test_requirements.txt --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} --index-strategy unsafe-best-match
+          uv pip install --no-cache setuptools
+        env:
+          NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT: ${{ needs.versions.outputs.cuda-default }}
+          NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION: ${{ needs.versions.outputs.torch-full-version }}
+
+      # Test the default Python/CUDA cell of the publish matrix; the other cells
+      # differ only in interpreter and CUDA minor version.
+      - name: Download package
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: fvdb-nightly-${{ needs.versions.outputs.python-default }}-torch${{ needs.versions.outputs.torch-version }}-cu${{ needs.versions.outputs.cuda-default }}
+          path: ./dist
+
+      - name: Install package
+        run: |
+          source .venv/bin/activate
+          pip install ./dist/*.whl
+
+      - name: Run unit tests
+        run: |
+          source .venv/bin/activate
+          NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
+          pip install "$NANOVDB_EDITOR_SPEC"
+          cd tests;
+          pytest -v unit
+
+      - name: Cleanup
+        if: always()
+        run: |
+          echo "Cleaning up /__w/_temp directory"
+          rm -rf /__w/_temp/*
+          echo "Cleanup completed"
+
+  ##############################################################################
+  # STOP NIGHTLY TESTS GPU RUNNER
+  ##############################################################################
+  stop-tests-gpu-runner:
+    name: Stop GPU EC2 runner for nightly tests
+    needs:
+      - start-tests-gpu-runner # required to get output from the start-tests-gpu-runner job
+      - nightly-unit-tests # required to wait when the main job is done
+      - versions
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # Required for AWS OIDC
+    # required to stop the runner even if the error happened in the previous jobs,
+    # but only if the start-tests-gpu-runner job was not skipped
+    if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1
+        with:
+          role-to-assume: ${{ needs.versions.outputs.aws-role }}
+          aws-region: ${{ needs.versions.outputs.aws-region }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1
+        with:
+          mode: stop
+          github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
+          label: ${{ needs.start-tests-gpu-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }}

.github/workflows/tests.yml

@@ -132,7 +132,9 @@ jobs:
           micromamba activate fvdb_build
           NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py)
           pip install "$NANOVDB_EDITOR_SPEC"
-          ./build.sh wheel verbose gtests benchmarks --cuda-arch-list '8.9+PTX'
+          # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances
+          # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc.
+          ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list '8.9+PTX'
 
       - name: Upload wheel
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -204,8 +206,8 @@ jobs:
         with:
           mode: start
           github-token: ${{ secrets.EC2_RUNNER_TOKEN }}
-          ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU
-          availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }}
+          ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU
+          availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }}
 
   ##############################################################################
   # RUN FVDB GTESTS

README.md

@@ -179,6 +179,9 @@ The build script supports the following build modifiers:
 - `debug`: Build in debug mode with full debug symbols and no optimizations.
 - `strip_symbols`: Strip symbols from the build (will be ignored if debug is enabled).
 - `verbose`: Enable verbose build output for pip and CMake.
+- `sync_cuda_malloc`: Compile with `-DNANOVDB_USE_SYNC_CUDA_MALLOC` so NanoVDB allocates with
+  `cudaMalloc`/`cudaFree` instead of the async variants. Required on vGPU slices, where async
+  allocation is unavailable because it depends on GPU unified memory.
 
 ### Running Tests
 

build.sh

@@ -26,6 +26,8 @@ usage() {
   echo "  debug          Build in debug mode with full debug symbols and no optimizations."
   echo "  lineinfo       Enable CUDA lineinfo (sets FVDB_LINEINFO=ON)."
   echo "  strip_symbols  Strip symbols from the build (will be ignored if debug is enabled)."
+  echo "  sync_cuda_malloc  Compile with -DNANOVDB_USE_SYNC_CUDA_MALLOC so NanoVDB uses"
+  echo "                    synchronous cudaMalloc/cudaFree (required on vGPU slices)."
   echo "  verbose        Enable verbose build output for pip and CMake."
   echo "  trace          Enable CMake trace output for debugging configuration."
   echo ""
@@ -316,6 +318,10 @@ while (( "$#" )); do
       echo "Enabling strip symbols build"
       CONFIG_SETTINGS+=" --config-settings=cmake.define.FVDB_STRIP_SYMBOLS=ON"
       is_config_arg_handled=true
+    elif [[ "$1" == "sync_cuda_malloc" ]]; then
+      echo "Enabling synchronous NanoVDB CUDA allocation (-DNANOVDB_USE_SYNC_CUDA_MALLOC)"
+      CONFIG_SETTINGS+=" --config-settings=cmake.define.FVDB_USE_SYNC_CUDA_MALLOC=ON"
+      is_config_arg_handled=true
     fi
   fi
 
@@ -422,22 +428,32 @@ elif [ "$BUILD_TYPE" == "ctest" ]; then
 
     # --- Find and Run Tests ---
     echo "Searching for test build directory..."
-    # Find CMakeCache.txt to locate the build root
-    CMAKE_CACHE=$(find build -name CMakeCache.txt -type f -print -quit 2>/dev/null)
+    # Find CMakeCache.txt to locate the build root. Every vendored dependency that
+    # CPM/FetchContent configures leaves a CMakeCache.txt of its own under _deps, so
+    # that subtree is pruned; otherwise find can return a dependency's cache first,
+    # since it walks directories in filesystem order rather than a defined one.
+    FOUND_CMAKE_CACHE=false
+    BUILD_DIR=""
+    while IFS= read -r cmake_cache; do
+        FOUND_CMAKE_CACHE=true
+        # Construct the test directory path (where CTestTestfile.cmake is generated)
+        # This discovers all tests from both src/tests/ and src/dispatch/
+        candidate_dir="$(dirname "$cmake_cache")/src"
+        if [ -f "$candidate_dir/CTestTestfile.cmake" ]; then
+            BUILD_DIR="$candidate_dir"
+            break
+        fi
+    done < <(find build -type d -name _deps -prune -o -name CMakeCache.txt -type f -print 2>/dev/null)
 
-    if [ -z "$CMAKE_CACHE" ]; then
+    if [ "$FOUND_CMAKE_CACHE" = false ]; then
         echo "Error: Could not find CMakeCache.txt in build directory"
         echo "Please build the project first with tests enabled:"
         echo "pip install . -C cmake.define.FVDB_BUILD_TESTS=ON"
         exit 1
     fi
 
-    # Construct the test directory path (where CTestTestfile.cmake is generated)
-    # This discovers all tests from both src/tests/ and src/dispatch/
-    BUILD_DIR="$(dirname "$CMAKE_CACHE")/src"
-
-    if [ ! -f "$BUILD_DIR/CTestTestfile.cmake" ]; then
-        echo "Error: No CTestTestfile.cmake found in $BUILD_DIR"
+    if [ -z "$BUILD_DIR" ]; then
+        echo "Error: No CTestTestfile.cmake found under build/*/src"
         echo "Please enable tests by building with:"
         echo "pip install . -C cmake.define.FVDB_BUILD_TESTS=ON"
         exit 1

src/cmake/get_nanovdb.cmake

@@ -4,7 +4,7 @@
 CPMAddPackage(
     NAME nanovdb
     GITHUB_REPOSITORY AcademySoftwareFoundation/openvdb
-    GIT_TAG f9754140ba6031813b37d8e1b239ed0253ebd96d
+    GIT_TAG b7fc4fc7af73e84071b5f625482cfad4c50eb247
     SOURCE_SUBDIR nanovdb/nanovdb
     DOWNLOAD_ONLY YES
 )
@@ -15,3 +15,19 @@ if(nanovdb_ADDED)
     add_library(nanovdb INTERFACE)
     target_include_directories(nanovdb INTERFACE ${nanovdb_SOURCE_DIR}/nanovdb)
 endif()
+
+# nanovdb::util::cuda::mallocAsync resolves to either cudaMallocAsync or plain
+# cudaMalloc depending on whether NANOVDB_USE_SYNC_CUDA_MALLOC is defined. The
+# choice is made in a header, so the macro has to be visible to every C++/CUDA
+# translation unit that includes NanoVDB or an inline function ends up with two
+# different bodies across the build. Hence a directory-wide compile definition
+# rather than a per-target one.
+#
+# Async allocation depends on GPU unified memory, which is unavailable on vGPU
+# slices (e.g. fractional-GPU cloud instances), so builds targeting those need
+# the synchronous path.
+option(FVDB_USE_SYNC_CUDA_MALLOC "Define NANOVDB_USE_SYNC_CUDA_MALLOC to force synchronous NanoVDB CUDA allocation" OFF)
+if(FVDB_USE_SYNC_CUDA_MALLOC)
+    message(STATUS "FVDB: compiling with NANOVDB_USE_SYNC_CUDA_MALLOC")
+    add_compile_definitions(NANOVDB_USE_SYNC_CUDA_MALLOC)
+endif()