Enable Dynamic MIG tests in Lambda CI

Add the existing Dynamic MIG test suite (`test_gpu_dynmig.bats`) to the
Lambda CI `tests-gpu-single` target, with GPU-type-aware filtering to
auto-skip on non-MIG-capable instances.

Changes:
- Tag all DynMIG tests with `dynmig` for selective filtering
- Tag DynMIG attribute inspection tests with `version-specific`
  (attribute list varies by driver version)
- Add `dynmig` filter to `e2e-test.sh` based on `LAMBDA_GPU_TYPE`:
  tests run on A100/H100/GH200/B200, skipped on V100/A10
- Add `DISABLE_COMPUTE_DOMAINS` handling to DynMIG `setup_file()`
  and TimeSlicing test `iupgrade_wait()` calls
- Make `nvmm` MIG cleanup non-fatal in `cleanup-from-previous-run.sh`
  (MIG not supported on all GPU types)

Known limitation: Dynamic MIG fails on single-GPU A100 instances with
"In use by another client" because the driver holds the GPU via NVML,
preventing MIG mode toggle. Works on H100 and newer GPUs.

Signed-off-by: Davanum Srinivas <davanum@gmail.com>

Author: dims

hack/ci/lambda/e2e-test.sh

@@ -84,11 +84,13 @@ fi
 case "${LAMBDA_GPU_TYPE}" in
   *v100*|*a10) FILTER="${FILTER},!gpu-busgrind" ;;
 esac
+# Dynamic MIG requires MIG-capable GPUs (A100/H100/GH200/B200).
+case "${LAMBDA_GPU_TYPE}" in
+  *a100*|*h100*|*gh200*|*b200*) ;;
+  *) FILTER="${FILTER},!dynmig" ;;
+esac
 
 # Detect whether compute domains should be disabled.
-# Compute domains (IMEX channels) require NVSwitch fabric with fabric manager,
-# which is only available on GB200/GB300 NVL systems. On all other GPU types,
-# disable compute domains to avoid the compute-domains container crashing.
 DISABLE_CD=true
 case "${LAMBDA_GPU_TYPE}" in
   *gb200*|*gb300*|*b200*) DISABLE_CD=false ;;
@@ -97,8 +99,8 @@ echo "Test filter: ${FILTER}, compute domains disabled: ${DISABLE_CD}"
 
 # --- Pre-cleanup: MIG teardown on host ---
 # Run MIG cleanup directly on the host where nvidia-smi is available.
-# The BATS Docker container doesn't have nvidia-smi, so cleanup-from-previous-run.sh
-# uses the lambda/nvmm no-op stub. We handle MIG cleanup here instead.
+# The BATS Docker container uses the lambda nvmm stub (no-op).
+# We handle MIG cleanup here instead.
 # IMPORTANT: Skip on A100 — disabling MIG mode on cloud VM A100s can put the
 # GPU in an unrecoverable state (#883).
 case "${LAMBDA_GPU_TYPE}" in
@@ -126,7 +128,7 @@ export TEST_CHART_LOCAL=true
 export DISABLE_COMPUTE_DOMAINS=${DISABLE_CD}
 export TEST_FILTER_TAGS='${FILTER}'
 export GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT}
-# Use lambda nvmm stub (no GPU Operator). MIG cleanup is handled above on the host.
+# Use lambda nvmm stub (no GPU Operator). MIG cleanup handled above.
 export NVMM_PATH=/cwd/tests/bats/lib/lambda
 
 make -f tests/bats/Makefile tests-gpu-single GIT_COMMIT_SHORT=${GIT_COMMIT_SHORT}

tests/bats/Makefile

@@ -177,6 +177,7 @@ tests-gpu-single: runner-image
 	$(call RUN_BATS, \
 		tests/bats/test_gpu_basic.bats \
 		tests/bats/test_gpu_cuda_workloads.bats \
+		tests/bats/test_gpu_dynmig.bats \
 		tests/bats/test_gpu_sharing.bats \
 		tests/bats/test_gpu_robustness.bats)
 

tests/bats/cleanup-from-previous-run.sh

@@ -104,7 +104,15 @@ set -e
 bash tests/bats/clean-state-dirs-all-nodes.sh
 
 # Remove any stray MIG devices and disable MIG mode on all nodes.
-nvmm all sh -c 'nvidia-smi mig -dci; nvidia-smi mig -dgi; nvidia-smi -mig 0'
+# Skip on A100 cloud VMs — disabling MIG mode can put the GPU in an
+# unrecoverable state (#883). Non-fatal for GPUs that don't support MIG.
+nvmm all sh -c '
+  gpu_name=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)
+  case "$gpu_name" in
+    *A100*) echo "Skipping MIG cleanup on A100 (#883)"; exit 0 ;;
+  esac
+  nvidia-smi mig -dci; nvidia-smi mig -dgi; nvidia-smi -mig 0
+' || echo "nvmm MIG cleanup skipped (non-fatal)"
 
 set +x
 echo "cleanup: done"

tests/bats/test_gpu_dynmig.bats

@@ -13,6 +13,9 @@ setup_file () {
   kubectl delete resourceslices.resource.k8s.io --all
 
   local _iargs=("--set" "logVerbosity=6" "--set" "featureGates.DynamicMIG=true")
+  if [ "${DISABLE_COMPUTE_DOMAINS:-}" = "true" ]; then
+    _iargs+=("--set" "resources.computeDomains.enabled=false")
+  fi
   iupgrade_wait "${TEST_CHART_REPO}" "${TEST_CHART_VERSION}" _iargs
   run kubectl logs \
     -l dra-driver-nvidia-gpu-component=kubelet-plugin \
@@ -57,7 +60,7 @@ confirm_mig_mode_disabled_all_nodes() {
 }
 
 
-# bats test_tags=fastfeedback
+# bats test_tags=fastfeedback,dynmig,version-specific
 @test "DynMIG: inspect device attributes in resource slice (gpu)" {
   local reference=(
     "architecture"
@@ -80,7 +83,7 @@ confirm_mig_mode_disabled_all_nodes() {
 }
 
 
-# bats test_tags=fastfeedback
+# bats test_tags=fastfeedback,dynmig,version-specific
 @test "DynMIG: inspect device attributes in resource slice (mig)" {
   local reference=(
     "architecture"
@@ -104,7 +107,7 @@ confirm_mig_mode_disabled_all_nodes() {
 }
 
 
-# bats test_tags=fastfeedback
+# bats test_tags=fastfeedback,dynmig
 @test "DynMIG: 1 pod, 1 MIG" {
   confirm_mig_mode_disabled_all_nodes
   kubectl apply -f tests/bats/specs/gpu-simple-mig.yaml
@@ -127,7 +130,7 @@ confirm_mig_mode_disabled_all_nodes() {
 }
 
 
-# bats test_tags=fastfeedback
+# bats test_tags=fastfeedback,dynmig
 @test "DynMIG: 1 pod, 2 containers (1 MIG each)" {
   confirm_mig_mode_disabled_all_nodes
 
@@ -159,9 +162,12 @@ confirm_mig_mode_disabled_all_nodes() {
 }
 
 
-# bats test_tags=fastfeedback
+# bats test_tags=fastfeedback,dynmig
 @test "DynMIG: 1 pod, 1 MIG + TimeSlicing config" {
   local _iargs=("--set" "logVerbosity=6" "--set" "featureGates.DynamicMIG=true" "--set" "featureGates.TimeSlicingSettings=true")
+  if [ "${DISABLE_COMPUTE_DOMAINS:-}" = "true" ]; then
+    _iargs+=("--set" "resources.computeDomains.enabled=false")
+  fi
   iupgrade_wait "${TEST_CHART_REPO}" "${TEST_CHART_VERSION}" _iargs
 
   confirm_mig_mode_disabled_all_nodes