feat(e2e): Under testing kai hack deployments, Add hami testing option, installing kai-resource-isolator

docs/gpu-sharing/hami/README.md

@@ -144,3 +144,81 @@ curl {pod-ip}:9394/metrics
 ### Memory value precision
 
 The `gpu-memory` annotation accepts an **integer in MiB** (no unit suffix). Internally, KAI-Scheduler converts this to a GPU fraction with 2-decimal precision, which is then multiplied against the total GPU memory to compute the actual limit. As a result, the value seen in `nvidia-smi` may differ slightly from the requested value. For example, requesting `4096` MiB on a `15360` MiB GPU (T4) rounds to a `0.27` fraction, yielding `4147m` as the enforced limit.
+
+## Local e2e testing
+
+The HAMi-core e2e suite lives at `test/e2e/suites/integrations/third_party/hamicore/`. It checks KAI’s isolation contract (`CUDA_DEVICE_MEMORY_LIMIT` injection and limited `nvidia-smi` visible memory). It is **not wired into CI**: KAI PR e2e runs on kind with the fake GPU operator and has no real GPUs, so these specs soft-skip unless the `hamicore` binder plugin and the `kai-resource-isolator` mutating webhook are present.
+
+Use a machine with a real NVIDIA GPU (for example minikube with `--gpus=all`). Docker must be able to run `docker run --rm --gpus all nvidia/cuda:12.6.0-base-ubuntu22.04 nvidia-smi` before you start.
+
+### 1. Start minikube with GPU access
+
+```bash
+minikube start --driver=docker --gpus=all --cpus=6 --memory=12288
+kubectl config use-context minikube
+```
+
+### 2. NVIDIA device plugin + node labels
+
+The binder and e2e helpers read `nvidia.com/gpu.memory` (MiB per GPU). Label the node with the card’s total from `nvidia-smi` (example below is an 11264 MiB 2080 Ti):
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.1/deployments/static/nvidia-device-plugin.yml
+
+kubectl label node minikube \
+  nvidia.com/gpu.present=true \
+  nvidia.com/gpu.memory=11264 \
+  --overwrite
+
+kubectl -n kube-system rollout status ds/nvidia-device-plugin-daemonset --timeout=180s
+kubectl get node minikube -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}{.metadata.labels.nvidia\.com/gpu.memory}{"\n"}'
+```
+
+### 3. Install KAI with GPU sharing + hamicore
+
+If the cluster has no `RuntimeClass` named `nvidia` (common on bare device-plugin minikube), clear the default runtime-class settings so admission does not reject fraction pods:
+
+```bash
+helm upgrade -i kai-scheduler \
+  oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler \
+  --namespace kai-scheduler --create-namespace \
+  --version v0.17.0 \
+  --set global.gpuSharing=true \
+  --set binder.plugins.hamicore.enabled=true \
+  --set binder.resourceReservation.runtimeClassName="" \
+  --set admission.gpuFractionRuntimeClassName="" \
+  --wait
+```
+
+### 4. Install kai-resource-isolator
+
+Prefer the helper under `hack/hami/` (also used by `--test-hami` in kind cluster setup):
+
+```bash
+# from the KAI-Scheduler repo root
+./hack/hami/deploy_isolator.sh
+
+# or a local isolator chart checkout:
+# ISOLATOR_CHART_REF=/path/to/KAI-resource-isolator/chart/kai-resource-isolator \
+#   ./hack/hami/deploy_isolator.sh
+```
+
+Confirm the webhook exists:
+
+```bash
+kubectl get mutatingwebhookconfiguration kai-resource-isolator-mutating
+kubectl -n kai-resource-isolator get deploy,ds,pods
+```
+
+### 5. Run the suite
+
+```bash
+export PATH="$(go env GOPATH)/bin:$PATH"
+go install github.com/onsi/ginkgo/v2/ginkgo@latest
+
+ginkgo -v --trace ./test/e2e/suites/integrations/third_party/hamicore/
+```
+
+### Optional: kind helper flag
+
+`hack/setup-e2e-cluster.sh --test-hami` (and `hack/run-e2e-kind.sh --test-hami`) enables `binder.plugins.hamicore.enabled=true` and runs `hack/hami/deploy_isolator.sh`. That is useful for install plumbing on kind, but the hamicore specs still need a real GPU and will skip or fail against the fake GPU operator alone.

hack/hami/deploy_isolator.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2026 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+#
+# Installs kai-resource-isolator with kai-vgpu-monitor enabled for HAMi/hamicore
+# e2e (see --test-hami in hack/setup-e2e-cluster.sh).
+#
+# Overrides (optional):
+#   ISOLATOR_CHART_REF       OCI ref or local chart path
+#                           (default: oci://docker.io/projecthami/kai-resource-isolator)
+#   ISOLATOR_CHART_VERSION   Chart version when using OCI (default: 1.1.0-chart)
+#   ISOLATOR_NAMESPACE       Install namespace (default: kai-resource-isolator)
+#   ISOLATOR_RELEASE         Helm release name (default: kai-resource-isolator)
+#   ISOLATOR_HELM_EXTRA_ARGS Extra args appended to helm upgrade (word-split)
+set -euo pipefail
+
+ISOLATOR_CHART_REF="${ISOLATOR_CHART_REF:-oci://docker.io/projecthami/kai-resource-isolator}"
+ISOLATOR_CHART_VERSION="${ISOLATOR_CHART_VERSION:-1.1.0-chart}"
+ISOLATOR_NAMESPACE="${ISOLATOR_NAMESPACE:-kai-resource-isolator}"
+ISOLATOR_RELEASE="${ISOLATOR_RELEASE:-kai-resource-isolator}"
+
+HELM_ARGS=(
+  upgrade --install "${ISOLATOR_RELEASE}" "${ISOLATOR_CHART_REF}"
+  --namespace "${ISOLATOR_NAMESPACE}"
+  --create-namespace
+  --set monitor.enabled=true
+  --wait
+  --timeout 5m
+)
+
+# --version only applies to OCI/repo charts, not a local filesystem chart path.
+if [[ "${ISOLATOR_CHART_REF}" == oci://* ]] || [[ "${ISOLATOR_CHART_REF}" == *://* ]]; then
+  HELM_ARGS+=(--version "${ISOLATOR_CHART_VERSION}")
+fi
+
+# shellcheck disable=SC2206
+if [[ -n "${ISOLATOR_HELM_EXTRA_ARGS:-}" ]]; then
+  EXTRA=( ${ISOLATOR_HELM_EXTRA_ARGS} )
+  HELM_ARGS+=("${EXTRA[@]}")
+fi
+
+echo "Installing kai-resource-isolator from ${ISOLATOR_CHART_REF} (monitor.enabled=true)..."
+helm "${HELM_ARGS[@]}"
+
+echo "Waiting for isolator webhook deployment..."
+kubectl -n "${ISOLATOR_NAMESPACE}" rollout status \
+  "deployment/${ISOLATOR_RELEASE}-webhook" --timeout=180s
+
+echo "kai-resource-isolator installed (mutating webhook + monitor chart resources)."

hack/run-e2e-kind.sh

@@ -11,6 +11,7 @@ GOBIN=${GOPATH}/bin
 
 # Parse named parameters
 TEST_THIRD_PARTY_INTEGRATIONS="false"
+TEST_HAMI="false"
 LOCAL_IMAGES_BUILD="false"
 PRESERVE_CLUSTER="false"
 
@@ -20,6 +21,10 @@ while [[ $# -gt 0 ]]; do
       TEST_THIRD_PARTY_INTEGRATIONS="true"
       shift
       ;;
+    --test-hami)
+      TEST_HAMI="true"
+      shift
+      ;;
     --local-images-build)
       LOCAL_IMAGES_BUILD="true"
       shift
@@ -29,8 +34,9 @@ while [[ $# -gt 0 ]]; do
       shift
       ;;
     -h|--help)
-      echo "Usage: $0 [--test-third-party-integrations] [--local-images-build] [--preserve-cluster]"
+      echo "Usage: $0 [--test-third-party-integrations] [--test-hami] [--local-images-build] [--preserve-cluster]"
       echo "  --test-third-party-integrations: Install third party operators for compatibility testing"
+      echo "  --test-hami: Enable HAMi/hamicore integration (binder hamicore plugin + kai-resource-isolator)"
       echo "  --local-images-build: Build and use local images instead of pulling from registry"
       echo "  --preserve-cluster: Keep the kind cluster after running the test suite"
       exit 0
@@ -48,6 +54,9 @@ SETUP_ARGS=""
 if [ "$TEST_THIRD_PARTY_INTEGRATIONS" = "true" ]; then
     SETUP_ARGS="$SETUP_ARGS --test-third-party-integrations"
 fi
+if [ "$TEST_HAMI" = "true" ]; then
+    SETUP_ARGS="$SETUP_ARGS --test-hami"
+fi
 if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then
     SETUP_ARGS="$SETUP_ARGS --local-images-build"
 fi

hack/setup-e2e-cluster.sh

@@ -31,6 +31,7 @@ trap cleanup EXIT
 
 # Parse named parameters
 TEST_THIRD_PARTY_INTEGRATIONS=${TEST_THIRD_PARTY_INTEGRATIONS:-"false"}
+TEST_HAMI=${TEST_HAMI:-"false"}
 LOCAL_IMAGES_BUILD=${LOCAL_IMAGES_BUILD:-"false"}
 INSTALL_VPA=${INSTALL_VPA:-"false"}
 SKIP_KAI_INSTALL=${SKIP_KAI_INSTALL:-"false"}
@@ -41,6 +42,10 @@ while [[ $# -gt 0 ]]; do
       TEST_THIRD_PARTY_INTEGRATIONS="true"
       shift
       ;;
+    --test-hami)
+      TEST_HAMI="true"
+      shift
+      ;;
     --local-images-build)
       LOCAL_IMAGES_BUILD="true"
       shift
@@ -62,8 +67,9 @@ while [[ $# -gt 0 ]]; do
       shift 2
       ;;
     -h|--help)
-      echo "Usage: $0 [--test-third-party-integrations] [--local-images-build] [--install-vpa] [--skip-kai-install] [--feature-config <config>] [--kind-config <path>]"
+      echo "Usage: $0 [--test-third-party-integrations] [--test-hami] [--local-images-build] [--install-vpa] [--skip-kai-install] [--feature-config <config>] [--kind-config <path>]"
       echo "  --test-third-party-integrations: Install third party operators for compatibility testing"
+      echo "  --test-hami: Enable HAMi/hamicore integration (binder hamicore plugin + kai-resource-isolator)"
       echo "  --local-images-build: Build and use local images instead of pulling from registry"
       echo "  --install-vpa: Install Vertical Pod Autoscaler and metrics-server"
       echo "  --skip-kai-install: Prepare the cluster (and images/chart with --local-images-build) without installing KAI (e.g. for gitops e2e tests)"
@@ -79,6 +85,14 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
+# Extra helm --set flags for HAMi/hamicore when --test-hami is set.
+HAMI_HELM_SETS=()
+if [ "$TEST_HAMI" = "true" ]; then
+  HAMI_HELM_SETS+=(
+    --set "binder.plugins.hamicore.enabled=true"
+  )
+fi
+
 if [[ -n "$KIND_CONFIG" && "$FEATURE_CONFIG" != "default" ]]; then
   echo "--feature-config cannot be used together with --kind-config"
   exit 1
@@ -204,7 +218,8 @@ if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then
     else
         helm upgrade -i kai-scheduler ./charts/kai-scheduler-$PACKAGE_VERSION.tgz -n kai-scheduler --create-namespace \
         --values ${REPO_ROOT}/hack/kai-scheduler-fake-npe-values.yaml \
-        --set "global.gpuSharing=true" --set "global.registry=localhost:30100" --set "prometheus.enabled=true" --debug --wait
+        --set "global.gpuSharing=true" --set "global.registry=localhost:30100" --set "prometheus.enabled=true" \
+        "${HAMI_HELM_SETS[@]}" --debug --wait
         rm -rf ./charts/kai-scheduler-$PACKAGE_VERSION.tgz
     fi
     cd ${REPO_ROOT}/hack
@@ -213,7 +228,8 @@ elif [ "$SKIP_KAI_INSTALL" = "true" ]; then
 else
     helm upgrade -i kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler -n kai-scheduler --create-namespace \
         --values ${REPO_ROOT}/hack/kai-scheduler-fake-npe-values.yaml \
-        --set "global.gpuSharing=true" --set "prometheus.enabled=true" --wait --version "$PACKAGE_VERSION"
+        --set "global.gpuSharing=true" --set "prometheus.enabled=true" \
+        "${HAMI_HELM_SETS[@]}" --wait --version "$PACKAGE_VERSION"
 fi
 
 if [ "$SKIP_KAI_INSTALL" != "true" ]; then
@@ -222,4 +238,11 @@ if [ "$SKIP_KAI_INSTALL" != "true" ]; then
     kubectl create rolebinding fake-status-updater --clusterrole=pods-patcher --serviceaccount=gpu-operator:status-updater -n kai-resource-reservation
 fi
 
+# HAMi resource isolation (kai-resource-isolator). Deeper than third-party CRD
+# operators: paired with binder hamicore helm values above. Soft-gated in
+# hamicore e2e when the webhook / monitor are absent.
+if [ "$TEST_HAMI" = "true" ]; then
+    ${REPO_ROOT}/hack/hami/deploy_isolator.sh
+fi
+
 echo "Cluster setup complete. Cluster name: $CLUSTER_NAME"