feat(e2e): Under testing kai hack deployments, Add hami testing option, installing kai-resource-isolator

hack/hami/deploy_isolator.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2026 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+#
+# Installs kai-resource-isolator with kai-vgpu-monitor enabled for HAMi/hamicore
+# e2e (see --test-hami in hack/setup-e2e-cluster.sh).
+#
+# Overrides (optional):
+#   ISOLATOR_CHART_REF       OCI ref or local chart path
+#                           (default: oci://docker.io/projecthami/kai-resource-isolator)
+#   ISOLATOR_CHART_VERSION   Chart version when using OCI (default: 1.1.0-chart)
+#   ISOLATOR_NAMESPACE       Install namespace (default: kai-resource-isolator)
+#   ISOLATOR_RELEASE         Helm release name (default: kai-resource-isolator)
+#   ISOLATOR_HELM_EXTRA_ARGS Extra args appended to helm upgrade (word-split)
+set -euo pipefail
+
+ISOLATOR_CHART_REF="${ISOLATOR_CHART_REF:-oci://docker.io/projecthami/kai-resource-isolator}"
+ISOLATOR_CHART_VERSION="${ISOLATOR_CHART_VERSION:-1.1.0-chart}"
+ISOLATOR_NAMESPACE="${ISOLATOR_NAMESPACE:-kai-resource-isolator}"
+ISOLATOR_RELEASE="${ISOLATOR_RELEASE:-kai-resource-isolator}"
+
+HELM_ARGS=(
+  upgrade --install "${ISOLATOR_RELEASE}" "${ISOLATOR_CHART_REF}"
+  --namespace "${ISOLATOR_NAMESPACE}"
+  --create-namespace
+  --set monitor.enabled=true
+  --wait
+  --timeout 5m
+)
+
+# --version only applies to OCI/repo charts, not a local filesystem chart path.
+if [[ "${ISOLATOR_CHART_REF}" == oci://* ]] || [[ "${ISOLATOR_CHART_REF}" == *://* ]]; then
+  HELM_ARGS+=(--version "${ISOLATOR_CHART_VERSION}")
+fi
+
+# shellcheck disable=SC2206
+if [[ -n "${ISOLATOR_HELM_EXTRA_ARGS:-}" ]]; then
+  EXTRA=( ${ISOLATOR_HELM_EXTRA_ARGS} )
+  HELM_ARGS+=("${EXTRA[@]}")
+fi
+
+echo "Installing kai-resource-isolator from ${ISOLATOR_CHART_REF} (monitor.enabled=true)..."
+helm "${HELM_ARGS[@]}"
+
+echo "Waiting for isolator webhook deployment..."
+kubectl -n "${ISOLATOR_NAMESPACE}" rollout status \
+  "deployment/${ISOLATOR_RELEASE}-webhook" --timeout=180s
+
+echo "kai-resource-isolator installed (mutating webhook + monitor chart resources)."

hack/run-e2e-kind.sh

@@ -11,6 +11,7 @@ GOBIN=${GOPATH}/bin
 
 # Parse named parameters
 TEST_THIRD_PARTY_INTEGRATIONS="false"
+TEST_HAMI="false"
 LOCAL_IMAGES_BUILD="false"
 PRESERVE_CLUSTER="false"
 
@@ -20,6 +21,10 @@ while [[ $# -gt 0 ]]; do
       TEST_THIRD_PARTY_INTEGRATIONS="true"
       shift
       ;;
+    --test-hami)
+      TEST_HAMI="true"
+      shift
+      ;;
     --local-images-build)
       LOCAL_IMAGES_BUILD="true"
       shift
@@ -29,8 +34,9 @@ while [[ $# -gt 0 ]]; do
       shift
       ;;
     -h|--help)
-      echo "Usage: $0 [--test-third-party-integrations] [--local-images-build] [--preserve-cluster]"
+      echo "Usage: $0 [--test-third-party-integrations] [--test-hami] [--local-images-build] [--preserve-cluster]"
       echo "  --test-third-party-integrations: Install third party operators for compatibility testing"
+      echo "  --test-hami: Enable HAMi/hamicore integration (binder hamicore plugin + kai-resource-isolator)"
       echo "  --local-images-build: Build and use local images instead of pulling from registry"
       echo "  --preserve-cluster: Keep the kind cluster after running the test suite"
       exit 0
@@ -48,6 +54,9 @@ SETUP_ARGS=""
 if [ "$TEST_THIRD_PARTY_INTEGRATIONS" = "true" ]; then
     SETUP_ARGS="$SETUP_ARGS --test-third-party-integrations"
 fi
+if [ "$TEST_HAMI" = "true" ]; then
+    SETUP_ARGS="$SETUP_ARGS --test-hami"
+fi
 if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then
     SETUP_ARGS="$SETUP_ARGS --local-images-build"
 fi

hack/setup-e2e-cluster.sh

@@ -31,6 +31,7 @@ trap cleanup EXIT
 
 # Parse named parameters
 TEST_THIRD_PARTY_INTEGRATIONS=${TEST_THIRD_PARTY_INTEGRATIONS:-"false"}
+TEST_HAMI=${TEST_HAMI:-"false"}
 LOCAL_IMAGES_BUILD=${LOCAL_IMAGES_BUILD:-"false"}
 INSTALL_VPA=${INSTALL_VPA:-"false"}
 SKIP_KAI_INSTALL=${SKIP_KAI_INSTALL:-"false"}
@@ -41,6 +42,10 @@ while [[ $# -gt 0 ]]; do
       TEST_THIRD_PARTY_INTEGRATIONS="true"
       shift
       ;;
+    --test-hami)
+      TEST_HAMI="true"
+      shift
+      ;;
     --local-images-build)
       LOCAL_IMAGES_BUILD="true"
       shift
@@ -62,8 +67,9 @@ while [[ $# -gt 0 ]]; do
       shift 2
       ;;
     -h|--help)
-      echo "Usage: $0 [--test-third-party-integrations] [--local-images-build] [--install-vpa] [--skip-kai-install] [--feature-config <config>] [--kind-config <path>]"
+      echo "Usage: $0 [--test-third-party-integrations] [--test-hami] [--local-images-build] [--install-vpa] [--skip-kai-install] [--feature-config <config>] [--kind-config <path>]"
       echo "  --test-third-party-integrations: Install third party operators for compatibility testing"
+      echo "  --test-hami: Enable HAMi/hamicore integration (binder hamicore plugin + kai-resource-isolator)"
       echo "  --local-images-build: Build and use local images instead of pulling from registry"
       echo "  --install-vpa: Install Vertical Pod Autoscaler and metrics-server"
       echo "  --skip-kai-install: Prepare the cluster (and images/chart with --local-images-build) without installing KAI (e.g. for gitops e2e tests)"
@@ -79,6 +85,17 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
+# Extra helm --set flags for HAMi/hamicore. Kind e2e has no nvidia RuntimeClass, so clear
+# the defaults that would otherwise reject GPU-fraction pods at admission.
+HAMI_HELM_SETS=()
+if [ "$TEST_HAMI" = "true" ]; then
+  HAMI_HELM_SETS+=(
+    --set "binder.plugins.hamicore.enabled=true"
+    --set "binder.resourceReservation.runtimeClassName="
+    --set "admission.gpuFractionRuntimeClassName="
+  )
+fi
+
 if [[ -n "$KIND_CONFIG" && "$FEATURE_CONFIG" != "default" ]]; then
   echo "--feature-config cannot be used together with --kind-config"
   exit 1
@@ -204,7 +221,8 @@ if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then
     else
         helm upgrade -i kai-scheduler ./charts/kai-scheduler-$PACKAGE_VERSION.tgz -n kai-scheduler --create-namespace \
         --values ${REPO_ROOT}/hack/kai-scheduler-fake-npe-values.yaml \
-        --set "global.gpuSharing=true" --set "global.registry=localhost:30100" --set "prometheus.enabled=true" --debug --wait
+        --set "global.gpuSharing=true" --set "global.registry=localhost:30100" --set "prometheus.enabled=true" \
+        "${HAMI_HELM_SETS[@]}" --debug --wait
         rm -rf ./charts/kai-scheduler-$PACKAGE_VERSION.tgz
     fi
     cd ${REPO_ROOT}/hack
@@ -213,7 +231,8 @@ elif [ "$SKIP_KAI_INSTALL" = "true" ]; then
 else
     helm upgrade -i kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler -n kai-scheduler --create-namespace \
         --values ${REPO_ROOT}/hack/kai-scheduler-fake-npe-values.yaml \
-        --set "global.gpuSharing=true" --set "prometheus.enabled=true" --wait --version "$PACKAGE_VERSION"
+        --set "global.gpuSharing=true" --set "prometheus.enabled=true" \
+        "${HAMI_HELM_SETS[@]}" --wait --version "$PACKAGE_VERSION"
 fi
 
 if [ "$SKIP_KAI_INSTALL" != "true" ]; then
@@ -222,4 +241,11 @@ if [ "$SKIP_KAI_INSTALL" != "true" ]; then
     kubectl create rolebinding fake-status-updater --clusterrole=pods-patcher --serviceaccount=gpu-operator:status-updater -n kai-resource-reservation
 fi
 
+# HAMi resource isolation (kai-resource-isolator). Deeper than third-party CRD
+# operators: paired with binder hamicore helm values above. Soft-gated in
+# hamicore e2e when the webhook / monitor are absent.
+if [ "$TEST_HAMI" = "true" ]; then
+    ${REPO_ROOT}/hack/hami/deploy_isolator.sh
+fi
+
 echo "Cluster setup complete. Cluster name: $CLUSTER_NAME"

test/e2e/suites/integrations/third_party/hamicore/hamicore_test.go

@@ -5,17 +5,23 @@ SPDX-License-Identifier: Apache-2.0
 package hamicore
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/ptr"
 
 	kaiv1binder "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1/binder"
@@ -32,13 +38,51 @@ import (
 
 const (
 	kaiResourceIsolatorWebhookName = "kai-resource-isolator-mutating"
+	kaiResourceIsolatorNamespace   = "kai-resource-isolator"
+	kaiVGPUMonitorDaemonSetName    = "kai-resource-isolator-monitor"
+	kaiVGPUMonitorLabel            = "app.kubernetes.io/component=kai-vgpu-monitor"
+	kaiVGPUMonitorMetricsPort      = "9394"
 	binderDeploymentName           = "binder"
 	binderDeploymentNamespace      = "kai-scheduler"
 	binderPluginsFlag              = "--plugins"
 	cudaImage                      = "nvidia/cuda:12.6.0-base-ubuntu22.04"
-	gpuMemoryRequestMiB            = 2000
+	// devel image is needed so the metrics workload can nvcc a tiny cudaMalloc hold binary.
+	cudaDevelImage      = "nvidia/cuda:12.6.0-devel-ubuntu22.04"
+	gpuMemoryRequestMiB = 2000
+	cudaAllocHoldMiB    = 64
+	vgpuMetricsTimeout  = 60 * time.Second
+	vgpuMetricsInterval = 5 * time.Second
+	hamiVGPUMemoryUsed  = "hami_vgpu_memory_used_bytes"
 )
 
+// Compiles and runs a small CUDA alloc that holds memory so kai-vgpu-monitor can
+// scrape hami_vgpu_memory_used_bytes > 0. Kept as a container entrypoint (not
+// ExecInPod) so the allocation outlives the ready wait.
+const cudaAllocHoldScript = `set -euo pipefail
+cat >/tmp/hold.cu <<'EOF'
+#include <cuda_runtime.h>
+#include <stdio.h>
+#include <unistd.h>
+int main(void) {
+  void *p = NULL;
+  size_t bytes = (size_t)64 * 1024 * 1024;
+  cudaError_t err = cudaMalloc(&p, bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(err));
+    return 1;
+  }
+  printf("allocated %zu bytes at %p\n", bytes, p);
+  fflush(stdout);
+  for (;;) {
+    sleep(3600);
+  }
+  return 0;
+}
+EOF
+nvcc -O0 -o /tmp/hold /tmp/hold.cu
+exec /tmp/hold
+`
+
 var _ = Describe("HAMi-core resource isolation", Ordered, func() {
 	var testCtx *testcontext.TestContext
 
@@ -191,6 +235,45 @@ var _ = Describe("HAMi-core resource isolation", Ordered, func() {
 				"nvidia-smi visible memory (%d MiB) should match CUDA_DEVICE_MEMORY_LIMIT (%d MiB)",
 				visibleMemMiB, limitMiB)
 		})
+
+	It("gpu-memory: kai-vgpu-monitor reports hami_vgpu_memory_used_bytes > 0",
+		Label(labels.ReservationPod), func(ctx context.Context) {
+			if !isKaiVGPUMonitorInstalled(ctx, testCtx.KubeClientset) {
+				Skip(fmt.Sprintf(
+					"kai-vgpu-monitor DaemonSet %q not found in namespace %q; "+
+						"install via hack/hami/deploy_isolator.sh (--test-hami)",
+					kaiVGPUMonitorDaemonSetName, kaiResourceIsolatorNamespace,
+				))
+			}
+
+			pod := rd.CreatePodObject(testCtx.Queues[0], v1.ResourceRequirements{})
+			pod.Annotations[constants.GpuMemory] = strconv.Itoa(gpuMemoryRequestMiB)
+			pod.Spec.Containers[0].Image = cudaDevelImage
+			pod.Spec.Containers[0].Command = []string{"bash", "-c"}
+			pod.Spec.Containers[0].Args = []string{cudaAllocHoldScript}
+
+			_, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
+			Expect(err).NotTo(HaveOccurred())
+			wait.ForPodReady(ctx, testCtx.ControllerClient, pod)
+
+			pod, err = rd.GetPod(ctx, testCtx.KubeClientset, pod.Namespace, pod.Name)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(pod.Spec.NodeName).NotTo(BeEmpty(), "pod should be scheduled to a node")
+			containerName := pod.Spec.Containers[0].Name
+
+			By("waiting for kai-vgpu-monitor to expose hami_vgpu_memory_used_bytes for the workload")
+			Eventually(func(g Gomega) {
+				used, scrapeErr := scrapeHamiVGPUMemoryUsedBytes(
+					ctx, testCtx.KubeClientset, pod.Spec.NodeName, pod.Namespace, pod.Name, containerName,
+				)
+				g.Expect(scrapeErr).NotTo(HaveOccurred())
+				GinkgoLogr.Info("hami_vgpu_memory_used_bytes",
+					"namespace", pod.Namespace, "pod", pod.Name, "container", containerName, "bytes", used)
+				g.Expect(used).To(BeNumerically(">", 0),
+					"expected hami_vgpu_memory_used_bytes > 0 for %s/%s container %s (held ~%d MiB)",
+					pod.Namespace, pod.Name, containerName, cudaAllocHoldMiB)
+			}, vgpuMetricsTimeout, vgpuMetricsInterval).Should(Succeed())
+		})
 })
 
 func printNvidiaSmi(ctx context.Context, testCtx *testcontext.TestContext, pod *v1.Pod) {
@@ -262,3 +345,88 @@ func hamiCoreEnabledInPluginsJSON(raw string) bool {
 	}
 	return ptr.Deref(cfg.Enabled, false)
 }
+
+func isKaiVGPUMonitorInstalled(ctx context.Context, client kubernetes.Interface) bool {
+	_, err := client.AppsV1().DaemonSets(kaiResourceIsolatorNamespace).
+		Get(ctx, kaiVGPUMonitorDaemonSetName, metav1.GetOptions{})
+	return err == nil
+}
+
+func scrapeHamiVGPUMemoryUsedBytes(
+	ctx context.Context,
+	client kubernetes.Interface,
+	nodeName, namespace, podName, containerName string,
+) (float64, error) {
+	monitorPod, err := findMonitorPodOnNode(ctx, client, nodeName)
+	if err != nil {
+		return 0, err
+	}
+
+	raw, err := client.CoreV1().Pods(monitorPod.Namespace).
+		ProxyGet("http", monitorPod.Name, kaiVGPUMonitorMetricsPort, "/metrics", nil).
+		DoRaw(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("proxy GET /metrics from monitor pod %s/%s: %w",
+			monitorPod.Namespace, monitorPod.Name, err)
+	}
+
+	parser := expfmt.NewTextParser(model.UTF8Validation)
+	families, err := parser.TextToMetricFamilies(bytes.NewReader(raw))
+	if err != nil {
+		return 0, fmt.Errorf("parse prometheus metrics: %w", err)
+	}
+
+	family, ok := families[hamiVGPUMemoryUsed]
+	if !ok {
+		return 0, fmt.Errorf("metric %s not present in monitor scrape", hamiVGPUMemoryUsed)
+	}
+
+	want := map[string]string{
+		"namespace": namespace,
+		"pod":       podName,
+		"container": containerName,
+	}
+	for _, metric := range family.GetMetric() {
+		if !metricHasLabels(metric, want) {
+			continue
+		}
+		if metric.GetGauge() == nil {
+			return 0, fmt.Errorf("metric %s for %v is not a gauge", hamiVGPUMemoryUsed, want)
+		}
+		return metric.GetGauge().GetValue(), nil
+	}
+	return 0, fmt.Errorf("metric %s with labels %v not found", hamiVGPUMemoryUsed, want)
+}
+
+func findMonitorPodOnNode(ctx context.Context, client kubernetes.Interface, nodeName string) (*v1.Pod, error) {
+	pods, err := client.CoreV1().Pods(kaiResourceIsolatorNamespace).List(ctx, metav1.ListOptions{
+		LabelSelector: kaiVGPUMonitorLabel,
+		FieldSelector: "spec.nodeName=" + nodeName,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list kai-vgpu-monitor pods on node %s: %w", nodeName, err)
+	}
+	for i := range pods.Items {
+		pod := &pods.Items[i]
+		if rd.IsPodReady(pod) {
+			return pod, nil
+		}
+	}
+	if len(pods.Items) == 0 {
+		return nil, fmt.Errorf("no kai-vgpu-monitor pods scheduled on node %s", nodeName)
+	}
+	return nil, fmt.Errorf("kai-vgpu-monitor pod(s) on node %s are not Ready", nodeName)
+}
+
+func metricHasLabels(metric *dto.Metric, want map[string]string) bool {
+	got := make(map[string]string, len(metric.GetLabel()))
+	for _, label := range metric.GetLabel() {
+		got[label.GetName()] = label.GetValue()
+	}
+	for k, v := range want {
+		if got[k] != v {
+			return false
+		}
+	}
+	return true
+}