Add hami metrics e2e test

Signed-off-by: dttung2905 <ttdao.2015@accountancy.smu.edu.sg>

Author: dttung2905

hack/setup-e2e-cluster.sh

@@ -147,6 +147,9 @@ if [ "$TEST_THIRD_PARTY_INTEGRATIONS" = "true" ]; then
     ${REPO_ROOT}/hack/third_party_integrations/deploy_knative.sh
     ${REPO_ROOT}/hack/third_party_integrations/deploy_lws.sh
     ${REPO_ROOT}/hack/third_party_integrations/deploy_jobset.sh
+    # HAMi-core / GPU-sharing isolation + optional VRAM metrics (:9394).
+    # Soft-gated in hamicore e2e when the webhook / monitor are absent.
+    ${REPO_ROOT}/hack/third_party_integrations/deploy_isolator.sh
 fi
 
 # Build and install kai-scheduler

hack/third_party_integrations/deploy_isolator.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2026 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+#
+# Installs kai-resource-isolator with kai-vgpu-monitor enabled so hamicore e2e
+# can scrape hami_* per-container VRAM metrics on :9394.
+#
+# Overrides (optional):
+#   ISOLATOR_CHART_REF       OCI ref or local chart path
+#                           (default: oci://docker.io/projecthami/kai-resource-isolator)
+#   ISOLATOR_CHART_VERSION   Chart version when using OCI (default: 1.1.0-chart)
+#   ISOLATOR_NAMESPACE       Install namespace (default: kai-resource-isolator)
+#   ISOLATOR_RELEASE         Helm release name (default: kai-resource-isolator)
+#   ISOLATOR_HELM_EXTRA_ARGS Extra args appended to helm upgrade (word-split)
+set -euo pipefail
+
+ISOLATOR_CHART_REF="${ISOLATOR_CHART_REF:-oci://docker.io/projecthami/kai-resource-isolator}"
+ISOLATOR_CHART_VERSION="${ISOLATOR_CHART_VERSION:-1.1.0-chart}"
+ISOLATOR_NAMESPACE="${ISOLATOR_NAMESPACE:-kai-resource-isolator}"
+ISOLATOR_RELEASE="${ISOLATOR_RELEASE:-kai-resource-isolator}"
+
+HELM_ARGS=(
+  upgrade --install "${ISOLATOR_RELEASE}" "${ISOLATOR_CHART_REF}"
+  --namespace "${ISOLATOR_NAMESPACE}"
+  --create-namespace
+  --set monitor.enabled=true
+  --wait
+  --timeout 5m
+)
+
+# --version only applies to OCI/repo charts, not a local filesystem chart path.
+if [[ "${ISOLATOR_CHART_REF}" == oci://* ]] || [[ "${ISOLATOR_CHART_REF}" == *://* ]]; then
+  HELM_ARGS+=(--version "${ISOLATOR_CHART_VERSION}")
+fi
+
+# shellcheck disable=SC2206
+if [[ -n "${ISOLATOR_HELM_EXTRA_ARGS:-}" ]]; then
+  EXTRA=( ${ISOLATOR_HELM_EXTRA_ARGS} )
+  HELM_ARGS+=("${EXTRA[@]}")
+fi
+
+echo "Installing kai-resource-isolator from ${ISOLATOR_CHART_REF} (monitor.enabled=true)..."
+helm "${HELM_ARGS[@]}"
+
+echo "Waiting for isolator webhook deployment..."
+kubectl -n "${ISOLATOR_NAMESPACE}" rollout status \
+  "deployment/${ISOLATOR_RELEASE}-webhook" --timeout=180s
+
+echo "kai-resource-isolator installed (mutating webhook + monitor chart resources)."

test/e2e/suites/integrations/third_party/hamicore/hamicore_test.go

@@ -5,17 +5,23 @@ SPDX-License-Identifier: Apache-2.0
 package hamicore
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/ptr"
 
 	kaiv1binder "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1/binder"
@@ -32,13 +38,51 @@ import (
 
 const (
 	kaiResourceIsolatorWebhookName = "kai-resource-isolator-mutating"
+	kaiResourceIsolatorNamespace   = "kai-resource-isolator"
+	kaiVGPUMonitorDaemonSetName    = "kai-resource-isolator-monitor"
+	kaiVGPUMonitorLabel            = "app.kubernetes.io/component=kai-vgpu-monitor"
+	kaiVGPUMonitorMetricsPort      = "9394"
 	binderDeploymentName           = "binder"
 	binderDeploymentNamespace      = "kai-scheduler"
 	binderPluginsFlag              = "--plugins"
 	cudaImage                      = "nvidia/cuda:12.6.0-base-ubuntu22.04"
-	gpuMemoryRequestMiB            = 2000
+	// devel image is needed so the metrics workload can nvcc a tiny cudaMalloc hold binary.
+	cudaDevelImage      = "nvidia/cuda:12.6.0-devel-ubuntu22.04"
+	gpuMemoryRequestMiB = 2000
+	cudaAllocHoldMiB    = 64
+	vgpuMetricsTimeout  = 60 * time.Second
+	vgpuMetricsInterval = 5 * time.Second
+	hamiVGPUMemoryUsed  = "hami_vgpu_memory_used_bytes"
 )
 
+// Compiles and runs a small CUDA alloc that holds memory so kai-vgpu-monitor can
+// scrape hami_vgpu_memory_used_bytes > 0. Kept as a container entrypoint (not
+// ExecInPod) so the allocation outlives the ready wait.
+const cudaAllocHoldScript = `set -euo pipefail
+cat >/tmp/hold.cu <<'EOF'
+#include <cuda_runtime.h>
+#include <stdio.h>
+#include <unistd.h>
+int main(void) {
+  void *p = NULL;
+  size_t bytes = (size_t)64 * 1024 * 1024;
+  cudaError_t err = cudaMalloc(&p, bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc failed: %s\n", cudaGetErrorString(err));
+    return 1;
+  }
+  printf("allocated %zu bytes at %p\n", bytes, p);
+  fflush(stdout);
+  for (;;) {
+    sleep(3600);
+  }
+  return 0;
+}
+EOF
+nvcc -O0 -o /tmp/hold /tmp/hold.cu
+exec /tmp/hold
+`
+
 var _ = Describe("HAMi-core resource isolation", Ordered, func() {
 	var testCtx *testcontext.TestContext
 
@@ -191,6 +235,45 @@ var _ = Describe("HAMi-core resource isolation", Ordered, func() {
 				"nvidia-smi visible memory (%d MiB) should match CUDA_DEVICE_MEMORY_LIMIT (%d MiB)",
 				visibleMemMiB, limitMiB)
 		})
+
+	It("gpu-memory: kai-vgpu-monitor reports hami_vgpu_memory_used_bytes > 0",
+		Label(labels.ReservationPod), func(ctx context.Context) {
+			if !isKaiVGPUMonitorInstalled(ctx, testCtx.KubeClientset) {
+				Skip(fmt.Sprintf(
+					"kai-vgpu-monitor DaemonSet %q not found in namespace %q; "+
+						"install via hack/third_party_integrations/deploy_isolator.sh",
+					kaiVGPUMonitorDaemonSetName, kaiResourceIsolatorNamespace,
+				))
+			}
+
+			pod := rd.CreatePodObject(testCtx.Queues[0], v1.ResourceRequirements{})
+			pod.Annotations[constants.GpuMemory] = strconv.Itoa(gpuMemoryRequestMiB)
+			pod.Spec.Containers[0].Image = cudaDevelImage
+			pod.Spec.Containers[0].Command = []string{"bash", "-c"}
+			pod.Spec.Containers[0].Args = []string{cudaAllocHoldScript}
+
+			_, err := rd.CreatePod(ctx, testCtx.KubeClientset, pod)
+			Expect(err).NotTo(HaveOccurred())
+			wait.ForPodReady(ctx, testCtx.ControllerClient, pod)
+
+			pod, err = rd.GetPod(ctx, testCtx.KubeClientset, pod.Namespace, pod.Name)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(pod.Spec.NodeName).NotTo(BeEmpty(), "pod should be scheduled to a node")
+			containerName := pod.Spec.Containers[0].Name
+
+			By("waiting for kai-vgpu-monitor to expose hami_vgpu_memory_used_bytes for the workload")
+			Eventually(func(g Gomega) {
+				used, scrapeErr := scrapeHamiVGPUMemoryUsedBytes(
+					ctx, testCtx.KubeClientset, pod.Spec.NodeName, pod.Namespace, pod.Name, containerName,
+				)
+				g.Expect(scrapeErr).NotTo(HaveOccurred())
+				GinkgoLogr.Info("hami_vgpu_memory_used_bytes",
+					"namespace", pod.Namespace, "pod", pod.Name, "container", containerName, "bytes", used)
+				g.Expect(used).To(BeNumerically(">", 0),
+					"expected hami_vgpu_memory_used_bytes > 0 for %s/%s container %s (held ~%d MiB)",
+					pod.Namespace, pod.Name, containerName, cudaAllocHoldMiB)
+			}, vgpuMetricsTimeout, vgpuMetricsInterval).Should(Succeed())
+		})
 })
 
 func printNvidiaSmi(ctx context.Context, testCtx *testcontext.TestContext, pod *v1.Pod) {
@@ -262,3 +345,88 @@ func hamiCoreEnabledInPluginsJSON(raw string) bool {
 	}
 	return ptr.Deref(cfg.Enabled, false)
 }
+
+func isKaiVGPUMonitorInstalled(ctx context.Context, client kubernetes.Interface) bool {
+	_, err := client.AppsV1().DaemonSets(kaiResourceIsolatorNamespace).
+		Get(ctx, kaiVGPUMonitorDaemonSetName, metav1.GetOptions{})
+	return err == nil
+}
+
+func scrapeHamiVGPUMemoryUsedBytes(
+	ctx context.Context,
+	client kubernetes.Interface,
+	nodeName, namespace, podName, containerName string,
+) (float64, error) {
+	monitorPod, err := findMonitorPodOnNode(ctx, client, nodeName)
+	if err != nil {
+		return 0, err
+	}
+
+	raw, err := client.CoreV1().Pods(monitorPod.Namespace).
+		ProxyGet("http", monitorPod.Name, kaiVGPUMonitorMetricsPort, "/metrics", nil).
+		DoRaw(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("proxy GET /metrics from monitor pod %s/%s: %w",
+			monitorPod.Namespace, monitorPod.Name, err)
+	}
+
+	parser := expfmt.NewTextParser(model.UTF8Validation)
+	families, err := parser.TextToMetricFamilies(bytes.NewReader(raw))
+	if err != nil {
+		return 0, fmt.Errorf("parse prometheus metrics: %w", err)
+	}
+
+	family, ok := families[hamiVGPUMemoryUsed]
+	if !ok {
+		return 0, fmt.Errorf("metric %s not present in monitor scrape", hamiVGPUMemoryUsed)
+	}
+
+	want := map[string]string{
+		"namespace": namespace,
+		"pod":       podName,
+		"container": containerName,
+	}
+	for _, metric := range family.GetMetric() {
+		if !metricHasLabels(metric, want) {
+			continue
+		}
+		if metric.GetGauge() == nil {
+			return 0, fmt.Errorf("metric %s for %v is not a gauge", hamiVGPUMemoryUsed, want)
+		}
+		return metric.GetGauge().GetValue(), nil
+	}
+	return 0, fmt.Errorf("metric %s with labels %v not found", hamiVGPUMemoryUsed, want)
+}
+
+func findMonitorPodOnNode(ctx context.Context, client kubernetes.Interface, nodeName string) (*v1.Pod, error) {
+	pods, err := client.CoreV1().Pods(kaiResourceIsolatorNamespace).List(ctx, metav1.ListOptions{
+		LabelSelector: kaiVGPUMonitorLabel,
+		FieldSelector: "spec.nodeName=" + nodeName,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list kai-vgpu-monitor pods on node %s: %w", nodeName, err)
+	}
+	for i := range pods.Items {
+		pod := &pods.Items[i]
+		if rd.IsPodReady(pod) {
+			return pod, nil
+		}
+	}
+	if len(pods.Items) == 0 {
+		return nil, fmt.Errorf("no kai-vgpu-monitor pods scheduled on node %s", nodeName)
+	}
+	return nil, fmt.Errorf("kai-vgpu-monitor pod(s) on node %s are not Ready", nodeName)
+}
+
+func metricHasLabels(metric *dto.Metric, want map[string]string) bool {
+	got := make(map[string]string, len(metric.GetLabel()))
+	for _, label := range metric.GetLabel() {
+		got[label.GetName()] = label.GetValue()
+	}
+	for k, v := range want {
+		if got[k] != v {
+			return false
+		}
+	}
+	return true
+}