diff --git a/docs/gpu-sharing/hami/README.md b/docs/gpu-sharing/hami/README.md index 7d265c109..acfc45512 100644 --- a/docs/gpu-sharing/hami/README.md +++ b/docs/gpu-sharing/hami/README.md @@ -144,3 +144,81 @@ curl {pod-ip}:9394/metrics ### Memory value precision The `gpu-memory` annotation accepts an **integer in MiB** (no unit suffix). Internally, KAI-Scheduler converts this to a GPU fraction with 2-decimal precision, which is then multiplied against the total GPU memory to compute the actual limit. As a result, the value seen in `nvidia-smi` may differ slightly from the requested value. For example, requesting `4096` MiB on a `15360` MiB GPU (T4) rounds to a `0.27` fraction, yielding `4147m` as the enforced limit. + +## Local e2e testing + +The HAMi-core e2e suite lives at `test/e2e/suites/integrations/third_party/hamicore/`. It checks KAI’s isolation contract (`CUDA_DEVICE_MEMORY_LIMIT` injection and limited `nvidia-smi` visible memory). It is **not wired into CI**: KAI PR e2e runs on kind with the fake GPU operator and has no real GPUs, so these specs soft-skip unless the `hamicore` binder plugin and the `kai-resource-isolator` mutating webhook are present. + +Use a machine with a real NVIDIA GPU (for example minikube with `--gpus=all`). Docker must be able to run `docker run --rm --gpus all nvidia/cuda:12.6.0-base-ubuntu22.04 nvidia-smi` before you start. + +### 1. Start minikube with GPU access + +```bash +minikube start --driver=docker --gpus=all --cpus=6 --memory=12288 +kubectl config use-context minikube +``` + +### 2. NVIDIA device plugin + node labels + +The binder and e2e helpers read `nvidia.com/gpu.memory` (MiB per GPU). Label the node with the card’s total from `nvidia-smi` (example below is an 11264 MiB 2080 Ti): + +```bash +kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.1/deployments/static/nvidia-device-plugin.yml + +kubectl label node minikube \ + nvidia.com/gpu.present=true \ + nvidia.com/gpu.memory=11264 \ + --overwrite + +kubectl -n kube-system rollout status ds/nvidia-device-plugin-daemonset --timeout=180s +kubectl get node minikube -o jsonpath='{.status.allocatable.nvidia\.com/gpu}{"\n"}{.metadata.labels.nvidia\.com/gpu.memory}{"\n"}' +``` + +### 3. Install KAI with GPU sharing + hamicore + +If the cluster has no `RuntimeClass` named `nvidia` (common on bare device-plugin minikube), clear the default runtime-class settings so admission does not reject fraction pods: + +```bash +helm upgrade -i kai-scheduler \ + oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler \ + --namespace kai-scheduler --create-namespace \ + --version v0.17.0 \ + --set global.gpuSharing=true \ + --set binder.plugins.hamicore.enabled=true \ + --set binder.resourceReservation.runtimeClassName="" \ + --set admission.gpuFractionRuntimeClassName="" \ + --wait +``` + +### 4. Install kai-resource-isolator + +Prefer the helper under `hack/hami/` (also used by `--test-hami` in kind cluster setup): + +```bash +# from the KAI-Scheduler repo root +./hack/hami/deploy_isolator.sh + +# or a local isolator chart checkout: +# ISOLATOR_CHART_REF=/path/to/KAI-resource-isolator/chart/kai-resource-isolator \ +# ./hack/hami/deploy_isolator.sh +``` + +Confirm the webhook exists: + +```bash +kubectl get mutatingwebhookconfiguration kai-resource-isolator-mutating +kubectl -n kai-resource-isolator get deploy,ds,pods +``` + +### 5. Run the suite + +```bash +export PATH="$(go env GOPATH)/bin:$PATH" +go install github.com/onsi/ginkgo/v2/ginkgo@latest + +ginkgo -v --trace ./test/e2e/suites/integrations/third_party/hamicore/ +``` + +### Optional: kind helper flag + +`hack/setup-e2e-cluster.sh --test-hami` (and `hack/run-e2e-kind.sh --test-hami`) enables `binder.plugins.hamicore.enabled=true` and runs `hack/hami/deploy_isolator.sh`. That is useful for install plumbing on kind, but the hamicore specs still need a real GPU and will skip or fail against the fake GPU operator alone. diff --git a/hack/hami/deploy_isolator.sh b/hack/hami/deploy_isolator.sh new file mode 100755 index 000000000..0ace54c31 --- /dev/null +++ b/hack/hami/deploy_isolator.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2026 NVIDIA CORPORATION +# SPDX-License-Identifier: Apache-2.0 +# +# Installs kai-resource-isolator with kai-vgpu-monitor enabled for HAMi/hamicore +# e2e (see --test-hami in hack/setup-e2e-cluster.sh). +# +# Overrides (optional): +# ISOLATOR_CHART_REF OCI ref or local chart path +# (default: oci://docker.io/projecthami/kai-resource-isolator) +# ISOLATOR_CHART_VERSION Chart version when using OCI (default: 1.1.0-chart) +# ISOLATOR_NAMESPACE Install namespace (default: kai-resource-isolator) +# ISOLATOR_RELEASE Helm release name (default: kai-resource-isolator) +# ISOLATOR_HELM_EXTRA_ARGS Extra args appended to helm upgrade (word-split) +set -euo pipefail + +ISOLATOR_CHART_REF="${ISOLATOR_CHART_REF:-oci://docker.io/projecthami/kai-resource-isolator}" +ISOLATOR_CHART_VERSION="${ISOLATOR_CHART_VERSION:-1.1.0-chart}" +ISOLATOR_NAMESPACE="${ISOLATOR_NAMESPACE:-kai-resource-isolator}" +ISOLATOR_RELEASE="${ISOLATOR_RELEASE:-kai-resource-isolator}" + +HELM_ARGS=( + upgrade --install "${ISOLATOR_RELEASE}" "${ISOLATOR_CHART_REF}" + --namespace "${ISOLATOR_NAMESPACE}" + --create-namespace + --set monitor.enabled=true + --wait + --timeout 5m +) + +# --version only applies to OCI/repo charts, not a local filesystem chart path. +if [[ "${ISOLATOR_CHART_REF}" == oci://* ]] || [[ "${ISOLATOR_CHART_REF}" == *://* ]]; then + HELM_ARGS+=(--version "${ISOLATOR_CHART_VERSION}") +fi + +# shellcheck disable=SC2206 +if [[ -n "${ISOLATOR_HELM_EXTRA_ARGS:-}" ]]; then + EXTRA=( ${ISOLATOR_HELM_EXTRA_ARGS} ) + HELM_ARGS+=("${EXTRA[@]}") +fi + +echo "Installing kai-resource-isolator from ${ISOLATOR_CHART_REF} (monitor.enabled=true)..." +helm "${HELM_ARGS[@]}" + +echo "Waiting for isolator webhook deployment..." +kubectl -n "${ISOLATOR_NAMESPACE}" rollout status \ + "deployment/${ISOLATOR_RELEASE}-webhook" --timeout=180s + +echo "kai-resource-isolator installed (mutating webhook + monitor chart resources)." diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 0e5b8002e..496332653 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -11,6 +11,7 @@ GOBIN=${GOPATH}/bin # Parse named parameters TEST_THIRD_PARTY_INTEGRATIONS="false" +TEST_HAMI="false" LOCAL_IMAGES_BUILD="false" PRESERVE_CLUSTER="false" @@ -20,6 +21,10 @@ while [[ $# -gt 0 ]]; do TEST_THIRD_PARTY_INTEGRATIONS="true" shift ;; + --test-hami) + TEST_HAMI="true" + shift + ;; --local-images-build) LOCAL_IMAGES_BUILD="true" shift @@ -29,8 +34,9 @@ while [[ $# -gt 0 ]]; do shift ;; -h|--help) - echo "Usage: $0 [--test-third-party-integrations] [--local-images-build] [--preserve-cluster]" + echo "Usage: $0 [--test-third-party-integrations] [--test-hami] [--local-images-build] [--preserve-cluster]" echo " --test-third-party-integrations: Install third party operators for compatibility testing" + echo " --test-hami: Enable HAMi/hamicore integration (binder hamicore plugin + kai-resource-isolator)" echo " --local-images-build: Build and use local images instead of pulling from registry" echo " --preserve-cluster: Keep the kind cluster after running the test suite" exit 0 @@ -48,6 +54,9 @@ SETUP_ARGS="" if [ "$TEST_THIRD_PARTY_INTEGRATIONS" = "true" ]; then SETUP_ARGS="$SETUP_ARGS --test-third-party-integrations" fi +if [ "$TEST_HAMI" = "true" ]; then + SETUP_ARGS="$SETUP_ARGS --test-hami" +fi if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then SETUP_ARGS="$SETUP_ARGS --local-images-build" fi diff --git a/hack/setup-e2e-cluster.sh b/hack/setup-e2e-cluster.sh index 321d5088d..87f02bc9c 100755 --- a/hack/setup-e2e-cluster.sh +++ b/hack/setup-e2e-cluster.sh @@ -31,6 +31,7 @@ trap cleanup EXIT # Parse named parameters TEST_THIRD_PARTY_INTEGRATIONS=${TEST_THIRD_PARTY_INTEGRATIONS:-"false"} +TEST_HAMI=${TEST_HAMI:-"false"} LOCAL_IMAGES_BUILD=${LOCAL_IMAGES_BUILD:-"false"} INSTALL_VPA=${INSTALL_VPA:-"false"} SKIP_KAI_INSTALL=${SKIP_KAI_INSTALL:-"false"} @@ -41,6 +42,10 @@ while [[ $# -gt 0 ]]; do TEST_THIRD_PARTY_INTEGRATIONS="true" shift ;; + --test-hami) + TEST_HAMI="true" + shift + ;; --local-images-build) LOCAL_IMAGES_BUILD="true" shift @@ -62,8 +67,9 @@ while [[ $# -gt 0 ]]; do shift 2 ;; -h|--help) - echo "Usage: $0 [--test-third-party-integrations] [--local-images-build] [--install-vpa] [--skip-kai-install] [--feature-config ] [--kind-config ]" + echo "Usage: $0 [--test-third-party-integrations] [--test-hami] [--local-images-build] [--install-vpa] [--skip-kai-install] [--feature-config ] [--kind-config ]" echo " --test-third-party-integrations: Install third party operators for compatibility testing" + echo " --test-hami: Enable HAMi/hamicore integration (binder hamicore plugin + kai-resource-isolator)" echo " --local-images-build: Build and use local images instead of pulling from registry" echo " --install-vpa: Install Vertical Pod Autoscaler and metrics-server" echo " --skip-kai-install: Prepare the cluster (and images/chart with --local-images-build) without installing KAI (e.g. for gitops e2e tests)" @@ -79,6 +85,14 @@ while [[ $# -gt 0 ]]; do esac done +# Extra helm --set flags for HAMi/hamicore when --test-hami is set. +HAMI_HELM_SETS=() +if [ "$TEST_HAMI" = "true" ]; then + HAMI_HELM_SETS+=( + --set "binder.plugins.hamicore.enabled=true" + ) +fi + if [[ -n "$KIND_CONFIG" && "$FEATURE_CONFIG" != "default" ]]; then echo "--feature-config cannot be used together with --kind-config" exit 1 @@ -204,7 +218,8 @@ if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then else helm upgrade -i kai-scheduler ./charts/kai-scheduler-$PACKAGE_VERSION.tgz -n kai-scheduler --create-namespace \ --values ${REPO_ROOT}/hack/kai-scheduler-fake-npe-values.yaml \ - --set "global.gpuSharing=true" --set "global.registry=localhost:30100" --set "prometheus.enabled=true" --debug --wait + --set "global.gpuSharing=true" --set "global.registry=localhost:30100" --set "prometheus.enabled=true" \ + "${HAMI_HELM_SETS[@]}" --debug --wait rm -rf ./charts/kai-scheduler-$PACKAGE_VERSION.tgz fi cd ${REPO_ROOT}/hack @@ -213,7 +228,8 @@ elif [ "$SKIP_KAI_INSTALL" = "true" ]; then else helm upgrade -i kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler -n kai-scheduler --create-namespace \ --values ${REPO_ROOT}/hack/kai-scheduler-fake-npe-values.yaml \ - --set "global.gpuSharing=true" --set "prometheus.enabled=true" --wait --version "$PACKAGE_VERSION" + --set "global.gpuSharing=true" --set "prometheus.enabled=true" \ + "${HAMI_HELM_SETS[@]}" --wait --version "$PACKAGE_VERSION" fi if [ "$SKIP_KAI_INSTALL" != "true" ]; then @@ -222,4 +238,11 @@ if [ "$SKIP_KAI_INSTALL" != "true" ]; then kubectl create rolebinding fake-status-updater --clusterrole=pods-patcher --serviceaccount=gpu-operator:status-updater -n kai-resource-reservation fi +# HAMi resource isolation (kai-resource-isolator). Deeper than third-party CRD +# operators: paired with binder hamicore helm values above. Soft-gated in +# hamicore e2e when the webhook / monitor are absent. +if [ "$TEST_HAMI" = "true" ]; then + ${REPO_ROOT}/hack/hami/deploy_isolator.sh +fi + echo "Cluster setup complete. Cluster name: $CLUSTER_NAME"