diff --git a/Makefile b/Makefile index 1571dc252..318ac9588 100644 --- a/Makefile +++ b/Makefile @@ -330,6 +330,11 @@ test-coverage: check-coverage-threshold test ## Runs tests and enforces coverage fi; \ echo "Coverage check passed" +.PHONY: update-goldens +update-goldens: ## Regenerates golden test fixtures (catalog/coverage/render parity); review with git diff before committing + @GOFLAGS="-mod=readonly" AICR_UPDATE_GOLDEN=1 go test -count=1 -run 'TestCatalogParityGolden|TestCoverageGoldenMatrix' ./pkg/recipe/ + @GOFLAGS="-mod=readonly" AICR_UPDATE_GOLDEN=1 go test -count=1 -run 'TestStockRenderParityGolden' ./pkg/bundler/ + .PHONY: bench bench: ## Runs benchmarks @echo "Running benchmarks..." diff --git a/docs/README.md b/docs/README.md index da5da7873..4112d266c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -46,7 +46,7 @@ For pipelines and platforms that call AICR programmatically or host | Add or modify recipe metadata | [Recipe Development](integrator/recipe-development.md) | | Verify artifacts (SLSA, SBOM, attestations) | [Supply Chain Verification](integrator/supply-chain-verification.md) | | Ship custom validators via `--data` | [Validator Extension](integrator/validator-extension.md) | -| Cloud-specific GPU setup | [AKS](integrator/aks-gpu-setup.md), [GKE](integrator/gke-gpu-setup.md), [EKS networking](integrator/eks-dynamo-networking.md), [GKE networking](integrator/gke-tcpxo-networking.md), [Talos](integrator/talos-integration.md) | +| Cloud-specific GPU setup | [AKS](integrator/aks-gpu-setup.md), [GKE](integrator/gke-gpu-setup.md), [EKS networking](integrator/eks-dynamo-networking.md), [GKE TCPXO networking](integrator/gke-tcpxo-networking.md), [GKE GB200 networking](integrator/gke-gb200-networking.md), [Talos](integrator/talos-integration.md) | ### Contributor Guide diff --git a/docs/contributor/validator.md b/docs/contributor/validator.md index 78d1a2973..1d28fb8cd 100644 --- a/docs/contributor/validator.md +++ b/docs/contributor/validator.md @@ -804,7 +804,7 @@ default** (`Qwen/Qwen3-8B` at 256/GPU). A non-positive / non-integer | `AICR_INFERENCE_PERF_WORKLOAD_READY_TIMEOUT` | `10m` | Wait for the `DynamoGraphDeployment` to become ready (image pull + model load + worker health). Large models load slower — raise this **and** the catalog entry's `timeout` in tandem, or the parent deadline caps it. | | `AICR_INFERENCE_PERF_HEALTH_TIMEOUT` | `5m` | Wait for the endpoint to serve a real chat-completion *after* the workload reports Ready. Concurrent first-load from one RWO cache PVC can push first-serve past 5m; raise it (bounded by the catalog `timeout`). | | `AICR_INFERENCE_PERF_MODEL_CACHE_SIZE` | `100Gi` (on) | The PVC-backed model-weights cache is **on by default**. Set a different K8s quantity to resize, or a disable sentinel (`off`/`0`/`none`/`disabled`) to turn it off and download from HF directly. | -| `AICR_INFERENCE_PERF_MODEL_CACHE_STORAGE_CLASS` | cluster default | StorageClass for the cache PVC. On a cluster with **no default SC and no value here**, the check **fails fast** with guidance rather than leaving the PVC `Pending` until timeout. AICR-deployed EKS gets a default `gp3` SC from `aws-ebs-csi-driver`; GKE has `standard-rwo`. | +| `AICR_INFERENCE_PERF_MODEL_CACHE_STORAGE_CLASS` | cluster default | StorageClass for the cache PVC. On a cluster with **no default SC and no value here**, the check **fails fast** with guidance rather than leaving the PVC `Pending` until timeout. AICR-deployed EKS gets a default `gp3` SC from `aws-ebs-csi-driver`; GKE has `standard-rwo`, **except A4X/GB200 nodes**, which reject `standard-rwo`'s `pd-balanced` disks and need a Hyperdisk-backed class (see [GKE GB200 Storage Prerequisites](../integrator/gke-gb200-networking.md#storage-prerequisites)). | | `AICR_INFERENCE_PERF_MODEL_CACHE_POPULATE_TIMEOUT` | `13m` | Wait for the one-time model-cache populate Job (cold image pull + first-ever Hugging Face download into the PVC). Separate from — and larger than — `AICR_INFERENCE_PERF_WORKLOAD_READY_TIMEOUT` because the populate Job pays a cold pull *and* a multi-GB download; provide the optional HF-token secret to remove anonymous-download throttling. Raise it (and the catalog `timeout`) for very large models. **Migration:** the cache-populate wait no longer honors `AICR_INFERENCE_PERF_WORKLOAD_READY_TIMEOUT` (which now bounds only the DynamoGraphDeployment readiness wait) — set this knob instead to widen the populate budget. | For gated models, or to lift Hugging Face rate limits on large downloads, diff --git a/docs/index.yml b/docs/index.yml index 162175988..d79e180a1 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -77,6 +77,8 @@ navigation: path: integrator/eks-dynamo-networking.md - page: GKE TCPXO Networking path: integrator/gke-tcpxo-networking.md + - page: GKE GB200 Networking + path: integrator/gke-gb200-networking.md - page: OpenShift Deployment path: integrator/openshift.md - page: Talos Integration diff --git a/docs/integrator/components/nodewright.md b/docs/integrator/components/nodewright.md index 62092593c..a3f467244 100644 --- a/docs/integrator/components/nodewright.md +++ b/docs/integrator/components/nodewright.md @@ -87,6 +87,7 @@ The table below is generated from the recipes by `make tuning-docs` — **do not | eks | rtx-pro-6000 | generic | - | nvidia-tuned 0.3.2 | | gke | a100 | h100 | - | nvidia-tuning-gke 0.1.2 | | gke | b200 | - | - | nvidia-tuning-gke 0.1.2 | +| gke | gb200 | - | - | nvidia-tuning-gke 0.1.2 | | gke | h100 | - | - | nvidia-tuning-gke 0.1.2 | {/* END AICR-TUNING */} diff --git a/docs/integrator/gke-gb200-networking.md b/docs/integrator/gke-gb200-networking.md new file mode 100644 index 000000000..16b18fa75 --- /dev/null +++ b/docs/integrator/gke-gb200-networking.md @@ -0,0 +1,248 @@ +# GKE GB200 (A4X) Networking Prerequisites + +For the **GB200 GKE COS** recipes (`gb200-gke-cos-training`, +`gb200-gke-cos-training-kubeflow`, `gb200-gke-cos-training-slurm`, +`gb200-gke-cos-inference`, and `gb200-gke-cos-inference-dynamo`, all on +`a4x-highgpu-4g` nodes), +GPUDirect-RDMA over RoCE enables high-speed inter-node GPU communication on +GKE. The recipe's NCCL workloads set `NCCL_NET=gIB` explicitly (see +`recipes/components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml`) +rather than letting NCCL auto-select a plugin, so a missing or +misconfigured RDMA fabric doesn't silently fall back to a slower network +path: it fails outright. + +GPUDirect RDMA on `a4x-highgpu-4g` is also incompatible with NCCL Fast +Socket and the GPUDirect TCPX/TCPXO plugin (see +[GKE TCPXO Networking](gke-tcpxo-networking.md) for that alternative, +non-RDMA path); don't enable either on a cluster that uses RDMA. + +## Infrastructure Prerequisites + +GKE clusters must have multi-networking configured before deploying AICR bundles: + +- Multi-networking enabled (1 gVNIC + 4 RDMA NICs per `a4x-highgpu-4g` node) +- `Network` + `GKENetworkParamSet` CRs for the gVNIC and 4 RDMA NICs (cluster-specific + VPC/subnet values, but fixed object names; see below, not managed by AICR) +- `nccl-rdma-installer` DaemonSet on GPU nodes (included in the AICR bundle) +- Each GPUDirect-RDMA workload Pod must request all 4 GPUs and use all 4 RDMA NICs + on a single node; RDMA can't be shared between Pods on the same node (a GKE + `a4x-highgpu-4g` constraint, not an AICR-specific one). AICR's own recipes + already request whole nodes this way; a custom workload built against this + component must too. + +The `nccl-rdma-installer` DaemonSet ships in the AICR bundle. The `Network`/ +`GKENetworkParamSet` CRs and the multi-networking/VPC fabric underneath them +are **cluster provisioning**: AICR's `gke-gb200-rdma` health check detects +them but does not create them. + +### Provisioning multi-networking + +These steps are ordered, following Google's +[A4X custom setup guide](https://docs.cloud.google.com/ai-hypercomputer/docs/create/gke-ai-hypercompute-custom-a4x): + +1. **Create the VPCs and subnets**: two VPCs in the cluster's region, one for + the gVNIC (with one subnet) and one RDMA VPC (with four subnets, one per + RDMA NIC); five subnets total across the two VPCs, not five separate VPCs. +2. **Create the cluster** with multi-networking enabled (HIPPO's `GKECluster` CR + does this via `spec.networks.managed.gb200NetworkStrategy`). +3. **Create the GPU node pool** on an `a4x-highgpu-4g` machine type, attaching + the five network/subnet pairs as `additionalNodeNetworkConfigs` (the RDMA + VPC repeated across its four subnets, plus the gVNIC VPC/subnet). +4. **Apply the `Network` and `GKENetworkParamSet` CRs**: one pair per NIC, + binding each additional node network into the cluster so pods can reference + it. Unlike TCPXO (see [GKE TCPXO Networking](gke-tcpxo-networking.md)), the + **object names are fixed, not cluster-specific**: `gvnic-1` for the gVNIC and + `rdma-0` through `rdma-3` for the RDMA NICs. Only the `vpc`/`vpcSubnet` fields + inside each `GKENetworkParamSet` vary per cluster (they name the VPC/subnet + your cluster actually has): + +```yaml +apiVersion: networking.gke.io/v1 +kind: GKENetworkParamSet +metadata: + name: gvnic-1 +spec: + vpc: "PREFIX-gvnic" + vpcSubnet: "PREFIX-gvnic" + deviceMode: NetDevice +--- +apiVersion: networking.gke.io/v1 +kind: Network +metadata: + name: gvnic-1 +spec: + type: "Device" + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: gvnic-1 +``` + + Repeat for `rdma-0` through `rdma-3`, pointing `vpc` at the single RDMA VPC + from step 1 (the same value for all four) and `vpcSubnet` at that VPC's + four subnets (`PREFIX-rdma-sub-0` through `PREFIX-rdma-sub-3`, or whatever + names your subnets were given in step 1, with `PREFIX` replaced by your + own), and set **`deviceMode: RDMA`** on all four, not `NetDevice` (that + value is only correct for `gvnic-1` above). + +> **The fixed naming is a requirement, not a convention.** AICR's +> `checks/gke-gb200-rdma/health-check.yaml` asserts these five objects by exact +> name (`gvnic-1`, `rdma-0`..`rdma-3`), including `spec.deviceMode` and +> `spec.parametersRef` linkage. A cluster provisioned with different `Network` +> names passes Google's own setup guide but fails this check; rename to match +> before running `aicr validate`. + +AICR installs the `nccl-rdma-installer` DaemonSet and detects the CRs; it does +not provision the networking itself. These steps are a summary of the +prerequisite AICR depends on, not a complete provisioning runbook; follow +Google's guide above for the full procedure, including firewall rules and +supported GKE version floors. + +Separately from GKE's own networking version floor, all AICR GB200 GKE +recipes (including `gb200-gke-cos-training-slurm`, which inherits it from +`gb200-gke-cos-training`) enforce `K8s.server.version >= 1.34`: NVLS +provisions the IMEX channel through a DRA `ComputeDomain`, which requires +the GA `resource.k8s.io/v1` API. `aicr validate` fails readiness on an +older control plane with this constraint by name. + +### Verifying + +```shell +kubectl get network.networking.gke.io \ + -o custom-columns='NAME:.metadata.name,PARAMETERS-REF:.spec.parametersRef.name' +kubectl get gkenetworkparamset.networking.gke.io \ + -o custom-columns='NAME:.metadata.name,DEVICE-MODE:.spec.deviceMode' +``` + +Expect `gvnic-1` and `rdma-0` through `rdma-3` (the five prerequisite +`Network`s from step 4), each bound to its `GKENetworkParamSet` via +`spec.parametersRef` (shown in the `PARAMETERS-REF` column above). Fewer +than five, or a `GKENetworkParamSet` with the wrong `DEVICE-MODE`, means +the prerequisite is incomplete or misconfigured; `aicr validate` (via the +`gke-gb200-rdma` health check) reports the shortfall by name. + +You'll also see a `default` network/`GKENetworkParamSet` pair in the same +output; that one is GKE-managed (created automatically once +multi-networking is enabled), not part of this prerequisite, and isn't +checked by name. + +## Driver Installer + +`a4x-highgpu-4g` recipes generated with `--profile gpuStack=bundle-installer` +(see [GKE GPU Setup](gke-gpu-setup.md#alternative-let-the-bundle-own-the-gpu-stack)) +get the driver from the bundle's `gcp-driver-installer` component — Google's +cos-gpu-installer DaemonSet, deployed and versioned by AICR alongside the +rest of the bundle, no manual DaemonSet apply required. This presumes the +node-pool prerequisite (pools created with `gpu-driver-version=disabled` +plus the `gke-no-default-nvidia-gpu-device-plugin=true` label) is already in +place; the component's `nodeAffinity` requires that label itself, so it +never schedules onto a pool that hasn't opted out of GKE's managed install. + +**GB200-specific wrinkle:** the component's default `partitionGpuImage` (its +`partition-gpus` init container, Google's `nvidia-partition-gpu` MIG tool — +a no-op here since this recipe allocates whole GPUs per node rather than +configuring MIG) is pinned to an amd64-only digest and fails with +`exec format error` on GB200's arm64 nodes. AICR's `gb200-gke-cos-training` +and `gb200-gke-cos-inference` overlays (and everything based on them) set +`gcp-driver-installer.partitionGpuImage` to a multi-arch digest +automatically — found via live GB200 GKE validation, no action needed: + +```yaml +partitionGpuImage: "gcr.io/gke-release/nvidia-partition-gpu@sha256:de12f85ebfb4fb6c1893cd30c23aab662a72fa0448f97ef74fccb82d7522ef17" +``` + +The driver version itself (`gcp-driver-installer.driverVersion`) needs no +GB200-specific override: the component's default is +[COS-qualified](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#cos) +for GB200 (see the component's `values.yaml` for the qualified COS builds). + +### Validate the RDMA prerequisite before deploying + +The `gcp-driver-installer` component deploys and orders itself ahead of the +GPU Operator automatically within `deploy.sh` (see [Driver +Installer](#driver-installer)) — there's nothing to hand-apply or pre-check +for the driver anymore. The RDMA `Network`/`GKENetworkParamSet` CRs are +still a true cluster-provisioning prerequisite applied before the node pool +exists (see [Provisioning multi-networking](#provisioning-multi-networking) +above); confirm they're in place before running the bundle's full +`deploy.sh`: + +```shell +aicr validate --recipe recipe.yaml --phase deployment --fail-fast +``` + +The `gke-gb200-rdma` health check only needs the RDMA CRs to exist, not the +rest of the bundle deployed, so this catches un-applied CRs in seconds +instead of surfacing them deep into a 20-component deploy. `--fail-fast` +stops there instead of continuing on to conformance and performance (see +[Validation](../user/validation.md)). Note `check-nvidia-smi` in the same +`deployment` phase only passes once the driver installer has actually run +(inside `deploy.sh`, not before it), so don't run this check before the +node pool itself exists — expect `check-nvidia-smi` to fail until +`deploy.sh` has deployed `gcp-driver-installer`. + +## Storage Prerequisites + +`a4x-highgpu-4g` nodes can't attach Persistent Disk at all (regional or +zonal, any type, including `pd-balanced`); only Hyperdisk. On a stock GKE +Standard cluster the default StorageClass is `standard-rwo` +(`pd.csi.storage.gke.io`, `pd-balanced`), but "default" isn't inherent to +GKE Standard itself: a cluster admin can repoint the +`storageclass.kubernetes.io/is-default-class` annotation to any +StorageClass. Run `kubectl get storageclass` first and check which one is +annotated `(default)`, its `PROVISIONER`, and (via `kubectl get +storageclass -o yaml`) its `parameters.type`; don't assume it's +`standard-rwo`/`pd-balanced`. Any PVC scheduled onto a GB200 node with no +`storageClassName` set (which binds it to the cluster default) fails +this way unless that default's `parameters.type` is already +Hyperdisk-backed: `pd-balanced disk type cannot be used by +a4x-highgpu-4g machine type` (or the equivalent for whatever `pd-*` type +the default actually provisions). + +This includes the `inference-perf` validator's model-weights cache PVC +when `AICR_INFERENCE_PERF_MODEL_CACHE_STORAGE_CLASS` (see +[Validation](../user/validation.md)) is left unset, it then falls back +to the cluster default too. Set that variable to name a Hyperdisk-backed +StorageClass explicitly (for example `hyperdisk-balanced`, applied below) +and the cache PVC uses it directly via `storageClassName`, independent of +whatever the cluster default resolves to. + +If the cluster default isn't already Hyperdisk-backed, apply one. Like +the RDMA CRs above, this is a cluster prerequisite AICR does not +provision: + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +parameters: + type: hyperdisk-balanced +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +``` + +Apply it once per cluster, then point the validator's model cache at it via +an `AICR_INFERENCE_PERF_MODEL_CACHE_STORAGE_CLASS=hyperdisk-balanced` entry +on the `inference-perf` catalog entry's `env` (or a catalog overlay in the +`aicr validate --data ` directory). + +## Running the NCCL Benchmark + +The GB200 GKE training recipe (`gb200-gke-cos-training`) selects the +NVLS-variant performance check (`nccl-all-reduce-bw-nvls`): MNNVL across the +A4X nodes' IMEX domain is the fabric that carries all-reduce traffic; gIB is the +transport driver underneath, not the NCCL algorithm itself. Run it via: + +```shell +aicr validate --recipe recipes/overlays/gb200-gke-cos-training.yaml \ + --phase performance +``` + +## References + +- [GKE A4X custom setup guide](https://docs.cloud.google.com/ai-hypercomputer/docs/create/gke-ai-hypercompute-custom-a4x) +- [Component Catalog](../user/component-catalog.md) +- [Validation readiness gate](../user/validation.md) +- [GKE TCPXO Networking](gke-tcpxo-networking.md) diff --git a/docs/integrator/gke-gpu-setup.md b/docs/integrator/gke-gpu-setup.md index d8f028d3e..4cd2e0aa7 100644 --- a/docs/integrator/gke-gpu-setup.md +++ b/docs/integrator/gke-gpu-setup.md @@ -219,6 +219,21 @@ the request against the COS build's curated per-GPU-type list and rejects unqualified versions. Version bumps take effect on replaced or rebooted nodes only (the installer skips nodes with a loaded nvidia module). +On A4X/GB200 (`a4x-highgpu-4g`, arm64) nodes, one override is required: the +component's default `partitionGpuImage` (the `partition-gpus` init +container) is an amd64-only digest and fails with `exec format error` on +arm64. AICR's GB200 GKE recipes set `gcp-driver-installer.partitionGpuImage` +to a multi-arch digest automatically — see +[GKE GB200 Networking › Driver Installer](gke-gb200-networking.md#driver-installer) +for why and the exact digest. + +Before deploying the rest of the bundle, confirm the driver actually landed: +`aicr validate --recipe recipe.yaml --phase deployment --fail-fast` runs +`check-nvidia-smi`, which only needs the GPU nodes to exist, not the rest of +the bundle deployed, so a missing driver fails in seconds instead of +surfacing later as the GPU Operator's toolkit/driver-validation init +containers looping forever. + Set the label when you create the GPU node pool, alongside the disabled managed install: @@ -440,3 +455,4 @@ confirm exactly which advertiser owns each node. - [Component Catalog › GKE Device-Plugin Ownership](../user/component-catalog.md#gke-device-plugin-ownership) - [Validation readiness gate](../user/validation.md) - [GKE TCPXO Networking](gke-tcpxo-networking.md) +- [GKE GB200 Networking](gke-gb200-networking.md) diff --git a/docs/integrator/index.md b/docs/integrator/index.md index c554a1909..913e45a18 100644 --- a/docs/integrator/index.md +++ b/docs/integrator/index.md @@ -22,6 +22,7 @@ This section is for integrators who: | [Kubernetes Deployment](kubernetes-deployment.md) | Self-hosted API server deployment with Kubernetes manifests | | [EKS Dynamo Networking](eks-dynamo-networking.md) | Security group prerequisites for Dynamo overlays on EKS | | [GKE TCPXO Networking](gke-tcpxo-networking.md) | GPUDirect TCPXO prerequisites for GKE training overlays | +| [GKE GB200 Networking](gke-gb200-networking.md) | GPUDirect-RDMA prerequisites for GB200 (A4X) GKE overlays | | [AKS GPU Setup](aks-gpu-setup.md) | AKS prerequisites: Kubernetes 1.34+ (DRA GA), GPU driver setup, DRA configuration | | [GKE GPU Setup](gke-gpu-setup.md) | GKE device-plugin ownership: the `gpuStack` profile, node-pool setup for both values, verification, and troubleshooting | | [Talos Integration](talos-integration.md) | Running AICR on Talos Linux | diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 112201971..be7f1b46b 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -19,8 +19,8 @@ A machine-readable **CycloneDX 1.6 JSON** companion to this page is produced by ## Summary -- Components: **44** -- Unique images: **100** +- Components: **45** +- Unique images: **101** - Distinct registries: **11** Registries: `602401143452.dkr.ecr.us-west-2.amazonaws.com`, `cr.agentgateway.dev`, `docker.io`, `gcr.io`, `ghcr.io`, `gke.gcr.io`, `nvcr.io`, `public.ecr.aws`, `quay.io`, `registry.k8s.io`, `us-docker.pkg.dev` @@ -41,6 +41,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | dynamo-platform | helm | dynamo-platform | 1.2.1 | 3 | | gatekeeper | helm | gatekeeper/gatekeeper | 3.22.2 | 3 | | gcp-driver-installer | manifest | — | — | 3 | +| gke-gb200-rdma | manifest | — | — | 2 | | gke-nccl-tcpxo | manifest | — | — | 4 | | gpu-operator | helm | nvidia/gpu-operator | v26.3.3 | 15 | | gpu-operator-ocp | manifest | — | — | 0 | @@ -141,6 +142,11 @@ _No images extracted._ - `gcr.io/gke-release/nvidia-partition-gpu@sha256:e226275da6c45816959fe43cde907ee9a85c6a2aa8a429418a4cadef8ecdb86a` - `gke.gcr.io/pause:3.8@sha256:880e63f94b145e46f1b1082bb71b85e21f16b99b180b9996407d61240ceb9830` +### gke-gb200-rdma + +- `gke.gcr.io/pause:3.8@sha256:880e63f94b145e46f1b1082bb71b85e21f16b99b180b9996407d61240ceb9830` +- `us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-arm64:v1.1.2@sha256:6b7950cac6e6833661d4206920f5633b6e361b18bfd5315b63f9bf4a4b84a80e` + ### gke-nccl-tcpxo - `gcr.io/gke-release/nri-device-injector:1.0.25-gke.6@sha256:7704e2bd74b8edbb76b6913c7904cc2362f1fa887c4d4aba7b19778ea353537c` diff --git a/docs/user/recipe-health.md b/docs/user/recipe-health.md index 659e22ee9..bb5f011da 100644 --- a/docs/user/recipe-health.md +++ b/docs/user/recipe-health.md @@ -40,8 +40,8 @@ The deep-link is the current Evidence rendering. It is distinct from — and coe {/* BEGIN AICR-HEALTH */} ## Summary -- Recipes: **48** -- Pass: **48** · Warn: **0** · Fail: **0** · Unknown: **0** +- Recipes: **51** +- Pass: **51** · Warn: **0** · Fail: **0** · Unknown: **0** ## Recipes @@ -80,6 +80,9 @@ The deep-link is the current Evidence rendering. It is distinct from — and coe | a100-gke-cos-training-kubeflow | gke | a100 | cos | training | kubeflow | pass | R:0 D:4 P:0 C:10 | pending | | b200-gke-cos-inference-dynamo | gke | b200 | cos | inference | dynamo | pass | R:0 D:4 P:0 C:11 | pending | | b200-gke-cos-training-kubeflow | gke | b200 | cos | training | kubeflow | pass | R:0 D:4 P:0 C:10 | pending | +| gb200-gke-cos-inference-dynamo | gke | gb200 | cos | inference | dynamo | pass | R:0 D:4 P:1 C:11 | pending | +| gb200-gke-cos-training-kubeflow | gke | gb200 | cos | training | kubeflow | pass | R:0 D:4 P:1 C:10 | pending | +| gb200-gke-cos-training-slurm | gke | gb200 | cos | training | slurm | pass | R:0 D:4 P:0 C:12 | pending | | h100-gke-cos-inference-dynamo | gke | h100 | cos | inference | dynamo | pass | R:0 D:4 P:1 C:11 | pending | | h100-gke-cos-training-kubeflow | gke | h100 | cos | training | kubeflow | pass | R:0 D:5 P:1 C:10 | pending | | h100-gke-cos-training-slurm | gke | h100 | cos | training | slurm | pass | R:0 D:5 P:0 C:11 | pending | diff --git a/docs/user/validation.md b/docs/user/validation.md index 936497bd2..0089b7a38 100644 --- a/docs/user/validation.md +++ b/docs/user/validation.md @@ -49,9 +49,9 @@ ones) that match the target fabric: | Check | Transport | Default applicability (from recipe criteria) | |---|---|---| -| `nccl-all-reduce-bw` | Auto-detect (whatever NCCL picks) | H100/H200 on EKS, H100 on GKE, H100 on AKS (ND-series InfiniBand — NCCL's built-in IB/verbs transport over the `rdma/hca_shared_devices_a` shared device pool), and B200/GB200 on self-managed clusters (`service=any`). Preserves the pre-variant behavior. | +| `nccl-all-reduce-bw` | Auto-detect (whatever NCCL picks) | H100/H200 on EKS, H100 on GKE (GPUDirect TCPXO), H100 on AKS (ND-series InfiniBand, NCCL's built-in IB/verbs transport over the `rdma/hca_shared_devices_a` shared device pool), and B200/GB200 on self-managed clusters (`service=any`). Preserves the pre-variant behavior. | | `nccl-all-reduce-bw-net` | NET (EFA on EKS by default; ConnectX RoCE via `AICR_NCCL_FABRIC=roce`) | GB200 + EKS. Asserts EFA actually carried traffic — catches silent fallback to Socket when the NVIDIA driver is missing `NVreg_GrdmaPciTopoCheckOverride=1`. | -| `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS, and GB200 + OKE. Asserts the NVLS communicator actually initialized — catches silent fallback to EFA (EKS) or Socket (OKE) when the IMEX domain is misconfigured. | +| `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS, GB200 + OKE, and GB200 + GKE (A4X GPUDirect-RDMA/gIB carries the IMEX/NVLink fabric traffic; gIB is the transport driver, not the NCCL algorithm). Asserts the NVLS communicator actually initialized, catching silent fallback to EFA (EKS), Socket (OKE), or gIB's own fallback path (GKE) when the IMEX domain is misconfigured. | The applicability column is the *default*, derived from the recipe's `criteria`. A recipe whose criteria fall outside it can still run these @@ -176,7 +176,7 @@ the GPU nodes, exactly as `service: any` recipes do. When `--node-selector` is passed it replaces the automatic filters rather than narrowing them. Valid profiles are the pairs in the applicability table above: `b200/any`, -`gb200/any`, `gb200/eks`, `gb200/oke`, `h100/aks`, `h100/eks`, `h100/gke`, +`gb200/any`, `gb200/eks`, `gb200/gke`, `gb200/oke`, `h100/aks`, `h100/eks`, `h100/gke`, `h200/eks`. A malformed or unknown value **fails** the check rather than silently skipping it. A valid profile that doesn't implement a requested variant (e.g. @@ -399,7 +399,10 @@ uses the cluster's **default** StorageClass unless you set StorageClass** (common on EKS — e.g. only a non-default `gp2`) and no value set, the check **fails fast** in seconds with guidance rather than hanging; set `AICR_INFERENCE_PERF_MODEL_CACHE_STORAGE_CLASS=` (e.g. `gp2`/`gp3` on EKS, -`standard-rwo` on GKE) on the `inference-perf` catalog entry's `env` (or via a +`standard-rwo` on GKE, **except A4X/GB200 `a4x-highgpu-4g` nodes, which reject +`standard-rwo`'s `pd-balanced` disks and need a Hyperdisk-backed class instead; +see [GKE GB200 Storage Prerequisites](../integrator/gke-gb200-networking.md#storage-prerequisites)**) +on the `inference-perf` catalog entry's `env` (or via a catalog overlay in the `aicr validate --data ` directory), or disable the cache with `AICR_INFERENCE_PERF_MODEL_CACHE_SIZE=off`. Like the other `AICR_INFERENCE_PERF_*` knobs, this is a **catalog/`--data`** setting — it is diff --git a/pkg/bundler/stock_render_parity_golden_test.go b/pkg/bundler/stock_render_parity_golden_test.go index ab4ca6bf2..04700c683 100644 --- a/pkg/bundler/stock_render_parity_golden_test.go +++ b/pkg/bundler/stock_render_parity_golden_test.go @@ -112,6 +112,9 @@ func TestStockRenderParityGolden(t *testing.T) { } if os.Getenv("AICR_UPDATE_GOLDEN") == "1" { + if t.Failed() { + t.Fatal("not writing golden: one or more leaves failed to resolve or render (see errors above)") + } writeStockRenderGolden(t, got) t.Logf("golden updated: %d leaves", len(got)) return diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index d43d9f13f..d5ac8e1bc 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,51 +3,54 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: 265940a7ef6f82bfe9752e5e488b416d5d54aa0245b29f0262796978547af68d -a100-any: 457dd24ba4f8b9d44af7f68ddc1f4e9d46d0e6eac30b0eceb2e979e1e9c2d2db -a100-eks-ubuntu-training-kubeflow: 9a77e69e7ea639be6100c4c2c0cb49bf87059348020c98f5c561158a22d452e7 -a100-gke-cos-training-kubeflow: 0fb904af67e4ad85cb1b71efed94c8cd3966fc36f14cc80d9b50d5ffe7babde5 -a100-oke-ubuntu-training-kubeflow: 92c1a420bccca84c7ca7e74d8960af6c3662bce4c708c45ccaa386f209dc1c9e -b200-any: b8e028ff78b7f142c3e157143a258510c4f44579e31f13c183796767de53d862 -b200-gke-cos-inference-dynamo: cd3c475c9bb709b8164571be18ec625b38263a762d91d16f89049e75fc934eb4 -b200-gke-cos-training-kubeflow: f35da200bec023578aaaeebe337775d9b62718cfd2ac0c568c4a56f69575fbb3 -bcm-inference: 2a7d525459b585c3b1639eb556de8feb682b27f6bb8be3a7015de7773ba7c235 -gb200-any: ff2c8128bc91fce4a48d650d160659b08272a346a044f7955d3b23109fde6e8d -gb200-eks-ubuntu-inference-dynamo: 637a4fd32933c220453d3732363c9a56bd75dd91cdd886a9b0571b666670744d -gb200-eks-ubuntu-training-kubeflow: e4d2b1cc86cf33b7741e881a984f6943353bf2e1b85d918b8d3e82cf50c8fa60 -gb200-eks-ubuntu-training-slurm: d356fec2562b9d3f8adcccb2344402f41270ca301f353affd6529aaa86e430d4 -gb200-oke-ubuntu-inference-dynamo: 0304fffcd27d04827ae561b32ced55b9ff87e0f6049bda30a075afd49f24ad11 -gb200-oke-ubuntu-training-kubeflow: 3ed8582775ca602b48eb88ff05d228db3f4189f4230f31f72cf355be8d5241b0 -gb300-any: a67ce96e16a0a22d85f22c2803f029e49b1befb4226a60cdf9820fb6b6e77ea2 -gb300-eks-ubuntu-inference-dynamo: c725636167448e64ef1d567613aded4bcd7ede0137a3d1e4b9c70be1ec20c61a -gb300-eks-ubuntu-training-kubeflow: e84019cb236bae67eef79a8166214295b066d1514ebb869619898925be48522d -h100-aks-ubuntu-inference-dynamo: 54061313326c300b83a0627f415002bd080c7c0e338e19bcce7996446a791065 -h100-aks-ubuntu-training-kubeflow: ea0180c91126bdda49e3641538cfef03961358ebe4d883e893d53a82b26d4bd7 -h100-aks-ubuntu-training-slurm: 7f9c80295d770456adca24b66bc7681e42fd9db54fcb97d2541c657c939645c9 -h100-any: 7e98352317f4d2a753255ec776e7b7bc605ad288f3856b4a467a15f8844aa903 -h100-bcm-ubuntu-training: c73ab59f00b5425a0be7c8aaac6107469c8fb59e54eecce9cf4937704821611d -h100-eks-ubuntu-inference-dynamo: 6a3e47afa2018591b22f2fd96105d75ec23b631fafb4c0f30f42ba77af6f537a -h100-eks-ubuntu-inference-nim: 3b9b4fe53c967fa95e56e2cf6e631a55f044d50f76423e11ac176d839e005d2d -h100-eks-ubuntu-training-kubeflow: 18be43e357c98753a069d5db99f2dbe79f6bbd7c98c91286715968f183639cb0 -h100-eks-ubuntu-training-slurm: fdf26391dd6d2e10b9e853f3bd480c8968f2749948cb58a4a1cf5a1b34b85325 -h100-gke-cos-inference-dynamo: a8527e38ea90f346eee045c6b1b448263c83258c3d99864f73b4e9f86d28d520 -h100-gke-cos-training-kubeflow: 304639b08cf1899fc5cb8bb7a5c1a4f9ff652896ae18a92123c802b2f4cfc341 -h100-gke-cos-training-slurm: 0efee10e13589930111b9f5064430d2dcac463d2d2cf6032a6a7687be2085958 -h100-kind-inference-dynamo: 8cea3d208001c5d7bb542d83a75e2ab641722981b279db9391a300b8f9d1f20c -h100-kind-training-kubeflow: 1203d2e475a51a3e07d3a62a87ad2dca91c24eda3aea800aca86bf2dfee5a80a -h100-kind-training-slurm: 8a27b1ae4e91cda80c4a32d456552952f2ec0a28af10e9e0317219b551ddd133 -h200-any: ceee05a6ed9dd0f218919a90187de851e16e93cf89293c9746d5f0e46bbda222 -h200-eks-inference: 2900b7d96bc48a27eb5fedda75b99b010b9165bc6058372d2de3b991f64c4982 -h200-eks-training: 4b3074954d10a81d057ac746c7bddf3664637dfd002cd0955348ec3dee11eb20 -l40s-any: dabbb23da635f2b6db7803be55841a7305f884ce7798dbf79d952686004f09d8 -l40s-oke-inference: 8a8884dcf64b9feaa708b152cc4eaf1b9c09d790de295cf2b0e5a5ca464154f1 -l40s-oke-training: f9b741b390f9f29451c7ad16da56602bbf6497bec067be6b19242cffdf1e1de7 -monitoring-hpa: 32917a470d982044c3d49960dbd5e62ab56cd870375bfac23c35d7a4b3f4298a -ocp-inference-nim: 016fc59c13e901de5556f09f3cb3d5a90f91b9826d34f126fde1e066b311605f -ocp-training: 652f7ccdf52009adafe993da111857aa67390678f382b4c0caf0aecd0d613635 -rtx-pro-6000-any: 9115c5d61d9941130e448f29b9fbcd834f6d15a46fa4edf3239afc44fcdd36f0 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 8c35c21289dc4a56f3e035da9d35427c9a1f2402c643febda2c6b397692b2a49 -rtx-pro-6000-eks-ubuntu-inference-nim: feb5f1d72ebd95459c9df0891420d20fac42c6b8c16f0e8fa551da61a19aef1d -rtx-pro-6000-eks-ubuntu-training-kubeflow: 1dfe905ac8d891fba4ce797de6532320024fe2616653e4703be5fc8e29716d77 -rtx-pro-6000-lke-ubuntu-inference: c5d53e0adbe9c500ecd5eb45d2482e632f1c414657a08ebef053a8aab90fc3cb -rtx-pro-6000-lke-ubuntu-training: 39e9cc1636a11d2568d9d50ecf3744f5773c4870772727661c40be63bac33280 +a100-aks-ubuntu-training-kubeflow: 058f805dbe8c05f3ae4bc0a0d0cb52340e8b33f6214ab4c9219503d80f673a28 +a100-any: 96af5cdc3525ebc8b624e995fe3f6dce03f70e8781fb2de4d67a58be1b4e876a +a100-eks-ubuntu-training-kubeflow: ee1cf33caf6ac5b924137834301059ee581ed35d783a70b2d7324096fd2c2856 +a100-gke-cos-training-kubeflow: 040f7646a91a8425696606691b483ccb984a4265ae3dbdd52f417396081cd870 +a100-oke-ubuntu-training-kubeflow: 44303bb3ed1a2e23cb5fdb6ed9b466de9f884e59629cefaa75c9858d5562ace1 +b200-any: 0eeabfb6cf7d2831e1440411e007a7a1205f151c3ce4c1972c0604799dd4d101 +b200-gke-cos-inference-dynamo: 35483862da82b84eb6a851d5e476a89c686aedc1a975f981900e06c75248a413 +b200-gke-cos-training-kubeflow: 60243ba23272f37ffdd2bdda617b429d933bc8bacf95a2cf39ae71867b574025 +bcm-inference: afcfa4c15440dcf5b3aa8d340b1e16ba1fe3068acb0b2b93d1a36ed5eabafa31 +gb200-any: 0a3b8ab3fc037ea881e6c11b866ae4e44e1e395387c2648335b2f26fe5772c63 +gb200-eks-ubuntu-inference-dynamo: d49ef07155e427bca68be623c3a7cbea85c62b4579f7319b43123d7aa02ec7c0 +gb200-eks-ubuntu-training-kubeflow: 4527b24746be69a0e5fdda2c0f5c7958db578b3edf858c1eb521a2eb4ff9563d +gb200-eks-ubuntu-training-slurm: 042d1c3c46325f9beefd681c7953b89f942f8f575a02afd462037fa83a116a7d +gb200-gke-cos-inference-dynamo: e95fe52c3e82d00fe716f39442d7d201661ef1be03700b1e4fe8bfa432b8988c +gb200-gke-cos-training-kubeflow: c0c0fc4e25dff2ea5bbc6c885c67ecc056a4410462af134592087635a7e08086 +gb200-gke-cos-training-slurm: 5f15fe3878f23b78e899ab442837418f1357806586fa5943ee87c3870b41ef31 +gb200-oke-ubuntu-inference-dynamo: bf3394536cdaed952a0dd04b78f93a5ea6dd47fba565b74fdf24b72253e5292d +gb200-oke-ubuntu-training-kubeflow: c81d72866d411c00bd06aa923375839c8b50eb95d8999a9a13d8f16d9767504a +gb300-any: 5d7d401d581ae07357934780737bca92d697b3cd5de58a8f490db81ed2a6543a +gb300-eks-ubuntu-inference-dynamo: 01e4e82dbe63e5343e2e315e05155781cb4fd7fa6235bd4030d42a1285038a54 +gb300-eks-ubuntu-training-kubeflow: 097ed23086946fee4bdffdc0931fd7826166a07aa32d7a0925f0bd1fbe38bcac +h100-aks-ubuntu-inference-dynamo: 34b85c9b061535fc333a2e9d8b4606dc25e399851028228139c1f3f084f0595f +h100-aks-ubuntu-training-kubeflow: ea91d117ed3d82b8ac3f6b0a920eb3d98732cb0d7c875392692023c18547533c +h100-aks-ubuntu-training-slurm: 9e8d7b6467af20513f54604623d7bfeb1a424d9aa5acd64dd5bc503f4b70b7f4 +h100-any: 62faf5dfb85d2a09b4f752ac8394f1513fbfd6b8a7b82312a78842770423e38e +h100-bcm-ubuntu-training: 1bd659f86f7cb77495601ae618c2bfd3f4809e08f36e649b30010613b6aa386b +h100-eks-ubuntu-inference-dynamo: 67f1dc6dd3c4ed160ad95c6b8cf7327e74d404cab9d58f9b6e46d4dcac0d3569 +h100-eks-ubuntu-inference-nim: 89a245d0e6c545c1eba3ba48e12b5bbe62dcc77f4ed3167599ad7d55505218ea +h100-eks-ubuntu-training-kubeflow: c103f4a94f7fe059e54438ab736229aad791794fef2df69a48091db78ddc0ed4 +h100-eks-ubuntu-training-slurm: 2a27572ac0dee23f5acb366a6ffc9700044e4c22268f075a787af1ca5b95238a +h100-gke-cos-inference-dynamo: 92615c0bf18ea5a9faa4d78c3d16acd475b945f3f79cab9648308da563135c0d +h100-gke-cos-training-kubeflow: e8aede7d22ed4627b2742e4864ef1a5fe6f62f1b5d2a73f8354dbd2f18855fc8 +h100-gke-cos-training-slurm: 3037c63b6ddf99da71e8d9fa648d7d35f2260771898e8f0234c58c310e9cf276 +h100-kind-inference-dynamo: fb048166eec541bf9744689a08779ce692ecf6113777da90e08a7e36a2383534 +h100-kind-training-kubeflow: 22d923cd34508a63648008b533b9412827154ab78af5ae924468bbb2edf1e2a6 +h100-kind-training-slurm: ffe9053a3e31089f73a5c377a5c03e41d5a89f0ca4f20a593f16c89957a2abfa +h200-any: a19b70296b9c2d5ca7959529d8140fb5ba9369350ab82407c91b131e9de2abd0 +h200-eks-inference: e4de5892f46c9632108cf08b57fb88e749c375848243028abb634ea7e8b2859a +h200-eks-training: 6682ee3e7cd98b471232568acae440d6f1f139a9de2822f27406c82371057da0 +l40s-any: 668f283dbb42891cda12b7e3eb86171e93959b7dbccfe654631444f5c6411769 +l40s-oke-inference: a90232c1d5552ae220f5db3a8aff9010da29a29a93414c6971bb330f9c4538be +l40s-oke-training: 4c4645efa950ccb91bd02a45d9e8504850833f0bb3b6f1c2bd86bd8ecc8ba10e +monitoring-hpa: ae15aee21233b0fc9b92bead8133856abf0da767bbea2a74f36575f6c2e7185b +ocp-inference-nim: 18d952153387a2097527b85785b887134d7fe229282a0ae861b6b11072c5bfde +ocp-training: 779b68a9aaf1c9ab6ac674064982bc5305f1b9624cd75fba0ac997acc21a36d9 +rtx-pro-6000-any: 37221dd7b4237d842021049a0fa44088d8f6a8b4add1444fc148e9dc1d355ffb +rtx-pro-6000-eks-ubuntu-inference-dynamo: 9904a484918e2bffc905caf21e47fd27cf62c5bc32fac6a6f2fab6f073f31337 +rtx-pro-6000-eks-ubuntu-inference-nim: 402e814e5c73f878378fbf2d3fd335d2f6b0f1a42f956ec64c0554436c747e69 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 5ff9c138b9d2f97adb7975fa2d6df5a143ba968e401610919c99ecbcd13775e8 +rtx-pro-6000-lke-ubuntu-inference: dcebe600dc4f7adee1d36d4f858dd8f801df65182dcd74036848c6cd1ad07776 +rtx-pro-6000-lke-ubuntu-training: be74ee802993300261c93015b43c17ab6417efb861054647b9a4057aec76d447 diff --git a/pkg/chainsaw/gke_gb200_rdma_check_states_test.go b/pkg/chainsaw/gke_gb200_rdma_check_states_test.go new file mode 100644 index 000000000..b20299fc3 --- /dev/null +++ b/pkg/chainsaw/gke_gb200_rdma_check_states_test.go @@ -0,0 +1,197 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chainsaw + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/NVIDIA/aicr/pkg/recipe" +) + +// TestGKEGB200RDMAHealthCheckClusterStates drives the shipped health check +// through the in-process executor against synthetic cluster states, covering +// a missing or misbound GKENetworkParamSet/Network on any of the 5 objects. +// +// Each case pins wantOutput, not just the pass/fail verdict, so a case +// cannot pass for the wrong reason (see TestK8sAIBOMHealthCheckClusterStates +// for the same rationale). +func TestGKEGB200RDMAHealthCheckClusterStates(t *testing.T) { + t.Parallel() + + provider := recipe.NewEmbeddedDataProvider(recipe.GetEmbeddedFS(), "") + data, err := provider.ReadFile(context.Background(), "checks/gke-gb200-rdma/health-check.yaml") + if err != nil { + t.Fatalf("read health check: %v", err) + } + + gkeNetworkParamSet := func(name, deviceMode string) map[string]any { + return map[string]any{ + "apiVersion": "networking.gke.io/v1", + "kind": "GKENetworkParamSet", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{ + "vpc": "prefix-net", + "vpcSubnet": "prefix-sub", + "deviceMode": deviceMode, + }, + } + } + gkeNetwork := func(name string) map[string]any { + return map[string]any{ + "apiVersion": "networking.gke.io/v1", + "kind": "Network", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{ + "type": "Device", + "parametersRef": map[string]any{ + "group": "networking.gke.io", + "kind": "GKENetworkParamSet", + "name": name, + }, + }, + } + } + + tests := []struct { + name string + // mutate lets a case delete or corrupt one fixture from the healthy + // baseline before the check runs. + mutate func(f *fakeFetcher) + wantPass bool + wantOutput string + }{ + { + name: "fully healthy cluster", + wantPass: true, + }, + { + name: "missing rdma-1 GKENetworkParamSet fails closed", + mutate: func(f *fakeFetcher) { + delete(f.gets, "networking.gke.io/v1/GKENetworkParamSet//rdma-1") + }, + wantOutput: "rdma-1", + }, + { + name: "rdma-2 GKENetworkParamSet has the wrong deviceMode fails closed", + mutate: func(f *fakeFetcher) { + f.gets["networking.gke.io/v1/GKENetworkParamSet//rdma-2"] = gkeNetworkParamSet("rdma-2", "NetDevice") + }, + wantOutput: "rdma-2", + }, + { + name: "missing gvnic-1 Network fails closed", + mutate: func(f *fakeFetcher) { + delete(f.gets, "networking.gke.io/v1/Network//gvnic-1") + }, + wantOutput: "gvnic-1", + }, + { + name: "gvnic-1 GKENetworkParamSet bound as RDMA instead of NetDevice fails closed", + mutate: func(f *fakeFetcher) { + f.gets["networking.gke.io/v1/GKENetworkParamSet//gvnic-1"] = gkeNetworkParamSet("gvnic-1", "RDMA") + }, + wantOutput: "gvnic-1", + }, + { + name: "rdma-3 Network parametersRef points at the wrong GKENetworkParamSet fails closed", + mutate: func(f *fakeFetcher) { + n := gkeNetwork("rdma-3") + n["spec"].(map[string]any)["parametersRef"].(map[string]any)["name"] = "rdma-0" + f.gets["networking.gke.io/v1/Network//rdma-3"] = n + }, + wantOutput: "rdma-3", + }, + { + name: "nccl-rdma-installer DaemonSet not fully rolled out fails closed", + mutate: func(f *fakeFetcher) { + f.gets["apps/v1/DaemonSet/kube-system/nccl-rdma-installer"] = map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": "nccl-rdma-installer", "namespace": "kube-system", "generation": 2}, + "status": map[string]any{"desiredNumberScheduled": 2, "numberReady": 1, "updatedNumberScheduled": 1, "observedGeneration": 2}, + } + }, + wantOutput: "DaemonSet", + }, + { + // numberReady alone can't tell a fully-current rollout from one + // where a node still runs the previous revision's pod: that pod + // reports Ready too, so desired/ready both read 2/2 while only 1 + // node has the new revision. updatedNumberScheduled is the field + // that catches it. + name: "nccl-rdma-installer DaemonSet with a stale-revision node fails closed", + mutate: func(f *fakeFetcher) { + f.gets["apps/v1/DaemonSet/kube-system/nccl-rdma-installer"] = map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": "nccl-rdma-installer", "namespace": "kube-system", "generation": 2}, + "status": map[string]any{"desiredNumberScheduled": 2, "numberReady": 2, "updatedNumberScheduled": 1, "observedGeneration": 2}, + } + }, + wantOutput: "DaemonSet", + }, + { + name: "nccl-rdma-installer DaemonSet status not yet observed at current generation fails closed", + mutate: func(f *fakeFetcher) { + f.gets["apps/v1/DaemonSet/kube-system/nccl-rdma-installer"] = map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": "nccl-rdma-installer", "namespace": "kube-system", "generation": 3}, + "status": map[string]any{"desiredNumberScheduled": 2, "numberReady": 2, "updatedNumberScheduled": 2, "observedGeneration": 2}, + } + }, + wantOutput: "DaemonSet", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fetcher := newFakeFetcher() + fetcher.addGet("networking.gke.io/v1", "GKENetworkParamSet", "", "gvnic-1", gkeNetworkParamSet("gvnic-1", "NetDevice")) + fetcher.addGet("networking.gke.io/v1", "Network", "", "gvnic-1", gkeNetwork("gvnic-1")) + for _, n := range []string{"rdma-0", "rdma-1", "rdma-2", "rdma-3"} { + fetcher.addGet("networking.gke.io/v1", "GKENetworkParamSet", "", n, gkeNetworkParamSet(n, "RDMA")) + fetcher.addGet("networking.gke.io/v1", "Network", "", n, gkeNetwork(n)) + } + fetcher.addGet("apps/v1", "DaemonSet", "kube-system", "nccl-rdma-installer", map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": "nccl-rdma-installer", "namespace": "kube-system", "generation": 2}, + "status": map[string]any{"desiredNumberScheduled": 2, "numberReady": 2, "updatedNumberScheduled": 2, "observedGeneration": 2}, + }) + fetcher.addList("v1", "Pod", "kube-system", nil) + + if tt.mutate != nil { + tt.mutate(fetcher) + } + + result := runChainsawTestInProcess( + context.Background(), "gke-gb200-rdma", string(data), 2*time.Second, fetcher, + ) + if result.Passed != tt.wantPass { + t.Fatalf("passed = %v, want %v (output: %s)", result.Passed, tt.wantPass, result.Output) + } + if tt.wantOutput != "" && !strings.Contains(result.Output, tt.wantOutput) { + t.Fatalf("output = %q, want it to name %q (the wrong assertion caught this state)", + result.Output, tt.wantOutput) + } + }) + } +} diff --git a/pkg/chainsaw/nvsentinel_check_states_test.go b/pkg/chainsaw/nvsentinel_check_states_test.go index dc50a9dee..248a7908d 100644 --- a/pkg/chainsaw/nvsentinel_check_states_test.go +++ b/pkg/chainsaw/nvsentinel_check_states_test.go @@ -54,13 +54,24 @@ func TestNVSentinelHealthCheckClusterStates(t *testing.T) { "metadata": map[string]any{"name": "labeler", "namespace": "nvsentinel"}, "status": map[string]any{"availableReplicas": int64(1)}, } - ds := func(name string, desired, ready int64) map[string]any { + // dsGen builds a DaemonSet with explicit updatedNumberScheduled/ + // generation/observedGeneration, for the stale-rollout cases below. ds + // defaults those to a fully-current rollout (updated == desired, + // observedGeneration == generation) so the existing ready/desired-only + // cases are unaffected by the new checks. + dsGen := func(name string, desired, ready, updated, generation, observedGeneration int64) map[string]any { return map[string]any{ "apiVersion": "apps/v1", "kind": "DaemonSet", - "metadata": map[string]any{"name": name, "namespace": "nvsentinel"}, - "status": map[string]any{"desiredNumberScheduled": desired, "numberReady": ready}, + "metadata": map[string]any{"name": name, "namespace": "nvsentinel", "generation": generation}, + "status": map[string]any{ + "desiredNumberScheduled": desired, "numberReady": ready, + "updatedNumberScheduled": updated, "observedGeneration": observedGeneration, + }, } } + ds := func(name string, desired, ready int64) map[string]any { + return dsGen(name, desired, ready, desired, 1, 1) + } healthySyslog := ds("syslog-health-monitor-regular", 2, 2) tests := []struct { @@ -121,6 +132,42 @@ func TestNVSentinelHealthCheckClusterStates(t *testing.T) { wantPass: false, wantContains: "syslog-health-monitor-regular", }, + { + // Stale rollout: every pod reports Ready (numberReady == + // desiredNumberScheduled), but one node is still running the + // previous revision (updatedNumberScheduled < desired) — the + // gap the numberReady-only check above cannot see. + name: "metadata-collector stale rollout (ready but not updated) → fail naming it", + collector: dsGen("metadata-collector", 2, 2, 1, 2, 2), + syslog: healthySyslog, + wantPass: false, + wantContains: "metadata-collector", + }, + { + // observedGeneration lagging metadata.generation: the + // controller hasn't yet processed the latest spec change, so + // numberReady/updatedNumberScheduled can still show the OLD + // revision's already-complete rollout. + name: "metadata-collector observedGeneration stale → fail naming it", + collector: dsGen("metadata-collector", 2, 2, 2, 2, 1), + syslog: healthySyslog, + wantPass: false, + wantContains: "metadata-collector", + }, + { + name: "syslog stale rollout (ready but not updated) → fail naming it", + collector: ds("metadata-collector", 2, 2), + syslog: dsGen("syslog-health-monitor-regular", 2, 2, 1, 2, 2), + wantPass: false, + wantContains: "syslog-health-monitor-regular", + }, + { + name: "syslog observedGeneration stale → fail naming it", + collector: ds("metadata-collector", 2, 2), + syslog: dsGen("syslog-health-monitor-regular", 2, 2, 2, 2, 1), + wantPass: false, + wantContains: "syslog-health-monitor-regular", + }, } for _, tt := range tests { diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index daeea3b9b..a8ddd1ed4 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -667,7 +667,7 @@ const ( // TrainerControllerReadyTimeout is the time to wait for the Kubeflow Trainer // controller-manager Deployment to have at least one ready replica after installation. - TrainerControllerReadyTimeout = 2 * time.Minute + TrainerControllerReadyTimeout = 3 * time.Minute // TrainerInstallPollInterval is the sleep between checks that a // recipe-declared Kubeflow Trainer installation has become complete. The diff --git a/pkg/recipe/catalog_parity_golden_test.go b/pkg/recipe/catalog_parity_golden_test.go index 2f9b66e15..74f26d10f 100644 --- a/pkg/recipe/catalog_parity_golden_test.go +++ b/pkg/recipe/catalog_parity_golden_test.go @@ -99,6 +99,9 @@ func TestCatalogParityGolden(t *testing.T) { } if os.Getenv("AICR_UPDATE_GOLDEN") == "1" { + if t.Failed() { + t.Fatal("not writing golden: one or more leaves failed to resolve (see errors above)") + } writeCatalogParityGolden(t, got) t.Logf("golden updated: %d leaves", len(got)) return diff --git a/pkg/recipe/metadata_test.go b/pkg/recipe/metadata_test.go index 866a0689a..7bb543d0a 100644 --- a/pkg/recipe/metadata_test.go +++ b/pkg/recipe/metadata_test.go @@ -2309,6 +2309,8 @@ func TestNFDTopologyUpdater_OverlayCoverage(t *testing.T) { {"rtx-pro-6000-lke-inference", criteria{CriteriaServiceLKE, CriteriaAcceleratorRTXPro6000, "", CriteriaIntentInference, ""}, true}, {"b200-gke-cos-training", criteria{CriteriaServiceGKE, CriteriaAcceleratorB200, CriteriaOSCOS, CriteriaIntentTraining, ""}, true}, {"b200-gke-cos-inference", criteria{CriteriaServiceGKE, CriteriaAcceleratorB200, CriteriaOSCOS, CriteriaIntentInference, ""}, true}, + {"gb200-gke-cos-training", criteria{CriteriaServiceGKE, CriteriaAcceleratorGB200, CriteriaOSCOS, CriteriaIntentTraining, ""}, true}, + {"gb200-gke-cos-inference", criteria{CriteriaServiceGKE, CriteriaAcceleratorGB200, CriteriaOSCOS, CriteriaIntentInference, ""}, true}, // Deeper specialized leaves — inherited via base: chain; a future overlay // that replaces (rather than deep-merges) componentRefs would break these. // H100 EKS Ubuntu variants @@ -2534,3 +2536,53 @@ func TestRecipeResultNormalizeKindNilReceiver(t *testing.T) { t.Errorf("NormalizeKind() on nil receiver = %v, want nil", err) } } + +// TestGB200GKEIncludesRDMA: gke+gb200+cos must resolve to the GB200 GKE +// leaf (not gke-cos + gb200-any) and keep gke-gb200-rdma plus the +// dma-buf kernel-module ConfigMap. +func TestGB200GKEIncludesRDMA(t *testing.T) { + builder := NewBuilder() + if builder == nil { + t.Fatal("NewBuilder() returned nil") + } + ctx := context.Background() + for _, intent := range []CriteriaIntentType{CriteriaIntentTraining, CriteriaIntentInference} { + cr := NewCriteria() + cr.Service = CriteriaServiceGKE + cr.Accelerator = CriteriaAcceleratorGB200 + cr.OS = CriteriaOSCOS + cr.Intent = intent + result, err := builder.BuildFromCriteria(ctx, cr) + if err != nil { + t.Fatalf("BuildFromCriteria(gke/gb200/cos/%s): %v", intent, err) + } + if result.GetComponentRef("gke-gb200-rdma") == nil { + t.Errorf("gke-gb200-rdma missing from resolved gke/gb200/cos/%s recipe", intent) + } + gpuOp := result.GetComponentRef("gpu-operator") + if gpuOp == nil { + t.Fatalf("gpu-operator missing from resolved gke/gb200/cos/%s recipe", intent) + } + km, ok := gpuOp.Overrides["driver"].(map[string]any) + if !ok { + t.Errorf("gpu-operator.driver override missing for gke/gb200/cos/%s", intent) + continue + } + cfg, ok := km["kernelModuleConfig"].(map[string]any) + if !ok || cfg["name"] != "nvidia-kernel-module-params" { + t.Errorf("kernelModuleConfig.name = %v, want nvidia-kernel-module-params for gke/gb200/cos/%s", km["kernelModuleConfig"], intent) + } + checkPresent := performanceCheckPresent(result.Validation, "nccl-all-reduce-bw-nvls") + floor, floorFound := findPerformanceConstraint(result.Validation, "nccl-all-reduce-bw-nvls") + if intent == CriteriaIntentTraining { + if !checkPresent { + t.Errorf("performance check nccl-all-reduce-bw-nvls missing for gke/gb200/cos/training") + } + if !floorFound || floor != ">= 250" { + t.Errorf("nccl-all-reduce-bw-nvls = %q found=%v, want >= 250 for gke/gb200/cos/training", floor, floorFound) + } + } else if checkPresent || floorFound { + t.Errorf("inference must not declare NCCL performance; check=%v floor=%q", checkPresent, floor) + } + } +} diff --git a/pkg/recipe/nccl_bandwidth_floor_test.go b/pkg/recipe/nccl_bandwidth_floor_test.go index 90bdb8141..b2a9b9042 100644 --- a/pkg/recipe/nccl_bandwidth_floor_test.go +++ b/pkg/recipe/nccl_bandwidth_floor_test.go @@ -129,7 +129,81 @@ func TestH100GKENCCLBandwidthFloor(t *testing.T) { t.Fatalf("performance constraint %q not found; expected value %q", checkName, tt.wantValue) } if gotValue != tt.wantValue { - t.Errorf("nccl-all-reduce-bw = %q, want %q", gotValue, tt.wantValue) + t.Errorf("%s = %q, want %q", checkName, gotValue, tt.wantValue) + } + } else { + if checkPresent { + t.Errorf("performance check %q should be cleared but is present", checkName) + } + if found { + t.Errorf("performance constraint %q should be cleared but resolved to %q", checkName, gotValue) + } + } + }) + } +} + +// TestGB200GKENCCLBandwidthFloor pins the GKE A4X (NVLS) all-reduce floor +// on the training leaf. Inference has no NCCL phase. +func TestGB200GKENCCLBandwidthFloor(t *testing.T) { + const checkName = "nccl-all-reduce-bw-nvls" + + tests := []struct { + name string + criteria *Criteria + wantValue string + wantPerf bool + }{ + { + name: "gb200-gke-cos-training", + criteria: &Criteria{ + Service: CriteriaServiceGKE, + Accelerator: CriteriaAcceleratorGB200, + OS: CriteriaOSCOS, + Intent: CriteriaIntentTraining, + Platform: CriteriaPlatformAny, + }, + wantValue: ">= 250", + wantPerf: true, + }, + { + name: "gb200-gke-cos-inference", + criteria: &Criteria{ + Service: CriteriaServiceGKE, + Accelerator: CriteriaAcceleratorGB200, + OS: CriteriaOSCOS, + Intent: CriteriaIntentInference, + Platform: CriteriaPlatformAny, + }, + wantPerf: false, + }, + } + + ctx := context.Background() + store, err := loadMetadataStore(ctx) + if err != nil { + t.Fatalf("loadMetadataStore: %v", err) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := store.BuildRecipeResult(ctx, tt.criteria) + if err != nil { + t.Fatalf("BuildRecipeResult: %v", err) + } + + gotValue, found := findPerformanceConstraint(result.Validation, checkName) + checkPresent := performanceCheckPresent(result.Validation, checkName) + + if tt.wantPerf { + if !checkPresent { + t.Errorf("performance check %q not present in resolved checks", checkName) + } + if !found { + t.Fatalf("performance constraint %q not found; expected value %q", checkName, tt.wantValue) + } + if gotValue != tt.wantValue { + t.Errorf("%s = %q, want %q", checkName, gotValue, tt.wantValue) } } else { if checkPresent { diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 95fe07389..b5ca71918 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,51 +3,54 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 3d8d77ac7ac29bf13253410e0b0486bc322d69d66d8778884df3375619566d58 -a100-any: 62d1581b21fecd3465b69659bac3dc9e5438eb2612c98c1acdd7b19cc9237084 -a100-eks-ubuntu-training-kubeflow: 0196adfa0c77f230d95ed3ebee2bdade8ac720f631cd3bbe2b8cec7d55094a10 -a100-gke-cos-training-kubeflow: 514f996360b251a6ef657ba1a715b5705be03d6b6dccb53887ee96554a83ffb6 -a100-oke-ubuntu-training-kubeflow: 310a6ee2abdd7f8508882429d657c01bd23458ba635236c410934825e4f617dc -b200-any: 63dbc4fe27b84321e8b1991cd98c13f85395a3919d13e0c6891599eaa1f1e729 -b200-gke-cos-inference-dynamo: 8174db8270ea447d9b307f21be9945b919ed3bcdacc702d2961d2ff75fb60c35 -b200-gke-cos-training-kubeflow: 96723738cb0baaaec300c95d5cfbc08a18d999fd0adb52aef3c88436561da1e7 -bcm-inference: 92fa64d3af61026891b37cbf692d2663be7b2484078ab65e2653d18fc2bfe6f8 -gb200-any: 9ee6ab187b4c1b93cd859d86ada35e922d0873ffc6fe892cb084ad98baf09727 -gb200-eks-ubuntu-inference-dynamo: 8babae6413d104da16060f18d919c80fd97ff983bd32f9876bd5f72da7fd6be6 -gb200-eks-ubuntu-training-kubeflow: 63b2fbd3a195bac682bb45263e5d5d0ec479a6571dd38ef0505eeb4f287b29e9 -gb200-eks-ubuntu-training-slurm: c35e33618c0a8bb548bbe3c5d6765faf90cebe1427478ef97089f7a30bb37a6c -gb200-oke-ubuntu-inference-dynamo: 6f01848824dcbefb0f58ec56ee9e911fff1470c9f67c0d0a9ba0de1dd1e648d2 -gb200-oke-ubuntu-training-kubeflow: 3959f47474edfac9b53a65939117d4364551fde0c9324246f195f144fe3beb12 -gb300-any: 7f68607dffbbfc912f19b00ffa833caa81ca675d25bc22cf612dbd0b76f39475 -gb300-eks-ubuntu-inference-dynamo: b14a3d99503949b8bfcdc81455da4af389a339deb24ea784637a7901297d70ad -gb300-eks-ubuntu-training-kubeflow: 5303e5ebf8dc2ec13c2cff60c7caba2ef53d1a65bfd69d06f4f78cbb93835c49 -h100-aks-ubuntu-inference-dynamo: 9fe4d82412d250d4004efc56fa4ddd6b8b1a9166387455e96ac2c2284d323107 -h100-aks-ubuntu-training-kubeflow: 72a8ca777c333e0db82717f53a706ae9155163ff0c73fd4e5ed8d71badda5811 -h100-aks-ubuntu-training-slurm: ef48d77171817bb5b24815d128457c246e0e5dc0fe850aa6881276952e590055 -h100-any: f5a55f03948075adde9b1b24fd650d98fa4d35b85f394d071ad3365ba05588a5 -h100-bcm-ubuntu-training: 76746f45add1fb6b0d7b6441b4d9581acceb3b5d3d4800b943c1cf460c92fbc5 -h100-eks-ubuntu-inference-dynamo: 0a31f03010849f76b385ce81c21e499af884f41bee89de47b9bacf03f8b6e311 -h100-eks-ubuntu-inference-nim: 9c6a4839bf5b59f620093427de0e138a9fafaf1f55f3fa7f555b1ee33b914a3a -h100-eks-ubuntu-training-kubeflow: bfcbd5e14fe43209f2401e429d39b5f7a52d049832db07e5b9595d130ab6e2fc -h100-eks-ubuntu-training-slurm: 8d29c2be5c3c0dc8396b0df53111a485f6d57771394df8ef135e4b052d681064 -h100-gke-cos-inference-dynamo: 45190af6c5b5d76f69a4f36d9c0cb800c450bdef54ca5b522ad49f605d1bad2a -h100-gke-cos-training-kubeflow: 43de2291cc0adb9b8d89494179d86eeef35994b2ad4fd4062a87265d6753ec6b -h100-gke-cos-training-slurm: 414f95e1a231b1c86bcbe70729b820a95533367dec0c9e3f7e36514431fad1d9 -h100-kind-inference-dynamo: 0e2d552b62b55b91f960c9721ed128b1d07f68be8517799473fbaa3baa1cc6e5 -h100-kind-training-kubeflow: 4bb03659faaa6a9a66d58dfc95572bdb54b75010c219c4bbefbd0eafeb8ed867 -h100-kind-training-slurm: 0245c786638ade1023ba4dc49e0f2efd56d2f789bdf447e946bbc3a3e652e734 -h200-any: acf1986d76037dddd025125eaa34eb7ecf3d31fe62cb80e7dc89dd042ebb42f5 -h200-eks-inference: d1ff148001722e9d5e23606bd7770f8120506f08c10f0c132e0c379f535609c5 -h200-eks-training: 98d04b3c81a059a0321451a39772af8d0d3310ad1fb861837f87d82dd997c4a1 -l40s-any: 89210ba5815f93ce3d7b1a2a78f7ba75d083556e940603a1bc812eac9aeccd0a -l40s-oke-inference: 6c75ef93e86fb21dbdf73ac97089b894ca8da832164dc807e94592493ab7f7cc -l40s-oke-training: 2a35011bd59cf3c727cea158d504c51bd59cba069620d3d6a4988d68972dee74 -monitoring-hpa: 15e93304e7e68997e7b15a0be4af360a29f10c869502e6dccb6a6cefb6519fd5 -ocp-inference-nim: 49245ecfc91f67fcc2ed0b305b00046a4164628cafc75ab98542e6dafa626821 -ocp-training: aaa8cd7f45ef42235b075936461d715e2e017ea2802df9e8017de38304a17915 -rtx-pro-6000-any: 9639d67639256a44be8566f3f8eda49ceb4025b6b140a3bc2a4b6d4768d07ffa -rtx-pro-6000-eks-ubuntu-inference-dynamo: cbe11051d979ae20e83471142c4582e1504fd24f55898a00afcededbe143bba6 -rtx-pro-6000-eks-ubuntu-inference-nim: 08b73d58d0d60681be55ee1402bdec5bf1367a0cb137a273ca27bec0788cc020 -rtx-pro-6000-eks-ubuntu-training-kubeflow: c2595e2994d56a735714103eedd30c6afd45921f458bf026c3432017b5eba60e -rtx-pro-6000-lke-ubuntu-inference: 28cdab0531f8ed6018391aba7cab1fbfad8ba551392de1b9138f3b88042f5340 -rtx-pro-6000-lke-ubuntu-training: 312c4cf49d0e8cec0377cd9858338d7106226a2252d81828a75c1d426a194a2f +a100-aks-ubuntu-training-kubeflow: 47f2c604ad1491346fc4923dacafe6d636fce9fc62b7312e1972f2286e7a1184 +a100-any: 3a9e12086e343324236f49d0d4e4e344f9b9309eb2fb5680b7de9265eda6a73b +a100-eks-ubuntu-training-kubeflow: e9ba3616f5a49924fda0d94f1035932c1d44b286d9d642f5b9823b6b4e98f04b +a100-gke-cos-training-kubeflow: b67b53baacf63289e0528d62ed94412f048c819313d60bdf7a0f9372f5bda1f6 +a100-oke-ubuntu-training-kubeflow: c202ac0aef742bcefb7252b3e5c073d94f3fbbbff3a3fc2572565d318b58d46c +b200-any: b1da720ca5ca65a854f86cf643b669dab0bc1ec4642d8b58a1b2e049d7e916c9 +b200-gke-cos-inference-dynamo: e2caa97c79a7a1e689edaa145d9a5c0f111067cd6061702fbd993d2679dc1caf +b200-gke-cos-training-kubeflow: 8119e255a77e1f36c3e6be8a5a2cb9fa1b878244eb3e955b26eed888eb9a0e33 +bcm-inference: 799569869ffff42841c5e758d6da1bc6f474af41a3cca5f74f806597e5be2f10 +gb200-any: bcc0972c4fe7b0a9e325d36f3dd1f1118119ecbe5b1c8a54da97e239029e775c +gb200-eks-ubuntu-inference-dynamo: 71e0707b3669b3084fb6cec75a8bd1d6c9493d9148fec8cddc5771be10680c91 +gb200-eks-ubuntu-training-kubeflow: 36468d7c502b13e3ae6e441eecd0c6da7901fc2c89dfeac014d5df184cd74700 +gb200-eks-ubuntu-training-slurm: c660bbe8b1a97e250cdc7a61e75ea58be49dc96061626715f516bcbf885f2d13 +gb200-gke-cos-inference-dynamo: 208ab56b75504cfaa3f00eaa898c4518667662182f402d2386d74670ca4ccb4b +gb200-gke-cos-training-kubeflow: 11e9a6c91b0aa0009cc3ad6c39f7889fc2c2db3015c5aafc0630e99581a936dd +gb200-gke-cos-training-slurm: 18595351d46083e023eed89e6d97c222ff73b547df50a6910e0b535e9c646e47 +gb200-oke-ubuntu-inference-dynamo: 12da75460cf668b2822557eccc998d815395649d6dfec42bea68e31e5480c621 +gb200-oke-ubuntu-training-kubeflow: a2828e176878a36533d39a6ba720418bb6cb3eff34a6de7b2e7db83197248d69 +gb300-any: 1274f6c19c26b29e064885f14b518e18c8469f2361caaac4832813af12fbb779 +gb300-eks-ubuntu-inference-dynamo: 631e54ea1f2297e33a7884fb173fd60a3244b0d7954ebbdda92e0d51ac4aee88 +gb300-eks-ubuntu-training-kubeflow: 33b8adf01bc622198ba9d46731c23efb88ab44991b84c8e37eea4ddf19aaeea9 +h100-aks-ubuntu-inference-dynamo: fa53ee5ecde84329429fd43b194a0828eeed354e471a1744b1b073f6fbbc189a +h100-aks-ubuntu-training-kubeflow: 3be5d233181d6dea5bf7127f554be827c579bac2fc893defca81621bbcc7ee77 +h100-aks-ubuntu-training-slurm: 2d797149b58ce853e7150d908be331c61dfdb477220fae9c15f6ebea22f9fe22 +h100-any: 233b2e31cbd94e7a5c1ac647648dce5f29fe07a4cc1674745be0e8847775f3fc +h100-bcm-ubuntu-training: 85c125e8440059a5087e3835551edb0196bf9256f27f84cb4fa094b3517536e9 +h100-eks-ubuntu-inference-dynamo: fa55376d388a78d961aa76b7343379f2bdbfad4d827e32af19eb3795962e4ffe +h100-eks-ubuntu-inference-nim: 53a96e375f75eed6508e57978e1e07fd05f740688dc4ca5a0e2fd04b90d716f3 +h100-eks-ubuntu-training-kubeflow: ecac42280c08d1eccaa49c31dee97009c803c360465d942de68a4d9786e8334c +h100-eks-ubuntu-training-slurm: 5fc03eb17bbb1934a78c03576d4aeeda8e723ca0937ca2fb0f6e0e40c19d38f2 +h100-gke-cos-inference-dynamo: 378d9fb68e1872185b7bb0a4c8f1a6f8d7747821b7144abc506ed8e7bd34920d +h100-gke-cos-training-kubeflow: 1e362c99329d7e7b4743ab1f727c509e334f5d9ddc8a63f98598e26c32e430c9 +h100-gke-cos-training-slurm: 622830ffd41d2a8311a76032eda64c9c100b4b09727d62d70879eee1127638c3 +h100-kind-inference-dynamo: 7188e00985d9e89992ceae579cf60b2e373846678d51d69aa78cfec15b748e46 +h100-kind-training-kubeflow: 5dd6d4efb2585c41c58dba1f373a5738c19be44618d109d5e31a53cf56482bd8 +h100-kind-training-slurm: 355c088589f5c948e025483d71468509addec60544b137653bf4df07f222ccc1 +h200-any: cea4403e292b8a2b3e44562c6770d6ca7366f4e99d409aece4c7e965bb6d2b20 +h200-eks-inference: 68b97d51d0cbc982564175114cb8626824e7643ac2fb6babf62073311c7eeb60 +h200-eks-training: 6db44e06178a0f74fab4d554406f988fe2e6c119b9523e458eb470ab3ed0cc8d +l40s-any: 78724a03fa25315bf8a7ef600dd5e869f8e25bc3c1d98500b5356e23a9daeffb +l40s-oke-inference: fbffa7c7d2e73b281b98fc010bb198549f4ccf5358830cefcc88579aad9102d2 +l40s-oke-training: b5af79f5f556627326759a7e8c9d44c35d215956b6d7c61cb959e26473164eed +monitoring-hpa: f45fa89ce40731a71cffe175fae6eb7d84788d8864f05329d2095688a0b6c807 +ocp-inference-nim: 8ce510d9612e039c09bf8bc298c70b10020af44782c86cd2e65b9a2894193ebd +ocp-training: 9d07f73a9070f7631d9e5faab124ed058d909a107a8cbe4220e16bbd8dd8b9ed +rtx-pro-6000-any: af194881071cad5943d884e2f2a54b930cf1c384ce69e8023356a701bef9ebe1 +rtx-pro-6000-eks-ubuntu-inference-dynamo: f3d27f5f3bd7bf6cedd53a73df14984478d1351fa4422d245b8a3b4b13580e7a +rtx-pro-6000-eks-ubuntu-inference-nim: 11f5cfe19a40b3e37ac63fb8c324ab4aa3eb74d05da50e4e0b10328e1426633e +rtx-pro-6000-eks-ubuntu-training-kubeflow: d2b5ceb741c24ed111457174755f0fe7e25c72dc8844112f0ce28954a7b7c76b +rtx-pro-6000-lke-ubuntu-inference: 0093d27aea8993870fe0c449eae9b7851660d7cdbec4fd1fe7182fbe0fc33a5e +rtx-pro-6000-lke-ubuntu-training: dfc4de4e93e1cab191c88a57ba969393e4acab024e86fffd57fd82b5c625988b diff --git a/pkg/recipe/testdata/coverage_golden.yaml b/pkg/recipe/testdata/coverage_golden.yaml index d01038108..e68342a27 100644 --- a/pkg/recipe/testdata/coverage_golden.yaml +++ b/pkg/recipe/testdata/coverage_golden.yaml @@ -11,6 +11,7 @@ criteria(service=, accelerator=, intent=, os=, platform=dynamo): - service=eks, accelerator=h100, intent=inference, os=ubuntu - service=eks, accelerator=rtx-pro-6000, intent=inference, os=ubuntu - service=gke, accelerator=b200, intent=inference, os=cos + - service=gke, accelerator=gb200, intent=inference, os=cos - service=gke, accelerator=h100, intent=inference, os=cos - service=oke, accelerator=gb200, intent=inference, os=ubuntu criteria(service=, accelerator=, intent=, os=, platform=kubeflow): @@ -29,6 +30,7 @@ criteria(service=, accelerator=, intent=, os=, platform=kubeflow): - service=eks, accelerator=rtx-pro-6000, intent=training, os=ubuntu - service=gke, accelerator=a100, intent=training, os=cos - service=gke, accelerator=b200, intent=training, os=cos + - service=gke, accelerator=gb200, intent=training, os=cos - service=gke, accelerator=h100, intent=training, os=cos - service=oke, accelerator=a100, intent=training, os=ubuntu - service=oke, accelerator=gb200, intent=training, os=ubuntu @@ -51,6 +53,7 @@ criteria(service=, accelerator=, intent=, os=, platform=slurm): - service=aks, accelerator=h100, intent=training, os=ubuntu - service=eks, accelerator=gb200, intent=training, os=ubuntu - service=eks, accelerator=h100, intent=training, os=ubuntu + - service=gke, accelerator=gb200, intent=training, os=cos - service=gke, accelerator=h100, intent=training, os=cos criteria(service=, accelerator=, intent=, os=cos, platform=): outcome: error @@ -69,6 +72,7 @@ criteria(service=, accelerator=, intent=, os=cos, platform=dynamo): - service=gke platform: - service=gke, accelerator=b200, intent=inference + - service=gke, accelerator=gb200, intent=inference - service=gke, accelerator=h100, intent=inference - service=kind, accelerator=h100, intent=inference criteria(service=, accelerator=, intent=, os=cos, platform=kubeflow): @@ -82,6 +86,7 @@ criteria(service=, accelerator=, intent=, os=cos, platform=kubeflow): platform: - service=gke, accelerator=a100, intent=training - service=gke, accelerator=b200, intent=training + - service=gke, accelerator=gb200, intent=training - service=gke, accelerator=h100, intent=training - service=kind, accelerator=h100, intent=training criteria(service=, accelerator=, intent=, os=cos, platform=slurm): @@ -93,6 +98,7 @@ criteria(service=, accelerator=, intent=, os=cos, platform=slurm): os: - service=gke platform: + - service=gke, accelerator=gb200, intent=training - service=gke, accelerator=h100, intent=training - service=kind, accelerator=h100, intent=training criteria(service=, accelerator=, intent=, os=ol, platform=): @@ -293,6 +299,7 @@ criteria(service=, accelerator=, intent=inference, os=, platform=dynamo): - service=eks, accelerator=h100, os=ubuntu - service=eks, accelerator=rtx-pro-6000, os=ubuntu - service=gke, accelerator=b200, os=cos + - service=gke, accelerator=gb200, os=cos - service=gke, accelerator=h100, os=cos - service=oke, accelerator=gb200, os=ubuntu criteria(service=, accelerator=, intent=inference, os=, platform=nim): @@ -350,6 +357,7 @@ criteria(service=, accelerator=, intent=inference, os=cos, platform=dynamo): - service=gke platform: - service=gke, accelerator=b200 + - service=gke, accelerator=gb200 - service=gke, accelerator=h100 - service=kind, accelerator=h100 criteria(service=, accelerator=, intent=inference, os=ol, platform=): @@ -492,6 +500,7 @@ criteria(service=, accelerator=, intent=training, os=, platform=kubeflow): - service=eks, accelerator=rtx-pro-6000, os=ubuntu - service=gke, accelerator=a100, os=cos - service=gke, accelerator=b200, os=cos + - service=gke, accelerator=gb200, os=cos - service=gke, accelerator=h100, os=cos - service=oke, accelerator=a100, os=ubuntu - service=oke, accelerator=gb200, os=ubuntu @@ -517,6 +526,7 @@ criteria(service=, accelerator=, intent=training, os=, platform=slurm): - service=aks, accelerator=h100, os=ubuntu - service=eks, accelerator=gb200, os=ubuntu - service=eks, accelerator=h100, os=ubuntu + - service=gke, accelerator=gb200, os=cos - service=gke, accelerator=h100, os=cos criteria(service=, accelerator=, intent=training, os=cos, platform=): outcome: error @@ -554,6 +564,7 @@ criteria(service=, accelerator=, intent=training, os=cos, platform=kubeflow): platform: - service=gke, accelerator=a100 - service=gke, accelerator=b200 + - service=gke, accelerator=gb200 - service=gke, accelerator=h100 - service=kind, accelerator=h100 criteria(service=, accelerator=, intent=training, os=cos, platform=slurm): @@ -574,6 +585,7 @@ criteria(service=, accelerator=, intent=training, os=cos, platform=slurm): os: - service=gke platform: + - service=gke, accelerator=gb200 - service=gke, accelerator=h100 - service=kind, accelerator=h100 criteria(service=, accelerator=, intent=training, os=ol, platform=): @@ -1051,6 +1063,7 @@ criteria(service=, accelerator=gb200, intent=, os=, platform=dynamo): validCompletions: platform: - service=eks, intent=inference, os=ubuntu + - service=gke, intent=inference, os=cos - service=oke, intent=inference, os=ubuntu criteria(service=, accelerator=gb200, intent=, os=, platform=kubeflow): outcome: error @@ -1059,6 +1072,7 @@ criteria(service=, accelerator=gb200, intent=, os=, platform=kubeflow): validCompletions: platform: - service=eks, intent=training, os=ubuntu + - service=gke, intent=training, os=cos - service=oke, intent=training, os=ubuntu criteria(service=, accelerator=gb200, intent=, os=, platform=slurm): outcome: error @@ -1067,6 +1081,44 @@ criteria(service=, accelerator=gb200, intent=, os=, platform=slurm): validCompletions: platform: - service=eks, intent=training, os=ubuntu + - service=gke, intent=training, os=cos +criteria(service=, accelerator=gb200, intent=, os=cos, platform=): + outcome: error + uncovered: + - os + validCompletions: + os: + - service=gke +criteria(service=, accelerator=gb200, intent=, os=cos, platform=dynamo): + outcome: error + uncovered: + - os + - platform + validCompletions: + os: + - service=gke + platform: + - service=gke, intent=inference +criteria(service=, accelerator=gb200, intent=, os=cos, platform=kubeflow): + outcome: error + uncovered: + - os + - platform + validCompletions: + os: + - service=gke + platform: + - service=gke, intent=training +criteria(service=, accelerator=gb200, intent=, os=cos, platform=slurm): + outcome: error + uncovered: + - os + - platform + validCompletions: + os: + - service=gke + platform: + - service=gke, intent=training criteria(service=, accelerator=gb200, intent=, os=ol, platform=): outcome: error uncovered: @@ -1158,7 +1210,43 @@ criteria(service=, accelerator=gb200, intent=inference, os=, platform=dynamo): - service=oke, os=ubuntu platform: - service=eks, os=ubuntu + - service=gke, os=cos - service=oke, os=ubuntu +criteria(service=, accelerator=gb200, intent=inference, os=cos, platform=): + outcome: error + uncovered: + - intent + - os + validCompletions: + intent: + - service=aks + - service=bcm + - service=eks + - service=gke + - service=kind + - service=lke + - service=ocp + os: + - service=gke +criteria(service=, accelerator=gb200, intent=inference, os=cos, platform=dynamo): + outcome: error + uncovered: + - intent + - os + - platform + validCompletions: + intent: + - service=aks + - service=bcm + - service=eks + - service=gke + - service=kind + - service=lke + - service=ocp + os: + - service=gke + platform: + - service=gke criteria(service=, accelerator=gb200, intent=inference, os=ol, platform=): outcome: error uncovered: @@ -1244,6 +1332,7 @@ criteria(service=, accelerator=gb200, intent=training, os=, platform=kubeflow): - service=oke, os=ubuntu platform: - service=eks, os=ubuntu + - service=gke, os=cos - service=oke, os=ubuntu criteria(service=, accelerator=gb200, intent=training, os=, platform=slurm): outcome: error @@ -1262,6 +1351,58 @@ criteria(service=, accelerator=gb200, intent=training, os=, platform=slurm): - service=oke, os=ubuntu platform: - service=eks, os=ubuntu + - service=gke, os=cos +criteria(service=, accelerator=gb200, intent=training, os=cos, platform=): + outcome: error + uncovered: + - intent + - os + validCompletions: + intent: + - service=aks + - service=bcm + - service=eks + - service=gke + - service=lke + - service=ocp + os: + - service=gke +criteria(service=, accelerator=gb200, intent=training, os=cos, platform=kubeflow): + outcome: error + uncovered: + - intent + - os + - platform + validCompletions: + intent: + - service=aks + - service=bcm + - service=eks + - service=gke + - service=lke + - service=ocp + os: + - service=gke + platform: + - service=gke +criteria(service=, accelerator=gb200, intent=training, os=cos, platform=slurm): + outcome: error + uncovered: + - intent + - os + - platform + validCompletions: + intent: + - service=aks + - service=bcm + - service=eks + - service=gke + - service=lke + - service=ocp + os: + - service=gke + platform: + - service=gke criteria(service=, accelerator=gb200, intent=training, os=ol, platform=): outcome: error uncovered: @@ -3339,6 +3480,7 @@ criteria(service=gke, accelerator=, intent=, os=, platform=dynamo): validCompletions: platform: - accelerator=b200, intent=inference, os=cos + - accelerator=gb200, intent=inference, os=cos - accelerator=h100, intent=inference, os=cos service: - os=cos @@ -3351,6 +3493,7 @@ criteria(service=gke, accelerator=, intent=, os=, platform=kubeflow): platform: - accelerator=a100, intent=training, os=cos - accelerator=b200, intent=training, os=cos + - accelerator=gb200, intent=training, os=cos - accelerator=h100, intent=training, os=cos service: - os=cos @@ -3361,6 +3504,7 @@ criteria(service=gke, accelerator=, intent=, os=, platform=slurm): - platform validCompletions: platform: + - accelerator=gb200, intent=training, os=cos - accelerator=h100, intent=training, os=cos service: - os=cos @@ -3373,6 +3517,7 @@ criteria(service=gke, accelerator=, intent=, os=cos, platform=dynamo): validCompletions: platform: - accelerator=b200, intent=inference + - accelerator=gb200, intent=inference - accelerator=h100, intent=inference criteria(service=gke, accelerator=, intent=, os=cos, platform=kubeflow): outcome: error @@ -3382,6 +3527,7 @@ criteria(service=gke, accelerator=, intent=, os=cos, platform=kubeflow): platform: - accelerator=a100, intent=training - accelerator=b200, intent=training + - accelerator=gb200, intent=training - accelerator=h100, intent=training criteria(service=gke, accelerator=, intent=, os=cos, platform=slurm): outcome: error @@ -3389,6 +3535,7 @@ criteria(service=gke, accelerator=, intent=, os=cos, platform=slurm): - platform validCompletions: platform: + - accelerator=gb200, intent=training - accelerator=h100, intent=training criteria(service=gke, accelerator=, intent=inference, os=, platform=): outcome: error @@ -3411,6 +3558,7 @@ criteria(service=gke, accelerator=, intent=inference, os=, platform=dynamo): - os=cos platform: - accelerator=b200, os=cos + - accelerator=gb200, os=cos - accelerator=h100, os=cos service: - os=cos @@ -3423,6 +3571,7 @@ criteria(service=gke, accelerator=, intent=inference, os=cos, platform=dynamo): validCompletions: platform: - accelerator=b200 + - accelerator=gb200 - accelerator=h100 criteria(service=gke, accelerator=, intent=training, os=, platform=): outcome: error @@ -3446,6 +3595,7 @@ criteria(service=gke, accelerator=, intent=training, os=, platform=kubeflow): platform: - accelerator=a100, os=cos - accelerator=b200, os=cos + - accelerator=gb200, os=cos - accelerator=h100, os=cos service: - os=cos @@ -3459,6 +3609,7 @@ criteria(service=gke, accelerator=, intent=training, os=, platform=slurm): intent: - os=cos platform: + - accelerator=gb200, os=cos - accelerator=h100, os=cos service: - os=cos @@ -3472,6 +3623,7 @@ criteria(service=gke, accelerator=, intent=training, os=cos, platform=kubeflow): platform: - accelerator=a100 - accelerator=b200 + - accelerator=gb200 - accelerator=h100 criteria(service=gke, accelerator=, intent=training, os=cos, platform=slurm): outcome: error @@ -3479,6 +3631,7 @@ criteria(service=gke, accelerator=, intent=training, os=cos, platform=slurm): - platform validCompletions: platform: + - accelerator=gb200 - accelerator=h100 criteria(service=gke, accelerator=a100, intent=, os=, platform=): outcome: error @@ -3630,6 +3783,135 @@ criteria(service=gke, accelerator=b200, intent=training, os=cos, platform=): outcome: success criteria(service=gke, accelerator=b200, intent=training, os=cos, platform=kubeflow): outcome: success +criteria(service=gke, accelerator=gb200, intent=, os=, platform=): + outcome: error + uncovered: + - service + validCompletions: + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=, os=, platform=dynamo): + outcome: error + uncovered: + - service + - platform + validCompletions: + platform: + - intent=inference, os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=, os=, platform=kubeflow): + outcome: error + uncovered: + - service + - platform + validCompletions: + platform: + - intent=training, os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=, os=, platform=slurm): + outcome: error + uncovered: + - service + - platform + validCompletions: + platform: + - intent=training, os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=, os=cos, platform=): + outcome: success +criteria(service=gke, accelerator=gb200, intent=, os=cos, platform=dynamo): + outcome: error + uncovered: + - platform + validCompletions: + platform: + - intent=inference +criteria(service=gke, accelerator=gb200, intent=, os=cos, platform=kubeflow): + outcome: error + uncovered: + - platform + validCompletions: + platform: + - intent=training +criteria(service=gke, accelerator=gb200, intent=, os=cos, platform=slurm): + outcome: error + uncovered: + - platform + validCompletions: + platform: + - intent=training +criteria(service=gke, accelerator=gb200, intent=inference, os=, platform=): + outcome: error + uncovered: + - service + - intent + validCompletions: + intent: + - os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=inference, os=, platform=dynamo): + outcome: error + uncovered: + - service + - intent + - platform + validCompletions: + intent: + - os=cos + platform: + - os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=inference, os=cos, platform=): + outcome: success +criteria(service=gke, accelerator=gb200, intent=inference, os=cos, platform=dynamo): + outcome: success +criteria(service=gke, accelerator=gb200, intent=training, os=, platform=): + outcome: error + uncovered: + - service + - intent + validCompletions: + intent: + - os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=training, os=, platform=kubeflow): + outcome: error + uncovered: + - service + - intent + - platform + validCompletions: + intent: + - os=cos + platform: + - os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=training, os=, platform=slurm): + outcome: error + uncovered: + - service + - intent + - platform + validCompletions: + intent: + - os=cos + platform: + - os=cos + service: + - os=cos +criteria(service=gke, accelerator=gb200, intent=training, os=cos, platform=): + outcome: success +criteria(service=gke, accelerator=gb200, intent=training, os=cos, platform=kubeflow): + outcome: success +criteria(service=gke, accelerator=gb200, intent=training, os=cos, platform=slurm): + outcome: success criteria(service=gke, accelerator=h100, intent=, os=, platform=): outcome: error uncovered: diff --git a/pkg/tuning/compute_test.go b/pkg/tuning/compute_test.go index 8a1ef55f2..c418ebedd 100644 --- a/pkg/tuning/compute_test.go +++ b/pkg/tuning/compute_test.go @@ -51,6 +51,7 @@ func TestCompute_Structure(t *testing.T) { {"eks", "rtx-pro-6000", "generic", "", "nvidia-tuned"}, {"gke", "a100", "h100", "", "nvidia-tuning-gke"}, {"gke", "b200", "-", "", "nvidia-tuning-gke"}, + {"gke", "gb200", "-", "", "nvidia-tuning-gke"}, {"gke", "h100", "-", "", "nvidia-tuning-gke"}, } if len(report.Rows) != len(want) { diff --git a/recipes/checks/aws-efa/health-check.yaml b/recipes/checks/aws-efa/health-check.yaml index aae6c6932..e88ba603d 100644 --- a/recipes/checks/aws-efa/health-check.yaml +++ b/recipes/checks/aws-efa/health-check.yaml @@ -20,9 +20,13 @@ # as a guard against vacuous pass on a 0-node selector). The previous # check gated only on numberReady > 0, which passed on partial # failures; see #1222 / recipes/checks/nfd/health-check.yaml for the -# same rationale. Container-state errors (CrashLoopBackOff, -# ImagePullBackOff, ErrImagePull, CreateContainerConfigError) are -# label-scoped because kube-system is a shared namespace. +# same rationale. updatedNumberScheduled/observedGeneration additionally +# guard against a stale rollout: a node still running the previous +# revision's pod also reports numberReady, so that alone can't +# distinguish it from a fully current rollout. +# Container-state errors (CrashLoopBackOff, ImagePullBackOff, +# ErrImagePull, CreateContainerConfigError) are label-scoped because +# kube-system is a shared namespace. apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -40,9 +44,10 @@ spec: metadata: name: aws-efa-k8s-device-plugin namespace: kube-system - status: - (desiredNumberScheduled > `0`): true - (numberReady == desiredNumberScheduled): true + ((status.desiredNumberScheduled || `0`) > `0`): true + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true - name: validate-all-pods-healthy try: - error: diff --git a/recipes/checks/gke-gb200-rdma/health-check.yaml b/recipes/checks/gke-gb200-rdma/health-check.yaml new file mode 100644 index 000000000..24e33e534 --- /dev/null +++ b/recipes/checks/gke-gb200-rdma/health-check.yaml @@ -0,0 +1,193 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GKE GB200 RDMA/RoCE Health Check +# +# gvnic-1/rdma-0..3 are a cluster provisioning prerequisite (see +# docs/integrator/gke-gb200-networking.md), not created by AICR. vpc/vpcSubnet +# aren't asserted since their names are cluster-specific. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gke-gb200-rdma-health-check +spec: + timeouts: + assert: 5m + steps: + - name: validate-gvnic-network-objects + try: + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: GKENetworkParamSet + metadata: + name: gvnic-1 + spec: + deviceMode: NetDevice + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: Network + metadata: + name: gvnic-1 + spec: + type: Device + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: gvnic-1 + - name: validate-rdma-network-objects + try: + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: GKENetworkParamSet + metadata: + name: rdma-0 + spec: + deviceMode: RDMA + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: Network + metadata: + name: rdma-0 + spec: + type: Device + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: rdma-0 + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: GKENetworkParamSet + metadata: + name: rdma-1 + spec: + deviceMode: RDMA + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: Network + metadata: + name: rdma-1 + spec: + type: Device + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: rdma-1 + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: GKENetworkParamSet + metadata: + name: rdma-2 + spec: + deviceMode: RDMA + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: Network + metadata: + name: rdma-2 + spec: + type: Device + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: rdma-2 + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: GKENetworkParamSet + metadata: + name: rdma-3 + spec: + deviceMode: RDMA + - assert: + resource: + apiVersion: networking.gke.io/v1 + kind: Network + metadata: + name: rdma-3 + spec: + type: Device + parametersRef: + group: networking.gke.io + kind: GKENetworkParamSet + name: rdma-3 + - name: validate-nccl-rdma-installer-fully-rolled-out + try: + - assert: + resource: + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: nccl-rdma-installer + namespace: kube-system + # A DaemonSet has no spec.replicas; desiredNumberScheduled (node + # count matching its selector) is both the desired count and the + # floor. numberReady alone accepts a stale rollout: a node still + # running the previous revision's pod counts as ready too, so + # updatedNumberScheduled must also match. observedGeneration + # confirms status reflects the current spec, not a reconcile + # from before the latest update. + ((status.desiredNumberScheduled || `0`) > `0`): true + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true + - name: validate-all-pods-healthy + try: + - error: + resource: + apiVersion: v1 + kind: Pod + metadata: + namespace: kube-system + labels: + k8s-app: nccl-rdma-installer + status: + phase: Pending + - error: + resource: + apiVersion: v1 + kind: Pod + metadata: + namespace: kube-system + labels: + k8s-app: nccl-rdma-installer + status: + phase: Failed + - error: + resource: + apiVersion: v1 + kind: Pod + metadata: + namespace: kube-system + labels: + k8s-app: nccl-rdma-installer + status: + phase: Unknown + - error: + resource: + apiVersion: v1 + kind: Pod + metadata: + namespace: kube-system + labels: + k8s-app: nccl-rdma-installer + status: + ((initContainerStatuses || `[]`)[?state.waiting.reason == 'CrashLoopBackOff' || state.waiting.reason == 'ImagePullBackOff' || state.waiting.reason == 'ErrImagePull' || state.waiting.reason == 'CreateContainerConfigError'] | length(@) > `0`): true diff --git a/recipes/checks/gke-nccl-tcpxo/health-check.yaml b/recipes/checks/gke-nccl-tcpxo/health-check.yaml index 7d5f79f90..26068397b 100644 --- a/recipes/checks/gke-nccl-tcpxo/health-check.yaml +++ b/recipes/checks/gke-nccl-tcpxo/health-check.yaml @@ -31,6 +31,10 @@ # `numberUnavailable` is `omitempty` and disappears from the status # when zero — making the equality compare null on a healthy DaemonSet # (see recipes/checks/nfd/health-check.yaml for the same rationale). +# updatedNumberScheduled/observedGeneration additionally guard against a +# stale rollout: a node still running the previous revision's pod also +# reports numberReady, so that alone can't distinguish it from a fully +# current rollout. apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: @@ -48,9 +52,10 @@ spec: metadata: name: nccl-tcpxo-installer namespace: kube-system - status: - (desiredNumberScheduled > `0`): true - (numberReady == desiredNumberScheduled): true + ((status.desiredNumberScheduled || `0`) > `0`): true + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true - name: validate-device-injector-daemonset try: - assert: @@ -60,6 +65,7 @@ spec: metadata: name: device-injector namespace: kube-system - status: - (desiredNumberScheduled > `0`): true - (numberReady == desiredNumberScheduled): true + ((status.desiredNumberScheduled || `0`) > `0`): true + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true diff --git a/recipes/checks/nfd/health-check.yaml b/recipes/checks/nfd/health-check.yaml index e5485009e..ad1b033fe 100644 --- a/recipes/checks/nfd/health-check.yaml +++ b/recipes/checks/nfd/health-check.yaml @@ -16,8 +16,10 @@ # # Validates the standalone NFD deployment by checking: # 1. NFD Master Deployment has at least one ready replica -# 2. NFD Worker DaemonSet has fully rolled out (desiredNumberScheduled > 0 -# and numberReady == desiredNumberScheduled, so partial failures don't pass) +# 2. NFD Worker DaemonSet has fully rolled out and is current +# (desiredNumberScheduled > 0, numberReady/updatedNumberScheduled both +# match it, and observedGeneration matches metadata.generation, so +# neither a partial nor a stale rollout passes) # 3. NFD GC Deployment has at least one ready replica (gc.enable: true) # # Resource names: AICR deploys the kubernetes-sigs/node-feature-discovery @@ -60,17 +62,21 @@ spec: metadata: name: nfd-node-feature-discovery-worker namespace: node-feature-discovery - status: - # Guard against vacuous pass when 0 nodes match the - # DaemonSet: require at least one scheduled pod. - (desiredNumberScheduled > `0`): true - # Full rollout: every scheduled pod is ready. Combined with - # the guard above, this rejects partial failures. We assert on - # numberReady (always present in DaemonSetStatus) rather than - # numberUnavailable, which is `omitempty` and disappears from - # the status when zero — making `numberUnavailable == 0` - # evaluate `null == 0` (false) on a fully-healthy DaemonSet. - (numberReady == desiredNumberScheduled): true + # Guard against vacuous pass when 0 nodes match the + # DaemonSet: require at least one scheduled pod. + ((status.desiredNumberScheduled || `0`) > `0`): true + # Full rollout: every scheduled pod is ready. Combined with + # the guard above, this rejects partial failures. We assert on + # numberReady (always present in DaemonSetStatus) rather than + # numberUnavailable, which is `omitempty` and disappears from + # the status when zero — making `numberUnavailable == 0` + # evaluate `null == 0` (false) on a fully-healthy DaemonSet. + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + # numberReady alone doesn't distinguish a node still on the + # previous revision (also Ready) from a fully current rollout; + # updatedNumberScheduled/observedGeneration close that gap. + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true - name: validate-gc-deployment try: - assert: diff --git a/recipes/checks/nvidia-dra-driver-gpu/health-check.yaml b/recipes/checks/nvidia-dra-driver-gpu/health-check.yaml index a255bc0fd..b02a1ab5d 100644 --- a/recipes/checks/nvidia-dra-driver-gpu/health-check.yaml +++ b/recipes/checks/nvidia-dra-driver-gpu/health-check.yaml @@ -20,7 +20,10 @@ # desiredNumberScheduled (with desiredNumberScheduled > 0 as a guard). # Previous check gated only on numberReady > 0, passing on partial # failures; see #1222 and recipes/checks/nfd/health-check.yaml for -# the same rationale. +# the same rationale. updatedNumberScheduled/observedGeneration +# additionally guard against a stale rollout: a node still running the +# previous revision's pod also reports numberReady, so that alone +# can't distinguish it from a fully current rollout. # # Resource names: the DaemonSet name is release-derived # (-kubelet-plugin). AICR's release name is @@ -46,9 +49,10 @@ spec: metadata: name: nvidia-dra-driver-gpu-kubelet-plugin namespace: nvidia-dra-driver - status: - (desiredNumberScheduled > `0`): true - (numberReady == desiredNumberScheduled): true + ((status.desiredNumberScheduled || `0`) > `0`): true + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true - name: validate-all-pods-healthy try: - error: diff --git a/recipes/checks/nvsentinel/health-check.yaml b/recipes/checks/nvsentinel/health-check.yaml index 7d04d78b0..1fff80ca0 100644 --- a/recipes/checks/nvsentinel/health-check.yaml +++ b/recipes/checks/nvsentinel/health-check.yaml @@ -119,6 +119,30 @@ spec: namespace: nvsentinel status: (numberReady < desiredNumberScheduled): true + # Stale-rollout guard: numberReady alone can't distinguish a node + # still running the previous revision's pod (also Ready) from a + # fully current rollout. Flat (not status:-nested) so + # metadata.generation is addressable below. `<` (not `!=`) states + # the violated invariant directly: fewer nodes updated than desired. + # updatedNumberScheduled is itself omitempty (unlike + # desiredNumberScheduled/numberReady), so it needs the `|| `0`` + # null-coalescing too. + - error: + resource: + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: metadata-collector + namespace: nvsentinel + ((status.updatedNumberScheduled || `0`) < status.desiredNumberScheduled): true + - error: + resource: + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: metadata-collector + namespace: nvsentinel + ((status.observedGeneration || `0`) < metadata.generation): true # syslog-health-monitor-regular: same negative form — the # syslog-health-monitor subchart is switchable via its chart # condition (global.syslogHealthMonitor.enabled) exactly like @@ -145,6 +169,28 @@ spec: namespace: nvsentinel status: (numberReady < desiredNumberScheduled): true + # Stale-rollout guard: numberReady alone can't distinguish a node + # still running the previous revision's pod (also Ready) from a + # fully current rollout. Flat (not status:-nested) so + # metadata.generation is addressable below. updatedNumberScheduled + # is itself omitempty (unlike desiredNumberScheduled/numberReady), + # so it needs the `|| `0`` null-coalescing too. + - error: + resource: + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: syslog-health-monitor-regular + namespace: nvsentinel + ((status.updatedNumberScheduled || `0`) < status.desiredNumberScheduled): true + - error: + resource: + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: syslog-health-monitor-regular + namespace: nvsentinel + ((status.observedGeneration || `0`) < metadata.generation): true - name: validate-all-pods-healthy try: # Assert no pods are in unhealthy phases (Pending / Failed / diff --git a/recipes/checks/slinky-topograph/health-check.yaml b/recipes/checks/slinky-topograph/health-check.yaml index 4253d68f9..2dda9383f 100644 --- a/recipes/checks/slinky-topograph/health-check.yaml +++ b/recipes/checks/slinky-topograph/health-check.yaml @@ -81,11 +81,15 @@ spec: metadata: name: slinky-topograph-node-data-broker namespace: topograph - status: - # Non-omitempty fields only: NumberUnavailable is omitempty and - # absent when healthy. desired > 0 guards the vacuous pass. - (desiredNumberScheduled > `0`): true - (numberReady == desiredNumberScheduled): true + # Non-omitempty fields only: NumberUnavailable is omitempty and + # absent when healthy. desired > 0 guards the vacuous pass. + ((status.desiredNumberScheduled || `0`) > `0`): true + ((status.numberReady || `0`) == status.desiredNumberScheduled): true + # updatedNumberScheduled/observedGeneration guard against a stale + # rollout: a node still on the previous revision also reports + # numberReady, so that alone can't distinguish current from stale. + ((status.updatedNumberScheduled || `0`) == status.desiredNumberScheduled): true + ((status.observedGeneration || `0`) == metadata.generation): true - name: validate-all-pods-healthy try: # Assert no pods are in unhealthy phases (Pending / Failed / diff --git a/recipes/components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml b/recipes/components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml new file mode 100644 index 000000000..cb88b8dc4 --- /dev/null +++ b/recipes/components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml @@ -0,0 +1,120 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NCCL gIB (GPUDirect IB/RoCE) plugin installer for GB200 (A4X, ARM64) GKE nodes. +# +# Vendored from GoogleCloudPlatform/container-engine-accelerators +# `gpudirect-rdma/nccl-rdma-installer-a4x.yaml`, adapted to AICR conventions +# (app.kubernetes.io labels, enabled gate, acceleratedTolerations). +# +# Installs RDMA binaries into /home/kubernetes/bin/gib and the NCCL library +# into /home/kubernetes/bin/nvidia/lib64. Workloads select it via NCCL_NET=gIB +# + LD_LIBRARY_PATH=/home/kubernetes/bin/nvidia/lib64:/home/kubernetes/bin/gib. +{{- $vals := index .Values "gke-gb200-rdma" }} +{{- if ne (toString (index $vals "enabled")) "false" }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nccl-rdma-installer + namespace: kube-system + labels: + app.kubernetes.io/name: nccl-rdma-installer + app.kubernetes.io/instance: gke-gb200-rdma + app.kubernetes.io/managed-by: aicr + k8s-app: nccl-rdma-installer +spec: + selector: + matchLabels: + k8s-app: nccl-rdma-installer + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: nccl-rdma-installer + app.kubernetes.io/name: nccl-rdma-installer + app.kubernetes.io/instance: gke-gb200-rdma + app.kubernetes.io/managed-by: aicr + k8s-app: nccl-rdma-installer + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-accelerator + operator: In + values: + - nvidia-gb200 + - key: kubernetes.io/arch + operator: In + values: + - arm64 + {{- if $vals.acceleratedNodeSelector }} + nodeSelector: + {{- toYaml $vals.acceleratedNodeSelector | nindent 8 }} + {{- end }} + {{- if $vals.acceleratedTolerations }} + tolerations: + {{- toYaml $vals.acceleratedTolerations | nindent 8 }} + {{- else }} + tolerations: + - operator: "Exists" + {{- end }} + hostNetwork: true + hostPID: true + volumes: + - name: library-dir-host + hostPath: + path: /home/kubernetes/bin/nvidia/lib64 + type: DirectoryOrCreate + - name: gib + hostPath: + path: /home/kubernetes/bin/gib + # Unmounted by design (matches upstream): type: Directory (not + # DirectoryOrCreate) makes this hostPath a strict-existence pod + # admission precondition, gating this DaemonSet's pods on the + # driver installer having already created the directory, without + # requiring any container to actually read/write it. + - name: nvidia-dir + hostPath: + path: /home/kubernetes/bin/nvidia + type: Directory + initContainers: + - image: us-docker.pkg.dev/gce-ai-infra/gpudirect-gib/nccl-plugin-gib-arm64:v1.1.2@sha256:6b7950cac6e6833661d4206920f5633b6e361b18bfd5315b63f9bf4a4b84a80e + name: nccl-rdma-installer + resources: + requests: + cpu: 150m + securityContext: + privileged: true + volumeMounts: + - name: library-dir-host + mountPath: /usr/local/home/kubernetes/bin/nvidia/lib64 + - name: gib + mountPath: /usr/local/home/kubernetes/bin/gib + command: ["/bin/sh", "-c"] + args: + - | + set -ex + /scripts/container_entry.sh install + cp -r /var/lib/gib/lib64/. /usr/local/home/kubernetes/bin/nvidia/lib64 + cp -r /var/lib/gib/. /usr/local/home/kubernetes/bin/gib + echo "installation finishes" + containers: + - image: "gke.gcr.io/pause:3.8@sha256:880e63f94b145e46f1b1082bb71b85e21f16b99b180b9996407d61240ceb9830" + name: pause +{{- end }} diff --git a/recipes/gke_gb200_rdma_test.go b/recipes/gke_gb200_rdma_test.go new file mode 100644 index 000000000..a3c23fbf4 --- /dev/null +++ b/recipes/gke_gb200_rdma_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipes + +import ( + "testing" + + "gopkg.in/yaml.v3" + + "github.com/NVIDIA/aicr/pkg/manifest" +) + +// renderGB200RDMAInstaller renders the gIB installer DaemonSet manifest with +// the given component values and returns the parsed DaemonSet pod spec. +func renderGB200RDMAInstaller(t *testing.T, values map[string]any) map[string]any { + t.Helper() + raw, err := FS.ReadFile("components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml") + if err != nil { + t.Fatalf("read nccl-gib-installer-arm64.yaml: %v", err) + } + out, err := manifest.Render(raw, manifest.RenderInput{ + ComponentName: "gke-gb200-rdma", + Values: values, + }) + if err != nil { + t.Fatalf("render: %v", err) + } + + var doc map[string]any + if err := yaml.Unmarshal(out, &doc); err != nil { + t.Fatalf("rendered YAML does not parse: %v\n%s", err, out) + } + specNode, ok := doc["spec"].(map[string]any) + if !ok { + t.Fatalf("DaemonSet spec not found in rendered manifest:\n%s", out) + } + templateNode, ok := specNode["template"].(map[string]any) + if !ok { + t.Fatalf("DaemonSet spec.template not found in rendered manifest:\n%s", out) + } + spec, ok := templateNode["spec"].(map[string]any) + if !ok { + t.Fatalf("DaemonSet pod spec not found in rendered manifest:\n%s", out) + } + return spec +} + +// TestGB200RDMAInstallerAcceleratedNodeSelector verifies +// --accelerated-node-selector (values["acceleratedNodeSelector"], see +// registry.yaml's nodeSelectorPaths) scopes the DaemonSet to the selected +// pool instead of running on every GB200/ARM64 node, and that an unset +// selector omits the nodeSelector field rather than rendering an empty one. +func TestGB200RDMAInstallerAcceleratedNodeSelector(t *testing.T) { + tests := []struct { + name string + values map[string]any + wantSelector map[string]any + }{ + { + name: "present scopes render", + values: map[string]any{ + "acceleratedNodeSelector": map[string]any{"cloud.google.com/gke-nodepool": "a4x-pool-a"}, + }, + wantSelector: map[string]any{"cloud.google.com/gke-nodepool": "a4x-pool-a"}, + }, + { + name: "absent omits field", + values: map[string]any{}, + wantSelector: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := renderGB200RDMAInstaller(t, tt.values) + nodeSelector, present := spec["nodeSelector"] + if tt.wantSelector == nil { + if present { + t.Errorf("expected no nodeSelector field when acceleratedNodeSelector is unset, got spec: %v", spec) + } + return + } + got, ok := nodeSelector.(map[string]any) + if !ok { + t.Fatalf("expected rendered pod spec to carry nodeSelector, got spec: %v", spec) + } + for k, want := range tt.wantSelector { + if got[k] != want { + t.Errorf("nodeSelector[%s] = %v, want %v", k, got[k], want) + } + } + }) + } +} diff --git a/recipes/overlays/gb200-gke-cos-inference-dynamo.yaml b/recipes/overlays/gb200-gke-cos-inference-dynamo.yaml new file mode 100644 index 000000000..42b00f170 --- /dev/null +++ b/recipes/overlays/gb200-gke-cos-inference-dynamo.yaml @@ -0,0 +1,99 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: RecipeMetadata +apiVersion: aicr.run/v1alpha2 +metadata: + name: gb200-gke-cos-inference-dynamo + +spec: + # Inherits from gb200-gke-cos-inference (GB200 + GKE COS inference settings) + # Adds Dynamo inference platform components. + base: gb200-gke-cos-inference + + criteria: + service: gke + accelerator: gb200 + os: cos + intent: inference + platform: dynamo + + # DRA requires Kubernetes 1.34+ (GA) + constraints: + - name: K8s.server.version + value: ">= 1.34" + + componentRefs: + - name: grove + type: Helm + source: oci://ghcr.io/ai-dynamo/grove + valuesFile: components/grove/values.yaml + + - name: dynamo-platform + type: Helm + source: https://helm.ngc.nvidia.com/nvidia/ai-dynamo + valuesFile: components/dynamo-platform/values.yaml + dependencyRefs: + - grove + - cert-manager + - kube-prometheus-stack + - gpu-operator + - kai-scheduler + + validation: + deployment: + checks: + - operator-health + - expected-resources + - gpu-operator-version + - check-nvidia-smi + constraints: + - name: Deployment.gpu-operator.version + value: ">= v25.10.0" + performance: + checks: + - inference-perf + # Gate thresholds mirrored from the GB200 EKS/OKE Dynamo overlays + # (same accelerator, same 4-GPU-per-node shape) until a GKE-specific + # measured baseline is published. Model + concurrency are pinned so + # the gate stays valid independent of the compiled defaults. + # + # NOTE: this throughput floor is a fixed absolute full-node value (here a + # 4-GPU GB200 node) and is not normalized for GPU count, so a smaller SKU + # of this accelerator can false-fail a healthy run. A normalized per-GPU + # floor is tracked in https://github.com/NVIDIA/aicr/issues/1254. + constraints: + - name: inference-model + value: Qwen/Qwen3-8B + - name: inference-concurrency-per-gpu + value: "256" + - name: inference-routing-mode + value: dynamo-router + - name: inference-throughput + value: ">= 50000" + - name: inference-ttft-p99 + value: "<= 2000" + conformance: + checks: + - platform-health + - gpu-operator-health + - dra-support + - accelerator-metrics + - ai-service-metrics + - inference-gateway + - gang-scheduling + - pod-autoscaling + - cluster-autoscaling + - robust-controller + - secure-accelerator-access diff --git a/recipes/overlays/gb200-gke-cos-inference.yaml b/recipes/overlays/gb200-gke-cos-inference.yaml new file mode 100644 index 000000000..2ccd9f76f --- /dev/null +++ b/recipes/overlays/gb200-gke-cos-inference.yaml @@ -0,0 +1,110 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GKE GB200 (A4X) inference: RDMA/RoCE networking. +# +# Same rationale as gb200-gke-cos-training: GB200 on GKE needs the NCCL gIB +# plugin over the cluster-provisioned RDMA fabric. Multi-node tensor-parallel +# serving crosses the same RoCE fabric as training. + +kind: RecipeMetadata +apiVersion: aicr.run/v1alpha2 +metadata: + name: gb200-gke-cos-inference + +spec: + base: gke-cos-inference + + criteria: + service: gke + accelerator: gb200 + os: cos + intent: inference + + constraints: + # >= 1.34: GB200 multi-node serving provisions the IMEX channel through a + # DRA ComputeDomain that requires the GA resource.k8s.io/v1 API. + - name: K8s.server.version + value: ">= 1.34" + + componentRefs: + - name: gpu-operator + type: Helm + preManifestFiles: + - components/gpu-operator/manifests/kernel-module-params.yaml + dependencyRefs: + - nfd + - cert-manager + - kube-prometheus-stack + - nodewright-customizations + overrides: + cdi: + enabled: true + gdrcopy: + enabled: true + driver: + kernelModuleConfig: + name: nvidia-kernel-module-params + + # gcp-driver-installer's pinned partition-gpus init container image is + # amd64-only and fails with "exec format error" on arm64 (found via live + # GB200 GKE validation) — override with a multi-arch digest. Only takes + # effect under --profile gpuStack=bundle-installer (installer.enabled); + # a no-op under the gke-default profile. + - name: gcp-driver-installer + type: Helm + overrides: + partitionGpuImage: "gcr.io/gke-release/nvidia-partition-gpu@sha256:de12f85ebfb4fb6c1893cd30c23aab662a72fa0448f97ef74fccb82d7522ef17" + + - name: gke-gb200-rdma + type: Helm + manifestFiles: + - components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml + + - name: nodewright-customizations + type: Helm + manifestFiles: + - components/nodewright-customizations/manifests/tuning-gke.yaml + overrides: + accelerator: gb200 + intent: inference + dependencyRefs: + - nodewright-operator + + - name: nfd + type: Helm + overrides: + topologyUpdater: + enable: true + + validation: + deployment: + checks: + - operator-health + - expected-resources + - gpu-operator-version + - check-nvidia-smi + constraints: + - name: Deployment.gpu-operator.version + value: ">= v25.10.0" + conformance: + checks: + - platform-health + - gpu-operator-health + - dra-support + - accelerator-metrics + - ai-service-metrics + - gang-scheduling + - pod-autoscaling + - cluster-autoscaling diff --git a/recipes/overlays/gb200-gke-cos-training-kubeflow.yaml b/recipes/overlays/gb200-gke-cos-training-kubeflow.yaml new file mode 100644 index 000000000..62c374576 --- /dev/null +++ b/recipes/overlays/gb200-gke-cos-training-kubeflow.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: RecipeMetadata +apiVersion: aicr.run/v1alpha2 +metadata: + name: gb200-gke-cos-training-kubeflow + +spec: + # Inherits from gb200-gke-cos-training recipe (GB200 + GKE COS + training settings) + # This overlay adds Kubeflow Training Operator for distributed training with TrainJob + base: gb200-gke-cos-training + + criteria: + service: gke + accelerator: gb200 + os: cos + intent: training + platform: kubeflow + + # Constraints for GB200 on GKE COS for Kubeflow training workloads + constraints: + - name: K8s.server.version + value: ">= 1.34" + + # Kubeflow Training Operator for TrainJob support + componentRefs: + - name: kubeflow-trainer + type: Helm + valuesFile: components/kubeflow-trainer/values.yaml + manifestFiles: + - components/kubeflow-trainer/manifests/torch-distributed-cluster-training-runtime.yaml + dependencyRefs: + - cert-manager + - kube-prometheus-stack + - gpu-operator diff --git a/recipes/overlays/gb200-gke-cos-training-slurm.yaml b/recipes/overlays/gb200-gke-cos-training-slurm.yaml new file mode 100644 index 000000000..718bc3468 --- /dev/null +++ b/recipes/overlays/gb200-gke-cos-training-slurm.yaml @@ -0,0 +1,158 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: RecipeMetadata +apiVersion: aicr.run/v1alpha2 +metadata: + name: gb200-gke-cos-training-slurm + +spec: + # GB200 + GKE + COS + training with the Slinky operator and a + # Slinky-managed Slurm cluster. GB200 GPU GRES, task isolation, and the + # NVLS/IMEX ComputeDomain wiring mirror the gb200-eks-ubuntu-training-slurm + # leaf (same accelerator, same 4-GPU-per-node shape); remaining GKE-specific + # tuning is layered at install time via `aicr bundle ... --set + # slinkyslurm:...` or a valuesFile. + base: gb200-gke-cos-training + + criteria: + service: gke + accelerator: gb200 + os: cos + intent: training + platform: slurm + + # Unlike the EKS slurm leaf, no os-ubuntu / os-cos mixin is needed: gke-cos + # already disables GPU driver installation and pins DRA / nodewright paths + # for COS's read-only rootfs. + # + # K8s.server.version (>= 1.34) is inherited from gb200-gke-cos-training.yaml; + # Slinky on GKE has no tighter floor than the parent leaf, so we don't + # restate it here (cf. the EKS slurm leaf, which restates the same floor to + # match its parent). + + # The Slinky operator (CRDs + operator + cluster instance) is declared + # inline per slurm leaf, mirroring the dynamo-platform pattern in + # h100-*-inference-dynamo leaves. Inlining lets each leaf carry its + # own GPU/GRES tuning without fighting the mixin-vs-leaf identity-field + # guard in mixinComponentRefSafeForMerge (pkg/recipe/metadata_store.go), + # and keeps base.yaml free of platform-specific components. + # + # GPU GRES and device isolation on slinky-slurm must be declared in four + # places because the chart does not derive GRES config in slurm.conf from + # pod resource limits + # (see comment in components/slinky-slurm/values.yaml): + # 1. controller.extraConfMap.GresTypes: enables the `gpu` GRES type. + # 2. controller.extraConfMap.TaskPlugin: activates task/cgroup so + # cgroup.conf's ConstrainDevices setting enforces GPU allocations. + # 3. nodesets.slinky.extraConfMap.Gres: adds `Gres=gpu:gb200:4` to + # slurmd's --conf so slurmctld knows it has GPUs to allocate via + # `srun --gres=gpu:N`. + # 4. nodesets.slinky.slurmd.resources.limits.nvidia.com/gpu: reserves + # 4 GB200s on the slurmd pod so the NVIDIA device plugin injects + # /dev/nvidia* into the container. Without this `gres.conf`'s + # AutoDetect=nvidia finds nothing. `requests` is omitted: Kubernetes + # auto-mirrors requests=limits for extended resources. + # The count is per Kubernetes GPU node / slurmd pod (a4x-highgpu-4g exposes + # nvidia.com/gpu.count=4), not cluster-total or rack-total capacity. + # Accelerated nodeSelector/tolerations on slurmd are injected via the + # registry's nodesets.slinky.podSpec.{nodeSelector,tolerations} paths. + # + # No slinky-topograph here: unlike the H100 GKE slurm leaf (which needs + # Slurm topology-awareness over TCPXO), GB200's NVLS all-reduce path runs + # over the IMEX ComputeDomain below, the same reason + # gb200-eks-ubuntu-training-slurm.yaml omits it too. + componentRefs: + - name: mariadb-operator-crds + type: Helm + valuesFile: components/mariadb-operator-crds/values.yaml + + - name: mariadb-operator + type: Helm + valuesFile: components/mariadb-operator/values.yaml + dependencyRefs: + - mariadb-operator-crds + + - name: slurm-accounting-mariadb + type: Helm + valuesFile: components/slurm-accounting-mariadb/values.yaml + dependencyRefs: + - mariadb-operator + - mariadb-operator-crds + + - name: slinky-slurm-operator-crds + type: Helm + valuesFile: components/slinky-slurm-operator-crds/values.yaml + + - name: slinky-slurm-operator + type: Helm + valuesFile: components/slinky-slurm-operator/values.yaml + dependencyRefs: + - cert-manager + - slinky-slurm-operator-crds + + - name: slinky-slurm + type: Helm + valuesFile: components/slinky-slurm/values.yaml + preManifestFiles: + - components/slinky-slurm/manifests/enroot-config.yaml + - components/slinky-slurm/manifests/shared-storage-pvcs.yaml + # Submit this immutable ComputeDomain after the NVIDIA DRA driver and + # before the Slinky chart creates NodeSet pods that consume its RCT. + # RCT reconciliation may complete asynchronously after this apply. + - components/slinky-slurm/manifests/compute-domain.yaml + dependencyRefs: + - slurm-accounting-mariadb + - nvidia-dra-driver-gpu + - slinky-slurm-operator + - slinky-slurm-operator-crds + overrides: + controller: + extraConfMap: + GresTypes: "gpu" + TaskPlugin: "task/cgroup,task/affinity" + SwitchType: "switch/nvidia_imex" + nodesets: + slinky: + extraConfMap: + Gres: "gpu:gb200:4" + podSpec: + resourceClaims: + # The claim name and ResourceClaimTemplate name are internal + # integration values and must not be overridden. They must + # stay aligned with the ComputeDomain pre-manifest; a mismatch + # can leave Slurm node pods Pending. + - name: imex-channels + resourceClaimTemplateName: slinky-slurm-imex-channels + slurmd: + resources: + limits: + nvidia.com/gpu: 4 + claims: + - name: imex-channels + + # K8s-native nccl-all-reduce-bw checks are dropped on Slinky leaves: + # those checks launch Pods against the cluster scheduler, so on a + # Slinky-managed cluster they bypass slurmd entirely and measure the + # wrong path. Slurm-specific health is covered by the conformance check. + # Deployment checks are inherited unchanged from gb200-gke-cos-training. + validation: + conformance: + checks: + - slinky-slurm-health + # Selected only by IMEX-capable Slinky Slurm recipes. + - slinky-slurm-imex-channel + performance: + checks: [] + constraints: [] diff --git a/recipes/overlays/gb200-gke-cos-training.yaml b/recipes/overlays/gb200-gke-cos-training.yaml new file mode 100644 index 000000000..a062a5b87 --- /dev/null +++ b/recipes/overlays/gb200-gke-cos-training.yaml @@ -0,0 +1,123 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GKE GB200 (A4X): RDMA/RoCE networking. +# +# GCP reference: "Create a custom AI-optimized GKE cluster which uses A4X" +# https://docs.cloud.google.com/ai-hypercomputer/docs/create/gke-ai-hypercompute-custom-a4x + +kind: RecipeMetadata +apiVersion: aicr.run/v1alpha2 +metadata: + name: gb200-gke-cos-training + +spec: + base: gke-cos-training + + criteria: + service: gke + accelerator: gb200 + os: cos + intent: training + + constraints: + # >= 1.34: GB200 NVLS provisions the IMEX channel through a DRA + # ComputeDomain that requires the GA resource.k8s.io/v1 API. + - name: K8s.server.version + value: ">= 1.34" + + componentRefs: + - name: gpu-operator + type: Helm + preManifestFiles: + # NVreg flag required for dma-buf attach over the RoCE fabric. + - components/gpu-operator/manifests/kernel-module-params.yaml + dependencyRefs: + - nfd + - cert-manager + - kube-prometheus-stack + - nodewright-customizations + overrides: + cdi: + enabled: true + gdrcopy: + enabled: true + driver: + kernelModuleConfig: + name: nvidia-kernel-module-params + + # gcp-driver-installer's pinned partition-gpus init container image is + # amd64-only and fails with "exec format error" on arm64 (found via live + # GB200 GKE validation) — override with a multi-arch digest. Only takes + # effect under --profile gpuStack=bundle-installer (installer.enabled); + # a no-op under the gke-default profile. + - name: gcp-driver-installer + type: Helm + overrides: + partitionGpuImage: "gcr.io/gke-release/nvidia-partition-gpu@sha256:de12f85ebfb4fb6c1893cd30c23aab662a72fa0448f97ef74fccb82d7522ef17" + + # The GKENetworkParamSet/Network CRs (gvnic-1, rdma-0..3) are a cluster + # provisioning prerequisite, not created by this component; see + # docs/integrator/gke-gb200-networking.md. + - name: gke-gb200-rdma + type: Helm + manifestFiles: + - components/gke-gb200-rdma/manifests/nccl-gib-installer-arm64.yaml + + - name: nodewright-customizations + type: Helm + manifestFiles: + - components/nodewright-customizations/manifests/tuning-gke.yaml + overrides: + accelerator: gb200 + intent: multiNodeTraining + dependencyRefs: + - nodewright-operator + + - name: nfd + type: Helm + overrides: + topologyUpdater: + enable: true + + validation: + deployment: + checks: + - operator-health + - expected-resources + - gpu-operator-version + - check-nvidia-smi + constraints: + - name: Deployment.gpu-operator.version + value: ">= v25.10.0" + performance: + checks: + - nccl-all-reduce-bw-nvls + # Calibrated on a4x-highgpu-4g (4x GB200/node): 2-node/8-GPU + # all_reduce_perf measured 281.936 GB/s avg bus bandwidth. + constraints: + - name: nccl-all-reduce-bw-nvls + value: ">= 250" + conformance: + checks: + - platform-health + - gpu-operator-health + - dra-support + - accelerator-metrics + - ai-service-metrics + - gang-scheduling + - pod-autoscaling + - cluster-autoscaling + - robust-controller + - secure-accelerator-access diff --git a/recipes/registry.yaml b/recipes/registry.yaml index abbf297a2..b5eee0e61 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -208,6 +208,26 @@ components: tolerationPaths: - acceleratedTolerations + # gvnic-1/rdma-0..3 are a cluster provisioning prerequisite (see + # docs/integrator/gke-gb200-networking.md), not created by this component; + # this only ships the gIB installer DaemonSet. + - name: gke-gb200-rdma + displayName: gke-gb200-rdma + valueOverrideKeys: + - gkegb200rdma + nodeScheduling: + accelerated: + nodeSelectorPaths: + - acceleratedNodeSelector + tolerationPaths: + - acceleratedTolerations + healthCheck: + assertFile: checks/gke-gb200-rdma/health-check.yaml + helm: + # Manifest-only component - no external Helm chart, uses manifestFiles + defaultRepository: "" + defaultNamespace: kube-system + - name: aws-efa displayName: aws-efa valueOverrideKeys: diff --git a/validators/performance/consts.go b/validators/performance/consts.go index 097cdac24..8bdd93169 100644 --- a/validators/performance/consts.go +++ b/validators/performance/consts.go @@ -21,6 +21,7 @@ const ( versionV1alpha1 = "v1alpha1" versionV1beta1 = "v1beta1" keyName = "name" + keyOperator = "operator" checkNameNCCLAllReduceBW = "nccl-all-reduce-bw" // nodeJobName is the name of both the NCCL worker replicatedJob and its diff --git a/validators/performance/inference_perf_constraint.go b/validators/performance/inference_perf_constraint.go index d4fa5f883..604982e3d 100644 --- a/validators/performance/inference_perf_constraint.go +++ b/validators/performance/inference_perf_constraint.go @@ -405,6 +405,7 @@ type inferenceWorkloadConfig struct { deployedByUs bool // true if we (or a prior run we own) created the workload modelCacheSize string // PVC size (e.g. "100Gi") enabling the model-weights cache; empty = disabled modelCacheStorageClass string // StorageClass for the cache PVC; empty = cluster default + gpuNodeInstanceType string // chosen node's node.kubernetes.io/instance-type; empty if unlabeled routingMode inferenceRoutingMode routerMode string // Dynamo frontend DYN_ROUTER_MODE (dynamo-router path only); env > default (see resolveRouterMode) @@ -770,6 +771,7 @@ func buildInferenceConfig(ctx *validators.Context, mode *allocmode.Mode) (*infer model: model, modelCacheSize: cacheSize, modelCacheStorageClass: strings.TrimSpace(os.Getenv(envModelCacheStorageClass)), + gpuNodeInstanceType: chosen.Labels[instanceTypeLabel], routingMode: routingMode, routerMode: routerMode, gpuAllocMode: mode, @@ -1823,7 +1825,7 @@ func tolerationsToUnstructured(tolerations []v1.Toleration) []any { tolList := make([]any, 0, len(tolerations)) for _, t := range tolerations { tolMap := map[string]any{ - "operator": string(t.Operator), + keyOperator: string(t.Operator), } if t.Key != "" { tolMap["key"] = t.Key diff --git a/validators/performance/model_cache.go b/validators/performance/model_cache.go index 229b46987..cb4235884 100644 --- a/validators/performance/model_cache.go +++ b/validators/performance/model_cache.go @@ -28,6 +28,7 @@ import ( "github.com/NVIDIA/aicr/validators" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -125,6 +126,36 @@ const ( cacheWorkerFSGroup = int64(1000) ) +// storageCompatibilityRule declares that on a given CSI provisioner, the +// listed machine families can only attach a StorageClass whose +// parameters.type either carries compatibleTypePrefix or exactly matches +// autoSelectType (a driver-specific value that resolves to a compatible disk +// per-node rather than naming one directly, e.g. GKE's "dynamic"; leave empty +// for a provisioner with no such value). Anything else provisioned by that +// driver is rejected at attach time. Add a rule here for any other +// cloud/provisioner with the same shape of restriction; nothing else in this +// file needs to change. +type storageCompatibilityRule struct { + provisioner string + families map[string]bool // node.kubernetes.io/instance-type family segment, e.g. "a4x" for "a4x-highgpu-4g" + compatibleTypePrefix string + autoSelectType string + docsRef string +} + +var storageCompatibilityRules = []storageCompatibilityRule{ + { + provisioner: "pd.csi.storage.gke.io", // GKE Persistent Disk CSI driver (also provisions Hyperdisk) + families: map[string]bool{"a4x": true}, + compatibleTypePrefix: "hyperdisk-", + // "dynamic" auto-selects Hyperdisk vs Persistent Disk per the node's + // machine type (GKE 1.35.3-gke.1290000+); a4x can't attach Persistent + // Disk at all, so on a4x nodes it always resolves to Hyperdisk. + autoSelectType: "dynamic", + docsRef: "docs/integrator/gke-gb200-networking.md#storage-prerequisites", + }, +} + // modelCacheEnabled reports whether the PVC cache is active for this run. // config.modelCacheSize is "" only when the operator explicitly disabled it // (see parseModelCacheSize); the unset case has already been defaulted on. @@ -162,23 +193,65 @@ func parseModelCacheSize(raw string) (string, bool, error) { return raw, true, nil } -// clusterHasDefaultStorageClass reports whether any StorageClass on the cluster -// is annotated as the default. Used to fail fast before provisioning a cache -// PVC with no StorageClass on a cluster that has no default. -func clusterHasDefaultStorageClass(ctx *validators.Context) (bool, error) { +// defaultStorageClass returns the cluster's effective default StorageClass, +// or nil if none is annotated default. Kubernetes tolerates more than one +// StorageClass annotated default; its own DefaultStorageClass admission +// controller resolves the ambiguity by picking the most recently created one +// (https://kubernetes.io/docs/concepts/storage/storage-classes/#default-storageclass). +// Matching that here means this pre-flight checks the same StorageClass a PVC +// with no storageClassName would actually bind to. +func defaultStorageClass(ctx *validators.Context) (*storagev1.StorageClass, error) { listCtx, cancel := context.WithTimeout(ctx.Ctx, defaults.DiagnosticTimeout) defer cancel() scs, err := ctx.Clientset.StorageV1().StorageClasses().List(listCtx, metav1.ListOptions{}) if err != nil { - return false, errors.Wrap(errors.ErrCodeInternal, "failed to list StorageClasses for cache pre-flight", err) + return nil, errors.Wrap(errors.ErrCodeInternal, "failed to list StorageClasses for cache pre-flight", err) } + var best *storagev1.StorageClass for i := range scs.Items { - ann := scs.Items[i].Annotations - if ann[defaultStorageClassAnnotation] == defaultStorageClassAnnotationValue || ann[defaultStorageClassAnnotationBeta] == defaultStorageClassAnnotationValue { - return true, nil + sc := &scs.Items[i] + ann := sc.Annotations + if ann[defaultStorageClassAnnotation] != defaultStorageClassAnnotationValue && ann[defaultStorageClassAnnotationBeta] != defaultStorageClassAnnotationValue { + continue + } + if best == nil || sc.CreationTimestamp.After(best.CreationTimestamp.Time) { + best = sc + } + } + return best, nil //nolint:nilnil // nil, nil means no default StorageClass is set, not an error +} + +// machineFamily returns the leading segment of a node.kubernetes.io/instance-type +// value, e.g. "a4x" for "a4x-highgpu-4g". Empty for an empty or family-less input. +func machineFamily(instanceType string) string { + family, _, _ := strings.Cut(instanceType, "-") + return family +} + +// checkStorageClassNodeCompatibility reports an error when sc's disk type +// can't attach to the worker node's machine family, per +// storageCompatibilityRules. A nil sc (not found, e.g. a typo in the +// explicit override) is not an error here; that surfaces via the normal +// PVC-create path instead. +func checkStorageClassNodeCompatibility(instanceType string, sc *storagev1.StorageClass) error { + if sc == nil { + return nil + } + family := machineFamily(instanceType) + for _, rule := range storageCompatibilityRules { + if sc.Provisioner != rule.provisioner || !rule.families[family] { + continue } + typ := sc.Parameters["type"] + if strings.HasPrefix(typ, rule.compatibleTypePrefix) || (rule.autoSelectType != "" && typ == rule.autoSelectType) { + continue + } + return errors.New(errors.ErrCodeInvalidRequest, fmt.Sprintf( + "model-weights cache PVC would bind to StorageClass %q (provisioner %s), which node machine family %q can't attach; "+ + "set %s to a StorageClass whose parameters.type starts with %q, or disable the cache with %s=off; see %s", + sc.Name, sc.Provisioner, family, envModelCacheStorageClass, rule.compatibleTypePrefix, envModelCacheSize, rule.docsRef)) } - return false, nil + return nil } // ensureModelCache provisions the model-weights cache when enabled: an RWO PVC @@ -207,21 +280,39 @@ func ensureModelCache(ctx *validators.Context, config *inferenceWorkloadConfig) fmt.Sprintf("invalid %s=%q: must be a Kubernetes quantity (e.g. 100Gi)", envModelCacheSize, config.modelCacheSize), err) } - // Fail fast when there is no StorageClass to bind the cache PVC to: with no - // explicit MODEL_CACHE_STORAGE_CLASS, the PVC relies on a cluster default, - // and without one it sits Pending until the populate Job times out (minutes). - // Surface an actionable error immediately instead. - if strings.TrimSpace(config.modelCacheStorageClass) == "" { - hasDefault, derr := clusterHasDefaultStorageClass(ctx) + // Resolve the StorageClass the cache PVC will bind to (explicit name, or + // the cluster default) for the checks below. + explicitSC := strings.TrimSpace(config.modelCacheStorageClass) + var resolvedSC *storagev1.StorageClass + if explicitSC == "" { + sc, derr := defaultStorageClass(ctx) if derr != nil { return derr } - if !hasDefault { + if sc == nil { return errors.New(errors.ErrCodeInvalidRequest, fmt.Sprintf("model-weights cache is enabled but the cluster has no default StorageClass and %s is unset; "+ "set %s= (e.g. gp2/gp3 on EKS, standard-rwo on GKE) or disable the cache with %s=off", envModelCacheStorageClass, envModelCacheStorageClass, envModelCacheSize)) } + resolvedSC = sc + } else { + getCtx, getCancel := context.WithTimeout(ctx.Ctx, defaults.DiagnosticTimeout) + sc, gerr := ctx.Clientset.StorageV1().StorageClasses().Get(getCtx, explicitSC, metav1.GetOptions{}) + getCancel() + switch { + case gerr == nil: + resolvedSC = sc + case apierrors.IsNotFound(gerr): + // Leave resolvedSC nil: fall through to the existing PVC-create + // path, which surfaces a nonexistent StorageClass the same way + // it always has. + default: + return errors.Wrap(errors.ErrCodeInternal, "failed to get StorageClass for cache pre-flight", gerr) + } + } + if cerr := checkStorageClassNodeCompatibility(config.gpuNodeInstanceType, resolvedSC); cerr != nil { + return cerr } // Bound the create calls so a slow/wedged apiserver can't burn the check diff --git a/validators/performance/model_cache_test.go b/validators/performance/model_cache_test.go index ce3fd6618..0af37145f 100644 --- a/validators/performance/model_cache_test.go +++ b/validators/performance/model_cache_test.go @@ -29,6 +29,7 @@ import ( storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" ) @@ -310,22 +311,6 @@ func TestWrapPopulateJobError(t *testing.T) { } } -// TestEnsureModelCache_DisabledNoop verifies that with the cache disabled no PVC -// or Job is created — the default behavior is unchanged. - -func TestEnsureModelCache_DisabledNoop(t *testing.T) { - client := fake.NewClientset() - ctx := &validators.Context{Ctx: context.Background(), Clientset: client} - cfg := &inferenceWorkloadConfig{namespace: "ns", modelCacheSize: ""} - if err := ensureModelCache(ctx, cfg); err != nil { - t.Fatalf("unexpected error: %v", err) - } - pvcs, _ := client.CoreV1().PersistentVolumeClaims("ns").List(context.Background(), metav1.ListOptions{}) - if len(pvcs.Items) != 0 { - t.Errorf("no PVC should be created when cache disabled, got %d", len(pvcs.Items)) - } -} - // TestParseModelCacheSize verifies the on-by-default policy: unset → default // size (enabled), the disable sentinels → disabled, an explicit quantity passes // through, and garbage fails closed. @@ -368,55 +353,218 @@ func TestParseModelCacheSize(t *testing.T) { } } -// TestClusterHasDefaultStorageClass verifies detection of a default-annotated -// StorageClass (the cache pre-flight's signal). -func TestClusterHasDefaultStorageClass(t *testing.T) { - def := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ - Name: "gp3", Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}} - nondef := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "gp2"}} +// TestDefaultStorageClass verifies detection of a default-annotated +// StorageClass (the cache pre-flight's signal). When more than one +// StorageClass is annotated default, Kubernetes' own admission controller +// picks the most recently created one — the two "multiple defaults" cases +// below list the same pair in both orders to prove selection follows +// CreationTimestamp, not list order. +func TestDefaultStorageClass(t *testing.T) { + older := metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + newer := metav1.NewTime(time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)) - t.Run("has default", func(t *testing.T) { - ctx := &validators.Context{Ctx: context.Background(), Clientset: fake.NewClientset(def, nondef)} - got, err := clusterHasDefaultStorageClass(ctx) - if err != nil || !got { - t.Errorf("got (%v,%v), want (true,nil)", got, err) - } - }) - t.Run("no default", func(t *testing.T) { - ctx := &validators.Context{Ctx: context.Background(), Clientset: fake.NewClientset(nondef)} - got, err := clusterHasDefaultStorageClass(ctx) - if err != nil || got { - t.Errorf("got (%v,%v), want (false,nil)", got, err) - } - }) - t.Run("legacy beta annotation counts as default", func(t *testing.T) { - beta := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ - Name: "gp2", Annotations: map[string]string{defaultStorageClassAnnotationBeta: "true"}}} - ctx := &validators.Context{Ctx: context.Background(), Clientset: fake.NewClientset(beta)} - got, err := clusterHasDefaultStorageClass(ctx) - if err != nil || !got { - t.Errorf("got (%v,%v), want (true,nil) for beta is-default-class annotation", got, err) + tests := []struct { + name string + classes []runtime.Object + wantName string // "" means want nil (no default) + }{ + { + name: "has default", + classes: []runtime.Object{ + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "gp3", Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}}, + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "gp2"}}, + }, + wantName: "gp3", + }, + { + name: "no default", + classes: []runtime.Object{ + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "gp2"}}, + }, + wantName: "", + }, + { + name: "legacy beta annotation counts as default", + classes: []runtime.Object{ + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "gp2", Annotations: map[string]string{defaultStorageClassAnnotationBeta: "true"}}}, + }, + wantName: "gp2", + }, + { + name: "multiple defaults, newer listed first: newer still wins", + classes: []runtime.Object{ + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "newer", CreationTimestamp: newer, Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}}, + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "older", CreationTimestamp: older, Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}}, + }, + wantName: "newer", + }, + { + name: "multiple defaults, older listed first: newer still wins", + classes: []runtime.Object{ + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "older", CreationTimestamp: older, Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}}, + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "newer", CreationTimestamp: newer, Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}}, + }, + wantName: "newer", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := &validators.Context{Ctx: context.Background(), Clientset: fake.NewClientset(tt.classes...)} + got, err := defaultStorageClass(ctx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + gotName := "" + if got != nil { + gotName = got.Name + } + if gotName != tt.wantName { + t.Errorf("got %q, want %q", gotName, tt.wantName) + } + }) + } +} + +// TestMachineFamily verifies the family segment extracted from a +// node.kubernetes.io/instance-type value. +func TestMachineFamily(t *testing.T) { + tests := []struct { + instanceType string + want string + }{ + {"a4x-highgpu-4g", "a4x"}, + {"n2-standard-4", "n2"}, + {"", ""}, + } + for _, tt := range tests { + if got := machineFamily(tt.instanceType); got != tt.want { + t.Errorf("machineFamily(%q) = %q, want %q", tt.instanceType, got, tt.want) } - }) + } +} + +// TestCheckStorageClassNodeCompatibility verifies the rule-table lookup: a +// machine family listed under a rule can only attach a StorageClass whose +// parameters.type carries that rule's compatibleTypePrefix; every other +// family/provisioner/type combination passes. +func TestCheckStorageClassNodeCompatibility(t *testing.T) { + pdBalanced := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "standard-rwo"}, + Provisioner: "pd.csi.storage.gke.io", + Parameters: map[string]string{"type": "pd-balanced"}, + } + hyperdiskBalanced := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "hyperdisk-balanced"}, + Provisioner: "pd.csi.storage.gke.io", + Parameters: map[string]string{"type": "hyperdisk-balanced"}, + } + dynamicSelect := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "dynamic-volume"}, + Provisioner: "pd.csi.storage.gke.io", + Parameters: map[string]string{"type": "dynamic", "pd-type": "pd-balanced", "hyperdisk-type": "hyperdisk-balanced"}, + } + otherProvisioner := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "gp3"}, + Provisioner: "ebs.csi.aws.com", + Parameters: map[string]string{"type": "gp3"}, + } + + tests := []struct { + name string + instanceType string + sc *storagev1.StorageClass + wantErr bool + }{ + {"a4x with Persistent Disk is rejected", "a4x-highgpu-4g", pdBalanced, true}, + {"a4x with explicit Hyperdisk selection is fine", "a4x-highgpu-4g", hyperdiskBalanced, false}, + {"a4x with dynamic disk-type selection is fine", "a4x-highgpu-4g", dynamicSelect, false}, + {"non-a4x family with Persistent Disk is fine", "n2-standard-4", pdBalanced, false}, + {"non-a4x family with dynamic disk-type selection is fine", "n2-standard-4", dynamicSelect, false}, + {"a4x with an unrelated provisioner is fine", "a4x-highgpu-4g", otherProvisioner, false}, + {"nil StorageClass is fine (not this function's concern)", "a4x-highgpu-4g", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkStorageClassNodeCompatibility(tt.instanceType, tt.sc) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr && !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error code = %v, want ErrCodeInvalidRequest", err) + } + }) + } } -// TestEnsureModelCache_NoDefaultStorageClassFailsFast verifies that with the -// cache enabled, no explicit StorageClass, and no cluster default, the validator -// fails fast (ErrCodeInvalidRequest) without creating the PVC — rather than -// leaving it Pending until the populate-Job timeout. -func TestEnsureModelCache_NoDefaultStorageClassFailsFast(t *testing.T) { - ctx := &validators.Context{Ctx: context.Background(), Clientset: fake.NewClientset()} - cfg := &inferenceWorkloadConfig{namespace: "ns", model: "Qwen/Qwen3-8B", modelCacheSize: defaultModelCacheSize} - err := ensureModelCache(ctx, cfg) - if err == nil { - t.Fatal("expected fast-fail error when cache enabled with no default StorageClass") - } - if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { - t.Errorf("error code = %v, want ErrCodeInvalidRequest", err) - } - pvcs, _ := ctx.Clientset.CoreV1().PersistentVolumeClaims("ns").List(context.Background(), metav1.ListOptions{}) - if len(pvcs.Items) != 0 { - t.Errorf("no PVC should be created on fast-fail, got %d", len(pvcs.Items)) +// TestEnsureModelCache covers the pre-flight's error paths: disabled is a +// no-op, an enabled cache with no resolvable StorageClass errors, and an +// enabled cache whose resolved StorageClass (cluster-default or explicit +// override) is incompatible with the node's machine family errors too, +// in every case before a PVC is created, rather than that surfacing later +// as a Pending claim or FailedAttachVolume. +func TestEnsureModelCache(t *testing.T) { + pdBalanced := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: "standard-rwo", Annotations: map[string]string{defaultStorageClassAnnotation: "true"}}, + Provisioner: "pd.csi.storage.gke.io", + Parameters: map[string]string{"type": "pd-balanced"}, + } + + tests := []struct { + name string + classes []runtime.Object + cfg *inferenceWorkloadConfig + wantErr bool + }{ + { + name: "disabled is a no-op", + cfg: &inferenceWorkloadConfig{namespace: "ns", modelCacheSize: ""}, + }, + { + name: "no default StorageClass errors", + cfg: &inferenceWorkloadConfig{namespace: "ns", model: "Qwen/Qwen3-8B", modelCacheSize: defaultModelCacheSize}, + wantErr: true, + }, + { + name: "incompatible cluster default is rejected", + classes: []runtime.Object{pdBalanced}, + cfg: &inferenceWorkloadConfig{ + namespace: "ns", model: "Qwen/Qwen3-8B", modelCacheSize: defaultModelCacheSize, + gpuNodeInstanceType: "a4x-highgpu-4g", + }, + wantErr: true, + }, + { + name: "incompatible explicit override is rejected", + classes: []runtime.Object{pdBalanced}, + cfg: &inferenceWorkloadConfig{ + namespace: "ns", model: "Qwen/Qwen3-8B", modelCacheSize: defaultModelCacheSize, + gpuNodeInstanceType: "a4x-highgpu-4g", modelCacheStorageClass: "standard-rwo", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewClientset(tt.classes...) + ctx := &validators.Context{Ctx: context.Background(), Clientset: client} + err := ensureModelCache(ctx, tt.cfg) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr && !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error code = %v, want ErrCodeInvalidRequest", err) + } + pvcs, _ := client.CoreV1().PersistentVolumeClaims(tt.cfg.namespace).List(context.Background(), metav1.ListOptions{}) + if len(pvcs.Items) != 0 { + t.Errorf("no PVC should be created, got %d", len(pvcs.Items)) + } + }) } } diff --git a/validators/performance/nccl_all_reduce_bw_constraint.go b/validators/performance/nccl_all_reduce_bw_constraint.go index 8d2297ba8..fc5de8636 100644 --- a/validators/performance/nccl_all_reduce_bw_constraint.go +++ b/validators/performance/nccl_all_reduce_bw_constraint.go @@ -63,6 +63,14 @@ const ( // ncclTrainingRuntimeName is the name of the TrainingRuntime resource. // Must stay in sync with runtime.yaml. ncclTrainingRuntimeName = "nccl-all-reduce-runtime" + + // ncclWorkloadNamespacePrefix is the base for the per-run benchmark + // namespace (see runNCCLTrainJob). Isolating each run in its own + // namespace, the same pattern inferenceWorkloadNamespacePrefix uses, + // means the fixed resource names below never collide across concurrent + // or crashed runs: uniqueness only has to hold within a namespace, and + // cleanup is a single namespace delete instead of per-resource tracking. + ncclWorkloadNamespacePrefix = "aicr-nccl-perf" ) // skipMsg* are the constraint-result strings returned when the NCCL check cannot @@ -251,6 +259,7 @@ var supportedNCCLCombinations = map[ncclVariant]map[recipe.CriteriaServiceType][ variantNVLS: { recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200}, recipe.CriteriaServiceOKE: {recipe.CriteriaAcceleratorGB200}, + recipe.CriteriaServiceGKE: {recipe.CriteriaAcceleratorGB200}, }, } @@ -478,6 +487,26 @@ func runNCCLTrainJob(ctx *validators.Context, gpuConfig *gpuConfiguration, dynamicClient := ctx.DynamicClient + // Isolate this run in its own namespace, the same pattern + // inferenceWorkloadConfig uses (see deriveRunID/ensureNamespace): every + // fixed resource name below only has to be unique within it, so two + // concurrent (or one crashed, one retried) aicr validate runs can never + // collide, adopt, or delete each other's resources — no lock required. + gpuConfig.Namespace = fmt.Sprintf("%s-%s", ncclWorkloadNamespacePrefix, deriveRunID()) + if err = ensureNamespace(ctx, gpuConfig.Namespace); err != nil { + return "", aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to create NCCL benchmark namespace", err) + } + + // Clean up the per-run namespace (and everything created in it) on every + // exit path from here on, including a failed Trainer install below. + // NotFound-tolerant, so running it after an early/partial-apply failure is + // safe. A cleanup failure only overrides a nil benchErr — see + // foldCleanupError — so it never masks a real benchmark failure. + defer func() { + err = foldCleanupError(err, cleanupNCCLResources(ctx.Clientset, gpuConfig.Namespace), + "NCCL benchmark succeeded but NCCL resource cleanup failed") + }() + // Ensure a usable Kubeflow Trainer. Whether an incomplete installation is a // failure or something to install over is decided by the recipe, not by what // happens to be on the cluster: a recipe that ships the component must have a @@ -492,21 +521,11 @@ func runNCCLTrainJob(ctx *validators.Context, gpuConfig *gpuConfiguration, } if len(installedResources) > 0 { defer func() { - err = foldCleanupError(err, deleteTrainer(dynamicClient, installedResources)) + err = foldCleanupError(err, deleteTrainer(dynamicClient, installedResources), + "NCCL benchmark succeeded but Kubeflow Trainer cleanup failed") }() } - // Clean up NCCL resources on every exit path. Registered after the trainer - // install block but before the apply: defers run LIFO, so this runs *before* - // the conditional deleteTrainer above — the NCCL TrainJob/TrainingRuntime CRs - // are deleted while their CRDs still exist, rather than relying on CRD-delete - // cascade GC. Registering it before applyNCCLResources still guarantees a - // partial-apply failure (e.g. the RoCE claim is created, then the runtime or - // TrainJob apply fails) doesn't leak nccl-roce-rct into the persistent, reused - // validation namespace. cleanupNCCLResources is NotFound-tolerant for every - // resource it deletes, so running it after an early failure is safe. - defer cleanupNCCLResources(dynamicClient, gpuConfig.Namespace) - // Apply runtime and trainjob resources. Propagate an inner code rather than // forcing ErrCodeInternal — a recipe-supplied runtime that fails to render is // an ErrCodeInvalidRequest (recipe-authoring error), not an internal fault. @@ -516,7 +535,7 @@ func runNCCLTrainJob(ctx *validators.Context, gpuConfig *gpuConfiguration, podHelper := &helper.PodLifecycle{ ClientSet: ctx.Clientset, - Namespace: ctx.Namespace, + Namespace: gpuConfig.Namespace, } // Wait for launcher pod and get logs. @@ -533,8 +552,11 @@ type gpuConfiguration struct { WorkerCount int GPUCountPerNode int TotalGPUCount int - Namespace string - Nodes []v1.Node + // Namespace is the per-run benchmark namespace; unset by determineGPUConfig + // and filled in by runNCCLTrainJob once it derives one (see + // ncclWorkloadNamespacePrefix). + Namespace string + Nodes []v1.Node } // parseThreshold extracts the numeric threshold value from a constraint value. @@ -754,7 +776,6 @@ func determineGPUConfig(ctx *validators.Context, service recipe.CriteriaServiceT WorkerCount: len(targetNodes), GPUCountPerNode: gpuCountPerNode, TotalGPUCount: totalGPUs, - Namespace: ctx.Namespace, Nodes: targetNodes, }, nil } @@ -837,9 +858,9 @@ func applyNCCLResources(ctx *validators.Context, dynamicClient dynamic.Interface var instanceType string - // For GKE, discover GPU NIC network names (cluster-specific prefixes). - // Skipped for a recipe-supplied runtime, which owns its own fabric wiring. - if customRuntime == "" && service == recipe.CriteriaServiceGKE { + // GKE GPU NIC discovery only applies to the TCPXO (gpu-nic-*) fabric; + // GB200 uses the gke-gb200-rdma Network CRs instead. + if customRuntime == "" && service == recipe.CriteriaServiceGKE && accelerator != recipe.CriteriaAcceleratorGB200 { gpuNICs, err := gkenet.DiscoverGPUNICNetworks(ctx.Ctx, dynamicClient) if err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to discover GKE GPU NIC networks", err) @@ -929,10 +950,10 @@ func applyNCCLResources(ctx *validators.Context, dynamicClient dynamic.Interface if err != nil { return err } - if err := applyNCCLWorkerScheduling(runtimeObj, effectiveNodeSelector, effectiveTolerations); err != nil { + if err = applyNCCLWorkerScheduling(runtimeObj, effectiveNodeSelector, effectiveTolerations); err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to apply NCCL worker scheduling", err) } - if err := createUnstructured(ctx.Ctx, dynamicClient, trainingRuntimeGVR, config.Namespace, runtimeObj); err != nil { + if err = createUnstructured(ctx.Ctx, dynamicClient, trainingRuntimeGVR, config.Namespace, runtimeObj); err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to apply training runtime", err) } slog.Info("Applied TrainingRuntime", "service", service) @@ -940,7 +961,7 @@ func applyNCCLResources(ctx *validators.Context, dynamicClient dynamic.Interface // Wait for the runtime to be visible to the Trainer admission webhook. // The webhook validates that the referenced runtime exists before allowing // TrainJob creation; without this wait we hit a race condition. - if err := waitForTrainingRuntime(ctx.Ctx, dynamicClient, config.Namespace); err != nil { + if err = waitForTrainingRuntime(ctx.Ctx, dynamicClient, config.Namespace); err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "TrainingRuntime not ready", err) } @@ -953,10 +974,10 @@ func applyNCCLResources(ctx *validators.Context, dynamicClient dynamic.Interface // runtime: IMEX/ComputeDomain wiring is part of the fabric contract the // runtime owns, so it must declare any ComputeDomain/ResourceClaim it needs. if customRuntime == "" && variant == variantNVLS { - if err := applyNCCLComputeDomain(ctx.Ctx, dynamicClient, config.Namespace); err != nil { + if err = applyNCCLComputeDomain(ctx.Ctx, dynamicClient, config.Namespace); err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to apply ComputeDomain", err) } - if err := waitForIMEXClaimTemplate(ctx.Ctx, dynamicClient, config.Namespace); err != nil { + if err = waitForIMEXClaimTemplate(ctx.Ctx, dynamicClient, config.Namespace); err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "IMEX ResourceClaimTemplate not ready", err) } } @@ -968,7 +989,7 @@ func applyNCCLResources(ctx *validators.Context, dynamicClient dynamic.Interface // rejected with "TrainingRuntime not found". applyTrainJobWithRetry retries // on exactly that denial until the webhook cache catches up. trainjobPath := filepath.Join("testdata", "trainjob.yaml") - if err := applyTrainJobWithRetry(ctx.Ctx, dynamicClient, config.Namespace, trainjobPath, templateData); err != nil { + if err = applyTrainJobWithRetry(ctx.Ctx, dynamicClient, config.Namespace, trainjobPath, templateData); err != nil { return err } slog.Info("Applied TrainJob") @@ -1107,7 +1128,10 @@ func applyNCCLComputeDomain(ctx context.Context, dynamicClient dynamic.Interface // AlreadyExists: fetch the current resourceVersion and Update in place. // Required because Update rejects an empty resourceVersion to prevent - // lost updates. + // lost updates. Adopting an existing ComputeDomain here is intentional: + // it's how a stale one left by a prior run under this same per-run + // namespace (e.g. a retry reusing AICR_RUN_ID after a hard kill before + // cleanup ran) gets reclaimed instead of failing with AlreadyExists. existing, err := client.Get(applyCtx, ncclComputeDomainName, metav1.GetOptions{}) if err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to get existing ComputeDomain", err) @@ -1300,12 +1324,12 @@ func renderYAMLTemplate(content string, data map[string]string) (*unstructured.U return obj, nil } -// createUnstructured creates a namespaced resource from an unstructured object with a timeout. +// createUnstructured creates a namespaced resource from an unstructured object +// with a timeout. func createUnstructured(ctx context.Context, dynamicClient dynamic.Interface, gvr schema.GroupVersionResource, namespace string, obj *unstructured.Unstructured) error { applyCtx, cancel := context.WithTimeout(ctx, defaults.DiagnosticTimeout) defer cancel() - _, err := dynamicClient.Resource(gvr).Namespace(namespace).Create(applyCtx, obj, metav1.CreateOptions{}) - if err != nil { + if _, err := dynamicClient.Resource(gvr).Namespace(namespace).Create(applyCtx, obj, metav1.CreateOptions{}); err != nil { return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, "failed to create resource", err) } return nil @@ -1451,7 +1475,7 @@ func applyNCCLWorkerScheduling(obj *unstructured.Unstructured, nodeSelector map[ tolList := make([]any, 0, len(tolerations)) for _, t := range tolerations { tolMap := map[string]any{ - "operator": string(t.Operator), + keyOperator: string(t.Operator), } if t.Key != "" { tolMap["key"] = t.Key @@ -1479,7 +1503,7 @@ func applyNCCLWorkerScheduling(obj *unstructured.Unstructured, nodeSelector map[ return unstructured.SetNestedSlice(obj.Object, replicatedJobs, "spec", "template", "spec", "replicatedJobs") } -// nestedMap navigates a chain of string keys through nested map[string]interface{} values. +// nestedMap navigates a chain of string keys through nested map[string]any values. // Returns the target map and true if found, nil and false otherwise. func nestedMap(m map[string]any, keys ...string) (map[string]any, bool) { current := m @@ -1505,7 +1529,7 @@ func waitForLauncherPodAndGetLogs(ctx *validators.Context, podHelper *helper.Pod launcherPod, err := waitForPodByLabelSelector( ctx.Ctx, ctx.Clientset, - ctx.Namespace, + podHelper.Namespace, fmt.Sprintf("jobset.sigs.k8s.io/jobset-name=%s,jobset.sigs.k8s.io/replicatedjob-name=launcher", ncclTrainJobName), defaults.NCCLLauncherPodTimeout, ) @@ -1557,14 +1581,14 @@ func waitForLauncherPodAndGetLogs(ctx *validators.Context, podHelper *helper.Pod // the pod object and survives the container GC that GetPodLogs loses to. // Either way the fetchNote reason is preserved in the payload. if launcherLogs == "" { - if term := launcherTerminationTail(ctx.Ctx, ctx.Clientset, ctx.Namespace, launcherPod.Name); term != "" { + if term := launcherTerminationTail(ctx.Ctx, ctx.Clientset, podHelper.Namespace, launcherPod.Name); term != "" { launcherLogs = fmt.Sprintf("<%s; container termination-message tail follows>\n%s", fetchNote, tailLines(term, maxDiagLogLines)) } else { launcherLogs = fmt.Sprintf("<%s; no termination message captured>", fetchNote) } } - workerDiag := collectNCCLWorkerDiagnostics(ctx.Ctx, ctx.Clientset, ctx.Namespace) + workerDiag := collectNCCLWorkerDiagnostics(ctx.Ctx, ctx.Clientset, podHelper.Namespace) // Surface the diagnostics via slog, not just the return value: every // caller on this error path (runNCCLTrainJob, validateNcclAllReduceBw, @@ -1600,7 +1624,7 @@ func waitForLauncherPodAndGetLogs(ctx *validators.Context, podHelper *helper.Pod // rotation-proof source of truth while the streamed log remains available for // transport verification and diagnostics. Empty for launchers that don't write // results there (other platforms), leaving behavior unchanged. - term := launcherTerminationTail(ctx.Ctx, ctx.Clientset, ctx.Namespace, launcherPod.Name) + term := launcherTerminationTail(ctx.Ctx, ctx.Clientset, podHelper.Namespace, launcherPod.Name) if term != "" { slog.Info("Appending launcher termination message (rotation-proof results)", "termBytes", len(term)) } @@ -2075,66 +2099,53 @@ func verifyTransportFromLogs(logs string, variant ncclVariant) error { } } -// cleanupNCCLResources removes the trainjob, runtime, and (if present) the -// ComputeDomain CR using the dynamic client. Deleting the ComputeDomain -// cascades to its auto-generated ResourceClaimTemplate via the DRA driver; -// NotFound on the ComputeDomain is expected for the default/NET variants -// and is logged at debug rather than error. -func cleanupNCCLResources(dynamicClient dynamic.Interface, namespace string) { - slog.Info("Cleaning up NCCL test resources...") +// cleanupNCCLResources deletes the per-run benchmark namespace, cascading +// away the trainjob, runtime, and (if present) the ComputeDomain and RoCE +// ResourceClaimTemplate CRs this run created in it — mirroring +// cleanupInferenceWorkload's pattern for the sibling inference-perf check. +// Since runNCCLTrainJob gives every run its own namespace (see +// ncclWorkloadNamespacePrefix), there is no shared state to pin deletes +// against: nothing else ever lives in this namespace. +// +// Namespaces().Delete only starts asynchronous deletion — it returns as soon +// as the delete is accepted, not once the namespace (and the +// ComputeDomain/ResourceClaimTemplate/TrainJob finalizers cascading through +// it) is actually gone. Waiting here for the deletion to finish, via the +// same waitForNamespaceGone helper ensureNamespace already uses on the +// create side for exactly this reason (see its doc comment), means a +// successful benchmark can't report clean teardown while resources are +// still leaking. +// +// Unlike cleanupInferenceWorkload, a delete failure here is returned rather +// than only logged, so foldCleanupError can still fail an otherwise-passing +// check on it. NotFound is tolerated (nothing to clean up). +func cleanupNCCLResources(clientset kubernetes.Interface, namespace string) error { + slog.Info("Cleaning up NCCL test resources...", "namespace", namespace) - cleanupCtx, cancel := context.WithTimeout(context.Background(), defaults.DiagnosticTimeout) + deleteCtx, cancel := context.WithTimeout(context.Background(), defaults.K8sCleanupTimeout) defer cancel() - // Delete trainjob. NotFound is expected and logged at debug: this runs as a - // deferred cleanup registered before the apply, so an early/partial-apply - // failure (or the install-trainer path, where deleteTrainer may already have - // cascade-removed the CRs) legitimately leaves no TrainJob to delete. - err := dynamicClient.Resource(trainJobGVR).Namespace(namespace).Delete(cleanupCtx, ncclTrainJobName, metav1.DeleteOptions{}) - switch { - case err == nil: - slog.Info("Deleted TrainJob") - case apierrors.IsNotFound(err): - slog.Debug("TrainJob not present, skipping", "name", ncclTrainJobName) - default: - slog.Warn("failed to delete TrainJob", "error", err) - } - - // Delete runtime. NotFound is expected and logged at debug (see TrainJob above). - err = dynamicClient.Resource(trainingRuntimeGVR).Namespace(namespace).Delete(cleanupCtx, ncclTrainingRuntimeName, metav1.DeleteOptions{}) - switch { - case err == nil: - slog.Info("Deleted TrainingRuntime") - case apierrors.IsNotFound(err): - slog.Debug("TrainingRuntime not present, skipping", "name", ncclTrainingRuntimeName) - default: - slog.Warn("failed to delete TrainingRuntime", "error", err) + nsClient := clientset.CoreV1().Namespaces() + err := nsClient.Delete(deleteCtx, namespace, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + slog.Info("NCCL benchmark namespace already gone", "namespace", namespace) + return nil + } + return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, + fmt.Sprintf("failed to delete NCCL benchmark namespace %q", namespace), err) } - // Delete ComputeDomain if this was the NVLS variant. NotFound is the - // expected path for default/NET and is ignored here; other errors bubble - // up as a warning because the RCT and IMEX daemons otherwise leak. - err = dynamicClient.Resource(computeDomainGVR).Namespace(namespace).Delete(cleanupCtx, ncclComputeDomainName, metav1.DeleteOptions{}) - switch { - case err == nil: - slog.Info("Deleted ComputeDomain") - case apierrors.IsNotFound(err): - slog.Debug("ComputeDomain not present (non-NVLS variant), skipping", "name", ncclComputeDomainName) - default: - slog.Warn("failed to delete ComputeDomain", "error", err, "name", ncclComputeDomainName) + // Same bound as ensureNamespace's wait on the create side (see + // defaults.InferenceNamespaceTerminationWait doc comment) — this cascade + // is the same finalizer chain, just observed from the delete side. + waitCtx, waitCancel := context.WithTimeout(context.Background(), defaults.InferenceNamespaceTerminationWait) + defer waitCancel() + if err := waitForNamespaceGone(waitCtx, nsClient, namespace); err != nil { + return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, + fmt.Sprintf("NCCL benchmark namespace %q did not finish terminating", namespace), err) } - // Delete the RoCE ResourceClaimTemplate (RoCE NET variant only). The - // validator namespace is persistent and reused across runs, so leaving it - // behind makes the next RoCE run fail with AlreadyExists when - // applyNCCLResources re-creates it. NotFound is expected for EFA/NVLS runs. - err = dynamicClient.Resource(resourceClaimTemplateGVR).Namespace(namespace).Delete(cleanupCtx, ncclRoceClaimName, metav1.DeleteOptions{}) - switch { - case err == nil: - slog.Info("Deleted RoCE ResourceClaimTemplate", "name", ncclRoceClaimName) - case apierrors.IsNotFound(err): - slog.Debug("RoCE ResourceClaimTemplate not present (non-RoCE variant), skipping", "name", ncclRoceClaimName) - default: - slog.Warn("failed to delete RoCE ResourceClaimTemplate", "error", err, "name", ncclRoceClaimName) - } + slog.Info("Deleted NCCL benchmark namespace", "namespace", namespace) + return nil } diff --git a/validators/performance/nccl_all_reduce_bw_constraint_test.go b/validators/performance/nccl_all_reduce_bw_constraint_test.go index 06d4109a7..41ebcc42c 100644 --- a/validators/performance/nccl_all_reduce_bw_constraint_test.go +++ b/validators/performance/nccl_all_reduce_bw_constraint_test.go @@ -21,10 +21,13 @@ import ( "strings" "testing" + "github.com/NVIDIA/aicr/validators" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) func TestEmitDiagnosticBlock(t *testing.T) { @@ -243,3 +246,35 @@ func TestCollectNCCLWorkerDiagnostics(t *testing.T) { }) } } + +// TestRunNCCLTrainJob_TrainerInstallFailureCleansUpNamespace is the regression +// guard for the namespace-cleanup defer's registration point: it must be +// registered right after ensureNamespace succeeds, not after +// ensureTrainerInstalled, or a Trainer-install failure returns before the +// defer is ever registered and leaks the per-run namespace forever. +func TestRunNCCLTrainJob_TrainerInstallFailureCleansUpNamespace(t *testing.T) { + dynamicClient := newTrainerFakeClient(completeTrainerInstall()...) + dynamicClient.PrependReactor("get", resourceCRDs, func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("apiserver is down") + }) + + clientset := fake.NewClientset() + vctx := &validators.Context{ + Ctx: context.Background(), + Clientset: clientset, + DynamicClient: dynamicClient, + } + gpuConfig := &gpuConfiguration{WorkerCount: 2, GPUCountPerNode: 4, TotalGPUCount: 8} + + _, err := runNCCLTrainJob(vctx, gpuConfig, "", "", variantDefault, fabricEFA, "") + if err == nil { + t.Fatal("expected an error from the failed Trainer install probe, got nil") + } + if gpuConfig.Namespace == "" { + t.Fatal("gpuConfig.Namespace was never set; ensureNamespace apparently wasn't reached") + } + + if _, getErr := clientset.CoreV1().Namespaces().Get(context.Background(), gpuConfig.Namespace, metav1.GetOptions{}); !apierrors.IsNotFound(getErr) { + t.Errorf("namespace %q was not cleaned up after Trainer install failure: get err = %v", gpuConfig.Namespace, getErr) + } +} diff --git a/validators/performance/nccl_benchmark_profile_test.go b/validators/performance/nccl_benchmark_profile_test.go index bedc0f876..d9fb4e7e9 100644 --- a/validators/performance/nccl_benchmark_profile_test.go +++ b/validators/performance/nccl_benchmark_profile_test.go @@ -152,6 +152,7 @@ func TestNCCLCombinationSupported(t *testing.T) { {"default H100 EKS", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorH100, recipe.CriteriaServiceEKS), true}, {"default H200 EKS", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorH200, recipe.CriteriaServiceEKS), true}, {"default H100 GKE", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorH100, recipe.CriteriaServiceGKE), true}, + {"default GB200 GKE not covered", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceGKE), false}, {"default H100 AKS", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorH100, recipe.CriteriaServiceAKS), true}, {"default B200 any", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorB200, recipe.CriteriaServiceAny), true}, {"default GB200 EKS not covered", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), false}, @@ -159,6 +160,7 @@ func TestNCCLCombinationSupported(t *testing.T) { {"NET GB200 OKE not covered", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), false}, {"NVLS GB200 EKS", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), true}, {"NVLS GB200 OKE", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), true}, + {"NVLS GB200 GKE", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceGKE), true}, {"unknown service", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, "custom-svc"), false}, {"unknown accelerator", variantNET, fabricEFA, target("gb300", recipe.CriteriaServiceEKS), false}, // RoCE NET is service-keyed and accelerator-agnostic. @@ -187,6 +189,7 @@ func TestKnownBenchmarkProfiles(t *testing.T) { "b200/any", "gb200/any", "gb200/eks", + "gb200/gke", "gb200/oke", "h100/aks", "h100/eks", diff --git a/validators/performance/nccl_roce_apply_test.go b/validators/performance/nccl_roce_apply_test.go index 099996098..9c84f9f73 100644 --- a/validators/performance/nccl_roce_apply_test.go +++ b/validators/performance/nccl_roce_apply_test.go @@ -16,10 +16,15 @@ package main import ( "context" + stderrors "errors" "path/filepath" + "sync/atomic" "testing" + "time" + aicrErrors "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/validators" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -27,11 +32,13 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) -// ncclGVRListKinds maps every GVR cleanupNCCLResources / applyNCCLResources -// touch to a fake list kind, so the dynamic fake client can serve Create/Get/ -// Update/Delete for these CRDs without a real REST mapper. +// ncclGVRListKinds maps every GVR applyNCCLResources touches to a fake list +// kind, so the dynamic fake client can serve Create/Get/Update for these CRDs +// without a real REST mapper. var ncclGVRListKinds = map[schema.GroupVersionResource]string{ resourceClaimTemplateGVR: "ResourceClaimTemplateList", trainJobGVR: "TrainJobList", @@ -115,47 +122,139 @@ func TestCreateOrUpdateFromTemplate_RoCEClaimIdempotent(t *testing.T) { } // TestCleanupNCCLResources_ToleratesMissing verifies the deferred cleanup is -// safe to run after an early/partial-apply failure: with no resources present, -// every Delete hits NotFound and the function must complete without panicking. +// safe to run after an early/partial-apply failure: with no namespace ever +// created, deleting it must be treated as success (NotFound-tolerant), not +// an error. func TestCleanupNCCLResources_ToleratesMissing(t *testing.T) { - const ns = "aicr-validation" - // No objects seeded — every Delete returns NotFound. - fakeClient := newFakeDynamicClient() - cleanupNCCLResources(fakeClient, ns) + const ns = "aicr-nccl-perf-deadbeef" + fakeClient := fake.NewClientset() + if err := cleanupNCCLResources(fakeClient, ns); err != nil { + t.Fatalf("cleanup of a namespace that was never created should not error, got: %v", err) + } +} - // Cleanup must tolerate the absence, not resurrect anything. - _, err := fakeClient.Resource(resourceClaimTemplateGVR).Namespace(ns). - Get(context.Background(), ncclRoceClaimName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - t.Fatalf("claim should remain absent after cleanup of empty namespace, got err=%v", err) +// TestCleanupNCCLResources_DeletesNamespace verifies the happy path: the +// per-run namespace this run created is deleted, cascading away everything +// created in it (TrainJob, TrainingRuntime, ComputeDomain, RoCE claim) via +// ordinary Kubernetes namespace garbage collection — no per-resource +// tracking required, since nothing else ever shares this namespace. +func TestCleanupNCCLResources_DeletesNamespace(t *testing.T) { + const ns = "aicr-nccl-perf-deadbeef" + fakeClient := fake.NewClientset(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}) + + if err := cleanupNCCLResources(fakeClient, ns); err != nil { + t.Fatalf("cleanup should not error, got: %v", err) + } + + if _, err := fakeClient.CoreV1().Namespaces().Get(context.Background(), ns, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("namespace should be deleted after cleanup, got err=%v", err) } } -// TestCleanupNCCLResources_DeletesRoCEClaim verifies the happy path: a RoCE -// claim left in the persistent namespace is deleted by cleanup, so the next run -// does not collide with it. -func TestCleanupNCCLResources_DeletesRoCEClaim(t *testing.T) { - const ns = "aicr-validation" +// TestCleanupNCCLResources_ReturnsErrorOnDeleteFailure verifies a delete +// failure that is not NotFound (e.g. a transient apiserver error) is +// returned to the caller instead of only logged, so foldCleanupError can +// still fail an otherwise-passing check on it. +func TestCleanupNCCLResources_ReturnsErrorOnDeleteFailure(t *testing.T) { + const ns = "aicr-nccl-perf-deadbeef" + fakeClient := fake.NewClientset(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}) + fakeClient.PrependReactor("delete", "namespaces", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("apiserver is down") + }) - claim := &unstructured.Unstructured{} - claim.SetAPIVersion("resource.k8s.io/v1") - claim.SetKind("ResourceClaimTemplate") - claim.SetName(ncclRoceClaimName) - claim.SetNamespace(ns) + err := cleanupNCCLResources(fakeClient, ns) + if err == nil { + t.Fatal("expected an error from a non-NotFound namespace delete failure, got nil") + } + if !stderrors.Is(err, aicrErrors.New(aicrErrors.ErrCodeInternal, "")) { + t.Errorf("got %v, want an ErrCodeInternal-wrapped delete failure", err) + } +} + +// TestCleanupNCCLResources_WaitsForFinalizerHeldNamespace is the regression +// guard for the wait-for-termination fix: a namespace whose first delete +// only stamps a DeletionTimestamp (finalizers still cascading, as the +// ComputeDomain/ResourceClaimTemplate CRs this namespace can hold commonly +// do) must not be reported as cleaned up until it actually disappears. +// Before this fix, cleanupNCCLResources returned nil as soon as the first +// Delete call was accepted, even though the namespace -- and everything +// still finalizing inside it -- was still there. +func TestCleanupNCCLResources_WaitsForFinalizerHeldNamespace(t *testing.T) { + const ns = "aicr-nccl-perf-deadbeef" + const holdFinalizer = 200 * time.Millisecond + fakeClient := fake.NewClientset(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}) + nsGVR := v1.SchemeGroupVersion.WithResource("namespaces") + + var deleteCalls int32 + fakeClient.PrependReactor("delete", "namespaces", func(k8stesting.Action) (bool, runtime.Object, error) { + if atomic.AddInt32(&deleteCalls, 1) == 1 { + // First delete: simulate a still-cascading finalizer by stamping + // DeletionTimestamp via the tracker directly instead of actually + // removing the object, then tell the caller the delete request + // was accepted (handled=true, err=nil) -- matching what a real + // apiserver does for a namespace with finalizers. + now := metav1.Now() + held := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: ns, + Finalizers: []string{"kubernetes"}, + DeletionTimestamp: &now, + }} + if err := fakeClient.Tracker().Update(nsGVR, held, ""); err != nil { + return true, nil, err + } + return true, nil, nil + } + // Second delete (fired by the goroutine below once the "finalizer" + // clears): let the default reactor perform the real delete, which + // also emits the watch.Deleted event waitForNamespaceGone is + // blocked on. + return false, nil, nil + }) - fakeClient := newFakeDynamicClient(claim) + go func() { + time.Sleep(holdFinalizer) + _ = fakeClient.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) + }() + + start := time.Now() + if err := cleanupNCCLResources(fakeClient, ns); err != nil { + t.Fatalf("cleanup should succeed once the finalizer clears, got: %v", err) + } + elapsed := time.Since(start) - // Sanity: the claim exists before cleanup. - if _, err := fakeClient.Resource(resourceClaimTemplateGVR).Namespace(ns). - Get(context.Background(), ncclRoceClaimName, metav1.GetOptions{}); err != nil { - t.Fatalf("precondition: claim should exist before cleanup: %v", err) + if got := atomic.LoadInt32(&deleteCalls); got < 2 { + t.Fatalf("expected cleanup to observe the namespace still present and wait for a second delete, got %d delete call(s)", got) } + if elapsed < holdFinalizer { + t.Errorf("cleanup returned after %v, want it to have blocked at least %v for the finalizer to clear (it returned before the namespace actually disappeared)", elapsed, holdFinalizer) + } +} - cleanupNCCLResources(fakeClient, ns) +// TestWaitForNamespaceGone_TimesOutWhenNeverDeleted is the regression guard +// for the "wait boundedly, then fail" half of the fix: if a namespace's +// finalizers never clear within the caller's deadline, the wait must return +// a timeout error rather than either hang indefinitely or (the pre-fix +// cleanupNCCLResources behavior this mirrors) silently report success while +// the namespace is still there. Calls waitForNamespaceGone directly with a +// short local context so the test doesn't have to wait out +// cleanupNCCLResources's real 5-minute production timeout. +func TestWaitForNamespaceGone_TimesOutWhenNeverDeleted(t *testing.T) { + const ns = "aicr-nccl-perf-deadbeef" + now := metav1.Now() + fakeClient := fake.NewClientset(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: ns, + Finalizers: []string{"kubernetes"}, + DeletionTimestamp: &now, + }}) - _, err := fakeClient.Resource(resourceClaimTemplateGVR).Namespace(ns). - Get(context.Background(), ncclRoceClaimName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - t.Fatalf("claim should be deleted after cleanup, got err=%v", err) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + err := waitForNamespaceGone(ctx, fakeClient.CoreV1().Namespaces(), ns) + if err == nil { + t.Fatal("expected a timeout error waiting for a namespace that never finishes terminating, got nil") + } + if !stderrors.Is(err, aicrErrors.New(aicrErrors.ErrCodeTimeout, "")) { + t.Errorf("got %v, want an ErrCodeTimeout-wrapped wait failure", err) } } diff --git a/validators/performance/nccl_test.go b/validators/performance/nccl_test.go index 5db706ac9..bea1050ea 100644 --- a/validators/performance/nccl_test.go +++ b/validators/performance/nccl_test.go @@ -960,6 +960,12 @@ func TestSupportedNCCLCombinations_Variants(t *testing.T) { service: recipe.CriteriaServiceOKE, want: []recipe.CriteriaAcceleratorType{recipe.CriteriaAcceleratorGB200}, }, + { + name: "NVLS GKE GB200", + variant: variantNVLS, + service: recipe.CriteriaServiceGKE, + want: []recipe.CriteriaAcceleratorType{recipe.CriteriaAcceleratorGB200}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -983,6 +989,10 @@ func TestSupportedNCCLCombinations_Variants(t *testing.T) { if accels := supportedNCCLCombinations[variantDefault][recipe.CriteriaServiceAny]; len(accels) != 2 { t.Errorf("variantDefault Any count = %d, want 2 (B200, GB200)", len(accels)) } + wantGKE := []recipe.CriteriaAcceleratorType{recipe.CriteriaAcceleratorH100} + if accels := supportedNCCLCombinations[variantDefault][recipe.CriteriaServiceGKE]; !reflect.DeepEqual(accels, wantGKE) { + t.Errorf("variantDefault GKE = %v, want %v", accels, wantGKE) + } // AKS ND-series H100 runs the default variant over NCCL's built-in // IB/verbs transport (testdata/h100/aks/runtime.yaml). wantAKS := []recipe.CriteriaAcceleratorType{recipe.CriteriaAcceleratorH100} diff --git a/validators/performance/testdata/gb200/gke/runtime-nvls.yaml b/validators/performance/testdata/gb200/gke/runtime-nvls.yaml new file mode 100644 index 000000000..dc994632d --- /dev/null +++ b/validators/performance/testdata/gb200/gke/runtime-nvls.yaml @@ -0,0 +1,223 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NCCL all-reduce TrainingRuntime for GB200 on GKE, NVLS-transport variant. +# +# Validates Multi-Node NVLink (MNNVL / NVLS) across the A4X nodes' IMEX domain. +# +# IMEX wiring is handled by the validator itself: before the TrainJob is +# applied, applyNCCLComputeDomain() creates a resource.nvidia.com/v1beta1 +# ComputeDomain CR in this namespace. The NVIDIA DRA driver reconciles the CR +# into a ResourceClaimTemplate named "nccl-all-reduce-imex" (matching +# ncclIMEXClaimTemplateName in the validator Go code), and the worker pods +# below reference that template via resourceClaims. At pod admission the +# kubelet therefore mounts /dev/nvidia-caps-imex-channels into each worker, +# which is what MNNVL needs to open NVLS multicast groups across nodes. +# +# Cluster prerequisite: nvidia-dra-driver-gpu must be installed with the +# resource.nvidia.com/v1beta1 ComputeDomain CRD present. Without it the +# validator's CR creation fails up front with a clear NotFound error. + +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainingRuntime +metadata: + name: nccl-all-reduce-runtime + namespace: ${NAMESPACE} + labels: + trainer.kubeflow.org/framework: mpi +spec: + mlPolicy: + mpi: + mpiImplementation: OpenMPI + numProcPerNode: ${GPU_COUNT_PER_NODE} + runLauncherAsNode: false + sshAuthMountPath: /tmp/mpi-keys + template: + spec: + network: + enableDNSHostnames: true + publishNotReadyAddresses: true + replicatedJobs: + - name: launcher + replicas: 1 + template: + spec: + template: + spec: + tolerations: + - operator: Exists + initContainers: + - name: fix-ssh-perms + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: + - /bin/sh + - -c + - | + set -x + mkdir -p /root/.ssh + cp /tmp/mpi-keys/id_rsa /root/.ssh/id_rsa + cp /tmp/mpi-keys/authorized_keys /root/.ssh/authorized_keys + chmod 700 /root/.ssh + chmod 600 /root/.ssh/id_rsa /root/.ssh/authorized_keys + volumeMounts: + - name: mpi-ssh-auth + mountPath: /tmp/mpi-keys + readOnly: true + - name: ssh-config + mountPath: /root/.ssh + containers: + - name: node + image: nvcr.io/nvidia/pytorch:25.06-py3 + terminationMessagePolicy: FallbackToLogsOnError + env: + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" + command: + - /bin/bash + - -c + - | + set -o pipefail + /usr/local/mpi/bin/mpirun "$@" 2>&1 | tee /tmp/nccl-results.out + rc=${PIPESTATUS[0]} + if [ "$rc" -eq 0 ]; then + pat='^[[:space:]]*[0-9]+[[:space:]]+[0-9]+[[:space:]]+[[:alpha:]]+|Avg bus bandwidth' + grep -E "$pat" /tmp/nccl-results.out 2>/dev/null | tail -c 3900 > /dev/termination-log 2>/dev/null || true + fi + exit "$rc" + - bash + args: + - -np + - "${GPU_COUNT}" + - --allow-run-as-root + - --mca + - plm_rsh_args + - -o StrictHostKeyChecking=no -o ConnectTimeout=10 -o ConnectionAttempts=300 + - --mca + - oob_tcp_if_include + - eth0 + - --mca + - btl_tcp_if_include + - eth0 + - -x + - UCX_NET_DEVICES=eth0 + - -x + - LD_LIBRARY_PATH=/home/kubernetes/bin/nvidia/lib64:/home/kubernetes/bin/gib + # INFO emits the "NVLS multicast support is available" + # hardware-capability banner and, when NCCL actually builds + # an NVLS communicator, a "NVLS comm 0x" init line. + # verifyTransportFromLogs greps the latter as proof of use + # (per-channel "[send] via NVLS" lines were dropped in + # NCCL 2.27). + - -x + - NCCL_DEBUG=INFO + - -x + - NCCL_NET=gIB + # Force MNNVL/NVLS on. If the IMEX domain isn't wired up, + # NCCL initialization will surface the error loudly. + - -x + - NCCL_NVLS_ENABLE=1 + - -x + - NCCL_SOCKET_IFNAME=eth0 + - /usr/local/bin/${TEST_TYPE}_mpi + - -b + - ${MIN_MESSAGE_SIZE} + - -e + - ${MAX_MESSAGE_SIZE} + - -f + - "2" + - -g + - "1" + resources: + limits: + cpu: "2" + memory: 128Mi + volumeMounts: + - name: ssh-config + mountPath: /root/.ssh + volumes: + - name: ssh-config + emptyDir: {} + - name: node + template: + spec: + template: + metadata: + annotations: + networking.gke.io/default-interface: eth0 + networking.gke.io/interfaces: '[{"interfaceName":"eth0","network":"default"},{"interfaceName":"eth1","network":"gvnic-1"},{"interfaceName":"eth2","network":"rdma-0"},{"interfaceName":"eth3","network":"rdma-1"},{"interfaceName":"eth4","network":"rdma-2"},{"interfaceName":"eth5","network":"rdma-3"}]' + spec: + # IMEX channel claim: the DRA driver auto-generated this + # ResourceClaimTemplate (name matches ncclIMEXClaimTemplateName + # in the validator Go code) when the ComputeDomain CR was + # reconciled. Without this block, /dev/nvidia-caps-imex-channels + # would not be mounted and NCCL would fail with + # "Cuda failure 800 'operation not permitted'" at MNNVL init. + resourceClaims: + - name: imex-channel + resourceClaimTemplateName: nccl-all-reduce-imex + containers: + - name: node + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: ["sh", "-c"] + args: + - | + set -x && + apt-get update && + apt-get install -y --no-install-recommends openssh-server && + mkdir -p /var/run/sshd && + chmod 0755 /var/run/sshd && + mkdir -p /root/.ssh && + chmod 700 /root/.ssh && + cp /tmp/mpi-keys/* /root/.ssh/ && + chmod 600 /root/.ssh/id_rsa && + chmod 644 /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && + export LD_LIBRARY_PATH=/home/kubernetes/bin/nvidia/lib64:/home/kubernetes/bin/gib:${LD_LIBRARY_PATH} && + export NCCL_NET=gIB && + if [ -f /home/kubernetes/bin/gib/scripts/set_nccl_env.sh ]; then . /home/kubernetes/bin/gib/scripts/set_nccl_env.sh; fi && + env | grep -E '^NCCL_|^CUDA_|^LD_LIBRARY_PATH' > /root/.ssh/environment && + echo "PermitUserEnvironment yes" >> /etc/ssh/sshd_config && + /usr/sbin/sshd -De + resources: + claims: + - name: imex-channel + limits: + nvidia.com/gpu: ${GPU_COUNT_PER_NODE} + requests: + nvidia.com/gpu: ${GPU_COUNT_PER_NODE} + securityContext: + capabilities: + add: ["IPC_LOCK"] + volumeMounts: + - name: dshm + mountPath: /dev/shm + - name: nvidia-host + mountPath: /home/kubernetes/bin/nvidia + readOnly: true + - name: gib-host + mountPath: /home/kubernetes/bin/gib + readOnly: true + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: nvidia-host + hostPath: + path: /home/kubernetes/bin/nvidia + - name: gib-host + hostPath: + path: /home/kubernetes/bin/gib + successPolicy: + operator: All + targetReplicatedJobs: + - launcher diff --git a/validators/performance/trainer_ensure_test.go b/validators/performance/trainer_ensure_test.go index 294976afd..31f308f6f 100644 --- a/validators/performance/trainer_ensure_test.go +++ b/validators/performance/trainer_ensure_test.go @@ -212,7 +212,7 @@ func TestFoldCleanupError(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := foldCleanupError(tt.bench, tt.cleanup) + got := foldCleanupError(tt.bench, tt.cleanup, "NCCL benchmark succeeded but Kubeflow Trainer cleanup failed") if tt.want == nil { if got != nil { t.Fatalf("got %v, want nil", got) @@ -231,7 +231,7 @@ func TestFoldCleanupError(t *testing.T) { func TestFoldCleanupError_PreservesCleanupCode(t *testing.T) { cleanupErr := aicrErrors.New(aicrErrors.ErrCodeUnavailable, "apiserver is down") - got := foldCleanupError(nil, cleanupErr) + got := foldCleanupError(nil, cleanupErr, "fallback message") if !stderrors.Is(got, aicrErrors.New(aicrErrors.ErrCodeUnavailable, "")) { t.Errorf("cleanup error code was flattened: %v", got) } diff --git a/validators/performance/trainer_lifecycle.go b/validators/performance/trainer_lifecycle.go index d51ec0b5d..88a51cced 100644 --- a/validators/performance/trainer_lifecycle.go +++ b/validators/performance/trainer_lifecycle.go @@ -32,8 +32,10 @@ import ( "strings" "time" + "github.com/NVIDIA/aicr/pkg/component" "github.com/NVIDIA/aicr/pkg/defaults" aicrErrors "github.com/NVIDIA/aicr/pkg/errors" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -67,6 +69,11 @@ const ( // trainerControllerDeployment is the Deployment name for the Trainer controller-manager. trainerControllerDeployment = "kubeflow-trainer-controller-manager" + // jobSetControllerDeployment is the JobSet controller-manager Deployment name + // emitted by this package's kustomize overlay (see jobSetNameLabel for why + // the Helm chart's release-derived name doesn't apply here). + jobSetControllerDeployment = "jobset-controller-manager" + // trainerControllerService is the Service fronting the controller-manager's // webhook port. Without it the admission webhooks have no endpoints and every // TrainJob create is rejected. @@ -159,6 +166,57 @@ const ( jobSetPromotedImageRepo = "registry.k8s.io/jobset/jobset" ) +// controllerTolerateAll lets a Trainer/JobSet controller-manager Deployment +// schedule on any node pool, regardless of taints. Built through +// component.TolerationsToPodSpec, the same converter used for Helm-values +// toleration overrides, so there is one canonical place that knows the +// toleration-to-map shape. +var controllerTolerateAll = tolerationsToAnySlice( + component.TolerationsToPodSpec([]corev1.Toleration{{Operator: corev1.TolerationOpExists}}), +) + +// tolerationsToAnySlice widens []map[string]any to []any: unstructured pod +// specs (podSpec["tolerations"]) must hold []any, not []map[string]any, to +// match how NestedSlice reads and how JSON round-tripping serializes it. +func tolerationsToAnySlice(tolerations []map[string]any) []any { + result := make([]any, len(tolerations)) + for i, t := range tolerations { + result[i] = t + } + return result +} + +// applyControllerTolerations stamps controllerTolerateAll onto the Trainer and +// JobSet controller-manager Deployments' pod template, unless one already +// declares tolerations. Scoped to those two names so an unrelated Deployment +// in the manifest set never gets a blanket toleration it didn't ask for. +func applyControllerTolerations(obj *unstructured.Unstructured) error { + if obj.GroupVersionKind().Kind != "Deployment" { + return nil + } + switch obj.GetName() { + case trainerControllerDeployment, jobSetControllerDeployment: + default: + return nil + } + + if existing, found, err := unstructured.NestedSlice(obj.Object, "spec", "template", "spec", "tolerations"); err != nil { + return aicrErrors.Wrap(aicrErrors.ErrCodeInternal, + fmt.Sprintf("failed to read tolerations from Deployment %q", obj.GetName()), err) + } else if found && len(existing) > 0 { + return nil + } + + podSpec, found := nestedMap(obj.Object, "spec", "template", "spec") + if !found { + return aicrErrors.New(aicrErrors.ErrCodeInternal, + fmt.Sprintf("pod spec not found in Deployment %q", obj.GetName())) + } + podSpec["tolerations"] = controllerTolerateAll + slog.Info("Applying blanket toleration to controller Deployment", "name", obj.GetName()) + return nil +} + // GVRs for the objects the Trainer lifecycle probes and waits on. var ( trainerCRDGVR = schema.GroupVersionResource{ @@ -732,15 +790,18 @@ func waitForDeclaredTrainer(ctx context.Context, dynamicClient dynamic.Interface } // foldCleanupError decides the check's verdict when teardown fails. A cleanup -// failure leaks cluster-scoped CRDs, RBAC, and webhook configurations that would -// silently poison the next run, so it fails an otherwise-passing check — but it -// never masks a real benchmark failure, which is always the more useful signal. -func foldCleanupError(benchErr, cleanupErr error) error { +// failure leaks cluster-scoped CRDs, RBAC, and webhook configurations (or, for +// the NCCL resource cleanup caller, poisons the next run's fixed-named +// resources) that would silently break a later run, so it fails an +// otherwise-passing check — but it never masks a real benchmark failure, +// which is always the more useful signal. fallbackMsg is used only when +// cleanupErr isn't already a *StructuredError (PropagateOrWrap preserves an +// existing one's own message/code as-is). +func foldCleanupError(benchErr, cleanupErr error, fallbackMsg string) error { if cleanupErr == nil || benchErr != nil { return benchErr } - return aicrErrors.PropagateOrWrap(cleanupErr, aicrErrors.ErrCodeInternal, - "NCCL benchmark succeeded but Kubeflow Trainer cleanup failed") + return aicrErrors.PropagateOrWrap(cleanupErr, aicrErrors.ErrCodeInternal, fallbackMsg) } // installTrainer downloads the Kubeflow Trainer v2.2.0 archive from GitHub, builds the @@ -807,6 +868,9 @@ func decodeTrainerObjects(resources []*resource.Resource) ([]*unstructured.Unstr if obj.GroupVersionKind().Kind == "" { continue } + if tolErr := applyControllerTolerations(obj); tolErr != nil { + return nil, tolErr + } objs = append(objs, obj) } return objs, nil diff --git a/validators/performance/trainer_lifecycle_test.go b/validators/performance/trainer_lifecycle_test.go index b03cfe7ef..71a63221c 100644 --- a/validators/performance/trainer_lifecycle_test.go +++ b/validators/performance/trainer_lifecycle_test.go @@ -17,6 +17,8 @@ package main import ( "strings" "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func TestRewriteJobSetStagingImage(t *testing.T) { @@ -76,3 +78,141 @@ func TestRewriteJobSetStagingImage_PreservesTag(t *testing.T) { t.Errorf("got %q, want %q", got, want) } } + +// deploymentFixture returns a minimal unstructured Deployment, optionally with an +// existing tolerations list, for exercising applyControllerTolerations. +func deploymentFixture(name string, existingTolerations []any) *unstructured.Unstructured { + podSpec := map[string]any{ + "containers": []any{ + map[string]any{"name": "manager", "image": "example/manager:latest"}, + }, + } + if existingTolerations != nil { + podSpec["tolerations"] = existingTolerations + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{ + "template": map[string]any{ + "spec": podSpec, + }, + }, + }} +} + +// TestApplyControllerTolerations covers both controller names, the two +// mutation-failure paths, and that an unrelated Deployment is left untouched. +func TestApplyControllerTolerations(t *testing.T) { + tests := []struct { + name string + obj *unstructured.Unstructured + wantErr bool + // wantTolerations is checked only when wantErr is false. nil means "the + // tolerations field must not be present at all" (untouched, not merely + // empty). + wantTolerations []any + }{ + { + name: "Trainer controller Deployment with no tolerations gets tolerate-all", + obj: deploymentFixture(trainerControllerDeployment, nil), + wantTolerations: []any{ + map[string]any{"operator": "Exists"}, + }, + }, + { + name: "JobSet controller Deployment with no tolerations gets tolerate-all", + obj: deploymentFixture(jobSetControllerDeployment, nil), + wantTolerations: []any{ + map[string]any{"operator": "Exists"}, + }, + }, + { + name: "Deployment with existing tolerations is left untouched", + obj: deploymentFixture(trainerControllerDeployment, []any{ + map[string]any{"key": "dedicated", "operator": "Equal", "value": "trainer", "effect": "NoSchedule"}, + }), + wantTolerations: []any{ + map[string]any{"key": "dedicated", "operator": "Equal", "value": "trainer", "effect": "NoSchedule"}, + }, + }, + { + name: "non-controller Deployment is left untouched", + obj: deploymentFixture("some-other-deployment", nil), + wantTolerations: nil, + }, + { + name: "non-Deployment resource is left untouched", + obj: &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Service", + "metadata": map[string]any{"name": trainerControllerDeployment}, + "spec": map[string]any{}, + }}, + wantTolerations: nil, + }, + { + name: "missing pod spec fails closed", + obj: &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": trainerControllerDeployment}, + "spec": map[string]any{}, + }}, + wantErr: true, + }, + { + name: "malformed tolerations field fails closed", + obj: &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": trainerControllerDeployment}, + "spec": map[string]any{ + "template": map[string]any{ + "spec": map[string]any{ + // A string, not a slice: NestedSlice's type assertion fails. + "tolerations": "not-a-slice", + }, + }, + }, + }}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := applyControllerTolerations(tt.obj) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + + got, found, _ := unstructured.NestedSlice(tt.obj.Object, "spec", "template", "spec", "tolerations") + if tt.wantTolerations == nil { + if found { + t.Errorf("expected no tolerations field, got %v", got) + } + return + } + if !found { + t.Fatalf("expected tolerations %v, found none", tt.wantTolerations) + } + if len(got) != len(tt.wantTolerations) { + t.Fatalf("got %d toleration(s) %v, want %d %v", len(got), got, len(tt.wantTolerations), tt.wantTolerations) + } + for i := range got { + gotTol, _ := got[i].(map[string]any) + wantTol, _ := tt.wantTolerations[i].(map[string]any) + for k, v := range wantTol { + if gotTol[k] != v { + t.Errorf("toleration[%d][%q] = %v, want %v", i, k, gotTol[k], v) + } + } + } + }) + } +}