diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 112201971..261f1ddf4 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -20,7 +20,7 @@ A machine-readable **CycloneDX 1.6 JSON** companion to this page is produced by ## Summary - Components: **44** -- Unique images: **100** +- Unique images: **104** - Distinct registries: **11** Registries: `602401143452.dkr.ecr.us-west-2.amazonaws.com`, `cr.agentgateway.dev`, `docker.io`, `gcr.io`, `ghcr.io`, `gke.gcr.io`, `nvcr.io`, `public.ecr.aws`, `quay.io`, `registry.k8s.io`, `us-docker.pkg.dev` @@ -56,7 +56,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | kueue | helm | kueue | 0.18.2 | 1 | | mariadb-operator | helm | mariadb-operator | 26.6.0 | 1 | | mariadb-operator-crds | helm | mariadb-operator-crds | 26.6.0 | 0 | -| network-operator | helm | nvidia/network-operator | 26.4.1 | 5 | +| network-operator | helm | nvidia/network-operator | 26.4.1 | 9 | | network-operator-ocp | manifest | — | — | 0 | | network-operator-ocp-olm | manifest | — | — | 0 | | nfd | helm | node-feature-discovery | 0.19.0 | 1 | @@ -240,6 +240,10 @@ _No images extracted._ ### network-operator - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616` +- `ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.1` +- `ghcr.io/k8snetworkplumbingwg/plugins:v1.6.2-update.1` +- `ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0` +- `ghcr.io/mellanox/nvidia-k8s-ipam:v0.2.0` - `nvcr.io/nvidia/cloud-native/network-operator:v26.4.1` - `nvcr.io/nvidia/doca/doca_telemetry:1.22.5-doca3.1.0-host` - `nvcr.io/nvidia/mellanox/doca-driver:doca3.2.0-25.10-1.2.8.0-2` diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index ed2c86b67..45c7c9d20 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -17,7 +17,7 @@ gb200-eks-ubuntu-inference-dynamo: 9eed535acde123eb3930dbde32926311ff2b548de998e gb200-eks-ubuntu-training-kubeflow: 1c63d58186ab1ede9aa2200a9990beae419e4e11a7681235593e64ba460ad7e3 gb200-eks-ubuntu-training-slurm: 391fc653aa7d4d88a612582cf95d9283f466f271573559ef161338726f95b2b4 gb200-oke-ubuntu-inference-dynamo: bc44cf45aba15cfa21b467c4cf01d1bb5f49c452e96e95d30581810665055888 -gb200-oke-ubuntu-training-kubeflow: 9f12bc4ad245e5e8454895967b4de9aeb034f9d97f2dd77540077f2438509fc4 +gb200-oke-ubuntu-training-kubeflow: 4ccb92e45324caaac38c2f45420f03754965a7c5eff563b76d46aec659988638 gb300-any: 88827e93ce413b83f4a69c0cddd6fb40cff3411d304ee19e898b952e2bbf42d8 gb300-eks-ubuntu-inference-dynamo: 867925ffc95b6ea43421b92e5cbe07d1a1c1cf10bed3d761ee9e8fb324be553e gb300-eks-ubuntu-training-kubeflow: e69085cfe1377bf8916084ae273770a8a712c100d3d77ec6b4d830bf7437ea36 @@ -41,7 +41,7 @@ h200-eks-inference: 638d96aea0b45a916eac6cbd1311d4a7aa76de781d221e34b0c7e268e752 h200-eks-training: 55a97efb5818c25775801b65691e9be1065eb799c1c4dbdae9b8e9b9cdbf9f9a l40s-any: 401c86beef896b571de151e139f939c9dcea34cda58015bbabee936e632bff21 l40s-oke-inference: 79c9fdc18720d96bd2c4f54d779e5d3087703cea39262b1168a80ba4b9664d5e -l40s-oke-training: 357e29252b607132e3fdfcaae3787c633fcda3440e0440077175aa5c27c0be56 +l40s-oke-training: 8c205e91a8521f08e340d29e783130dff1d04c1b0bda8f38c79c67844b98da40 monitoring-hpa: f44796edcb112c0b34971661b9c6574a8bc1843fbffaa3e90ef9ddfa82c39bc8 ocp-inference-nim: bb00cdb191823b32da334bea70826c8a92c11b62d0e096918d43a8e1b043c361 ocp-training: d7a213263630f2c25982d6f4a144df7d9d5784428d296ccda1b4dab5a42b98bb diff --git a/pkg/recipe/performance_goals_oke_test.go b/pkg/recipe/performance_goals_oke_test.go index b6e8e77d8..18a32d742 100644 --- a/pkg/recipe/performance_goals_oke_test.go +++ b/pkg/recipe/performance_goals_oke_test.go @@ -34,22 +34,25 @@ func TestOKEPerformanceGoalsFollowTrainingInferencePattern(t *testing.T) { }{ { name: "gb200-oke-training", - wantChecks: []string{"nccl-all-reduce-bw-nvls"}, + wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"}, wantConstraints: map[string]string{ + "nccl-all-reduce-bw-net": ">= 40", "nccl-all-reduce-bw-nvls": ">= 500", }, }, { name: "gb200-oke-ubuntu-training", - wantChecks: []string{"nccl-all-reduce-bw-nvls"}, + wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"}, wantConstraints: map[string]string{ + "nccl-all-reduce-bw-net": ">= 40", "nccl-all-reduce-bw-nvls": ">= 500", }, }, { name: "gb200-oke-ubuntu-training-kubeflow", - wantChecks: []string{"nccl-all-reduce-bw-nvls"}, + wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"}, wantConstraints: map[string]string{ + "nccl-all-reduce-bw-net": ">= 40", "nccl-all-reduce-bw-nvls": ">= 500", }, }, diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 5ec73c328..5e3dc2c71 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -17,7 +17,7 @@ gb200-eks-ubuntu-inference-dynamo: 8babae6413d104da16060f18d919c80fd97ff983bd32f gb200-eks-ubuntu-training-kubeflow: 63b2fbd3a195bac682bb45263e5d5d0ec479a6571dd38ef0505eeb4f287b29e9 gb200-eks-ubuntu-training-slurm: c35e33618c0a8bb548bbe3c5d6765faf90cebe1427478ef97089f7a30bb37a6c gb200-oke-ubuntu-inference-dynamo: 6f01848824dcbefb0f58ec56ee9e911fff1470c9f67c0d0a9ba0de1dd1e648d2 -gb200-oke-ubuntu-training-kubeflow: 3959f47474edfac9b53a65939117d4364551fde0c9324246f195f144fe3beb12 +gb200-oke-ubuntu-training-kubeflow: 5eda952a4f20b526db67a2210ce79549627cbf5e44bac6eea6e4e499af8de16c gb300-any: 7f68607dffbbfc912f19b00ffa833caa81ca675d25bc22cf612dbd0b76f39475 gb300-eks-ubuntu-inference-dynamo: b14a3d99503949b8bfcdc81455da4af389a339deb24ea784637a7901297d70ad gb300-eks-ubuntu-training-kubeflow: 5303e5ebf8dc2ec13c2cff60c7caba2ef53d1a65bfd69d06f4f78cbb93835c49 @@ -41,7 +41,7 @@ h200-eks-inference: d1ff148001722e9d5e23606bd7770f8120506f08c10f0c132e0c379f5356 h200-eks-training: 98d04b3c81a059a0321451a39772af8d0d3310ad1fb861837f87d82dd997c4a1 l40s-any: 89210ba5815f93ce3d7b1a2a78f7ba75d083556e940603a1bc812eac9aeccd0a l40s-oke-inference: 6c75ef93e86fb21dbdf73ac97089b894ca8da832164dc807e94592493ab7f7cc -l40s-oke-training: 2a35011bd59cf3c727cea158d504c51bd59cba069620d3d6a4988d68972dee74 +l40s-oke-training: 167eeb34d6b5f2d03a3c6605f623883c45261f49e9d2eb5b6e1463aee931f4e3 monitoring-hpa: 15e93304e7e68997e7b15a0be4af360a29f10c869502e6dccb6a6cefb6519fd5 ocp-inference-nim: 49245ecfc91f67fcc2ed0b305b00046a4164628cafc75ab98542e6dafa626821 ocp-training: aaa8cd7f45ef42235b075936461d715e2e017ea2802df9e8017de38304a17915 diff --git a/recipes/components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml new file mode 100644 index 000000000..66161dbca --- /dev/null +++ b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml @@ -0,0 +1,48 @@ +# NicClusterPolicy for GB200 OKE (OCI) — rdmaSharedDevicePlugin over InfiniBand. +# +# Mirrors the AOR OCI GB200 config validated on gb200-ew. No ofedDriver (host +# MOFED), no SR-IOV: the NVL72 east-west fabric is IB on the rdma0-3 netdevs +# (oci_hpc.rdma_device_names_mode=2 kernel cmdline names them deterministically). +# +# The IB devices are advertised as nvidia.com/mlnxnics — the same resource +# name the L40S SR-IOV path uses, so workloads request RDMA uniformly +# across OKE fabrics. +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +spec: + rdmaSharedDevicePlugin: + image: k8s-rdma-shared-dev-plugin + repository: nvcr.io/nvidia/mellanox + version: network-operator-v26.4.1 + config: | + { + "configList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "mlnxnics", + "rdmaHcaMax": 63, + "selectors": { + "linkTypes": ["infiniband"], + "ifNames": ["rdma0", "rdma1", "rdma2", "rdma3"] + } + } + ] + } + deploymentTolerations: + - key: CriticalAddonsOnly + operator: Exists + tolerations: + # RDMA DaemonSets must land on tainted GPU nodes. + - key: nvidia.com/gpu + operator: Exists + - key: CriticalAddonsOnly + operator: Exists diff --git a/recipes/components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml new file mode 100644 index 000000000..0a78aacca --- /dev/null +++ b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml @@ -0,0 +1,73 @@ +# NicClusterPolicy for L40S OKE (OCI) SR-IOV RoCE. +# +# The network-operator Helm chart installs the operator + CRD but does not template +# a NicClusterPolicy CR (values-oke-l40s.yaml sets deployCR: false). This manifest +# creates it so the operator reconciles the RoCE fabric stack. Hand-rendered from +# AOR's network-operator/nicclusterpolicy.yaml.tmpl (provider: oci branch, with +# network.type == roce → nvIpam + secondaryNetwork included). +# +# OCI specifics (vs Forge IB): NO ofedDriver — OCI nodes carry host MOFED, consumed +# by the GPU Operator driver via driver.rdma.useHostMofed (l40s-oke-ubuntu leaf). One +# sriovDevicePlugin resource, nvidia.com/mlnxnics, selecting the OCI ConnectX VF +# device IDs (101a = ConnectX-5 Ex VF, 101e = mlx5Gen VF). RoCE also needs nv-ipam +# (VF IP allocation) + secondaryNetwork/multus (attach the VF into workload pods). +# vendor 15b3 = Mellanox. +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +spec: + # RoCE: allocate IPs for the RDMA VFs and wire them into pods via multus. + nvIpam: + image: nvidia-k8s-ipam + repository: ghcr.io/mellanox + version: v0.2.0 + enableWebhook: false + containerResources: + - name: nv-ipam-node + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + secondaryNetwork: + cniPlugins: + image: plugins + repository: ghcr.io/k8snetworkplumbingwg + version: v1.6.2-update.1 + multus: + image: multus-cni + repository: ghcr.io/k8snetworkplumbingwg + version: v4.2.1 + sriovDevicePlugin: + image: sriov-network-device-plugin + repository: ghcr.io/k8snetworkplumbingwg + version: v3.9.0 + config: | + { + "resourceList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "mlnxnics", + "selectors": {"isRdma":true,"vendors":["15b3"],"devices":["101a","101e"]} + } + ] + } + # Operator DaemonSet placement: system/monitoring nodes only (matches AOR). + deploymentTolerations: + - key: CriticalAddonsOnly + operator: Exists + tolerations: + # RDMA DaemonSets must land on tainted GPU nodes. + - key: nvidia.com/gpu + operator: Exists + - key: CriticalAddonsOnly + operator: Exists diff --git a/recipes/components/network-operator/values-oke-gb200.yaml b/recipes/components/network-operator/values-oke-gb200.yaml new file mode 100644 index 000000000..6211096f4 --- /dev/null +++ b/recipes/components/network-operator/values-oke-gb200.yaml @@ -0,0 +1,25 @@ +# network-operator Helm values for GB200 OKE (OCI) InfiniBand. +# +# OCI GB200 NVL72 model (vs L40S RoCE / Forge IB): NO ofedDriver — nodes carry host +# MOFED — and no SR-IOV/nv-ipam/multus either. East-west is InfiniBand (rdma0-3), +# served by rdmaSharedDevicePlugin from the post-install NicClusterPolicy manifest, +# NOT the chart. deployCR off so that manifest CR is authoritative. +# nfd.enabled: false — GPU Operator's NFD is used; no second NFD. +deployCR: false +nvIpam: + enabled: false +secondaryNetwork: + deploy: false +nfd: + enabled: false +operator: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + # Operator placement comes from the bundler's system-node scheduling + # injection (registry nodeScheduling: operator.nodeSelector / + # operator.tolerations) — no hardcoded affinity here. diff --git a/recipes/components/network-operator/values-oke-l40s.yaml b/recipes/components/network-operator/values-oke-l40s.yaml new file mode 100644 index 000000000..109a16ac9 --- /dev/null +++ b/recipes/components/network-operator/values-oke-l40s.yaml @@ -0,0 +1,34 @@ +# network-operator Helm values for L40S OKE (OCI) SR-IOV RoCE. +# Hand-rendered from AOR's network-operator/values.yaml.tmpl (provider: oci) + +# nicclusterpolicy.yaml.tmpl (oci branch, network.type == roce). +# +# OCI model (vs Forge IB / Mistral DOCA): NO ofedDriver — OCI bare-metal nodes carry +# host MOFED, so the GPU Operator uses it via driver.rdma.useHostMofed (set on the +# l40s-oke-ubuntu leaf). network-operator's job here is the SR-IOV VF device plugin +# (advertises nvidia.com/mlnxnics RDMA VFs) plus nv-ipam + secondaryNetwork (multus) +# for RoCE — all supplied by the post-install NicClusterPolicy manifest, NOT the chart. +# +# deployCR/nvIpam/secondaryNetwork: AICR's wrapper defaults are on (deployCR: true, +# nvIpam.enabled: true, secondaryNetwork.deploy: true) — they template the wrapper's +# own NicClusterPolicy. Turn deployCR off so our manifest CR is authoritative (it is +# the only place the OCI VF selectors 101a/101e can be expressed); the operator +# reconciles nv-ipam + secondaryNetwork + sriovDevicePlugin from that CR regardless. +# nfd.enabled: false — GPU Operator's NFD is used; no second NFD. +deployCR: false +nvIpam: + enabled: false +secondaryNetwork: + deploy: false +nfd: + enabled: false +operator: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + # Operator placement comes from the bundler's system-node scheduling + # injection (registry nodeScheduling: operator.nodeSelector / + # operator.tolerations) — no hardcoded affinity here. diff --git a/recipes/manifest_images_test.go b/recipes/manifest_images_test.go index 23461a37e..c5c44a11e 100644 --- a/recipes/manifest_images_test.go +++ b/recipes/manifest_images_test.go @@ -87,25 +87,43 @@ func TestComponentManifestImagesAreFullyQualified(t *testing.T) { // these refs is delivered by admission-time digest or signature // verification at deploy time (#745) plus the upstream signing requests // filed under the supply-chain epic (#739). -var imageDigestExemptions = map[string]string{ +// imageDigestExemption scopes an exemption to the manifest that carries +// the reference: the exemption applies only when the walked path contains +// Manifest, so another resource reusing the same tag elsewhere cannot ride +// an existing exemption past the digest check. +type imageDigestExemption struct { + // Manifest is a path substring under components/ that must appear in + // the manifest path for the exemption to apply. + Manifest string + Reason string +} + +var imageDigestExemptions = map[string]imageDigestExemption{ // NicClusterPolicy (network-operator AKS): repository/image/version // triplet schema; no digest field. - "nvcr.io/nvidia/mellanox/doca-driver:doca3.2.0-25.10-1.2.8.0-2": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", - "nvcr.io/nvidia/mellanox/k8s-rdma-shared-dev-plugin:network-operator-v26.4.1": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", - "nvcr.io/nvidia/doca/doca_telemetry:1.22.5-doca3.1.0-host": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", + "nvcr.io/nvidia/mellanox/doca-driver:doca3.2.0-25.10-1.2.8.0-2": {"network-operator/manifests/nic-cluster-policy-aks", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, + "nvcr.io/nvidia/mellanox/k8s-rdma-shared-dev-plugin:network-operator-v26.4.1": {"network-operator/manifests/nic-cluster-policy-", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, + "nvcr.io/nvidia/doca/doca_telemetry:1.22.5-doca3.1.0-host": {"network-operator/manifests/nic-cluster-policy-aks", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, // Skyhook Package (nodewright-customizations no-op): shellscript package // is pinned by tag only — `containerSHA` is not surfaced for this // upstream image. nodewright-packages/* refs are digest-pinned via the // Skyhook Package `containerSHA` field (issue #1031), folded into the // extracted image ref as `@sha256:...` by pkg/bom.ExtractImagesFromYAML. - "ghcr.io/nvidia/skyhook-packages/shellscript:1.1.1": "Skyhook Package CRD does not accept image digests; tracked via #745 and NVIDIA/nodewright#224", + "ghcr.io/nvidia/skyhook-packages/shellscript:1.1.1": {"nodewright-customizations/manifests/", "Skyhook Package CRD does not accept image digests; tracked via #745 and NVIDIA/nodewright#224"}, // cos-gpu-installer (gcp-driver-installer): node-local image preloaded on // every COS node and referenced with imagePullPolicy: Never — there is no // registry to pin a digest against, the "digest" differs per COS build, // and the ref must never be mirrored or pulled. Issue #1716. - "cos-nvidia-installer:fixed": "COS-node-local preloaded image (imagePullPolicy: Never); no registry digest exists and it must not be mirrored; issue #1716", + "cos-nvidia-installer:fixed": {"gcp-driver-installer/manifests/", "COS-node-local preloaded image (imagePullPolicy: Never); no registry digest exists and it must not be mirrored; issue #1716"}, + + // NicClusterPolicy (network-operator OKE): same repository/image/version + // triplet schema as the AKS entries above — no digest field in the CRD. + "ghcr.io/mellanox/nvidia-k8s-ipam:v0.2.0": {"network-operator/manifests/nic-cluster-policy-oke-", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, + "ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.1": {"network-operator/manifests/nic-cluster-policy-oke-", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, + "ghcr.io/k8snetworkplumbingwg/plugins:v1.6.2-update.1": {"network-operator/manifests/nic-cluster-policy-oke-", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, + "ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0": {"network-operator/manifests/nic-cluster-policy-oke-", "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555"}, } // TestComponentManifestImagesAreDigestPinned asserts that every image @@ -151,8 +169,8 @@ func TestComponentManifestImagesAreDigestPinned(t *testing.T) { t.Errorf("%s: image %q uses non-sha256 digest %q; ADR-006 requires @sha256:", p, img, ref.Digest) continue } - if reason, ok := imageDigestExemptions[img]; ok { - t.Logf("exempted: %s — %s", img, reason) + if exemption, ok := imageDigestExemptions[img]; ok && strings.Contains(p, exemption.Manifest) { + t.Logf("exempted: %s (%s) — %s", img, exemption.Manifest, exemption.Reason) continue } t.Errorf("%s: image %q is not digest-pinned and not in the documented exemption set (recipes/manifest_images_test.go::imageDigestExemptions); per ADR-006 layer 2, append an @sha256: or add an exemption with a reason", p, img) diff --git a/recipes/overlays/gb200-oke-training.yaml b/recipes/overlays/gb200-oke-training.yaml index d9543c6b7..eeff79b0d 100644 --- a/recipes/overlays/gb200-oke-training.yaml +++ b/recipes/overlays/gb200-oke-training.yaml @@ -39,9 +39,15 @@ spec: value: ">= 1.34" componentRefs: - # GB200-specific GPU Operator overrides (inherits valuesFile from oke-training) + # GB200-specific GPU Operator overrides (inherits valuesFile from oke-training). + # kernel-module-params sets NVreg_GrdmaPciTopoCheckOverride=1, required + # for dma-buf attach over the IB fabric (GPUDirect RDMA without + # nvidia-peermem, whose chroot modprobe fails to build against the -64k + # Grace kernel). - name: gpu-operator type: Helm + preManifestFiles: + - components/gpu-operator/manifests/kernel-module-params.yaml dependencyRefs: - nfd - cert-manager @@ -49,6 +55,9 @@ spec: overrides: gdrcopy: enabled: true + driver: + kernelModuleConfig: + name: nvidia-kernel-module-params - name: nfd type: Helm @@ -56,13 +65,33 @@ spec: topologyUpdater: enable: true + # InfiniBand east-west fabric (NVL72 rdma0-3). rdmaSharedDevicePlugin + # advertises the shared HCAs as nvidia.com/mlnxnics; no SR-IOV/nv-ipam + # (that is the L40S RoCE path) and no ofedDriver (OCI nodes carry host + # MOFED). NicClusterPolicy is manifest-supplied (chart deployCR off). + # Present in every gpuStack value; incompatible with Oracle's opt-in + # NvidiaNetworkOperator add-on. + - name: network-operator + type: Helm + valuesFile: components/network-operator/values-oke-gb200.yaml + manifestFiles: + - components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml + dependencyRefs: + - nfd + - cert-manager + validation: performance: - # NVLS runtime support is OKE-specific. NET/RDMA is intentionally left - # out until OCI-specific pod RDMA exposure is verified on the testbed. + # Both transport variants: NVLS (MNNVL across the NVL72 IMEX domain) + # and NET (the IB east-west fabric this leaf's NicClusterPolicy + # exposes — validated on a BM.GPU.GB200.4 NVL72 rack). Constraints + # match gb200-eks-training. checks: + - nccl-all-reduce-bw-net - nccl-all-reduce-bw-nvls constraints: + - name: nccl-all-reduce-bw-net + value: ">= 40" - name: nccl-all-reduce-bw-nvls value: ">= 500" conformance: diff --git a/recipes/overlays/l40s-oke-training.yaml b/recipes/overlays/l40s-oke-training.yaml index 6314819d2..e3c756e75 100644 --- a/recipes/overlays/l40s-oke-training.yaml +++ b/recipes/overlays/l40s-oke-training.yaml @@ -54,6 +54,26 @@ spec: topologyUpdater: enable: true + # RDMA fabric (RoCE over SR-IOV VFs). Every L40S OCI cluster runs RoCE; + # the SR-IOV VF device plugin advertises nvidia.com/mlnxnics RDMA VFs, + # with nv-ipam + multus attaching the VFs into workload pods. The + # NicClusterPolicy is supplied by the manifest (the chart's deployCR is + # off — the manifest is the only place the OCI VF selectors 101a/101e + # can be expressed). OCI nodes carry host MOFED, so there is no + # ofedDriver in any configuration. GPUDirect RDMA works via DMA-BUF; + # nvidia-peermem stays off (base default) — inert on this topology. + # Present in every gpuStack value: the fabric is orthogonal to GPU + # driver/plugin ownership. Incompatible with Oracle's opt-in + # NvidiaNetworkOperator add-on (two lifecycle managers, one release). + - name: network-operator + type: Helm + valuesFile: components/network-operator/values-oke-l40s.yaml + manifestFiles: + - components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml + dependencyRefs: + - nfd + - cert-manager + # Validation checks for L40S on OKE training workloads. # Defined at the intent layer (not OS-specific) so all OS variants inherit them. # diff --git a/validators/performance/nccl_all_reduce_bw_constraint.go b/validators/performance/nccl_all_reduce_bw_constraint.go index 92365f519..d50f49e26 100644 --- a/validators/performance/nccl_all_reduce_bw_constraint.go +++ b/validators/performance/nccl_all_reduce_bw_constraint.go @@ -247,6 +247,10 @@ var supportedNCCLCombinations = map[ncclVariant]map[recipe.CriteriaServiceType][ }, variantNET: { recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200}, + // OKE GB200 NVL72: IB east-west (rdma0-3) via the + // rdmaSharedDevicePlugin's nvidia.com/mlnxnics shared HCAs — + // see testdata/gb200/oke/runtime-net.yaml. + recipe.CriteriaServiceOKE: {recipe.CriteriaAcceleratorGB200}, }, variantNVLS: { recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200}, diff --git a/validators/performance/nccl_benchmark_profile_test.go b/validators/performance/nccl_benchmark_profile_test.go index bedc0f876..99044c2ac 100644 --- a/validators/performance/nccl_benchmark_profile_test.go +++ b/validators/performance/nccl_benchmark_profile_test.go @@ -156,7 +156,7 @@ func TestNCCLCombinationSupported(t *testing.T) { {"default B200 any", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorB200, recipe.CriteriaServiceAny), true}, {"default GB200 EKS not covered", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), false}, {"NET GB200 EKS", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), true}, - {"NET GB200 OKE not covered", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), false}, + {"NET GB200 OKE (IB via rdmaSharedDevicePlugin)", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), true}, {"NVLS GB200 EKS", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), true}, {"NVLS GB200 OKE", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), true}, {"unknown service", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, "custom-svc"), false}, diff --git a/validators/performance/testdata/gb200/oke/runtime-net.yaml b/validators/performance/testdata/gb200/oke/runtime-net.yaml new file mode 100644 index 000000000..7ae7aea32 --- /dev/null +++ b/validators/performance/testdata/gb200/oke/runtime-net.yaml @@ -0,0 +1,208 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NCCL all-reduce TrainingRuntime for GB200 on OKE, NET/IB-transport variant. +# +# Measures the InfiniBand east-west fabric (rdma0-3, served by the +# rdmaSharedDevicePlugin from the gb200-oke NicClusterPolicy) instead of +# NVLink: NVLS and MNNVL are forced off so NCCL falls back to its built-in +# IB transport over the nvidia.com/mlnxnics shared HCAs. NCCL_DEBUG stays +# INFO — verifyTransportFromLogs confirms the transport by parsing the +# "NCCL INFO Using network" banner from launcher logs. +# +# Composition: the gb200/oke runtime-nvls.yaml scaffold (pytorch image, +# mpirun ssh wiring) with the IMEX resource claims removed and the +# net-variant env from gb200/eks runtime-net.yaml, minus the EFA provider +# knobs. Node pods request nvidia.com/mlnxnics alongside GPUs. +# +# Cluster prerequisites: rdmaSharedDevicePlugin healthy (nvidia.com/mlnxnics +# allocatable on every GPU node — the gb200-oke leaves' NicClusterPolicy), +# and NVreg_GrdmaPciTopoCheckOverride=1 from the leaf's kernel-module-params +# ConfigMap (dma-buf attach over the IB fabric). + +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainingRuntime +metadata: + name: nccl-all-reduce-runtime + namespace: ${NAMESPACE} + labels: + trainer.kubeflow.org/framework: mpi +spec: + mlPolicy: + mpi: + mpiImplementation: OpenMPI + numProcPerNode: ${GPU_COUNT_PER_NODE} + runLauncherAsNode: false + sshAuthMountPath: /tmp/mpi-keys + template: + spec: + network: + enableDNSHostnames: true + publishNotReadyAddresses: true + replicatedJobs: + - name: launcher + replicas: 1 + template: + spec: + template: + spec: + tolerations: + - operator: Exists + initContainers: + - name: fix-ssh-perms + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: + - /bin/sh + - -c + - | + mkdir -p /root/.ssh + cp /tmp/mpi-keys/id_rsa /root/.ssh/id_rsa + cp /tmp/mpi-keys/authorized_keys /root/.ssh/authorized_keys + chmod 700 /root/.ssh + chmod 600 /root/.ssh/id_rsa /root/.ssh/authorized_keys + volumeMounts: + - name: mpi-ssh-auth + mountPath: /tmp/mpi-keys + readOnly: true + - name: ssh-config + mountPath: /root/.ssh + containers: + - name: node + image: nvcr.io/nvidia/pytorch:25.06-py3 + env: + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" + command: + - /usr/local/mpi/bin/mpirun + args: + - -np + - "${GPU_COUNT}" + - --allow-run-as-root + - --mca + - plm_rsh_args + - -o StrictHostKeyChecking=no -o ConnectionAttempts=10 + - --mca + - btl + - ^openib + - --mca + - btl_tcp_if_include + - eth0 + - --mca + - oob_tcp_if_include + - eth0 + - -x + - LD_LIBRARY_PATH + - -x + - NCCL_DEBUG=INFO + # Force NCCL onto the NET/IB transport: disable NVLS (NVLink + # SHARP) and MNNVL (multi-node NVLink) so traffic crosses the + # IB fabric; NCCL_NET_PLUGIN=none selects the built-in IB + # verbs transport over the shared mlx5 HCAs. + - -x + - NCCL_NVLS_ENABLE=0 + - -x + - NCCL_MNNVL_ENABLE=0 + - -x + - NCCL_NET_PLUGIN=none + - -x + - NCCL_SOCKET_IFNAME=eth0 + - -x + - NCCL_IGNORE_DISABLED_P2P=1 + - /usr/local/bin/${TEST_TYPE}_mpi + - -b + - ${MIN_MESSAGE_SIZE} + - -e + - ${MAX_MESSAGE_SIZE} + - -f + - "2" + - -g + - "1" + resources: + limits: + cpu: "2" + memory: 128Mi + volumeMounts: + - name: ssh-config + mountPath: /root/.ssh + volumes: + - name: ssh-config + emptyDir: {} + - name: node + template: + spec: + template: + spec: + tolerations: + - operator: Exists + initContainers: + - name: fix-ssh-perms + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: + - /bin/sh + - -c + - | + apt-get update && + apt-get install -y --no-install-recommends openssh-server && + mkdir -p /var/run/sshd && + chmod 0755 /var/run/sshd && + mkdir -p /root/.ssh && + cp /tmp/mpi-keys/authorized_keys /root/.ssh/authorized_keys && + chmod 700 /root/.ssh && + chmod 600 /root/.ssh/authorized_keys + volumeMounts: + - name: mpi-ssh-auth + mountPath: /tmp/mpi-keys + readOnly: true + - name: ssh-config + mountPath: /root/.ssh + containers: + - name: node + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: ["sh", "-c"] + args: + - | + apt-get update && + apt-get install -y --no-install-recommends openssh-server && + mkdir -p /var/run/sshd && + chmod 0755 /var/run/sshd && + mkdir -p /root/.ssh && + cp /tmp/mpi-keys/* /root/.ssh/ && + chmod 700 /root/.ssh && + chmod 600 /root/.ssh/authorized_keys && + /usr/sbin/sshd -De + resources: + limits: + nvidia.com/gpu: ${GPU_COUNT_PER_NODE} + nvidia.com/mlnxnics: "1" + requests: + nvidia.com/gpu: ${GPU_COUNT_PER_NODE} + nvidia.com/mlnxnics: "1" + securityContext: + capabilities: + add: ["IPC_LOCK"] + volumeMounts: + - name: ssh-config + mountPath: /root/.ssh + - name: dshm + mountPath: /dev/shm + volumes: + - name: ssh-config + emptyDir: {} + - name: dshm + emptyDir: + medium: Memory + successPolicy: + operator: All + targetReplicatedJobs: + - launcher