feat(ci): onboard GB200 AWS UAT reservation

njhensley · njhensley · commit 19ee39acfa98 · 2026-07-15T16:16:06.000-07:00
Add the aws-gb200 reservation row (cross-account GB200 Capacity Block),
its EKS cluster-config and Kubeflow training test-config, and generalize
the AWS UAT pipeline's test-config selection from h100-&lt;intent&gt; to
&lt;accelerator&gt;-&lt;intent&gt; so per-accelerator configs are picked up.

Derive AWS_ACCOUNT_ID from the cluster-config's .deployment.tenancy
rather than a caller-supplied input: the OIDC role can only be assumed
into the account being provisioned, removing the two-source drift risk
that a separate account input carried.

Signed-off-by: Nathan Hensley &lt;nhensley@nvidia.com&gt;
diff --git a/.github/workflows/uat-aws.yaml b/.github/workflows/uat-aws.yaml
@@ -33,7 +33,7 @@ on:
         type: string
         default: ''
       intent:
-        description: 'Recipe intent — selects the per-intent test config (h100-<intent>-config.yaml).'
+        description: 'Recipe intent — selects the per-intent test config (<accelerator>-<intent>-config.yaml).'
         type: string
         default: training
       lifecycle:
@@ -48,6 +48,10 @@ on:
         description: 'Directory holding the AICRConfig test files (from infra/uat/reservations.yaml).'
         type: string
         required: true
+      accelerator:
+        description: 'GPU accelerator family (from the reservation); selects the test config <accelerator>-<intent>-config.yaml.'
+        type: string
+        default: h100
       skip_delete:
         description: 'Skip cluster teardown (manual runs only; the nightly batch always deletes).'
         type: boolean
@@ -90,7 +94,15 @@ jobs:
       # the ingest-evidence job below. Empty when no bundle was produced.
       bundle_ref: ${{ steps.evidence_ref.outputs.ref }}
     env:
-      AWS_ACCOUNT_ID: "615299774277"
+      # AWS_ACCOUNT_ID (the tenancy the OIDC role is assumed into, and where the
+      # cluster + Terraform state live) is NOT set here and is NOT a caller
+      # input: it is the single-source-of-truth `.deployment.tenancy` from the
+      # cluster-config, exported to $GITHUB_ENV by the "Validate inputs" step
+      # below. Deriving it — rather than passing it in — makes it impossible to
+      # assume role in one account while provisioning/state land in another
+      # (which a separate `account` input could silently do). A cross-account
+      # reservation (e.g. a GB200 Capacity Block in a separate tenancy) is
+      # selected purely by its cluster-config's tenancy field.
       AWS_REGION: "us-east-1"
       GITHUB_ACTIONS_ROLE_NAME: "github-actions-role-aicr"
       # Cluster name (EKS actuator derives it from .deployment.id). The nightly
@@ -104,9 +116,10 @@ jobs:
         && format('aicr-uat-day-{0}', inputs.reservation)
         || format('aicr-uat-{0}', github.run_id) }}
       CLUSTER_CONFIG: ${{ inputs.cluster_config_path }}
-      # DC2 selects the AICRConfig by intent; both intents drive the same
+      # The AICRConfig is selected by accelerator + intent
+      # (<accelerator>-<intent>-config.yaml); both intents drive the same
       # cluster-config (GPU pool from the reservation, system/CPU pools dynamic).
-      TEST_CONFIG: ${{ inputs.test_config_dir }}/h100-${{ inputs.intent }}-config.yaml
+      TEST_CONFIG: ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}-config.yaml
       # v0.4.27 — required for node-group taints in the cluster-config schema
       # (GPU pool carries skyhook.nvidia.com=runtime-required:NoSchedule until
       # NodeWrite finishes tuning + reboot). Earlier v0.4.x pins predate taint
@@ -155,7 +168,20 @@ jobs:
           if [[ ! -f "${TEST_CONFIG}" ]]; then
             echo "::error::test config not found: ${TEST_CONFIG}"; exit 1
           fi
-          echo "intent=${INTENT} lifecycle=${LIFECYCLE} test-config=${TEST_CONFIG} cluster=${DEPLOYMENT_ID}"
+          # The account the OIDC role is assumed into (AWS_ACCOUNT_ID) is the
+          # cluster-config's .deployment.tenancy — the single source of truth
+          # for which tenancy this run provisions + keeps state in. Deriving it
+          # here (rather than accepting it as an input) means the role can never
+          # be assumed into a different account than the one being provisioned.
+          # A cross-account reservation (e.g. a GB200 Capacity Block in a
+          # separate tenancy) is selected purely by its cluster-config. Uses the
+          # runner-preinstalled yq (Install tools runs later).
+          TENANCY=$(yq eval '.deployment.tenancy' "${CLUSTER_CONFIG}")
+          if [[ -z "${TENANCY}" || "${TENANCY}" == "null" ]]; then
+            echo "::error::${CLUSTER_CONFIG} has no .deployment.tenancy"; exit 1
+          fi
+          echo "AWS_ACCOUNT_ID=${TENANCY}" >> "${GITHUB_ENV}"
+          echo "intent=${INTENT} lifecycle=${LIFECYCLE} test-config=${TEST_CONFIG} cluster=${DEPLOYMENT_ID} account=${TENANCY}"
 
       # Versions
       - name: Load versions
diff --git a/.github/workflows/uat-run.yaml b/.github/workflows/uat-run.yaml
@@ -130,6 +130,11 @@ jobs:
       cloud: ${{ steps.row.outputs.cloud }}
       cluster_config_path: ${{ steps.row.outputs['cluster-config-path'] }}
       test_config_dir: ${{ steps.row.outputs['test-config-dir'] }}
+      # Accelerator selects the per-accelerator test-config filename
+      # (<accelerator>-<intent>-config.yaml) in the AWS pipeline. The AWS
+      # account (tenancy) is NOT plumbed here: the AWS pipeline derives it from
+      # the cluster-config's .deployment.tenancy, the single source of truth.
+      accelerator: ${{ steps.row.outputs.accelerator }}
     steps:
       - name: Checkout
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
@@ -183,6 +188,7 @@ jobs:
       lifecycle: ${{ inputs.lifecycle }}
       cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
       test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
+      accelerator: ${{ needs.resolve.outputs.accelerator }}
       skip_delete: ${{ inputs.skip_delete }}
       skip_tests: ${{ inputs.skip_tests }}
 
diff --git a/infra/uat/reservations.yaml b/infra/uat/reservations.yaml
@@ -109,3 +109,28 @@ reservations:
     # rows run both, so add it here rather than removing the key (a bare
     # `nightly-intents:` decodes as absent and defaults to [training]).
     nightly-intents: [training]
+  - name: aws-gb200
+    cloud: aws
+    # CROSS-ACCOUNT: the GB200 Capacity Block lives in a SEPARATE tenancy
+    # (dgx-cloud-runai-prod-22, account 897722687756), not the standard UAT
+    # account. The account is NOT a field here: the AWS pipeline derives the
+    # tenancy it assumes the OIDC role into from the target cluster-config's
+    # .deployment.tenancy (cluster-config-gb200.yaml sets it to 897722687756),
+    # the single source of truth. Prereq: apply infra/uat-aws-account in
+    # 897722687756 (GitHub OIDC provider + github-actions-role-aicr).
+    # GB200 reservations are time-bounded Capacity Blocks (not standing
+    # on-demand CRs like the h100 rows) — a run must fall inside the block's
+    # active window, else actuator apply fails to acquire capacity.
+    reservation-id: cr-0e2f3833a602809a6
+    accelerator: gb200
+    # 4 Blackwell GPUs per p6e-gb200.36xlarge node; the cluster-config runs 2
+    # nodes (activates the multi-node NCCL East-West fabric check).
+    gpu-count: 4
+    cluster-config-path: tests/uat/aws/cluster-config-gb200.yaml
+    test-config-dir: tests/uat/aws/tests
+    # Opted OUT of the nightly batch during bring-up (explicit empty list, NOT
+    # a bare `nightly-intents:` which would default to [training] and provision
+    # real GB200 capacity nightly). Stays manually dispatchable via uat-run.yaml.
+    # Enroll ([training]) after a green manual acceptance run and once the
+    # Capacity-Block scheduling window is wired into the nightly broker.
+    nightly-intents: []
diff --git a/pkg/uatbroker/registry_test.go b/pkg/uatbroker/registry_test.go
@@ -511,14 +511,15 @@ func TestDaytimeAssignmentsNone(t *testing.T) {
 }
 
 // TestCommittedRegistryValid guards the actual checked-in registry: it must
-// parse, validate, and carry the two launch reservations. A bad data edit
-// fails here before it can break the broker workflows.
+// parse, validate, and carry the launch reservations plus the aws-gb200
+// bring-up row. A bad data edit fails here before it can break the broker
+// workflows.
 func TestCommittedRegistryValid(t *testing.T) {
 	reg, err := LoadRegistryFile(filepath.Join("..", "..", "infra", "uat", "reservations.yaml"))
 	if err != nil {
 		t.Fatalf("committed reservations.yaml invalid: %v", err)
 	}
-	want := map[string]string{"aws-h100": CloudAWS, "gcp-h100": CloudGCP, "azure-h100": CloudAzure}
+	want := map[string]string{"aws-h100": CloudAWS, "gcp-h100": CloudGCP, "azure-h100": CloudAzure, "aws-gb200": CloudAWS}
 	for name, cloud := range want {
 		res, err := reg.Lookup(name)
 		if err != nil {
@@ -569,6 +570,11 @@ func TestCommittedRegistryValid(t *testing.T) {
 		// acceptance run (29125390442); inference joins after a green
 		// manual intent=inference dispatch.
 		"azure-h100": {IntentTraining},
+		// aws-gb200 is OPTED OUT of the nightly batch during bring-up (explicit
+		// empty list). Locked here so an accidental edit to a bare
+		// `nightly-intents:` (which defaults to [training]) — provisioning real
+		// GB200 Capacity-Block capacity nightly — fails this guard instead.
+		"aws-gb200": {},
 	}
 	for name, want := range wantNightly {
 		res, lookupErr := reg.Lookup(name)
diff --git a/tests/uat/aws/cluster-config-gb200.yaml b/tests/uat/aws/cluster-config-gb200.yaml
@@ -0,0 +1,147 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# EKS GB200 UAT cluster (reservation aws-gb200, infra/uat/reservations.yaml).
+#
+# Differs from the p5 cluster-config.yaml (aws-h100) in three structural ways,
+# all forced by GB200 hardware and its Capacity Block:
+#   1. CROSS-ACCOUNT — the Capacity Block is held in tenancy 897722687756
+#      (dgx-cloud-runai-prod-22), not the standard UAT account. The AWS
+#      pipeline assumes the OIDC role in .deployment.tenancy below (which must
+#      match reservations.yaml `account`). Prereq: apply infra/uat-aws-account
+#      in 897722687756 first (GitHub OIDC provider + github-actions-role-aicr).
+#   2. LOCAL ZONE — p6e-gb200 capacity lives in us-east-1-dfw-2a (parent AZ
+#      us-east-1d). The EKS control plane cannot live in a Local Zone, so the
+#      GPU worker + pod subnets are in the Local Zone while control-plane /
+#      system / public subnets stay in region AZs (incl. us-east-1d, the
+#      parent). Local-Zone worker egress routes through a region NAT.
+#   3. arm64 + Capacity Block — GB200 is Grace/arm64 and the reservation is a
+#      time-bounded Capacity Block (marketType below), not a standing CR.
+#
+# Two mchmarny/cluster actuator workarounds are load-bearing (see the inline
+# notes on the worker nodeGroup and adminRoles).
+
+apiVersion: github.com/mchmarny/cluster/v1alpha1
+kind: Cluster
+
+deployment:
+  id: aicr-uat-gb200            # workflow overrides with a run-scoped id
+  provider: eks
+  tenancy: "897722687756"       # dgx-cloud-runai-prod-22 (GB200 Capacity Block)
+  location: us-east-1
+  state: tenancy                # remote TF state in this account's S3
+  destroy: false
+  tags:
+    env: dev
+
+cluster:
+  eks:
+    version: "1.35"
+    # Empty: the CI OIDC role (github-actions-role-aicr) is the cluster
+    # CREATOR and gets cluster-admin automatically
+    # (bootstrap_cluster_creator_admin_permissions=true). The actuator ALSO
+    # writes an explicit access entry per adminRole; listing the creator's own
+    # role here collides with the bootstrap entry and 409s. Human admins can
+    # add this account's SSO admin role once its name is known (it differs from
+    # the standard account's AWSReservedSSO_CS-Admin_55d42b5a9a2a8c35).
+    adminRoles: []
+    controlPlane:
+      allowedCidrs:
+        - 216.228.127.128/30
+        - 52.41.222.58/32
+        - 54.68.11.139/32
+        - 128.77.49.32/30
+    addOns:
+      coreDns: ""
+      kubeProxy: ""
+      vpcCni: ""
+
+network:
+  eks:
+    cidrs:
+      host: 10.0.0.0/16
+      pod: 100.65.0.0/16
+    subnets:
+      # Control-plane / system / public subnets MUST be in region AZs (the EKS
+      # control plane cannot live in a Local Zone). Include us-east-1d — the
+      # Local Zone's parent — so cross-zone routing to the workers is local.
+      public:
+        - cidr: 10.0.1.0/27
+          zone: us-east-1a
+        - cidr: 10.0.2.0/27
+          zone: us-east-1d
+      system:
+        - cidr: 10.0.4.0/22
+          zone: us-east-1a
+        - cidr: 10.0.8.0/22
+          zone: us-east-1d
+      # GPU workers + pods live in the Local Zone where the Capacity Block is.
+      worker:
+        - cidr: 10.0.128.0/17
+          zone: us-east-1-dfw-2a
+          disableEndpoints: true
+      pod:
+        - cidr: 100.65.0.0/16
+          zone: us-east-1-dfw-2a
+          disableEndpoints: true
+    securityGroups:
+      additionalRules:
+        - target: system
+          direction: ingress
+          fromPort: 4222
+          toPort: 4222
+          protocol: tcp
+          # VPC-CNI does not SNAT in-VPC pod-to-pod traffic, so NATS on the
+          # system node sees the pod source IP (pod subnet), not the node IP.
+          cidrBlocks: ["10.0.128.0/17", "100.65.0.0/16"]  # worker node + pod subnet CIDRs
+          description: "Allow NATS from worker nodes and pods"
+
+compute:
+  eks:
+    nodeGroups:
+      system:
+        instanceType: m7i.xlarge
+        capacity:
+          desired: 3
+        labels:
+          nodeGroup: system-worker
+          dedicated: system-workload
+      workers:
+        - name: gpu-worker
+          instanceType: p6e-gb200.36xlarge
+          architecture: arm64               # GB200 = Grace/arm64 (arm64 Ubuntu EKS AMI)
+          gpuType: gb200
+          # REQUIRED: auto-derivation breaks on the "p6e-gb200.*" name — the
+          # actuator keys its accelerator map on "p6e" but split(".")[0] yields
+          # "p6e-gb200", so it falls through to the generic-GPU EFA branch and
+          # emits an invalid launch template (device_index=1 on a 1-device
+          # network card) → CreateAutoScalingGroup ValidationError. The explicit
+          # value selects the correct GB200 branch.
+          accelerator: gb200
+          capacity:
+            # Two GPU nodes: activates the multi-node NCCL East-West fabric
+            # check (nccl-all-reduce-bw-net) and lets the TrainJob (numNodes: 2)
+            # span both nodes.
+            desired: 2
+            reservation:
+              preference: capacity-reservations-only
+              target: cr-0e2f3833a602809a6
+              marketType: capacity-block    # REQUIRED — GB200 CRs are Capacity Blocks
+          labels:
+            nodeGroup: gpu-worker
+            dedicated: user-workload
+          taints:
+            - key: skyhook.nvidia.com
+              value: runtime-required
+              effect: NoSchedule
diff --git a/tests/uat/aws/tests/gb200-training-config.yaml b/tests/uat/aws/tests/gb200-training-config.yaml
