UAT aws-h100 @ v0.17.0 #29674208952-1-aws-h100-3 #254
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: UAT Run | |
| # A distinct run title per dispatch so the nightly controller can identify the | |
| # run it dispatched — `gh workflow run` does not return a run id, so the | |
| # controller matches on this display title. The controller supplies a unique | |
| # dispatch_key (its run id + cell index) that is appended here, so a concurrent | |
| # manual dispatch of the same reservation/version can never be mistaken for the | |
| # controller's run. A manual dispatch omits the key and the suffix is dropped. | |
| # Quoted so YAML does not treat the ` #` in the dispatch_key suffix as a comment. | |
| run-name: "UAT ${{ inputs.reservation }} @ ${{ inputs.aicr_version || 'main' }}${{ inputs.dispatch_key != '' && format(' #{0}', inputs.dispatch_key) || '' }}" | |
| # The shared UAT dispatch surface and reservation-lease owner (#1274, DC1). | |
| # A human (workflow_dispatch) or the nightly batch (workflow_call) requests a | |
| # run against a named reservation; this workflow holds the per-reservation | |
| # concurrency lease, resolves the reservation row from the checked-in registry | |
| # (infra/uat/reservations.yaml) via the uat-broker helper, and dispatches the | |
| # cloud-appropriate reusable pipeline. Because every run for a reservation | |
| # routes through here and shares the lease, contending runs QUEUE instead of | |
| # racing the hardware. | |
| # | |
| # DC2 (#1275) adds two orthogonal dials, both forwarded to the cloud pipeline: | |
| # - intent (training|inference) selects the per-intent test config + | |
| # recipe criteria. | |
| # - lifecycle (nightly|daytime-up|daytime-down) selects the cluster | |
| # lifecycle: the nightly provision→CUJ→teardown, the daytime | |
| # provision-and-hold (stable lease-tagged name, no teardown), or the | |
| # evening teardown of that held daytime cluster. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| reservation: | |
| description: 'Reservation name from infra/uat/reservations.yaml (e.g. aws-h100, gcp-h100, azure-h100).' | |
| type: string | |
| required: true | |
| aicr_version: | |
| description: 'Released aicr version to test (e.g. v1.2.3); empty = build from source (main tip).' | |
| type: string | |
| default: '' | |
| dispatch_key: | |
| description: 'Unique correlation key set by the nightly controller so it can find this run; leave empty for manual runs.' | |
| type: string | |
| default: '' | |
| intent: | |
| description: 'Recipe intent — selects the per-intent test config (h100-<intent>-config.yaml) and recipe criteria.' | |
| type: choice | |
| options: [training, inference] | |
| default: training | |
| lifecycle: | |
| description: 'Cluster lifecycle. nightly: provision→CUJ→teardown. daytime-up: provision+deploy+HOLD (stable name, no teardown). daytime-down: tear down the held daytime cluster.' | |
| type: choice | |
| options: [nightly, daytime-up, daytime-down] | |
| default: nightly | |
| skip_delete: | |
| description: 'Skip cluster teardown (manual debugging only; ignored for daytime lifecycles).' | |
| type: boolean | |
| default: false | |
| skip_tests: | |
| description: 'Skip UAT test execution (manual debugging only).' | |
| type: boolean | |
| default: false | |
| workflow_call: | |
| inputs: | |
| reservation: | |
| type: string | |
| required: true | |
| aicr_version: | |
| type: string | |
| default: '' | |
| dispatch_key: | |
| type: string | |
| default: '' | |
| intent: | |
| type: string | |
| default: training | |
| lifecycle: | |
| type: string | |
| default: nightly | |
| skip_delete: | |
| type: boolean | |
| default: false | |
| skip_tests: | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| # The reservation lease. Every run targeting a given reservation serializes on | |
| # this group, so a contender waits rather than racing the reservation; | |
| # cancel-in-progress:false keeps an in-progress run (and its teardown) from | |
| # being killed. | |
| # | |
| # NOTE: this is a single-slot lease, not a full queue. GitHub concurrency holds | |
| # at most one in-progress + one PENDING run per group; a third contender | |
| # supersedes (cancels) the older pending one — cancel-in-progress:false only | |
| # protects the in-progress run, not the pending slot. Acceptable at launch (two | |
| # reservations, at most cron + one ad-hoc each); preserving every request would | |
| # need the deferred standing broker (see docs/contributor/uat.md, #1264). | |
| # | |
| # github.event.inputs is used for the dispatch path because the inputs context | |
| # is unreliable in a top-level concurrency group on workflow_dispatch | |
| # (community #45734/#35341); inputs.* covers the workflow_call path, where | |
| # github.event.inputs is empty. | |
| concurrency: | |
| group: uat-${{ github.event.inputs.reservation || inputs.reservation }} | |
| cancel-in-progress: false | |
| jobs: | |
| # Resolve the reservation row to the cloud + on-disk config paths the | |
| # cloud-specific pipeline consumes. A typo'd reservation name fails here | |
| # (uat-broker exits NOT_FOUND), so the registry is the single source of truth. | |
| resolve: | |
| if: github.repository == 'nvidia/aicr' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cloud: ${{ steps.row.outputs.cloud }} | |
| cluster_config_path: ${{ steps.row.outputs['cluster-config-path'] }} | |
| test_config_dir: ${{ steps.row.outputs['test-config-dir'] }} | |
| # Accelerator selects the per-accelerator test-config filename | |
| # (<accelerator>-<intent>-config.yaml) in the AWS pipeline. The AWS | |
| # account (tenancy) is NOT plumbed here: the AWS pipeline derives it from | |
| # the cluster-config's .deployment.tenancy, the single source of truth. | |
| accelerator: ${{ steps.row.outputs.accelerator }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| - name: Setup Go | |
| uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version: '${{ steps.versions.outputs.go }}' | |
| cache: true | |
| cache-dependency-path: | | |
| go.sum | |
| vendor/modules.txt | |
| - name: Resolve reservation row | |
| id: row | |
| env: | |
| GOFLAGS: -mod=vendor | |
| # Pass via env (not inline interpolation) to avoid expression | |
| # injection through the dispatch input. | |
| RESERVATION: ${{ inputs.reservation }} | |
| run: | | |
| set -euo pipefail | |
| go build -o ./bin/uat-broker ./tools/uat-broker | |
| ./bin/uat-broker reservations --name "$RESERVATION" >> "$GITHUB_OUTPUT" | |
| # One of run-aws / run-gcp / run-azure fires based on the resolved cloud | |
| # (the registry validates cloud membership, so at most one matches; the | |
| # unmapped-cloud job below fails closed if a registry cloud has no | |
| # run-<cloud> job here yet). The caller job grants the permission SUPERSET | |
| # its callee needs — a reusable workflow's jobs are ceilinged by the calling | |
| # job's permissions, and the called pipeline does OIDC, GHCR push, and | |
| # (nested) evidence-ingest GCS WIF. | |
| run-aws: | |
| needs: resolve | |
| if: needs.resolve.outputs.cloud == 'aws' | |
| # Superset ceiling for the nested pipeline (a reusable workflow's jobs | |
| # cannot exceed the caller's permissions). | |
| permissions: | |
| contents: read # checkout inside the called pipeline | |
| actions: read # required by the nested uat-{aws,gcp} + evidence-ingest jobs | |
| id-token: write # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish | |
| packages: write # push validator + snapshot-agent images to GHCR | |
| uses: ./.github/workflows/uat-aws.yaml | |
| with: | |
| reservation: ${{ inputs.reservation }} | |
| aicr_version: ${{ inputs.aicr_version }} | |
| intent: ${{ inputs.intent }} | |
| lifecycle: ${{ inputs.lifecycle }} | |
| cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }} | |
| test_config_dir: ${{ needs.resolve.outputs.test_config_dir }} | |
| accelerator: ${{ needs.resolve.outputs.accelerator }} | |
| skip_delete: ${{ inputs.skip_delete }} | |
| skip_tests: ${{ inputs.skip_tests }} | |
| run-gcp: | |
| needs: resolve | |
| if: needs.resolve.outputs.cloud == 'gcp' | |
| # Superset ceiling for the nested pipeline (a reusable workflow's jobs | |
| # cannot exceed the caller's permissions). | |
| permissions: | |
| contents: read # checkout inside the called pipeline | |
| actions: read # required by the nested uat-{aws,gcp} + evidence-ingest jobs | |
| id-token: write # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish | |
| packages: write # push validator + snapshot-agent images to GHCR | |
| uses: ./.github/workflows/uat-gcp.yaml | |
| with: | |
| reservation: ${{ inputs.reservation }} | |
| aicr_version: ${{ inputs.aicr_version }} | |
| intent: ${{ inputs.intent }} | |
| lifecycle: ${{ inputs.lifecycle }} | |
| cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }} | |
| test_config_dir: ${{ needs.resolve.outputs.test_config_dir }} | |
| skip_delete: ${{ inputs.skip_delete }} | |
| skip_tests: ${{ inputs.skip_tests }} | |
| run-azure: | |
| needs: resolve | |
| if: needs.resolve.outputs.cloud == 'azure' | |
| # Superset ceiling for the nested pipeline (a reusable workflow's jobs | |
| # cannot exceed the caller's permissions). | |
| permissions: | |
| contents: read # checkout inside the called pipeline | |
| actions: read # required by the nested uat-* + evidence-ingest jobs | |
| id-token: write # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish | |
| packages: write # push validator + snapshot-agent images to GHCR | |
| uses: ./.github/workflows/uat-azure.yaml | |
| with: | |
| reservation: ${{ inputs.reservation }} | |
| aicr_version: ${{ inputs.aicr_version }} | |
| intent: ${{ inputs.intent }} | |
| lifecycle: ${{ inputs.lifecycle }} | |
| cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }} | |
| test_config_dir: ${{ needs.resolve.outputs.test_config_dir }} | |
| skip_delete: ${{ inputs.skip_delete }} | |
| skip_tests: ${{ inputs.skip_tests }} | |
| # Fail closed on an unmapped cloud. The broker validates cloud membership | |
| # against its own set, but a cloud can be added there (and to the registry) | |
| # before its run-<cloud> job exists here — without this job the dispatch | |
| # would silently no-op: the lease is taken, no pipeline runs, and the | |
| # requester sees a green run that did nothing. | |
| unmapped-cloud: | |
| needs: resolve | |
| if: >- | |
| needs.resolve.outputs.cloud != 'aws' && | |
| needs.resolve.outputs.cloud != 'gcp' && | |
| needs.resolve.outputs.cloud != 'azure' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fail on unmapped cloud | |
| env: | |
| CLOUD: ${{ needs.resolve.outputs.cloud }} | |
| run: | | |
| set -euo pipefail | |
| echo "::error::reservation resolved to cloud '${CLOUD}', which has no run-<cloud> job in uat-run.yaml — add one (see run-azure for the shape)." | |
| exit 1 |