UAT Nightly Batch #23
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: UAT Nightly Batch | |
| # The night side of the day/night cycle (#1274, DC1 + DC5): on a nightly cron, | |
| # run the version matrix per reservation. For each reservation the controller | |
| # expands an ordered, time-boxed schedule — main first, then the previous N | |
| # stable releases in descending semver order (uat-broker schedule) — and runs | |
| # the cells SEQUENTIALLY, dispatching the shared uat-run.yaml per cell and | |
| # waiting for it before the next. Different reservations run in parallel | |
| # (independent hardware); uat-run.yaml's per-reservation lease serializes runs | |
| # that share a reservation. When the time-box closes, remaining cells are | |
| # dropped oldest-release-first (they are ordered oldest-last, so the controller | |
| # simply stops). Dispatch (not workflow_call) keeps uat-run.yaml top-level, so | |
| # its own lease + superseded observer apply and the reusable-nesting depth stays | |
| # at 3 (uat-run -> uat-{aws,gcp} -> evidence-ingest). | |
| # | |
| # aicr_version is threaded per cell; the release cells install the released | |
| # aicr + validator/agent images at that version (DC5), main builds from source. | |
| on: | |
| schedule: | |
| - cron: '0 4 * * *' # 04:00 UTC daily (evening in US Pacific); default branch only | |
| workflow_dispatch: | |
| inputs: | |
| previous_n: | |
| description: 'Previous stable releases below main to run per reservation (0 = main only).' | |
| type: string | |
| default: '1' | |
| deadline_offset_hours: | |
| description: 'Time-box: hours after batch start to stop dispatching new cells. Keep below the drive job timeout (GitHub caps a hosted job at 6h) so the graceful drop-oldest is reachable.' | |
| type: string | |
| default: '5' | |
| permissions: | |
| contents: read | |
| # Serialize batch runs: a manual dispatch should not fan out a second batch | |
| # alongside the cron one. (The per-reservation leases in uat-run.yaml would | |
| # queue the duplicates anyway, but there is no reason to start them.) | |
| concurrency: | |
| group: uat-nightly-batch | |
| cancel-in-progress: false | |
| jobs: | |
| # Enumerate reservation names from the registry into a JSON array for the | |
| # matrix below — keeps the batch data-driven (no per-reservation YAML). | |
| enumerate: | |
| if: github.repository == 'nvidia/aicr' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| reservations: ${{ steps.list.outputs.reservations }} | |
| # Single batch-start timestamp shared by every matrix leg, so the time-box | |
| # deadline is identical regardless of when each leg's runner starts. | |
| batch_start: ${{ steps.list.outputs.batch_start }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| - name: Setup Go | |
| uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version: '${{ steps.versions.outputs.go }}' | |
| cache: true | |
| cache-dependency-path: | | |
| go.sum | |
| vendor/modules.txt | |
| - name: Enumerate reservations | |
| id: list | |
| env: | |
| GOFLAGS: -mod=vendor | |
| run: | | |
| set -euo pipefail | |
| go build -o ./bin/uat-broker ./tools/uat-broker | |
| mapfile -t names < <(./bin/uat-broker reservations --list) | |
| if [ "${#names[@]}" -eq 0 ]; then | |
| echo "::error::no reservations found in infra/uat/reservations.yaml" | |
| exit 1 | |
| fi | |
| json=$(printf '%s\n' "${names[@]}" | jq -R . | jq -cs .) | |
| echo "reservations=${json}" >> "$GITHUB_OUTPUT" | |
| echo "Reservations: ${json}" | |
| # Capture the batch start once; every leg derives its deadline from this. | |
| echo "batch_start=$(date +%s)" >> "$GITHUB_OUTPUT" | |
| # One matrix leg per reservation (parallel). Each leg drives its own version | |
| # matrix sequentially. It only DISPATCHES uat-run.yaml (workflow_dispatch is | |
| # exempt from the GITHUB_TOKEN recursion rule and always creates a run), so | |
| # this job needs actions:write to dispatch + read to watch; the dispatched | |
| # uat-run.yaml runs top-level with its own permissions. | |
| drive: | |
| needs: enumerate | |
| runs-on: ubuntu-latest | |
| # The controller blocks on `gh run watch` per cell sequentially, so this job | |
| # runs as long as the batch. GitHub hard-caps a hosted job at 6h; set an | |
| # explicit timeout just under that so the job ends cleanly, and keep | |
| # deadline_offset_hours below it so the graceful drop-oldest fires first — | |
| # otherwise the job is killed mid-cell and the leg hard-fails. | |
| timeout-minutes: 350 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| reservation: ${{ fromJSON(needs.enumerate.outputs.reservations) }} | |
| permissions: | |
| contents: read # checkout + build uat-broker | |
| actions: write # dispatch uat-run.yaml and watch the dispatched runs | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 # full tag history for `git tag` -> uat-broker schedule | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| - name: Setup Go | |
| uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version: '${{ steps.versions.outputs.go }}' | |
| cache: true | |
| cache-dependency-path: | | |
| go.sum | |
| vendor/modules.txt | |
| - name: Run version-matrix cells (sequential, time-boxed) | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GOFLAGS: -mod=vendor | |
| REPO: ${{ github.repository }} | |
| REF: ${{ github.ref_name }} | |
| RESERVATION: ${{ matrix.reservation }} | |
| PREVIOUS_N: ${{ inputs.previous_n || '1' }} | |
| DEADLINE_OFFSET_HOURS: ${{ inputs.deadline_offset_hours || '5' }} | |
| BATCH_START: ${{ needs.enumerate.outputs.batch_start }} | |
| run: | | |
| set -euo pipefail | |
| # Poll budget for finding a dispatched run (gh workflow run returns no | |
| # run id): POLL_MAX_ATTEMPTS x POLL_INTERVAL_SECONDS seconds. | |
| POLL_MAX_ATTEMPTS=24 | |
| POLL_INTERVAL_SECONDS=5 | |
| go build -o ./bin/uat-broker ./tools/uat-broker | |
| # Nightly intents for this reservation (#1276, DC3). The batch runs | |
| # EACH listed intent (training and/or inference) as its own full CUJ | |
| # cell per version, dispatched SEQUENTIALLY through the shared | |
| # per-reservation lease — so both intents run nightly with no | |
| # contention and no second cron. The broker resolves an ABSENT | |
| # nightly-intents to the training default; an EXPLICIT empty list is | |
| # a bring-up opt-out (manual dispatch only) and resolves to an empty | |
| # value. Distinguish the two failure shapes: a MISSING key is a | |
| # broker/registry regression (fail closed); a present-but-EMPTY | |
| # value is the opt-out (skip this leg gracefully). | |
| ROW=$(./bin/uat-broker reservations --name "$RESERVATION") | |
| if ! grep -q '^nightly-intents=' <<<"$ROW"; then | |
| echo "::error::could not resolve nightly-intents for ${RESERVATION} (key missing from broker output)" >&2 | |
| exit 1 | |
| fi | |
| NIGHTLY_INTENTS_CSV=$(sed -n 's/^nightly-intents=//p' <<<"$ROW") | |
| if [[ -z "$NIGHTLY_INTENTS_CSV" ]]; then | |
| echo "::notice::Reservation ${RESERVATION} opts out of the nightly batch (nightly-intents: []); skipping this leg." | |
| exit 0 | |
| fi | |
| # The actual per-cell intents come from the schedule below, which | |
| # applies the reservation's nightly-intent-min-versions gate per | |
| # version; this line is just the leg-level summary of what's enrolled. | |
| echo "Nightly intents for ${RESERVATION}: ${NIGHTLY_INTENTS_CSV} (release cells further gated by nightly-intent-min-versions)" | |
| # Validate previous_n before the broker consumes it. `--previous-n` is | |
| # an int flag so it rejects non-integers, but it would accept a | |
| # negative and mis-shape the schedule; reject non-integer or negative | |
| # values here too, symmetric with the deadline_offset_hours guard. | |
| if [[ ! "$PREVIOUS_N" =~ ^[0-9]+$ ]]; then | |
| echo "::error::previous_n must be a non-negative integer; got '${PREVIOUS_N}'." >&2 | |
| exit 1 | |
| fi | |
| # Ordered cells for this reservation: main first, then the previous-N | |
| # stable releases in descending semver order (oldest last). | |
| schedule=$(git tag -l 'v*' | ./bin/uat-broker schedule \ | |
| --reservations "$RESERVATION" --previous-n "$PREVIOUS_N") | |
| mapfile -t versions < <(jq -r --arg r "$RESERVATION" '.[$r][].aicr_version' <<<"$schedule") | |
| echo "Cells for ${RESERVATION}: ${versions[*]:-<none>}" | |
| # Reject a non-integer / negative deadline_offset_hours before the | |
| # arithmetic below — workflow_dispatch passes it as a free-form string | |
| # and Bash would silently accept e.g. -1, moving the cutoff before | |
| # batch start and dropping every cell immediately. | |
| if [[ ! "$DEADLINE_OFFSET_HOURS" =~ ^[0-9]+$ ]]; then | |
| echo "::error::deadline_offset_hours must be a non-negative integer; got '${DEADLINE_OFFSET_HOURS}'." >&2 | |
| exit 1 | |
| fi | |
| # Anchor the time-box to the single batch-start timestamp (captured | |
| # once in the enumerate job) so every reservation leg shares one cutoff | |
| # regardless of when its runner started. | |
| deadline=$(( BATCH_START + DEADLINE_OFFSET_HOURS * 3600 )) | |
| cell=0 | |
| leg_failed="" | |
| # Version outer-loop, intent inner-loop: `main` runs every intent | |
| # before any release cell, so a time-box drop always sheds the OLDEST | |
| # release cells first — never main's inference. Each (version × intent) | |
| # is a full provision→CUJ→teardown dispatched through the shared lease, | |
| # so the intents serialize automatically (no contention). | |
| # | |
| # Intents are per-CELL: the schedule already applied each | |
| # reservation's nightly-intent-min-versions gate, so a release that | |
| # predates an intent's minimum version carries a shorter (or empty) | |
| # .intents list and that (version × intent) never dispatches; `main` | |
| # always carries every listed intent. | |
| for ver in "${versions[@]}"; do | |
| mapfile -t cell_intents < <(jq -r --arg r "$RESERVATION" --arg v "$ver" \ | |
| '.[$r][] | select(.aicr_version == $v) | .intents[]' <<<"$schedule") | |
| # A fully-gated release cell (no eligible intents) contributes | |
| # nothing — skip it without consuming the time-box or cell counter. | |
| (( ${#cell_intents[@]} == 0 )) && continue | |
| for intent in "${cell_intents[@]}"; do | |
| if (( $(date +%s) >= deadline )); then | |
| echo "::notice::Time-box reached — dropping remaining cells for ${RESERVATION} (oldest release first)." | |
| break 2 | |
| fi | |
| cell=$((cell + 1)) | |
| label="${ver:-main}" | |
| # Unique per-dispatch key (controller run id + attempt + reservation | |
| # + cell index) so the resolver below watches THIS run and never a | |
| # concurrent manual dispatch of the same reservation/version. The | |
| # cell index increments per (version × intent), so the two intents | |
| # of one version get distinct keys and distinct run titles. | |
| dispatch_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RESERVATION}-${cell}" | |
| title="UAT ${RESERVATION} @ ${label} #${dispatch_key}" | |
| echo "::group::${title} [intent=${intent}]" | |
| gh workflow run uat-run.yaml --repo "$REPO" --ref "$REF" \ | |
| -f reservation="$RESERVATION" -f aicr_version="$ver" \ | |
| -f intent="$intent" \ | |
| -f dispatch_key="$dispatch_key" | |
| # Resolve the dispatched run by its run-name — this title MUST match | |
| # uat-run.yaml's `run-name:` (which appends the same dispatch_key). | |
| # dispatch_key makes the title globally unique, so an exact title | |
| # match is unambiguous; no createdAt filter (a runner clock running | |
| # ahead of GitHub's API clock would falsely reject this dispatch). | |
| run_id="" | |
| for _ in $(seq 1 "$POLL_MAX_ATTEMPTS"); do | |
| sleep "$POLL_INTERVAL_SECONDS" | |
| run_id=$(gh run list --repo "$REPO" --workflow uat-run.yaml \ | |
| --event workflow_dispatch --limit 50 \ | |
| --json databaseId,displayTitle,createdAt 2>/dev/null \ | |
| | jq -r --arg t "$title" \ | |
| 'map(select(.displayTitle == $t)) | |
| | sort_by(.createdAt) | .[-1].databaseId // empty' || true) | |
| [[ -n "$run_id" ]] && break | |
| done | |
| if [[ -z "$run_id" ]]; then | |
| # Fail the leg rather than continue: an unwatched run can still | |
| # hold the reservation's pending slot and let a later (lower- | |
| # priority) cell supersede it, breaking version ordering. | |
| echo "::error::Dispatched ${title} but could not resolve its run id; stopping this reservation leg to preserve version ordering." | |
| echo "::endgroup::" | |
| exit 1 | |
| fi | |
| echo "Waiting on run ${run_id} ..." | |
| # --exit-status is intentionally not fatal here; classify the run's | |
| # conclusion below so we can distinguish a benign superseded run from | |
| # a real failure without aborting mid-loop. | |
| # Output is discarded: in a non-TTY log `gh run watch` APPENDS a full | |
| # status render every refresh (no in-place redraw), which flooded the | |
| # drive log with tens of thousands of duplicate lines per multi-hour | |
| # cell. We only need it to BLOCK until the run finishes; the | |
| # conclusion is read from `gh run view` below, so the live render is | |
| # redundant. --interval 60 keeps the polling gentle on the API. | |
| gh run watch "$run_id" --repo "$REPO" --interval 60 --exit-status >/dev/null 2>&1 || true | |
| conclusion=$(gh run view "$run_id" --repo "$REPO" --json conclusion --jq '.conclusion' 2>/dev/null || echo "") | |
| njobs=$(gh api "repos/${REPO}/actions/runs/${run_id}/jobs" --jq '.total_count' 2>/dev/null || echo "") | |
| if [[ "$conclusion" == "cancelled" && "$njobs" == "0" ]]; then | |
| # Benign: the single-slot reservation lease dropped a pending run. | |
| # Surface it (must not be silent) but do not fail the leg — the | |
| # superseded-run observer complements this (DC1 superseded surfacing). | |
| echo "::warning::${title} [intent=${intent}] was superseded while pending (dropped by the reservation lease); re-dispatch when the reservation frees." | |
| elif [[ "$conclusion" != "success" ]]; then | |
| # Real cell failure/timeout — record it so the leg (and thus the | |
| # nightly batch) fails, but keep testing the remaining cells. | |
| echo "::error::${title} [intent=${intent}] finished with conclusion '${conclusion:-unknown}'." | |
| leg_failed=1 | |
| fi | |
| echo "::endgroup::" | |
| done | |
| done | |
| if [[ -n "$leg_failed" ]]; then | |
| echo "::error::One or more version cells failed for ${RESERVATION}." | |
| exit 1 | |
| fi |