KWOK Cluster Validation #2676
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: KWOK Cluster Validation | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'recipes/**' | |
| - 'kwok/**' | |
| - 'tests/chainsaw/kwok/**' | |
| # The aicr binary that renders/bundles recipes is built from Go source; | |
| # a recipe-rendering regression or dependency bump must trigger KWOK e2e. | |
| - 'pkg/recipe/**' | |
| - 'pkg/bundler/**' | |
| - 'pkg/cli/**' | |
| - 'cmd/aicr/**' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'vendor/**' | |
| - '.github/workflows/kwok-recipes.yaml' | |
| - '.github/workflows/kwok-tier3-shard.yaml' | |
| - '.github/actions/kwok-test/**' | |
| - '!**.md' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - 'recipes/**' | |
| - 'kwok/**' | |
| - 'tests/chainsaw/kwok/**' | |
| # The aicr binary that renders/bundles recipes is built from Go source; | |
| # a recipe-rendering regression or dependency bump must trigger KWOK e2e. | |
| - 'pkg/recipe/**' | |
| - 'pkg/bundler/**' | |
| - 'pkg/cli/**' | |
| - 'cmd/aicr/**' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'vendor/**' | |
| - '.github/workflows/kwok-recipes.yaml' | |
| - '.github/workflows/kwok-tier3-shard.yaml' | |
| - '.github/actions/kwok-test/**' | |
| - '!**.md' | |
| schedule: | |
| # Nightly at 03:00 UTC — Tier 3 full matrix backstop | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| recipe: | |
| description: 'Specific recipe to test (leave empty to test all with service criteria)' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| # PR concurrency: cancel in-progress runs for the same PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| discover: | |
| name: Discover Recipes | |
| if: github.event_name != 'schedule' || github.repository == 'nvidia/aicr' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| tier1: ${{ steps.classify.outputs.tier1 }} | |
| tier2: ${{ steps.classify.outputs.tier2 }} | |
| tier3: ${{ steps.classify.outputs.tier3 }} | |
| tier3_batches: ${{ steps.classify.outputs.tier3_batches }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| # Full checkout needed for diff-aware Tier 2 discovery | |
| fetch-depth: 0 | |
| - name: Script unit tests (kwok/scripts/lib) | |
| run: bash kwok/scripts/lib/sync-budget_test.sh | |
| - name: Classify recipes into tiers | |
| id: classify | |
| shell: bash | |
| env: | |
| DISPATCH_RECIPE: ${{ github.event.inputs.recipe }} | |
| run: | | |
| set -euo pipefail | |
| # --- workflow_dispatch: test exactly the requested recipe --- | |
| if [[ -n "${DISPATCH_RECIPE}" ]]; then | |
| single=$(jq -nc '[$r]' --arg r "${DISPATCH_RECIPE}") | |
| echo "tier1=${single}" >> "$GITHUB_OUTPUT" | |
| echo "tier2=[]" >> "$GITHUB_OUTPUT" | |
| echo "tier3=[]" >> "$GITHUB_OUTPUT" | |
| echo "tier3_batches=[]" >> "$GITHUB_OUTPUT" | |
| echo "Manual dispatch: ${DISPATCH_RECIPE}" | |
| exit 0 | |
| fi | |
| # --- Scan all overlays and classify --- | |
| tier1="[]" # generic overlays (no accelerator specialization) | |
| all="[]" # every testable overlay | |
| for overlay in recipes/overlays/*.yaml; do | |
| name=$(basename "$overlay" .yaml) | |
| service=$(yq eval '.spec.criteria.service // ""' "$overlay" 2>/dev/null || true) | |
| # Skip non-testable overlays (no service, or OCP — needs OpenShift operators) | |
| if [[ -z "$service" || "$service" == "null" || "$service" == "any" || "$service" == "ocp" ]]; then | |
| continue | |
| fi | |
| all=$(echo "$all" | jq -c --arg r "$name" '. + [$r]') | |
| accel=$(yq eval '.spec.criteria.accelerator // ""' "$overlay" 2>/dev/null || true) | |
| if [[ -z "$accel" || "$accel" == "null" || "$accel" == "any" ]]; then | |
| tier1=$(echo "$tier1" | jq -c --arg r "$name" '. + [$r]') | |
| fi | |
| done | |
| # --- Tier 2: diff-aware accelerator tests (PR only) --- | |
| tier2="[]" | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| base_ref="${{ github.event.pull_request.base.sha }}" | |
| head_ref="${{ github.event.pull_request.head.sha }}" | |
| if ! changed_files=$(git diff --name-only "${base_ref}..${head_ref}" 2>&1); then | |
| echo "::error::git diff failed — cannot compute Tier 2 affected overlays" | |
| echo "git diff output: ${changed_files}" | |
| exit 1 | |
| fi | |
| # If registry.yaml or base.yaml changed, all accelerator overlays are | |
| # potentially affected — promote the full accelerator set to Tier 2. | |
| promote_all=false | |
| if echo "$changed_files" | grep -qE '^recipes/(registry\.yaml|overlays/base\.yaml)$'; then | |
| promote_all=true | |
| echo "registry.yaml or base.yaml changed — all accelerator overlays promoted to Tier 2" | |
| fi | |
| # Build set of affected accelerator-specific overlays | |
| for overlay in recipes/overlays/*.yaml; do | |
| name=$(basename "$overlay" .yaml) | |
| service=$(yq eval '.spec.criteria.service // ""' "$overlay" 2>/dev/null || true) | |
| accel=$(yq eval '.spec.criteria.accelerator // ""' "$overlay" 2>/dev/null || true) | |
| # Only accelerator-specific, KWOK-testable overlays belong in Tier 2 | |
| if [[ -z "$service" || "$service" == "null" || "$service" == "any" || "$service" == "ocp" ]]; then | |
| continue | |
| fi | |
| if [[ -z "$accel" || "$accel" == "null" || "$accel" == "any" ]]; then | |
| continue | |
| fi | |
| # If registry.yaml or base.yaml changed, include all accelerator overlays | |
| if [[ "$promote_all" == "true" ]]; then | |
| tier2=$(echo "$tier2" | jq -c --arg r "$name" '. + [$r]') | |
| continue | |
| fi | |
| include=false | |
| # Rule 1: overlay file itself changed | |
| if echo "$changed_files" | grep -qF "recipes/overlays/${name}.yaml"; then | |
| include=true | |
| fi | |
| # Rule 2: check if any ancestor overlay in the base chain changed | |
| if [[ "$include" == "false" ]]; then | |
| current="$overlay" | |
| while true; do | |
| parent=$(yq eval '.spec.base // ""' "$current" 2>/dev/null || true) | |
| if [[ -z "$parent" || "$parent" == "null" ]]; then | |
| break | |
| fi | |
| if echo "$changed_files" | grep -qF "recipes/overlays/${parent}.yaml"; then | |
| include=true | |
| break | |
| fi | |
| current="recipes/overlays/${parent}.yaml" | |
| if [[ ! -f "$current" ]]; then | |
| break | |
| fi | |
| done | |
| fi | |
| # Rule 3: a component values file referenced by this overlay changed | |
| if [[ "$include" == "false" ]]; then | |
| # Collect valuesFile references from this overlay and its base chain | |
| values_files="" | |
| current="$overlay" | |
| while true; do | |
| vf=$(yq eval '.spec.componentRefs[].valuesFile // ""' "$current" 2>/dev/null | grep -v '^$' || true) | |
| if [[ -n "$vf" ]]; then | |
| values_files="${values_files}"$'\n'"${vf}" | |
| fi | |
| parent=$(yq eval '.spec.base // ""' "$current" 2>/dev/null || true) | |
| if [[ -z "$parent" || "$parent" == "null" ]]; then | |
| break | |
| fi | |
| current="recipes/overlays/${parent}.yaml" | |
| if [[ ! -f "$current" ]]; then | |
| break | |
| fi | |
| done | |
| # Check if any referenced values file changed | |
| for vf in $values_files; do | |
| # valuesFile paths are relative to recipes/ | |
| if echo "$changed_files" | grep -qF "recipes/${vf}"; then | |
| include=true | |
| break | |
| fi | |
| done | |
| fi | |
| if [[ "$include" == "true" ]]; then | |
| tier2=$(echo "$tier2" | jq -c --arg r "$name" '. + [$r]') | |
| fi | |
| done | |
| fi | |
| # --- Tier 3: full matrix (all testable overlays) --- | |
| tier3="$all" | |
| # --- Tier 3 batching --- | |
| # GitHub caps a single job's matrix at 256 configurations. Tier 3 | |
| # crosses every testable recipe with every deployer, so the raw | |
| # cross-product (recipes × deployers) outgrew the cap. Split the | |
| # {recipe, deployer} pairs into batches of <= TIER3_BATCH_SIZE; the | |
| # caller fans each batch out to kwok-tier3-shard.yaml, keeping every | |
| # shard's matrix under the limit. Keep this deployer list in sync | |
| # with the test-tier1 matrix above and the input doc in | |
| # .github/actions/kwok-test/action.yml. | |
| deployers='["helm","argocd-oci","argocd-helm-oci","argocd-git","flux-oci","flux-git"]' | |
| readonly TIER3_BATCH_SIZE=200 # headroom under GitHub's 256 cap | |
| # Fail closed if the batch size is ever raised past the hard limit — | |
| # a too-large batch would otherwise resurface the opaque | |
| # "produced N configurations which exceeds the maximum of 256" error. | |
| if (( TIER3_BATCH_SIZE > 256 )); then | |
| echo "::error::TIER3_BATCH_SIZE=${TIER3_BATCH_SIZE} exceeds GitHub's 256-configuration matrix limit" | |
| exit 1 | |
| fi | |
| tier3_batches=$(jq -cn \ | |
| --argjson recipes "$tier3" \ | |
| --argjson deployers "$deployers" \ | |
| --argjson size "$TIER3_BATCH_SIZE" ' | |
| [ $recipes[] as $r | $deployers[] as $d | {recipe: $r, deployer: $d} ] | |
| | [ range(0; length; $size) as $i | |
| # id = zero-based batch index; used as the concurrency-group suffix | |
| | {id: ($i / $size | floor), pairs: .[$i:$i+$size]} ] | |
| ') | |
| # --- Output --- | |
| echo "tier1=${tier1}" >> "$GITHUB_OUTPUT" | |
| echo "tier2=${tier2}" >> "$GITHUB_OUTPUT" | |
| echo "tier3=${tier3}" >> "$GITHUB_OUTPUT" | |
| echo "tier3_batches=${tier3_batches}" >> "$GITHUB_OUTPUT" | |
| tier3_pairs=$(echo "$tier3_batches" | jq '[.[].pairs[]] | length') | |
| tier3_batch_count=$(echo "$tier3_batches" | jq 'length') | |
| echo "Tier 1 (generic): $(echo "$tier1" | jq 'length') recipe(s)" | |
| echo "Tier 2 (diff-aware): $(echo "$tier2" | jq 'length') recipe(s)" | |
| echo "Tier 3 (full matrix): $(echo "$tier3" | jq 'length') recipe(s) × $(echo "$deployers" | jq 'length') deployer(s) = ${tier3_pairs} pair(s) in ${tier3_batch_count} batch(es)" | |
| # ── Tier 1: PR gate — generic overlays (PR + push, skip on schedule) ── | |
| test-tier1: | |
| name: 'Tier 1: ${{ matrix.recipe }} (${{ matrix.deployer }})' | |
| needs: discover | |
| if: >- | |
| github.event_name != 'schedule' && | |
| needs.discover.outputs.tier1 != '[]' && | |
| needs.discover.outputs.tier1 != '' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 18 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| recipe: ${{ fromJSON(needs.discover.outputs.tier1) }} | |
| deployer: [helm, argocd-oci, argocd-helm-oci, argocd-git, flux-oci, flux-git] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| - name: Run KWOK test | |
| uses: ./.github/actions/kwok-test | |
| with: | |
| recipe: ${{ matrix.recipe }} | |
| deployer: ${{ matrix.deployer }} | |
| go_version: ${{ steps.versions.outputs.go }} | |
| goreleaser_version: ${{ steps.versions.outputs.goreleaser }} | |
| kind_version: ${{ steps.versions.outputs.kind }} | |
| helm_version: ${{ steps.versions.outputs.helm }} | |
| kwok_version: ${{ steps.versions.outputs.kwok }} | |
| kubectl_version: ${{ steps.versions.outputs.kubectl }} | |
| yq_version: ${{ steps.versions.outputs.yq }} | |
| flux_version: ${{ steps.versions.outputs.flux }} | |
| chainsaw_version: ${{ steps.versions.outputs.chainsaw }} | |
| chainsaw_sha256: ${{ steps.versions.outputs.chainsaw_sha256_linux_amd64 }} | |
| kind_node_image: ${{ steps.versions.outputs.kind_node_image }} | |
| # keep == this job's timeout-minutes (asserted by sync-budget_test.sh) | |
| job_timeout_minutes: '18' | |
| # ── Tier 2: diff-aware accelerator tests (PR only, conditional) ── | |
| test-tier2: | |
| name: 'Tier 2: ${{ matrix.recipe }}' | |
| needs: discover | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| needs.discover.outputs.tier2 != '[]' && | |
| needs.discover.outputs.tier2 != '' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 18 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| recipe: ${{ fromJSON(needs.discover.outputs.tier2) }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Load versions | |
| id: versions | |
| uses: ./.github/actions/load-versions | |
| - name: Run KWOK test | |
| uses: ./.github/actions/kwok-test | |
| with: | |
| recipe: ${{ matrix.recipe }} | |
| go_version: ${{ steps.versions.outputs.go }} | |
| goreleaser_version: ${{ steps.versions.outputs.goreleaser }} | |
| kind_version: ${{ steps.versions.outputs.kind }} | |
| helm_version: ${{ steps.versions.outputs.helm }} | |
| kwok_version: ${{ steps.versions.outputs.kwok }} | |
| kubectl_version: ${{ steps.versions.outputs.kubectl }} | |
| yq_version: ${{ steps.versions.outputs.yq }} | |
| flux_version: ${{ steps.versions.outputs.flux }} | |
| chainsaw_version: ${{ steps.versions.outputs.chainsaw }} | |
| chainsaw_sha256: ${{ steps.versions.outputs.chainsaw_sha256_linux_amd64 }} | |
| kind_node_image: ${{ steps.versions.outputs.kind_node_image }} | |
| # keep == this job's timeout-minutes (asserted by sync-budget_test.sh) | |
| job_timeout_minutes: '18' | |
| # ── Tier 3: full matrix (push to main + nightly schedule) ── | |
| # The recipe × deployer cross-product exceeds GitHub's 256-config matrix cap, | |
| # so discover splits it into batches and we fan each batch out to the | |
| # kwok-tier3-shard reusable workflow (one shard per batch, each <= 256). | |
| # Per ADR-003: the concurrency group is keyed by SHA so successive merges to | |
| # main never cancel in-flight Tier 3 runs; the batch id keeps every shard of a | |
| # single run in its own group so they all run in parallel. | |
| test-tier3: | |
| needs: discover | |
| concurrency: | |
| group: kwok-tier3-${{ github.sha }}-${{ matrix.batch.id }} | |
| cancel-in-progress: false | |
| if: >- | |
| (github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'nvidia/aicr')) && | |
| needs.discover.outputs.tier3_batches != '[]' && | |
| needs.discover.outputs.tier3_batches != '' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| batch: ${{ fromJSON(needs.discover.outputs.tier3_batches) }} | |
| uses: ./.github/workflows/kwok-tier3-shard.yaml | |
| with: | |
| pairs: ${{ toJSON(matrix.batch.pairs) }} | |
| # ── Summary: aggregate all tiers ── | |
| summary: | |
| name: KWOK Test Summary | |
| needs: [test-tier1, test-tier2, test-tier3] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| steps: | |
| - name: Check test results | |
| run: | | |
| echo "## KWOK Cluster Validation Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| tier1="${{ needs.test-tier1.result }}" | |
| tier2="${{ needs.test-tier2.result }}" | |
| tier3="${{ needs.test-tier3.result }}" | |
| echo "| Tier | Result |" >> $GITHUB_STEP_SUMMARY | |
| echo "|------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tier 1 (generic) | ${tier1} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tier 2 (diff-aware) | ${tier2} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Tier 3 (full matrix) | ${tier3} |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| failed=false | |
| # Tier 1 must pass (unless skipped on schedule-only with no recipes) | |
| if [[ "$tier1" == "failure" || "$tier1" == "cancelled" ]]; then | |
| echo "Tier 1 (generic) failed or was cancelled" >> $GITHUB_STEP_SUMMARY | |
| failed=true | |
| fi | |
| # Tier 2 must pass when it runs (skipped is OK — means no diff-affected overlays) | |
| if [[ "$tier2" == "failure" || "$tier2" == "cancelled" ]]; then | |
| echo "Tier 2 (diff-aware) failed or was cancelled" >> $GITHUB_STEP_SUMMARY | |
| failed=true | |
| fi | |
| # Tier 3 must pass on push/schedule (skipped is OK on PR) | |
| if [[ "$tier3" == "failure" || "$tier3" == "cancelled" ]]; then | |
| echo "Tier 3 (full matrix) failed or was cancelled" >> $GITHUB_STEP_SUMMARY | |
| failed=true | |
| fi | |
| if [[ "$failed" == "true" ]]; then | |
| exit 1 | |
| fi | |
| echo "All recipe validations passed" >> $GITHUB_STEP_SUMMARY |