kwok-recipes.yaml

# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: KWOK Cluster Validation

on:
  push:
    branches:
      - main
    paths:
      - 'recipes/**'
      - 'kwok/**'
      - 'tests/chainsaw/kwok/**'
      # The aicr binary that renders/bundles recipes is built from Go source;
      # a recipe-rendering regression or dependency bump must trigger KWOK e2e.
      - 'pkg/recipe/**'
      - 'pkg/bundler/**'
      - 'pkg/cli/**'
      - 'cmd/aicr/**'
      - 'go.mod'
      - 'go.sum'
      - 'vendor/**'
      - '.github/workflows/kwok-recipes.yaml'
      - '.github/workflows/kwok-tier3-shard.yaml'
      - '.github/actions/kwok-test/**'
      - '!**.md'
  pull_request:
    branches:
      - main
    paths:
      - 'recipes/**'
      - 'kwok/**'
      - 'tests/chainsaw/kwok/**'
      # The aicr binary that renders/bundles recipes is built from Go source;
      # a recipe-rendering regression or dependency bump must trigger KWOK e2e.
      - 'pkg/recipe/**'
      - 'pkg/bundler/**'
      - 'pkg/cli/**'
      - 'cmd/aicr/**'
      - 'go.mod'
      - 'go.sum'
      - 'vendor/**'
      - '.github/workflows/kwok-recipes.yaml'
      - '.github/workflows/kwok-tier3-shard.yaml'
      - '.github/actions/kwok-test/**'
      - '!**.md'
  schedule:
    # Nightly at 03:00 UTC — Tier 3 full matrix backstop
    - cron: '0 3 * * *'
  workflow_dispatch:
    inputs:
      recipe:
        description: 'Specific recipe to test (leave empty to test all with service criteria)'
        required: false
        type: string

permissions:
  contents: read

# PR concurrency: cancel in-progress runs for the same PR
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  discover:
    name: Discover Recipes
    if: github.event_name != 'schedule' || github.repository == 'nvidia/aicr'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      tier1: ${{ steps.classify.outputs.tier1 }}
      tier2: ${{ steps.classify.outputs.tier2 }}
      tier3: ${{ steps.classify.outputs.tier3 }}
      tier3_batches: ${{ steps.classify.outputs.tier3_batches }}
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false
          # Full checkout needed for diff-aware Tier 2 discovery
          fetch-depth: 0

      - name: Script unit tests (kwok/scripts/lib)
        run: bash kwok/scripts/lib/sync-budget_test.sh

      - name: Classify recipes into tiers
        id: classify
        shell: bash
        env:
          DISPATCH_RECIPE: ${{ github.event.inputs.recipe }}
        run: |
          set -euo pipefail

          # --- workflow_dispatch: test exactly the requested recipe ---
          if [[ -n "${DISPATCH_RECIPE}" ]]; then
            single=$(jq -nc '[$r]' --arg r "${DISPATCH_RECIPE}")
            echo "tier1=${single}"   >> "$GITHUB_OUTPUT"
            echo "tier2=[]"          >> "$GITHUB_OUTPUT"
            echo "tier3=[]"          >> "$GITHUB_OUTPUT"
            echo "tier3_batches=[]"  >> "$GITHUB_OUTPUT"
            echo "Manual dispatch: ${DISPATCH_RECIPE}"
            exit 0
          fi

          # --- Scan all overlays and classify ---
          tier1="[]"   # generic overlays (no accelerator specialization)
          all="[]"     # every testable overlay

          for overlay in recipes/overlays/*.yaml; do
            name=$(basename "$overlay" .yaml)
            service=$(yq eval '.spec.criteria.service // ""' "$overlay" 2>/dev/null || true)

            # Skip non-testable overlays (no service, or OCP — needs OpenShift operators)
            if [[ -z "$service" || "$service" == "null" || "$service" == "any" || "$service" == "ocp" ]]; then
              continue
            fi

            all=$(echo "$all" | jq -c --arg r "$name" '. + [$r]')

            accel=$(yq eval '.spec.criteria.accelerator // ""' "$overlay" 2>/dev/null || true)
            if [[ -z "$accel" || "$accel" == "null" || "$accel" == "any" ]]; then
              tier1=$(echo "$tier1" | jq -c --arg r "$name" '. + [$r]')
            fi
          done

          # --- Tier 2: diff-aware accelerator tests (PR only) ---
          tier2="[]"

          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            base_ref="${{ github.event.pull_request.base.sha }}"
            head_ref="${{ github.event.pull_request.head.sha }}"
            if ! changed_files=$(git diff --name-only "${base_ref}..${head_ref}" 2>&1); then
              echo "::error::git diff failed — cannot compute Tier 2 affected overlays"
              echo "git diff output: ${changed_files}"
              exit 1
            fi

            # If registry.yaml or base.yaml changed, all accelerator overlays are
            # potentially affected — promote the full accelerator set to Tier 2.
            promote_all=false
            if echo "$changed_files" | grep -qE '^recipes/(registry\.yaml|overlays/base\.yaml)$'; then
              promote_all=true
              echo "registry.yaml or base.yaml changed — all accelerator overlays promoted to Tier 2"
            fi

            # Build set of affected accelerator-specific overlays
            for overlay in recipes/overlays/*.yaml; do
              name=$(basename "$overlay" .yaml)
              service=$(yq eval '.spec.criteria.service // ""' "$overlay" 2>/dev/null || true)
              accel=$(yq eval '.spec.criteria.accelerator // ""' "$overlay" 2>/dev/null || true)

              # Only accelerator-specific, KWOK-testable overlays belong in Tier 2
              if [[ -z "$service" || "$service" == "null" || "$service" == "any" || "$service" == "ocp" ]]; then
                continue
              fi
              if [[ -z "$accel" || "$accel" == "null" || "$accel" == "any" ]]; then
                continue
              fi

              # If registry.yaml or base.yaml changed, include all accelerator overlays
              if [[ "$promote_all" == "true" ]]; then
                tier2=$(echo "$tier2" | jq -c --arg r "$name" '. + [$r]')
                continue
              fi

              include=false

              # Rule 1: overlay file itself changed
              if echo "$changed_files" | grep -qF "recipes/overlays/${name}.yaml"; then
                include=true
              fi

              # Rule 2: check if any ancestor overlay in the base chain changed
              if [[ "$include" == "false" ]]; then
                current="$overlay"
                while true; do
                  parent=$(yq eval '.spec.base // ""' "$current" 2>/dev/null || true)
                  if [[ -z "$parent" || "$parent" == "null" ]]; then
                    break
                  fi
                  if echo "$changed_files" | grep -qF "recipes/overlays/${parent}.yaml"; then
                    include=true
                    break
                  fi
                  current="recipes/overlays/${parent}.yaml"
                  if [[ ! -f "$current" ]]; then
                    break
                  fi
                done
              fi

              # Rule 3: a component values file referenced by this overlay changed
              if [[ "$include" == "false" ]]; then
                # Collect valuesFile references from this overlay and its base chain
                values_files=""
                current="$overlay"
                while true; do
                  vf=$(yq eval '.spec.componentRefs[].valuesFile // ""' "$current" 2>/dev/null | grep -v '^$' || true)
                  if [[ -n "$vf" ]]; then
                    values_files="${values_files}"$'\n'"${vf}"
                  fi
                  parent=$(yq eval '.spec.base // ""' "$current" 2>/dev/null || true)
                  if [[ -z "$parent" || "$parent" == "null" ]]; then
                    break
                  fi
                  current="recipes/overlays/${parent}.yaml"
                  if [[ ! -f "$current" ]]; then
                    break
                  fi
                done

                # Check if any referenced values file changed
                for vf in $values_files; do
                  # valuesFile paths are relative to recipes/
                  if echo "$changed_files" | grep -qF "recipes/${vf}"; then
                    include=true
                    break
                  fi
                done
              fi

              if [[ "$include" == "true" ]]; then
                tier2=$(echo "$tier2" | jq -c --arg r "$name" '. + [$r]')
              fi
            done
          fi

          # --- Tier 3: full matrix (all testable overlays) ---
          tier3="$all"

          # --- Tier 3 batching ---
          # GitHub caps a single job's matrix at 256 configurations. Tier 3
          # crosses every testable recipe with every deployer, so the raw
          # cross-product (recipes × deployers) outgrew the cap. Split the
          # {recipe, deployer} pairs into batches of <= TIER3_BATCH_SIZE; the
          # caller fans each batch out to kwok-tier3-shard.yaml, keeping every
          # shard's matrix under the limit. Keep this deployer list in sync
          # with the test-tier1 matrix above and the input doc in
          # .github/actions/kwok-test/action.yml.
          deployers='["helm","argocd-oci","argocd-helm-oci","argocd-git","flux-oci","flux-git"]'
          readonly TIER3_BATCH_SIZE=200  # headroom under GitHub's 256 cap

          # Fail closed if the batch size is ever raised past the hard limit —
          # a too-large batch would otherwise resurface the opaque
          # "produced N configurations which exceeds the maximum of 256" error.
          if (( TIER3_BATCH_SIZE > 256 )); then
            echo "::error::TIER3_BATCH_SIZE=${TIER3_BATCH_SIZE} exceeds GitHub's 256-configuration matrix limit"
            exit 1
          fi

          tier3_batches=$(jq -cn \
            --argjson recipes "$tier3" \
            --argjson deployers "$deployers" \
            --argjson size "$TIER3_BATCH_SIZE" '
              [ $recipes[] as $r | $deployers[] as $d | {recipe: $r, deployer: $d} ]
              | [ range(0; length; $size) as $i
                  # id = zero-based batch index; used as the concurrency-group suffix
                  | {id: ($i / $size | floor), pairs: .[$i:$i+$size]} ]
            ')

          # --- Output ---
          echo "tier1=${tier1}" >> "$GITHUB_OUTPUT"
          echo "tier2=${tier2}" >> "$GITHUB_OUTPUT"
          echo "tier3=${tier3}" >> "$GITHUB_OUTPUT"
          echo "tier3_batches=${tier3_batches}" >> "$GITHUB_OUTPUT"

          tier3_pairs=$(echo "$tier3_batches" | jq '[.[].pairs[]] | length')
          tier3_batch_count=$(echo "$tier3_batches" | jq 'length')
          echo "Tier 1 (generic):      $(echo "$tier1" | jq 'length') recipe(s)"
          echo "Tier 2 (diff-aware):   $(echo "$tier2" | jq 'length') recipe(s)"
          echo "Tier 3 (full matrix):  $(echo "$tier3" | jq 'length') recipe(s) × $(echo "$deployers" | jq 'length') deployer(s) = ${tier3_pairs} pair(s) in ${tier3_batch_count} batch(es)"

  # ── Tier 1: PR gate — generic overlays (PR + push, skip on schedule) ──
  test-tier1:
    name: 'Tier 1: ${{ matrix.recipe }} (${{ matrix.deployer }})'
    needs: discover
    if: >-
      github.event_name != 'schedule' &&
      needs.discover.outputs.tier1 != '[]' &&
      needs.discover.outputs.tier1 != ''
    runs-on: ubuntu-latest
    timeout-minutes: 18
    strategy:
      fail-fast: false
      matrix:
        recipe: ${{ fromJSON(needs.discover.outputs.tier1) }}
        deployer: [helm, argocd-oci, argocd-helm-oci, argocd-git, flux-oci, flux-git]
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Load versions
        id: versions
        uses: ./.github/actions/load-versions

      - name: Run KWOK test
        uses: ./.github/actions/kwok-test
        with:
          recipe: ${{ matrix.recipe }}
          deployer: ${{ matrix.deployer }}
          go_version: ${{ steps.versions.outputs.go }}
          goreleaser_version: ${{ steps.versions.outputs.goreleaser }}
          kind_version: ${{ steps.versions.outputs.kind }}
          helm_version: ${{ steps.versions.outputs.helm }}
          kwok_version: ${{ steps.versions.outputs.kwok }}
          kubectl_version: ${{ steps.versions.outputs.kubectl }}
          yq_version: ${{ steps.versions.outputs.yq }}
          flux_version: ${{ steps.versions.outputs.flux }}
          chainsaw_version: ${{ steps.versions.outputs.chainsaw }}
          chainsaw_sha256: ${{ steps.versions.outputs.chainsaw_sha256_linux_amd64 }}
          kind_node_image: ${{ steps.versions.outputs.kind_node_image }}
          # keep == this job's timeout-minutes (asserted by sync-budget_test.sh)
          job_timeout_minutes: '18'

  # ── Tier 2: diff-aware accelerator tests (PR only, conditional) ──
  test-tier2:
    name: 'Tier 2: ${{ matrix.recipe }}'
    needs: discover
    if: >-
      github.event_name == 'pull_request' &&
      needs.discover.outputs.tier2 != '[]' &&
      needs.discover.outputs.tier2 != ''
    runs-on: ubuntu-latest
    timeout-minutes: 18
    strategy:
      fail-fast: false
      matrix:
        recipe: ${{ fromJSON(needs.discover.outputs.tier2) }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          persist-credentials: false

      - name: Load versions
        id: versions
        uses: ./.github/actions/load-versions

      - name: Run KWOK test
        uses: ./.github/actions/kwok-test
        with:
          recipe: ${{ matrix.recipe }}
          go_version: ${{ steps.versions.outputs.go }}
          goreleaser_version: ${{ steps.versions.outputs.goreleaser }}
          kind_version: ${{ steps.versions.outputs.kind }}
          helm_version: ${{ steps.versions.outputs.helm }}
          kwok_version: ${{ steps.versions.outputs.kwok }}
          kubectl_version: ${{ steps.versions.outputs.kubectl }}
          yq_version: ${{ steps.versions.outputs.yq }}
          flux_version: ${{ steps.versions.outputs.flux }}
          chainsaw_version: ${{ steps.versions.outputs.chainsaw }}
          chainsaw_sha256: ${{ steps.versions.outputs.chainsaw_sha256_linux_amd64 }}
          kind_node_image: ${{ steps.versions.outputs.kind_node_image }}
          # keep == this job's timeout-minutes (asserted by sync-budget_test.sh)
          job_timeout_minutes: '18'

  # ── Tier 3: full matrix (push to main + nightly schedule) ──
  # The recipe × deployer cross-product exceeds GitHub's 256-config matrix cap,
  # so discover splits it into batches and we fan each batch out to the
  # kwok-tier3-shard reusable workflow (one shard per batch, each <= 256).
  # Per ADR-003: the concurrency group is keyed by SHA so successive merges to
  # main never cancel in-flight Tier 3 runs; the batch id keeps every shard of a
  # single run in its own group so they all run in parallel.
  test-tier3:
    needs: discover
    concurrency:
      group: kwok-tier3-${{ github.sha }}-${{ matrix.batch.id }}
      cancel-in-progress: false
    if: >-
      (github.event_name == 'push' || (github.event_name == 'schedule' && github.repository == 'nvidia/aicr')) &&
      needs.discover.outputs.tier3_batches != '[]' &&
      needs.discover.outputs.tier3_batches != ''
    strategy:
      fail-fast: false
      matrix:
        batch: ${{ fromJSON(needs.discover.outputs.tier3_batches) }}
    uses: ./.github/workflows/kwok-tier3-shard.yaml
    with:
      pairs: ${{ toJSON(matrix.batch.pairs) }}

  # ── Summary: aggregate all tiers ──
  summary:
    name: KWOK Test Summary
    needs: [test-tier1, test-tier2, test-tier3]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    if: always()
    steps:
      - name: Check test results
        run: |
          echo "## KWOK Cluster Validation Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          tier1="${{ needs.test-tier1.result }}"
          tier2="${{ needs.test-tier2.result }}"
          tier3="${{ needs.test-tier3.result }}"

          echo "| Tier | Result |" >> $GITHUB_STEP_SUMMARY
          echo "|------|--------|" >> $GITHUB_STEP_SUMMARY
          echo "| Tier 1 (generic) | ${tier1} |" >> $GITHUB_STEP_SUMMARY
          echo "| Tier 2 (diff-aware) | ${tier2} |" >> $GITHUB_STEP_SUMMARY
          echo "| Tier 3 (full matrix) | ${tier3} |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          failed=false

          # Tier 1 must pass (unless skipped on schedule-only with no recipes)
          if [[ "$tier1" == "failure" || "$tier1" == "cancelled" ]]; then
            echo "Tier 1 (generic) failed or was cancelled" >> $GITHUB_STEP_SUMMARY
            failed=true
          fi

          # Tier 2 must pass when it runs (skipped is OK — means no diff-affected overlays)
          if [[ "$tier2" == "failure" || "$tier2" == "cancelled" ]]; then
            echo "Tier 2 (diff-aware) failed or was cancelled" >> $GITHUB_STEP_SUMMARY
            failed=true
          fi

          # Tier 3 must pass on push/schedule (skipped is OK on PR)
          if [[ "$tier3" == "failure" || "$tier3" == "cancelled" ]]; then
            echo "Tier 3 (full matrix) failed or was cancelled" >> $GITHUB_STEP_SUMMARY
            failed=true
          fi

          if [[ "$failed" == "true" ]]; then
            exit 1
          fi

          echo "All recipe validations passed" >> $GITHUB_STEP_SUMMARY