test-playbooks.yml

# Copyright Advanced Micro Devices, Inc.
# 
# SPDX-License-Identifier: MIT

name: Test Playbooks

on:
  schedule:
    - cron: '0 1 * * *'
  workflow_dispatch:
    inputs:
      playbook_id:
        description: 'Specific playbook ID to test (leave empty to test all playbooks)'
        required: false
        type: string
  pull_request:
    paths:
      - 'playbooks/**'

permissions:
  contents: read
  # Required so the failure-reporting step can open issues on behalf of the
  # workflow. Note we still gate the issue creation step itself on
  # github.ref == 'refs/heads/main' so PR runs never create issues.
  issues: write

jobs:
  detect-changes:
    name: Detect Changed Playbooks
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.build-matrix.outputs.matrix }}
      has_entries: ${{ steps.build-matrix.outputs.has_entries }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Detect changed playbooks
        id: detect
        run: |
          if [ -n "${{ github.event.inputs.playbook_id }}" ]; then
            echo "playbooks=[\"${{ github.event.inputs.playbook_id }}\"]" >> $GITHUB_OUTPUT
          elif [ "${{ github.event_name }}" = "pull_request" ]; then
            CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} -- 'playbooks/core/*' 'playbooks/supplemental/*' 2>/dev/null || echo "")
            
            PLAYBOOK_IDS=$(echo "$CHANGED_FILES" | grep -E '^playbooks/(core|supplemental)/[^/]+/' | sed 's|playbooks/[^/]*/\([^/]*\)/.*|\1|' | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))')
            
            if [ "$PLAYBOOK_IDS" = "[]" ] || [ -z "$PLAYBOOK_IDS" ]; then
              echo "playbooks=[]" >> $GITHUB_OUTPUT
            else
              echo "playbooks=$PLAYBOOK_IDS" >> $GITHUB_OUTPUT
            fi
          else
            ALL_PLAYBOOKS=$(find playbooks/core playbooks/supplemental -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sed 's|playbooks/[^/]*/||' | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))')
            
            if [ "$ALL_PLAYBOOKS" = "[]" ] || [ -z "$ALL_PLAYBOOKS" ]; then
              echo "playbooks=[]" >> $GITHUB_OUTPUT
            else
              echo "playbooks=$ALL_PLAYBOOKS" >> $GITHUB_OUTPUT
            fi
          fi

      - name: Build test matrix from playbook metadata
        id: build-matrix
        env:
          PLAYBOOKS: ${{ steps.detect.outputs.playbooks }}
        run: |
          if [ "$PLAYBOOKS" = "[]" ] || [ -z "$PLAYBOOKS" ]; then
            echo "matrix=[]" >> $GITHUB_OUTPUT
            echo "has_entries=false" >> $GITHUB_OUTPUT
            exit 0
          fi

          MATRIX=$(python3 << 'PYEOF'
          import json, os
          from pathlib import Path

          playbooks = json.loads(os.environ["PLAYBOOKS"])
          matrix = []

          for pb_id in playbooks:
              for cat in ["core", "supplemental"]:
                  pb_file = Path(f"playbooks/{cat}/{pb_id}/playbook.json")
                  if pb_file.exists():
                      meta = json.loads(pb_file.read_text())
                      tested = meta.get("tested_platforms", {})
                      if not tested:
                          continue
                      required = meta.get("required_platforms", {})
                      for device, platform_list in tested.items():
                          required_platforms = set(required.get(device, []))
                          for platform in platform_list:
                              os_label = "Windows" if platform == "windows" else "Linux"
                              matrix.append({
                                  "playbook": pb_id,
                                  "platform": platform,
                                  "arch": device,
                                  "runner": json.dumps(["self-hosted", os_label, device]),
                                  "required": platform in required_platforms,
                              })
                      break

          print(json.dumps(matrix))
          PYEOF
          )

          if [ "$MATRIX" = "[]" ] || [ -z "$MATRIX" ]; then
            echo "matrix=[]" >> $GITHUB_OUTPUT
            echo "has_entries=false" >> $GITHUB_OUTPUT
          else
            echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
            echo "has_entries=true" >> $GITHUB_OUTPUT
          fi

      - name: Display test matrix
        run: |
          echo "Test matrix: ${{ steps.build-matrix.outputs.matrix }}"

  test-playbooks:
    name: "Test ${{ matrix.playbook }} (${{ matrix.platform }}/${{ matrix.arch }})${{ matrix.required && ' ✦' || '' }}"
    needs: detect-changes
    if: needs.detect-changes.outputs.has_entries == 'true'
    runs-on: ${{ fromJson(matrix.runner) }}
    continue-on-error: ${{ !matrix.required }}
    env:
      PYTHONUTF8: 1
      PYTHONIOENCODING: utf-8
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.detect-changes.outputs.matrix) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ (matrix.playbook == 'open-webui-chat' || matrix.playbook == 'vllm-inference') && '3.12' || '3.13' }}

      - name: Install test dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pyyaml

      - name: Extract and run tests (Windows)
        if: matrix.platform == 'windows'
        id: run_tests_windows
        shell: powershell
        run: |
          python .github/scripts/run_playbook_tests.py --playbook "${{ matrix.playbook }}" --platform windows --device "${{ matrix.arch }}"

      - name: Extract and run tests (Linux)
        if: matrix.platform == 'linux'
        id: run_tests_linux
        shell: bash
        run: |
          python .github/scripts/run_playbook_tests.py --playbook "${{ matrix.playbook }}" --platform linux --device "${{ matrix.arch }}"

      - name: Upload test artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ matrix.playbook }}-${{ matrix.platform }}-${{ matrix.arch }}
          path: test-results/
          if-no-files-found: ignore

      # Open / update a tracking issue for every failed test, but ONLY on the
      # main branch. PRs (whose base is main) intentionally don't trigger this:
      # those failures are usually the PR author's responsibility and would
      # otherwise spam the tracker. `failure()` only fires when a previous step
      # in this job failed, so passing matrix entries are skipped automatically.
      #
      # We invoke python directly (no shell-specific line continuations) so the
      # same step works on both Windows and Linux self-hosted runners.
      - name: Create failure issues (Linux)
        if: failure() && matrix.platform == 'linux' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RUNNER_NAME: ${{ runner.name }}
        run: |
          python .github/scripts/create_failure_issues.py \
            --repo "${{ github.repository }}" \
            --playbook "${{ matrix.playbook }}" \
            --workflow-file "test-playbooks.yml" \
            --runner-labels "self-hosted,Linux,${{ matrix.arch }}" \
            --runner-name "${{ runner.name }}" \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --sha "${{ github.sha }}"

      - name: Create failure issues (Windows)
        if: failure() && matrix.platform == 'windows' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
        shell: powershell
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RUNNER_NAME: ${{ runner.name }}
        run: |
          python .github/scripts/create_failure_issues.py `
            --repo "${{ github.repository }}" `
            --playbook "${{ matrix.playbook }}" `
            --workflow-file "test-playbooks.yml" `
            --runner-labels "self-hosted,Windows,${{ matrix.arch }}" `
            --runner-name "${{ runner.name }}" `
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" `
            --sha "${{ github.sha }}"

      - name: Test Summary (Windows)
        if: always() && matrix.platform == 'windows'
        shell: powershell
        run: |
          $summary = @"
          ### Test Results for ``${{ matrix.playbook }}`` (${{ matrix.platform }}/${{ matrix.arch }})
          
          See the test artifacts for detailed logs.
          "@
          Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value $summary

      - name: Test Summary (Linux)
        if: always() && matrix.platform == 'linux'
        shell: bash
        run: |
          cat >> $GITHUB_STEP_SUMMARY << 'EOF'
          ### Test Results for `${{ matrix.playbook }}` (${{ matrix.platform }}/${{ matrix.arch }})

          See the test artifacts for detailed logs.
          EOF

  test-gate:
    name: Test Gate (Required Tests)
    needs: [detect-changes, test-playbooks]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check required tests
        run: |
          echo "test-playbooks result: ${{ needs.test-playbooks.result }}"
          if [ "${{ needs.test-playbooks.result }}" = "failure" ]; then
            echo "::error::One or more required tests failed"
            exit 1
          fi
          echo "All required tests passed (or no tests were needed)"