llm-d-benchmark/.github/workflows/ci-pr-plan-rendering-validation.yaml at 24dae84a57934b7a5dfb4f4ac423ba0c3a5eddfe · hubunt/llm-d-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
name: CI - PR Plan Rendering Validation

on:
  pull_request:

jobs:
  render-templates:
    name: Render All Templates
    runs-on: ubuntu-latest
    timeout-minutes: 10

    env:
      LLMDBENCH_WORKSPACE: /tmp/render-check

    steps:
      - name: Checkout Code
        uses: actions/checkout@v7.0.0
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Install llmdbenchmark
        run: |
          ./install.sh -y 2>&1

      - name: Render all specification templates
        run: |
          FAILED=0
          PASSED=0
          SPECS=$(find config/specification -name "*.yaml.j2" | sort)

          for spec_path in $SPECS; do
            # Strip config/specification/ prefix and .yaml.j2 suffix
            spec_name="${spec_path#config/specification/}"
            spec_name="${spec_name%.yaml.j2}"

            echo ""
            echo "=========================================="
            echo "Rendering: $spec_name"
            echo "=========================================="

            if llmdbenchmark --spec "$spec_name" --dry-run plan -p render-check 2>&1; then
              echo "  ✅ $spec_name"
              PASSED=$((PASSED + 1))
            else
              echo "  ❌ $spec_name"
              FAILED=$((FAILED + 1))
            fi
          done

          echo ""
          echo "=========================================="
          echo "RESULTS: $PASSED passed, $FAILED failed ($(echo "$SPECS" | wc -l) total)"
          echo "=========================================="

          if [ "$FAILED" -gt 0 ]; then
            echo "::error::$FAILED specification(s) failed to render"
            exit 1
          fi