|
63 | 63 | exit 1 |
64 | 64 | fi |
65 | 65 |
|
66 | | - # Judge mode hits the backend. Asserts the LLM rubric actually ran. |
| 66 | + # Judge mode hits the backend. Asserts the LLM rubric actually ran and |
| 67 | + # that build-comment.sh renders valid markdown (written to step summary, |
| 68 | + # since push events have no PR to comment on). |
67 | 69 | judge-backend: |
68 | 70 | name: judge · good-skill (backend LLM) |
69 | 71 | runs-on: ubuntu-latest |
|
75 | 77 | with: |
76 | 78 | mode: judge |
77 | 79 | path: tests/fixtures/good-skill |
78 | | - comment: false |
| 80 | + comment: true |
79 | 81 | - name: Assert judge_score present |
80 | 82 | env: |
81 | 83 | RESULTS: ${{ steps.run.outputs.results }} |
|
91 | 93 | echo "judge_score=$score" |
92 | 94 | echo "total-cost=${{ steps.run.outputs.total-cost }}" |
93 | 95 |
|
| 96 | + # Optimize mode — exercises the rewrite path end-to-end. |
| 97 | + optimize-backend: |
| 98 | + name: optimize · good-skill (backend LLM) |
| 99 | + runs-on: ubuntu-latest |
| 100 | + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository |
| 101 | + steps: |
| 102 | + - uses: actions/checkout@v4 |
| 103 | + - uses: ./ |
| 104 | + id: run |
| 105 | + with: |
| 106 | + mode: optimize |
| 107 | + path: tests/fixtures/good-skill |
| 108 | + comment: true |
| 109 | + - name: Assert optimized_content present |
| 110 | + env: |
| 111 | + RESULTS_FILE: ${{ steps.run.outputs.results-file }} |
| 112 | + run: | |
| 113 | + orig=$(jq -r '.[0].optimize.original_score // "null"' "$RESULTS_FILE") |
| 114 | + opt=$(jq -r '.[0].optimize.optimized_score // "null"' "$RESULTS_FILE") |
| 115 | + content_len=$(jq -r '.[0].optimize.optimized_content | length' "$RESULTS_FILE") |
| 116 | + err=$(jq -r '.[0].error // "null"' "$RESULTS_FILE") |
| 117 | + echo "original_score=$orig optimized_score=$opt content_length=$content_len" |
| 118 | + if [ "$orig" = "null" ] || [ "$opt" = "null" ] || [ "$content_len" -lt 100 ]; then |
| 119 | + echo "::error::optimize output missing or truncated" |
| 120 | + echo "Error field: $err" |
| 121 | + exit 1 |
| 122 | + fi |
| 123 | +
|
94 | 124 | # BYOK mode — exercises sklab CLI. Runs only when ANTHROPIC_API_KEY is set. |
95 | 125 | judge-byok: |
96 | 126 | name: judge · good-skill (BYOK) |
|
0 commit comments