Skip to content

Commit ffb5199

Browse files
committed
Add optimize smoke test; exercise comment rendering via step summary
On push events the action falls back to $GITHUB_STEP_SUMMARY when there's no PR to comment on, so flipping comment:true on the judge and optimize test jobs verifies build-comment.sh renders live without needing a PR. optimize-backend asserts original_score, optimized_score, and a non-trivial optimized_content length — catches any regression in the /optimize call or response parsing.
1 parent 7d44f68 commit ffb5199

1 file changed

Lines changed: 32 additions & 2 deletions

File tree

.github/workflows/test.yml

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,9 @@ jobs:
6363
exit 1
6464
fi
6565
66-
# Judge mode hits the backend. Asserts the LLM rubric actually ran.
66+
# Judge mode hits the backend. Asserts the LLM rubric actually ran and
67+
# that build-comment.sh renders valid markdown (written to step summary,
68+
# since push events have no PR to comment on).
6769
judge-backend:
6870
name: judge · good-skill (backend LLM)
6971
runs-on: ubuntu-latest
@@ -75,7 +77,7 @@ jobs:
7577
with:
7678
mode: judge
7779
path: tests/fixtures/good-skill
78-
comment: false
80+
comment: true
7981
- name: Assert judge_score present
8082
env:
8183
RESULTS: ${{ steps.run.outputs.results }}
@@ -91,6 +93,34 @@ jobs:
9193
echo "judge_score=$score"
9294
echo "total-cost=${{ steps.run.outputs.total-cost }}"
9395
96+
# Optimize mode — exercises the rewrite path end-to-end.
97+
optimize-backend:
98+
name: optimize · good-skill (backend LLM)
99+
runs-on: ubuntu-latest
100+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
101+
steps:
102+
- uses: actions/checkout@v4
103+
- uses: ./
104+
id: run
105+
with:
106+
mode: optimize
107+
path: tests/fixtures/good-skill
108+
comment: true
109+
- name: Assert optimized_content present
110+
env:
111+
RESULTS_FILE: ${{ steps.run.outputs.results-file }}
112+
run: |
113+
orig=$(jq -r '.[0].optimize.original_score // "null"' "$RESULTS_FILE")
114+
opt=$(jq -r '.[0].optimize.optimized_score // "null"' "$RESULTS_FILE")
115+
content_len=$(jq -r '.[0].optimize.optimized_content | length' "$RESULTS_FILE")
116+
err=$(jq -r '.[0].error // "null"' "$RESULTS_FILE")
117+
echo "original_score=$orig optimized_score=$opt content_length=$content_len"
118+
if [ "$orig" = "null" ] || [ "$opt" = "null" ] || [ "$content_len" -lt 100 ]; then
119+
echo "::error::optimize output missing or truncated"
120+
echo "Error field: $err"
121+
exit 1
122+
fi
123+
94124
# BYOK mode — exercises sklab CLI. Runs only when ANTHROPIC_API_KEY is set.
95125
judge-byok:
96126
name: judge · good-skill (BYOK)

0 commit comments

Comments
 (0)