feat: add per-skill semantic-release workflow #26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tessl Skill Eval | |
| on: | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| tessl-eval: | |
| runs-on: ubuntu-latest | |
| # Uses the "eval" environment for protection rules (e.g. reviewer approval) | |
| # to guard the TESSL_TOKEN secret against exfiltration via same-repo branches. | |
| environment: eval | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Check eval trigger | |
| id: trigger | |
| run: | | |
| LAST_MSG=$(git log -1 --format=%s) | |
| echo "commit_message=$LAST_MSG" | |
| if [[ ! "$LAST_MSG" =~ ^eval: ]]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "Last commit does not start with 'eval:' — skipping eval." | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| echo "Eval triggered by commit: $LAST_MSG" | |
| fi | |
| - name: Post skip summary (no eval trigger) | |
| if: steps.trigger.outputs.skip == 'true' | |
| run: | | |
| { | |
| echo "## Skill Eval" | |
| echo "" | |
| echo "Eval not requested — last commit message does not start with \`eval:\`." | |
| echo "To trigger evals, create a commit with a message like:" | |
| echo "\`\`\`" | |
| echo 'git commit --allow-empty -m "eval: test building-blocks changes"' | |
| echo "\`\`\`" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Detect changed skills with tiles | |
| if: steps.trigger.outputs.skip != 'true' | |
| id: detect | |
| run: | | |
| CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) | |
| # Build list of all tile directories (parents of tile.json) | |
| ALL_TILE_DIRS=$(find skills -name tile.json -exec dirname {} \; 2>/dev/null | sort) | |
| if [ -z "$ALL_TILE_DIRS" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "No tile.json files found in repo." | |
| else | |
| # For each changed file, find which tile directory it belongs to | |
| DIRS="" | |
| for changed_file in $CHANGED_FILES; do | |
| for tile_dir in $ALL_TILE_DIRS; do | |
| case "$changed_file" in | |
| "$tile_dir"/*) | |
| # Check not already added | |
| case " $DIRS " in | |
| *" $tile_dir "*) ;; | |
| *) DIRS="$DIRS $tile_dir" ;; | |
| esac | |
| break | |
| ;; | |
| esac | |
| done | |
| done | |
| DIRS=$(echo "$DIRS" | xargs) # trim whitespace | |
| # Check for unmatched changes under skills/ | |
| UNMATCHED=false | |
| for changed_file in $CHANGED_FILES; do | |
| case "$changed_file" in | |
| skills/*) | |
| MATCHED=false | |
| for tile_dir in $ALL_TILE_DIRS; do | |
| case "$changed_file" in | |
| "$tile_dir"/*) MATCHED=true; break ;; | |
| esac | |
| done | |
| if [ "$MATCHED" = "false" ]; then | |
| UNMATCHED=true | |
| break | |
| fi | |
| ;; | |
| esac | |
| done | |
| if [ "$UNMATCHED" = "true" ]; then | |
| echo "Unmatched changes under skills/ detected — evaluating all tiles" | |
| DIRS=$(find skills -name tile.json -exec dirname {} \; 2>/dev/null | tr "\n" " " | xargs) | |
| fi | |
| if [ -z "$DIRS" ]; then | |
| echo "skip=true" >> "$GITHUB_OUTPUT" | |
| echo "Changed files don't belong to any tile — nothing to eval." | |
| else | |
| echo "skip=false" >> "$GITHUB_OUTPUT" | |
| echo "dirs=$DIRS" >> "$GITHUB_OUTPUT" | |
| echo "Changed tiles: $DIRS" | |
| fi | |
| fi | |
| - name: Post skip summary (no qualifying tiles) | |
| if: steps.trigger.outputs.skip != 'true' && steps.detect.outputs.skip == 'true' | |
| run: | | |
| { | |
| echo "## Skill Eval" | |
| echo "" | |
| echo "No qualifying skills to evaluate — either no skill files were changed or changed skills have no \`tile.json\`." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - uses: tesslio/setup-tessl@v2 | |
| if: steps.trigger.outputs.skip != 'true' && steps.detect.outputs.skip != 'true' | |
| with: | |
| token: ${{ secrets.TESSL_TOKEN }} | |
| - name: Run evals | |
| if: steps.trigger.outputs.skip != 'true' && steps.detect.outputs.skip != 'true' | |
| run: | | |
| PASS=0 | |
| FAIL=0 | |
| SUMMARY_ROWS="" | |
| for tile_dir in ${{ steps.detect.outputs.dirs }}; do | |
| TILE_NAME=$(basename "$tile_dir") | |
| echo "::group::Evaluating $TILE_NAME ($tile_dir)" | |
| EXIT_CODE=0 | |
| OUTPUT=$(tessl eval run "$tile_dir" 2>&1) || EXIT_CODE=$? | |
| echo "$OUTPUT" | |
| echo "::endgroup::" | |
| if [ "$EXIT_CODE" -ne 0 ]; then | |
| echo "::warning::tessl eval run failed for $TILE_NAME (exit code $EXIT_CODE)" | |
| FAIL=$((FAIL + 1)) | |
| SUMMARY_ROWS="$SUMMARY_ROWS| $TILE_NAME | error | ❌ |\n" | |
| else | |
| PASS=$((PASS + 1)) | |
| SUMMARY_ROWS="$SUMMARY_ROWS| $TILE_NAME | passed | ✅ |\n" | |
| # Show detailed results only for successful eval | |
| echo "::group::Eval results for $TILE_NAME" | |
| tessl eval view --last 2>&1 || true | |
| echo "::endgroup::" | |
| fi | |
| done | |
| TOTAL=$((PASS + FAIL)) | |
| { | |
| echo "## Skill Eval" | |
| echo "" | |
| echo "| Tile | Result | Status |" | |
| echo "|------|--------|--------|" | |
| echo -e "$SUMMARY_ROWS" | |
| echo "| **Total** | **$PASS/$TOTAL passed** | $([ "$FAIL" -eq 0 ] && echo '✅' || echo '❌') |" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| echo "" | |
| echo "=============================" | |
| echo " Skill Eval Summary" | |
| echo "=============================" | |
| echo " Total: $TOTAL" | |
| echo " Passed: $PASS" | |
| echo " Failed: $FAIL" | |
| echo "=============================" | |
| if [ "$FAIL" -gt 0 ]; then | |
| echo "::error::$FAIL tile(s) failed evaluation" | |
| exit 1 | |
| fi | |