prepare-release-plan skill updates #283
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Skill Evaluations | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/skills/**' | |
| pull_request: | |
| paths: | |
| - '.github/skills/**' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| check-skills: | |
| name: Skill Compliance Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install Azure Developer CLI | |
| uses: Azure/setup-azd@v2 | |
| - name: Install waza extension | |
| run: | | |
| azd config set alpha.extensions on | |
| azd ext source add -n waza -t url -l https://raw.githubusercontent.com/microsoft/waza/main/registry.json | |
| azd ext install microsoft.azd.waza | |
| - name: Check skill compliance | |
| run: | | |
| echo "## Skill Compliance Report" > compliance.md | |
| echo "" >> compliance.md | |
| failed=0 | |
| for skill_dir in .github/skills/*/; do | |
| skill_name=$(basename "$skill_dir") | |
| if [ -f "$skill_dir/SKILL.md" ]; then | |
| echo "### $skill_name" >> compliance.md | |
| if ! azd waza check "$skill_dir" 2>&1 | tee -a compliance.md; then | |
| failed=1 | |
| fi | |
| echo "" >> compliance.md | |
| fi | |
| done | |
| cat compliance.md | |
| if [ "$failed" -eq 1 ]; then | |
| echo "::error::One or more skills failed compliance checks" | |
| exit 1 | |
| fi | |
| - name: Upload compliance report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: skill-compliance | |
| path: compliance.md | |
| retention-days: 30 | |
| run-evals: | |
| name: Run Skill Evaluations | |
| runs-on: ubuntu-latest | |
| needs: check-skills | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check if skill files changed | |
| id: changes | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "SHOULD_RUN=true" >> $GITHUB_OUTPUT | |
| elif [ "${{ github.event_name }}" = "pull_request" ]; then | |
| # Compare PR head against base branch to detect all changes in the PR | |
| BASE_SHA="${{ github.event.pull_request.base.sha }}" | |
| if git diff --name-only "$BASE_SHA"...HEAD | grep -q "^\.github/skills/"; then | |
| echo "SHOULD_RUN=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "SHOULD_RUN=false" >> $GITHUB_OUTPUT | |
| echo "No skill file changes detected in PR — skipping evals" | |
| fi | |
| elif git diff --name-only HEAD~1 HEAD | grep -q "^\.github/skills/"; then | |
| echo "SHOULD_RUN=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "SHOULD_RUN=false" >> $GITHUB_OUTPUT | |
| echo "No skill file changes detected — skipping evals" | |
| fi | |
| - name: Install Azure Developer CLI | |
| if: steps.changes.outputs.SHOULD_RUN == 'true' | |
| uses: Azure/setup-azd@v2 | |
| - name: Install waza extension | |
| if: steps.changes.outputs.SHOULD_RUN == 'true' | |
| run: | | |
| azd config set alpha.extensions on | |
| azd ext source add -n waza -t url -l https://raw.githubusercontent.com/microsoft/waza/main/registry.json | |
| azd ext install microsoft.azd.waza | |
| - name: Run evaluations | |
| if: steps.changes.outputs.SHOULD_RUN == 'true' | |
| continue-on-error: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| mkdir -p results | |
| for eval_file in .github/skills/*/eval.yaml; do | |
| if [ -f "$eval_file" ]; then | |
| skill_name=$(basename "$(dirname "$eval_file")") | |
| echo "=== Running evals for $skill_name ===" | |
| azd waza run "$eval_file" --output-dir "results/${skill_name}" | |
| fi | |
| done | |
| - name: Upload eval results | |
| if: always() && steps.changes.outputs.SHOULD_RUN == 'true' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: eval-results-${{ github.run_id }} | |
| path: results/ | |
| retention-days: 30 |