Add shared Copilot skills and skill development tooling #11
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Skill Evaluations | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/skills/**' | |
| pull_request: | |
| paths: | |
| - '.github/skills/**' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| check-skills: | |
| name: Skill Compliance Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install waza | |
| run: | | |
| curl -fsSL https://raw.githubusercontent.com/microsoft/waza/refs/tags/v0.12.0/install.sh | bash | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Check skill compliance | |
| run: | | |
| echo "## Skill Compliance Report" > compliance.md | |
| echo "" >> compliance.md | |
| failed=0 | |
| for skill_dir in .github/skills/*/; do | |
| skill_name=$(basename "$skill_dir") | |
| if [ -f "$skill_dir/SKILL.md" ]; then | |
| echo "### $skill_name" >> compliance.md | |
| if ! waza check "$skill_dir" 2>&1 | tee -a compliance.md; then | |
| failed=1 | |
| fi | |
| echo "" >> compliance.md | |
| fi | |
| done | |
| cat compliance.md | |
| if [ "$failed" -eq 1 ]; then | |
| echo "::error::One or more skills failed compliance checks" | |
| exit 1 | |
| fi | |
| - name: Upload compliance report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skill-compliance | |
| path: compliance.md | |
| retention-days: 30 | |
| run-evals: | |
| name: Run Skill Evaluations | |
| runs-on: ubuntu-latest | |
| needs: check-skills | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check if skill files changed | |
| id: changes | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "SHOULD_RUN=true" >> $GITHUB_OUTPUT | |
| elif [ "${{ github.event_name }}" = "pull_request" ]; then | |
| # Compare PR head against base branch to detect all changes in the PR | |
| BASE_SHA="${{ github.event.pull_request.base.sha }}" | |
| if git diff --name-only "$BASE_SHA"...HEAD | grep -q "^\.github/skills/"; then | |
| echo "SHOULD_RUN=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "SHOULD_RUN=false" >> $GITHUB_OUTPUT | |
| echo "No skill file changes detected in PR — skipping evals" | |
| fi | |
| elif git diff --name-only HEAD~1 HEAD | grep -q "^\.github/skills/"; then | |
| echo "SHOULD_RUN=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "SHOULD_RUN=false" >> $GITHUB_OUTPUT | |
| echo "No skill file changes detected — skipping evals" | |
| fi | |
| - name: Install waza | |
| if: steps.changes.outputs.SHOULD_RUN == 'true' | |
| run: | | |
| curl -fsSL https://raw.githubusercontent.com/microsoft/waza/refs/tags/v0.12.0/install.sh | bash | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Run evaluations | |
| if: steps.changes.outputs.SHOULD_RUN == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| mkdir -p results | |
| for eval_file in .github/skills/*/eval.yaml; do | |
| if [ -f "$eval_file" ]; then | |
| skill_name=$(basename "$(dirname "$eval_file")") | |
| echo "=== Running evals for $skill_name ===" | |
| waza run "$eval_file" \ | |
| --model "gpt-4o" \ | |
| --parallel \ | |
| -v \ | |
| -o "results/${skill_name}.json" | |
| fi | |
| done | |
| - name: Upload eval results | |
| if: always() && steps.changes.outputs.SHOULD_RUN == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval-results-${{ github.run_id }} | |
| path: results/ | |
| retention-days: 30 |