summarize-results.yml

name: Summarize Evaluation Results

on:
  workflow_call:
    inputs:
      results-dir:
        description: Directory to download results to
        required: true
        type: string
      model:
        description: "model to use for evaluation"
        required: true
        type: string
      agent:
        description: "type of agent used for evaluation"
        required: true
        type: string
      mock:
        description: "whether a mock evaluation was used"
        required: false
        type: boolean
        default: false
      category:
        description: "Evaluation category"
        required: true
        type: string
      skip-leaderboard:
        description: "Whether to skip updating the leaderboard"
        required: false
        type: boolean
        default: false

env:
  BCEVAL_RESULT_FILE: bceval_results.jsonl
  SUMMARY_OUTPUT_FILE: evaluation_summary.json

jobs:
  summarize-results:
    name: Results
    environment:
      name: ado-read
      deployment: false
    permissions:
      contents: write
      id-token: write
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v5

      - name: Setup Python with UV
        uses: ./.github/actions/setup-python-uv

      - name: Download all evaluation results
        uses: actions/download-artifact@v7
        with:
          path: ${{ inputs.results-dir }}/
          merge-multiple: true

      - name: Summarize evaluation results
        run: uv run bcbench result summarize --category "${{ inputs.category }}" --result-dir "${{ inputs.results-dir }}" --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}"

      - name: Upload evaluation summary to artifacts
        uses: actions/upload-artifact@v6
        with:
          name: evaluation-summary
          path: ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }}
          if-no-files-found: error
          retention-days: ${{ inputs.mock && 1 || 30 }}

      - name: Azure Login with OIDC for bceval package
        uses: azure/login@v3
        with:
          client-id: ${{ secrets.AZURE_CLIENT_ID }}
          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          allow-no-subscriptions: true

      - name: Resolve bceval evaluator config for category
        id: bceval
        run: uv run bcbench category bceval-config --category "${{ inputs.category }}"

      - name: Upload result using bceval
        env:
          BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
        run: |
          # Get Azure DevOps access token from Azure CLI (uses the OIDC token from azure/login)
          ADO_TOKEN=$(az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv)
          echo "::add-mask::$ADO_TOKEN"

          uv tool install bc-eval[capi]==0.3.6 --python 3.12 --index "https://anything:${ADO_TOKEN}@dynamicssmb2.pkgs.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_packaging/BC_PythonFeed/pypi/simple/"

          # Upload summary using bc-eval
          MODEL_TAG="${{ inputs.model }}"
          MODEL_TAG="${MODEL_TAG//./-}"
          bceval metrics calculate \
            --feature-name "BC-Bench" \
            --eval-suite-name "${{ inputs.category }}" \
            --eval-run-name "${{ inputs.agent }} (${{ inputs.model }}) - #${{ github.run_id }}" \
            --tags "${{ inputs.agent }},${MODEL_TAG}" \
            --source "${{ github.sha }}" \
            --input-file "${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}" \
            --evaluator-definitions "${{ github.workspace }}/evaluator/scores.py" \
            --evaluators "${{ steps.bceval.outputs.evaluators }}" \
            --core-score "${{ steps.bceval.outputs.core_score }}" \
            --metric-definitions "${{ github.workspace }}/evaluator/metrics.py" \
            --metrics "bc_bench_metrics" \
            --use-capi ${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }}

      - name: Update leaderboard in a new branch
        # WIP for code-review category
        if: ${{ !inputs.mock && !inputs.skip-leaderboard && inputs.category != 'code-review' }}
        run: |
          git fetch origin main

          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Derive a stable branch name for the entire multi-run chain.
          # When running from a requeue tag (eval-run-<agent>-<original_run_id>),
          # extract the original run_id so all runs in the chain share one branch.
          REF="${{ github.ref }}"
          if [[ "${REF}" == refs/tags/eval-run-* ]]; then
            TAG_NAME="${{ github.ref_name }}"
            STABLE_ID="${TAG_NAME##*-}"
          else
            STABLE_ID="${{ github.run_id }}"
          fi

          branchName="leaderboard/${{ inputs.category }}/${STABLE_ID}"

          # Reuse existing branch from a prior run in this chain, or create new from main
          if git ls-remote --exit-code --heads origin "$branchName" >/dev/null 2>&1; then
            git fetch origin "$branchName"
            git checkout -b "$branchName" "origin/$branchName"
          else
            git checkout -b "$branchName" origin/main
          fi

          uv run bcbench result update "${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }}"

          git add docs/_data/${{ inputs.category }}.json
          git commit -m "Update leaderboard: ${{ inputs.agent }} (${{ inputs.model }}) - Run ${{ github.run_id }}"

          git push --set-upstream origin $branchName