benchmark.yml

name: Autorouting Benchmark

on:
  issue_comment:
    types: [created]
  pull_request:
    types: [opened, reopened, synchronize, edited]
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      solver_name:
        description: 'Solver name to benchmark (optional, default: AutoroutingPipelineSolver; use "all" for all solvers)'
        required: false
        type: string
      scenario_limit:
        description: Number of scenarios to run (optional)
        required: false
        type: string
      effort:
        description: Effort multiplier to apply to each scenario (optional)
        required: false
        type: string
      concurrency:
        description: Number of workers per solver (or "auto")
        required: false
        default: "auto"
        type: string
      include_assignable:
        description: Include assignable pipelines
        required: false
        default: false
        type: boolean
      ref:
        description: Git ref (branch, tag, or SHA) to benchmark
        required: false
        type: string

permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  benchmark:
    name: Run benchmark
    if: |
      github.event_name == 'workflow_dispatch' || (
        github.event_name == 'push' &&
        github.ref_name == 'main'
      ) || (
        github.event_name == 'pull_request' &&
        contains(github.event.pull_request.title, '[BENCHMARK TEST]')
      ) || (
        github.event_name == 'issue_comment' &&
        github.event.issue.pull_request &&
        github.event.comment.user.type != 'Bot' &&
        startsWith(github.event.comment.body, '/benchmark') &&
        (
          github.event.comment.author_association == 'OWNER' ||
          github.event.comment.author_association == 'MEMBER' ||
          github.event.comment.author_association == 'COLLABORATOR'
        )
      )
    runs-on: ${{ vars.BENCHMARK_RUNNER || 'blacksmith-32vcpu-ubuntu-2404-arm' }}
    timeout-minutes: 60

    steps:
      - name: Parse benchmark command
        id: parse
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
          script: |
            const isComment = context.eventName === 'issue_comment'
            const allSolversAliases = new Set(['all', '_'])
            const defaultSolver = 'AutoroutingPipelineSolver'
            let solverName = defaultSolver
            let scenarioLimit = ''
            let effort = ''
            let concurrency = 'auto'
            let ref = context.sha
            let statusCommentId = ''
            let includeAssignable = false

            if (isComment) {
              const body = context.payload.comment.body.trim()
              const tokens = body.split(/\s+/).slice(1)
              const positional = []

              for (let i = 0; i < tokens.length; i += 1) {
                const token = tokens[i]
                if (token === '--concurrency') {
                  concurrency = tokens[i + 1] || concurrency
                  i += 1
                  continue
                }
                if (token === '--effort') {
                  effort = tokens[i + 1] || effort
                  i += 1
                  continue
                }
                if (token === '--include-assignable') {
                  includeAssignable = true
                  continue
                }
                positional.push(token)
              }

              const requestedSolver = positional[0] || ''
              const usesAllSolvers =
                requestedSolver !== '' &&
                allSolversAliases.has(requestedSolver.toLowerCase())

              if (requestedSolver && !usesAllSolvers) {
                solverName = positional[0]
              }
              if (positional[1]) {
                scenarioLimit = positional[1]
              }
              if (usesAllSolvers) {
                solverName = ''
              }

              const pr = await github.rest.pulls.get({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: context.issue.number,
              })
              ref = pr.data.head.sha

              const statusComment = await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: `## 🏃 Autorouting Benchmark\n\n⏳ Running benchmark on \`${ref.slice(0, 7)}\`...\n\n🔗 Workflow: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`,
              })

              statusCommentId = String(statusComment.data.id)
            }


            if (context.eventName === 'workflow_dispatch') {
              const inputs = context.payload.inputs || {}
              const requestedSolver = (inputs.solver_name || '').trim()
              if (requestedSolver) {
                solverName = allSolversAliases.has(requestedSolver.toLowerCase())
                  ? ''
                  : requestedSolver
              }
              scenarioLimit = (inputs.scenario_limit || '').trim()
              effort = (inputs.effort || '').trim()
              concurrency = (inputs.concurrency || concurrency).trim()
              includeAssignable = inputs.include_assignable === true || inputs.include_assignable === 'true'
              ref = (inputs.ref || '').trim() || ref
            }

            if (context.eventName === 'pull_request') {
              ref = context.payload.pull_request.head.sha
            }

            core.setOutput('solver_name', solverName)
            core.setOutput('scenario_limit', scenarioLimit)
            core.setOutput('effort', effort)
            core.setOutput('concurrency', concurrency)
            core.setOutput('include_assignable', includeAssignable ? 'true' : 'false')
            core.setOutput('ref', ref)
            core.setOutput('status_comment_id', statusCommentId)

      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ steps.parse.outputs.ref }}

      - name: Setup bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Install dependencies
        run: bun install

      - name: Run benchmark (PR)
        env:
          SOLVER_NAME: ${{ steps.parse.outputs.solver_name }}
          SCENARIO_LIMIT: ${{ steps.parse.outputs.scenario_limit }}
          EFFORT: ${{ steps.parse.outputs.effort }}
          CONCURRENCY: ${{ steps.parse.outputs.concurrency }}
          INCLUDE_ASSIGNABLE: ${{ steps.parse.outputs.include_assignable }}
          BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS: ${{ vars.BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS || vars.BENCHMARK_TASK_TIMEOUT_MS || '60000' }}
          BENCHMARK_HEARTBEAT_INTERVAL_MS: ${{ vars.BENCHMARK_HEARTBEAT_INTERVAL_MS || '30000' }}
          BENCHMARK_TERMINATE_TIMEOUT_MS: ${{ vars.BENCHMARK_TERMINATE_TIMEOUT_MS || '1000' }}
        run: |
          chmod +x ./benchmark.sh
          RESOLVED_CONCURRENCY="$CONCURRENCY"
          if [ -z "$RESOLVED_CONCURRENCY" ] || [ "$RESOLVED_CONCURRENCY" = "auto" ]; then
            RESOLVED_CONCURRENCY="$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 4)"
          fi
          echo "Using benchmark concurrency: $RESOLVED_CONCURRENCY"
          CMD=(./benchmark.sh --concurrency "$RESOLVED_CONCURRENCY")
          if [ -n "$SOLVER_NAME" ]; then
            CMD+=(--solver "$SOLVER_NAME")
          fi
          if [ -n "$SCENARIO_LIMIT" ]; then
            CMD+=(--scenario-limit "$SCENARIO_LIMIT")
          fi
          if [ -n "$EFFORT" ]; then
            CMD+=(--effort "$EFFORT")
          fi
          if [ "$INCLUDE_ASSIGNABLE" = "true" ]; then
            CMD+=(--include-assignable)
          fi
          "${CMD[@]}"
          cp benchmark-result.txt benchmark-result-pr.txt

      - name: Download main branch benchmark result
        if: github.event_name == 'issue_comment'
        env:
          GH_TOKEN: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
        run: |
          RUN_ID=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmark.yml/runs?branch=main&event=push&status=success&per_page=1" --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
          if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
            echo "(no main branch benchmark result available)" > benchmark-result-main.txt
            exit 0
          fi
          gh run download "$RUN_ID" --repo "${{ github.repository }}" --name benchmark-result --dir ./main-artifact 2>/dev/null || true
          if [ -f ./main-artifact/benchmark-result.txt ]; then
            cp ./main-artifact/benchmark-result.txt benchmark-result-main.txt
          else
            echo "(no main branch benchmark result available)" > benchmark-result-main.txt
          fi

      - name: Upload benchmark result
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-result
          path: |
            ./benchmark-result.txt
            ./benchmark-result-pr.txt
            ./benchmark-result-main.txt
          overwrite: true

      - name: Post benchmark result comment
        if: github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
          script: |
            const fs = require('node:fs')
            const maxLength = 60000
            const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s

            const prText = fs.existsSync('benchmark-result-pr.txt')
              ? fs.readFileSync('benchmark-result-pr.txt', 'utf8').trim()
              : fs.readFileSync('benchmark-result.txt', 'utf8').trim()
            const mainText = fs.existsSync('benchmark-result-main.txt')
              ? fs.readFileSync('benchmark-result-main.txt', 'utf8').trim()
              : '(not available)'

            const body = [
              '## 🏃 Autorouting Benchmark Results',
              '',
              '<details>',
              '<summary>📊 PR Results</summary>',
              '',
              '```',
              truncate(prText, maxLength),
              '```',
              '</details>',
              '',
              '<details open>',
              '<summary>📊 Main Branch Results</summary>',
              '',
              '```',
              truncate(mainText, maxLength),
              '```',
              '</details>',
              '',
              `📦 Artifact: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
            ].join('\n')

            await github.rest.issues.updateComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: Number('${{ steps.parse.outputs.status_comment_id }}'),
              body,
            })