Introduce zdwiel benchmark dataset support and allow dataset selectio… #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Autorouting Benchmark | |
| on: | |
| issue_comment: | |
| types: [created] | |
| pull_request: | |
| types: [opened, reopened, synchronize, edited] | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| solver_name: | |
| description: 'Solver name to benchmark (optional, default: AutoroutingPipelineSolver; use "all" for all solvers)' | |
| required: false | |
| type: string | |
| scenario_limit: | |
| description: Number of scenarios to run (optional) | |
| required: false | |
| type: string | |
| effort: | |
| description: Effort multiplier to apply to each scenario (optional) | |
| required: false | |
| type: string | |
| concurrency: | |
| description: Number of workers per solver (or "auto") | |
| required: false | |
| default: "auto" | |
| type: string | |
| include_assignable: | |
| description: Include assignable pipelines | |
| required: false | |
| default: false | |
| type: boolean | |
| ref: | |
| description: Git ref (branch, tag, or SHA) to benchmark | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| benchmark: | |
| name: Run benchmark | |
| if: | | |
| github.event_name == 'workflow_dispatch' || ( | |
| github.event_name == 'push' && | |
| github.ref_name == 'main' | |
| ) || ( | |
| github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.title, '[BENCHMARK TEST]') | |
| ) || ( | |
| github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| github.event.comment.user.type != 'Bot' && | |
| startsWith(github.event.comment.body, '/benchmark') && | |
| ( | |
| github.event.comment.author_association == 'OWNER' || | |
| github.event.comment.author_association == 'MEMBER' || | |
| github.event.comment.author_association == 'COLLABORATOR' | |
| ) | |
| ) | |
| runs-on: ${{ vars.BENCHMARK_RUNNER || 'blacksmith-32vcpu-ubuntu-2404-arm' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Parse benchmark command | |
| id: parse | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| script: | | |
| const isComment = context.eventName === 'issue_comment' | |
| const allSolversAliases = new Set(['all', '_']) | |
| const defaultSolver = 'AutoroutingPipelineSolver' | |
| let solverName = defaultSolver | |
| let scenarioLimit = '' | |
| let effort = '' | |
| let concurrency = 'auto' | |
| let ref = context.sha | |
| let statusCommentId = '' | |
| let includeAssignable = false | |
| if (isComment) { | |
| const body = context.payload.comment.body.trim() | |
| const tokens = body.split(/\s+/).slice(1) | |
| const positional = [] | |
| for (let i = 0; i < tokens.length; i += 1) { | |
| const token = tokens[i] | |
| if (token === '--concurrency') { | |
| concurrency = tokens[i + 1] || concurrency | |
| i += 1 | |
| continue | |
| } | |
| if (token === '--effort') { | |
| effort = tokens[i + 1] || effort | |
| i += 1 | |
| continue | |
| } | |
| if (token === '--include-assignable') { | |
| includeAssignable = true | |
| continue | |
| } | |
| positional.push(token) | |
| } | |
| const requestedSolver = positional[0] || '' | |
| const usesAllSolvers = | |
| requestedSolver !== '' && | |
| allSolversAliases.has(requestedSolver.toLowerCase()) | |
| if (requestedSolver && !usesAllSolvers) { | |
| solverName = positional[0] | |
| } | |
| if (positional[1]) { | |
| scenarioLimit = positional[1] | |
| } | |
| if (usesAllSolvers) { | |
| solverName = '' | |
| } | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| }) | |
| ref = pr.data.head.sha | |
| const statusComment = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `## 🏃 Autorouting Benchmark\n\n⏳ Running benchmark on \`${ref.slice(0, 7)}\`...\n\n🔗 Workflow: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`, | |
| }) | |
| statusCommentId = String(statusComment.data.id) | |
| } | |
| if (context.eventName === 'workflow_dispatch') { | |
| const inputs = context.payload.inputs || {} | |
| const requestedSolver = (inputs.solver_name || '').trim() | |
| if (requestedSolver) { | |
| solverName = allSolversAliases.has(requestedSolver.toLowerCase()) | |
| ? '' | |
| : requestedSolver | |
| } | |
| scenarioLimit = (inputs.scenario_limit || '').trim() | |
| effort = (inputs.effort || '').trim() | |
| concurrency = (inputs.concurrency || concurrency).trim() | |
| includeAssignable = inputs.include_assignable === true || inputs.include_assignable === 'true' | |
| ref = (inputs.ref || '').trim() || ref | |
| } | |
| if (context.eventName === 'pull_request') { | |
| ref = context.payload.pull_request.head.sha | |
| } | |
| core.setOutput('solver_name', solverName) | |
| core.setOutput('scenario_limit', scenarioLimit) | |
| core.setOutput('effort', effort) | |
| core.setOutput('concurrency', concurrency) | |
| core.setOutput('include_assignable', includeAssignable ? 'true' : 'false') | |
| core.setOutput('ref', ref) | |
| core.setOutput('status_comment_id', statusCommentId) | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.parse.outputs.ref }} | |
| - name: Setup bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Run benchmark (PR) | |
| env: | |
| SOLVER_NAME: ${{ steps.parse.outputs.solver_name }} | |
| SCENARIO_LIMIT: ${{ steps.parse.outputs.scenario_limit }} | |
| EFFORT: ${{ steps.parse.outputs.effort }} | |
| CONCURRENCY: ${{ steps.parse.outputs.concurrency }} | |
| INCLUDE_ASSIGNABLE: ${{ steps.parse.outputs.include_assignable }} | |
| BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS: ${{ vars.BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS || vars.BENCHMARK_TASK_TIMEOUT_MS || '60000' }} | |
| BENCHMARK_HEARTBEAT_INTERVAL_MS: ${{ vars.BENCHMARK_HEARTBEAT_INTERVAL_MS || '30000' }} | |
| BENCHMARK_TERMINATE_TIMEOUT_MS: ${{ vars.BENCHMARK_TERMINATE_TIMEOUT_MS || '1000' }} | |
| run: | | |
| chmod +x ./benchmark.sh | |
| RESOLVED_CONCURRENCY="$CONCURRENCY" | |
| if [ -z "$RESOLVED_CONCURRENCY" ] || [ "$RESOLVED_CONCURRENCY" = "auto" ]; then | |
| RESOLVED_CONCURRENCY="$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 4)" | |
| fi | |
| echo "Using benchmark concurrency: $RESOLVED_CONCURRENCY" | |
| CMD=(./benchmark.sh --concurrency "$RESOLVED_CONCURRENCY") | |
| if [ -n "$SOLVER_NAME" ]; then | |
| CMD+=(--solver "$SOLVER_NAME") | |
| fi | |
| if [ -n "$SCENARIO_LIMIT" ]; then | |
| CMD+=(--scenario-limit "$SCENARIO_LIMIT") | |
| fi | |
| if [ -n "$EFFORT" ]; then | |
| CMD+=(--effort "$EFFORT") | |
| fi | |
| if [ "$INCLUDE_ASSIGNABLE" = "true" ]; then | |
| CMD+=(--include-assignable) | |
| fi | |
| "${CMD[@]}" | |
| cp benchmark-result.txt benchmark-result-pr.txt | |
| - name: Download main branch benchmark result | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| GH_TOKEN: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| run: | | |
| RUN_ID=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmark.yml/runs?branch=main&event=push&status=success&per_page=1" --jq '.workflow_runs[0].id' 2>/dev/null || echo "") | |
| if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then | |
| echo "(no main branch benchmark result available)" > benchmark-result-main.txt | |
| exit 0 | |
| fi | |
| gh run download "$RUN_ID" --repo "${{ github.repository }}" --name benchmark-result --dir ./main-artifact 2>/dev/null || true | |
| if [ -f ./main-artifact/benchmark-result.txt ]; then | |
| cp ./main-artifact/benchmark-result.txt benchmark-result-main.txt | |
| else | |
| echo "(no main branch benchmark result available)" > benchmark-result-main.txt | |
| fi | |
| - name: Upload benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-result | |
| path: | | |
| ./benchmark-result.txt | |
| ./benchmark-result-pr.txt | |
| ./benchmark-result-main.txt | |
| overwrite: true | |
| - name: Post benchmark result comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| script: | | |
| const fs = require('node:fs') | |
| const maxLength = 60000 | |
| const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s | |
| const prText = fs.existsSync('benchmark-result-pr.txt') | |
| ? fs.readFileSync('benchmark-result-pr.txt', 'utf8').trim() | |
| : fs.readFileSync('benchmark-result.txt', 'utf8').trim() | |
| const mainText = fs.existsSync('benchmark-result-main.txt') | |
| ? fs.readFileSync('benchmark-result-main.txt', 'utf8').trim() | |
| : '(not available)' | |
| const body = [ | |
| '## 🏃 Autorouting Benchmark Results', | |
| '', | |
| '<details>', | |
| '<summary>📊 Main Branch Results</summary>', | |
| '', | |
| '```', | |
| truncate(mainText, maxLength), | |
| '```', | |
| '</details>', | |
| '', | |
| '<details open>', | |
| '<summary>📊 PR Results</summary>', | |
| '', | |
| '```', | |
| truncate(prText, maxLength), | |
| '```', | |
| '</details>', | |
| '', | |
| `📦 Artifact: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`, | |
| ].join('\n') | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: Number('${{ steps.parse.outputs.status_comment_id }}'), | |
| body, | |
| }) |