This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Autorouting Benchmark | |
| on: | |
| issue_comment: | |
| types: [created] | |
| pull_request: | |
| types: [opened, reopened, synchronize, edited] | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| solver_name: | |
| description: 'Solver name to benchmark (optional, default: AutoroutingPipelineSolver; use "all" for all solvers)' | |
| required: false | |
| type: string | |
| scenario_limit: | |
| description: Number of scenarios to run (optional) | |
| required: false | |
| type: string | |
| effort: | |
| description: Effort multiplier to apply to each scenario (optional) | |
| required: false | |
| type: string | |
| concurrency: | |
| description: Number of workers per solver (or "auto") | |
| required: false | |
| default: "auto" | |
| type: string | |
| include_assignable: | |
| description: Include assignable pipelines | |
| required: false | |
| default: false | |
| type: boolean | |
| dataset_name: | |
| description: 'Dataset to benchmark (optional: dataset01, zdwiel, or srj05)' | |
| required: false | |
| type: string | |
| ref: | |
| description: Git ref (branch, tag, or SHA) to benchmark | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| benchmark: | |
| name: Run benchmark | |
| if: | | |
| github.event_name == 'workflow_dispatch' || ( | |
| github.event_name == 'push' && | |
| github.ref_name == 'main' | |
| ) || ( | |
| github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.title, '[BENCHMARK TEST]') | |
| ) || ( | |
| github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| github.event.comment.user.type != 'Bot' && | |
| startsWith(github.event.comment.body, '/benchmark') && | |
| ( | |
| github.event.comment.author_association == 'OWNER' || | |
| github.event.comment.author_association == 'MEMBER' || | |
| github.event.comment.author_association == 'COLLABORATOR' | |
| ) | |
| ) | |
| runs-on: ${{ vars.BENCHMARK_RUNNER || 'blacksmith-32vcpu-ubuntu-2404-arm' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Parse benchmark command | |
| id: parse | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| script: | | |
| const isComment = context.eventName === 'issue_comment' | |
| const allSolversAliases = new Set(['all', '_']) | |
| const splitShellArgs = (input) => { | |
| const args = [] | |
| let current = '' | |
| let quote = null | |
| let escaping = false | |
| let tokenStarted = false | |
| const pushCurrent = () => { | |
| if (!tokenStarted) return | |
| args.push(current) | |
| current = '' | |
| tokenStarted = false | |
| } | |
| for (const char of input) { | |
| if (escaping) { | |
| if (quote === '"' && char === '\n') { | |
| escaping = false | |
| continue | |
| } | |
| if (quote === '"' && !['"', '\\', '$', '`'].includes(char)) { | |
| current += '\\' | |
| } | |
| current += char | |
| tokenStarted = true | |
| escaping = false | |
| continue | |
| } | |
| if (quote === "'") { | |
| if (char === "'") { | |
| quote = null | |
| } else { | |
| current += char | |
| } | |
| tokenStarted = true | |
| continue | |
| } | |
| if (quote === '"') { | |
| if (char === '"') { | |
| quote = null | |
| } else if (char === '\\') { | |
| escaping = true | |
| } else { | |
| current += char | |
| } | |
| tokenStarted = true | |
| continue | |
| } | |
| if (/\s/.test(char)) { | |
| pushCurrent() | |
| continue | |
| } | |
| if (char === "'" || char === '"') { | |
| quote = char | |
| tokenStarted = true | |
| continue | |
| } | |
| if (char === '\\') { | |
| escaping = true | |
| tokenStarted = true | |
| continue | |
| } | |
| current += char | |
| tokenStarted = true | |
| } | |
| if (escaping) { | |
| current += '\\' | |
| } | |
| if (quote !== null) { | |
| throw new Error('Unterminated quote in /benchmark command') | |
| } | |
| pushCurrent() | |
| return args | |
| } | |
| let benchmarkArgs = [] | |
| let benchmarkConcurrency = '' | |
| let ref = context.sha | |
| let statusCommentId = '' | |
| if (isComment) { | |
| const body = context.payload.comment.body.trim() | |
| const commentArgs = body.replace(/^\/benchmark\b/, '').trim() | |
| benchmarkArgs = splitShellArgs(commentArgs) | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| }) | |
| ref = pr.data.head.sha | |
| const statusComment = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `## 🏃 Autorouting Benchmark\n\n⏳ Running benchmark on \`${ref.slice(0, 7)}\`...\n\n🔗 Workflow: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`, | |
| }) | |
| statusCommentId = String(statusComment.data.id) | |
| } | |
| if (context.eventName === 'workflow_dispatch') { | |
| const inputs = context.payload.inputs || {} | |
| const requestedSolver = (inputs.solver_name || '').trim() | |
| if (requestedSolver) { | |
| if (allSolversAliases.has(requestedSolver.toLowerCase())) { | |
| benchmarkArgs.push('all') | |
| } else { | |
| benchmarkArgs.push('--solver', requestedSolver) | |
| } | |
| } | |
| const scenarioLimit = (inputs.scenario_limit || '').trim() | |
| const effort = (inputs.effort || '').trim() | |
| const concurrency = (inputs.concurrency || '').trim() | |
| const includeAssignable = inputs.include_assignable === true || inputs.include_assignable === 'true' | |
| const datasetName = (inputs.dataset_name || '').trim() | |
| if (scenarioLimit) { | |
| benchmarkArgs.push('--scenario-limit', scenarioLimit) | |
| } | |
| if (effort) { | |
| benchmarkArgs.push('--effort', effort) | |
| } | |
| if (includeAssignable) { | |
| benchmarkArgs.push('--include-assignable') | |
| } | |
| if (datasetName) { | |
| benchmarkArgs.push('--dataset', datasetName) | |
| } | |
| if (concurrency && concurrency.toLowerCase() !== 'auto') { | |
| benchmarkConcurrency = concurrency | |
| } | |
| ref = (inputs.ref || '').trim() || ref | |
| } | |
| if (context.eventName === 'pull_request') { | |
| ref = context.payload.pull_request.head.sha | |
| } | |
| core.setOutput('benchmark_args_json', JSON.stringify(benchmarkArgs)) | |
| core.setOutput('benchmark_concurrency', benchmarkConcurrency) | |
| core.setOutput('ref', ref) | |
| core.setOutput('status_comment_id', statusCommentId) | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.parse.outputs.ref }} | |
| - name: Setup bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Run benchmark (PR) | |
| env: | |
| BENCHMARK_ARGS_JSON: ${{ steps.parse.outputs.benchmark_args_json }} | |
| BENCHMARK_CONCURRENCY: ${{ steps.parse.outputs.benchmark_concurrency }} | |
| BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS: ${{ vars.BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS || vars.BENCHMARK_TASK_TIMEOUT_MS || '60000' }} | |
| BENCHMARK_HEARTBEAT_INTERVAL_MS: ${{ vars.BENCHMARK_HEARTBEAT_INTERVAL_MS || '30000' }} | |
| BENCHMARK_TERMINATE_TIMEOUT_MS: ${{ vars.BENCHMARK_TERMINATE_TIMEOUT_MS || '1000' }} | |
| run: | | |
| chmod +x ./benchmark.sh | |
| node <<'NODE' | |
| const { spawnSync } = require('node:child_process') | |
| const args = JSON.parse(process.env.BENCHMARK_ARGS_JSON || '[]') | |
| const renderedArgs = args.map((arg) => JSON.stringify(arg)).join(' ') | |
| console.log(`Running benchmark command: ./benchmark.sh${renderedArgs ? ` ${renderedArgs}` : ''}`) | |
| const result = spawnSync('./benchmark.sh', args, { | |
| stdio: 'inherit', | |
| env: process.env, | |
| }) | |
| if (result.error) { | |
| throw result.error | |
| } | |
| process.exit(result.status ?? 1) | |
| NODE | |
| cp benchmark-result.txt benchmark-result-pr.txt | |
| - name: Download main branch benchmark result | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| GH_TOKEN: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| run: | | |
| RUN_ID=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmark.yml/runs?branch=main&event=push&status=success&per_page=1" --jq '.workflow_runs[0].id' 2>/dev/null || echo "") | |
| if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then | |
| echo "(no main branch benchmark result available)" > benchmark-result-main.txt | |
| exit 0 | |
| fi | |
| gh run download "$RUN_ID" --repo "${{ github.repository }}" --name benchmark-result --dir ./main-artifact 2>/dev/null || true | |
| if [ -f ./main-artifact/benchmark-result.txt ]; then | |
| cp ./main-artifact/benchmark-result.txt benchmark-result-main.txt | |
| else | |
| echo "(no main branch benchmark result available)" > benchmark-result-main.txt | |
| fi | |
| - name: Upload benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-result | |
| path: | | |
| ./benchmark-result.txt | |
| ./benchmark-result-pr.txt | |
| ./benchmark-result-main.txt | |
| overwrite: true | |
| - name: Post benchmark result comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| script: | | |
| const fs = require('node:fs') | |
| const maxLength = 60000 | |
| const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s | |
| const prText = fs.existsSync('benchmark-result-pr.txt') | |
| ? fs.readFileSync('benchmark-result-pr.txt', 'utf8').trim() | |
| : fs.readFileSync('benchmark-result.txt', 'utf8').trim() | |
| const mainText = fs.existsSync('benchmark-result-main.txt') | |
| ? fs.readFileSync('benchmark-result-main.txt', 'utf8').trim() | |
| : '(not available)' | |
| const body = [ | |
| '## 🏃 Autorouting Benchmark Results', | |
| '', | |
| '<details>', | |
| '<summary>📊 Main Branch Results</summary>', | |
| '', | |
| '```', | |
| truncate(mainText, maxLength), | |
| '```', | |
| '</details>', | |
| '', | |
| '<details open>', | |
| '<summary>📊 PR Results</summary>', | |
| '', | |
| '```', | |
| truncate(prText, maxLength), | |
| '```', | |
| '</details>', | |
| '', | |
| `📦 Artifact: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`, | |
| ].join('\n') | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: Number('${{ steps.parse.outputs.status_comment_id }}'), | |
| body, | |
| }) |