wip (#804) #57
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Autorouting Benchmark | |
| on: | |
| issue_comment: | |
| types: [created] | |
| pull_request: | |
| types: [opened, reopened, synchronize, edited] | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| solver_name: | |
| description: 'Solver name to benchmark (optional, default: AutoroutingPipelineSolver4; use "all" for all solvers)' | |
| required: false | |
| type: string | |
| scenario_limit: | |
| description: Number of scenarios to run (optional) | |
| required: false | |
| type: string | |
| effort: | |
| description: Effort multiplier to apply to each scenario (optional) | |
| required: false | |
| type: string | |
| concurrency: | |
| description: Number of workers per solver (or "auto") | |
| required: false | |
| default: "auto" | |
| type: string | |
| include_assignable: | |
| description: Include assignable pipelines | |
| required: false | |
| default: false | |
| type: boolean | |
| dataset_name: | |
| description: 'Dataset to benchmark (optional: dataset01, zdwiel, or srj05)' | |
| required: false | |
| type: string | |
| ref: | |
| description: Git ref (branch, tag, or SHA) to benchmark | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| issues: write | |
| pull-requests: write | |
| jobs: | |
| benchmark: | |
| name: Run benchmark | |
| if: | | |
| github.event_name == 'workflow_dispatch' || ( | |
| github.event_name == 'push' && | |
| github.ref_name == 'main' | |
| ) || ( | |
| github.event_name == 'pull_request' && | |
| contains(github.event.pull_request.title, '[BENCHMARK TEST]') | |
| ) || ( | |
| github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| github.event.comment.user.type != 'Bot' && | |
| startsWith(github.event.comment.body, '/benchmark') && | |
| ( | |
| github.event.comment.author_association == 'OWNER' || | |
| github.event.comment.author_association == 'MEMBER' || | |
| github.event.comment.author_association == 'COLLABORATOR' | |
| ) | |
| ) | |
| runs-on: ${{ vars.BENCHMARK_RUNNER || 'blacksmith-32vcpu-ubuntu-2404-arm' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Parse benchmark command | |
| id: parse | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| script: | | |
| const isComment = context.eventName === 'issue_comment' | |
| const allSolversAliases = new Set(['all', '_']) | |
| const splitShellArgs = (input) => { | |
| const args = [] | |
| let current = '' | |
| let quote = null | |
| let escaping = false | |
| let tokenStarted = false | |
| const pushCurrent = () => { | |
| if (!tokenStarted) return | |
| args.push(current) | |
| current = '' | |
| tokenStarted = false | |
| } | |
| for (const char of input) { | |
| if (escaping) { | |
| if (quote === '"' && char === '\n') { | |
| escaping = false | |
| continue | |
| } | |
| if (quote === '"' && !['"', '\\', '$', '`'].includes(char)) { | |
| current += '\\' | |
| } | |
| current += char | |
| tokenStarted = true | |
| escaping = false | |
| continue | |
| } | |
| if (quote === "'") { | |
| if (char === "'") { | |
| quote = null | |
| } else { | |
| current += char | |
| } | |
| tokenStarted = true | |
| continue | |
| } | |
| if (quote === '"') { | |
| if (char === '"') { | |
| quote = null | |
| } else if (char === '\\') { | |
| escaping = true | |
| } else { | |
| current += char | |
| } | |
| tokenStarted = true | |
| continue | |
| } | |
| if (/\s/.test(char)) { | |
| pushCurrent() | |
| continue | |
| } | |
| if (char === "'" || char === '"') { | |
| quote = char | |
| tokenStarted = true | |
| continue | |
| } | |
| if (char === '\\') { | |
| escaping = true | |
| tokenStarted = true | |
| continue | |
| } | |
| current += char | |
| tokenStarted = true | |
| } | |
| if (escaping) { | |
| current += '\\' | |
| } | |
| if (quote !== null) { | |
| throw new Error('Unterminated quote in /benchmark command') | |
| } | |
| pushCurrent() | |
| return args | |
| } | |
| let benchmarkArgs = [] | |
| let benchmarkConcurrency = '' | |
| let ref = context.sha | |
| let statusCommentId = '' | |
| if (isComment) { | |
| const body = context.payload.comment.body.trim() | |
| const commentArgs = body.replace(/^\/benchmark\b/, '').trim() | |
| benchmarkArgs = splitShellArgs(commentArgs) | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| }) | |
| ref = pr.data.head.sha | |
| const statusComment = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `## 🏃 Autorouting Benchmark\n\n⏳ Running benchmark on \`${ref.slice(0, 7)}\`...\n\n🔗 Workflow: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`, | |
| }) | |
| statusCommentId = String(statusComment.data.id) | |
| } | |
| if (context.eventName === 'workflow_dispatch') { | |
| const inputs = context.payload.inputs || {} | |
| const requestedSolver = (inputs.solver_name || '').trim() | |
| if (requestedSolver) { | |
| if (allSolversAliases.has(requestedSolver.toLowerCase())) { | |
| benchmarkArgs.push('all') | |
| } else { | |
| benchmarkArgs.push('--solver', requestedSolver) | |
| } | |
| } | |
| const scenarioLimit = (inputs.scenario_limit || '').trim() | |
| const effort = (inputs.effort || '').trim() | |
| const concurrency = (inputs.concurrency || '').trim() | |
| const includeAssignable = inputs.include_assignable === true || inputs.include_assignable === 'true' | |
| const datasetName = (inputs.dataset_name || '').trim() | |
| if (scenarioLimit) { | |
| benchmarkArgs.push('--scenario-limit', scenarioLimit) | |
| } | |
| if (effort) { | |
| benchmarkArgs.push('--effort', effort) | |
| } | |
| if (includeAssignable) { | |
| benchmarkArgs.push('--include-assignable') | |
| } | |
| if (datasetName) { | |
| benchmarkArgs.push('--dataset', datasetName) | |
| } | |
| if (concurrency && concurrency.toLowerCase() !== 'auto') { | |
| benchmarkConcurrency = concurrency | |
| } | |
| ref = (inputs.ref || '').trim() || ref | |
| } | |
| if (context.eventName === 'pull_request') { | |
| ref = context.payload.pull_request.head.sha | |
| } | |
| core.setOutput('benchmark_args_json', JSON.stringify(benchmarkArgs)) | |
| core.setOutput('benchmark_concurrency', benchmarkConcurrency) | |
| core.setOutput('ref', ref) | |
| core.setOutput('status_comment_id', statusCommentId) | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.parse.outputs.ref }} | |
| - name: Setup bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Run benchmark (PR) | |
| env: | |
| BENCHMARK_ARGS_JSON: ${{ steps.parse.outputs.benchmark_args_json }} | |
| BENCHMARK_CONCURRENCY: ${{ steps.parse.outputs.benchmark_concurrency }} | |
| BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS: ${{ vars.BENCHMARK_TASK_TIMEOUT_PER_EFFORT_MS || vars.BENCHMARK_TASK_TIMEOUT_MS || '60000' }} | |
| BENCHMARK_HEARTBEAT_INTERVAL_MS: ${{ vars.BENCHMARK_HEARTBEAT_INTERVAL_MS || '30000' }} | |
| BENCHMARK_TERMINATE_TIMEOUT_MS: ${{ vars.BENCHMARK_TERMINATE_TIMEOUT_MS || '1000' }} | |
| run: | | |
| chmod +x ./benchmark.sh | |
| node <<'NODE' | |
| const { spawnSync } = require('node:child_process') | |
| const args = JSON.parse(process.env.BENCHMARK_ARGS_JSON || '[]') | |
| const renderedArgs = args.map((arg) => JSON.stringify(arg)).join(' ') | |
| console.log(`Running benchmark command: ./benchmark.sh${renderedArgs ? ` ${renderedArgs}` : ''}`) | |
| const result = spawnSync('./benchmark.sh', args, { | |
| stdio: 'inherit', | |
| env: process.env, | |
| }) | |
| if (result.error) { | |
| throw result.error | |
| } | |
| process.exit(result.status ?? 1) | |
| NODE | |
| cp benchmark-result.txt benchmark-result-pr.txt | |
| cp benchmark-result.json benchmark-result-pr.json | |
| - name: Download main branch benchmark result | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| GH_TOKEN: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| run: | | |
| RUN_ID=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmark.yml/runs?branch=main&event=push&status=success&per_page=1" --jq '.workflow_runs[0].id' 2>/dev/null || echo "") | |
| if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then | |
| echo "(no main branch benchmark result available)" > benchmark-result-main.txt | |
| echo "" > benchmark-result-main.json | |
| exit 0 | |
| fi | |
| gh run download "$RUN_ID" --repo "${{ github.repository }}" --name benchmark-result --dir ./main-artifact 2>/dev/null || true | |
| if [ -f ./main-artifact/benchmark-result.txt ]; then | |
| cp ./main-artifact/benchmark-result.txt benchmark-result-main.txt | |
| else | |
| echo "(no main branch benchmark result available)" > benchmark-result-main.txt | |
| fi | |
| if [ -f ./main-artifact/benchmark-result.json ]; then | |
| cp ./main-artifact/benchmark-result.json benchmark-result-main.json | |
| else | |
| echo "" > benchmark-result-main.json | |
| fi | |
| - name: Upload benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-result | |
| path: | | |
| ./benchmark-result.txt | |
| ./benchmark-result.json | |
| ./benchmark-result-pr.txt | |
| ./benchmark-result-pr.json | |
| ./benchmark-result-main.txt | |
| ./benchmark-result-main.json | |
| overwrite: true | |
| - name: Post benchmark result comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }} | |
| script: | | |
| const fs = require('node:fs') | |
| const maxLength = 60000 | |
| const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s | |
| const formatTime = (timeMs) => { | |
| if (typeof timeMs !== 'number' || !Number.isFinite(timeMs)) { | |
| return 'n/a' | |
| } | |
| return timeMs < 1000 ? `${Math.round(timeMs)}ms` : `${(timeMs / 1000).toFixed(1)}s` | |
| } | |
| const getStatus = (test) => { | |
| if (test.didTimeout) return '❌' | |
| if (test.didSolve) return '✅' | |
| return '❌' | |
| } | |
| const getDrc = (test) => { | |
| if (!test.didSolve) return '❌' | |
| return test.relaxedDrcPassed ? '✅' : '❌' | |
| } | |
| const renderSummary = (report) => { | |
| if (!Array.isArray(report?.summary) || report.summary.length === 0) { | |
| return [] | |
| } | |
| return [ | |
| `Dataset: ${report.datasetName}`, | |
| `Scenarios: ${report.scenarioCount}`, | |
| `Effort: ${report.effortLabel}`, | |
| '', | |
| '| Solver | Completed % | Relaxed DRC Pass % | Timed Out | P50 Time | P95 Time |', | |
| '| --- | --- | --- | --- | --- | --- |', | |
| ...report.summary.map((row) => | |
| `| ${row.solverName} | ${row.completedRateLabel} | ${row.relaxedDrcRateLabel} | ${row.timedOutLabel} | ${formatTime(row.p50TimeMs)} | ${formatTime(row.p95TimeMs)} |`, | |
| ), | |
| ] | |
| } | |
| const getOutcomeScore = (test) => { | |
| if (!test?.didSolve) return 0 | |
| if (test.relaxedDrcPassed) return 2 | |
| return 1 | |
| } | |
| const buildMainIndex = (report) => { | |
| if (!Array.isArray(report?.tests)) return new Map() | |
| return new Map( | |
| report.tests.map((test) => [ | |
| `${test.solverName}::${test.scenarioName}`, | |
| test, | |
| ]), | |
| ) | |
| } | |
| const getDelta = (test, mainIndex) => { | |
| if (!mainIndex) return '' | |
| const key = `${test.solverName}::${test.scenarioName}` | |
| const mainTest = mainIndex.get(key) | |
| if (!mainTest) return '' | |
| const prScore = getOutcomeScore(test) | |
| const mainScore = getOutcomeScore(mainTest) | |
| if (prScore > mainScore) return '✅' | |
| if (prScore < mainScore) return '❌' | |
| return '' | |
| } | |
| const renderTests = (report, options = {}) => { | |
| const includeDelta = Boolean(options.includeDelta) | |
| const mainIndex = options.mainIndex | |
| if (!Array.isArray(report?.tests) || report.tests.length === 0) { | |
| return ['Per-test results unavailable.'] | |
| } | |
| const header = includeDelta | |
| ? [ | |
| '| Solver | Scenario | Status | Time | Relaxed DRC | Delta |', | |
| '| --- | --- | --- | --- | --- | --- |', | |
| ] | |
| : [ | |
| '| Solver | Scenario | Status | Time | Relaxed DRC |', | |
| '| --- | --- | --- | --- | --- |', | |
| ] | |
| return [ | |
| ...header, | |
| ...report.tests.map((test) => | |
| includeDelta | |
| ? `| ${test.solverName} | ${test.scenarioName} | ${getStatus(test)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getDelta(test, mainIndex)} |` | |
| : `| ${test.solverName} | ${test.scenarioName} | ${getStatus(test)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} |`, | |
| ), | |
| ] | |
| } | |
| const renderReport = (report, fallbackText, options = {}) => { | |
| if (!report) { | |
| return ['```', truncate(fallbackText, maxLength), '```'] | |
| } | |
| return [ | |
| ...renderSummary(report), | |
| '', | |
| '<details>', | |
| '<summary>Details</summary>', | |
| '', | |
| ...renderTests(report, options), | |
| '</details>', | |
| ] | |
| } | |
| const readJson = (path) => { | |
| if (!fs.existsSync(path)) return null | |
| const raw = fs.readFileSync(path, 'utf8').trim() | |
| if (!raw) return null | |
| try { | |
| return JSON.parse(raw) | |
| } catch { | |
| return null | |
| } | |
| } | |
| const prText = fs.existsSync('benchmark-result-pr.txt') | |
| ? fs.readFileSync('benchmark-result-pr.txt', 'utf8').trim() | |
| : fs.readFileSync('benchmark-result.txt', 'utf8').trim() | |
| const mainText = fs.existsSync('benchmark-result-main.txt') | |
| ? fs.readFileSync('benchmark-result-main.txt', 'utf8').trim() | |
| : '(not available)' | |
| const prReport = readJson('benchmark-result-pr.json') ?? readJson('benchmark-result.json') | |
| const mainReport = readJson('benchmark-result-main.json') | |
| const mainIndex = buildMainIndex(mainReport) | |
| const body = [ | |
| '## 🏃 Autorouting Benchmark Results', | |
| '', | |
| '<details>', | |
| '<summary>📊 Main Branch Results</summary>', | |
| '', | |
| ...renderReport(mainReport, mainText), | |
| '</details>', | |
| '', | |
| '<details open>', | |
| '<summary>📊 PR Results</summary>', | |
| '', | |
| ...renderReport(prReport, prText, { includeDelta: true, mainIndex }), | |
| '</details>', | |
| '', | |
| `📦 Artifact: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`, | |
| ].join('\n') | |
| const finalBody = body.length > maxLength ? truncate(body, maxLength) : body | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: Number('${{ steps.parse.outputs.status_comment_id }}'), | |
| body: finalBody, | |
| }) |