benchmark-instructions.yml

name: Benchmark Instructions

on:
  pull_request_target:
    types: [opened]

permissions:
  issues: write
  pull-requests: write

jobs:
  post-instructions:
    name: Post benchmark instructions
    runs-on: ubuntu-latest
    steps:
      - name: Comment benchmark usage
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
          script: |
            const body = [
              "## 🏃 Benchmark This PR",
              "",
              "Run benchmarks by commenting on this PR:",
              "",
              "```",
              "/benchmark [benchmark.sh args...]",
              "/benchmark [solver-name|all] [scenario-limit] --concurrency <n> --effort <n> --dataset <dataset01|zdwiel|srj05>",
              "```",
              "",
              "Everything after `/benchmark` is forwarded directly to `./benchmark.sh`.",
              "",
              "Examples:",
              "- `/benchmark` -> `AutoroutingPipelineSolver4`, all scenarios (default concurrency uses the benchmark runner CPU count)",
              "- `/benchmark AutoroutingPipelineSolver4` -> one solver, all scenarios",
              "- `/benchmark all 20` -> all solvers, first 20 scenarios",
              "- `/benchmark AutoroutingPipelineSolver4 20 --concurrency 8` -> one solver, 20 scenarios, 8 workers",
              "- `/benchmark AutoroutingPipelineSolver4 20 --effort 2` -> one solver, 20 scenarios, 2x effort",
              "- `/benchmark AutoroutingPipelineSolver4 --dataset zdwiel` -> one solver, all scenarios, zdwiel dataset",
              "- `/benchmark AutoroutingPipelineSolver4 --dataset srj05` -> one solver, all scenarios, srj05 dataset",
              "- `/benchmark all 20 --dataset dataset01 --concurrency 8` -> all solvers, 20 scenarios, dataset01, 8 workers",
              "- `/benchmark --pipeline 4` -> same as `./benchmark.sh --pipeline 4`",
              "",
              "Any PR whose title contains `[BENCHMARK TEST]` will automatically run the benchmark workflow on PR updates.",
            ].join("\n")

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.payload.pull_request.number,
              body,
            })