Only smoke-test the benchmarks on CI

Three CI jobs ran benchmarks on the PR and on the target branch, 12 to 23
minutes each, and nothing compared the results. One run on a shared runner
is too noisy to show a regression, and performance PRs measure locally.
One job now runs every benchmark once with minimal iterations, which still
catches benchmarks that break or hang. runBenchmarks takes JMH options from
benchmark.jmhOptions.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: lrytz

.github/workflows/ci.yml

@@ -30,14 +30,6 @@ jobs:
             java: 21
             distribution: temurin
             jobtype: 4
-          - os: ubuntu-latest
-            java: 21
-            distribution: temurin
-            jobtype: 5
-          - os: ubuntu-latest
-            java: 21
-            distribution: temurin
-            jobtype: 6
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
@@ -65,38 +57,8 @@ jobs:
       run: |
         sbt -v --server -Dfile.encoding=UTF-8 scalafmtCheckRepo headerCheck "Test/headerCheck" generateContrabands
         git diff --exit-code
-    - name: Benchmark (Scalac) (4)
+    - name: Benchmark smoke test (4)
       if: ${{ matrix.jobtype == 4 }}
       shell: bash
       run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
-    - name: Benchmark (Shapeless) (5)
-      if: ${{ matrix.jobtype == 5 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
-    - name: Benchmark (AnalysisFormatBenchmark) (6)
-      if: ${{ matrix.jobtype == 6 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks"
-    - name: Checkout Target Branch (4-6)
-      if: ${{ github.event_name == 'pull_request' && (matrix.jobtype >= 4 && matrix.jobtype <= 6) }}
-      uses: actions/checkout@v7
-      with:
-        ref: ${{ github.event.pull_request.base.ref }}
-    - name: Benchmark (Scalac) against Target Branch (4)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
-    - name: Benchmark (Shapeless) against Target Branch (5)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
-    - name: Benchmark (AnalysisFormatBenchmark) against Target Branch (6)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 6 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks"
+        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.jmhOptions=-f 1 -wi 0 -i 1 -w 1s -r 1s" "runBenchmarks"

build.sbt

@@ -690,11 +690,12 @@ addCommandAlias(
   "runBenchmarks", {
     val dir = IO.createTemporaryDirectory.getAbsolutePath
     val pattern = sys.props.getOrElse("benchmark.pattern", "")
+    val jmhOptions = sys.props.getOrElse("benchmark.jmhOptions", "")
     Seq(
       s"${compilerBridge213.id}/packageBin",
       s"${compilerBridge212.id}/packageBin",
       s"${zincBenchmarks.jvm(scala3).id}/Test/run $dir $pattern",
-      s"${zincBenchmarks.jvm(scala3).id}/Jmh/run -p _tempDir=$dir -prof gc -foe true $pattern",
+      s"${zincBenchmarks.jvm(scala3).id}/Jmh/run -p _tempDir=$dir -prof gc -foe true $jmhOptions $pattern",
       s"""eval IO.delete(file("$dir"))""",
     ).mkString(";", ";", "")
   }

contributing-docs/04_benchmark_tests.md

@@ -6,6 +6,11 @@ it will run a benchmark for Shapeless, but all benchmarks are welcome to be run
 on the Scala standard library and other well-known projects in the community,
 like Akka.
 
+`-Dbenchmark.pattern=<regex>` selects benchmarks, and `-Dbenchmark.jmhOptions` passes options to
+JMH. CI only checks that every benchmark runs, with
+`-Dbenchmark.jmhOptions="-f 1 -wi 0 -i 1 -w 1s -r 1s"`; its numbers mean nothing. Measure
+performance locally, comparing against the base branch on the same machine.
+
 If you add a new benchmark, make sure that you define the new benchmarking repo
 in [BenchmarkProjects.scala](https://github.com/sbt/zinc/blob/d532d15139f9f6e8346c8ffb649e564b25d7e897/internal/zinc-benchmarks/src/main/scala/xsbt/BenchmarkProjects.scala)
 and that you define how the benchmarks should be run (have a look at the