Merge pull request #1831 from lrytz/bench-smoke-only

Only smoke-test the benchmarks on CI

Author: eed3si9n

.github/workflows/ci.yml

@@ -30,14 +30,6 @@ jobs:
             java: 21
             distribution: temurin
             jobtype: 4
-          - os: ubuntu-latest
-            java: 21
-            distribution: temurin
-            jobtype: 5
-          - os: ubuntu-latest
-            java: 21
-            distribution: temurin
-            jobtype: 6
     runs-on: ${{ matrix.os }}
     steps:
     - name: Checkout
@@ -65,38 +57,8 @@ jobs:
       run: |
         sbt -v --server -Dfile.encoding=UTF-8 scalafmtCheckRepo headerCheck "Test/headerCheck" generateContrabands
         git diff --exit-code
-    - name: Benchmark (Scalac) (4)
+    - name: Benchmark smoke test (4)
       if: ${{ matrix.jobtype == 4 }}
       shell: bash
       run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
-    - name: Benchmark (Shapeless) (5)
-      if: ${{ matrix.jobtype == 5 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
-    - name: Benchmark (AnalysisFormatBenchmark) (6)
-      if: ${{ matrix.jobtype == 6 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks"
-    - name: Checkout Target Branch (4-6)
-      if: ${{ github.event_name == 'pull_request' && (matrix.jobtype >= 4 && matrix.jobtype <= 6) }}
-      uses: actions/checkout@v7
-      with:
-        ref: ${{ github.event.pull_request.base.ref }}
-    - name: Benchmark (Scalac) against Target Branch (4)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 4 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Scalac.*" "runBenchmarks"
-    - name: Benchmark (Shapeless) against Target Branch (5)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 5 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*Shapeless.*" "runBenchmarks"
-    - name: Benchmark (AnalysisFormatBenchmark) against Target Branch (6)
-      if: ${{ github.event_name == 'pull_request' && matrix.jobtype == 6 }}
-      shell: bash
-      run: |
-        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.pattern=.*AnalysisFormatBenchmark.*" "runBenchmarks"
+        sbt -v --server -Dfile.encoding=UTF-8 "-Dbenchmark.jmhOptions=-f 1 -wi 0 -i 1 -w 1s -r 1s" "runBenchmarks"

build.sbt

@@ -690,11 +690,12 @@ addCommandAlias(
   "runBenchmarks", {
     val dir = IO.createTemporaryDirectory.getAbsolutePath
     val pattern = sys.props.getOrElse("benchmark.pattern", "")
+    val jmhOptions = sys.props.getOrElse("benchmark.jmhOptions", "")
     Seq(
       s"${compilerBridge213.id}/packageBin",
       s"${compilerBridge212.id}/packageBin",
       s"${zincBenchmarks.jvm(scala3).id}/Test/run $dir $pattern",
-      s"${zincBenchmarks.jvm(scala3).id}/Jmh/run -p _tempDir=$dir -prof gc -foe true $pattern",
+      s"${zincBenchmarks.jvm(scala3).id}/Jmh/run -p _tempDir=$dir -prof gc -foe true $jmhOptions $pattern",
       s"""eval IO.delete(file("$dir"))""",
     ).mkString(";", ";", "")
   }

contributing-docs/04_benchmark_tests.md

@@ -6,6 +6,11 @@ it will run a benchmark for Shapeless, but all benchmarks are welcome to be run
 on the Scala standard library and other well-known projects in the community,
 like Akka.
 
+`-Dbenchmark.pattern=<regex>` selects benchmarks, and `-Dbenchmark.jmhOptions` passes options to
+JMH. CI only checks that every benchmark runs, with
+`-Dbenchmark.jmhOptions="-f 1 -wi 0 -i 1 -w 1s -r 1s"`; its numbers mean nothing. Measure
+performance locally, comparing against the base branch on the same machine.
+
 If you add a new benchmark, make sure that you define the new benchmarking repo
 in [BenchmarkProjects.scala](https://github.com/sbt/zinc/blob/d532d15139f9f6e8346c8ffb649e564b25d7e897/internal/zinc-benchmarks/src/main/scala/xsbt/BenchmarkProjects.scala)
 and that you define how the benchmarks should be run (have a look at the