chore: Use Codspeed runners for wallclock benches

.github/actions/nss/action.yml

@@ -14,7 +14,7 @@ runs:
   steps:
     - name: Install system NSS (Linux)
       shell: bash
-      if: ${{ runner.os == 'Linux' && runner.environment == 'github-hosted' && inputs.target == '' }}
+      if: ${{ runner.os == 'Linux' && inputs.target == '' }}
       env:
         DEBIAN_FRONTEND: noninteractive
       run: |
@@ -23,7 +23,7 @@ runs:
 
     - name: Install system NSS (MacOS)
       shell: bash
-      if: ${{ runner.os == 'MacOS' && runner.environment == 'github-hosted' && inputs.target == '' }}
+      if: ${{ runner.os == 'MacOS' && inputs.target == '' }}
       run: |
         [ "$BREW_UPDATED" ] || brew update && echo "BREW_UPDATED=1" >> "$GITHUB_ENV"
         brew install nss
@@ -118,15 +118,15 @@ runs:
     - name: Store Ubuntu release code name (Linux)
       id: ubuntu_release
       shell: bash
-      if: ${{ runner.os == 'Linux' && !steps.system_nss.outputs.suitable && runner.environment == 'github-hosted' }}
+      if: ${{ runner.os == 'Linux' && !steps.system_nss.outputs.suitable }}
       run: |
         # Store Ubuntu release codename for use in cache key.
         . /etc/os-release
         echo "codename=-$UBUNTU_CODENAME" >> "$GITHUB_OUTPUT"
 
     - name: Cache NSS
       id: cache
-      if: ${{ !steps.system_nss.outputs.suitable && runner.environment == 'github-hosted' }}
+      if: ${{ !steps.system_nss.outputs.suitable }}
       uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
       with:
         path: dist
@@ -137,10 +137,9 @@ runs:
       if: ${{ !steps.system_nss.outputs.suitable }}
       env:
         CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
-        RUNNER_ENVIRONMENT: ${{ runner.environment }}
       shell: bash
       run: |
-        if [ "$RUNNER_ENVIRONMENT" != "github-hosted" ] || [ ! "$CACHE_HIT" ]; then
+        if [ ! "$CACHE_HIT" ]; then
           echo "Building NSS from source"
           echo "build_nss=1" >> "$GITHUB_OUTPUT"
         else
@@ -149,7 +148,7 @@ runs:
 
     - name: Install build dependencies (Linux)
       shell: bash
-      if: ${{ runner.os == 'Linux' && steps.check_build.outputs.build_nss && runner.environment == 'github-hosted' }}
+      if: ${{ runner.os == 'Linux' && steps.check_build.outputs.build_nss }}
       env:
         DEBIAN_FRONTEND: noninteractive
       run: sudo apt-get install -y --no-install-recommends gyp ninja-build

.github/workflows/bench.yml

@@ -9,8 +9,9 @@ on:
         type: choice
         description: Which benchmarking testbed to run on.
         required: true
-        default: on-prem
+        default: codspeed
         options:
+          - codspeed
           - on-prem
           - gcp
 env:
@@ -35,7 +36,7 @@ permissions:
 jobs:
   bench:
     name: cargo bench
-    runs-on: ${{ inputs.bencher == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || 'self-hosted' }}
+    runs-on: ${{ (inputs.bencher || 'codspeed') == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || (inputs.bencher || 'codspeed') == 'codspeed' && 'Neqo' || 'self-hosted' }}
     defaults:
       run:
         shell: bash
@@ -164,7 +165,7 @@ jobs:
         id: results
         env:
           EVENT_PATH: ${{ github.event_path }}
-          TESTBED: ${{ inputs.bencher == 'gcp' && 'GCP' || 'On-prem' }}
+          TESTBED: ${{ (inputs.bencher || 'codspeed') == 'gcp' && 'GCP' || (inputs.bencher || 'codspeed') == 'codspeed' && 'CodSpeed' || 'On-prem' }}
         run: |
           SHA=$(cd neqo && git log origin/main -1 --format=%H)
           echo "$SHA" > main-sha.txt

.github/workflows/codspeed.yml

@@ -1,5 +1,7 @@
 name: CodSpeed
 on:
+  push:
+    branches: ["main"] # To generate a performance baseline.
   pull_request:
     branches: ["main"]
   merge_group:
@@ -33,39 +35,30 @@ jobs:
       - id: benches
         run: |
           # Create a GitHub matrix where each entry is a bench with its
-          # associated crate and benchmarking mode taken from the Cargo
-          # metadata.
+          # associated crate, benchmarking mode, and runner taken from the
+          # Cargo metadata. Walltime benchmarks run on Codspeed runners,
+          # simulation benchmarks run on GitHub runners.
           {
-            echo -n "benches=["
-            first=true
-            while IFS=: read -r crate manifest_path; do
-              dir=$(dirname "$manifest_path")
-              if [ -e "$dir/benches" ]; then
-                for bench in "$dir/benches"/*.rs; do
-                  bench=$(basename -s .rs "$bench")
-                  # Set measurement mode from Cargo.toml metadata, default to "instrumentation".
-                  # The following jq pipeline:
-                  # - Selects the package with name == $crate
-                  # - Looks up .metadata.bench[$bench].codspeed.mode for the benchmark
-                  # - If not set, defaults to "instrumentation"
-                  mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"')
-                  # FIXME: Always use "instrumentation" mode to eliminate CI VM effects.
-                  mode="instrumentation"
-                  if [ "$first" = true ]; then
-                    first=false
-                  else
-                    echo -n ","
-                  fi
-                  echo -n "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}"
-                done
-              fi
-            done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path')
-            echo -n "]"
+            echo -n "benches="
+            cargo metadata --no-deps --format-version 1 | jq -c '
+              [.packages[] |
+                . as $pkg |
+                .targets[] |
+                select(.kind[] == "bench") |
+                select(has("required-features")) |
+                ($pkg.metadata.bench[.name].codspeed.mode // "simulation") as $mode |
+                {
+                  crate: $pkg.name,
+                  bench: .name,
+                  mode: $mode,
+                  runner: (if $mode == "walltime" then {group: "codspeed", labels: "codspeed-macro"} else "ubuntu-24.04" end)
+                }
+              ]'
           } >> "${GITHUB_OUTPUT}"
 
   benchmarks:
     name: Run bench
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.bench.runner }}
     strategy:
       matrix:
         bench: ${{ fromJson(needs.bench-matrix.outputs.benches) }}
@@ -94,7 +87,9 @@ jobs:
           BENCH: ${{ matrix.bench.bench }}
           CRATE: ${{ matrix.bench.crate }}
           MODE: ${{ matrix.bench.mode }}
-        run: cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE"
+        run: |
+          # FIXME: Codspeed is switching from "instrumentation" to "simulation", but the command line flag has not yet been updated.
+          cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "${MODE/simulation/instrumentation}"
 
       - name: Run bench & upload results
         uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1

.github/workflows/perfcompare.yml

@@ -9,8 +9,9 @@ on:
         type: choice
         description: Which benchmarking testbed to run on.
         required: true
-        default: on-prem
+        default: codspeed
         options:
+          - codspeed
           - on-prem
           - gcp
 env:
@@ -33,7 +34,7 @@ permissions:
 jobs:
   perfcompare:
     name: Performance comparison
-    runs-on: ${{ inputs.bencher == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || 'self-hosted' }}
+    runs-on: ${{ (inputs.bencher || 'codspeed') == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || (inputs.bencher || 'codspeed') == 'codspeed' && 'Neqo' || 'self-hosted' }}
     defaults:
       run:
         shell: bash
@@ -209,7 +210,7 @@ jobs:
         id: results
         env:
           EVENT_PATH: ${{ github.event_path }}
-          TESTBED: ${{ inputs.bencher == 'gcp' && 'GCP' || 'On-prem' }}
+          TESTBED: ${{ (inputs.bencher || 'codspeed') == 'gcp' && 'GCP' || (inputs.bencher || 'codspeed') == 'codspeed' && 'CodSpeed' || 'On-prem' }}
         run: |
           SHA=$(cd neqo && git log origin/main -1 --format=%H)
           echo "$SHA" > main-sha.txt

neqo-bin/Cargo.toml

@@ -11,6 +11,7 @@ license.workspace = true
 keywords.workspace = true
 categories.workspace = true
 readme.workspace = true
+autobenches = false
 
 [[bin]]
 name = "neqo-client"
@@ -65,7 +66,7 @@ fast-apple-datapath = ["quinn-udp/fast-apple-datapath"]
 draft-29 = ["neqo-http3/draft-29", "neqo-transport/draft-29"]
 
 [package.metadata.cargo-machete]
-ignored = ["log"]
+ignored = ["criterion"]
 
 [lib]
 # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options

neqo-common/Cargo.toml

@@ -11,6 +11,7 @@ description.workspace = true
 keywords.workspace = true
 categories.workspace = true
 readme.workspace = true
+autobenches = false
 
 [lints]
 workspace = true
@@ -39,6 +40,9 @@ build-fuzzing-corpus = ["hex/alloc", "sha1"]
 ci = []
 test-fixture = []
 
+[package.metadata.cargo-machete]
+ignored = ["criterion"]
+
 [lib]
 # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
 bench = false

neqo-http3/Cargo.toml

@@ -62,10 +62,18 @@ ignored = ["criterion", "log"]
 # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
 bench = false
 
-[package.metadata.bench.streams]
-codspeed.mode = "walltime" # Use "walltime" mode for "streams" bench with codspeed.
+[package.metadata.bench.streams_walltime]
+codspeed.mode = "walltime"
+
+[package.metadata.bench.streams_simulated]
+codspeed.mode = "simulation"
+
+[[bench]]
+name = "streams_walltime"
+harness = false
+required-features = ["bench"]
 
 [[bench]]
-name = "streams"
+name = "streams_simulated"
 harness = false
 required-features = ["bench"]

neqo-http3/benches/common.rs

@@ -0,0 +1,53 @@
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::time::Duration;
+
+use criterion::{BenchmarkGroup, Criterion};
+use test_fixture::{
+    boxed, fixture_init,
+    sim::{
+        http3_connection::{Node, Requests, Responses},
+        network::{RandomDelay, TailDrop},
+        ReadySimulator, Simulator,
+    },
+};
+
+const ZERO: Duration = Duration::from_millis(0);
+const JITTER: Duration = Duration::from_millis(10);
+
+/// Benchmark parameters: `(streams, data_size)`.
+const BENCHMARK_PARAMS: [(usize, usize); 3] = [(1, 1_000), (1_000, 1), (1_000, 1_000)];
+
+/// Creates a ready simulator for benchmarking HTTP/3 streams.
+pub fn setup(streams: usize, data_size: usize) -> ReadySimulator {
+    let nodes = boxed![
+        Node::default_client(boxed![Requests::new(streams, data_size)]),
+        TailDrop::dsl_uplink(),
+        RandomDelay::new(ZERO..JITTER),
+        Node::default_server(boxed![Responses::new(streams, data_size)]),
+        TailDrop::dsl_uplink(),
+        RandomDelay::new(ZERO..JITTER),
+    ];
+    Simulator::new("", nodes).setup()
+}
+
+/// Runs benchmarks for all parameter combinations.
+///
+/// The closure receives the benchmark group and parameters, allowing each
+/// benchmark to define its own measurement approach.
+pub fn benchmark<M>(c: &mut Criterion, mut measure: M)
+where
+    M: FnMut(&mut BenchmarkGroup<'_, criterion::measurement::WallTime>, usize, usize),
+{
+    fixture_init();
+
+    for (streams, data_size) in BENCHMARK_PARAMS {
+        let mut group = c.benchmark_group(format!("{streams}-streams/each-{data_size}-bytes"));
+        measure(&mut group, streams, data_size);
+        group.finish();
+    }
+}