diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index bd79512128..e80d74aa42 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -14,7 +14,7 @@ runs: steps: - name: Install system NSS (Linux) shell: bash - if: ${{ runner.os == 'Linux' && runner.environment == 'github-hosted' && inputs.target == '' }} + if: ${{ runner.os == 'Linux' && inputs.target == '' }} env: DEBIAN_FRONTEND: noninteractive run: | @@ -23,7 +23,7 @@ runs: - name: Install system NSS (MacOS) shell: bash - if: ${{ runner.os == 'MacOS' && runner.environment == 'github-hosted' && inputs.target == '' }} + if: ${{ runner.os == 'MacOS' && inputs.target == '' }} run: | [ "$BREW_UPDATED" ] || brew update && echo "BREW_UPDATED=1" >> "$GITHUB_ENV" brew install nss @@ -118,7 +118,7 @@ runs: - name: Store Ubuntu release code name (Linux) id: ubuntu_release shell: bash - if: ${{ runner.os == 'Linux' && !steps.system_nss.outputs.suitable && runner.environment == 'github-hosted' }} + if: ${{ runner.os == 'Linux' && !steps.system_nss.outputs.suitable }} run: | # Store Ubuntu release codename for use in cache key. . /etc/os-release @@ -126,7 +126,7 @@ runs: - name: Cache NSS id: cache - if: ${{ !steps.system_nss.outputs.suitable && runner.environment == 'github-hosted' }} + if: ${{ !steps.system_nss.outputs.suitable }} uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: dist @@ -137,10 +137,9 @@ runs: if: ${{ !steps.system_nss.outputs.suitable }} env: CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} - RUNNER_ENVIRONMENT: ${{ runner.environment }} shell: bash run: | - if [ "$RUNNER_ENVIRONMENT" != "github-hosted" ] || [ ! "$CACHE_HIT" ]; then + if [ ! "$CACHE_HIT" ]; then echo "Building NSS from source" echo "build_nss=1" >> "$GITHUB_OUTPUT" else @@ -149,7 +148,7 @@ runs: - name: Install build dependencies (Linux) shell: bash - if: ${{ runner.os == 'Linux' && steps.check_build.outputs.build_nss && runner.environment == 'github-hosted' }} + if: ${{ runner.os == 'Linux' && steps.check_build.outputs.build_nss }} env: DEBIAN_FRONTEND: noninteractive run: sudo apt-get install -y --no-install-recommends gyp ninja-build diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 78114fd15f..857d9f3e2e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -9,8 +9,9 @@ on: type: choice description: Which benchmarking testbed to run on. required: true - default: on-prem + default: codspeed options: + - codspeed - on-prem - gcp env: @@ -35,7 +36,7 @@ permissions: jobs: bench: name: cargo bench - runs-on: ${{ inputs.bencher == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || 'self-hosted' }} + runs-on: ${{ (inputs.bencher || 'codspeed') == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || (inputs.bencher || 'codspeed') == 'codspeed' && 'Neqo' || 'self-hosted' }} defaults: run: shell: bash @@ -164,7 +165,7 @@ jobs: id: results env: EVENT_PATH: ${{ github.event_path }} - TESTBED: ${{ inputs.bencher == 'gcp' && 'GCP' || 'On-prem' }} + TESTBED: ${{ (inputs.bencher || 'codspeed') == 'gcp' && 'GCP' || (inputs.bencher || 'codspeed') == 'codspeed' && 'CodSpeed' || 'On-prem' }} run: | SHA=$(cd neqo && git log origin/main -1 --format=%H) echo "$SHA" > main-sha.txt diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 4a310b90dc..8712ac730c 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -1,5 +1,7 @@ name: CodSpeed on: + push: + branches: ["main"] # To generate a performance baseline. pull_request: branches: ["main"] merge_group: @@ -33,39 +35,30 @@ jobs: - id: benches run: | # Create a GitHub matrix where each entry is a bench with its - # associated crate and benchmarking mode taken from the Cargo - # metadata. + # associated crate, benchmarking mode, and runner taken from the + # Cargo metadata. Walltime benchmarks run on Codspeed runners, + # simulation benchmarks run on GitHub runners. { - echo -n "benches=[" - first=true - while IFS=: read -r crate manifest_path; do - dir=$(dirname "$manifest_path") - if [ -e "$dir/benches" ]; then - for bench in "$dir/benches"/*.rs; do - bench=$(basename -s .rs "$bench") - # Set measurement mode from Cargo.toml metadata, default to "instrumentation". - # The following jq pipeline: - # - Selects the package with name == $crate - # - Looks up .metadata.bench[$bench].codspeed.mode for the benchmark - # - If not set, defaults to "instrumentation" - mode=$(cargo metadata --no-deps --format-version 1 | jq -r --arg crate "$crate" --arg bench "$bench" '.packages[] | select(.name == $crate) | .metadata.bench[$bench].codspeed.mode // "instrumentation"') - # FIXME: Always use "instrumentation" mode to eliminate CI VM effects. - mode="instrumentation" - if [ "$first" = true ]; then - first=false - else - echo -n "," - fi - echo -n "{\"crate\":\"$crate\",\"bench\":\"$bench\",\"mode\":\"$mode\"}" - done - fi - done < <(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | .name + ":" + .manifest_path') - echo -n "]" + echo -n "benches=" + cargo metadata --no-deps --format-version 1 | jq -c ' + [.packages[] | + . as $pkg | + .targets[] | + select(.kind[] == "bench") | + select(has("required-features")) | + ($pkg.metadata.bench[.name].codspeed.mode // "simulation") as $mode | + { + crate: $pkg.name, + bench: .name, + mode: $mode, + runner: (if $mode == "walltime" then {group: "codspeed", labels: "codspeed-macro"} else "ubuntu-24.04" end) + } + ]' } >> "${GITHUB_OUTPUT}" benchmarks: name: Run bench - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.bench.runner }} strategy: matrix: bench: ${{ fromJson(needs.bench-matrix.outputs.benches) }} @@ -94,7 +87,9 @@ jobs: BENCH: ${{ matrix.bench.bench }} CRATE: ${{ matrix.bench.crate }} MODE: ${{ matrix.bench.mode }} - run: cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "$MODE" + run: | + # FIXME: Codspeed is switching from "instrumentation" to "simulation", but the command line flag has not yet been updated. + cargo codspeed build --package "$CRATE" --locked --features bench --bench "$BENCH" --measurement-mode "${MODE/simulation/instrumentation}" - name: Run bench & upload results uses: CodSpeedHQ/action@346a2d8a8d9d38909abd0bc3d23f773110f076ad # v4.4.1 diff --git a/.github/workflows/perfcompare.yml b/.github/workflows/perfcompare.yml index d11f3458f7..1ef13ac336 100644 --- a/.github/workflows/perfcompare.yml +++ b/.github/workflows/perfcompare.yml @@ -9,8 +9,9 @@ on: type: choice description: Which benchmarking testbed to run on. required: true - default: on-prem + default: codspeed options: + - codspeed - on-prem - gcp env: @@ -33,7 +34,7 @@ permissions: jobs: perfcompare: name: Performance comparison - runs-on: ${{ inputs.bencher == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || 'self-hosted' }} + runs-on: ${{ (inputs.bencher || 'codspeed') == 'gcp' && format('cirun-gcp-bencher--{0}', github.run_id) || (inputs.bencher || 'codspeed') == 'codspeed' && 'Neqo' || 'self-hosted' }} defaults: run: shell: bash @@ -209,7 +210,7 @@ jobs: id: results env: EVENT_PATH: ${{ github.event_path }} - TESTBED: ${{ inputs.bencher == 'gcp' && 'GCP' || 'On-prem' }} + TESTBED: ${{ (inputs.bencher || 'codspeed') == 'gcp' && 'GCP' || (inputs.bencher || 'codspeed') == 'codspeed' && 'CodSpeed' || 'On-prem' }} run: | SHA=$(cd neqo && git log origin/main -1 --format=%H) echo "$SHA" > main-sha.txt diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml index 51920ce869..6dbd536bd8 100644 --- a/neqo-bin/Cargo.toml +++ b/neqo-bin/Cargo.toml @@ -11,6 +11,7 @@ license.workspace = true keywords.workspace = true categories.workspace = true readme.workspace = true +autobenches = false [[bin]] name = "neqo-client" @@ -65,7 +66,7 @@ fast-apple-datapath = ["quinn-udp/fast-apple-datapath"] draft-29 = ["neqo-http3/draft-29", "neqo-transport/draft-29"] [package.metadata.cargo-machete] -ignored = ["log"] +ignored = ["criterion"] [lib] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml index a651b1b67f..8e9228ad2d 100644 --- a/neqo-common/Cargo.toml +++ b/neqo-common/Cargo.toml @@ -11,6 +11,7 @@ description.workspace = true keywords.workspace = true categories.workspace = true readme.workspace = true +autobenches = false [lints] workspace = true @@ -39,6 +40,9 @@ build-fuzzing-corpus = ["hex/alloc", "sha1"] ci = [] test-fixture = [] +[package.metadata.cargo-machete] +ignored = ["criterion"] + [lib] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml index 16a9592dd3..3aeccab1ba 100644 --- a/neqo-http3/Cargo.toml +++ b/neqo-http3/Cargo.toml @@ -62,10 +62,18 @@ ignored = ["criterion", "log"] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false -[package.metadata.bench.streams] -codspeed.mode = "walltime" # Use "walltime" mode for "streams" bench with codspeed. +[package.metadata.bench.streams_walltime] +codspeed.mode = "walltime" + +[package.metadata.bench.streams_simulated] +codspeed.mode = "simulation" + +[[bench]] +name = "streams_walltime" +harness = false +required-features = ["bench"] [[bench]] -name = "streams" +name = "streams_simulated" harness = false required-features = ["bench"] diff --git a/neqo-http3/benches/common.rs b/neqo-http3/benches/common.rs new file mode 100644 index 0000000000..7202779d1b --- /dev/null +++ b/neqo-http3/benches/common.rs @@ -0,0 +1,53 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::time::Duration; + +use criterion::{BenchmarkGroup, Criterion}; +use test_fixture::{ + boxed, fixture_init, + sim::{ + http3_connection::{Node, Requests, Responses}, + network::{RandomDelay, TailDrop}, + ReadySimulator, Simulator, + }, +}; + +const ZERO: Duration = Duration::from_millis(0); +const JITTER: Duration = Duration::from_millis(10); + +/// Benchmark parameters: `(streams, data_size)`. +const BENCHMARK_PARAMS: [(usize, usize); 3] = [(1, 1_000), (1_000, 1), (1_000, 1_000)]; + +/// Creates a ready simulator for benchmarking HTTP/3 streams. +pub fn setup(streams: usize, data_size: usize) -> ReadySimulator { + let nodes = boxed![ + Node::default_client(boxed![Requests::new(streams, data_size)]), + TailDrop::dsl_uplink(), + RandomDelay::new(ZERO..JITTER), + Node::default_server(boxed![Responses::new(streams, data_size)]), + TailDrop::dsl_uplink(), + RandomDelay::new(ZERO..JITTER), + ]; + Simulator::new("", nodes).setup() +} + +/// Runs benchmarks for all parameter combinations. +/// +/// The closure receives the benchmark group and parameters, allowing each +/// benchmark to define its own measurement approach. +pub fn benchmark(c: &mut Criterion, mut measure: M) +where + M: FnMut(&mut BenchmarkGroup<'_, criterion::measurement::WallTime>, usize, usize), +{ + fixture_init(); + + for (streams, data_size) in BENCHMARK_PARAMS { + let mut group = c.benchmark_group(format!("{streams}-streams/each-{data_size}-bytes")); + measure(&mut group, streams, data_size); + group.finish(); + } +} diff --git a/neqo-http3/benches/streams.rs b/neqo-http3/benches/streams.rs deleted file mode 100644 index c9ff4c6ac3..0000000000 --- a/neqo-http3/benches/streams.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![expect( - clippy::significant_drop_tightening, - reason = "Inherent in codspeed criterion_group! macro." -)] - -use std::{hint::black_box, time::Duration}; - -use criterion::{criterion_group, criterion_main, Criterion, Throughput}; -use test_fixture::{ - boxed, fixture_init, - sim::{ - http3_connection::{Node, Requests, Responses}, - network::{RandomDelay, TailDrop}, - ReadySimulator, Simulator, - }, -}; - -const ZERO: Duration = Duration::from_millis(0); -const JITTER: Duration = Duration::from_millis(10); - -fn criterion_benchmark(c: &mut Criterion) { - fixture_init(); - - for (streams, data_size) in [(1usize, 1_000usize), (1_000, 1), (1_000, 1_000)] { - let setup = || { - let nodes = boxed![ - Node::default_client(boxed![Requests::new(streams, data_size)]), - TailDrop::dsl_uplink(), - RandomDelay::new(ZERO..JITTER), - Node::default_server(boxed![Responses::new(streams, data_size)]), - TailDrop::dsl_uplink(), - RandomDelay::new(ZERO..JITTER), - ]; - Simulator::new("", nodes).setup() - }; - let routine = |sim: ReadySimulator| black_box(sim.run()); - - let mut group = c.benchmark_group(format!("{streams}-streams/each-{data_size}-bytes")); - - // Benchmark with wallclock time, i.e. measure the compute efficiency. - group.bench_function("wallclock-time", |b| { - b.iter_batched(setup, routine, criterion::BatchSize::SmallInput); - }); - - // Benchmark with simulated time, i.e. measure the network protocol - // efficiency. - // - // Note: Given that this is using simulated time, we can measure actual - // throughput. - group.throughput(Throughput::Bytes((streams * data_size) as u64)); - group.bench_function("simulated-time", |b| { - b.iter_custom(|iters| { - let mut d_sum = Duration::ZERO; - for _i in 0..iters { - d_sum += setup().run(); - } - - d_sum - }); - }); - - group.finish(); - } -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/neqo-http3/benches/streams_simulated.rs b/neqo-http3/benches/streams_simulated.rs new file mode 100644 index 0000000000..3cc0fdd751 --- /dev/null +++ b/neqo-http3/benches/streams_simulated.rs @@ -0,0 +1,39 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Benchmark with simulated time, i.e., measure the network protocol efficiency. +//! +//! Given that this uses simulated time, we can measure actual throughput. + +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion, Throughput}; + +#[path = "common.rs"] +mod common; + +fn benchmark(c: &mut Criterion) { + common::benchmark(c, |group, streams, data_size| { + group.throughput(Throughput::Bytes((streams * data_size) as u64)); + group.bench_function("simulated-time", |b| { + b.iter_custom(|iters| { + let mut d_sum = Duration::ZERO; + for _i in 0..iters { + d_sum += common::setup(streams, data_size).run(); + } + d_sum + }); + }); + }); +} + +criterion_group!(benches, benchmark); +criterion_main!(benches); diff --git a/neqo-http3/benches/streams_walltime.rs b/neqo-http3/benches/streams_walltime.rs new file mode 100644 index 0000000000..4060f451d5 --- /dev/null +++ b/neqo-http3/benches/streams_walltime.rs @@ -0,0 +1,34 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Benchmark with walltime, i.e., measure the compute efficiency. + +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, Criterion}; + +#[path = "common.rs"] +mod common; + +fn benchmark(c: &mut Criterion) { + common::benchmark(c, |group, streams, data_size| { + group.bench_function("walltime", |b| { + b.iter_batched( + || common::setup(streams, data_size), + |sim| black_box(sim.run()), + criterion::BatchSize::SmallInput, + ); + }); + }); +} + +criterion_group!(benches, benchmark); +criterion_main!(benches); diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml index 4be52ab21d..5c5707f285 100644 --- a/neqo-transport/Cargo.toml +++ b/neqo-transport/Cargo.toml @@ -11,6 +11,7 @@ description.workspace = true keywords.workspace = true categories.workspace = true readme.workspace = true +autobenches = false [lints] workspace = true @@ -48,12 +49,26 @@ disable-encryption = ["neqo-crypto/disable-encryption"] draft-29 = [] gecko = ["mtu/gecko"] +[package.metadata.cargo-machete] +ignored = ["criterion"] + [lib] # See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options bench = false +[package.metadata.bench.transfer_walltime] +codspeed.mode = "walltime" + +[package.metadata.bench.transfer_simulated] +codspeed.mode = "simulation" + +[[bench]] +name = "transfer_walltime" +harness = false +required-features = ["bench"] + [[bench]] -name = "transfer" +name = "transfer_simulated" harness = false required-features = ["bench"] diff --git a/neqo-transport/benches/transfer.rs b/neqo-transport/benches/transfer.rs deleted file mode 100644 index 19509fe545..0000000000 --- a/neqo-transport/benches/transfer.rs +++ /dev/null @@ -1,121 +0,0 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![expect( - clippy::significant_drop_tightening, - reason = "Inherent in codspeed criterion_group! macro." -)] - -use std::{hint::black_box, time::Duration}; - -use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion}; -use neqo_transport::{ConnectionParameters, State}; -use test_fixture::{ - boxed, - sim::{ - connection::{Node, ReachState, ReceiveData, SendData}, - network::{Delay, TailDrop}, - Simulator, - }, -}; - -const DELAY: Duration = Duration::from_millis(10); -const TRANSFER_AMOUNT: usize = 1 << 22; // 4Mbyte - -#[expect( - clippy::needless_pass_by_value, - reason = "Passing String where &str would do is fine here." -)] -fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option>) { - for pacing in [false, true] { - let setup = || { - let nodes = boxed![ - Node::new_client( - ConnectionParameters::default() - .pmtud(true) - .pacing(pacing) - .mlkem(false), - boxed![ReachState::new(State::Confirmed)], - boxed![SendData::new(TRANSFER_AMOUNT)] - ), - TailDrop::dsl_uplink(), - Delay::new(DELAY), - Node::new_server( - ConnectionParameters::default() - .pmtud(true) - .pacing(pacing) - .mlkem(false), - boxed![ReachState::new(State::Confirmed)], - boxed![ReceiveData::new(TRANSFER_AMOUNT)] - ), - TailDrop::dsl_downlink(), - Delay::new(DELAY), - ]; - let mut sim = Simulator::new(label, nodes); - if let Some(seed) = &seed { - sim.seed_str(seed); - } - sim.setup() - }; - - // Benchmark with wallclock time, i.e. measure the compute efficiency. - { - let mut group = - c.benchmark_group(format!("transfer/pacing-{pacing}/{label}/wallclock-time")); - group.noise_threshold(0.03); - group.bench_function("run", |b| { - b.iter_batched( - setup, - |s| { - black_box(s.run()); - }, - SmallInput, - ); - }); - } - - // Benchmark with simulated time, i.e. measure the network protocol - // efficiency. - // - // Note: Given that this is using simulated time, we can measure actual - // throughput. - { - let mut group = - c.benchmark_group(format!("transfer/pacing-{pacing}/{label}/simulated-time")); - group.throughput(criterion::Throughput::Bytes(TRANSFER_AMOUNT as u64)); - group.bench_function("run", |b| { - b.iter_custom(|iters| { - let mut d_sum = Duration::ZERO; - for _i in 0..iters { - // run() returns the simulated time, excluding setup time. - // No need for black_box(), because this doesn't measure compute. - d_sum += setup().run(); - } - d_sum - }); - }); - } - } -} - -fn benchmark_transfer_variable(c: &mut Criterion) { - benchmark_transfer(c, "varying-seeds", std::env::var("SIMULATION_SEED").ok()); -} - -fn benchmark_transfer_fixed(c: &mut Criterion) { - benchmark_transfer( - c, - "same-seed", - Some("62df6933ba1f543cece01db8f27fb2025529b27f93df39e19f006e1db3b8c843"), - ); -} - -criterion_group! { - name = transfer; - config = Criterion::default().warm_up_time(Duration::from_secs(5)).measurement_time(Duration::from_secs(15)); - targets = benchmark_transfer_variable, benchmark_transfer_fixed -} -criterion_main!(transfer); diff --git a/neqo-transport/benches/transfer_common.rs b/neqo-transport/benches/transfer_common.rs new file mode 100644 index 0000000000..0b0bc1fbfc --- /dev/null +++ b/neqo-transport/benches/transfer_common.rs @@ -0,0 +1,88 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::time::Duration; + +use criterion::{BenchmarkGroup, Criterion}; +use neqo_transport::{ConnectionParameters, State}; +use test_fixture::{ + boxed, + sim::{ + connection::{Node, ReachState, ReceiveData, SendData}, + network::{Delay, TailDrop}, + ReadySimulator, Simulator, + }, +}; + +const DELAY: Duration = Duration::from_millis(10); +pub const TRANSFER_AMOUNT: usize = 1 << 22; // 4Mbyte + +const FIXED_SEED: &str = "62df6933ba1f543cece01db8f27fb2025529b27f93df39e19f006e1db3b8c843"; + +/// Creates a ready simulator for benchmarking transfer. +#[must_use] +pub fn setup(label: &str, seed: Option<&str>, pacing: bool) -> ReadySimulator { + let nodes = boxed![ + Node::new_client( + ConnectionParameters::default() + .pmtud(true) + .pacing(pacing) + .mlkem(false), + boxed![ReachState::new(State::Confirmed)], + boxed![SendData::new(TRANSFER_AMOUNT)] + ), + TailDrop::dsl_uplink(), + Delay::new(DELAY), + Node::new_server( + ConnectionParameters::default() + .pmtud(true) + .pacing(pacing) + .mlkem(false), + boxed![ReachState::new(State::Confirmed)], + boxed![ReceiveData::new(TRANSFER_AMOUNT)] + ), + TailDrop::dsl_downlink(), + Delay::new(DELAY), + ]; + let mut sim = Simulator::new(label, nodes); + if let Some(seed) = seed { + sim.seed_str(seed); + } + sim.setup() +} + +/// Runs transfer benchmarks for all configurations. +/// +/// The closure receives the benchmark group, label, seed, and pacing flag, +/// allowing each benchmark to define its own measurement approach. +pub fn benchmark(c: &mut Criterion, mut measure: M) +where + M: FnMut(&mut BenchmarkGroup<'_, criterion::measurement::WallTime>, &str, Option<&str>, bool), +{ + // Handle SIMULATION_SEED environment variable for varying-seeds config + let env_seed = std::env::var("SIMULATION_SEED").ok(); + let configs: [(&str, Option<&str>); 2] = [ + ("varying-seeds", env_seed.as_deref()), + ("same-seed", Some(FIXED_SEED)), + ]; + + for (label, seed) in configs { + for pacing in [false, true] { + let mut group = c.benchmark_group(format!("transfer/pacing-{pacing}/{label}")); + group.noise_threshold(0.03); + measure(&mut group, label, seed, pacing); + group.finish(); + } + } +} + +/// Returns the criterion configuration for transfer benchmarks. +#[must_use] +pub fn criterion_config() -> Criterion { + Criterion::default() + .warm_up_time(Duration::from_secs(5)) + .measurement_time(Duration::from_secs(15)) +} diff --git a/neqo-transport/benches/transfer_simulated.rs b/neqo-transport/benches/transfer_simulated.rs new file mode 100644 index 0000000000..6ea45ffc9c --- /dev/null +++ b/neqo-transport/benches/transfer_simulated.rs @@ -0,0 +1,43 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Benchmark with simulated time, i.e., measure the network protocol efficiency. +//! +//! Given that this uses simulated time, we can measure actual throughput. + +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Throughput}; + +#[path = "transfer_common.rs"] +mod common; + +fn benchmark(c: &mut criterion::Criterion) { + common::benchmark(c, |group, label, seed, pacing| { + group.throughput(Throughput::Bytes(common::TRANSFER_AMOUNT as u64)); + group.bench_function("simulated-time", |b| { + b.iter_custom(|iters| { + let mut d_sum = Duration::ZERO; + for _i in 0..iters { + d_sum += common::setup(label, seed, pacing).run(); + } + d_sum + }); + }); + }); +} + +criterion_group! { + name = transfer; + config = common::criterion_config(); + targets = benchmark +} +criterion_main!(transfer); diff --git a/neqo-transport/benches/transfer_walltime.rs b/neqo-transport/benches/transfer_walltime.rs new file mode 100644 index 0000000000..c71d6dbbce --- /dev/null +++ b/neqo-transport/benches/transfer_walltime.rs @@ -0,0 +1,38 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Benchmark with walltime, i.e., measure the compute efficiency. + +#![expect( + clippy::significant_drop_tightening, + reason = "Inherent in codspeed criterion_group! macro." +)] + +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, BatchSize::SmallInput}; + +#[path = "transfer_common.rs"] +mod common; + +fn benchmark(c: &mut criterion::Criterion) { + common::benchmark(c, |group, label, seed, pacing| { + group.bench_function("walltime", |b| { + b.iter_batched( + || common::setup(label, seed, pacing), + |sim| black_box(sim.run()), + SmallInput, + ); + }); + }); +} + +criterion_group! { + name = transfer; + config = common::criterion_config(); + targets = benchmark +} +criterion_main!(transfer);