monorepo/.github/benchmark-tracking.toml at 6e2e06633eefc0ed92bf9d5642e9ae375c70539b · commonwarexyz/monorepo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
# Benchmark tracking configuration.
#
# Each variant is passed to Criterion with `--exact`, so names must match the full benchmark name
# printed by `cargo bench -- --list` or `cargo bench -- --output-format bencher`.
#
# Add one `[[packages]]` table per Cargo package/feature configuration. The same package may appear
# multiple times with different `cargo_flags`, for example to run storage benchmarks with and
# without io_uring. When doing this, set a distinct `baseline_suite` for each feature
# configuration so results are matched against the correct main-branch baseline.
#
# Thresholds can be set globally with `default_threshold_percent`, per package, per benchmark, or
# per variant table. More specific settings override less specific settings.
#
# Variants may be specified as strings:
#
#   variants = ["crate::bench/key=value"]
#
# or as tables when per-variant overrides are needed:
#
#   [[packages.benchmarks.variants]]
#   name = "crate::bench/key=value"
#   threshold_percent = 20.0
#
# Example duplicate package configuration:
#
#   [[packages]]
#   name = "commonware-storage"
#   baseline_suite = "commonware-storage"
#   cargo_flags = ["--features", "test-traits"]
#
#   [[packages.benchmarks]]
#   name = "qmdb"
#   variants = ["qmdb::merkleize/variant=... keys=10000 ch=false sync=false"]
#
#   [[packages]]
#   name = "commonware-storage"
#   baseline_suite = "commonware-storage-iouring"
#   cargo_flags = ["--features", "commonware-runtime/iouring-storage,test-traits"]
#
#   [[packages.benchmarks]]
#   name = "qmdb"
#   variants = ["qmdb::merkleize/variant=... keys=10000 ch=false sync=false"]

default_threshold_percent = 10.0

[[packages]]
name = "commonware-storage"
cargo_flags = ["--features", "test-traits"]

[[packages.benchmarks]]
name = "qmdb"
criterion_args = ["--sample-size", "100", "--significance-level", "0.01"]
variants = [
  "qmdb::merkleize/variant=any::unordered::fixed::mmr keys=10000 ch=false sync=false",
  "qmdb::merkleize/variant=current::ordered::fixed::mmb keys=10000 ch=true sync=false",
]