Skip to content

Commit 4bd58be

Browse files
roberto-bayardoroberto-bayardo
authored
[storage/qmdb] appropriate items-per-blob in apply_batch benchmark (#3766)
1 parent e1c64d0 commit 4bd58be

2 files changed

Lines changed: 100 additions & 22 deletions

File tree

‎storage/src/qmdb/benches/apply_batch.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,24 @@
11
//! Benchmarks for applying already-merkleized QMDB batches.
22
3-
use crate::common::{any_fix_cfg, make_fixed_value, seed_db, AnyUFixDb, CHUNK_SIZE};
3+
use crate::common::{any_fix_cfg_with, make_fixed_value, seed_db, AnyUFixDb, CHUNK_SIZE};
44
use commonware_cryptography::{Hasher as _, Sha256};
55
use commonware_runtime::{
66
benchmarks::{context, tokio},
77
tokio::{Config, Context},
88
Supervisor,
99
};
1010
use commonware_storage::{merkle::mmb::Family as Mmb, qmdb::any::traits::BatchableDb};
11+
use commonware_utils::NZU64;
1112
use criterion::{criterion_group, Criterion};
1213
use rand::{rngs::StdRng, RngCore, SeedableRng};
13-
use std::time::{Duration, Instant};
14+
use std::{
15+
num::NonZeroU64,
16+
time::{Duration, Instant},
17+
};
1418

1519
const NUM_KEYS: u64 = 65_536;
1620
const UPDATES: [u64; 1] = [16_384];
21+
const ITEMS_PER_BLOB: NonZeroU64 = NZU64!(10_000_000);
1722

1823
type Db = AnyUFixDb<Mmb>;
1924

@@ -31,7 +36,7 @@ fn write_updates(
3136
}
3237

3338
async fn open_db(ctx: &Context) -> Db {
34-
Db::init(ctx.child("storage"), any_fix_cfg(ctx))
39+
Db::init(ctx.child("storage"), any_fix_cfg_with(ctx, ITEMS_PER_BLOB))
3540
.await
3641
.unwrap()
3742
}

‎storage/src/qmdb/benches/common.rs‎

Lines changed: 92 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ use std::num::{NonZeroU16, NonZeroU64, NonZeroUsize};
2929

3030
pub type Digest = <Sha256 as Hasher>::Digest;
3131

32+
/// Default items per blob for benchmarks. This is small enough that blob boundary crossings
33+
/// (which trigger fsync) can dominate benchmark time. Benchmarks that don't want to measure
34+
/// that cost should override via the `_with` config generators.
3235
pub const ITEMS_PER_BLOB: NonZeroU64 = NZU64!(50_000);
3336
pub const CHUNK_SIZE: usize = 32;
3437
pub const THREADS: NonZeroUsize = NZUsize!(8);
@@ -84,30 +87,36 @@ fn merkle_cfg(
8487
suffix: &str,
8588
ctx: &(impl BufferPooler + ThreadPooler),
8689
page_cache: CacheRef,
90+
items_per_blob: NonZeroU64,
8791
) -> MerkleConfig<Rayon> {
8892
MerkleConfig {
8993
journal_partition: format!("journal-{suffix}"),
9094
metadata_partition: format!("metadata-{suffix}"),
91-
items_per_blob: ITEMS_PER_BLOB,
95+
items_per_blob,
9296
write_buffer: WRITE_BUFFER_SIZE,
9397
strategy: ctx.create_strategy(THREADS).unwrap(),
9498
page_cache,
9599
}
96100
}
97101

98-
fn fix_log_cfg(suffix: &str, page_cache: CacheRef) -> FConfig {
102+
fn fix_log_cfg(suffix: &str, page_cache: CacheRef, items_per_blob: NonZeroU64) -> FConfig {
99103
FConfig {
100104
partition: format!("log-journal-{suffix}"),
101-
items_per_blob: ITEMS_PER_BLOB,
105+
items_per_blob,
102106
page_cache,
103107
write_buffer: WRITE_BUFFER_SIZE,
104108
}
105109
}
106110

107-
fn var_log_cfg<C>(suffix: &str, page_cache: CacheRef, codec_config: C) -> VConfig<C> {
111+
fn var_log_cfg<C>(
112+
suffix: &str,
113+
page_cache: CacheRef,
114+
codec_config: C,
115+
items_per_section: NonZeroU64,
116+
) -> VConfig<C> {
108117
VConfig {
109118
partition: format!("log-journal-{suffix}"),
110-
items_per_section: ITEMS_PER_BLOB,
119+
items_per_section,
111120
compression: None,
112121
codec_config,
113122
page_cache,
@@ -116,44 +125,72 @@ fn var_log_cfg<C>(suffix: &str, page_cache: CacheRef, codec_config: C) -> VConfi
116125
}
117126

118127
pub fn any_fix_cfg(ctx: &(impl BufferPooler + ThreadPooler)) -> AnyFixedConfig<EightCap, Rayon> {
128+
any_fix_cfg_with(ctx, ITEMS_PER_BLOB)
129+
}
130+
131+
pub fn any_fix_cfg_with(
132+
ctx: &(impl BufferPooler + ThreadPooler),
133+
items_per_blob: NonZeroU64,
134+
) -> AnyFixedConfig<EightCap, Rayon> {
119135
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
120136
AnyFixedConfig {
121-
merkle_config: merkle_cfg(PARTITION_FIX, ctx, page_cache.clone()),
122-
journal_config: fix_log_cfg(PARTITION_FIX, page_cache),
137+
merkle_config: merkle_cfg(PARTITION_FIX, ctx, page_cache.clone(), items_per_blob),
138+
journal_config: fix_log_cfg(PARTITION_FIX, page_cache, items_per_blob),
123139
translator: EightCap,
124140
}
125141
}
126142

127143
pub fn cur_fix_cfg(
128144
ctx: &(impl BufferPooler + ThreadPooler),
145+
) -> CurrentFixedConfig<EightCap, Rayon> {
146+
cur_fix_cfg_with(ctx, ITEMS_PER_BLOB)
147+
}
148+
149+
pub fn cur_fix_cfg_with(
150+
ctx: &(impl BufferPooler + ThreadPooler),
151+
items_per_blob: NonZeroU64,
129152
) -> CurrentFixedConfig<EightCap, Rayon> {
130153
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
131154
CurrentFixedConfig {
132-
merkle_config: merkle_cfg(PARTITION_FIX, ctx, page_cache.clone()),
133-
journal_config: fix_log_cfg(PARTITION_FIX, page_cache),
155+
merkle_config: merkle_cfg(PARTITION_FIX, ctx, page_cache.clone(), items_per_blob),
156+
journal_config: fix_log_cfg(PARTITION_FIX, page_cache, items_per_blob),
134157
grafted_metadata_partition: format!("grafted-metadata-{PARTITION_FIX}"),
135158
translator: EightCap,
136159
}
137160
}
138161

139162
pub fn any_var_digest_cfg(
140163
ctx: &(impl BufferPooler + ThreadPooler),
164+
) -> AnyVariableConfig<EightCap, ((), ()), Rayon> {
165+
any_var_digest_cfg_with(ctx, ITEMS_PER_BLOB)
166+
}
167+
168+
pub fn any_var_digest_cfg_with(
169+
ctx: &(impl BufferPooler + ThreadPooler),
170+
items_per_blob: NonZeroU64,
141171
) -> AnyVariableConfig<EightCap, ((), ()), Rayon> {
142172
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
143173
AnyVariableConfig {
144-
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone()),
145-
journal_config: var_log_cfg(PARTITION_VAR, page_cache, ((), ())),
174+
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone(), items_per_blob),
175+
journal_config: var_log_cfg(PARTITION_VAR, page_cache, ((), ()), items_per_blob),
146176
translator: EightCap,
147177
}
148178
}
149179

150180
pub fn cur_var_digest_cfg(
151181
ctx: &(impl BufferPooler + ThreadPooler),
182+
) -> CurrentVariableConfig<EightCap, ((), ()), Rayon> {
183+
cur_var_digest_cfg_with(ctx, ITEMS_PER_BLOB)
184+
}
185+
186+
pub fn cur_var_digest_cfg_with(
187+
ctx: &(impl BufferPooler + ThreadPooler),
188+
items_per_blob: NonZeroU64,
152189
) -> CurrentVariableConfig<EightCap, ((), ()), Rayon> {
153190
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
154191
CurrentVariableConfig {
155-
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone()),
156-
journal_config: var_log_cfg(PARTITION_VAR, page_cache, ((), ())),
192+
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone(), items_per_blob),
193+
journal_config: var_log_cfg(PARTITION_VAR, page_cache, ((), ()), items_per_blob),
157194
grafted_metadata_partition: format!("grafted-metadata-{PARTITION_VAR}"),
158195
translator: EightCap,
159196
}
@@ -164,34 +201,70 @@ type VarVecCfg = ((), (commonware_codec::RangeCfg<usize>, ()));
164201

165202
pub fn any_var_vec_cfg(
166203
ctx: &(impl BufferPooler + ThreadPooler),
204+
) -> AnyVariableConfig<EightCap, VarVecCfg, Rayon> {
205+
any_var_vec_cfg_with(ctx, ITEMS_PER_BLOB)
206+
}
207+
208+
pub fn any_var_vec_cfg_with(
209+
ctx: &(impl BufferPooler + ThreadPooler),
210+
items_per_blob: NonZeroU64,
167211
) -> AnyVariableConfig<EightCap, VarVecCfg, Rayon> {
168212
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
169213
AnyVariableConfig {
170-
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone()),
171-
journal_config: var_log_cfg(PARTITION_VAR, page_cache, ((), ((0..=10000).into(), ()))),
214+
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone(), items_per_blob),
215+
journal_config: var_log_cfg(
216+
PARTITION_VAR,
217+
page_cache,
218+
((), ((0..=10000).into(), ())),
219+
items_per_blob,
220+
),
172221
translator: EightCap,
173222
}
174223
}
175224

176225
pub fn cur_var_vec_cfg(
177226
ctx: &(impl BufferPooler + ThreadPooler),
227+
) -> CurrentVariableConfig<EightCap, VarVecCfg, Rayon> {
228+
cur_var_vec_cfg_with(ctx, ITEMS_PER_BLOB)
229+
}
230+
231+
pub fn cur_var_vec_cfg_with(
232+
ctx: &(impl BufferPooler + ThreadPooler),
233+
items_per_blob: NonZeroU64,
178234
) -> CurrentVariableConfig<EightCap, VarVecCfg, Rayon> {
179235
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
180236
CurrentVariableConfig {
181-
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone()),
182-
journal_config: var_log_cfg(PARTITION_VAR, page_cache, ((), ((0..=10000).into(), ()))),
237+
merkle_config: merkle_cfg(PARTITION_VAR, ctx, page_cache.clone(), items_per_blob),
238+
journal_config: var_log_cfg(
239+
PARTITION_VAR,
240+
page_cache,
241+
((), ((0..=10000).into(), ())),
242+
items_per_blob,
243+
),
183244
grafted_metadata_partition: format!("grafted-metadata-{PARTITION_VAR}"),
184245
translator: EightCap,
185246
}
186247
}
187248

188249
pub fn keyless_cfg(
189250
ctx: &(impl BufferPooler + ThreadPooler),
251+
) -> KeylessConfig<(commonware_codec::RangeCfg<usize>, ()), Rayon> {
252+
keyless_cfg_with(ctx, ITEMS_PER_BLOB)
253+
}
254+
255+
pub fn keyless_cfg_with(
256+
ctx: &(impl BufferPooler + ThreadPooler),
257+
items_per_blob: NonZeroU64,
190258
) -> KeylessConfig<(commonware_codec::RangeCfg<usize>, ()), Rayon> {
191259
let page_cache = CacheRef::from_pooler(ctx, PAGE_SIZE, PAGE_CACHE_SIZE);
192260
KeylessConfig {
193-
merkle: merkle_cfg(PARTITION_KEYLESS, ctx, page_cache.clone()),
194-
log: var_log_cfg(PARTITION_KEYLESS, page_cache, ((0..=10000).into(), ())),
261+
merkle: merkle_cfg(PARTITION_KEYLESS, ctx, page_cache.clone(), items_per_blob),
262+
log: var_log_cfg(
263+
PARTITION_KEYLESS,
264+
page_cache,
265+
((0..=10000).into(), ()),
266+
items_per_blob,
267+
),
195268
}
196269
}
197270

0 commit comments

Comments
 (0)