Skip to content

Commit bec3871

Browse files
danlainedanlaine
authored
[Storage] Consolidate storage configs (#3430)
1 parent 3f6fe55 commit bec3871

26 files changed

Lines changed: 587 additions & 796 deletions

‎examples/sync/src/databases/any.rs‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ use crate::{Hasher, Key, Translator, Value};
44
use commonware_cryptography::Hasher as CryptoHasher;
55
use commonware_runtime::{buffer, BufferPooler, Clock, Metrics, Storage};
66
use commonware_storage::{
7-
mmr::{Location, Proof},
7+
journal::contiguous::fixed::Config as FConfig,
8+
mmr::{journaled::Config as MmrConfig, Location, Proof},
89
qmdb::{
910
self,
1011
any::{
@@ -29,17 +30,23 @@ pub type Operation = FixedOperation<Key, Value>;
2930

3031
/// Create a database configuration for use in tests.
3132
pub fn create_config(context: &impl BufferPooler) -> Config<Translator> {
33+
let page_cache = buffer::paged::CacheRef::from_pooler(context, NZU16!(2048), NZUsize!(10));
3234
Config {
33-
mmr_journal_partition: "mmr-journal".into(),
34-
mmr_metadata_partition: "mmr-metadata".into(),
35-
mmr_items_per_blob: NZU64!(4096),
36-
mmr_write_buffer: NZUsize!(4096),
37-
log_journal_partition: "log-journal".into(),
38-
log_items_per_blob: NZU64!(4096),
39-
log_write_buffer: NZUsize!(4096),
35+
mmr: MmrConfig {
36+
journal_partition: "mmr-journal".into(),
37+
metadata_partition: "mmr-metadata".into(),
38+
items_per_blob: NZU64!(4096),
39+
write_buffer: NZUsize!(4096),
40+
thread_pool: None,
41+
page_cache: page_cache.clone(),
42+
},
43+
log: FConfig {
44+
partition: "log-journal".into(),
45+
items_per_blob: NZU64!(4096),
46+
write_buffer: NZUsize!(4096),
47+
page_cache,
48+
},
4049
translator: Translator::default(),
41-
thread_pool: None,
42-
page_cache: buffer::paged::CacheRef::from_pooler(context, NZU16!(2048), NZUsize!(10)),
4350
}
4451
}
4552

‎examples/sync/src/databases/current.rs‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ use commonware_codec::FixedSize;
1919
use commonware_cryptography::{sha256, Hasher as CryptoHasher};
2020
use commonware_runtime::{buffer, BufferPooler, Clock, Metrics, Storage};
2121
use commonware_storage::{
22-
mmr::{Location, Proof},
22+
journal::contiguous::fixed::Config as FConfig,
23+
mmr::{journaled::Config as MmrConfig, Location, Proof},
2324
qmdb::{
2425
self,
2526
any::unordered::{fixed::Operation as FixedOperation, Update},
@@ -42,18 +43,24 @@ pub type Operation = FixedOperation<Key, Value>;
4243

4344
/// Create a database configuration.
4445
pub fn create_config(context: &impl BufferPooler) -> Config<Translator> {
46+
let page_cache = buffer::paged::CacheRef::from_pooler(context, NZU16!(2048), NZUsize!(10));
4547
Config {
46-
mmr_journal_partition: "mmr-journal".into(),
47-
mmr_metadata_partition: "mmr-metadata".into(),
48-
mmr_items_per_blob: NZU64!(4096),
49-
mmr_write_buffer: NZUsize!(4096),
50-
log_journal_partition: "log-journal".into(),
51-
log_items_per_blob: NZU64!(4096),
52-
log_write_buffer: NZUsize!(4096),
48+
mmr: MmrConfig {
49+
journal_partition: "mmr-journal".into(),
50+
metadata_partition: "mmr-metadata".into(),
51+
items_per_blob: NZU64!(4096),
52+
write_buffer: NZUsize!(4096),
53+
thread_pool: None,
54+
page_cache: page_cache.clone(),
55+
},
56+
log: FConfig {
57+
partition: "log-journal".into(),
58+
items_per_blob: NZU64!(4096),
59+
write_buffer: NZUsize!(4096),
60+
page_cache,
61+
},
5362
grafted_mmr_metadata_partition: "grafted-mmr-metadata".into(),
5463
translator: Translator::default(),
55-
thread_pool: None,
56-
page_cache: buffer::paged::CacheRef::from_pooler(context, NZU16!(2048), NZUsize!(10)),
5764
}
5865
}
5966

‎examples/sync/src/databases/immutable.rs‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ use crate::{Hasher, Key, Translator, Value};
44
use commonware_cryptography::{Hasher as CryptoHasher, Sha256};
55
use commonware_runtime::{BufferPooler, Clock, Metrics, Storage};
66
use commonware_storage::{
7-
mmr::{Location, Proof},
7+
journal::contiguous::variable::Config as VConfig,
8+
mmr::{journaled::Config as MmrConfig, Location, Proof},
89
qmdb::{
910
self,
1011
immutable::{self, Config},
@@ -22,23 +23,29 @@ pub type Operation = immutable::Operation<Key, Value>;
2223

2324
/// Create a database configuration with appropriate partitioning for Immutable.
2425
pub fn create_config(context: &impl BufferPooler) -> Config<Translator, ()> {
26+
let page_cache = commonware_runtime::buffer::paged::CacheRef::from_pooler(
27+
context,
28+
NZU16!(2048),
29+
NZUsize!(10),
30+
);
2531
Config {
26-
mmr_journal_partition: "mmr-journal".into(),
27-
mmr_metadata_partition: "mmr-metadata".into(),
28-
mmr_items_per_blob: NZU64!(4096),
29-
mmr_write_buffer: NZUsize!(4096),
30-
log_partition: "log".into(),
31-
log_items_per_section: NZU64!(4096),
32-
log_compression: None,
33-
log_codec_config: (),
34-
log_write_buffer: NZUsize!(4096),
32+
mmr: MmrConfig {
33+
journal_partition: "mmr-journal".into(),
34+
metadata_partition: "mmr-metadata".into(),
35+
items_per_blob: NZU64!(4096),
36+
write_buffer: NZUsize!(4096),
37+
thread_pool: None,
38+
page_cache: page_cache.clone(),
39+
},
40+
log: VConfig {
41+
partition: "log".into(),
42+
items_per_section: NZU64!(4096),
43+
compression: None,
44+
codec_config: (),
45+
write_buffer: NZUsize!(4096),
46+
page_cache,
47+
},
3548
translator: commonware_storage::translator::EightCap,
36-
thread_pool: None,
37-
page_cache: commonware_runtime::buffer::paged::CacheRef::from_pooler(
38-
context,
39-
NZU16!(2048),
40-
NZUsize!(10),
41-
),
4249
}
4350
}
4451

‎storage/fuzz/fuzz_targets/current_crash_recovery.rs‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ use commonware_runtime::{
1414
Metrics as _, Runner,
1515
};
1616
use commonware_storage::{
17-
mmr::Location,
17+
journal::contiguous::variable::Config as VConfig,
18+
mmr::{journaled::Config as MmrConfig, Location},
1819
qmdb::current::{unordered::variable::Db as Current, VariableConfig},
1920
translator::TwoCap,
2021
};
@@ -95,20 +96,26 @@ fn make_config(
9596
log_items_per_blob: u64,
9697
write_buffer: NonZeroUsize,
9798
) -> VariableConfig<TwoCap, ((), ())> {
99+
let page_cache = CacheRef::from_pooler(ctx, page_size, page_cache_size);
98100
VariableConfig {
99-
mmr_journal_partition: format!("crash-mmr-journal-{suffix}"),
100-
mmr_metadata_partition: format!("crash-mmr-metadata-{suffix}"),
101-
mmr_items_per_blob: NZU64!(mmr_items_per_blob),
102-
mmr_write_buffer: write_buffer,
103-
log_partition: format!("crash-log-{suffix}"),
104-
log_items_per_blob: NZU64!(log_items_per_blob),
105-
log_write_buffer: write_buffer,
106-
log_compression: None,
107-
log_codec_config: ((), ()),
101+
mmr: MmrConfig {
102+
journal_partition: format!("crash-mmr-journal-{suffix}"),
103+
metadata_partition: format!("crash-mmr-metadata-{suffix}"),
104+
items_per_blob: NZU64!(mmr_items_per_blob),
105+
write_buffer,
106+
thread_pool: None,
107+
page_cache: page_cache.clone(),
108+
},
109+
log: VConfig {
110+
partition: format!("crash-log-{suffix}"),
111+
items_per_section: NZU64!(log_items_per_blob),
112+
write_buffer,
113+
compression: None,
114+
codec_config: ((), ()),
115+
page_cache,
116+
},
108117
grafted_mmr_metadata_partition: format!("crash-grafted-mmr-metadata-{suffix}"),
109118
translator: TwoCap,
110-
page_cache: CacheRef::from_pooler(ctx, page_size, page_cache_size),
111-
thread_pool: None,
112119
}
113120
}
114121

‎storage/fuzz/fuzz_targets/current_ordered_operations.rs‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ use arbitrary::Arbitrary;
44
use commonware_cryptography::{sha256::Digest, Hasher, Sha256};
55
use commonware_runtime::{buffer::paged::CacheRef, deterministic, Runner};
66
use commonware_storage::{
7-
mmr::Location,
7+
journal::contiguous::fixed::Config as FConfig,
8+
mmr::{journaled::Config as MmrConfig, Location},
89
qmdb::current::{ordered::fixed::Db as CurrentDb, FixedConfig as Config},
910
translator::TwoCap,
1011
};
@@ -110,22 +111,28 @@ fn fuzz(data: FuzzInput) {
110111

111112
runner.start(|context| async move {
112113
let mut hasher = Sha256::new();
114+
let page_cache = CacheRef::from_pooler(
115+
&context,
116+
PAGE_SIZE,
117+
NZUsize!(PAGE_CACHE_SIZE),
118+
);
113119
let cfg = Config {
114-
mmr_journal_partition: "fuzz-current-mmr-journal".into(),
115-
mmr_metadata_partition: "fuzz-current-mmr-metadata".into(),
116-
mmr_items_per_blob: NZU64!(MMR_ITEMS_PER_BLOB),
117-
mmr_write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
118-
log_journal_partition: "fuzz-current-log-journal".into(),
119-
log_items_per_blob: NZU64!(LOG_ITEMS_PER_BLOB),
120-
log_write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
120+
mmr: MmrConfig {
121+
journal_partition: "fuzz-current-mmr-journal".into(),
122+
metadata_partition: "fuzz-current-mmr-metadata".into(),
123+
items_per_blob: NZU64!(MMR_ITEMS_PER_BLOB),
124+
write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
125+
thread_pool: None,
126+
page_cache: page_cache.clone(),
127+
},
128+
log: FConfig {
129+
partition: "fuzz-current-log-journal".into(),
130+
items_per_blob: NZU64!(LOG_ITEMS_PER_BLOB),
131+
write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
132+
page_cache,
133+
},
121134
grafted_mmr_metadata_partition: "fuzz-current-grafted-mmr-metadata".into(),
122135
translator: TwoCap,
123-
page_cache: CacheRef::from_pooler(
124-
&context,
125-
PAGE_SIZE,
126-
NZUsize!(PAGE_CACHE_SIZE),
127-
),
128-
thread_pool: None,
129136
};
130137

131138
let mut db = Db::init(context.clone(), cfg)

‎storage/fuzz/fuzz_targets/current_unordered_operations.rs‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ use arbitrary::Arbitrary;
44
use commonware_cryptography::{sha256::Digest, Hasher, Sha256};
55
use commonware_runtime::{buffer::paged::CacheRef, deterministic, Runner};
66
use commonware_storage::{
7-
mmr::Location,
7+
journal::contiguous::fixed::Config as FConfig,
8+
mmr::{journaled::Config as MmrConfig, Location},
89
qmdb::current::{unordered::fixed::Db as CurrentDb, FixedConfig as Config},
910
translator::TwoCap,
1011
};
@@ -100,22 +101,28 @@ fn fuzz(data: FuzzInput) {
100101

101102
runner.start(|context| async move {
102103
let mut hasher = Sha256::new();
104+
let page_cache = CacheRef::from_pooler(
105+
&context,
106+
PAGE_SIZE,
107+
NZUsize!(PAGE_CACHE_SIZE),
108+
);
103109
let cfg = Config {
104-
mmr_journal_partition: "fuzz-current-mmr-journal".into(),
105-
mmr_metadata_partition: "fuzz-current-mmr-metadata".into(),
106-
mmr_items_per_blob: NZU64!(MMR_ITEMS_PER_BLOB),
107-
mmr_write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
108-
log_journal_partition: "fuzz-current-log-journal".into(),
109-
log_items_per_blob: NZU64!(LOG_ITEMS_PER_BLOB),
110-
log_write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
110+
mmr: MmrConfig {
111+
journal_partition: "fuzz-current-mmr-journal".into(),
112+
metadata_partition: "fuzz-current-mmr-metadata".into(),
113+
items_per_blob: NZU64!(MMR_ITEMS_PER_BLOB),
114+
write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
115+
thread_pool: None,
116+
page_cache: page_cache.clone(),
117+
},
118+
log: FConfig {
119+
partition: "fuzz-current-log-journal".into(),
120+
items_per_blob: NZU64!(LOG_ITEMS_PER_BLOB),
121+
write_buffer: NZUsize!(WRITE_BUFFER_SIZE),
122+
page_cache,
123+
},
111124
grafted_mmr_metadata_partition: "fuzz-current-grafted-mmr-metadata".into(),
112125
translator: TwoCap,
113-
page_cache: CacheRef::from_pooler(
114-
&context,
115-
PAGE_SIZE,
116-
NZUsize!(PAGE_CACHE_SIZE),
117-
),
118-
thread_pool: None,
119126
};
120127

121128
let mut db = Db::init(context.clone(), cfg)

‎storage/fuzz/fuzz_targets/qmdb_any_fixed_sync.rs‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ use arbitrary::Arbitrary;
44
use commonware_cryptography::Sha256;
55
use commonware_runtime::{buffer::paged::CacheRef, deterministic, BufferPooler, Metrics, Runner};
66
use commonware_storage::{
7+
journal::contiguous::fixed::Config as FConfig,
8+
mmr::journaled::Config as MmrConfig,
79
qmdb::{
810
any::{
911
unordered::fixed::{Db, Operation as FixedOperation},
@@ -89,17 +91,23 @@ impl<'a> Arbitrary<'a> for FuzzInput {
8991
const PAGE_SIZE: NonZeroU16 = NZU16!(129);
9092

9193
fn test_config(test_name: &str, pooler: &impl BufferPooler) -> Config<TwoCap> {
94+
let page_cache = CacheRef::from_pooler(pooler, PAGE_SIZE, NZUsize!(1));
9295
Config {
93-
mmr_journal_partition: format!("{test_name}-mmr"),
94-
mmr_metadata_partition: format!("{test_name}-meta"),
95-
mmr_items_per_blob: NZU64!(3),
96-
mmr_write_buffer: NZUsize!(1024),
97-
log_journal_partition: format!("{test_name}-log"),
98-
log_items_per_blob: NZU64!(3),
99-
log_write_buffer: NZUsize!(1024),
96+
mmr: MmrConfig {
97+
journal_partition: format!("{test_name}-mmr"),
98+
metadata_partition: format!("{test_name}-meta"),
99+
items_per_blob: NZU64!(3),
100+
write_buffer: NZUsize!(1024),
101+
thread_pool: None,
102+
page_cache: page_cache.clone(),
103+
},
104+
log: FConfig {
105+
partition: format!("{test_name}-log"),
106+
items_per_blob: NZU64!(3),
107+
write_buffer: NZUsize!(1024),
108+
page_cache,
109+
},
100110
translator: TwoCap,
101-
thread_pool: None,
102-
page_cache: CacheRef::from_pooler(pooler, PAGE_SIZE, NZUsize!(1)),
103111
}
104112
}
105113

‎storage/fuzz/fuzz_targets/qmdb_any_variable_sync.rs‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ use arbitrary::Arbitrary;
44
use commonware_cryptography::Sha256;
55
use commonware_runtime::{buffer::paged::CacheRef, deterministic, BufferPooler, Metrics, Runner};
66
use commonware_storage::{
7+
journal::contiguous::variable::Config as VConfig,
78
merkle::Family as _,
8-
mmr::{self, Family, StandardHasher as Standard},
9+
mmr::{self, journaled::Config as MmrConfig, Family, StandardHasher as Standard},
910
qmdb::{
1011
any::{unordered::variable::Db, VariableConfig as Config},
1112
verify_proof,
@@ -139,19 +140,25 @@ fn test_config(
139140
test_name: &str,
140141
pooler: &impl BufferPooler,
141142
) -> Config<TwoCap, ((), (commonware_codec::RangeCfg<usize>, ()))> {
143+
let page_cache = CacheRef::from_pooler(pooler, PAGE_SIZE, NZUsize!(1));
142144
Config {
143-
mmr_journal_partition: format!("{test_name}-mmr"),
144-
mmr_metadata_partition: format!("{test_name}-meta"),
145-
mmr_items_per_blob: NZU64!(3),
146-
mmr_write_buffer: NZUsize!(1024),
147-
log_partition: format!("{test_name}-log"),
148-
log_items_per_blob: NZU64!(3),
149-
log_write_buffer: NZUsize!(1024),
150-
log_compression: None,
151-
log_codec_config: ((), ((0..=100000).into(), ())),
145+
mmr: MmrConfig {
146+
journal_partition: format!("{test_name}-mmr"),
147+
metadata_partition: format!("{test_name}-meta"),
148+
items_per_blob: NZU64!(3),
149+
write_buffer: NZUsize!(1024),
150+
thread_pool: None,
151+
page_cache: page_cache.clone(),
152+
},
153+
log: VConfig {
154+
partition: format!("{test_name}-log"),
155+
items_per_section: NZU64!(3),
156+
write_buffer: NZUsize!(1024),
157+
compression: None,
158+
codec_config: ((), ((0..=100000).into(), ())),
159+
page_cache,
160+
},
152161
translator: TwoCap,
153-
thread_pool: None,
154-
page_cache: CacheRef::from_pooler(pooler, PAGE_SIZE, NZUsize!(1)),
155162
}
156163
}
157164

0 commit comments

Comments
 (0)