Skip to content

Commit 228b0db

Browse files
roberto-bayardoroberto-bayardo
authored
[storage] further generify storage crate on merkle family (#3463)
1 parent 20b3766 commit 228b0db

96 files changed

Lines changed: 11687 additions & 9116 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎examples/sync/src/databases/any.rs‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use commonware_cryptography::Hasher as CryptoHasher;
55
use commonware_runtime::{buffer, BufferPooler, Clock, Metrics, Storage};
66
use commonware_storage::{
77
journal::contiguous::fixed::Config as FConfig,
8-
mmr::{journaled::Config as MmrConfig, Location, Proof},
8+
mmr::{self, journaled::Config as MmrConfig, Location, Proof},
99
qmdb::{
1010
self,
1111
any::{
@@ -23,16 +23,16 @@ use std::{future::Future, num::NonZeroU64};
2323
use tracing::error;
2424

2525
/// Database type alias.
26-
pub type Database<E> = Db<E, Key, Value, Hasher, Translator>;
26+
pub type Database<E> = Db<mmr::Family, E, Key, Value, Hasher, Translator>;
2727

2828
/// Operation type alias.
29-
pub type Operation = FixedOperation<Key, Value>;
29+
pub type Operation = FixedOperation<mmr::Family, Key, Value>;
3030

3131
/// Create a database configuration for use in tests.
3232
pub fn create_config(context: &impl BufferPooler) -> Config<Translator> {
3333
let page_cache = buffer::paged::CacheRef::from_pooler(context, NZU16!(2048), NZUsize!(10));
3434
Config {
35-
mmr_config: MmrConfig {
35+
merkle_config: MmrConfig {
3636
journal_partition: "mmr-journal".into(),
3737
metadata_partition: "mmr-metadata".into(),
3838
items_per_blob: NZU64!(4096),
@@ -87,7 +87,7 @@ where
8787
async fn add_operations(
8888
&mut self,
8989
operations: Vec<Self::Operation>,
90-
) -> Result<(), commonware_storage::qmdb::Error> {
90+
) -> Result<(), qmdb::Error<mmr::Family>> {
9191
if operations.last().is_none() || !operations.last().unwrap().is_commit() {
9292
// Ignore bad inputs rather than return errors.
9393
error!("operations must end with a commit");
@@ -131,14 +131,15 @@ where
131131
op_count: Location,
132132
start_loc: Location,
133133
max_ops: NonZeroU64,
134-
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error>> + Send {
134+
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error<mmr::Family>>> + Send
135+
{
135136
self.historical_proof(op_count, start_loc, max_ops)
136137
}
137138

138139
fn pinned_nodes_at(
139140
&self,
140141
loc: Location,
141-
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error>> + Send {
142+
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error<mmr::Family>>> + Send {
142143
self.pinned_nodes_at(loc)
143144
}
144145

‎examples/sync/src/databases/current.rs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use commonware_cryptography::{sha256, Hasher as CryptoHasher};
2020
use commonware_runtime::{buffer, BufferPooler, Clock, Metrics, Storage};
2121
use commonware_storage::{
2222
journal::contiguous::fixed::Config as FConfig,
23-
mmr::{journaled::Config as MmrConfig, Location, Proof},
23+
mmr::{self, journaled::Config as MmrConfig, Location, Proof},
2424
qmdb::{
2525
self,
2626
any::unordered::{fixed::Operation as FixedOperation, Update},
@@ -39,7 +39,7 @@ const CHUNK_SIZE: usize = sha256::Digest::SIZE;
3939
pub type Database<E> = current::unordered::fixed::Db<E, Key, Value, Hasher, Translator, CHUNK_SIZE>;
4040

4141
/// Operation type alias. Same as the `any` operation type.
42-
pub type Operation = FixedOperation<Key, Value>;
42+
pub type Operation = FixedOperation<mmr::Family, Key, Value>;
4343

4444
/// Create a database configuration.
4545
pub fn create_config(context: &impl BufferPooler) -> Config<Translator> {
@@ -101,7 +101,7 @@ where
101101
async fn add_operations(
102102
&mut self,
103103
operations: Vec<Self::Operation>,
104-
) -> Result<(), qmdb::Error> {
104+
) -> Result<(), qmdb::Error<mmr::Family>> {
105105
if operations.last().is_none() || !operations.last().unwrap().is_commit() {
106106
error!("operations must end with a commit");
107107
return Ok(());
@@ -146,15 +146,16 @@ where
146146
op_count: Location,
147147
start_loc: Location,
148148
max_ops: NonZeroU64,
149-
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error>> + Send {
149+
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error<mmr::Family>>> + Send
150+
{
150151
// Return ops-level proofs (not grafted proofs) for the sync engine.
151152
self.ops_historical_proof(op_count, start_loc, max_ops)
152153
}
153154

154155
fn pinned_nodes_at(
155156
&self,
156157
loc: Location,
157-
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error>> + Send {
158+
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error<mmr::Family>>> + Send {
158159
self.pinned_nodes_at(loc)
159160
}
160161

‎examples/sync/src/databases/immutable.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use commonware_cryptography::{Hasher as CryptoHasher, Sha256};
55
use commonware_runtime::{BufferPooler, Clock, Metrics, Storage};
66
use commonware_storage::{
77
journal::contiguous::variable::Config as VConfig,
8-
mmr::{journaled::Config as MmrConfig, Location, Proof},
8+
mmr::{self, journaled::Config as MmrConfig, Location, Proof},
99
qmdb::{
1010
self,
1111
immutable::{self, Config},
@@ -16,7 +16,7 @@ use std::{future::Future, num::NonZeroU64};
1616
use tracing::error;
1717

1818
/// Database type alias.
19-
pub type Database<E> = immutable::Immutable<E, Key, Value, Hasher, Translator>;
19+
pub type Database<E> = immutable::Immutable<mmr::Family, E, Key, Value, Hasher, Translator>;
2020

2121
/// Operation type alias.
2222
pub type Operation = immutable::Operation<Key, Value>;
@@ -29,7 +29,7 @@ pub fn create_config(context: &impl BufferPooler) -> Config<Translator, ()> {
2929
NZUsize!(10),
3030
);
3131
Config {
32-
mmr: MmrConfig {
32+
merkle_config: MmrConfig {
3333
journal_partition: "mmr-journal".into(),
3434
metadata_partition: "mmr-metadata".into(),
3535
items_per_blob: NZU64!(4096),
@@ -93,7 +93,7 @@ where
9393
async fn add_operations(
9494
&mut self,
9595
operations: Vec<Self::Operation>,
96-
) -> Result<(), commonware_storage::qmdb::Error> {
96+
) -> Result<(), commonware_storage::qmdb::Error<mmr::Family>> {
9797
if operations.last().is_none() || !operations.last().unwrap().is_commit() {
9898
// Ignore bad inputs rather than return errors.
9999
error!("operations must end with a commit");
@@ -136,14 +136,15 @@ where
136136
op_count: Location,
137137
start_loc: Location,
138138
max_ops: NonZeroU64,
139-
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error>> + Send {
139+
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error<mmr::Family>>> + Send
140+
{
140141
self.historical_proof(op_count, start_loc, max_ops)
141142
}
142143

143144
fn pinned_nodes_at(
144145
&self,
145146
loc: Location,
146-
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error>> + Send {
147+
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error<mmr::Family>>> + Send {
147148
self.pinned_nodes_at(loc)
148149
}
149150

‎examples/sync/src/databases/mod.rs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use crate::Key;
44
use commonware_codec::Encode;
55
use commonware_storage::{
6-
mmr::{Location, Proof},
6+
mmr::{self, Location, Proof},
77
qmdb::{self, operation::Operation},
88
};
99
use std::{future::Future, num::NonZeroU64};
@@ -46,9 +46,10 @@ impl DatabaseType {
4646
}
4747

4848
/// Helper trait for databases that can be synced.
49+
#[allow(clippy::type_complexity)]
4950
pub trait Syncable: Sized {
5051
/// The type of operations in the database.
51-
type Operation: Operation + Encode + Sync + 'static;
52+
type Operation: Operation<mmr::Family> + Encode + Sync + 'static;
5253

5354
/// Create test operations with the given count and seed.
5455
/// The returned operations must end with a commit operation.
@@ -59,7 +60,7 @@ pub trait Syncable: Sized {
5960
fn add_operations(
6061
&mut self,
6162
operations: Vec<Self::Operation>,
62-
) -> impl Future<Output = Result<(), qmdb::Error>>;
63+
) -> impl Future<Output = Result<(), qmdb::Error<mmr::Family>>>;
6364

6465
/// Get the database's root digest.
6566
fn root(&self) -> Key;
@@ -76,13 +77,13 @@ pub trait Syncable: Sized {
7677
op_count: Location,
7778
start_loc: Location,
7879
max_ops: NonZeroU64,
79-
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error>> + Send;
80+
) -> impl Future<Output = Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error<mmr::Family>>> + Send;
8081

8182
/// Get the pinned MMR nodes for a lower operation boundary of `loc`.
8283
fn pinned_nodes_at(
8384
&self,
8485
loc: Location,
85-
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error>> + Send;
86+
) -> impl Future<Output = Result<Vec<Key>, qmdb::Error<mmr::Family>>> + Send;
8687

8788
/// Get the database type name for logging.
8889
fn name() -> &'static str;

‎examples/sync/src/error.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ pub enum Error {
2424

2525
/// Database operation failed
2626
#[error("database operation failed")]
27-
Database(#[from] commonware_storage::qmdb::Error),
27+
Database(#[from] commonware_storage::qmdb::Error<commonware_storage::mmr::Family>),
2828

2929
/// Request channel to I/O task closed unexpectedly
3030
#[error("request channel closed - I/O task may have terminated")]

‎storage/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,9 @@ harness = false
109109
path = "src/journal/benches/bench.rs"
110110

111111
[[bench]]
112-
name="mmr"
112+
name="merkle"
113113
harness = false
114-
path = "src/merkle/mmr/benches/bench.rs"
114+
path = "src/merkle/benches/bench.rs"
115115

116116
[[bench]]
117117
name="ordinal"

‎storage/conformance.toml‎

Lines changed: 13 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -90,42 +90,38 @@ hash = "3636363c592dddcff2eaeb4e6b01ec0cfcca654d69f9fdc1c57626c5adea0de1"
9090
n_cases = 65536
9191
hash = "f93b1dda40f6d9f1ccb3f05994be56189205dcb558551e056c2f3db03a79182d"
9292

93-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<Ordered<Fixed<U64,U64>>>"]
93+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<OrderedOperation<mmb::Family,U64,FixedEncoding<U64\n>>>"]
9494
n_cases = 65536
95-
hash = "f07e12cc9db44b417c1e936e0f36f06d93ec8390b59c4394811d422d11b32a7d"
95+
hash = "606f8c7d5ce073573e0e4d37006bb9068be93fdddeccdbc265cf41df0c9fcde6"
9696

97-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<Ordered<VariableValue<U64,Vec<u8>>>>"]
97+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<OrderedOperation<mmb::Family,U64,VariableEncoding<\nVec<u8>>>>"]
9898
n_cases = 65536
99-
hash = "1be8a11f1e27adb03ad60b57af16993cc571a4b8c2bdd1c086b1762f6a92f70e"
99+
hash = "1a3c262e874fb91dc65f8fe2cd9b0e64ea1e69f615e0ef2be128564cfca963ec"
100100

101-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<OrderedOperation<U64,FixedEncoding<U64>>>"]
101+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<OrderedOperation<mmr::Family,U64,FixedEncoding<U64\n>>>"]
102102
n_cases = 65536
103103
hash = "b96e969e3c2fa571101b20a527709627a46c7cbcabc21b6c34491d942c44e11a"
104104

105-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<OrderedOperation<U64,VariableEncoding<Vec<u8>>>>"]
105+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<OrderedOperation<mmr::Family,U64,VariableEncoding<\nVec<u8>>>>"]
106106
n_cases = 65536
107107
hash = "734013661e2e967b0ecad32fac076377319237c040b5deb2363268d23dc9c926"
108108

109-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<Unordered<Fixed<U64,U64>>>"]
109+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<UnorderedOperation<mmb::Family,U64,FixedEncoding<\nU64>>>"]
110110
n_cases = 65536
111-
hash = "92d97caa778f4825bb78af31114456bd2fadee745387914ca4754407638b518a"
111+
hash = "054574a1657cfb4d4d4b216adc7eddecc87565d0c6536f693a3ab82de7c8bcad"
112112

113-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<Unordered<VariableValue<U64,Vec<u8>>>>"]
113+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<UnorderedOperation<mmb::Family,U64,VariableEncoding<\nVec<u8>>>>"]
114114
n_cases = 65536
115-
hash = "36910b6b06e883c255d8e6f4ae688630c2bae894dd440e7a93985663119be574"
115+
hash = "80fce5b04a0d7c506724ea1e04e2c37efb475a0f080d321920cdf4cda9d2fa1a"
116116

117-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<UnorderedOperation<U64,FixedEncoding<U64>>>"]
117+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<UnorderedOperation<mmr::Family,U64,FixedEncoding<\nU64>>>"]
118118
n_cases = 65536
119119
hash = "0a7354fdc9af8a461126f61f85def0eb2e5929df0b0bb045431d16fe8376ff0b"
120120

121-
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<UnorderedOperation<U64,VariableEncoding<Vec<u8>>>\n>"]
121+
["commonware_storage::qmdb::any::operation::tests::conformance::CodecConformance<UnorderedOperation<mmr::Family,U64,VariableEncoding<\nVec<u8>>>>"]
122122
n_cases = 65536
123123
hash = "32b30ab90b530340cd3d86ba14a31a4d265095a4e78719529d3a55fae2ad0bf2"
124124

125-
["commonware_storage::qmdb::any::operation::update::tests::conformance::CodecConformance<OrderedUpdate<Fixed<U64,U64>>>"]
126-
n_cases = 65536
127-
hash = "457f73ee7d5f80f0d4a274509b374edddecfc881a9cbd0864691bf1a14f304a1"
128-
129125
["commonware_storage::qmdb::any::operation::update::tests::conformance::CodecConformance<OrderedUpdate<U64,FixedEncoding<U64>>>"]
130126
n_cases = 65536
131127
hash = "457f73ee7d5f80f0d4a274509b374edddecfc881a9cbd0864691bf1a14f304a1"
@@ -134,14 +130,6 @@ hash = "457f73ee7d5f80f0d4a274509b374edddecfc881a9cbd0864691bf1a14f304a1"
134130
n_cases = 65536
135131
hash = "2f8ebe2f7bc78454ba8d232c80c4f7d55797f26ba4cbcc0a4abcb33ca83c2b1c"
136132

137-
["commonware_storage::qmdb::any::operation::update::tests::conformance::CodecConformance<OrderedUpdate<VariableValue<U64,Vec<u8>>>>"]
138-
n_cases = 65536
139-
hash = "2f8ebe2f7bc78454ba8d232c80c4f7d55797f26ba4cbcc0a4abcb33ca83c2b1c"
140-
141-
["commonware_storage::qmdb::any::operation::update::tests::conformance::CodecConformance<UnorderedUpdate<Fixed<U64,U64>>>"]
142-
n_cases = 65536
143-
hash = "f5456580eb69727a23bfb0281e33f467acc0c91273efebd334d3e82c9770ae76"
144-
145133
["commonware_storage::qmdb::any::operation::update::tests::conformance::CodecConformance<UnorderedUpdate<U64,FixedEncoding<U64>>>"]
146134
n_cases = 65536
147135
hash = "f5456580eb69727a23bfb0281e33f467acc0c91273efebd334d3e82c9770ae76"
@@ -150,11 +138,7 @@ hash = "f5456580eb69727a23bfb0281e33f467acc0c91273efebd334d3e82c9770ae76"
150138
n_cases = 65536
151139
hash = "a3fbb5f749fa5b73a684e9f0bebd8973c456e5ee43a0a3db75a99aa550dee302"
152140

153-
["commonware_storage::qmdb::any::operation::update::tests::conformance::CodecConformance<UnorderedUpdate<VariableValue<U64,Vec<u8>>>>"]
154-
n_cases = 65536
155-
hash = "a3fbb5f749fa5b73a684e9f0bebd8973c456e5ee43a0a3db75a99aa550dee302"
156-
157-
["commonware_storage::qmdb::immutable::operation::tests::conformance::CodecConformance<Operation<U64,U64>>"]
141+
["commonware_storage::qmdb::immutable::operation::tests::conformance::CodecConformance<Op>"]
158142
n_cases = 65536
159143
hash = "fdca5df62d243b28676ee15034663694cd219d5ef80749079126b0ed73effe0d"
160144

‎storage/fuzz/Cargo.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,8 @@ doc = false
167167
bench = false
168168

169169
[[bin]]
170-
name = "mmr_journaled"
171-
path = "fuzz_targets/mmr_journaled.rs"
170+
name = "merkle_journaled"
171+
path = "fuzz_targets/merkle_journaled.rs"
172172
test = false
173173
doc = false
174174
bench = false
@@ -251,8 +251,8 @@ doc = false
251251
bench = false
252252

253253
[[bin]]
254-
name = "mmr_journaled_crash_recovery"
255-
path = "fuzz_targets/mmr_journaled_crash_recovery.rs"
254+
name = "merkle_journaled_crash_recovery"
255+
path = "fuzz_targets/merkle_journaled_crash_recovery.rs"
256256
test = false
257257
doc = false
258258
bench = false

0 commit comments

Comments
 (0)