[storage/qmdb/current] support split (pyramid) bagging in qmdb-current (#3693)

Author: roberto-bayardo

.github/benchmark-tracking.toml

@@ -49,6 +49,8 @@ cargo_flags = ["--features", "test-traits"]
 
 [[packages.benchmarks]]
 name = "qmdb"
+criterion_args = ["--sample-size", "100", "--significance-level", "0.01"]
 variants = [
   "qmdb::merkleize/variant=any::unordered::fixed::mmr keys=10000 ch=false sync=false",
+  "qmdb::merkleize/variant=current::ordered::fixed::mmb chunk=256 keys=10000 ch=true sync=false",
 ]

examples/sync/src/databases/current.rs

@@ -7,9 +7,9 @@
 //! documentation for more details.
 //!
 //! For sync, the engine targets the **ops root** (not the canonical root). The operations and proof
-//! format are identical to `any` -- the bitmap is reconstructed deterministically from the
-//! operations after sync completes. See the [Root structure](commonware_storage::qmdb::current)
-//! module documentation for details.
+//! format are identical to `any`; direct proof verifiers should use `qmdb::hasher`. The bitmap is
+//! reconstructed deterministically from the operations after sync completes. See the
+//! [Root structure](commonware_storage::qmdb::current) module documentation for details.
 //!
 //! This module re-uses the same [`Operation`] type as [`super::any`] since the underlying
 //! operations log is the same.

storage/conformance.toml

@@ -168,35 +168,35 @@ hash = "22374a7cbdfa0bdac17e2e285422a54375fec27017942ef7abea2cf8da298c70"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbOrderedFixedConf"]
 n_cases = 200
-hash = "f265ca1f2f7f571230ef4e576b949d3699b619f108cb0a4e2e28c352f7b32aee"
+hash = "c13c9100d657cd01fdf4cff19985bba9e65b7c057391785c5d401a13c976b0af"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbOrderedVariableConf"]
 n_cases = 200
-hash = "74d3a4b1721216ed1bdfc564f5ae07759476b0ceae7dbd1e953c35538a1b0755"
+hash = "8fed62bd40fa5ef36f6aebff5152e37d1089e4a66f8dfb9eec8cb00132487efb"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbUnorderedFixedConf"]
 n_cases = 200
-hash = "92b2d71ff19a0f24f337a934d43ee06e3f81a155cd4adc97c58b27cef5cde435"
+hash = "81fb100d508803580ae22a96b76955b6480f3526254703c9d75ebb56f0f053f1"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbUnorderedVariableConf"]
 n_cases = 200
-hash = "bad7ded44c7cedc95d202dc3334e3de88877d3aeb63b5808b18bd12c2f59c0d9"
+hash = "c46b60791c970b54e936d47a8e2eaae45e9e34c9fb0e1278e678f378392a02f1"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrOrderedFixedConf"]
 n_cases = 200
-hash = "95dc738adeb88d6546c62828e448213fe723d600f57bb4660ecf982ee76e29ef"
+hash = "0e5264ac6a0c0d56ace9ebc986635b128f9d7a118471875ad874b08e80f66ebb"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrOrderedVariableConf"]
 n_cases = 200
-hash = "a52a284b8aec813f99234a0545d7c8e651b6f6c93d1912394be3e46075a2681d"
+hash = "66cd3cc0299ef062e23d4f9812d5d45896cd808ee0bc65d9745a4265895ae8b5"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrUnorderedFixedConf"]
 n_cases = 200
-hash = "537f4c63d903b112fc045ed47f0127da3e2ca70e38c8406e6535a94b2dddc693"
+hash = "8ce052fef8ce8e4d817a2961144cf3a28aec4bd6eb75d32b013499116a6981da"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrUnorderedVariableConf"]
 n_cases = 200
-hash = "93926373351527268aead35fc1884ff27ae2a154df38309dbed50c2c388d4870"
+hash = "0d0591823b7d5ed8dcb940d086ec214062eb3cfffd018e9fbadf9e16ce8da0f7"
 
 ["commonware_storage::qmdb::conformance::ImmutableMmbCompactFixedConf"]
 n_cases = 200

storage/fuzz/fuzz_targets/current_ordered_operations.rs

@@ -51,6 +51,8 @@ enum CurrentOperation {
         bad_digests: Vec<[u8; 32]>,
         max_ops: NonZeroU64,
         bad_chunks: Vec<[u8; 32]>,
+        bad_prefix_peaks: Vec<[u8; 32]>,
+        bad_suffix_peaks: Vec<[u8; 32]>,
     },
     GetSpan {
         key: RawKey,
@@ -267,7 +269,14 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                     }
                 }
 
-                CurrentOperation::ArbitraryProof {start_loc, bad_digests, max_ops, bad_chunks} => {
+                CurrentOperation::ArbitraryProof {
+                    start_loc,
+                    bad_digests,
+                    max_ops,
+                    bad_chunks,
+                    bad_prefix_peaks,
+                    bad_suffix_peaks,
+                } => {
                     let current_op_count = db.bounds().await.end;
                     if current_op_count == 0 {
                         continue;
@@ -311,6 +320,36 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                                 &root
                             ), "proof with bad chunks should not verify");
                         }
+
+                        let bad_prefix_peaks =
+                            bad_prefix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_prefix_peaks != bad_prefix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_prefix_peaks = bad_prefix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad prefix peaks should not verify");
+                        }
+
+                        let bad_suffix_peaks =
+                            bad_suffix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_suffix_peaks != bad_suffix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_suffix_peaks = bad_suffix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad suffix peaks should not verify");
+                        }
                     }
                 }
 

storage/fuzz/fuzz_targets/current_unordered_operations.rs

@@ -51,6 +51,8 @@ enum CurrentOperation {
         bad_digests: Vec<[u8; 32]>,
         max_ops: NonZeroU64,
         bad_chunks: Vec<[u8; 32]>,
+        bad_prefix_peaks: Vec<[u8; 32]>,
+        bad_suffix_peaks: Vec<[u8; 32]>,
     },
 }
 
@@ -242,7 +244,14 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                     }
                 }
 
-                CurrentOperation::ArbitraryProof {start_loc, bad_digests, max_ops, bad_chunks} => {
+                CurrentOperation::ArbitraryProof {
+                    start_loc,
+                    bad_digests,
+                    max_ops,
+                    bad_chunks,
+                    bad_prefix_peaks,
+                    bad_suffix_peaks,
+                } => {
                     let current_op_count = db.bounds().await.end;
                     if current_op_count == 0 {
                         continue;
@@ -283,6 +292,36 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                                 &root
                             ), "proof with bad chunks should not verify");
                         }
+
+                        let bad_prefix_peaks =
+                            bad_prefix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_prefix_peaks != bad_prefix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_prefix_peaks = bad_prefix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad prefix peaks should not verify");
+                        }
+
+                        let bad_suffix_peaks =
+                            bad_suffix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_suffix_peaks != bad_suffix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_suffix_peaks = bad_suffix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad suffix peaks should not verify");
+                        }
                     }
                 }
 

storage/fuzz/fuzz_targets/merkle_family_operations.rs

@@ -4,7 +4,8 @@ use arbitrary::Arbitrary;
 use commonware_cryptography::{sha256::Digest, Sha256};
 use commonware_runtime::{deterministic, Runner};
 use commonware_storage::merkle::{
-    hasher::Standard, mem::Mem, mmb, mmr, Error, Family as MerkleFamily, Location, Position,
+    hasher::Standard, mem::Mem, mmb, mmr, Bagging::ForwardFold, Error, Family as MerkleFamily,
+    Location, Position,
 };
 use core::any::type_name;
 use libfuzzer_sys::fuzz_target;
@@ -128,7 +129,7 @@ fn fuzz_family<F: MerkleFamily>(operations: &[MerkleOperation]) {
     let runner = deterministic::Runner::default();
 
     runner.start(|_context| async move {
-        let hasher = Standard::<Sha256>::new();
+        let hasher = Standard::<Sha256>::new(ForwardFold);
         let mut merkle = Mem::<F, Digest>::new();
         let mut reference = ReferenceMerkle::<F>::new();
 

storage/fuzz/fuzz_targets/merkle_full.rs

@@ -5,8 +5,8 @@ use commonware_cryptography::Sha256;
 use commonware_parallel::Sequential;
 use commonware_runtime::{buffer::paged::CacheRef, deterministic, BufferPooler, Metrics, Runner};
 use commonware_storage::merkle::{
-    full::Config, hasher::Standard, mem::Mem, mmb, mmr, Error, Family as MerkleFamily, Location,
-    LocationRangeExt as _, Position,
+    full::Config, hasher::Standard, mem::Mem, mmb, mmr, Bagging::ForwardFold, Error,
+    Family as MerkleFamily, Location, LocationRangeExt as _, Position,
 };
 use commonware_utils::{non_empty_range, NZUsize, NZU16, NZU64};
 use libfuzzer_sys::fuzz_target;
@@ -92,7 +92,7 @@ fn historical_root<F: MerkleFamily>(
     leaves: &[Vec<u8>],
     requested_leaves: Location<F>,
 ) -> <Sha256 as commonware_cryptography::Hasher>::Digest {
-    let hasher = Standard::<Sha256>::new();
+    let hasher = Standard::<Sha256>::new(ForwardFold);
     let mut mem = Mem::<F, _>::new();
     let batch = {
         let mut batch = mem.new_batch();
@@ -115,7 +115,7 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
         let operations = input.operations.clone();
         async move {
             let mut leaves = Vec::new();
-            let hasher = Standard::<Sha256>::new();
+            let hasher = Standard::<Sha256>::new(ForwardFold);
             let mut merkle =
                 Merkle::<F, _, _>::init(context.clone(), &hasher, test_config(suffix, &context))
                     .await
@@ -258,7 +258,7 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
                             .await;
                         match result {
                             Ok(historical_proof) => {
-                                let verify_hasher = Standard::<Sha256>::new();
+                                let verify_hasher = Standard::<Sha256>::new(ForwardFold);
                                 assert!(historical_proof.verify_range_inclusion(
                                     &verify_hasher,
                                     &leaves[range.to_usize_range()],

storage/fuzz/fuzz_targets/merkle_full_crash_recovery.rs

@@ -10,7 +10,8 @@ use commonware_runtime::{
     buffer::paged::CacheRef, deterministic, BufferPooler, Metrics as _, Runner,
 };
 use commonware_storage::merkle::{
-    full::Config, hasher::Standard as StandardHasher, mmb, mmr, Family as MerkleFamily, Location,
+    full::Config, hasher::Standard as StandardHasher, mmb, mmr, Bagging::ForwardFold,
+    Family as MerkleFamily, Location,
 };
 use commonware_utils::NZU64;
 use libfuzzer_sys::fuzz_target;
@@ -235,7 +236,7 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
         let partition_suffix = partition_suffix.clone();
         let operations = operations.clone();
         async move {
-            let hasher = StandardHasher::<Sha256>::new();
+            let hasher = StandardHasher::<Sha256>::new(ForwardFold);
             let mut merkle = Merkle::<F>::init(
                 ctx.with_label("merkle"),
                 &hasher,
@@ -267,7 +268,7 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
     runner.start(|ctx| async move {
         *ctx.storage_fault_config().write() = deterministic::FaultConfig::default();
 
-        let hasher = StandardHasher::<Sha256>::new();
+        let hasher = StandardHasher::<Sha256>::new(ForwardFold);
         let mut merkle = Merkle::<F>::init(
             ctx.with_label("recovered"),
             &hasher,

storage/fuzz/fuzz_targets/mmr_bitmap.rs

@@ -4,7 +4,7 @@ use arbitrary::Arbitrary;
 use commonware_cryptography::{sha256, Digest, Sha256};
 use commonware_parallel::Sequential;
 use commonware_runtime::{deterministic, Clock, Metrics, Runner, Storage};
-use commonware_storage::{MerkleizedBitMap, UnmerkleizedBitMap};
+use commonware_storage::{merkle::Bagging::ForwardFold, MerkleizedBitMap, UnmerkleizedBitMap};
 use commonware_utils::bitmap::BitMap;
 use libfuzzer_sys::fuzz_target;
 
@@ -59,7 +59,7 @@ fn fuzz(input: FuzzInput) {
     const PARTITION: &str = "fuzz-mmr-bitmap-test-partition";
 
     runner.start(|context| async move {
-        let hasher = commonware_storage::mmr::StandardHasher::<Sha256>::new();
+        let hasher = commonware_storage::mmr::StandardHasher::<Sha256>::new(ForwardFold);
         let init_bitmap = MerkleizedBitMap::<_, _, CHUNK_SIZE>::init(
             context.with_label("bitmap"),
             PARTITION,

storage/fuzz/fuzz_targets/proof_store.rs

@@ -4,7 +4,8 @@ use arbitrary::{Arbitrary, Unstructured};
 use commonware_codec::Encode as _;
 use commonware_cryptography::{sha256::Digest, Sha256};
 use commonware_storage::merkle::{
-    self, mmb, mmr, verification::ProofStore, Family as MerkleFamily, Location, Position, Proof,
+    self, mmb, mmr, verification::ProofStore, Bagging::BackwardFold, Family as MerkleFamily,
+    Location, Position, Proof,
 };
 use libfuzzer_sys::fuzz_target;
 use std::ops::Range;
@@ -57,7 +58,7 @@ impl<'a, F: MerkleFamily> Arbitrary<'a> for FuzzInput<F> {
 }
 
 fn fuzz_family<F: MerkleFamily>(input: &FuzzInput<F>) {
-    let hasher = merkle::hasher::Standard::<Sha256>::with_bagging(merkle::Bagging::BackwardFold);
+    let hasher = merkle::hasher::Standard::<Sha256>::new(BackwardFold);
     let proof = Proof::<F, Digest> {
         leaves: input.proof_leaves,
         inactive_peaks: input.inactive_peaks,