[storage/qmdb/current] support split (pyramid) bagging in qmdb-current

.github/benchmark-tracking.toml

@@ -49,6 +49,8 @@ cargo_flags = ["--features", "test-traits"]
 
 [[packages.benchmarks]]
 name = "qmdb"
+criterion_args = ["--sample-size", "100", "--significance-level", "0.01"]
 variants = [
   "qmdb::merkleize/variant=any::unordered::fixed::mmr keys=10000 ch=false sync=false",
+  "qmdb::merkleize/variant=current::ordered::fixed::mmb keys=10000 ch=true sync=false",
 ]

storage/conformance.toml

@@ -168,35 +168,35 @@ hash = "22374a7cbdfa0bdac17e2e285422a54375fec27017942ef7abea2cf8da298c70"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbOrderedFixedConf"]
 n_cases = 200
-hash = "f265ca1f2f7f571230ef4e576b949d3699b619f108cb0a4e2e28c352f7b32aee"
+hash = "c13c9100d657cd01fdf4cff19985bba9e65b7c057391785c5d401a13c976b0af"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbOrderedVariableConf"]
 n_cases = 200
-hash = "74d3a4b1721216ed1bdfc564f5ae07759476b0ceae7dbd1e953c35538a1b0755"
+hash = "8fed62bd40fa5ef36f6aebff5152e37d1089e4a66f8dfb9eec8cb00132487efb"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbUnorderedFixedConf"]
 n_cases = 200
-hash = "92b2d71ff19a0f24f337a934d43ee06e3f81a155cd4adc97c58b27cef5cde435"
+hash = "81fb100d508803580ae22a96b76955b6480f3526254703c9d75ebb56f0f053f1"
 
 ["commonware_storage::qmdb::conformance::CurrentMmbUnorderedVariableConf"]
 n_cases = 200
-hash = "bad7ded44c7cedc95d202dc3334e3de88877d3aeb63b5808b18bd12c2f59c0d9"
+hash = "c46b60791c970b54e936d47a8e2eaae45e9e34c9fb0e1278e678f378392a02f1"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrOrderedFixedConf"]
 n_cases = 200
-hash = "95dc738adeb88d6546c62828e448213fe723d600f57bb4660ecf982ee76e29ef"
+hash = "0e5264ac6a0c0d56ace9ebc986635b128f9d7a118471875ad874b08e80f66ebb"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrOrderedVariableConf"]
 n_cases = 200
-hash = "a52a284b8aec813f99234a0545d7c8e651b6f6c93d1912394be3e46075a2681d"
+hash = "66cd3cc0299ef062e23d4f9812d5d45896cd808ee0bc65d9745a4265895ae8b5"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrUnorderedFixedConf"]
 n_cases = 200
-hash = "537f4c63d903b112fc045ed47f0127da3e2ca70e38c8406e6535a94b2dddc693"
+hash = "8ce052fef8ce8e4d817a2961144cf3a28aec4bd6eb75d32b013499116a6981da"
 
 ["commonware_storage::qmdb::conformance::CurrentMmrUnorderedVariableConf"]
 n_cases = 200
-hash = "93926373351527268aead35fc1884ff27ae2a154df38309dbed50c2c388d4870"
+hash = "0d0591823b7d5ed8dcb940d086ec214062eb3cfffd018e9fbadf9e16ce8da0f7"
 
 ["commonware_storage::qmdb::conformance::ImmutableMmbCompactFixedConf"]
 n_cases = 200

storage/fuzz/fuzz_targets/current_ordered_operations.rs

@@ -51,6 +51,8 @@ enum CurrentOperation {
         bad_digests: Vec<[u8; 32]>,
         max_ops: NonZeroU64,
         bad_chunks: Vec<[u8; 32]>,
+        bad_prefix_peaks: Vec<[u8; 32]>,
+        bad_suffix_peaks: Vec<[u8; 32]>,
     },
     GetSpan {
         key: RawKey,
@@ -267,7 +269,14 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                     }
                 }
 
-                CurrentOperation::ArbitraryProof {start_loc, bad_digests, max_ops, bad_chunks} => {
+                CurrentOperation::ArbitraryProof {
+                    start_loc,
+                    bad_digests,
+                    max_ops,
+                    bad_chunks,
+                    bad_prefix_peaks,
+                    bad_suffix_peaks,
+                } => {
                     let current_op_count = db.bounds().await.end;
                     if current_op_count == 0 {
                         continue;
@@ -311,6 +320,36 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                                 &root
                             ), "proof with bad chunks should not verify");
                         }
+
+                        let bad_prefix_peaks =
+                            bad_prefix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_prefix_peaks != bad_prefix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_prefix_peaks = bad_prefix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad prefix peaks should not verify");
+                        }
+
+                        let bad_suffix_peaks =
+                            bad_suffix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_suffix_peaks != bad_suffix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_suffix_peaks = bad_suffix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad suffix peaks should not verify");
+                        }
                     }
                 }
 

storage/fuzz/fuzz_targets/current_unordered_operations.rs

@@ -51,6 +51,8 @@ enum CurrentOperation {
         bad_digests: Vec<[u8; 32]>,
         max_ops: NonZeroU64,
         bad_chunks: Vec<[u8; 32]>,
+        bad_prefix_peaks: Vec<[u8; 32]>,
+        bad_suffix_peaks: Vec<[u8; 32]>,
     },
 }
 
@@ -242,7 +244,14 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                     }
                 }
 
-                CurrentOperation::ArbitraryProof {start_loc, bad_digests, max_ops, bad_chunks} => {
+                CurrentOperation::ArbitraryProof {
+                    start_loc,
+                    bad_digests,
+                    max_ops,
+                    bad_chunks,
+                    bad_prefix_peaks,
+                    bad_suffix_peaks,
+                } => {
                     let current_op_count = db.bounds().await.end;
                     if current_op_count == 0 {
                         continue;
@@ -283,6 +292,36 @@ fn fuzz_family<F: Graftable>(data: &FuzzInput, suffix: &str) {
                                 &root
                             ), "proof with bad chunks should not verify");
                         }
+
+                        let bad_prefix_peaks =
+                            bad_prefix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_prefix_peaks != bad_prefix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_prefix_peaks = bad_prefix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad prefix peaks should not verify");
+                        }
+
+                        let bad_suffix_peaks =
+                            bad_suffix_peaks.iter().map(|d| Digest::from(*d)).collect();
+                        if range_proof.unfolded_suffix_peaks != bad_suffix_peaks {
+                            let mut bad_proof = range_proof.clone();
+                            bad_proof.unfolded_suffix_peaks = bad_suffix_peaks;
+                            assert!(!Db::<F>::verify_range_proof(
+                                &mut hasher,
+                                &bad_proof,
+                                start_loc,
+                                &ops,
+                                &chunks,
+                                &root
+                            ), "proof with bad suffix peaks should not verify");
+                        }
                     }
                 }
 

storage/src/merkle/mod.rs

@@ -33,6 +33,8 @@ pub use position::Position;
 #[cfg(test)]
 pub(crate) use proof::build_range_proof;
 pub use proof::Proof;
+#[cfg(feature = "std")]
+pub(crate) use proof::{build_range_collection_proof, range_collection_nodes};
 pub use read::Readable;
 use thiserror::Error;
 

storage/src/merkle/proof.rs

@@ -592,6 +592,66 @@ impl<F: Family, D: Digest> Proof<F, D> {
             )
             .ok_or(ReconstructionError::InvalidProof)
     }
+
+    /// Authenticate the proven range without reconstructing the full generic Merkle root.
+    ///
+    /// This consumes only the sibling digests needed to rebuild the proven range peaks, then returns
+    /// those peak digests in `collected`. It deliberately does not consume peak-bagging witnesses
+    /// such as prefix peaks, after peaks, or backward-fold suffix accumulators.
+    ///
+    /// Current QMDB grafted proofs use this path because their final root is rebuilt by the wrapper
+    /// from the collected range peaks plus grafted prefix/suffix witnesses. Including generic
+    /// peak-bagging witnesses here would create proof bytes that the wrapper root ignores, making
+    /// those bytes malleable.
+    #[cfg(feature = "std")]
+    pub(crate) fn reconstruct_range_collecting<H, E>(
+        &self,
+        hasher: &H,
+        elements: &[E],
+        start_loc: Location<F>,
+        collected: &mut Vec<(Position<F>, D)>,
+    ) -> Result<(), ReconstructionError>
+    where
+        H: Hasher<F, Digest = D>,
+        E: AsRef<[u8]>,
+    {
+        if elements.is_empty() {
+            return Err(ReconstructionError::MissingElements);
+        }
+        if !start_loc.is_valid_index() {
+            return Err(ReconstructionError::InvalidStartLoc);
+        }
+        let end_loc = start_loc
+            .checked_add(elements.len() as u64)
+            .ok_or(ReconstructionError::InvalidEndLoc)?;
+        if end_loc > self.leaves {
+            return Err(ReconstructionError::InvalidEndLoc);
+        }
+
+        let range = start_loc..end_loc;
+        let peaks = range_peaks(self.leaves, self.inactive_peaks, &range)
+            .map_err(|_| ReconstructionError::InvalidSize)?;
+
+        let mut sibling_cursor = 0usize;
+        let mut elements_iter = elements.iter();
+        for peak in &peaks {
+            let peak_digest = peak.reconstruct_digest(
+                hasher,
+                &range,
+                &mut elements_iter,
+                &self.digests,
+                &mut sibling_cursor,
+                None,
+            )?;
+            collected.push((peak.pos, peak_digest));
+        }
+
+        if elements_iter.next().is_some() || sibling_cursor != self.digests.len() {
+            return Err(ReconstructionError::ExtraDigests);
+        }
+
+        Ok(())
+    }
 }
 
 /// A perfect binary subtree within a peak, identified by its root position, height,
@@ -778,6 +838,26 @@ impl<F: Family> Subtree<F> {
     }
 }
 
+/// Return the peaks of a tree of `leaves` that overlap `range`, validating both the range and the
+/// declared `inactive_peaks` boundary.
+///
+/// The returned subtrees are bagging-independent: `Blueprint::new`'s prefix/suffix accumulator
+/// layout depends on bagging, but the per-peak partition of the proven range does not.
+///
+/// # Errors
+///
+/// See [`Blueprint::new`].
+#[cfg(feature = "std")]
+pub(crate) fn range_peaks<F: Family>(
+    leaves: Location<F>,
+    inactive_peaks: usize,
+    range: &Range<Location<F>>,
+) -> Result<Vec<Subtree<F>>, super::Error<F>> {
+    // Bagging only affects `Blueprint`'s prefix/suffix bucketing; `range_peaks` is independent.
+    Blueprint::new(leaves, inactive_peaks, Bagging::ForwardFold, range.clone())
+        .map(|bp| bp.range_peaks)
+}
+
 /// Blueprint for a range proof, separating fold-prefix peaks from nodes that must be fetched.
 pub(crate) struct Blueprint<F: Family> {
     /// Total number of leaves in the structure this blueprint was built for.
@@ -1066,6 +1146,55 @@ where
     )
 }
 
+/// Return the node positions needed by [`build_range_collection_proof`].
+#[cfg(feature = "std")]
+pub(crate) fn range_collection_nodes<F: Family>(
+    leaves: Location<F>,
+    inactive_peaks: usize,
+    range: Range<Location<F>>,
+) -> Result<Vec<Position<F>>, super::Error<F>> {
+    let peaks = range_peaks(leaves, inactive_peaks, &range)?;
+    let mut fetch_nodes = Vec::new();
+    for peak in &peaks {
+        peak.collect_siblings(&range, &mut fetch_nodes);
+    }
+    Ok(fetch_nodes)
+}
+
+/// Build a proof containing only the sibling digests needed to authenticate the requested range.
+///
+/// `fetch_nodes` must be the slice returned by [`range_collection_nodes`] for the same `leaves` and
+/// `inactive_peaks`; the caller passes it in so the Blueprint traversal isn't repeated when the
+/// positions are already known (e.g., to drive concurrent fetching).
+///
+/// The resulting proof cannot reconstruct a complete generic Merkle root on its own. It is intended
+/// for wrappers that rebuild a custom root from separately supplied peak witnesses, such as current
+/// QMDB grafted proofs.
+#[cfg(feature = "std")]
+pub(crate) fn build_range_collection_proof<F, D, E>(
+    leaves: Location<F>,
+    inactive_peaks: usize,
+    fetch_nodes: &[Position<F>],
+    get_node: impl Fn(Position<F>) -> Option<D>,
+    element_pruned: impl Fn(Position<F>) -> E,
+) -> Result<Proof<F, D>, E>
+where
+    F: Family,
+    D: Digest,
+    E: From<super::Error<F>>,
+{
+    let mut digests = Vec::with_capacity(fetch_nodes.len());
+    for &pos in fetch_nodes {
+        digests.push(get_node(pos).ok_or_else(|| element_pruned(pos))?);
+    }
+
+    Ok(Proof {
+        leaves,
+        inactive_peaks,
+        digests,
+    })
+}
+
 /// Returns the positions of the minimal set of nodes whose digests are required to prove the
 /// inclusion of the elements at the specified `locations`, using the provided root bagging.
 #[cfg(any(feature = "std", test))]