[Storage] Support fixed-length values in qmdb::immutable (#3461)

Author: danlaine

examples/sync/src/databases/immutable.rs

@@ -5,7 +5,10 @@ use commonware_cryptography::{Hasher as CryptoHasher, Sha256};
 use commonware_runtime::{BufferPooler, Clock, Metrics, Storage};
 use commonware_storage::{
     journal::contiguous::variable::Config as VConfig,
-    mmr::{self, journaled::Config as MmrConfig, Location, Proof},
+    merkle::{
+        journaled::Config as MmrConfig,
+        mmr::{self, Location, Proof},
+    },
     qmdb::{
         self,
         immutable::{self, Config},
@@ -16,13 +19,13 @@ use std::{future::Future, num::NonZeroU64};
 use tracing::error;
 
 /// Database type alias.
-pub type Database<E> = immutable::Immutable<mmr::Family, E, Key, Value, Hasher, Translator>;
+pub type Database<E> = immutable::variable::Db<mmr::Family, E, Key, Value, Hasher, Translator>;
 
 /// Operation type alias.
-pub type Operation = immutable::Operation<Key, Value>;
+pub type Operation = immutable::variable::Operation<Key, Value>;
 
 /// Create a database configuration with appropriate partitioning for Immutable.
-pub fn create_config(context: &impl BufferPooler) -> Config<Translator, ((), ())> {
+pub fn create_config(context: &impl BufferPooler) -> Config<Translator, VConfig<((), ())>> {
     let page_cache = commonware_runtime::buffer::paged::CacheRef::from_pooler(
         context,
         NZU16!(2048),

storage/conformance.toml

@@ -138,14 +138,18 @@ hash = "f5456580eb69727a23bfb0281e33f467acc0c91273efebd334d3e82c9770ae76"
 n_cases = 65536
 hash = "a3fbb5f749fa5b73a684e9f0bebd8973c456e5ee43a0a3db75a99aa550dee302"
 
-["commonware_storage::qmdb::immutable::operation::tests::conformance::CodecConformance<Operation<U64,U64>>"]
+["commonware_storage::qmdb::immutable::operation::fixed::tests::conformance::CodecConformance<FixedOp>"]
 n_cases = 65536
-hash = "fdca5df62d243b28676ee15034663694cd219d5ef80749079126b0ed73effe0d"
+hash = "4f75cdf8952431729e7a3dfad38a8d5bfbb92bf6b54b1ca180a66745aed618d5"
 
-["commonware_storage::qmdb::immutable::operation::tests::conformance::CodecConformance<Operation<Vec<u8>,U64>>"]
+["commonware_storage::qmdb::immutable::operation::variable::tests::conformance::CodecConformance<VarKeyOp>"]
 n_cases = 65536
 hash = "cce5f888e506282f861e0e49e176b26c65f92e457f0c5f5353fc9b7196f07478"
 
+["commonware_storage::qmdb::immutable::operation::variable::tests::conformance::CodecConformance<VarOp>"]
+n_cases = 65536
+hash = "fdca5df62d243b28676ee15034663694cd219d5ef80749079126b0ed73effe0d"
+
 ["commonware_storage::qmdb::keyless::operation::tests::conformance::CodecConformance<Operation<U64>>"]
 n_cases = 65536
 hash = "b41d6c6ec560bde9caf2e206526864c618a0721af367585a1719617ca7ce9291"

storage/fuzz/fuzz_targets/qmdb_immutable.rs

@@ -9,7 +9,7 @@ use commonware_storage::{
     merkle::{hasher::Standard, mmb, mmr, Family as MerkleFamily, Location},
     mmr::journaled::Config as MerkleConfig,
     qmdb::{
-        immutable::{Config, Immutable},
+        immutable::{variable::Db as Immutable, Config},
         verify_proof,
     },
     translator::TwoCap,
@@ -92,10 +92,11 @@ fn generate_value(rng: &mut StdRng, size: usize) -> Vec<u8> {
     (0..actual_size).map(|_| rng.gen()).collect()
 }
 
+#[allow(clippy::type_complexity)]
 fn db_config(
     suffix: &str,
     pooler: &impl BufferPooler,
-) -> Config<TwoCap, ((), (RangeCfg<usize>, ()))> {
+) -> Config<TwoCap, VConfig<((), (RangeCfg<usize>, ()))>> {
     let page_cache = CacheRef::from_pooler(pooler, PAGE_SIZE, NZUsize!(PAGE_CACHE_SIZE));
     Config {
         merkle_config: MerkleConfig {

storage/src/qmdb/any/unordered/variable.rs

@@ -660,7 +660,7 @@ pub(crate) mod test {
     /// Regression test for https://github.com/commonwarexyz/monorepo/issues/2787
     #[allow(dead_code, clippy::manual_async_fn)]
     fn issue_2787_regression(
-        db: &crate::qmdb::immutable::Immutable<
+        db: &crate::qmdb::immutable::variable::Db<
             mmr::Family,
             deterministic::Context,
             Digest,

storage/src/qmdb/immutable/batch.rs

@@ -2,12 +2,13 @@
 
 use super::Immutable;
 use crate::{
-    journal::authenticated,
+    journal::{authenticated, contiguous::Mutable, Error as JournalError},
     merkle::{Family, Location, Position},
-    qmdb::{any::VariableValue, immutable::operation::Operation, operation::Key},
+    qmdb::{any::ValueEncoding, immutable::operation::Operation, operation::Key, Error},
     translator::Translator,
-    Context,
+    Context, Persistable,
 };
+use commonware_codec::EncodeShared;
 use commonware_cryptography::{Digest, Hasher as CHasher};
 use std::{collections::BTreeMap, sync::Arc};
 
@@ -33,17 +34,17 @@ pub struct UnmerkleizedBatch<F, H, K, V>
 where
     F: Family,
     K: Key,
-    V: VariableValue,
+    V: ValueEncoding,
     H: CHasher,
 {
     /// Authenticated journal batch for computing the speculative Merkle root.
     journal_batch: authenticated::UnmerkleizedBatch<F, H, Operation<K, V>>,
 
     /// Pending mutations.
-    mutations: BTreeMap<K, V>,
+    mutations: BTreeMap<K, V::Value>,
 
     /// Uncommitted key-level changes accumulated by prior batches in the chain.
-    base_diff: Arc<BTreeMap<K, DiffEntry<F, V>>>,
+    base_diff: Arc<BTreeMap<K, DiffEntry<F, V::Value>>>,
 
     /// Total operation count before this batch (committed DB + prior batches).
     /// This batch's i-th operation lands at location `base_size + i`.
@@ -55,12 +56,12 @@ where
 
 /// A speculative batch of operations whose root digest has been computed,
 /// in contrast to [`UnmerkleizedBatch`].
-pub struct MerkleizedBatch<F: Family, D: Digest, K: Key, V: VariableValue> {
+pub struct MerkleizedBatch<F: Family, D: Digest, K: Key, V: ValueEncoding> {
     /// Journal batch (Merkle state + accumulated operation segments).
     journal: authenticated::MerkleizedBatch<F, D, Operation<K, V>>,
 
     /// All uncommitted key-level changes from the batch chain.
-    diff: Arc<BTreeMap<K, DiffEntry<F, V>>>,
+    diff: Arc<BTreeMap<K, DiffEntry<F, V::Value>>>,
 
     /// Total operation count after this batch.
     total_size: u64,
@@ -70,7 +71,7 @@ pub struct MerkleizedBatch<F: Family, D: Digest, K: Key, V: VariableValue> {
 }
 
 /// An owned changeset that can be applied to the database.
-pub struct Changeset<F: Family, K: Key, D: Digest, V: VariableValue> {
+pub struct Changeset<F: Family, K: Key, D: Digest, V: ValueEncoding> {
     /// The finalized authenticated journal batch (Merkle changeset + item chain).
     pub(super) journal_finalized: authenticated::Changeset<F, D, Operation<K, V>>,
 
@@ -88,13 +89,19 @@ impl<F, H, K, V> UnmerkleizedBatch<F, H, K, V>
 where
     F: Family,
     K: Key,
-    V: VariableValue,
+    V: ValueEncoding,
     H: CHasher,
+    Operation<K, V>: EncodeShared,
 {
     /// Create a batch from a committed DB (no parent chain).
-    pub(super) fn new<E, T>(immutable: &Immutable<F, E, K, V, H, T>, journal_size: u64) -> Self
+    pub(super) fn new<E, C, T>(
+        immutable: &Immutable<F, E, K, V, C, H, T>,
+        journal_size: u64,
+    ) -> Self
     where
         E: Context,
+        C: Mutable<Item = Operation<K, V>> + Persistable<Error = JournalError>,
+        C::Item: EncodeShared,
         T: Translator,
     {
         Self {
@@ -108,21 +115,23 @@ where
 
     /// Set a key to a value.
     ///
-    /// Duplicate keys are not supported. The key must not already exist in the database or in any
-    /// ancestor batch in the chain. Setting a key that already exists is undefined behavior.
-    pub fn set(mut self, key: K, value: V) -> Self {
+    /// The key must not already exist in the database or in any ancestor batch
+    /// in the chain. Setting a key that already exists causes undefined behavior.
+    pub fn set(mut self, key: K, value: V::Value) -> Self {
         self.mutations.insert(key, value);
         self
     }
 
     /// Read through: mutations -> base diff -> committed DB.
-    pub async fn get<E, T>(
+    pub async fn get<E, C, T>(
         &self,
         key: &K,
-        db: &Immutable<F, E, K, V, H, T>,
-    ) -> Result<Option<V>, crate::qmdb::Error<F>>
+        db: &Immutable<F, E, K, V, C, H, T>,
+    ) -> Result<Option<V::Value>, Error<F>>
     where
         E: Context,
+        C: Mutable<Item = Operation<K, V>> + Persistable<Error = JournalError>,
+        C::Item: EncodeShared,
         T: Translator,
     {
         // Check this batch's pending mutations.
@@ -138,12 +147,12 @@ where
     }
 
     /// Resolve mutations into operations, merkleize, and return a [`MerkleizedBatch`].
-    pub fn merkleize(self, metadata: Option<V>) -> MerkleizedBatch<F, H::Digest, K, V> {
+    pub fn merkleize(self, metadata: Option<V::Value>) -> MerkleizedBatch<F, H::Digest, K, V> {
         let base = self.base_size;
 
         // Build operations: one Set per key (BTreeMap iterates in sorted order), then Commit.
         let mut ops: Vec<Operation<K, V>> = Vec::with_capacity(self.mutations.len() + 1);
-        let mut diff: BTreeMap<K, DiffEntry<F, V>> = BTreeMap::new();
+        let mut diff: BTreeMap<K, DiffEntry<F, V::Value>> = BTreeMap::new();
 
         for (key, value) in self.mutations {
             let loc = Location::new(base + ops.len() as u64);
@@ -179,20 +188,25 @@ where
     }
 }
 
-impl<F: Family, D: Digest, K: Key, V: VariableValue> MerkleizedBatch<F, D, K, V> {
+impl<F: Family, D: Digest, K: Key, V: ValueEncoding> MerkleizedBatch<F, D, K, V>
+where
+    Operation<K, V>: EncodeShared,
+{
     /// Return the speculative root.
     pub fn root(&self) -> D {
         self.journal.root()
     }
 
     /// Read through: diff -> committed DB.
-    pub async fn get<E, H, T>(
+    pub async fn get<E, C, H, T>(
         &self,
         key: &K,
-        db: &Immutable<F, E, K, V, H, T>,
-    ) -> Result<Option<V>, crate::qmdb::Error<F>>
+        db: &Immutable<F, E, K, V, C, H, T>,
+    ) -> Result<Option<V::Value>, Error<F>>
     where
         E: Context,
+        C: Mutable<Item = Operation<K, V>> + Persistable<Error = JournalError>,
+        C::Item: EncodeShared,
         H: CHasher<Digest = D>,
         T: Translator,
     {
@@ -277,12 +291,14 @@ impl<F: Family, D: Digest, K: Key, V: VariableValue> MerkleizedBatch<F, D, K, V>
     }
 }
 
-impl<F, E, K, V, H, T> Immutable<F, E, K, V, H, T>
+impl<F, E, K, V, C, H, T> Immutable<F, E, K, V, C, H, T>
 where
     F: Family,
     E: Context,
     K: Key,
-    V: VariableValue,
+    V: ValueEncoding,
+    C: Mutable<Item = Operation<K, V>> + Persistable<Error = JournalError>,
+    C::Item: EncodeShared,
     H: CHasher,
     T: Translator,
 {