Merge remote-tracking branch 'origin/main' into danlaine/keyless-inactivity-floor

Author: danlaine

examples/sync/src/databases/immutable.rs

@@ -22,7 +22,7 @@ use tracing::error;
 pub type Database<E> = immutable::variable::Db<mmr::Family, E, Key, Value, Hasher, Translator>;
 
 /// Operation type alias.
-pub type Operation = immutable::variable::Operation<Key, Value>;
+pub type Operation = immutable::variable::Operation<mmr::Family, Key, Value>;
 
 /// Create a database configuration with appropriate partitioning for Immutable.
 pub fn create_config(context: &impl BufferPooler) -> Config<Translator, VConfig<((), ())>> {
@@ -74,12 +74,12 @@ pub fn create_test_operations(count: usize, seed: u64) -> Vec<Operation> {
         operations.push(Operation::Set(key, value));
 
         if (i + 1) % 10 == 0 {
-            operations.push(Operation::Commit(None));
+            operations.push(Operation::Commit(None, Location::new(0)));
         }
     }
 
     // Always end with a commit
-    operations.push(Operation::Commit(Some(Sha256::fill(1))));
+    operations.push(Operation::Commit(Some(Sha256::fill(1)), Location::new(0)));
     operations
 }
 
@@ -110,8 +110,8 @@ where
                 Operation::Set(key, value) => {
                     batch = batch.set(key, value);
                 }
-                Operation::Commit(metadata) => {
-                    let merkleized = batch.merkleize(self, metadata);
+                Operation::Commit(metadata, floor) => {
+                    let merkleized = batch.merkleize(self, metadata, floor);
                     self.apply_batch(merkleized).await?;
                     self.commit().await?;
                     batch = self.new_batch();
@@ -130,7 +130,7 @@ where
     }
 
     async fn sync_boundary(&self) -> Location {
-        self.sync_boundary().await
+        self.sync_boundary()
     }
 
     fn historical_proof(

storage/conformance.toml

@@ -208,15 +208,15 @@ hash = "ba4c1ce0c2975f27231e13920129ecd63ae173c10a6e481af388ada2383ea15e"
 
 ["commonware_storage::qmdb::conformance::ImmutableMmbVariableConf"]
 n_cases = 200
-hash = "aee77e198354d5cee1dae5e572cfad11248b0e8635fdc9c19245442e6785fa25"
+hash = "d148ed1d4ce0fce52bfb24463e40f777bdc5915cbb7710d31e155d5b13237d1f"
 
 ["commonware_storage::qmdb::conformance::ImmutableMmrFixedConf"]
 n_cases = 200
 hash = "6a25e6efa15b15bfd6f39e588fca23c1ec431eec1f148722d8a00491876c4252"
 
 ["commonware_storage::qmdb::conformance::ImmutableMmrVariableConf"]
 n_cases = 200
-hash = "f32fda4f6c0bbde62f8f1959edeb2ac67b28f5347bd108e4118e740aa1fd8fd1"
+hash = "cafbb288de6f8d5c73ddb19df64d2e1429b4f2e273124cb1b20be3026c54d288"
 
 ["commonware_storage::qmdb::conformance::KeylessMmbFixedConf"]
 n_cases = 200
@@ -236,15 +236,15 @@ hash = "aee37bfef687a6855e53f2e4a2ef7190cf9d33cdb67b5e177a7f56f7b44e4b9b"
 
 ["commonware_storage::qmdb::immutable::operation::fixed::tests::conformance::CodecConformance<FixedOp>"]
 n_cases = 65536
-hash = "4f75cdf8952431729e7a3dfad38a8d5bfbb92bf6b54b1ca180a66745aed618d5"
+hash = "1737c49c112fc9dfdc4dde3626d8ab08a9df8353b9a702e488d3a1fd8e9428dc"
 
 ["commonware_storage::qmdb::immutable::operation::variable::tests::conformance::CodecConformance<VarKeyOp>"]
 n_cases = 65536
-hash = "cce5f888e506282f861e0e49e176b26c65f92e457f0c5f5353fc9b7196f07478"
+hash = "bdee433c53489ee67a42ad2029081d3f233799bd360d6e5e7f29cbaec87c9064"
 
 ["commonware_storage::qmdb::immutable::operation::variable::tests::conformance::CodecConformance<VarOp>"]
 n_cases = 65536
-hash = "fdca5df62d243b28676ee15034663694cd219d5ef80749079126b0ed73effe0d"
+hash = "7952a9bb3a9cec87a95af6dd96a5b88b535106f0241770e0bfdb8aeaf9421056"
 
 ["commonware_storage::qmdb::keyless::operation::tests::conformance::CodecConformance<Operation<mmr::Family,FixedEncoding<U64>>>"]
 n_cases = 65536

storage/fuzz/fuzz_targets/qmdb_immutable.rs

@@ -40,6 +40,7 @@ enum ImmutableOperation {
     Commit {
         has_metadata: bool,
         metadata_size: usize,
+        advance_floor: bool,
     },
     Prune {
         loc: u64,
@@ -179,24 +180,35 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
                     ImmutableOperation::Commit {
                         has_metadata,
                         metadata_size,
+                        advance_floor,
                     } => {
                         let metadata = if has_metadata {
                             Some(generate_value(&mut rng, metadata_size))
                         } else {
                             None
                         };
 
+                        let end = db.bounds().await.end;
+                        let pending_count = pending_sets.len() as u64;
                         assign_pending_locations(
                             &pending_sets,
-                            db.bounds().await.end,
+                            end,
                             &mut keys_set,
                             &mut set_locations,
                         );
                         let mut batch = db.new_batch();
                         for (k, v) in pending_sets.drain(..) {
                             batch = batch.set(k, v);
                         }
-                        let merkleized = batch.merkleize(&db, metadata);
+                        let floor = if advance_floor {
+                            // Advance floor to the commit location (end of this batch).
+                            // total_size = end + pending_count + 1 (commit op).
+                            // Floor at the commit op is the maximum valid value.
+                            Location::new(*end + pending_count)
+                        } else {
+                            db.inactivity_floor_loc()
+                        };
+                        let merkleized = batch.merkleize(&db, metadata, floor);
                         db.apply_batch(merkleized).await.unwrap();
                         db.commit().await.unwrap();
                         last_commit_loc = Some(db.bounds().await.end - 1);
@@ -216,7 +228,10 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
                             for (k, v) in pending_sets.drain(..) {
                                 batch = batch.set(k, v);
                             }
-                            let merkleized = batch.merkleize(&db, None);
+                            // Set the floor to at least safe_loc so the prune succeeds,
+                            // but never below the current floor (monotonicity).
+                            let floor = safe_loc.max(db.inactivity_floor_loc());
+                            let merkleized = batch.merkleize(&db, None, floor);
                             db.apply_batch(merkleized).await.unwrap();
                             db.commit().await.unwrap();
                             last_commit_loc = Some(db.bounds().await.end - 1);
@@ -247,7 +262,8 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
                             for (k, v) in pending_sets.drain(..) {
                                 batch = batch.set(k, v);
                             }
-                            let merkleized = batch.merkleize(&db, None);
+                            let floor = db.inactivity_floor_loc();
+                            let merkleized = batch.merkleize(&db, None, floor);
                             db.apply_batch(merkleized).await.unwrap();
                             db.commit().await.unwrap();
                             last_commit_loc = Some(db.bounds().await.end - 1);
@@ -272,7 +288,8 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
                             let safe_max_ops =
                                 NonZeroU64::new((max_ops % MAX_PROOF_OPS).max(1)).unwrap();
 
-                            let batch = db.new_batch().merkleize(&db, None);
+                            let floor = db.inactivity_floor_loc();
+                            let batch = db.new_batch().merkleize(&db, None, floor);
                             db.apply_batch(batch).await.unwrap();
                             db.commit().await.unwrap();
                             last_commit_loc = Some(db.bounds().await.end - 1);
@@ -307,7 +324,8 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
                         for (k, v) in pending_sets.drain(..) {
                             batch = batch.set(k, v);
                         }
-                        let merkleized = batch.merkleize(&db, None);
+                        let floor = db.inactivity_floor_loc();
+                        let merkleized = batch.merkleize(&db, None, floor);
                         db.apply_batch(merkleized).await.unwrap();
                         db.commit().await.unwrap();
                         last_commit_loc = Some(db.bounds().await.end - 1);
@@ -326,7 +344,8 @@ fn fuzz_family<F: MerkleFamily>(input: &FuzzInput, suffix: &str) {
             for (k, v) in pending_sets.drain(..) {
                 batch = batch.set(k, v);
             }
-            let merkleized = batch.merkleize(&db, None);
+            let floor = db.inactivity_floor_loc();
+            let merkleized = batch.merkleize(&db, None, floor);
             db.apply_batch(merkleized).await.unwrap();
             db.destroy().await.unwrap();
         }

storage/src/qmdb/conformance.rs

@@ -237,11 +237,12 @@ async fn apply_writes<F: Family, D: DbAny<F, Key = Digest, Value = Digest>>(
 /// Apply a batch of immutable sets to the database.
 macro_rules! apply_sets {
     ($db:ident, $ops:expr) => {{
+        let floor = $db.inactivity_floor_loc();
         let mut batch = $db.new_batch();
         for (k, v) in $ops {
             batch = batch.set(k, v);
         }
-        let merkleized = batch.merkleize(&$db, None);
+        let merkleized = batch.merkleize(&$db, None, floor);
         $db.apply_batch(merkleized).await.unwrap();
     }};
 }
@@ -642,18 +643,20 @@ macro_rules! assert_immutable_order_independent {
             ops.push((colliding_digest(0xCD, i), to_val(i, 100)));
         }
 
+        let fwd_floor = $fwd.inactivity_floor_loc();
         let mut batch = $fwd.new_batch();
         for &(k, v) in &ops {
             batch = batch.set(k, v);
         }
-        let merkleized = batch.merkleize(&$fwd, None);
+        let merkleized = batch.merkleize(&$fwd, None, fwd_floor);
         $fwd.apply_batch(merkleized).await.unwrap();
 
+        let rev_floor = $rev.inactivity_floor_loc();
         let mut batch = $rev.new_batch();
         for &(k, v) in ops.iter().rev() {
             batch = batch.set(k, v);
         }
-        let merkleized = batch.merkleize(&$rev, None);
+        let merkleized = batch.merkleize(&$rev, None, rev_floor);
         $rev.apply_batch(merkleized).await.unwrap();
 
         assert_eq!(