|
| 1 | +//! Keyless database types and helpers for the sync example. |
| 2 | +//! |
| 3 | +//! A `keyless` database is append-only: operations are stored by location rather than by key. |
| 4 | +//! It supports `Append(value)` and `Commit(metadata)` operations. For sync, the engine targets |
| 5 | +//! the Merkle root over all operations, and the client reconstructs the same state by replaying |
| 6 | +//! the fetched operations. |
| 7 | +
|
| 8 | +use crate::{Hasher, Key, Value}; |
| 9 | +use commonware_cryptography::{Hasher as CryptoHasher, Sha256}; |
| 10 | +use commonware_runtime::{buffer, BufferPooler, Clock, Metrics, Storage}; |
| 11 | +use commonware_storage::{ |
| 12 | + journal::contiguous::fixed::Config as FConfig, |
| 13 | + merkle::{ |
| 14 | + journaled::Config as MmrConfig, |
| 15 | + mmr::{self, Location, Proof}, |
| 16 | + }, |
| 17 | + qmdb::{ |
| 18 | + self, |
| 19 | + keyless::{self, fixed}, |
| 20 | + operation::Committable, |
| 21 | + }, |
| 22 | +}; |
| 23 | +use commonware_utils::{NZUsize, NZU16, NZU64}; |
| 24 | +use std::num::NonZeroU64; |
| 25 | +use tracing::error; |
| 26 | + |
| 27 | +/// Database type alias. |
| 28 | +pub type Database<E> = fixed::Db<mmr::Family, E, Value, Hasher>; |
| 29 | + |
| 30 | +/// Operation type alias. |
| 31 | +pub type Operation = fixed::Operation<Value>; |
| 32 | + |
| 33 | +/// Create a database configuration for the keyless variant. |
| 34 | +pub fn create_config(context: &(impl BufferPooler + commonware_runtime::Metrics)) -> fixed::Config { |
| 35 | + let page_cache = buffer::paged::CacheRef::from_pooler( |
| 36 | + &context.with_label("page_cache"), |
| 37 | + NZU16!(2048), |
| 38 | + NZUsize!(10), |
| 39 | + ); |
| 40 | + keyless::Config { |
| 41 | + merkle: MmrConfig { |
| 42 | + journal_partition: "mmr-journal".into(), |
| 43 | + metadata_partition: "mmr-metadata".into(), |
| 44 | + items_per_blob: NZU64!(4096), |
| 45 | + write_buffer: NZUsize!(4096), |
| 46 | + thread_pool: None, |
| 47 | + page_cache: page_cache.clone(), |
| 48 | + }, |
| 49 | + log: FConfig { |
| 50 | + partition: "log-journal".into(), |
| 51 | + items_per_blob: NZU64!(4096), |
| 52 | + write_buffer: NZUsize!(4096), |
| 53 | + page_cache, |
| 54 | + }, |
| 55 | + } |
| 56 | +} |
| 57 | + |
| 58 | +/// Create deterministic test operations for demonstration purposes. |
| 59 | +/// Generates Append operations and periodic Commit operations. |
| 60 | +pub fn create_test_operations(count: usize, seed: u64) -> Vec<Operation> { |
| 61 | + let mut operations = Vec::new(); |
| 62 | + let mut hasher = <Hasher as CryptoHasher>::new(); |
| 63 | + |
| 64 | + for i in 0..count { |
| 65 | + let value = { |
| 66 | + hasher.update(&i.to_be_bytes()); |
| 67 | + hasher.update(&seed.to_be_bytes()); |
| 68 | + hasher.finalize() |
| 69 | + }; |
| 70 | + |
| 71 | + operations.push(Operation::Append(value)); |
| 72 | + |
| 73 | + if (i + 1) % 10 == 0 { |
| 74 | + operations.push(Operation::Commit(None)); |
| 75 | + } |
| 76 | + } |
| 77 | + |
| 78 | + // Always end with a commit |
| 79 | + operations.push(Operation::Commit(Some(Sha256::fill(1)))); |
| 80 | + operations |
| 81 | +} |
| 82 | + |
| 83 | +impl<E> super::Syncable for Database<E> |
| 84 | +where |
| 85 | + E: Storage + Clock + Metrics, |
| 86 | +{ |
| 87 | + type Family = mmr::Family; |
| 88 | + type Operation = Operation; |
| 89 | + |
| 90 | + fn create_test_operations(count: usize, seed: u64) -> Vec<Self::Operation> { |
| 91 | + create_test_operations(count, seed) |
| 92 | + } |
| 93 | + |
| 94 | + async fn add_operations( |
| 95 | + &mut self, |
| 96 | + operations: Vec<Self::Operation>, |
| 97 | + ) -> Result<(), qmdb::Error<mmr::Family>> { |
| 98 | + if operations.last().is_none() || !operations.last().unwrap().is_commit() { |
| 99 | + // Ignore bad inputs rather than return errors. |
| 100 | + error!("operations must end with a commit"); |
| 101 | + return Ok(()); |
| 102 | + } |
| 103 | + |
| 104 | + let mut batch = self.new_batch(); |
| 105 | + for operation in operations { |
| 106 | + match operation { |
| 107 | + Operation::Append(value) => { |
| 108 | + batch = batch.append(value); |
| 109 | + } |
| 110 | + Operation::Commit(metadata) => { |
| 111 | + let merkleized = batch.merkleize(self, metadata); |
| 112 | + self.apply_batch(merkleized).await?; |
| 113 | + self.commit().await?; |
| 114 | + batch = self.new_batch(); |
| 115 | + } |
| 116 | + } |
| 117 | + } |
| 118 | + Ok(()) |
| 119 | + } |
| 120 | + |
| 121 | + fn root(&self) -> Key { |
| 122 | + self.root() |
| 123 | + } |
| 124 | + |
| 125 | + async fn size(&self) -> Location { |
| 126 | + self.bounds().await.end |
| 127 | + } |
| 128 | + |
| 129 | + async fn inactivity_floor(&self) -> Location { |
| 130 | + // Keyless databases have no inactivity floor concept. |
| 131 | + // Use the pruning boundary, same as immutable. |
| 132 | + self.bounds().await.start |
| 133 | + } |
| 134 | + |
| 135 | + async fn historical_proof( |
| 136 | + &self, |
| 137 | + op_count: Location, |
| 138 | + start_loc: Location, |
| 139 | + max_ops: NonZeroU64, |
| 140 | + ) -> Result<(Proof<Key>, Vec<Self::Operation>), qmdb::Error<mmr::Family>> { |
| 141 | + self.historical_proof(op_count, start_loc, max_ops).await |
| 142 | + } |
| 143 | + |
| 144 | + async fn pinned_nodes_at(&self, loc: Location) -> Result<Vec<Key>, qmdb::Error<mmr::Family>> { |
| 145 | + self.pinned_nodes_at(loc).await |
| 146 | + } |
| 147 | + |
| 148 | + fn name() -> &'static str { |
| 149 | + "keyless" |
| 150 | + } |
| 151 | +} |
| 152 | + |
| 153 | +#[cfg(test)] |
| 154 | +mod tests { |
| 155 | + use super::*; |
| 156 | + use crate::databases::Syncable; |
| 157 | + use commonware_runtime::deterministic; |
| 158 | + |
| 159 | + type KeylessDb = Database<deterministic::Context>; |
| 160 | + |
| 161 | + #[test] |
| 162 | + fn test_create_test_operations() { |
| 163 | + let ops = <KeylessDb as Syncable>::create_test_operations(5, 12345); |
| 164 | + assert_eq!(ops.len(), 6); // 5 operations + 1 commit |
| 165 | + |
| 166 | + if let Operation::Commit(Some(_)) = &ops[5] { |
| 167 | + // ok |
| 168 | + } else { |
| 169 | + panic!("last operation should be a commit with metadata"); |
| 170 | + } |
| 171 | + } |
| 172 | + |
| 173 | + #[test] |
| 174 | + fn test_deterministic_operations() { |
| 175 | + // Operations should be deterministic based on seed |
| 176 | + let ops1 = <KeylessDb as Syncable>::create_test_operations(3, 12345); |
| 177 | + let ops2 = <KeylessDb as Syncable>::create_test_operations(3, 12345); |
| 178 | + assert_eq!(ops1, ops2); |
| 179 | + |
| 180 | + // Different seeds should produce different operations |
| 181 | + let ops3 = <KeylessDb as Syncable>::create_test_operations(3, 54321); |
| 182 | + assert_ne!(ops1, ops3); |
| 183 | + } |
| 184 | +} |
0 commit comments