From ad890a09726657fe51ab2d36c7f156363c0e5212 Mon Sep 17 00:00:00 2001 From: Nelson Dominguez Date: Fri, 27 Feb 2026 14:57:27 +0100 Subject: [PATCH 1/2] Add storage abstraction layer for metadata persistence --- Cargo.lock | 32 ++++++++++++ Cargo.toml | 2 + minikv-core/Cargo.toml | 1 + minikv-core/src/error.rs | 5 ++ minikv-core/src/lib.rs | 4 ++ minikv-core/src/storage/mod.rs | 95 ++++++++++++++++++++++++++++++++++ 6 files changed, 139 insertions(+) create mode 100644 minikv-core/src/error.rs create mode 100644 minikv-core/src/storage/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ef31703..0822f21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,7 @@ dependencies = [ "dashmap", "thiserror", "tokio", + "tracing", ] [[package]] @@ -319,6 +320,37 @@ dependencies = [ "syn", ] +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/Cargo.toml b/Cargo.toml index 09fcb88..7758821 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ categories = ["database", "network-programming"] [workspace.dependencies] thiserror = "2" +tracing = "0.1" + base64 = "0.22" blake3 = "1.8" dashmap = "6.1" diff --git a/minikv-core/Cargo.toml b/minikv-core/Cargo.toml index bd3e8d0..6731581 100644 --- a/minikv-core/Cargo.toml +++ b/minikv-core/Cargo.toml @@ -12,6 +12,7 @@ categories.workspace = true [dependencies] thiserror.workspace = true +tracing.workspace = true base64 = { workspace = true } blake3 = { workspace = true } dashmap = { workspace = true } diff --git a/minikv-core/src/error.rs b/minikv-core/src/error.rs new file mode 100644 index 0000000..f4bc3ae --- /dev/null +++ b/minikv-core/src/error.rs @@ -0,0 +1,5 @@ +use thiserror::Error; + +/// Unified error type for minikv. +#[derive(Debug, Error)] +pub enum Error {} diff --git a/minikv-core/src/lib.rs b/minikv-core/src/lib.rs index 74e284f..babf06d 100644 --- a/minikv-core/src/lib.rs +++ b/minikv-core/src/lib.rs @@ -1,3 +1,7 @@ +pub mod error; pub mod hashing; pub mod locking; +pub mod storage; pub mod volumes; + +pub use error::Error; diff --git a/minikv-core/src/storage/mod.rs b/minikv-core/src/storage/mod.rs new file mode 100644 index 0000000..f3268c1 --- /dev/null +++ b/minikv-core/src/storage/mod.rs @@ -0,0 +1,95 @@ +//! Storage abstraction layer for metadata persistence. +//! +//! All interactions with LevelDB or in-memory stores go through the +//! `MetadataStore` trait. This isolates storage concerns from business logic +//! and allows swapping backends in tests or future implementations. + +use crate::error::Error; + +/// A single key-value pair in a metadata store. +/// +/// - The first element (`Vec`) is the key. +/// - The second element (`Vec`) is the associated value. +/// +/// This type is used throughout `MetadataStore` APIs for operations that +/// return or manipulate multiple entries, such as `scan_prefix` or `scan_all`. +pub type KeyValuePair = (Vec, Vec); + +/// Trait defining the interface for all metadata stores. +/// +/// Implementors must be `Send + Sync` to allow sharing via `Arc` +/// across async tasks. +pub trait MetadataStore: Send + Sync { + /// Retrieve the value associated with `key`. Returns `None` if the key is absent. + fn get(&self, key: &[u8]) -> Result>, Error>; + + /// Insert or update a key-value pair. + fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Error>; + + /// Delete the given key. Succeeds silently if the key does not exist. + fn delete(&self, key: &[u8]) -> Result<(), Error>; + + /// Return all key-value pairs with keys starting with `prefix`, + /// ordered lexicographically by key. + fn scan_prefix(&self, prefix: &[u8]) -> Result, Error>; + + /// Return all key-value pairs in the store. + fn scan_all(&self) -> Result, Error>; + + /// Remove all entries in the store. + /// Primarily used by `rebuild_all` to regenerate the database from scratch. + fn delete_all(&self) -> Result<(), Error>; +} + +/// In-memory `MetadataStore` backed by a `BTreeMap`. +/// +/// Useful for unit and integration tests that do not require persistent storage. +#[cfg(debug_assertions)] +pub mod mem { + use super::*; + use std::collections::BTreeMap; + use std::sync::Mutex; + + #[derive(Default)] + pub struct MemStore { + inner: Mutex, Vec>>, + } + + impl MetadataStore for MemStore { + fn get(&self, key: &[u8]) -> Result>, Error> { + Ok(self.inner.lock().unwrap().get(key).cloned()) + } + + fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Error> { + self.inner + .lock() + .unwrap() + .insert(key.to_vec(), value.to_vec()); + Ok(()) + } + + fn delete(&self, key: &[u8]) -> Result<(), Error> { + self.inner.lock().unwrap().remove(key); + Ok(()) + } + + fn scan_prefix(&self, prefix: &[u8]) -> Result, Error> { + let store = self.inner.lock().unwrap(); + Ok(store + .iter() + .filter(|(k, _)| k.starts_with(prefix)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect()) + } + + fn scan_all(&self) -> Result, Error> { + let store = self.inner.lock().unwrap(); + Ok(store.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + } + + fn delete_all(&self) -> Result<(), Error> { + self.inner.lock().unwrap().clear(); + Ok(()) + } + } +} From f1ccde01bad3111f09e6e27e04e635a1241ab2c2 Mon Sep 17 00:00:00 2001 From: Nelson Dominguez Date: Fri, 27 Feb 2026 15:01:03 +0100 Subject: [PATCH 2/2] Add LevelDB-backed implementation of `MetadataStore` --- Cargo.lock | 186 ++++++++++++++++++++++++++++- Cargo.toml | 3 + minikv-core/Cargo.toml | 1 + minikv-core/src/error.rs | 16 ++- minikv-core/src/storage/leveldb.rs | 119 ++++++++++++++++++ minikv-core/src/storage/mod.rs | 2 + 6 files changed, 321 insertions(+), 6 deletions(-) create mode 100644 minikv-core/src/storage/leveldb.rs diff --git a/Cargo.lock b/Cargo.lock index 0822f21..0ad3fd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,12 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "build_const" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" + [[package]] name = "bytes" version = "1.11.1" @@ -77,6 +83,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -97,6 +112,17 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + [[package]] name = "errno" version = "0.3.14" @@ -107,18 +133,55 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "libc" version = "0.2.182" @@ -145,6 +208,7 @@ dependencies = [ "base64", "blake3", "dashmap", + "rusty-leveldb", "thiserror", "tokio", "tracing", @@ -157,7 +221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -196,6 +260,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -214,6 +287,47 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -223,6 +337,20 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rusty-leveldb" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf78fa8e2741733b7b99d47864e893b7e52c5af39049c7a67305075f3ce6b6c4" +dependencies = [ + "crc", + "errno 0.2.8", + "fs2", + "integer-encoding", + "rand", + "snap", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -241,7 +369,7 @@ version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ - "errno", + "errno 0.3.14", "libc", ] @@ -251,6 +379,12 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.6.2" @@ -357,12 +491,40 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -451,3 +613,23 @@ name = "windows_x86_64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "zerocopy" +version = "0.8.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 7758821..be592ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,6 @@ base64 = "0.22" blake3 = "1.8" dashmap = "6.1" tokio = { version = "1.49", default-features = false } + +# Chosen over `leveldb` crate which requires C++ LevelDB via FFI. +rusty-leveldb = "1" diff --git a/minikv-core/Cargo.toml b/minikv-core/Cargo.toml index 6731581..7102b68 100644 --- a/minikv-core/Cargo.toml +++ b/minikv-core/Cargo.toml @@ -16,6 +16,7 @@ tracing.workspace = true base64 = { workspace = true } blake3 = { workspace = true } dashmap = { workspace = true } +rusty-leveldb = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["full"] } diff --git a/minikv-core/src/error.rs b/minikv-core/src/error.rs index f4bc3ae..e0453c2 100644 --- a/minikv-core/src/error.rs +++ b/minikv-core/src/error.rs @@ -1,5 +1,13 @@ -use thiserror::Error; - /// Unified error type for minikv. -#[derive(Debug, Error)] -pub enum Error {} +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Errors from the rusty-leveldb storage engine. + #[error("LevelDB error: {0}")] + LevelDb(String), +} + +impl From for Error { + fn from(s: rusty_leveldb::Status) -> Self { + Error::LevelDb(s.to_string()) + } +} diff --git a/minikv-core/src/storage/leveldb.rs b/minikv-core/src/storage/leveldb.rs new file mode 100644 index 0000000..19a747c --- /dev/null +++ b/minikv-core/src/storage/leveldb.rs @@ -0,0 +1,119 @@ +//! LevelDB-backed implementation of `MetadataStore`. +//! +//! This uses `rusty-leveldb`, a pure Rust LevelDB implementation with no C++ +//! FFI dependency. It was chosen because it exposes no `unsafe`, is fully +//! auditable, requires no C++ compiler, and provides sufficient performance +//! for metadata workloads. +//! +//! # Thread safety +//! +//! `rusty_leveldb::DB` is `Send` but not `Sync`. We wrap it in a `Mutex` to +//! allow sharing across threads. LevelDB operations are fast, so mutex +//! contention is minimal for typical metadata access. For bulk scans, which +//! hold the lock longer, callers may use `spawn_blocking` to avoid blocking +//! async tasks. + +use std::path::Path; +use std::sync::Mutex; + +use rusty_leveldb::{DB, LdbIterator, Options}; +use tracing::debug; + +use crate::error::Error; +use crate::storage::{KeyValuePair, MetadataStore}; + +/// LevelDB-backed metadata store. +pub struct LevelDbStore { + db: Mutex, +} + +impl LevelDbStore { + /// Open or create a LevelDB database at the specified `path`. + pub fn open(path: &Path) -> Result { + let opts = Options::default(); + let db = DB::open(path, opts).map_err(Error::from)?; + debug!(path = ?path, "LevelDB opened"); + Ok(LevelDbStore { db: Mutex::new(db) }) + } +} + +impl MetadataStore for LevelDbStore { + fn get(&self, key: &[u8]) -> Result>, Error> { + let mut db = self.db.lock().expect("LevelDB mutex poisoned"); + Ok(db.get(key)) + } + + fn put(&self, key: &[u8], value: &[u8]) -> Result<(), Error> { + let mut db = self.db.lock().expect("LevelDB mutex poisoned"); + db.put(key, value).map_err(Error::from) + } + + fn delete(&self, key: &[u8]) -> Result<(), Error> { + let mut db = self.db.lock().expect("LevelDB mutex poisoned"); + db.delete(key).map_err(Error::from) + } + + fn scan_prefix(&self, prefix: &[u8]) -> Result, Error> { + let mut db = self.db.lock().expect("LevelDB mutex poisoned"); + let mut iter = db.new_iter().map_err(Error::from)?; + iter.seek(prefix); + + let mut results = Vec::new(); + let mut key_buf = Vec::new(); + let mut val_buf = Vec::new(); + + while iter.current(&mut key_buf, &mut val_buf) { + if !key_buf.starts_with(prefix) { + break; + } + results.push((key_buf.clone(), val_buf.clone())); + if !iter.advance() { + break; + } + } + + Ok(results) + } + + fn scan_all(&self) -> Result, Error> { + let mut db = self.db.lock().expect("LevelDB mutex poisoned"); + let mut iter = db.new_iter().map_err(Error::from)?; + iter.seek(b""); + + let mut results = Vec::new(); + let mut key_buf = Vec::new(); + let mut val_buf = Vec::new(); + + while iter.current(&mut key_buf, &mut val_buf) { + results.push((key_buf.clone(), val_buf.clone())); + if !iter.advance() { + break; + } + } + + Ok(results) + } + + fn delete_all(&self) -> Result<(), Error> { + let mut db = self.db.lock().expect("LevelDB mutex poisoned"); + let mut iter = db.new_iter().map_err(Error::from)?; + iter.seek(b""); + + let mut keys = Vec::new(); + let mut key_buf = Vec::new(); + let mut val_buf = Vec::new(); + + while iter.current(&mut key_buf, &mut val_buf) { + keys.push(key_buf.clone()); + if !iter.advance() { + break; + } + } + drop(iter); // release borrow before mutating + + for k in keys { + db.delete(&k).map_err(Error::from)?; + } + Ok(()) + } +} diff --git a/minikv-core/src/storage/mod.rs b/minikv-core/src/storage/mod.rs index f3268c1..c324b00 100644 --- a/minikv-core/src/storage/mod.rs +++ b/minikv-core/src/storage/mod.rs @@ -4,6 +4,8 @@ //! `MetadataStore` trait. This isolates storage concerns from business logic //! and allows swapping backends in tests or future implementations. +pub mod leveldb; + use crate::error::Error; /// A single key-value pair in a metadata store.