Add NamespaceHasher (#22)

Author: jzombie

Cargo.lock

@@ -1,7 +1,7 @@
 [package]
 name = "simd-r-drive-extensions"
 authors = ["Jeremy Harris <[email protected]>"]
-version = "0.4.0-alpha.5"
+version = "0.4.0-alpha.6"
 edition = "2021"
 repository = "https://github.com/jzombie/rust-simd-r-drive"
 description = "Storage extensions for SIMD R Drive."

simd-r-drive-extensions/Cargo.toml

@@ -11,3 +11,6 @@ mod storage_cache_ext;
 pub use storage_cache_ext::*;
 
 mod constants;
+
+pub mod namespace_hasher;
+pub use namespace_hasher::NamespaceHasher;

simd-r-drive-extensions/src/lib.rs

@@ -0,0 +1,66 @@
+use simd_r_drive::storage_engine::digest::compute_hash;
+
+/// A utility struct for namespacing keys using XXH3 hashing.
+///
+/// This ensures that keys are uniquely identified within a given namespace,
+/// even if they share the same suffix. By hashing both the namespace and key
+/// separately before combining them, it prevents unintended collisions.
+///
+/// # Example:
+/// ```
+/// use simd_r_drive_extensions::NamespaceHasher;
+///
+/// let hasher = NamespaceHasher::new(b"opt");
+/// let namespaced_key = hasher.namespace(b"my_key");
+/// assert_eq!(namespaced_key.len(), 16, "Namespaced key should be exactly 16 bytes");
+/// ```
+pub struct NamespaceHasher {
+    prefix: u64,
+}
+
+impl NamespaceHasher {
+    /// Creates a new `NamespaceHasher` with a given prefix.
+    ///
+    /// The prefix itself is hashed using XXH3 to ensure a unique namespace identifier.
+    /// This avoids collisions between different namespaces while keeping the hashing fast.
+    ///
+    /// # Arguments
+    /// - `prefix`: A byte slice representing the namespace prefix.
+    ///
+    /// # Returns
+    /// - A `NamespaceHasher` instance with a precomputed prefix hash.
+    #[inline]
+    pub fn new(prefix: &[u8]) -> Self {
+        Self {
+            prefix: compute_hash(prefix),
+        }
+    }
+
+    /// Computes a namespaced key, returning it as a **16-byte vector**.
+    ///
+    /// The final namespaced key is derived by:
+    /// 1. Hashing the key separately to ensure uniqueness.
+    /// 2. Combining it with the precomputed namespace hash.
+    /// 3. Returning the **concatenation of both hashes** as a **16-byte key**.
+    ///
+    /// This ensures that:
+    /// - **Different namespaces** do not generate overlapping keys.
+    /// - **Keys within a namespace** remain **uniquely identifiable**.
+    ///
+    /// # Arguments
+    /// - `key`: A byte slice representing the key to be namespaced.
+    ///
+    /// # Returns
+    /// - A `Vec<u8>` containing the **16-byte** namespaced key (`8-byte prefix hash + 8-byte key hash`).
+    #[inline]
+    pub fn namespace(&self, key: &[u8]) -> Vec<u8> {
+        let key_hash = compute_hash(key);
+
+        // Combine both hashes into a 16-byte buffer
+        let mut buffer = Vec::with_capacity(16);
+        buffer.extend_from_slice(&self.prefix.to_le_bytes()); // Prefix hash (8 bytes)
+        buffer.extend_from_slice(&key_hash.to_le_bytes()); // Key hash (8 bytes)
+
+        buffer
+    }
+}

simd-r-drive-extensions/src/namespace_hasher.rs

@@ -1,11 +1,14 @@
 use crate::constants::TTL_PREFIX;
-use crate::utils::prefix_key;
+use crate::NamespaceHasher;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use simd_r_drive::DataStore;
 use std::io::{self, ErrorKind};
+use std::sync::{Arc, OnceLock};
 use std::time::{SystemTime, UNIX_EPOCH};
 
+static TTL_NAMESPACE_HASHER: OnceLock<Arc<NamespaceHasher>> = OnceLock::new();
+
 #[cfg(any(test, debug_assertions))]
 pub const TEST_TTL_PREFIX: &[u8] = TTL_PREFIX;
 
@@ -54,7 +57,9 @@ impl StorageCacheExt for DataStore {
         value: &T,
         ttl_secs: u64,
     ) -> io::Result<u64> {
-        let key = &prefix_key(TTL_PREFIX, key);
+        let namespace_hasher =
+            TTL_NAMESPACE_HASHER.get_or_init(|| Arc::new(NamespaceHasher::new(TTL_PREFIX)));
+        let namespaced_key = namespace_hasher.namespace(key);
 
         let expiration_timestamp = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -67,13 +72,15 @@ impl StorageCacheExt for DataStore {
             .map_err(|_| io::Error::new(ErrorKind::InvalidData, "Serialization failed"))?;
         data.extend_from_slice(&serialized_value);
 
-        self.write(key, &data)
+        self.write(&namespaced_key, &data)
     }
 
     fn read_with_ttl<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error> {
-        let key = &prefix_key(TTL_PREFIX, key);
+        let namespace_hasher =
+            TTL_NAMESPACE_HASHER.get_or_init(|| Arc::new(NamespaceHasher::new(TTL_PREFIX)));
+        let namespaced_key = namespace_hasher.namespace(key);
 
-        match self.read(key) {
+        match self.read(&namespaced_key) {
             Some(entry) => {
                 let data = entry.as_slice();
 
@@ -91,7 +98,7 @@ impl StorageCacheExt for DataStore {
                     .as_secs();
 
                 if now >= expiration_timestamp {
-                    self.delete_entry(key.as_slice()).ok(); // Remove expired entry
+                    self.delete_entry(&namespaced_key).ok(); // Remove expired entry
                     return Ok(None);
                 }
 

simd-r-drive-extensions/src/storage_cache_ext.rs

@@ -1,10 +1,13 @@
 use crate::constants::{OPTION_PREFIX, OPTION_TOMBSTONE_MARKER};
-use crate::utils::prefix_key;
+use crate::NamespaceHasher;
 use crate::{deserialize_option, serialize_option};
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use simd_r_drive::DataStore;
 use std::io::{self, ErrorKind};
+use std::sync::{Arc, OnceLock};
+
+static OPTION_NAMESPACE_HASHER: OnceLock<Arc<NamespaceHasher>> = OnceLock::new();
 
 #[cfg(any(test, debug_assertions))]
 pub const TEST_OPTION_TOMBSTONE_MARKER: [u8; 2] = OPTION_TOMBSTONE_MARKER;
@@ -124,16 +127,20 @@ pub trait StorageOptionExt {
 /// Implements `StorageOptionExt` for `DataStore`
 impl StorageOptionExt for DataStore {
     fn write_option<T: Serialize>(&self, key: &[u8], value: Option<&T>) -> io::Result<u64> {
-        let key = &prefix_key(OPTION_PREFIX, key);
+        let namespace_hasher =
+            OPTION_NAMESPACE_HASHER.get_or_init(|| Arc::new(NamespaceHasher::new(OPTION_PREFIX)));
+        let namespaced_key = namespace_hasher.namespace(key);
 
         let serialized = serialize_option(value)?;
-        self.write(key, &serialized)
+        self.write(&namespaced_key, &serialized)
     }
 
     fn read_option<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error> {
-        let key = &prefix_key(OPTION_PREFIX, key);
+        let namespace_hasher =
+            OPTION_NAMESPACE_HASHER.get_or_init(|| Arc::new(NamespaceHasher::new(OPTION_PREFIX)));
+        let namespaced_key = namespace_hasher.namespace(key);
 
-        match self.read(key) {
+        match self.read(&namespaced_key) {
             Some(entry) => deserialize_option::<T>(entry.as_slice()),
             None => Err(io::Error::new(
                 ErrorKind::NotFound,

simd-r-drive-extensions/src/storage_option_ext.rs

@@ -1,5 +1,2 @@
 pub mod option_serializer;
 pub use option_serializer::{deserialize_option, serialize_option};
-
-pub mod prefix_key;
-pub use prefix_key::prefix_key;

simd-r-drive-extensions/src/utils.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 use simd_r_drive::DataStore;
-use simd_r_drive_extensions::{utils::prefix_key, StorageCacheExt, TEST_TTL_PREFIX};
+use simd_r_drive_extensions::{NamespaceHasher, StorageCacheExt, TEST_TTL_PREFIX};
 use std::io::ErrorKind;
 use std::thread::sleep;
 use std::time::Duration;
@@ -258,8 +258,10 @@ fn test_write_and_read_option_with_ttl() {
 fn test_ttl_prefix_is_applied() {
     let (_dir, storage) = create_temp_storage();
 
+    let namespace_hasher = NamespaceHasher::new(TEST_TTL_PREFIX);
+
     let key = b"test_key";
-    let prefixed_key = prefix_key(TEST_TTL_PREFIX, key);
+    let namespaced_key = namespace_hasher.namespace(key);
     let test_value = TestData {
         id: 123,
         name: "Test Value".to_string(),
@@ -271,7 +273,7 @@ fn test_ttl_prefix_is_applied() {
         .expect("Failed to write with TTL");
 
     // Ensure the prefixed key exists in storage
-    let raw_data = storage.read(&prefixed_key);
+    let raw_data = storage.read(&namespaced_key);
     assert!(
         raw_data.is_some(),
         "Expected data to be stored under the prefixed key"
@@ -306,14 +308,16 @@ fn test_ttl_prefixing_does_not_affect_regular_storage() {
         name: "Non-TTL Value".to_string(),
     };
 
+    let namespace_hasher = NamespaceHasher::new(TEST_TTL_PREFIX);
+
     // Directly write without TTL
     storage
         .write(key, &bincode::serialize(&test_value).unwrap())
         .expect("Failed to write without TTL");
 
     // Ensure reading from TTL-prefixed key fails (since it was not stored with TTL)
-    let prefixed_key = prefix_key(TEST_TTL_PREFIX, key);
-    let raw_data_prefixed = storage.read(&prefixed_key);
+    let namespaced_key = namespace_hasher.namespace(key);
+    let raw_data_prefixed = storage.read(&namespaced_key);
     assert!(
         raw_data_prefixed.is_none(),
         "No TTL-prefixed entry should exist for a non-TTL write"

simd-r-drive-extensions/src/utils/prefix_key.rs

@@ -3,7 +3,7 @@ mod tests {
     use serde::{Deserialize, Serialize};
     use simd_r_drive::DataStore;
     use simd_r_drive_extensions::{
-        utils::prefix_key, StorageOptionExt, TEST_OPTION_PREFIX, TEST_OPTION_TOMBSTONE_MARKER,
+        NamespaceHasher, StorageOptionExt, TEST_OPTION_PREFIX, TEST_OPTION_TOMBSTONE_MARKER,
     };
     use std::io::ErrorKind;
     use tempfile::tempdir;
@@ -106,8 +106,11 @@ mod tests {
             "Entry should return None when tombstone is written"
         );
 
+        let namespace_hasher = NamespaceHasher::new(TEST_OPTION_PREFIX);
+        let namespaced_key = namespace_hasher.namespace(key);
+
         // Step 4: Ensure the entry still exists in storage (not fully deleted)
-        let raw_entry = storage.read(&prefix_key(&TEST_OPTION_PREFIX, key));
+        let raw_entry = storage.read(&namespaced_key);
         assert!(
             raw_entry.is_some(),
             "Entry should still exist in storage even after writing None"
@@ -219,8 +222,10 @@ mod tests {
     fn test_option_prefix_is_applied_for_some() {
         let (_dir, storage) = create_temp_storage();
 
+        let namespace_hasher = NamespaceHasher::new(TEST_OPTION_PREFIX);
+
         let key = b"test_key_option";
-        let prefixed_key = prefix_key(TEST_OPTION_PREFIX, key);
+        let namespaced_key = namespace_hasher.namespace(key);
         let test_value = Some(TestData {
             id: 456,
             name: "Test Option Value".to_string(),
@@ -232,7 +237,7 @@ mod tests {
             .expect("Failed to write option");
 
         // Ensure the prefixed key exists in storage
-        let raw_data = storage.read(&prefixed_key);
+        let raw_data = storage.read(&namespaced_key);
         assert!(
             raw_data.is_some(),
             "Expected data to be stored under the prefixed key"
@@ -260,16 +265,18 @@ mod tests {
     fn test_option_prefix_is_applied_for_none() {
         let (_dir, storage) = create_temp_storage();
 
+        let namespace_hasher = NamespaceHasher::new(TEST_OPTION_PREFIX);
+
         let key = b"test_key_none";
-        let prefixed_key = prefix_key(TEST_OPTION_PREFIX, key);
+        let namespaced_key = namespace_hasher.namespace(key);
 
         // Write `None`
         storage
             .write_option::<TestData>(key, None)
             .expect("Failed to write None with option handling");
 
         // Ensure the prefixed key exists in storage (tombstone marker stored)
-        let raw_data = storage.read(&prefixed_key);
+        let raw_data = storage.read(&namespaced_key);
         assert!(
             raw_data.is_some(),
             "Expected tombstone marker to be stored under the prefixed key"
@@ -308,9 +315,11 @@ mod tests {
             .write(key, &bincode::serialize(&test_value).unwrap())
             .expect("Failed to write non-option value");
 
+        let namespace_hasher = NamespaceHasher::new(TEST_OPTION_PREFIX);
+
         // Ensure reading from the option-prefixed key fails (since it was not stored as an option)
-        let prefixed_key = prefix_key(TEST_OPTION_PREFIX, key);
-        let raw_data_prefixed = storage.read(&prefixed_key);
+        let namespaced_key = namespace_hasher.namespace(key);
+        let raw_data_prefixed = storage.read(&namespaced_key);
         assert!(
             raw_data_prefixed.is_none(),
             "No option-prefixed entry should exist for a non-prefixed write"