Extension key namespaces (#21)

Author: jzombie

Cargo.lock

@@ -1,7 +1,7 @@
 [package]
 name = "simd-r-drive-extensions"
 authors = ["Jeremy Harris <[email protected]>"]
-version = "0.4.0-alpha.4"
+version = "0.4.0-alpha.5"
 edition = "2021"
 repository = "https://github.com/jzombie/rust-simd-r-drive"
 description = "Storage extensions for SIMD R Drive."

simd-r-drive-extensions/Cargo.toml

@@ -0,0 +1,42 @@
+/// Special marker for explicitly storing `None` values in binary storage.
+pub(crate) const OPTION_TOMBSTONE_MARKER: [u8; 2] = [0xFF, 0xFE];
+
+/// # Namespaced Prefixes for Storage Features
+///
+/// These prefixes are **not** used to differentiate values themselves;  
+/// SIMD R Drive already handles type and structure differentiation.  
+///
+/// Instead, these prefixes are used to distinguish **storage features** such as:
+/// - **Option handling** (explicit tombstones for `None` values)
+/// - **TTL-based auto-eviction** (keys prefixed with expiration timestamps)
+///
+/// By applying **feature-based** prefixes, we ensure that:
+/// - Different feature extensions do not naturally conflict.
+/// - Per-extensions read/write operations apply the correct logic.
+/// - Keys remain distinct even if their raw values are identical.
+///
+/// This ensures relatively safe, efficient, and collision-free feature separation  
+/// without interfering with the actual stored values.
+macro_rules! namespace_prefix {
+    ($name:expr) => {{
+        const PREFIX: &[u8] = &{
+            const LEN: usize = $name.len();
+            let mut arr = [0u8; LEN + 2]; // Boundary + Name + Boundary
+
+            arr[0] = 0xF7; // Start Boundary: Non-standard, forbidden high UTF-8 range
+            arr[LEN + 1] = 0xFD; // End Boundary: Another high, rarely used byte
+
+            let mut i = 0;
+            while i < LEN {
+                arr[i + 1] = $name[i];
+                i += 1;
+            }
+            arr
+        };
+        PREFIX
+    }};
+}
+
+/// Namespaced extension prefixes
+pub(crate) const OPTION_PREFIX: &[u8] = namespace_prefix!(b"option");
+pub(crate) const TTL_PREFIX: &[u8] = namespace_prefix!(b"ttl");

simd-r-drive-extensions/src/constants.rs

@@ -1,8 +1,13 @@
 #[cfg(doctest)]
 doc_comment::doctest!("../README.md");
 
+pub mod utils;
+pub use utils::option_serializer::*;
+
 mod storage_option_ext;
 pub use storage_option_ext::*;
 
 mod storage_cache_ext;
 pub use storage_cache_ext::*;
+
+mod constants;

simd-r-drive-extensions/src/lib.rs

@@ -1,11 +1,20 @@
+use crate::constants::TTL_PREFIX;
+use crate::utils::prefix_key;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use simd_r_drive::DataStore;
 use std::io::{self, ErrorKind};
 use std::time::{SystemTime, UNIX_EPOCH};
 
-/// **Prefix-based TTL storage for cache expiration**  
+#[cfg(any(test, debug_assertions))]
+pub const TEST_TTL_PREFIX: &[u8] = TTL_PREFIX;
+
+/// # Storage Utilities for Handling Auto-Evicting TTL Entries
+///
 /// Stores a timestamp (in seconds) before the actual value.
+///
+/// Note: Option types are *safely* handled by this without additional serialization
+/// as they are stored with the TTL value as well.
 pub trait StorageCacheExt {
     /// Writes a value with a TTL (Time-To-Live).
     ///
@@ -25,6 +34,7 @@ pub trait StorageCacheExt {
 
     /// Reads a value, checking TTL expiration.
     ///
+    /// - **⚠️ Non Zero-Copy Warning**: Requires deserialization.
     /// - If the TTL has expired, the key is **automatically evicted**, and `None` is returned.
     /// - If the key does not exist, returns `Err(ErrorKind::NotFound)`.
     /// - If deserialization fails, returns `Err(ErrorKind::InvalidData)`.
@@ -44,6 +54,8 @@ impl StorageCacheExt for DataStore {
         value: &T,
         ttl_secs: u64,
     ) -> io::Result<u64> {
+        let key = &prefix_key(TTL_PREFIX, key);
+
         let expiration_timestamp = SystemTime::now()
             .duration_since(UNIX_EPOCH)
             .expect("Time went backwards")
@@ -59,6 +71,8 @@ impl StorageCacheExt for DataStore {
     }
 
     fn read_with_ttl<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error> {
+        let key = &prefix_key(TTL_PREFIX, key);
+
         match self.read(key) {
             Some(entry) => {
                 let data = entry.as_slice();
@@ -77,7 +91,7 @@ impl StorageCacheExt for DataStore {
                     .as_secs();
 
                 if now >= expiration_timestamp {
-                    self.delete_entry(key).ok(); // Remove expired entry
+                    self.delete_entry(key.as_slice()).ok(); // Remove expired entry
                     return Ok(None);
                 }
 

simd-r-drive-extensions/src/storage_cache_ext.rs

@@ -1,16 +1,17 @@
-use bincode;
+use crate::constants::{OPTION_PREFIX, OPTION_TOMBSTONE_MARKER};
+use crate::utils::prefix_key;
+use crate::{deserialize_option, serialize_option};
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 use simd_r_drive::DataStore;
 use std::io::{self, ErrorKind};
 
-/// Special marker for explicitly storing `None` values in binary storage.
-/// This ensures that `None` is distinguishable from an empty or default value.
-const OPTION_TOMBSTONE_MARKER: [u8; 2] = [0xFF, 0xFE];
-
 #[cfg(any(test, debug_assertions))]
 pub const TEST_OPTION_TOMBSTONE_MARKER: [u8; 2] = OPTION_TOMBSTONE_MARKER;
 
+#[cfg(any(test, debug_assertions))]
+pub const TEST_OPTION_PREFIX: &[u8] = OPTION_PREFIX;
+
 /// # Storage Utilities for Handling `Option<T>`
 ///
 /// This trait provides methods to store and retrieve `Option<T>` values
@@ -122,32 +123,22 @@ pub trait StorageOptionExt {
 
 /// Implements `StorageOptionExt` for `DataStore`
 impl StorageOptionExt for DataStore {
-    fn write_option<T: Serialize>(&self, key: &[u8], value: Option<&T>) -> std::io::Result<u64> {
-        let serialized = match value {
-            Some(v) => bincode::serialize(v).unwrap_or_else(|_| OPTION_TOMBSTONE_MARKER.to_vec()),
-            None => OPTION_TOMBSTONE_MARKER.to_vec(),
-        };
+    fn write_option<T: Serialize>(&self, key: &[u8], value: Option<&T>) -> io::Result<u64> {
+        let key = &prefix_key(OPTION_PREFIX, key);
 
+        let serialized = serialize_option(value)?;
         self.write(key, &serialized)
     }
 
     fn read_option<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error> {
+        let key = &prefix_key(OPTION_PREFIX, key);
+
         match self.read(key) {
-            Some(entry) => {
-                let data = entry.as_slice();
-                if data == OPTION_TOMBSTONE_MARKER {
-                    return Ok(None);
-                }
-                bincode::deserialize::<T>(data)
-                    .map(Some)
-                    .map_err(|e| io::Error::new(ErrorKind::InvalidData, e))
-            }
-            None => {
-                return Err(io::Error::new(
-                    ErrorKind::NotFound,
-                    "Key not found in storage",
-                ))
-            }
+            Some(entry) => deserialize_option::<T>(entry.as_slice()),
+            None => Err(io::Error::new(
+                ErrorKind::NotFound,
+                "Key not found in storage",
+            )),
         }
     }
 }

simd-r-drive-extensions/src/storage_option_ext.rs

@@ -0,0 +1,5 @@
+pub mod option_serializer;
+pub use option_serializer::{deserialize_option, serialize_option};
+
+pub mod prefix_key;
+pub use prefix_key::prefix_key;

simd-r-drive-extensions/src/utils.rs

@@ -0,0 +1,66 @@
+use crate::constants::OPTION_TOMBSTONE_MARKER;
+use bincode;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+use std::io::{self, ErrorKind};
+
+/// Serializes an `Option<T>` into a binary representation compatible with `SIMD R Drive`.
+///
+/// - If `Some(value)`, serializes the value using `bincode`.
+/// - If `None`, returns a tombstone marker (`[0xFF, 0xFE]`).
+///
+/// ## Returns
+/// - `Vec<u8>` containing the serialized value or tombstone marker.
+/// - `Err(io::Error)`: If serialization fails.
+///
+/// ## Example
+/// ```
+/// use simd_r_drive_extensions::utils::option_serializer::serialize_option;
+///
+/// let some_value = serialize_option(Some(&42)).unwrap();
+/// let none_value = serialize_option::<i32>(None).unwrap();
+///
+/// assert_ne!(some_value, none_value);
+/// assert_eq!(none_value, vec![0xFF, 0xFE]); // Tombstone marker for `None`
+/// ```
+pub fn serialize_option<T: Serialize>(value: Option<&T>) -> io::Result<Vec<u8>> {
+    match value {
+        Some(v) => bincode::serialize(v)
+            .map_err(|_| io::Error::new(ErrorKind::InvalidData, "Failed to serialize Option<T>")),
+        None => Ok(OPTION_TOMBSTONE_MARKER.to_vec()),
+    }
+}
+
+/// Deserializes an `Option<T>` from binary storage.
+///
+/// - **⚠️ Non Zero-Copy Warning**: Requires deserialization.
+/// - If the data matches the tombstone marker, returns `Ok(None)`.
+/// - Otherwise, attempts to deserialize the stored value.
+///
+/// ## Returns
+/// - `Ok(Some(T))` if the data is valid.
+/// - `Ok(None)` if the tombstone marker is found.
+/// - `Err(io::Error)`: If deserialization fails.
+///
+/// ## Example
+/// ```
+/// use simd_r_drive_extensions::utils::option_serializer::{serialize_option, deserialize_option};
+///
+/// let some_value = serialize_option(Some(&42)).unwrap();
+/// let none_value = serialize_option::<i32>(None).unwrap();
+///
+/// let deserialized_some: Option<i32> = deserialize_option(&some_value).unwrap();
+/// let deserialized_none: Option<i32> = deserialize_option(&none_value).unwrap();
+///
+/// assert_eq!(deserialized_some, Some(42));
+/// assert_eq!(deserialized_none, None);
+/// ```
+pub fn deserialize_option<T: DeserializeOwned>(data: &[u8]) -> Result<Option<T>, io::Error> {
+    if data == OPTION_TOMBSTONE_MARKER {
+        return Ok(None);
+    }
+
+    bincode::deserialize::<T>(data)
+        .map(Some)
+        .map_err(|_| io::Error::new(ErrorKind::InvalidData, "Failed to deserialize Option<T>"))
+}

simd-r-drive-extensions/src/utils/option_serializer.rs

@@ -0,0 +1,27 @@
+/// Utility function to **prefix a binary key** with a given prefix.
+///
+/// - Ensures the prefixed key remains valid for storage.
+/// - Prevents key collisions by ensuring distinct namespaces.
+///
+/// ## Arguments
+/// - `prefix`: The binary prefix to prepend.
+/// - `key`: The original binary key.
+///
+/// ## Returns
+/// - A new `Vec<u8>` containing the prefixed key.
+///
+/// ## Example
+/// ```rust
+/// use simd_r_drive_extensions::utils::prefix_key;
+///
+/// let key = b"my_key";
+/// let prefixed = prefix_key(b"cache_", key);
+///
+/// assert_eq!(prefixed, b"cache_my_key".to_vec());
+/// ```
+pub fn prefix_key(prefix: &[u8], key: &[u8]) -> Vec<u8> {
+    let mut prefixed_key = Vec::with_capacity(prefix.len() + key.len());
+    prefixed_key.extend_from_slice(prefix);
+    prefixed_key.extend_from_slice(key);
+    prefixed_key
+}