TTL cache extension (#20)

Author: jzombie

README.md

@@ -220,7 +220,7 @@ By using SIMD for these performance-critical tasks, `SIMD R Drive` minimizes CPU
 
 ## Extensions
 
-[SIMD R Drive Extensions](./simd-r-drive-extensions/) provide additional functionality and introduce extra dependencies.
+[SIMD R Drive Extensions](./simd-r-drive-extensions/) provide additional functionality.
 
 ## License
 

simd-r-drive-extensions/README.md

@@ -2,7 +2,7 @@
 
 **Work in progress.**
 
-`simd-r-drive-extensions` provides optional utilities for working with `Option<T>` in [SIMD R Drive](https://crates.io/crates/simd-r-drive).
+`simd-r-drive-extensions` provides optional utilities for working with `Option<T>` and TTL-based caching in [SIMD R Drive](https://crates.io/crates/simd-r-drive).
 
 [Documentation](https://docs.rs/simd-r-drive-extensions/latest/simd_r_drive_extensions/)
 
@@ -14,6 +14,7 @@ cargo add simd-r-drive-extensions
 
 ## Usage
 
+### Working with `Option<T>`
 ```rust
 use simd_r_drive::DataStore;
 use simd_r_drive_extensions::StorageOptionExt;
@@ -35,28 +36,42 @@ assert_eq!(
     None
 );
 
-// Check if the key exists in storage, regardless of whether it's `Some` or `None`
-if let Ok(none_option) = storage.read_option::<i32>(b"key_with_none_value") {
-    assert!(none_option.is_none());
-} else {
-    // Just to check the example
-    panic!("Failed to read key: `key_with_none_value` does not exist or read error occurred.");
-}
-
-// Alternative, concise check
-let none_option = storage.read_option::<i32>(b"key_with_none_value").unwrap();
-assert!(none_option.is_none()); // Ensures `Option<T>` exists
-
 // Errors on non-existent keys
 assert!(storage.read_option::<i32>(b"non_existent_key").is_err());
+```
+
+### Working with TTL-based Caching
+```rust
+use simd_r_drive::DataStore;
+use simd_r_drive_extensions::StorageCacheExt;
+use std::path::PathBuf;
+use std::thread::sleep;
+use std::time::Duration;
 
+let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
+
+// Write value with a TTL of 5 seconds
+storage.write_with_ttl(b"key_with_ttl", &42, 5).unwrap();
+assert_eq!(
+    storage.read_with_ttl::<i32>(b"key_with_ttl").expect("Failed to read key"),
+    Some(42)
+);
+
+// Wait for TTL to expire
+sleep(Duration::from_secs(6));
+assert_eq!(
+    storage.read_with_ttl::<i32>(b"key_with_ttl").expect("Failed to read key"),
+    None // Key should be expired and removed
+);
 ```
 
 ## Implementation Details
 
 - Uses a predefined tombstone marker (`[0xFF, 0xFE]`) to represent `None`.
+- TTL values are stored as a **binary prefix** before the actual value.
 - Values are serialized using [bincode](https://crates.io/crates/bincode).
 - ⚠️ Unlike [SIMD R Drive](https://crates.io/crates/simd-r-drive), values are non-zero-copy, as they require deserialization.
+- TTL-based storage will **automatically evict expired values upon read** to prevent stale data.
 
 ## License
 

simd-r-drive-extensions/src/lib.rs

@@ -1,155 +1,8 @@
 #[cfg(doctest)]
 doc_comment::doctest!("../README.md");
 
-use bincode;
-use serde::de::DeserializeOwned;
-use serde::Serialize;
-use simd_r_drive::DataStore;
-use std::io::{self, ErrorKind};
+mod storage_option_ext;
+pub use storage_option_ext::*;
 
-/// Special marker for explicitly storing `None` values in binary storage.
-/// This ensures that `None` is distinguishable from an empty or default value.
-const OPTION_TOMBSTONE_MARKER: [u8; 2] = [0xFF, 0xFE];
-
-#[cfg(any(test, debug_assertions))]
-pub const TEST_OPTION_TOMBSTONE_MARKER: [u8; 2] = OPTION_TOMBSTONE_MARKER;
-
-/// # Storage Utilities for Handling `Option<T>`
-///
-/// This trait provides methods to store and retrieve `Option<T>` values
-/// in a `DataStore`, ensuring that `None` values are explicitly handled.
-///
-/// ## Purpose
-/// - **Prevents ambiguity**: Ensures `None` is stored and retrieved correctly.
-/// - **Efficient storage**: Uses a compact representation.
-/// - **Binary-safe**: Avoids unintended interpretation of missing values.
-///
-/// ## Implementation Details
-/// - **`Some(value)`**: Serialized using `bincode`.
-/// - **`None`**: Explicitly stored using a dedicated tombstone marker (`[0xFF, 0xFE]`).
-///
-/// ## Example Usage
-///
-/// ```rust
-/// use simd_r_drive::DataStore;
-/// use simd_r_drive_extensions::StorageOptionExt;
-/// use std::path::PathBuf;
-///
-/// let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
-///
-/// // Store `Some(value)`
-/// storage.write_option(b"key1", Some(&42)).unwrap();
-///
-/// // Store `None` (tombstone)
-/// storage.write_option::<i32>(b"key2", None).unwrap();
-///
-/// // Read values
-/// assert_eq!(storage.read_option::<i32>(b"key1").unwrap(), Some(42));
-/// assert_eq!(storage.read_option::<i32>(b"key2").unwrap(), None);
-/// ```
-pub trait StorageOptionExt {
-    fn write_option<T: Serialize>(&self, key: &[u8], value: Option<&T>) -> std::io::Result<u64>;
-    fn read_option<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, std::io::Error>;
-}
-
-/// Implements `StorageOptionExt` for `DataStore`
-impl StorageOptionExt for DataStore {
-    /// Writes an `Option<T>` into the `DataStore`, ensuring `None` values are preserved.
-    ///
-    /// - `Some(value)`: Serialized using `bincode`.
-    /// - `None`: Stored in a way that allows correct retrieval.
-    ///
-    /// ## Arguments
-    /// - `key`: The binary key under which the value is stored.
-    /// - `value`: An optional reference to `T`, where `None` is handled appropriately.
-    ///
-    /// ## Returns
-    /// - `Ok(offset)`: The **file offset** where the data was written.
-    /// - `Err(std::io::Error)`: If the write operation fails.
-    ///
-    /// ## Example
-    /// ```rust
-    /// use simd_r_drive::DataStore;
-    /// use simd_r_drive_extensions::StorageOptionExt;
-    /// use std::path::PathBuf;
-    ///
-    /// let storage = DataStore::open(&PathBuf::from("store.bin")).unwrap();
-    ///
-    /// // Write `Some(value)`
-    /// storage.write_option(b"key_with_some_value", Some(&123)).unwrap();
-    ///
-    /// // Write `None` (tombstone)
-    /// storage.write_option::<i32>(b"key_with_none_value", None).unwrap();
-    /// ```
-    fn write_option<T: Serialize>(&self, key: &[u8], value: Option<&T>) -> std::io::Result<u64> {
-        let serialized = match value {
-            Some(v) => bincode::serialize(v).unwrap_or_else(|_| OPTION_TOMBSTONE_MARKER.to_vec()),
-            None => OPTION_TOMBSTONE_MARKER.to_vec(),
-        };
-
-        self.write(key, &serialized)
-    }
-
-    /// Reads an `Option<T>` from storage.
-    ///
-    /// - **⚠️ Non Zero-Copy Warning**: Requires deserialization.
-    /// - **Returns `Ok(None)`** if the key exists and explicitly stores the tombstone marker (`[0xFF, 0xFE]`).
-    /// - **Returns `Err(ErrorKind::NotFound)`** if the key does not exist.
-    /// - **Returns `Err(ErrorKind::InvalidData)`** if deserialization fails.
-    ///
-    /// ## Arguments
-    /// - `key`: The binary key to retrieve.
-    ///
-    /// ## Returns
-    /// - `Ok(Some(T))`: If deserialization succeeds and is `Some`.
-    /// - `Ok(None)`: If the key represents `None`.
-    /// - `Err(std::io::Error)`: If the key does not exist or if deserialization fails.
-    ///
-    /// ## Example
-    /// ```rust
-    /// use simd_r_drive::DataStore;
-    /// use simd_r_drive_extensions::StorageOptionExt;
-    /// use std::path::PathBuf;
-    ///
-    /// let storage = DataStore::open(&PathBuf::from("store.bin")).unwrap();
-    ///
-    /// storage.write_option(b"key_with_some_value", Some(&789)).unwrap();
-    /// storage.write_option::<i32>(b"key_with_none_value", None).unwrap();
-    ///
-    /// assert_eq!(storage.read_option::<i32>(b"key_with_some_value").unwrap(), Some(789));
-    /// assert_eq!(storage.read_option::<i32>(b"key_with_none_value").unwrap(), None);
-    ///
-    /// if let Ok(none_option) = storage.read_option::<i32>(b"key_with_none_value") {
-    ///     assert!(none_option.is_some() || none_option.is_none()); // Explicitly checking Option type
-    /// }
-    ///
-    /// // Alternative, concise check
-    /// let none_option = storage.read_option::<i32>(b"key_with_none_value").unwrap();
-    /// assert!(none_option.is_none() || none_option.is_some()); // Ensures `Option<T>` exists
-    ///
-    /// // Errors on non-existent keys
-    /// assert!(storage.read_option::<i32>(b"non_existent_key").is_err());
-    /// ```
-    ///
-    /// # Safety
-    /// - This function **allocates memory** for deserialization.
-    fn read_option<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error> {
-        match self.read(key) {
-            Some(entry) => {
-                let data = entry.as_slice();
-                if data == OPTION_TOMBSTONE_MARKER {
-                    return Ok(None);
-                }
-                bincode::deserialize::<T>(data)
-                    .map(Some)
-                    .map_err(|e| io::Error::new(ErrorKind::InvalidData, e))
-            }
-            None => {
-                return Err(io::Error::new(
-                    ErrorKind::NotFound,
-                    "Key not found in storage",
-                ))
-            }
-        }
-    }
-}
+mod storage_cache_ext;
+pub use storage_cache_ext::*;

simd-r-drive-extensions/src/storage_cache_ext.rs

@@ -0,0 +1,91 @@
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+use simd_r_drive::DataStore;
+use std::io::{self, ErrorKind};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+/// **Prefix-based TTL storage for cache expiration**  
+/// Stores a timestamp (in seconds) before the actual value.
+pub trait StorageCacheExt {
+    /// Writes a value with a TTL (Time-To-Live).
+    ///
+    /// - Stores the expiration timestamp as a **binary prefix** before the actual data.
+    /// - If the key exists, it will be **overwritten**.
+    ///
+    /// ## Arguments
+    /// - `key`: The binary key to store.
+    /// - `value`: The value to be stored.
+    /// - `ttl_secs`: The TTL in **seconds** (relative to current time).
+    ///
+    /// ## Returns
+    /// - `Ok(offset)`: The **file offset** where the data was written.
+    /// - `Err(std::io::Error)`: If the write operation fails.
+    fn write_with_ttl<T: Serialize>(&self, key: &[u8], value: &T, ttl_secs: u64)
+        -> io::Result<u64>;
+
+    /// Reads a value, checking TTL expiration.
+    ///
+    /// - If the TTL has expired, the key is **automatically evicted**, and `None` is returned.
+    /// - If the key does not exist, returns `Err(ErrorKind::NotFound)`.
+    /// - If deserialization fails, returns `Err(ErrorKind::InvalidData)`.
+    ///
+    /// ## Returns
+    /// - `Ok(Some(T))`: If the TTL is still valid and the value is readable.
+    /// - `Ok(None)`: If the TTL has expired and the entry has been evicted.
+    /// - `Err(std::io::Error)`: If the key is missing or deserialization fails.
+    fn read_with_ttl<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error>;
+}
+
+/// Implements TTL-based caching for `DataStore`
+impl StorageCacheExt for DataStore {
+    fn write_with_ttl<T: Serialize>(
+        &self,
+        key: &[u8],
+        value: &T,
+        ttl_secs: u64,
+    ) -> io::Result<u64> {
+        let expiration_timestamp = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs()
+            .saturating_add(ttl_secs); // Avoid overflow
+
+        let mut data = expiration_timestamp.to_le_bytes().to_vec();
+        let serialized_value = bincode::serialize(value)
+            .map_err(|_| io::Error::new(ErrorKind::InvalidData, "Serialization failed"))?;
+        data.extend_from_slice(&serialized_value);
+
+        self.write(key, &data)
+    }
+
+    fn read_with_ttl<T: DeserializeOwned>(&self, key: &[u8]) -> Result<Option<T>, io::Error> {
+        match self.read(key) {
+            Some(entry) => {
+                let data = entry.as_slice();
+
+                if data.len() < 8 {
+                    return Err(io::Error::new(
+                        ErrorKind::InvalidData,
+                        "Data too short to contain TTL",
+                    ));
+                }
+
+                let expiration_timestamp = u64::from_le_bytes(data[..8].try_into().unwrap());
+                let now = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .expect("Time went backwards")
+                    .as_secs();
+
+                if now >= expiration_timestamp {
+                    self.delete_entry(key).ok(); // Remove expired entry
+                    return Ok(None);
+                }
+
+                bincode::deserialize::<T>(&data[8..])
+                    .map(Some)
+                    .map_err(|_| io::Error::new(ErrorKind::InvalidData, "Deserialization failed"))
+            }
+            None => Err(io::Error::new(ErrorKind::NotFound, "Key not found")),
+        }
+    }
+}