From 165544844982a68ca178e4b4fea5026663efce5a Mon Sep 17 00:00:00 2001 From: chrisfang Date: Sat, 22 Aug 2026 18:20:57 +0800 Subject: [PATCH 1/7] feat(ragfs): unify cache runtime providers --- Cargo.lock | 50 +- Cargo.toml | 2 - MANIFEST.in | 1 - crates/ragfs-cache-redis/Cargo.toml | 19 - crates/ragfs-cache-redis/src/config.rs | 133 --- crates/ragfs-cache-redis/src/lib.rs | 8 - crates/ragfs-cache-redis/src/provider.rs | 193 ---- .../tests/cached_filesystem.rs | 226 ----- .../tests/provider_contract.rs | 101 -- crates/ragfs-python-native/Cargo.toml | 42 - crates/ragfs-python/Cargo.toml | 7 +- crates/ragfs-python/src/lib.rs | 882 ++++++++---------- crates/ragfs/Cargo.toml | 19 +- crates/ragfs/src/cache/envelope.rs | 2 +- crates/ragfs/src/cache/memory.rs | 132 --- crates/ragfs/src/cache/mod.rs | 4 - crates/ragfs/src/cache/provider.rs | 103 -- crates/ragfs/src/cache/wrapper.rs | 66 +- crates/ragfs/src/cache_runtime/api.rs | 54 ++ crates/ragfs/src/cache_runtime/dynamic/abi.rs | 93 ++ .../ragfs/src/cache_runtime/dynamic/config.rs | 10 + crates/ragfs/src/cache_runtime/dynamic/mod.rs | 6 + .../src/cache_runtime/dynamic/provider.rs | 368 ++++++++ crates/ragfs/src/cache_runtime/error.rs | 36 + crates/ragfs/src/cache_runtime/executor.rs | 63 ++ crates/ragfs/src/cache_runtime/memory.rs | 212 +++++ crates/ragfs/src/cache_runtime/mod.rs | 230 +++++ crates/ragfs/src/cache_runtime/provider.rs | 46 + .../src/cache_runtime/redis}/client.rs | 122 ++- .../ragfs/src/cache_runtime/redis/config.rs | 123 +++ crates/ragfs/src/cache_runtime/redis/mod.rs | 9 + .../ragfs/src/cache_runtime/redis/provider.rs | 162 ++++ crates/ragfs/src/cache_runtime/script.rs | 72 ++ crates/ragfs/src/core/builder.rs | 221 ++++- crates/ragfs/src/core/mod.rs | 4 + crates/ragfs/src/core/mountable.rs | 133 ++- crates/ragfs/src/lib.rs | 2 + .../src/plugins/queuefs/cache_backend.rs | 645 +++++++++++++ crates/ragfs/src/plugins/queuefs/mod.rs | 314 ++++++- .../src/plugins/queuefs/redis_backend.rs | 50 +- crates/ragfs/tests/cache_runtime.rs | 127 +++ crates/ragfs/tests/cache_wrapper.rs | 340 ++----- crates/ragfs/tests/dynamic_provider.rs | 131 +++ .../fixtures/dynamic_provider/provider.c | 174 ++++ crates/ragfs/tests/redis_runtime.rs | 80 ++ openviking/utils/agfs_utils.py | 8 +- openviking_cli/utils/config/agfs_config.py | 112 +-- tests/misc/test_config_validation.py | 56 +- .../test_ragfs_python_manifest_isolation.py | 37 +- 49 files changed, 3970 insertions(+), 2060 deletions(-) delete mode 100644 crates/ragfs-cache-redis/Cargo.toml delete mode 100644 crates/ragfs-cache-redis/src/config.rs delete mode 100644 crates/ragfs-cache-redis/src/lib.rs delete mode 100644 crates/ragfs-cache-redis/src/provider.rs delete mode 100644 crates/ragfs-cache-redis/tests/cached_filesystem.rs delete mode 100644 crates/ragfs-cache-redis/tests/provider_contract.rs delete mode 100644 crates/ragfs-python-native/Cargo.toml delete mode 100644 crates/ragfs/src/cache/memory.rs delete mode 100644 crates/ragfs/src/cache/provider.rs create mode 100644 crates/ragfs/src/cache_runtime/api.rs create mode 100644 crates/ragfs/src/cache_runtime/dynamic/abi.rs create mode 100644 crates/ragfs/src/cache_runtime/dynamic/config.rs create mode 100644 crates/ragfs/src/cache_runtime/dynamic/mod.rs create mode 100644 crates/ragfs/src/cache_runtime/dynamic/provider.rs create mode 100644 crates/ragfs/src/cache_runtime/error.rs create mode 100644 crates/ragfs/src/cache_runtime/executor.rs create mode 100644 crates/ragfs/src/cache_runtime/memory.rs create mode 100644 crates/ragfs/src/cache_runtime/mod.rs create mode 100644 crates/ragfs/src/cache_runtime/provider.rs rename crates/{ragfs-cache-redis/src => ragfs/src/cache_runtime/redis}/client.rs (70%) create mode 100644 crates/ragfs/src/cache_runtime/redis/config.rs create mode 100644 crates/ragfs/src/cache_runtime/redis/mod.rs create mode 100644 crates/ragfs/src/cache_runtime/redis/provider.rs create mode 100644 crates/ragfs/src/cache_runtime/script.rs create mode 100644 crates/ragfs/src/plugins/queuefs/cache_backend.rs create mode 100644 crates/ragfs/tests/cache_runtime.rs create mode 100644 crates/ragfs/tests/dynamic_provider.rs create mode 100644 crates/ragfs/tests/fixtures/dynamic_provider/provider.c create mode 100644 crates/ragfs/tests/redis_runtime.rs diff --git a/Cargo.lock b/Cargo.lock index 7e134f9421..f01f14253e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3049,6 +3049,16 @@ dependencies = [ "cc", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -3979,13 +3989,14 @@ dependencies = [ "grep-searcher", "hkdf", "ignore", + "libloading", "lru", "mime_guess", "path-clean", "r2d2", "radix_trie", "rand 0.8.5", - "redis 1.5.0", + "redis", "regex", "rusqlite", "serde", @@ -3998,23 +4009,11 @@ dependencies = [ "tokio", "toml", "tracing", + "url", "uuid", "windows-sys 0.61.2", ] -[[package]] -name = "ragfs-cache-redis" -version = "0.1.0" -dependencies = [ - "async-trait", - "bytes", - "ragfs", - "redis 0.32.7", - "thiserror 1.0.69", - "tokio", - "url", -] - [[package]] name = "ragfs-python" version = "0.1.0" @@ -4023,7 +4022,6 @@ dependencies = [ "gix-hash", "pyo3", "ragfs", - "ragfs-cache-redis", "serde", "serde_json", "similar", @@ -4201,28 +4199,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redis" -version = "0.32.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44" -dependencies = [ - "bytes", - "cfg-if", - "combine", - "futures-util", - "itoa", - "num-bigint", - "percent-encoding", - "pin-project-lite", - "ryu", - "sha1_smol", - "socket2 0.6.3", - "tokio", - "tokio-util", - "url", -] - [[package]] name = "redis" version = "1.5.0" diff --git a/Cargo.toml b/Cargo.toml index 2831b663de..b8789f2aea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,14 +2,12 @@ members = [ "crates/ov_cli", "crates/ragfs", - "crates/ragfs-cache-redis", "crates/ragfs-python", ] exclude = [ "crates/ragfs-cache-mooncake", "crates/ragfs-cache-yuanrong", "crates/ragfs-cache-yuanrong-sys", - "crates/ragfs-python-native", ] resolver = "2" diff --git a/MANIFEST.in b/MANIFEST.in index 673316ab04..d93d175a55 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,7 +10,6 @@ include setup.py include Cargo.toml include Cargo.lock graft crates/ragfs -graft crates/ragfs-cache-redis graft crates/ragfs-python recursive-include openviking *.yaml diff --git a/crates/ragfs-cache-redis/Cargo.toml b/crates/ragfs-cache-redis/Cargo.toml deleted file mode 100644 index 408cf0a80d..0000000000 --- a/crates/ragfs-cache-redis/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "ragfs-cache-redis" -version = "0.1.0" -edition = "2021" -rust-version = "1.91.1" -license = "Apache-2.0" -description = "Redis cache provider adapter for RAGFS" - -[dependencies] -async-trait = "0.1" -bytes = "1.5" -ragfs = { path = "../ragfs", features = ["cache"] } -redis = { version = "0.32", features = ["tokio-comp"] } -thiserror = "1.0" -tokio = { version = "1.38", features = ["rt-multi-thread", "sync", "time", "macros"] } -url = "2" - -[dev-dependencies] -ragfs = { path = "../ragfs", features = ["cache"] } diff --git a/crates/ragfs-cache-redis/src/config.rs b/crates/ragfs-cache-redis/src/config.rs deleted file mode 100644 index faf2d7fad5..0000000000 --- a/crates/ragfs-cache-redis/src/config.rs +++ /dev/null @@ -1,133 +0,0 @@ -use ragfs::cache::{CacheError, CacheResult}; - -/// Connection and execution settings for a Redis provider. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RedisConfig { - /// Redis deployment mode. The first adapter stage supports standalone. - pub mode: String, - /// Redis endpoints. Standalone mode uses the first endpoint. - pub endpoints: Vec, - /// Optional ACL username. - pub username: String, - /// Environment variable name containing the Redis password. - pub password_env: String, - /// Maximum number of concurrent Redis commands issued by this provider. - pub pool_size: usize, - /// Timeout used while establishing the Redis connection. - pub connect_timeout_ms: u64, - /// Timeout applied to each Redis command. - pub command_timeout_ms: u64, - /// Prefix prepended to every cache key before it reaches Redis. - pub key_prefix: String, - /// Redis TTL in seconds. Zero disables native Redis expiration. - pub default_ttl_seconds: u64, - /// Whether reads may use replicas. The first adapter stage is primary-only. - pub read_from_replica: bool, -} - -impl Default for RedisConfig { - fn default() -> Self { - Self { - mode: "standalone".into(), - endpoints: vec!["redis://127.0.0.1:6379".into()], - username: String::new(), - password_env: String::new(), - pool_size: 32, - connect_timeout_ms: 1_000, - command_timeout_ms: 20, - key_prefix: "ragfs-cache".into(), - default_ttl_seconds: 3_600, - read_from_replica: false, - } - } -} - -impl RedisConfig { - pub(crate) fn validate(&self) -> CacheResult<()> { - if self.mode != "standalone" { - return Err(CacheError::InvalidArgument( - "Redis mode must be standalone in this adapter stage".into(), - )); - } - if self.endpoints.is_empty() { - return Err(CacheError::InvalidArgument( - "Redis endpoints must not be empty".into(), - )); - } - if self - .endpoints - .iter() - .any(|endpoint| endpoint.trim().is_empty()) - { - return Err(CacheError::InvalidArgument( - "Redis endpoints must not contain empty values".into(), - )); - } - if self.pool_size == 0 { - return Err(CacheError::InvalidArgument( - "Redis pool_size must be greater than zero".into(), - )); - } - if self.connect_timeout_ms == 0 { - return Err(CacheError::InvalidArgument( - "Redis connect_timeout_ms must be greater than zero".into(), - )); - } - if self.command_timeout_ms == 0 { - return Err(CacheError::InvalidArgument( - "Redis command_timeout_ms must be greater than zero".into(), - )); - } - if self.key_prefix.trim().is_empty() { - return Err(CacheError::InvalidArgument( - "Redis key_prefix must not be empty".into(), - )); - } - if self.read_from_replica { - return Err(CacheError::InvalidArgument( - "Redis read_from_replica is not supported in standalone mode".into(), - )); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn validates_required_redis_settings() { - let config = RedisConfig { - mode: "cluster".into(), - ..RedisConfig::default() - }; - assert!(matches!( - config.validate().unwrap_err(), - CacheError::InvalidArgument(_) - )); - - let mut config = RedisConfig::default(); - config.endpoints.clear(); - assert!(matches!( - config.validate().unwrap_err(), - CacheError::InvalidArgument(_) - )); - - let config = RedisConfig { - pool_size: 0, - ..RedisConfig::default() - }; - assert!(matches!( - config.validate().unwrap_err(), - CacheError::InvalidArgument(_) - )); - - let mut config = RedisConfig::default(); - config.key_prefix.clear(); - assert!(matches!( - config.validate().unwrap_err(), - CacheError::InvalidArgument(_) - )); - } -} diff --git a/crates/ragfs-cache-redis/src/lib.rs b/crates/ragfs-cache-redis/src/lib.rs deleted file mode 100644 index 6efa84140e..0000000000 --- a/crates/ragfs-cache-redis/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Redis cache provider adapter for RAGFS. - -mod client; -mod config; -mod provider; - -pub use config::RedisConfig; -pub use provider::RedisProvider; diff --git a/crates/ragfs-cache-redis/src/provider.rs b/crates/ragfs-cache-redis/src/provider.rs deleted file mode 100644 index 8c61f64b85..0000000000 --- a/crates/ragfs-cache-redis/src/provider.rs +++ /dev/null @@ -1,193 +0,0 @@ -use crate::client::RedisClient; -use crate::RedisConfig; -use async_trait::async_trait; -use bytes::Bytes; -use ragfs::cache::{CacheError, CacheProvider, CacheResult, ProviderCapabilities}; -use std::collections::HashSet; -use std::fmt; -use std::sync::{Arc, Mutex, MutexGuard}; - -/// Redis implementation of the common RAGFS cache provider contract. -pub struct RedisProvider { - client: Arc, - key_prefix: String, - ttl_ms: Option, - known_keys: Mutex>, -} - -impl fmt::Debug for RedisProvider { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RedisProvider") - .field("key_prefix", &self.key_prefix) - .field("ttl_ms", &self.ttl_ms) - .finish_non_exhaustive() - } -} - -impl RedisProvider { - /// Connect to Redis and validate the provider with PING. - pub async fn connect(config: RedisConfig) -> CacheResult { - config.validate()?; - let key_prefix = normalized_prefix(&config.key_prefix); - let ttl_ms = ttl_ms(config.default_ttl_seconds)?; - let client = Arc::new(RedisClient::connect(&config).await?); - Ok(Self { - client, - key_prefix, - ttl_ms, - known_keys: Mutex::new(HashSet::new()), - }) - } - - /// Check whether the connected Redis service is healthy. - pub async fn health_check(&self) -> CacheResult<()> { - self.client.health_check().await - } - - fn redis_key(&self, key: &str) -> String { - format!("{}{}", self.key_prefix, key) - } -} - -fn normalized_prefix(prefix: &str) -> String { - if prefix.ends_with(':') { - prefix.to_owned() - } else { - format!("{prefix}:") - } -} - -fn ttl_ms(ttl_seconds: u64) -> CacheResult> { - if ttl_seconds == 0 { - return Ok(None); - } - ttl_seconds - .checked_mul(1_000) - .map(Some) - .ok_or_else(|| CacheError::InvalidArgument("Redis default_ttl_seconds is too large".into())) -} - -fn lock_known_keys( - known_keys: &Mutex>, -) -> CacheResult>> { - known_keys - .lock() - .map_err(|_| CacheError::Internal("Redis key tracker is poisoned".into())) -} - -#[async_trait] -impl CacheProvider for RedisProvider { - fn name(&self) -> &'static str { - "redis" - } - - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities { - batch_get: true, - batch_put: true, - native_ttl: self.ttl_ms.is_some(), - } - } - - async fn get(&self, key: &str) -> CacheResult> { - Ok(self.client.get(self.redis_key(key)).await?.map(Bytes::from)) - } - - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> { - self.client - .set(self.redis_key(key), value.to_vec(), self.ttl_ms) - .await?; - lock_known_keys(&self.known_keys)?.insert(key.to_owned()); - Ok(()) - } - - async fn delete(&self, key: &str) -> CacheResult<()> { - self.client.delete(self.redis_key(key)).await?; - lock_known_keys(&self.known_keys)?.remove(key); - Ok(()) - } - - async fn exists(&self, key: &str) -> CacheResult { - self.client.exists(self.redis_key(key)).await - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - let redis_keys = keys.iter().map(|key| self.redis_key(key)).collect(); - Ok(self - .client - .batch_get(redis_keys) - .await? - .into_iter() - .map(|value| value.map(Bytes::from)) - .collect()) - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - let redis_entries = entries - .iter() - .map(|(key, value)| (self.redis_key(key), value.to_vec())) - .collect(); - self.client.batch_set(redis_entries, self.ttl_ms).await?; - lock_known_keys(&self.known_keys)?.extend(entries.into_iter().map(|(key, _)| key)); - Ok(()) - } - - async fn invalidate(&self, keys: &[String]) -> CacheResult<()> { - let redis_keys = keys.iter().map(|key| self.redis_key(key)).collect(); - self.client.batch_delete(redis_keys).await?; - let mut known_keys = lock_known_keys(&self.known_keys)?; - for key in keys { - known_keys.remove(key); - } - Ok(()) - } - - async fn flush(&self) -> CacheResult<()> { - let keys = lock_known_keys(&self.known_keys)? - .iter() - .cloned() - .collect::>(); - self.invalidate(&keys).await - } - - async fn close(&self) -> CacheResult<()> { - lock_known_keys(&self.known_keys)?.clear(); - self.client.close().await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::panic::{self, AssertUnwindSafe}; - - #[test] - fn normalizes_key_prefix_once() { - assert_eq!(normalized_prefix("ragfs-cache"), "ragfs-cache:"); - assert_eq!(normalized_prefix("ragfs-cache:"), "ragfs-cache:"); - } - - #[test] - fn ttl_zero_disables_native_expiration() { - assert_eq!(ttl_ms(0).unwrap(), None); - assert_eq!(ttl_ms(2).unwrap(), Some(2_000)); - } - - #[test] - fn lock_known_keys_returns_internal_error_when_poisoned() { - let known_keys = Mutex::new(HashSet::new()); - - let _ = panic::catch_unwind(AssertUnwindSafe(|| { - let _guard = known_keys.lock().unwrap(); - panic!("poison known_keys"); - })); - - let error = lock_known_keys(&known_keys).unwrap_err(); - - assert!(matches!( - error, - CacheError::Internal(message) if message == "Redis key tracker is poisoned" - )); - } -} diff --git a/crates/ragfs-cache-redis/tests/cached_filesystem.rs b/crates/ragfs-cache-redis/tests/cached_filesystem.rs deleted file mode 100644 index df3c2ceae8..0000000000 --- a/crates/ragfs-cache-redis/tests/cached_filesystem.rs +++ /dev/null @@ -1,226 +0,0 @@ -use async_trait::async_trait; -use ragfs::cache::{CacheNamespace, CachePolicy, CacheProvider, CachedFileSystem}; -use ragfs::core::{GrepResult, TreeEntry}; -use ragfs::plugins::MemFileSystem; -use ragfs::{FileInfo, FileSystem, Result as FsResult, WriteFlag}; -use ragfs_cache_redis::{RedisConfig, RedisProvider}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; - -#[derive(Clone)] -struct CountingFileSystem { - inner: Arc, - reads: Arc, - read_dirs: Arc, -} - -impl CountingFileSystem { - fn new() -> Self { - Self { - inner: Arc::new(MemFileSystem::new()), - reads: Arc::new(AtomicU64::new(0)), - read_dirs: Arc::new(AtomicU64::new(0)), - } - } -} - -#[async_trait] -impl FileSystem for CountingFileSystem { - async fn create(&self, path: &str) -> FsResult<()> { - self.inner.create(path).await - } - - async fn mkdir(&self, path: &str, mode: u32) -> FsResult<()> { - self.inner.mkdir(path, mode).await - } - - async fn remove(&self, path: &str) -> FsResult<()> { - self.inner.remove(path).await - } - - async fn remove_all(&self, path: &str) -> FsResult<()> { - self.inner.remove_all(path).await - } - - async fn read(&self, path: &str, offset: u64, size: u64) -> FsResult> { - self.reads.fetch_add(1, Ordering::SeqCst); - self.inner.read(path, offset, size).await - } - - async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> FsResult { - self.inner.write(path, data, offset, flags).await - } - - async fn read_dir(&self, path: &str) -> FsResult> { - self.read_dirs.fetch_add(1, Ordering::SeqCst); - self.inner.read_dir(path).await - } - - async fn stat(&self, path: &str) -> FsResult { - self.inner.stat(path).await - } - - async fn rename(&self, old_path: &str, new_path: &str) -> FsResult<()> { - self.inner.rename(old_path, new_path).await - } - - async fn chmod(&self, path: &str, mode: u32) -> FsResult<()> { - self.inner.chmod(path, mode).await - } - - async fn truncate(&self, path: &str, size: u64) -> FsResult<()> { - self.inner.truncate(path, size).await - } - - async fn grep( - &self, - path: &str, - pattern: &str, - recursive: bool, - case_insensitive: bool, - node_limit: Option, - exclude_path: Option<&str>, - level_limit: Option, - ) -> FsResult { - self.inner - .grep( - path, - pattern, - recursive, - case_insensitive, - node_limit, - exclude_path, - level_limit, - ) - .await - } - - async fn tree_directory( - &self, - path: &str, - show_hidden: bool, - node_limit: Option, - level_limit: Option, - ) -> FsResult> { - self.inner - .tree_directory(path, show_hidden, node_limit, level_limit) - .await - } -} - -fn config(test_name: &str) -> Option { - let endpoint = std::env::var("REDIS_URL").ok()?; - Some(RedisConfig { - endpoints: vec![endpoint], - key_prefix: format!("ragfs-cache-fs-test:{}:{}", std::process::id(), test_name), - connect_timeout_ms: 30_000, - command_timeout_ms: 1_000, - default_ttl_seconds: 60, - ..RedisConfig::default() - }) -} - -async fn cached_fs(backend: CountingFileSystem, test_name: &str) -> Option { - let provider: Arc = - Arc::new(RedisProvider::connect(config(test_name)?).await.unwrap()); - Some(CachedFileSystem::new( - Box::new(backend), - provider, - CacheNamespace::new(test_name), - CachePolicy::default(), - )) -} - -#[tokio::test] -async fn redis_hit_miss_fill_and_write_after_read_are_consistent() { - let backend = CountingFileSystem::new(); - backend - .write("/value.md", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let Some(fs) = cached_fs(backend, "redis-read-write").await else { - return; - }; - - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old"); - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old"); - assert_eq!(probe.reads.load(Ordering::SeqCst), 1); - - fs.write("/value.md", b"new", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"new"); - assert_eq!(probe.reads.load(Ordering::SeqCst), 1); - fs.provider().flush().await.unwrap(); - fs.provider().close().await.unwrap(); -} - -#[tokio::test] -async fn redis_directory_and_mutation_invalidation_stay_consistent() { - let backend = CountingFileSystem::new(); - backend.mkdir("/root", 0o755).await.unwrap(); - backend.mkdir("/root/tree", 0o755).await.unwrap(); - backend - .write("/root/tree/leaf", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let Some(fs) = cached_fs(backend, "redis-invalidation").await else { - return; - }; - - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 1); - fs.mkdir("/root/created", 0o755).await.unwrap(); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2); - assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 2); - - assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"old"); - fs.rename("/root/tree/leaf", "/root/tree/moved") - .await - .unwrap(); - assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err()); - assert_eq!(fs.read("/root/tree/moved", 0, 0).await.unwrap(), b"old"); - fs.remove("/root/tree/moved").await.unwrap(); - assert!(fs.read("/root/tree/moved", 0, 0).await.is_err()); - - direct - .write("/root/tree/leaf", b"stale", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"stale"); - fs.remove_all("/root/tree").await.unwrap(); - direct.mkdir("/root/tree", 0o755).await.unwrap(); - direct - .write("/root/tree/leaf", b"fresh", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"fresh"); - - fs.rename("/root/tree", "/root/renamed").await.unwrap(); - assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err()); - assert_eq!(fs.read("/root/renamed/leaf", 0, 0).await.unwrap(), b"fresh"); - fs.provider().flush().await.unwrap(); - fs.provider().close().await.unwrap(); -} - -#[tokio::test] -async fn closed_redis_provider_falls_back_without_breaking_backend_reads() { - let backend = CountingFileSystem::new(); - backend - .write("/available.md", b"backend", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let Some(fs) = cached_fs(backend, "redis-fallback").await else { - return; - }; - fs.provider().close().await.unwrap(); - - assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend"); - assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend"); - assert_eq!(probe.reads.load(Ordering::SeqCst), 2); - assert!(fs.metrics().snapshot().errors >= 1); -} diff --git a/crates/ragfs-cache-redis/tests/provider_contract.rs b/crates/ragfs-cache-redis/tests/provider_contract.rs deleted file mode 100644 index da130af547..0000000000 --- a/crates/ragfs-cache-redis/tests/provider_contract.rs +++ /dev/null @@ -1,101 +0,0 @@ -use bytes::Bytes; -use ragfs::cache::CacheProvider; -use ragfs_cache_redis::{RedisConfig, RedisProvider}; - -fn config(test_name: &str) -> Option { - let endpoint = std::env::var("REDIS_URL").ok()?; - Some(RedisConfig { - endpoints: vec![endpoint], - key_prefix: format!("ragfs-cache-test:{}:{}", std::process::id(), test_name), - connect_timeout_ms: 30_000, - command_timeout_ms: 1_000, - default_ttl_seconds: 60, - ..RedisConfig::default() - }) -} - -async fn provider(test_name: &str) -> Option { - Some(RedisProvider::connect(config(test_name)?).await.unwrap()) -} - -#[tokio::test] -async fn hit_miss_write_delete_and_exists_map_to_redis_operations() { - let Some(provider) = provider("contract-basic").await else { - return; - }; - - assert_eq!(provider.get("missing").await.unwrap(), None); - provider - .put("hit", Bytes::from_static(b"value")) - .await - .unwrap(); - assert!(provider.exists("hit").await.unwrap()); - assert_eq!( - provider.get("hit").await.unwrap(), - Some(Bytes::from_static(b"value")) - ); - - provider.delete("hit").await.unwrap(); - provider.delete("hit").await.unwrap(); - assert!(!provider.exists("hit").await.unwrap()); - assert_eq!(provider.get("hit").await.unwrap(), None); - provider.flush().await.unwrap(); - provider.close().await.unwrap(); -} - -#[tokio::test] -async fn batch_operations_preserve_order_and_flush_only_known_keys() { - let Some(provider) = provider("contract-batch").await else { - return; - }; - - provider - .batch_put(vec![ - ("one".into(), Bytes::from_static(b"1")), - ("two".into(), Bytes::from_static(b"2")), - ]) - .await - .unwrap(); - - assert_eq!( - provider - .batch_get(&["two".into(), "missing".into(), "one".into()]) - .await - .unwrap(), - vec![ - Some(Bytes::from_static(b"2")), - None, - Some(Bytes::from_static(b"1")) - ] - ); - assert!(provider.capabilities().batch_get); - assert!(provider.capabilities().batch_put); - assert!(provider.capabilities().native_ttl); - - provider.flush().await.unwrap(); - assert_eq!(provider.get("one").await.unwrap(), None); - assert_eq!(provider.get("two").await.unwrap(), None); - provider.close().await.unwrap(); -} - -#[tokio::test] -async fn set_px_ttl_expires_to_cache_miss() { - let Some(mut config) = config("contract-ttl") else { - return; - }; - config.default_ttl_seconds = 1; - let provider = RedisProvider::connect(config).await.unwrap(); - - provider - .put("ttl", Bytes::from_static(b"short")) - .await - .unwrap(); - assert_eq!( - provider.get("ttl").await.unwrap(), - Some(Bytes::from_static(b"short")) - ); - - tokio::time::sleep(std::time::Duration::from_millis(1_200)).await; - assert_eq!(provider.get("ttl").await.unwrap(), None); - provider.close().await.unwrap(); -} diff --git a/crates/ragfs-python-native/Cargo.toml b/crates/ragfs-python-native/Cargo.toml deleted file mode 100644 index 6f0702cc14..0000000000 --- a/crates/ragfs-python-native/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[workspace] -resolver = "2" - -[package] -name = "ragfs-python" -version = "0.1.0" -edition = "2021" -rust-version = "1.91.1" -description = "Python bindings for RAGFS with explicit native cache providers" -publish = false - -[lib] -name = "ragfs_python" -path = "../ragfs-python/src/lib.rs" -crate-type = ["cdylib"] - -[features] -default = ["extension-module", "s3", "cache-redis"] -extension-module = ["pyo3/extension-module"] -s3 = ["ragfs/s3"] -cache-redis = ["ragfs-cache-redis"] -yuanrong-native = [ - "ragfs-cache-yuanrong", - "ragfs-cache-yuanrong/yuanrong-native", -] -mooncake-native = [ - "ragfs-cache-mooncake", - "ragfs-cache-mooncake/mooncake-native", -] - -[dependencies] -ragfs = { path = "../ragfs", features = ["cache"] } -ragfs-cache-redis = { path = "../ragfs-cache-redis", optional = true } -ragfs-cache-yuanrong = { path = "../ragfs-cache-yuanrong", optional = true } -ragfs-cache-mooncake = { path = "../ragfs-cache-mooncake", optional = true } -pyo3 = { version = "0.27", features = ["abi3", "abi3-py310"] } -tokio = { version = "1", features = ["full"] } -serde_json = "1.0" -serde = { version = "1", features = ["derive"] } -toml = "0.8" -gix-hash = "0.14" -similar = "3.1" diff --git a/crates/ragfs-python/Cargo.toml b/crates/ragfs-python/Cargo.toml index 95d8668447..340130ae4e 100644 --- a/crates/ragfs-python/Cargo.toml +++ b/crates/ragfs-python/Cargo.toml @@ -13,14 +13,12 @@ name = "ragfs_python" crate-type = ["cdylib", "rlib"] [features] -default = ["extension-module", "s3", "cache-redis"] +default = ["extension-module", "s3"] extension-module = ["pyo3/extension-module"] s3 = ["ragfs/s3"] -cache-redis = ["ragfs-cache-redis"] [dependencies] ragfs = { path = "../ragfs", features = ["cache"] } -ragfs-cache-redis = { path = "../ragfs-cache-redis", optional = true } pyo3 = { version = "0.27", features = ["abi3", "abi3-py310"] } tokio = { version = "1", features = ["full"] } tracing = "0.1" @@ -32,6 +30,3 @@ toml = "0.8" gix-hash = "0.14" bytes = "1.5" similar = "3.1" - -[lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("yuanrong-native", "mooncake-native"))'] } diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs index 42eba817a3..0ba9fe0fbc 100644 --- a/crates/ragfs-python/src/lib.rs +++ b/crates/ragfs-python/src/lib.rs @@ -14,8 +14,8 @@ use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; use tracing::Level; -use tracing_subscriber::fmt::writer::BoxMakeWriter; use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::fmt::writer::BoxMakeWriter; /// Cached reference to the Python `openviking.storage.errors.LockAcquisitionError` class, /// imported at module init so that native and Python code share the same exception type. @@ -66,10 +66,7 @@ fn open_tracing_log_file(path: &Path) -> Result { } /// Swap the shared Rust tracing file handle to the current active log file. -fn replace_tracing_log_file( - shared: &Arc>, - path: &Path, -) -> Result<(), String> { +fn replace_tracing_log_file(shared: &Arc>, path: &Path) -> Result<(), String> { let new_file = open_tracing_log_file(path)?; let mut file = shared .lock() @@ -164,7 +161,8 @@ fn pathlock_err_to_py(err: PathLockError) -> PyErr { PathLockError::Conflict { .. } | PathLockError::Timeout { .. } | PathLockError::HandoffFailed(_) - | PathLockError::Busy { .. } => { + | PathLockError::Busy { .. } => + { #[allow(deprecated)] Python::with_gil(|py| { let ty = LOCK_ACQUISITION_ERROR_TYPE @@ -210,21 +208,21 @@ fn validate_expire_secs(v: f64) -> PyResult { } } -use ragfs::cache::{ - CacheError, CacheNamespace, CachePolicy, CacheProvider, CacheResult, CacheTraversalMode, - MemoryCacheProvider, +use ragfs::cache::{CachePolicy, CacheTraversalMode}; +use ragfs::cache_runtime::{DynamicProviderConfig, RedisProviderConfig}; +use ragfs::core::builder::{ + CacheFsConfig, CacheRuntimeProviderConfig, CacheStackConfig, EncryptionConfig, }; -use ragfs::core::builder::EncryptionConfig; use ragfs::core::{ - build_default_stack, build_stack_with_mountable, ConfigValue, FileInfo, FileSystem, - FilesystemStats, FsContext, FsContextInner, FsOperation, GlobPage, GrepResult, MountableFS, - OperationStats, PathLockContext, PluginConfig, RagfsConfig, TreeEntry, WriteFlag, FS_CTX, + build_configured_stack, ConfigValue, FileInfo, FileSystem, FilesystemStats, FsContext, + FsContextInner, FsOperation, GlobPage, GrepResult, MountableFS, OperationStats, + PathLockContext, PluginConfig, RagfsConfig, TreeEntry, WriteFlag, FS_CTX, }; +use ragfs::lock::types::PathLockError; use ragfs::lock::{ BorrowedPathLockLease, OwnedPathLockLease, PathLockConfig, PathLockHandoffRef, PathLockKind, PathLockManager, PathLockRequest, }; -use ragfs::lock::types::PathLockError; /// Parse one Python pathlock request without silently defaulting invalid fields. fn parse_pathlock_request(raw: &HashMap) -> PyResult { @@ -261,46 +259,27 @@ fn parse_pathlock_request_batch( #[derive(Debug, Clone, PartialEq, Eq)] enum CacheProviderKind { - Memory, - Yuanrong, - Mooncake, Redis, + Dynamic, } #[derive(Debug, Clone, PartialEq, Eq)] struct RagfsCacheConfig { enabled: bool, + runtime_enabled: bool, provider: CacheProviderKind, namespace: String, max_file_size_bytes: usize, traversal_mode: CacheTraversalMode, bypass_prefixes: Vec, - yuanrong: YuanrongCacheConfig, - mooncake: MooncakeCacheConfig, redis: RedisCacheConfig, + dynamic: DynamicCacheConfig, } #[derive(Debug, Clone, PartialEq, Eq)] -struct YuanrongCacheConfig { - host: String, - port: u16, - connect_timeout_ms: u64, - request_timeout_ms: u64, - sdk_concurrency: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct MooncakeCacheConfig { - local_hostname: String, - metadata_server: String, - master_server_addr: String, - protocol: String, - device_name: String, - global_segment_size: u64, - local_buffer_size: u64, - replica_num: usize, - sdk_concurrency: usize, - operation_timeout_ms: u64, +struct DynamicCacheConfig { + library: String, + params: serde_json::Value, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -321,43 +300,23 @@ impl Default for RagfsCacheConfig { fn default() -> Self { Self { enabled: false, - provider: CacheProviderKind::Memory, + runtime_enabled: false, + provider: CacheProviderKind::Redis, namespace: "openviking".to_string(), max_file_size_bytes: CachePolicy::default().max_file_size(), traversal_mode: CacheTraversalMode::Backend, bypass_prefixes: Vec::new(), - yuanrong: YuanrongCacheConfig::default(), - mooncake: MooncakeCacheConfig::default(), redis: RedisCacheConfig::default(), + dynamic: DynamicCacheConfig::default(), } } } -impl Default for YuanrongCacheConfig { - fn default() -> Self { - Self { - host: "127.0.0.1".to_string(), - port: 31501, - connect_timeout_ms: 5_000, - request_timeout_ms: 5_000, - sdk_concurrency: 4, - } - } -} - -impl Default for MooncakeCacheConfig { +impl Default for DynamicCacheConfig { fn default() -> Self { Self { - local_hostname: "127.0.0.1".to_string(), - metadata_server: "http://127.0.0.1:8080/metadata".to_string(), - master_server_addr: "127.0.0.1:50051".to_string(), - protocol: "tcp".to_string(), - device_name: String::new(), - global_segment_size: 512 << 20, - local_buffer_size: 128 << 20, - replica_num: 2, - sdk_concurrency: 4, - operation_timeout_ms: 5_000, + library: String::new(), + params: serde_json::Value::Object(serde_json::Map::new()), } } } @@ -372,124 +331,85 @@ impl Default for RedisCacheConfig { pool_size: 32, connect_timeout_ms: 1_000, command_timeout_ms: 20, - key_prefix: "ragfs-cache".to_string(), + key_prefix: String::new(), default_ttl_seconds: 3_600, read_from_replica: false, } } } -struct CacheProviderFactory; - -impl CacheProviderFactory { - async fn create(config: &RagfsCacheConfig) -> CacheResult> { - match config.provider { - CacheProviderKind::Memory => Ok(Arc::new(MemoryCacheProvider::new())), - CacheProviderKind::Yuanrong => create_yuanrong_provider(config).await, - CacheProviderKind::Mooncake => create_mooncake_provider(config).await, - CacheProviderKind::Redis => create_redis_provider(config).await, +impl RagfsCacheConfig { + fn stack_config(&self) -> Option { + if !self.enabled && !self.runtime_enabled { + return None; } + let provider = match self.provider { + CacheProviderKind::Redis => CacheRuntimeProviderConfig::Redis(RedisProviderConfig { + mode: self.redis.mode.clone(), + endpoints: self.redis.endpoints.clone(), + username: self.redis.username.clone(), + password_env: self.redis.password_env.clone(), + pool_size: self.redis.pool_size, + connect_timeout_ms: self.redis.connect_timeout_ms, + command_timeout_ms: self.redis.command_timeout_ms, + key_prefix: self.redis.key_prefix.clone(), + default_ttl_seconds: self.redis.default_ttl_seconds, + read_from_replica: self.redis.read_from_replica, + }), + CacheProviderKind::Dynamic => { + CacheRuntimeProviderConfig::Dynamic(DynamicProviderConfig { + library_path: PathBuf::from(&self.dynamic.library), + params_json: serde_json::to_string(&self.dynamic.params) + .expect("dynamic provider params are valid JSON"), + }) + } + }; + Some(CacheStackConfig { + provider: Some(provider), + cachefs: CacheFsConfig { + enabled: self.enabled, + namespace: self.namespace.clone(), + policy: cache_policy_from_config(self), + }, + }) } } -#[cfg(feature = "yuanrong-native")] -async fn create_yuanrong_provider( - config: &RagfsCacheConfig, -) -> CacheResult> { - use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongProvider}; - - let provider = YuanrongProvider::connect(YuanrongConfig { - host: config.yuanrong.host.clone(), - port: config.yuanrong.port, - connect_timeout_ms: config.yuanrong.connect_timeout_ms, - request_timeout_ms: config.yuanrong.request_timeout_ms, - sdk_concurrency: config.yuanrong.sdk_concurrency, - }) - .await?; - Ok(Arc::new(provider)) -} - -#[cfg(not(feature = "yuanrong-native"))] -async fn create_yuanrong_provider( - _config: &RagfsCacheConfig, -) -> CacheResult> { - Err(CacheError::Unavailable( - "Yuanrong support requires the yuanrong-native feature".to_string(), - )) -} - -#[cfg(feature = "mooncake-native")] -async fn create_mooncake_provider( - config: &RagfsCacheConfig, -) -> CacheResult> { - use ragfs_cache_mooncake::{MooncakeConfig, MooncakeProvider}; - - let provider = MooncakeProvider::connect(MooncakeConfig { - local_hostname: config.mooncake.local_hostname.clone(), - metadata_server: config.mooncake.metadata_server.clone(), - master_server_addr: config.mooncake.master_server_addr.clone(), - protocol: config.mooncake.protocol.clone(), - device_name: config.mooncake.device_name.clone(), - global_segment_size: config.mooncake.global_segment_size, - local_buffer_size: config.mooncake.local_buffer_size, - replica_num: config.mooncake.replica_num, - sdk_concurrency: config.mooncake.sdk_concurrency, - operation_timeout_ms: config.mooncake.operation_timeout_ms, - }) - .await?; - Ok(Arc::new(provider)) -} - -#[cfg(not(feature = "mooncake-native"))] -async fn create_mooncake_provider( - _config: &RagfsCacheConfig, -) -> CacheResult> { - Err(CacheError::Unavailable( - "Mooncake support requires the mooncake-native feature".to_string(), - )) -} - -#[cfg(feature = "cache-redis")] -async fn create_redis_provider(config: &RagfsCacheConfig) -> CacheResult> { - use ragfs_cache_redis::{RedisConfig, RedisProvider}; - - let provider = RedisProvider::connect(RedisConfig { - mode: config.redis.mode.clone(), - endpoints: config.redis.endpoints.clone(), - username: config.redis.username.clone(), - password_env: config.redis.password_env.clone(), - pool_size: config.redis.pool_size, - connect_timeout_ms: config.redis.connect_timeout_ms, - command_timeout_ms: config.redis.command_timeout_ms, - key_prefix: config.redis.key_prefix.clone(), - default_ttl_seconds: config.redis.default_ttl_seconds, - read_from_replica: config.redis.read_from_replica, - }) - .await?; - Ok(Arc::new(provider)) -} - -#[cfg(not(feature = "cache-redis"))] -async fn create_redis_provider(_config: &RagfsCacheConfig) -> CacheResult> { - Err(CacheError::Unavailable( - "Redis support requires the cache-redis feature".to_string(), - )) -} - fn cache_config_from_ov_conf(path: &str) -> Result { let raw = fs::read_to_string(path) .map_err(|error| format!("failed to read OpenViking config {path}: {error}"))?; let json: serde_json::Value = serde_json::from_str(&raw) .map_err(|error| format!("failed to parse OpenViking config {path}: {error}"))?; - match json + let configured_provider = json + .get("storage") + .and_then(|storage| storage.get("agfs")) + .and_then(|agfs| agfs.get("cache")) + .and_then(|cache| cache.get("provider")) + .and_then(serde_json::Value::as_str) + .unwrap_or("redis"); + + let mut config = match json .get("storage") .and_then(|storage| storage.get("agfs")) .and_then(|agfs| agfs.get("cache")) { Some(cache) => cache_config_from_value(cache), None => Ok(RagfsCacheConfig::default()), + }?; + if json + .get("storage") + .and_then(|storage| storage.get("agfs")) + .and_then(|agfs| agfs.get("queuefs")) + .and_then(|queuefs| queuefs.get("backend")) + .and_then(serde_json::Value::as_str) + == Some("cache") + { + config.runtime_enabled = true; + config.provider = provider_kind(configured_provider.to_string())?; } + validate_cache_runtime_config(&config)?; + Ok(config) } fn cache_config_from_value(cache: &serde_json::Value) -> Result { @@ -502,7 +422,16 @@ fn cache_config_from_value(cache: &serde_json::Value) -> Result CacheProviderKind::Redis, + _ => provider_kind(provider)?, + } + }; config.namespace = string_field(cache, "namespace", &config.namespace)?; if config.namespace.trim().is_empty() { return Err("storage.agfs.cache.namespace must not be empty".to_string()); @@ -512,66 +441,6 @@ fn cache_config_from_value(cache: &serde_json::Value) -> Result Result Result<(), String> { + if !config.enabled && !config.runtime_enabled { + return Ok(()); + } + if matches!(config.provider, CacheProviderKind::Dynamic) + && config.dynamic.library.trim().is_empty() + { + return Err("storage.agfs.cache.dynamic.library must not be empty".to_string()); + } + if config.runtime_enabled + && matches!(config.provider, CacheProviderKind::Redis) + && !config.redis.key_prefix.is_empty() + { + return Err( + "storage.agfs.cache.redis.key_prefix must be empty when QueueFS uses backend=cache" + .to_string(), + ); + } + Ok(()) +} + fn provider_kind(value: String) -> Result { match value.as_str() { - "memory" => Ok(CacheProviderKind::Memory), - "yuanrong" => Ok(CacheProviderKind::Yuanrong), - "mooncake" => Ok(CacheProviderKind::Mooncake), "redis" => Ok(CacheProviderKind::Redis), + "dynamic" => Ok(CacheProviderKind::Dynamic), other => Err(format!( - "unsupported storage.agfs.cache.provider: {other}; expected memory, yuanrong, mooncake, or redis" + "unsupported storage.agfs.cache.provider: {other}; expected redis or dynamic" )), } } @@ -676,15 +575,6 @@ fn u64_field_allow_zero( } } -fn u16_field( - object: &serde_json::Map, - key: &str, - default: u16, -) -> Result { - let value = u64_field(object, key, default as u64)?; - u16::try_from(value).map_err(|_| format!("{key} must fit in u16")) -} - fn usize_field( object: &serde_json::Map, key: &str, @@ -1050,7 +940,10 @@ fn build_fs_context(ctx: Option>) -> FsContext { let mut bypass_cache = false; if let Some(m) = &ctx { account_id = m.get("account_id").cloned().unwrap_or_default(); - disable_auto_pathlock = m.get("disable_auto_pathlock").map(|s| s == "true").unwrap_or(false); + disable_auto_pathlock = m + .get("disable_auto_pathlock") + .map(|s| s == "true") + .unwrap_or(false); lease_ref = m.get("lease_ref").cloned(); bypass_cache = m.get("bypass_cache").map(|s| s == "true").unwrap_or(false); } @@ -1135,10 +1028,7 @@ fn extract_lease_ref(py: Python<'_>, lease_ref: &Py) -> PyResult } /// Extract an owned PathLock lease and lifecycle capability from a typed dictionary. -fn extract_owned_lease_ref( - py: Python<'_>, - lease_ref: &Py, -) -> PyResult<(String, String)> { +fn extract_owned_lease_ref(py: Python<'_>, lease_ref: &Py) -> PyResult<(String, String)> { let ref_dict: HashMap> = lease_ref.extract(py)?; require_owned_lease_ref(py, &ref_dict) } @@ -1201,9 +1091,8 @@ fn load_git_from_config( fs: &Arc, rt: &tokio::runtime::Runtime, ) -> PyResult<(Option>, Option)> { - let body = std::fs::read_to_string(path).map_err(|e| { - PyRuntimeError::new_err(format!("read config_path {}: {}", path, e)) - })?; + let body = std::fs::read_to_string(path) + .map_err(|e| PyRuntimeError::new_err(format!("read config_path {}: {}", path, e)))?; let cfg: BindingConfig = toml::from_str(&body) .map_err(|e| PyRuntimeError::new_err(format!("parse config_path: {}", e)))?; match cfg.git { @@ -1267,7 +1156,7 @@ impl RAGFSBindingClient { py: Python<'_>, config_path: Option<&str>, config: Option>>, - git_config_path: Option<&str> + git_config_path: Option<&str>, ) -> PyResult { let rt = tokio::runtime::Runtime::new() .map_err(|e| PyRuntimeError::new_err(format!("Failed to create runtime: {}", e)))?; @@ -1365,24 +1254,16 @@ impl RAGFSBindingClient { }, }; - // Phase B: build the stack. Cache, when enabled, replaces the mountable - // layer with a cache-aware mountable while preserving the same plugin - // registration and per-backend encryption setup. - let stack = if cache_config.enabled { - let provider = rt - .block_on(CacheProviderFactory::create(&cache_config)) - .map_err(|error| { - PyRuntimeError::new_err(format!("Failed to initialize cache provider: {error}")) - })?; - let mountable = Arc::new(MountableFS::with_cache( - provider, - CacheNamespace::new(&cache_config.namespace), - cache_policy_from_config(&cache_config), - )); - rt.block_on(build_stack_with_mountable(ragfs_cfg, mountable)) - } else { - rt.block_on(build_default_stack(ragfs_cfg)) - }; + // Phase B: RAGFS owns Runtime construction and injects the same instance + // into CacheFS and QueueFS. The binding only translates configuration. + let stack = rt + .block_on(build_configured_stack( + ragfs_cfg, + cache_config.stack_config(), + )) + .map_err(|error| { + PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}")) + })?; // Build the git service from inline config when present; otherwise fall // back to loading the [git] section from a config file path. @@ -1410,7 +1291,11 @@ impl RAGFSBindingClient { m.insert("status".to_string(), "healthy".to_string()); m.insert( "git_enabled".to_string(), - if self.git_service.is_some() { "true".into() } else { "false".into() }, + if self.git_service.is_some() { + "true".into() + } else { + "false".into() + }, ); if let Some(b) = &self.git_backend { m.insert("git_backend".to_string(), b.clone()); @@ -1456,11 +1341,7 @@ impl RAGFSBindingClient { /// Read a commit's metadata or a blob's bytes at a path. #[pyo3(signature = (**kwargs))] - fn git_show( - &self, - py: Python<'_>, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { + fn git_show(&self, py: Python<'_>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult> { let svc = self.git_service.clone().ok_or_else(|| { git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into()) })?; @@ -1471,7 +1352,7 @@ impl RAGFSBindingClient { let resp = py_detach_blocking(py, move || { self.rt.block_on(svc.show_with_limit(req, max_blob_bytes)) }) - .map_err(|e| git::map_git_error(py, e))?; + .map_err(|e| git::map_git_error(py, e))?; git::show_response_to_pydict(py, resp) } @@ -1509,11 +1390,7 @@ impl RAGFSBindingClient { /// Walk snapshot history, optionally filtered to commits touching paths. #[pyo3(signature = (**kwargs))] - fn git_log( - &self, - py: Python<'_>, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { + fn git_log(&self, py: Python<'_>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult> { let svc = self.git_service.clone().ok_or_else(|| { git::new_py_err_pub(py, "AGFSNotSupportedError", "git feature disabled".into()) })?; @@ -2210,22 +2087,20 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let timeout = validate_timeout_secs(timeout_secs)?; - let owner_capability = - extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let path = path.clone(); - let capability = owner_capability.clone(); - async move { - let capability = capability - .as_ref() - .map(|(lease_ref, ownership_ref)| { + let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let path = path.clone(); + let capability = owner_capability.clone(); + async move { + let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| { (lease_ref.as_str(), ownership_ref.as_str()) }); - mgr.acquire_exact(&path, timeout, capability).await - } - }) - .map_err(pathlock_err_to_py)?; + mgr.acquire_exact(&path, timeout, capability).await + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2242,22 +2117,20 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let timeout = validate_timeout_secs(timeout_secs)?; - let owner_capability = - extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let paths = paths.clone(); - let capability = owner_capability.clone(); - async move { - let capability = capability - .as_ref() - .map(|(lease_ref, ownership_ref)| { + let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let paths = paths.clone(); + let capability = owner_capability.clone(); + async move { + let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| { (lease_ref.as_str(), ownership_ref.as_str()) }); - mgr.acquire_exact_batch(&paths, timeout, capability).await - } - }) - .map_err(pathlock_err_to_py)?; + mgr.acquire_exact_batch(&paths, timeout, capability).await + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2274,22 +2147,20 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let timeout = validate_timeout_secs(timeout_secs)?; - let owner_capability = - extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let path = path.clone(); - let capability = owner_capability.clone(); - async move { - let capability = capability - .as_ref() - .map(|(lease_ref, ownership_ref)| { + let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let path = path.clone(); + let capability = owner_capability.clone(); + async move { + let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| { (lease_ref.as_str(), ownership_ref.as_str()) }); - mgr.acquire_tree(&path, timeout, capability).await - } - }) - .map_err(pathlock_err_to_py)?; + mgr.acquire_tree(&path, timeout, capability).await + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2306,22 +2177,20 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let timeout = validate_timeout_secs(timeout_secs)?; - let owner_capability = - extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let paths = paths.clone(); - let capability = owner_capability.clone(); - async move { - let capability = capability - .as_ref() - .map(|(lease_ref, ownership_ref)| { + let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let paths = paths.clone(); + let capability = owner_capability.clone(); + async move { + let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| { (lease_ref.as_str(), ownership_ref.as_str()) }); - mgr.acquire_tree_batch(&paths, timeout, capability).await - } - }) - .map_err(pathlock_err_to_py)?; + mgr.acquire_tree_batch(&paths, timeout, capability).await + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2339,24 +2208,22 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let timeout = validate_timeout_secs(timeout_secs)?; - let owner_capability = - extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let exact = exact_paths.clone(); - let tree = tree_paths.clone(); - let capability = owner_capability.clone(); - async move { - let capability = capability - .as_ref() - .map(|(lease_ref, ownership_ref)| { + let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let exact = exact_paths.clone(); + let tree = tree_paths.clone(); + let capability = owner_capability.clone(); + async move { + let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| { (lease_ref.as_str(), ownership_ref.as_str()) }); - mgr.acquire_exact_tree_batch(&exact, &tree, timeout, capability) - .await - } - }) - .map_err(pathlock_err_to_py)?; + mgr.acquire_exact_tree_batch(&exact, &tree, timeout, capability) + .await + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2374,25 +2241,23 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let timeout = validate_timeout_secs(timeout_secs)?; - let owner_capability = - extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; + let owner_capability = extract_optional_owned_lease_ref(py, owner_lease_ref.as_ref())?; let lock_requests = parse_pathlock_request_batch(&requests)?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let reqs = lock_requests.clone(); - let capability = owner_capability.clone(); - async move { - let capability = capability - .as_ref() - .map(|(lease_ref, ownership_ref)| { + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let reqs = lock_requests.clone(); + let capability = owner_capability.clone(); + async move { + let capability = capability.as_ref().map(|(lease_ref, ownership_ref)| { (lease_ref.as_str(), ownership_ref.as_str()) }); - mgr.acquire_batch(&reqs, timeout, capability).await - } - }) - .map_err(pathlock_err_to_py)?; + mgr.acquire_batch(&reqs, timeout, capability).await + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2408,16 +2273,17 @@ impl RAGFSBindingClient { let mgr2 = mgr.clone(); let fs_ctx = build_fs_context(ctx); let lease_ref = extract_lease_ref(py, &owned_lease_ref)?; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr2.clone(); - let lr = lease_ref.clone(); - async move { - mgr.get_owned_lease_by_ref(&lr).await.ok_or_else(|| { - PathLockError::Internal(format!("lease not found for ref '{lr}'")) - }) - } - }) - .map_err(|e: PathLockError| PyRuntimeError::new_err(e.to_string()))?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr2.clone(); + let lr = lease_ref.clone(); + async move { + mgr.get_owned_lease_by_ref(&lr).await.ok_or_else(|| { + PathLockError::Internal(format!("lease not found for ref '{lr}'")) + }) + } + }) + .map_err(|e: PathLockError| PyRuntimeError::new_err(e.to_string()))?; let borrowed = mgr.as_borrowed(&lease); Python::attach(|py| borrowed_lease_to_py_dict(py, &borrowed)) } @@ -2433,23 +2299,24 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?; - let status = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let lr = lease_ref.clone(); - let ownership = ownership_ref.clone(); - async move { - let lease = mgr - .get_owned_lease_by_capability(&lr, &ownership) - .await - .ok_or_else(|| { - PathLockError::InvalidRequest(format!( - "owned lease capability does not match ref '{lr}'" - )) - })?; - mgr.refresh(&lease).await - } - }) - .map_err(pathlock_err_to_py)?; + let status = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let lr = lease_ref.clone(); + let ownership = ownership_ref.clone(); + async move { + let lease = mgr + .get_owned_lease_by_capability(&lr, &ownership) + .await + .ok_or_else(|| { + PathLockError::InvalidRequest(format!( + "owned lease capability does not match ref '{lr}'" + )) + })?; + mgr.refresh(&lease).await + } + }) + .map_err(pathlock_err_to_py)?; Ok(status) } @@ -2528,23 +2395,24 @@ impl RAGFSBindingClient { let mgr = self.clone_pathlock_manager(); let fs_ctx = build_fs_context(ctx); let (lease_ref, ownership_ref) = extract_owned_lease_ref(py, &owned_lease_ref)?; - let handoff = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let lr = lease_ref.clone(); - let ownership = ownership_ref.clone(); - async move { - let lease = mgr - .get_owned_lease_by_capability(&lr, &ownership) - .await - .ok_or_else(|| { - PathLockError::InvalidRequest(format!( - "owned lease capability does not match ref '{lr}'" - )) - })?; - Ok(mgr.to_handoff(&lease)) - } - }) - .map_err(pathlock_err_to_py)?; + let handoff = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let lr = lease_ref.clone(); + let ownership = ownership_ref.clone(); + async move { + let lease = mgr + .get_owned_lease_by_capability(&lr, &ownership) + .await + .ok_or_else(|| { + PathLockError::InvalidRequest(format!( + "owned lease capability does not match ref '{lr}'" + )) + })?; + Ok(mgr.to_handoff(&lease)) + } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| handoff_ref_to_py_dict(py, &handoff)) } @@ -2627,12 +2495,13 @@ impl RAGFSBindingClient { lock_paths, covered_paths, }; - let lease = self.run_scoped(py, fs_ctx, move || { - let mgr = mgr.clone(); - let h = handoff.clone(); - async move { mgr.adopt(&h).await } - }) - .map_err(pathlock_err_to_py)?; + let lease = self + .run_scoped(py, fs_ctx, move || { + let mgr = mgr.clone(); + let h = handoff.clone(); + async move { mgr.adopt(&h).await } + }) + .map_err(pathlock_err_to_py)?; Python::attach(|py| owned_lease_to_py_dict(py, &lease)) } @@ -2862,7 +2731,9 @@ mod tests { let _ = fs::remove_file(&rotated); let shared = Arc::new(Mutex::new(open_tracing_log_file(&active).unwrap())); - let mut writer = SharedFileWriter { file: shared.clone() }; + let mut writer = SharedFileWriter { + file: shared.clone(), + }; writer.write_all(b"before-rotate\n").unwrap(); writer.flush().unwrap(); @@ -2872,15 +2743,19 @@ mod tests { writer.write_all(b"after-rotate\n").unwrap(); writer.flush().unwrap(); - assert!(fs::read_to_string(&rotated).unwrap().contains("before-rotate")); - assert!(fs::read_to_string(&active).unwrap().contains("after-rotate")); + assert!(fs::read_to_string(&rotated) + .unwrap() + .contains("before-rotate")); + assert!(fs::read_to_string(&active) + .unwrap() + .contains("after-rotate")); let _ = fs::remove_file(&active); let _ = fs::remove_file(&rotated); } - #[tokio::test] - async fn cache_provider_factory_creates_memory_provider_from_ov_conf() { + #[test] + fn dynamic_cache_config_is_parsed_from_ov_conf() { let path = std::env::temp_dir().join(format!( "openviking-cache-config-{}.json", std::process::id() @@ -2892,9 +2767,13 @@ mod tests { "agfs": { "cache": { "enabled": true, - "provider": "memory", + "provider": "dynamic", "namespace": "ov-test", - "traversal_mode": "cached_traversal" + "traversal_mode": "cached_traversal", + "dynamic": { + "library": "/opt/openviking/libprovider.so", + "params": {"endpoint": "provider:1234"} + } } } } @@ -2903,22 +2782,23 @@ mod tests { .unwrap(); let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap(); - let provider = CacheProviderFactory::create(&cache_config).await.unwrap(); - assert!(cache_config.enabled); - assert_eq!(cache_config.provider, CacheProviderKind::Memory); + assert_eq!(cache_config.provider, CacheProviderKind::Dynamic); assert_eq!(cache_config.namespace, "ov-test"); assert_eq!( cache_config.traversal_mode, CacheTraversalMode::CachedTraversal ); - assert_eq!(provider.name(), "memory"); + assert_eq!( + cache_config.dynamic.library, + "/opt/openviking/libprovider.so" + ); fs::remove_file(path).unwrap(); } #[test] - fn missing_cache_config_defaults_to_disabled_memory_config() { + fn missing_cache_config_defaults_to_disabled_redis_config() { let path = std::env::temp_dir().join(format!( "openviking-no-cache-config-{}.json", std::process::id() @@ -2928,13 +2808,97 @@ mod tests { let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap(); assert!(!cache_config.enabled); - assert_eq!(cache_config.provider, CacheProviderKind::Memory); + assert_eq!(cache_config.provider, CacheProviderKind::Redis); assert_eq!(cache_config.namespace, "openviking"); assert_eq!(cache_config.traversal_mode, CacheTraversalMode::Backend); fs::remove_file(path).unwrap(); } + #[test] + fn queuefs_cache_runtime_does_not_enable_cachefs() { + let path = std::env::temp_dir().join(format!( + "openviking-queue-cache-config-{}.json", + std::process::id() + )); + fs::write( + &path, + r#"{ + "storage": { + "agfs": { + "cache": {"enabled": false, "provider": "redis"}, + "queuefs": {"backend": "cache"} + } + } + }"#, + ) + .unwrap(); + + let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap(); + let stack_config = cache_config.stack_config().unwrap(); + + assert!(cache_config.runtime_enabled); + assert!(!stack_config.cachefs.enabled); + + fs::remove_file(path).unwrap(); + } + + #[test] + fn queuefs_cache_runtime_rejects_redis_provider_key_prefix() { + let path = std::env::temp_dir().join(format!( + "openviking-queue-cache-prefix-config-{}.json", + std::process::id() + )); + fs::write( + &path, + r#"{ + "storage": { + "agfs": { + "cache": { + "enabled": false, + "provider": "redis", + "redis": {"key_prefix": "provider-prefix"} + }, + "queuefs": {"backend": "cache"} + } + } + }"#, + ) + .unwrap(); + + let error = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap_err(); + + assert!(error.contains("key_prefix")); + + fs::remove_file(path).unwrap(); + } + + #[test] + fn queuefs_cache_runtime_rejects_disabled_legacy_provider_name() { + let path = std::env::temp_dir().join(format!( + "openviking-queue-cache-legacy-provider-{}.json", + std::process::id() + )); + fs::write( + &path, + r#"{ + "storage": { + "agfs": { + "cache": {"enabled": false, "provider": "mooncake"}, + "queuefs": {"backend": "cache"} + } + } + }"#, + ) + .unwrap(); + + let error = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap_err(); + + assert!(error.contains("mooncake")); + + fs::remove_file(path).unwrap(); + } + #[test] fn cache_config_rejects_invalid_traversal_mode() { let path = std::env::temp_dir().join(format!( @@ -2970,8 +2934,8 @@ mod tests { Python::attach(|py| { let ty = py.get_type::(); let cache = PyDict::new(py); - cache.set_item("enabled", true).unwrap(); - cache.set_item("provider", "memory").unwrap(); + cache.set_item("enabled", false).unwrap(); + cache.set_item("provider", "redis").unwrap(); cache.set_item("namespace", "runtime-cache").unwrap(); let config = PyDict::new(py); config.set_item("cache", cache).unwrap(); @@ -3009,7 +2973,7 @@ mod tests { "pool_size": 8, "connect_timeout_ms": 1000, "command_timeout_ms": 20, - "key_prefix": "ragfs-cache", + "key_prefix": "", "default_ttl_seconds": 3600, "read_from_replica": false } @@ -3029,76 +2993,10 @@ mod tests { assert_eq!(cache_config.redis.pool_size, 8); assert_eq!(cache_config.redis.connect_timeout_ms, 1000); assert_eq!(cache_config.redis.command_timeout_ms, 20); - assert_eq!(cache_config.redis.key_prefix, "ragfs-cache"); + assert!(cache_config.redis.key_prefix.is_empty()); assert_eq!(cache_config.redis.default_ttl_seconds, 3600); assert!(!cache_config.redis.read_from_replica); fs::remove_file(path).unwrap(); } - - #[cfg(not(feature = "cache-redis"))] - #[tokio::test] - async fn redis_provider_requires_cache_redis_feature() { - let config = RagfsCacheConfig { - enabled: true, - provider: CacheProviderKind::Redis, - ..RagfsCacheConfig::default() - }; - - let error = match CacheProviderFactory::create(&config).await { - Ok(provider) => panic!("unexpected provider: {}", provider.name()), - Err(error) => error, - }; - - assert!(matches!( - error, - CacheError::Unavailable(message) - if message == "Redis support requires the cache-redis feature" - )); - } - - #[cfg(feature = "cache-redis")] - #[tokio::test] - async fn cache_provider_factory_creates_redis_provider_from_ov_conf() { - let Ok(endpoint) = std::env::var("REDIS_URL") else { - return; - }; - let path = std::env::temp_dir().join(format!( - "openviking-cache-redis-factory-{}.json", - std::process::id() - )); - fs::write( - &path, - format!( - r#"{{ - "storage": {{ - "agfs": {{ - "cache": {{ - "enabled": true, - "provider": "redis", - "namespace": "ov-test", - "redis": {{ - "mode": "standalone", - "endpoints": ["{endpoint}"], - "connect_timeout_ms": 30000, - "command_timeout_ms": 1000, - "key_prefix": "ragfs-python-cache-test", - "default_ttl_seconds": 60 - }} - }} - }} - }} - }}"# - ), - ) - .unwrap(); - - let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap(); - let provider = CacheProviderFactory::create(&cache_config).await.unwrap(); - - assert_eq!(provider.name(), "redis"); - provider.close().await.unwrap(); - - fs::remove_file(path).unwrap(); - } } diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml index c96e02bcee..7f6a0de073 100644 --- a/crates/ragfs/Cargo.toml +++ b/crates/ragfs/Cargo.toml @@ -23,6 +23,21 @@ name = "cache_wrapper" path = "tests/cache_wrapper.rs" required-features = ["cache"] +[[test]] +name = "cache_runtime" +path = "tests/cache_runtime.rs" +required-features = ["cache"] + +[[test]] +name = "redis_runtime" +path = "tests/redis_runtime.rs" +required-features = ["cache"] + +[[test]] +name = "dynamic_provider" +path = "tests/dynamic_provider.rs" +required-features = ["cache"] + [dependencies] # Async runtime tokio = { version = "1.38", features = ["full"] } @@ -84,6 +99,8 @@ lru = "0.12" regex = "1.10" mime_guess = "2.0" globset = "0.4" +url = "2.5" +libloading = { version = "0.8", optional = true } # Encryption (envelope encryption: AES-256-GCM + HKDF-SHA256) aes-gcm = "0.10" @@ -120,6 +137,6 @@ uuid = { version = "1.0", features = ["v4"] } [features] default = [] -cache = [] +cache = ["dep:libloading"] s3 = ["aws-sdk-s3", "aws-config", "aws-types"] full = ["s3", "cache"] diff --git a/crates/ragfs/src/cache/envelope.rs b/crates/ragfs/src/cache/envelope.rs index 3510fdbc8a..851bb2902e 100644 --- a/crates/ragfs/src/cache/envelope.rs +++ b/crates/ragfs/src/cache/envelope.rs @@ -1,6 +1,6 @@ //! Stable values stored behind the provider contract. -use super::{CacheError, CacheResult}; +use crate::cache_runtime::{CacheError, CacheResult}; use crate::core::FileInfo; use bytes::Bytes; use serde::{Deserialize, Serialize}; diff --git a/crates/ragfs/src/cache/memory.rs b/crates/ragfs/src/cache/memory.rs deleted file mode 100644 index 9f66730ec4..0000000000 --- a/crates/ragfs/src/cache/memory.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! In-process cache provider used by tests and smoke validation. - -use super::{CacheError, CacheProvider, CacheResult, ProviderCapabilities}; -use async_trait::async_trait; -use bytes::Bytes; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use tokio::sync::RwLock; - -/// A thread-safe in-memory implementation of [`CacheProvider`]. -pub struct MemoryCacheProvider { - values: RwLock>, - closed: AtomicBool, -} - -/// Test-oriented name for the in-process cache provider. -pub type MemoryMockProvider = MemoryCacheProvider; - -impl MemoryCacheProvider { - /// Create an empty provider. - pub fn new() -> Self { - Self { - values: RwLock::new(HashMap::new()), - closed: AtomicBool::new(false), - } - } - - fn ensure_open(&self) -> CacheResult<()> { - if self.closed.load(Ordering::Acquire) { - Err(CacheError::Unavailable( - "memory provider is closed".to_string(), - )) - } else { - Ok(()) - } - } - - /// Return the current number of stored objects. - pub async fn len(&self) -> usize { - self.values.read().await.len() - } - - /// Return whether the provider currently stores no objects. - pub async fn is_empty(&self) -> bool { - self.len().await == 0 - } - - /// Return a snapshot of stored keys for diagnostics and smoke tests. - pub async fn keys(&self) -> Vec { - self.values.read().await.keys().cloned().collect() - } -} - -impl Default for MemoryCacheProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl CacheProvider for MemoryCacheProvider { - fn name(&self) -> &'static str { - "memory" - } - - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities { - batch_get: true, - batch_put: true, - native_ttl: false, - } - } - - async fn get(&self, key: &str) -> CacheResult> { - self.ensure_open()?; - Ok(self.values.read().await.get(key).cloned()) - } - - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> { - self.ensure_open()?; - let mut values = self.values.write().await; - self.ensure_open()?; - values.insert(key.to_string(), value); - Ok(()) - } - - async fn delete(&self, key: &str) -> CacheResult<()> { - self.ensure_open()?; - let mut values = self.values.write().await; - self.ensure_open()?; - values.remove(key); - Ok(()) - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - self.ensure_open()?; - let values = self.values.read().await; - Ok(keys.iter().map(|key| values.get(key).cloned()).collect()) - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - self.ensure_open()?; - let mut values = self.values.write().await; - self.ensure_open()?; - values.extend(entries); - Ok(()) - } - - async fn invalidate(&self, keys: &[String]) -> CacheResult<()> { - self.ensure_open()?; - let mut values = self.values.write().await; - self.ensure_open()?; - for key in keys { - values.remove(key); - } - Ok(()) - } - - async fn flush(&self) -> CacheResult<()> { - self.ensure_open()?; - let mut values = self.values.write().await; - self.ensure_open()?; - values.clear(); - Ok(()) - } - - async fn close(&self) -> CacheResult<()> { - self.closed.store(true, Ordering::Release); - self.values.write().await.clear(); - Ok(()) - } -} diff --git a/crates/ragfs/src/cache/mod.rs b/crates/ragfs/src/cache/mod.rs index 2c58ea3962..6c3ca10e45 100644 --- a/crates/ragfs/src/cache/mod.rs +++ b/crates/ragfs/src/cache/mod.rs @@ -5,14 +5,10 @@ //! [`CachedFileSystem`]. mod envelope; -mod memory; mod metrics; mod policy; -mod provider; mod wrapper; -pub use memory::{MemoryCacheProvider, MemoryMockProvider}; pub use metrics::{CacheMetrics, CacheMetricsSnapshot}; pub use policy::{CacheDecision, CachePolicy, CacheTraversalMode, CacheTreeMode}; -pub use provider::{CacheError, CacheProvider, CacheResult, ProviderCapabilities}; pub use wrapper::{CacheNamespace, CachedFileSystem}; diff --git a/crates/ragfs/src/cache/provider.rs b/crates/ragfs/src/cache/provider.rs deleted file mode 100644 index 183a535b89..0000000000 --- a/crates/ragfs/src/cache/provider.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Provider-independent cache primitives. - -use async_trait::async_trait; -use bytes::Bytes; - -/// Result type returned by cache providers. -pub type CacheResult = std::result::Result; - -/// Errors shared by all cache providers. -#[derive(Debug, thiserror::Error)] -pub enum CacheError { - /// The provider is unavailable or has been closed. - #[error("cache provider unavailable: {0}")] - Unavailable(String), - - /// A provider operation timed out. - #[error("cache provider operation timed out: {0}")] - Timeout(String), - - /// A cache value could not be decoded or validated. - #[error("invalid cache data: {0}")] - InvalidData(String), - - /// A provider rejected an argument. - #[error("invalid cache argument: {0}")] - InvalidArgument(String), - - /// An unspecified provider failure occurred. - #[error("cache provider internal error: {0}")] - Internal(String), -} - -/// Optional features exposed by a provider. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct ProviderCapabilities { - /// Whether batch reads have a native implementation. - pub batch_get: bool, - /// Whether batch writes have a native implementation. - pub batch_put: bool, - /// Whether the provider supports native expiration. - pub native_ttl: bool, -} - -/// Minimal contract implemented by local and remote cache providers. -#[async_trait] -pub trait CacheProvider: Send + Sync { - /// Return a stable provider name for diagnostics. - fn name(&self) -> &'static str; - - /// Return optional provider capabilities. - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities::default() - } - - /// Read one cache object. - async fn get(&self, key: &str) -> CacheResult>; - - /// Write one cache object. - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()>; - - /// Delete one cache object. Missing keys are treated as success. - async fn delete(&self, key: &str) -> CacheResult<()>; - - /// Check whether one cache object exists. - async fn exists(&self, key: &str) -> CacheResult { - Ok(self.get(key).await?.is_some()) - } - - /// Read multiple objects while preserving input order. - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - let mut values = Vec::with_capacity(keys.len()); - for key in keys { - values.push(self.get(key).await?); - } - Ok(values) - } - - /// Write multiple objects. - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - for (key, value) in entries { - self.put(&key, value).await?; - } - Ok(()) - } - - /// Invalidate a known set of cache keys. - async fn invalidate(&self, keys: &[String]) -> CacheResult<()> { - for key in keys { - self.delete(key).await?; - } - Ok(()) - } - - /// Remove all objects owned by this provider instance. - async fn flush(&self) -> CacheResult<()> { - Ok(()) - } - - /// Release provider resources. - async fn close(&self) -> CacheResult<()> { - Ok(()) - } -} diff --git a/crates/ragfs/src/cache/wrapper.rs b/crates/ragfs/src/cache/wrapper.rs index 300912a3cf..3e93b0eca5 100644 --- a/crates/ragfs/src/cache/wrapper.rs +++ b/crates/ragfs/src/cache/wrapper.rs @@ -1,16 +1,15 @@ //! A transparent [`FileSystem`](crate::FileSystem) cache wrapper. use super::envelope::{CacheEnvelope, CacheObjectKind, GenerationSnapshot}; -use super::{ - CacheError, CacheMetrics, CachePolicy, CacheProvider, CacheResult, CacheTraversalMode, -}; +use super::{CacheMetrics, CachePolicy, CacheTraversalMode}; +use crate::cache_runtime::{AsyncCacheRuntime, CacheError, CacheResult, CacheRuntime, PutOptions}; use crate::core::filesystem::{ compile_grep_regex, is_excluded_path, normalize_prefix_path, relative_depth, - relative_match_file, + relative_match_file, sort_directory_entries, }; use crate::core::{ - FileInfo, FileSystem, GlobPage, GrepMatch, GrepResult, MultiWriteWrappedFS, Result, - TreeEntry, WriteFlag, + FileInfo, FileSystem, GlobPage, GrepMatch, GrepResult, MultiWriteWrappedFS, Result, TreeEntry, + WriteFlag, }; use async_trait::async_trait; use bytes::Bytes; @@ -50,7 +49,7 @@ impl CacheNamespace { /// default mount path, so existing filesystem behavior remains unchanged. pub struct CachedFileSystem { backend: Box, - provider: Arc, + runtime: Arc, namespace: CacheNamespace, policy: CachePolicy, metrics: Arc, @@ -62,16 +61,25 @@ pub struct CachedFileSystem { } impl CachedFileSystem { - /// Wrap an existing backend with a cache provider. - pub fn new( + /// Wrap an existing backend with the unified cache runtime. + pub fn with_runtime( + backend: Box, + runtime: Arc, + namespace: CacheNamespace, + policy: CachePolicy, + ) -> Self { + Self::build(backend, runtime, namespace, policy) + } + + fn build( backend: Box, - provider: Arc, + runtime: Arc, namespace: CacheNamespace, policy: CachePolicy, ) -> Self { Self { backend, - provider, + runtime, namespace, policy, metrics: Arc::new(CacheMetrics::default()), @@ -88,9 +96,9 @@ impl CachedFileSystem { Arc::clone(&self.metrics) } - /// Return the provider used by this wrapper. - pub fn provider(&self) -> Arc { - Arc::clone(&self.provider) + /// Return the unified cache runtime used by this wrapper. + pub fn runtime(&self) -> Arc { + Arc::clone(&self.runtime) } /// Return the wrapped filesystem for mount-stack capability discovery. @@ -140,7 +148,8 @@ impl CachedFileSystem { } } - let entries = self.read_dir(¤t_path).await?; + let mut entries = self.read_dir(¤t_path).await?; + sort_directory_entries(&mut entries); for entry in entries.into_iter().rev() { let is_hidden_file = !entry.is_dir && entry.name.starts_with('.'); if is_hidden_file && !show_hidden { @@ -368,7 +377,7 @@ impl CachedFileSystem { async fn cache_get(&self, key: &str) -> CacheResult> { let started = Instant::now(); - let result = self.provider.get(key).await; + let result = self.runtime.get(key).await; self.metrics.get(started.elapsed()); result } @@ -378,7 +387,7 @@ impl CachedFileSystem { return Ok(Vec::new()); } - if keys.len() == 1 || !self.provider.capabilities().batch_get { + if keys.len() == 1 { let mut values = Vec::with_capacity(keys.len()); for key in keys { values.push(self.cache_get(key).await?); @@ -387,7 +396,7 @@ impl CachedFileSystem { } let started = Instant::now(); - let result = self.provider.batch_get(keys).await; + let result = self.runtime.batch_get(keys).await; self.metrics.get(started.elapsed()); let values = result?; if values.len() != keys.len() { @@ -402,7 +411,7 @@ impl CachedFileSystem { async fn cache_put(&self, key: &str, value: Bytes, affected_path: &str) -> bool { let started = Instant::now(); - let result = self.provider.put(key, value).await; + let result = self.runtime.put(key, value, PutOptions::default()).await; self.metrics.put(started.elapsed()); match result { Ok(()) => true, @@ -416,7 +425,7 @@ impl CachedFileSystem { async fn cache_delete(&self, key: &str, affected_path: &str) { let started = Instant::now(); - let result = self.provider.delete(key).await; + let result = self.runtime.delete(key).await; self.metrics.delete(started.elapsed()); match result { Ok(()) => self.metrics.invalidation(), @@ -540,13 +549,13 @@ impl CachedFileSystem { return; } - if missing.len() > 1 && self.provider.capabilities().batch_put { + if missing.len() > 1 { let entries = missing .into_iter() .map(|(key, value)| (key, Bytes::copy_from_slice(&value.to_be_bytes()))) .collect(); let started = Instant::now(); - if self.provider.batch_put(entries).await.is_err() { + if self.runtime.batch_put(entries).await.is_err() { self.metrics.error(); } self.metrics.put(started.elapsed()); @@ -556,8 +565,12 @@ impl CachedFileSystem { for (key, value) in missing { let started = Instant::now(); if self - .provider - .put(&key, Bytes::copy_from_slice(&value.to_be_bytes())) + .runtime + .put( + &key, + Bytes::copy_from_slice(&value.to_be_bytes()), + PutOptions::default(), + ) .await .is_err() { @@ -1066,7 +1079,10 @@ impl FileSystem for CachedFileSystem { let key = self.file_key(&normalized); self.cache_delete(&key, &normalized).await; if offset == 0 - && matches!(flags, WriteFlag::Create | WriteFlag::CreateNew | WriteFlag::Truncate) + && matches!( + flags, + WriteFlag::Create | WriteFlag::CreateNew | WriteFlag::Truncate + ) && self.policy.cache_file(&normalized, data.len()) && !self.is_runtime_bypassed(&normalized).await { diff --git a/crates/ragfs/src/cache_runtime/api.rs b/crates/ragfs/src/cache_runtime/api.rs new file mode 100644 index 0000000000..35a91175b8 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/api.rs @@ -0,0 +1,54 @@ +//! Public CacheRuntime operation types and interfaces. + +use super::{CacheResult, ScriptRequest, ScriptResult}; +use async_trait::async_trait; +use bytes::Bytes; +use std::time::Duration; + +/// Options applied to one cache write. +#[derive(Debug, Clone, Copy, Default)] +pub struct PutOptions { + /// Optional provider-side expiration. + pub ttl: Option, +} + +/// Asynchronous primitive cache interface. +#[async_trait] +pub trait AsyncCacheRuntime: Send + Sync { + /// Read one value. + async fn get(&self, key: &str) -> CacheResult>; + /// Write one value. + async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()>; + /// Delete one value. + async fn delete(&self, key: &str) -> CacheResult<()>; + /// Check whether one value exists. + async fn exists(&self, key: &str) -> CacheResult; + /// Read multiple values while preserving input order. + async fn batch_get(&self, keys: &[String]) -> CacheResult>>; + /// Write multiple values. + async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()>; + /// Delete multiple values. + async fn batch_delete(&self, keys: &[String]) -> CacheResult<()>; + /// Execute one provider-specific named atomic program. + async fn execute_script(&self, request: ScriptRequest) -> CacheResult; +} + +/// Synchronous facade over the same CacheRuntime instance. +pub trait SyncCacheRuntime: Send + Sync { + /// Read one value. + fn get(&self, key: &str) -> CacheResult>; + /// Write one value. + fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()>; + /// Delete one value. + fn delete(&self, key: &str) -> CacheResult<()>; + /// Check whether one value exists. + fn exists(&self, key: &str) -> CacheResult; + /// Read multiple values while preserving input order. + fn batch_get(&self, keys: &[String]) -> CacheResult>>; + /// Write multiple values. + fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()>; + /// Delete multiple values. + fn batch_delete(&self, keys: &[String]) -> CacheResult<()>; + /// Execute one provider-specific named atomic program. + fn execute_script(&self, request: ScriptRequest) -> CacheResult; +} diff --git a/crates/ragfs/src/cache_runtime/dynamic/abi.rs b/crates/ragfs/src/cache_runtime/dynamic/abi.rs new file mode 100644 index 0000000000..07721f4e5f --- /dev/null +++ b/crates/ragfs/src/cache_runtime/dynamic/abi.rs @@ -0,0 +1,93 @@ +use std::ffi::{c_char, c_void}; + +pub(super) const ABI_VERSION_V1: u32 = 1; +pub(super) const STATUS_OK: i32 = 0; +pub(super) const STATUS_NOT_FOUND: i32 = 1; + +#[repr(C)] +#[derive(Clone, Copy)] +pub(super) struct OvSlice { + pub(super) ptr: *const u8, + pub(super) len: usize, +} + +impl OvSlice { + pub(super) fn new(value: &[u8]) -> Self { + Self { + ptr: value.as_ptr(), + len: value.len(), + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(super) struct OvBuffer { + pub(super) ptr: *mut u8, + pub(super) len: usize, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(super) struct OvPutOptions { + pub(super) ttl_ms: u64, + pub(super) has_ttl: u8, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(super) struct OvEntry { + pub(super) key: OvSlice, + pub(super) value: OvSlice, + pub(super) ttl_ms: u64, + pub(super) has_ttl: u8, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub(super) struct OvScriptRequest { + pub(super) script_id: OvSlice, + pub(super) keys: *const OvSlice, + pub(super) key_count: usize, + pub(super) args: *const OvSlice, + pub(super) arg_count: usize, +} + +pub(super) type InitFn = unsafe extern "C" fn(OvSlice, *mut *mut c_void) -> i32; +pub(super) type GetFn = unsafe extern "C" fn(*mut c_void, OvSlice, *mut OvBuffer) -> i32; +pub(super) type PutFn = + unsafe extern "C" fn(*mut c_void, OvSlice, OvSlice, *const OvPutOptions) -> i32; +pub(super) type DeleteFn = unsafe extern "C" fn(*mut c_void, OvSlice) -> i32; +pub(super) type ExistsFn = unsafe extern "C" fn(*mut c_void, OvSlice, *mut u8) -> i32; +pub(super) type BatchGetFn = + unsafe extern "C" fn(*mut c_void, *const OvSlice, usize, *mut OvBuffer) -> i32; +pub(super) type BatchPutFn = unsafe extern "C" fn(*mut c_void, *const OvEntry, usize) -> i32; +pub(super) type BatchDeleteFn = unsafe extern "C" fn(*mut c_void, *const OvSlice, usize) -> i32; +pub(super) type ExecuteScriptFn = + unsafe extern "C" fn(*mut c_void, *const OvScriptRequest, *mut OvBuffer) -> i32; +pub(super) type HealthFn = unsafe extern "C" fn(*mut c_void) -> i32; +pub(super) type FreeBufferFn = unsafe extern "C" fn(*mut OvBuffer); +pub(super) type CloseFn = unsafe extern "C" fn(*mut c_void); +pub(super) type LastErrorFn = unsafe extern "C" fn(*mut c_void) -> *const c_char; + +#[repr(C)] +#[derive(Clone, Copy)] +pub(super) struct OvCacheProviderV1 { + pub(super) abi_version: u32, + pub(super) struct_size: u32, + pub(super) init: Option, + pub(super) get: Option, + pub(super) put: Option, + pub(super) delete_key: Option, + pub(super) exists: Option, + pub(super) batch_get: Option, + pub(super) batch_put: Option, + pub(super) batch_delete: Option, + pub(super) execute_script: Option, + pub(super) health: Option, + pub(super) free_buffer: Option, + pub(super) close: Option, + pub(super) last_error: Option, +} + +pub(super) type ProviderEntryV1 = unsafe extern "C" fn() -> *const OvCacheProviderV1; diff --git a/crates/ragfs/src/cache_runtime/dynamic/config.rs b/crates/ragfs/src/cache_runtime/dynamic/config.rs new file mode 100644 index 0000000000..ca58d7c837 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/dynamic/config.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +/// Configuration for one dynamically loaded cache provider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DynamicProviderConfig { + /// Absolute or process-resolvable path to the provider library. + pub library_path: PathBuf, + /// Provider-owned JSON configuration passed unchanged to `init`. + pub params_json: String, +} diff --git a/crates/ragfs/src/cache_runtime/dynamic/mod.rs b/crates/ragfs/src/cache_runtime/dynamic/mod.rs new file mode 100644 index 0000000000..24e850c56a --- /dev/null +++ b/crates/ragfs/src/cache_runtime/dynamic/mod.rs @@ -0,0 +1,6 @@ +mod abi; +mod config; +mod provider; + +pub use config::DynamicProviderConfig; +pub(crate) use provider::DynamicProvider; diff --git a/crates/ragfs/src/cache_runtime/dynamic/provider.rs b/crates/ragfs/src/cache_runtime/dynamic/provider.rs new file mode 100644 index 0000000000..15f2e89ebf --- /dev/null +++ b/crates/ragfs/src/cache_runtime/dynamic/provider.rs @@ -0,0 +1,368 @@ +use super::abi::*; +use super::DynamicProviderConfig; +use crate::cache_runtime::provider::CacheProvider; +use crate::cache_runtime::{CacheError, CacheResult, PutOptions, ScriptRequest, ScriptResult}; +use async_trait::async_trait; +use bytes::Bytes; +use libloading::Library; +use std::ffi::{c_void, CStr}; +use std::mem::size_of; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use tokio::sync::Semaphore; + +const MAX_INFLIGHT: u32 = 1024; + +pub(crate) struct DynamicProvider { + inner: Arc, +} + +struct DynamicProviderInner { + _library: Library, + api: OvCacheProviderV1, + handle: NonNull, + closed: AtomicBool, + inflight: Arc, +} + +unsafe impl Send for DynamicProviderInner {} +unsafe impl Sync for DynamicProviderInner {} + +impl DynamicProvider { + pub(crate) async fn connect(config: DynamicProviderConfig) -> CacheResult { + let inner = tokio::task::spawn_blocking(move || unsafe { load(config) }) + .await + .map_err(|error| { + CacheError::Internal(format!("dynamic provider init failed: {error}")) + })??; + Ok(Self { + inner: Arc::new(inner), + }) + } + + async fn call(&self, operation: &'static str, call: F) -> CacheResult + where + T: Send + 'static, + F: FnOnce(Arc) -> CacheResult + Send + 'static, + { + if self.inner.closed.load(Ordering::Acquire) { + return Err(CacheError::Closed); + } + let permit = Arc::clone(&self.inner.inflight) + .acquire_owned() + .await + .map_err(|_| CacheError::Closed)?; + if self.inner.closed.load(Ordering::Acquire) { + return Err(CacheError::Closed); + } + let inner = Arc::clone(&self.inner); + tokio::task::spawn_blocking(move || { + let _permit = permit; + call(inner) + }) + .await + .map_err(|error| CacheError::Internal(format!("dynamic {operation} failed: {error}")))? + } +} + +#[async_trait] +impl CacheProvider for DynamicProvider { + async fn get(&self, key: &str) -> CacheResult> { + let key = key.as_bytes().to_vec(); + self.call("get", move |inner| unsafe { + let mut buffer = OvBuffer::default(); + let status = required(inner.api.get, "get")?( + inner.handle.as_ptr(), + OvSlice::new(&key), + &mut buffer, + ); + if status == STATUS_NOT_FOUND { + return Ok(None); + } + status_ok(&inner, "get", status)?; + take_buffer(&inner, buffer).map(Some) + }) + .await + } + + async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { + let key = key.as_bytes().to_vec(); + self.call("put", move |inner| unsafe { + let ttl_ms = options + .ttl + .map(|ttl| { + u64::try_from(ttl.as_millis()).map_err(|_| { + CacheError::InvalidArgument("dynamic provider TTL is too large".into()) + }) + }) + .transpose()?; + let options = OvPutOptions { + ttl_ms: ttl_ms.unwrap_or_default(), + has_ttl: u8::from(ttl_ms.is_some()), + }; + let status = required(inner.api.put, "put")?( + inner.handle.as_ptr(), + OvSlice::new(&key), + OvSlice::new(&value), + &options, + ); + status_ok(&inner, "put", status) + }) + .await + } + + async fn delete(&self, key: &str) -> CacheResult<()> { + let key = key.as_bytes().to_vec(); + self.call("delete", move |inner| unsafe { + let status = required(inner.api.delete_key, "delete")?( + inner.handle.as_ptr(), + OvSlice::new(&key), + ); + status_ok(&inner, "delete", status) + }) + .await + } + + async fn exists(&self, key: &str) -> CacheResult { + let key = key.as_bytes().to_vec(); + self.call("exists", move |inner| unsafe { + let mut exists = 0; + let status = required(inner.api.exists, "exists")?( + inner.handle.as_ptr(), + OvSlice::new(&key), + &mut exists, + ); + status_ok(&inner, "exists", status)?; + Ok(exists != 0) + }) + .await + } + + async fn batch_get(&self, keys: &[String]) -> CacheResult>> { + let keys = keys.to_vec(); + self.call("batch_get", move |inner| unsafe { + let slices = keys + .iter() + .map(|key| OvSlice::new(key.as_bytes())) + .collect::>(); + let mut buffer = OvBuffer::default(); + let status = required(inner.api.batch_get, "batch_get")?( + inner.handle.as_ptr(), + slices.as_ptr(), + slices.len(), + &mut buffer, + ); + status_ok(&inner, "batch_get", status)?; + let payload = take_buffer(&inner, buffer)?; + serde_json::from_slice::>>>(&payload) + .map(|values| { + values + .into_iter() + .map(|value| value.map(Bytes::from)) + .collect() + }) + .map_err(|error| CacheError::InvalidData(error.to_string())) + }) + .await + } + + async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + self.call("batch_put", move |inner| unsafe { + let ffi_entries = entries + .iter() + .map(|(key, value)| OvEntry { + key: OvSlice::new(key.as_bytes()), + value: OvSlice::new(value), + ttl_ms: 0, + has_ttl: 0, + }) + .collect::>(); + let status = required(inner.api.batch_put, "batch_put")?( + inner.handle.as_ptr(), + ffi_entries.as_ptr(), + ffi_entries.len(), + ); + status_ok(&inner, "batch_put", status) + }) + .await + } + + async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + let keys = keys.to_vec(); + self.call("batch_delete", move |inner| unsafe { + let slices = keys + .iter() + .map(|key| OvSlice::new(key.as_bytes())) + .collect::>(); + let status = required(inner.api.batch_delete, "batch_delete")?( + inner.handle.as_ptr(), + slices.as_ptr(), + slices.len(), + ); + status_ok(&inner, "batch_delete", status) + }) + .await + } + + async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + self.call("execute_script", move |inner| unsafe { + let key_slices = request + .keys + .iter() + .map(|key| OvSlice::new(key.as_bytes())) + .collect::>(); + let arg_slices = request + .args + .iter() + .map(|arg| OvSlice::new(arg)) + .collect::>(); + let ffi_request = OvScriptRequest { + script_id: OvSlice::new(request.script_id.as_bytes()), + keys: key_slices.as_ptr(), + key_count: key_slices.len(), + args: arg_slices.as_ptr(), + arg_count: arg_slices.len(), + }; + let mut buffer = OvBuffer::default(); + let status = required(inner.api.execute_script, "execute_script")?( + inner.handle.as_ptr(), + &ffi_request, + &mut buffer, + ); + status_ok(&inner, "execute_script", status)?; + take_buffer(&inner, buffer).map(|payload| ScriptResult { payload }) + }) + .await + } + + async fn close(&self) -> CacheResult<()> { + if self.inner.closed.swap(true, Ordering::AcqRel) { + return Ok(()); + } + let permits = Arc::clone(&self.inner.inflight) + .acquire_many_owned(MAX_INFLIGHT) + .await + .map_err(|_| CacheError::Closed)?; + let inner = Arc::clone(&self.inner); + tokio::task::spawn_blocking(move || unsafe { + required(inner.api.close, "close").map(|close| close(inner.handle.as_ptr())) + }) + .await + .map_err(|error| CacheError::Internal(format!("dynamic close failed: {error}")))??; + drop(permits); + Ok(()) + } +} + +impl Drop for DynamicProviderInner { + fn drop(&mut self) { + if !self.closed.swap(true, Ordering::AcqRel) { + if let Some(close) = self.api.close { + unsafe { close(self.handle.as_ptr()) }; + } + } + } +} + +unsafe fn load(config: DynamicProviderConfig) -> CacheResult { + let library = Library::new(&config.library_path).map_err(|error| { + CacheError::AbiMismatch(format!( + "failed to load {}: {error}", + config.library_path.display() + )) + })?; + let entry = library + .get::(b"openviking_cache_provider_v1\0") + .map_err(|error| CacheError::AbiMismatch(format!("missing provider entry: {error}")))?; + let api_ptr = entry(); + let api = api_ptr + .as_ref() + .copied() + .ok_or_else(|| CacheError::AbiMismatch("provider entry returned null".into()))?; + if api.abi_version != ABI_VERSION_V1 { + return Err(CacheError::AbiMismatch(format!( + "expected ABI {}, got {}", + ABI_VERSION_V1, api.abi_version + ))); + } + if api.struct_size as usize != size_of::() { + return Err(CacheError::AbiMismatch(format!( + "expected struct size {}, got {}", + size_of::(), + api.struct_size + ))); + } + validate_api(&api)?; + let mut handle = std::ptr::null_mut(); + let status = + required(api.init, "init")?(OvSlice::new(config.params_json.as_bytes()), &mut handle); + let handle = NonNull::new(handle) + .ok_or_else(|| CacheError::Unavailable("dynamic provider init returned null".into()))?; + let inner = DynamicProviderInner { + _library: library, + api, + handle, + closed: AtomicBool::new(false), + inflight: Arc::new(Semaphore::new(MAX_INFLIGHT as usize)), + }; + status_ok(&inner, "init", status)?; + let health = required(inner.api.health, "health")?(inner.handle.as_ptr()); + status_ok(&inner, "health", health)?; + Ok(inner) +} + +fn validate_api(api: &OvCacheProviderV1) -> CacheResult<()> { + required(api.init, "init")?; + required(api.get, "get")?; + required(api.put, "put")?; + required(api.delete_key, "delete")?; + required(api.exists, "exists")?; + required(api.batch_get, "batch_get")?; + required(api.batch_put, "batch_put")?; + required(api.batch_delete, "batch_delete")?; + required(api.execute_script, "execute_script")?; + required(api.health, "health")?; + required(api.free_buffer, "free_buffer")?; + required(api.close, "close")?; + required(api.last_error, "last_error")?; + Ok(()) +} + +fn required(function: Option, name: &str) -> CacheResult { + function.ok_or_else(|| CacheError::AbiMismatch(format!("missing function {name}"))) +} + +fn status_ok(inner: &DynamicProviderInner, operation: &str, status: i32) -> CacheResult<()> { + if status == STATUS_OK { + return Ok(()); + } + let last_error = required(inner.api.last_error, "last_error")?; + let error_ptr = unsafe { last_error(inner.handle.as_ptr()) }; + let message = if error_ptr.is_null() { + format!("status {status}") + } else { + unsafe { CStr::from_ptr(error_ptr) } + .to_string_lossy() + .into_owned() + }; + Err(CacheError::Unavailable(format!( + "dynamic provider {operation} failed: {message}" + ))) +} + +unsafe fn take_buffer(inner: &DynamicProviderInner, mut buffer: OvBuffer) -> CacheResult { + let result = if buffer.len == 0 { + Ok(Bytes::new()) + } else if buffer.ptr.is_null() { + Err(CacheError::InvalidData( + "dynamic provider returned a null buffer".into(), + )) + } else { + Ok(Bytes::copy_from_slice(std::slice::from_raw_parts( + buffer.ptr, buffer.len, + ))) + }; + required(inner.api.free_buffer, "free_buffer")?(&mut buffer); + result +} diff --git a/crates/ragfs/src/cache_runtime/error.rs b/crates/ragfs/src/cache_runtime/error.rs new file mode 100644 index 0000000000..3adf664605 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/error.rs @@ -0,0 +1,36 @@ +//! Errors returned by the unified cache runtime. + +/// Result type returned by CacheRuntime operations. +pub type CacheResult = std::result::Result; + +/// Provider-independent cache runtime error. +#[derive(Debug, thiserror::Error)] +pub enum CacheError { + /// The configured provider cannot be reached. + #[error("cache provider unavailable: {0}")] + Unavailable(String), + /// A provider operation exceeded its deadline. + #[error("cache provider operation timed out: {0}")] + Timeout(String), + /// A provider returned malformed data. + #[error("invalid cache data: {0}")] + InvalidData(String), + /// A caller supplied an invalid argument. + #[error("invalid cache argument: {0}")] + InvalidArgument(String), + /// The selected provider does not implement a named script. + #[error("unsupported cache script: {0}")] + UnsupportedScript(String), + /// A dynamic provider uses an incompatible ABI. + #[error("cache provider ABI mismatch: {0}")] + AbiMismatch(String), + /// The runtime has already been closed. + #[error("cache runtime is closed")] + Closed, + /// A synchronous facade was called from an asynchronous runtime thread. + #[error("synchronous cache calls are not allowed inside a Tokio runtime")] + InvalidExecutionContext, + /// An internal provider or executor failure occurred. + #[error("cache runtime internal error: {0}")] + Internal(String), +} diff --git a/crates/ragfs/src/cache_runtime/executor.rs b/crates/ragfs/src/cache_runtime/executor.rs new file mode 100644 index 0000000000..b261526a9e --- /dev/null +++ b/crates/ragfs/src/cache_runtime/executor.rs @@ -0,0 +1,63 @@ +//! Synchronous execution bridge for CacheRuntime. + +use super::{CacheError, CacheResult}; +use std::future::Future; +use std::sync::mpsc; + +type Job = Box; + +pub(crate) struct RuntimeExecutor { + sender: mpsc::Sender, +} + +impl RuntimeExecutor { + pub(crate) fn new() -> CacheResult { + let (sender, receiver) = mpsc::channel::(); + let (ready_sender, ready_receiver) = mpsc::sync_channel(1); + std::thread::Builder::new() + .name("ragfs-cache-runtime".into()) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build(); + match runtime { + Ok(runtime) => { + let _ = ready_sender.send(Ok(())); + while let Ok(job) = receiver.recv() { + job(&runtime); + } + } + Err(error) => { + let _ = ready_sender.send(Err(error.to_string())); + } + } + }) + .map_err(|error| CacheError::Internal(error.to_string()))?; + ready_receiver + .recv() + .map_err(|_| CacheError::Internal("cache runtime executor failed to start".into()))? + .map_err(CacheError::Internal)?; + Ok(Self { sender }) + } + + pub(crate) fn run(&self, future: F) -> CacheResult + where + T: Send + 'static, + F: Future> + Send + 'static, + { + if tokio::runtime::Handle::try_current().is_ok() { + return Err(CacheError::InvalidExecutionContext); + } + + let (result_sender, result_receiver) = mpsc::sync_channel(1); + self.sender + .send(Box::new(move |runtime| { + let _ = result_sender.send(runtime.block_on(future)); + })) + .map_err(|_| CacheError::Closed)?; + result_receiver + .recv() + .map_err(|_| CacheError::Internal("cache runtime executor stopped".into()))? + } +} diff --git a/crates/ragfs/src/cache_runtime/memory.rs b/crates/ragfs/src/cache_runtime/memory.rs new file mode 100644 index 0000000000..ea1634c665 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/memory.rs @@ -0,0 +1,212 @@ +//! In-process provider used by Runtime tests and smoke validation. + +use super::provider::CacheProvider; +use super::{CacheError, CacheResult, PutOptions}; +use async_trait::async_trait; +use bytes::Bytes; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::Duration; +use tokio::sync::RwLock; + +/// Controllable in-memory provider for tests and smoke validation. +pub struct MemoryMockProvider { + values: RwLock>, + closed: AtomicBool, + unavailable: AtomicBool, + delete_failure: AtomicBool, + gets: AtomicU64, + batch_gets: AtomicU64, + active_gets: AtomicU64, + max_active_gets: AtomicU64, + seen_get_keys: Mutex>, + seen_batch_get_keys: Mutex>>, + get_delay: Duration, +} + +impl MemoryMockProvider { + /// Create an empty provider. + pub fn new() -> Self { + Self { + values: RwLock::new(HashMap::new()), + closed: AtomicBool::new(false), + unavailable: AtomicBool::new(false), + delete_failure: AtomicBool::new(false), + gets: AtomicU64::new(0), + batch_gets: AtomicU64::new(0), + active_gets: AtomicU64::new(0), + max_active_gets: AtomicU64::new(0), + seen_get_keys: Mutex::new(Vec::new()), + seen_batch_get_keys: Mutex::new(Vec::new()), + get_delay: Duration::ZERO, + } + } + + /// Delay individual get calls to exercise inflight and concurrency behavior. + pub fn with_get_delay(mut self, delay: Duration) -> Self { + self.get_delay = delay; + self + } + + /// Make all provider operations fail or recover them again. + pub fn set_unavailable(&self, unavailable: bool) { + self.unavailable.store(unavailable, Ordering::Release); + } + + /// Make delete operations fail or recover them again. + pub fn set_delete_failure(&self, fail: bool) { + self.delete_failure.store(fail, Ordering::Release); + } + + /// Return the current number of stored objects. + pub async fn len(&self) -> usize { + self.values.read().await.len() + } + + /// Return whether the provider currently stores no objects. + pub async fn is_empty(&self) -> bool { + self.len().await == 0 + } + + /// Return a snapshot of stored keys. + pub async fn keys(&self) -> Vec { + self.values.read().await.keys().cloned().collect() + } + + /// Reset observed read calls and concurrency counters. + pub fn reset_observed_reads(&self) { + self.gets.store(0, Ordering::Relaxed); + self.batch_gets.store(0, Ordering::Relaxed); + self.active_gets.store(0, Ordering::Relaxed); + self.max_active_gets.store(0, Ordering::Relaxed); + self.seen_get_keys.lock().unwrap().clear(); + self.seen_batch_get_keys.lock().unwrap().clear(); + } + + /// Return the number of batch_get calls since the last reset. + pub fn batch_get_count(&self) -> u64 { + self.batch_gets.load(Ordering::Relaxed) + } + + /// Return all keys observed by get and batch_get calls. + pub fn observed_read_keys(&self) -> Vec { + let mut keys = self.seen_get_keys.lock().unwrap().clone(); + keys.extend( + self.seen_batch_get_keys + .lock() + .unwrap() + .iter() + .flat_map(|batch| batch.iter().cloned()), + ); + keys + } + + /// Return the maximum number of concurrent get calls since the last reset. + pub fn max_concurrent_gets(&self) -> u64 { + self.max_active_gets.load(Ordering::Relaxed) + } + + fn ensure_open(&self) -> CacheResult<()> { + if self.closed.load(Ordering::Acquire) { + Err(CacheError::Unavailable( + "memory provider is closed".to_string(), + )) + } else if self.unavailable.load(Ordering::Acquire) { + Err(CacheError::Unavailable( + "memory provider is unavailable".to_string(), + )) + } else { + Ok(()) + } + } + + fn enter_get(&self) { + let active = self.active_gets.fetch_add(1, Ordering::Relaxed) + 1; + let mut current = self.max_active_gets.load(Ordering::Relaxed); + while active > current { + match self.max_active_gets.compare_exchange_weak( + current, + active, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(observed) => current = observed, + } + } + } +} + +impl Default for MemoryMockProvider { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl CacheProvider for MemoryMockProvider { + async fn get(&self, key: &str) -> CacheResult> { + self.ensure_open()?; + self.gets.fetch_add(1, Ordering::Relaxed); + self.seen_get_keys.lock().unwrap().push(key.to_string()); + self.enter_get(); + if !self.get_delay.is_zero() { + tokio::time::sleep(self.get_delay).await; + } + let value = self.values.read().await.get(key).cloned(); + self.active_gets.fetch_sub(1, Ordering::Relaxed); + Ok(value) + } + + async fn put(&self, key: &str, value: Bytes, _options: PutOptions) -> CacheResult<()> { + self.ensure_open()?; + self.values.write().await.insert(key.to_string(), value); + Ok(()) + } + + async fn delete(&self, key: &str) -> CacheResult<()> { + self.ensure_open()?; + if self.delete_failure.load(Ordering::Acquire) { + return Err(CacheError::Unavailable( + "memory provider delete intentionally failed".to_string(), + )); + } + self.values.write().await.remove(key); + Ok(()) + } + + async fn batch_get(&self, keys: &[String]) -> CacheResult>> { + self.ensure_open()?; + self.batch_gets.fetch_add(1, Ordering::Relaxed); + self.seen_batch_get_keys.lock().unwrap().push(keys.to_vec()); + let values = self.values.read().await; + Ok(keys.iter().map(|key| values.get(key).cloned()).collect()) + } + + async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + self.ensure_open()?; + self.values.write().await.extend(entries); + Ok(()) + } + + async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + self.ensure_open()?; + if self.delete_failure.load(Ordering::Acquire) { + return Err(CacheError::Unavailable( + "memory provider delete intentionally failed".to_string(), + )); + } + let mut values = self.values.write().await; + for key in keys { + values.remove(key); + } + Ok(()) + } + + async fn close(&self) -> CacheResult<()> { + self.closed.store(true, Ordering::Release); + self.values.write().await.clear(); + Ok(()) + } +} diff --git a/crates/ragfs/src/cache_runtime/mod.rs b/crates/ragfs/src/cache_runtime/mod.rs new file mode 100644 index 0000000000..e1db4028fd --- /dev/null +++ b/crates/ragfs/src/cache_runtime/mod.rs @@ -0,0 +1,230 @@ +//! Unified provider-independent cache runtime. + +mod api; +mod dynamic; +mod error; +mod executor; +mod memory; +mod provider; +mod redis; +mod script; + +pub use api::{AsyncCacheRuntime, PutOptions, SyncCacheRuntime}; +pub use dynamic::DynamicProviderConfig; +pub use error::{CacheError, CacheResult}; +pub use memory::MemoryMockProvider; +pub use redis::RedisProviderConfig; + +use async_trait::async_trait; +use bytes::Bytes; +use executor::RuntimeExecutor; +use provider::CacheProvider; +pub(crate) use script::{ScriptDefinition, ScriptRegistry, ScriptValue}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +/// Request for one named provider-side atomic program. +#[derive(Debug, Clone)] +pub struct ScriptRequest { + /// Stable program identifier. + pub script_id: String, + /// Fully-qualified keys used by the program. + pub keys: Vec, + /// Opaque program arguments. + pub args: Vec, +} + +/// Opaque provider-side program result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScriptResult { + /// Encoded result payload interpreted by the business module. + pub payload: Bytes, +} + +/// One cache runtime bound to one provider instance. +pub struct CacheRuntime { + provider: Arc, + scripts: Arc, + executor: Arc, + closed: AtomicBool, +} + +impl CacheRuntime { + pub(crate) fn from_provider(provider: Arc) -> Arc { + Self::from_provider_with_scripts(provider, Arc::new(ScriptRegistry::default())) + } + + fn from_provider_with_scripts( + provider: Arc, + scripts: Arc, + ) -> Arc { + Arc::new(Self { + provider, + scripts, + executor: Arc::new( + RuntimeExecutor::new().expect("CacheRuntime executor must initialize"), + ), + closed: AtomicBool::new(false), + }) + } + + /// Build an in-process runtime for tests and smoke validation. + pub fn memory() -> Arc { + Self::memory_with_provider(Arc::new(MemoryMockProvider::new())) + } + + /// Build a Runtime around one controllable in-memory provider. + pub fn memory_with_provider(provider: Arc) -> Arc { + Self::from_provider(provider) + } + + /// Connect the built-in Redis provider and create one Runtime. + pub async fn redis(config: RedisProviderConfig) -> CacheResult> { + let scripts = Arc::new(ScriptRegistry::default()); + let provider = redis::RedisProvider::connect(config, Arc::clone(&scripts)).await?; + Ok(Self::from_provider_with_scripts( + Arc::new(provider), + scripts, + )) + } + + /// Load one provider through the versioned dynamic C ABI. + pub async fn dynamic(config: DynamicProviderConfig) -> CacheResult> { + let provider = dynamic::DynamicProvider::connect(config).await?; + Ok(Self::from_provider(Arc::new(provider))) + } + + pub(crate) fn register_script(&self, definition: ScriptDefinition) -> CacheResult<()> { + self.scripts.register(definition) + } + + /// Wrap the current Runtime with synchronous primitive operations. + pub fn sync_facade(self: &Arc) -> SyncCacheRuntimeFacade { + SyncCacheRuntimeFacade { + runtime: Arc::clone(self), + executor: Arc::clone(&self.executor), + } + } + + /// Close the provider and reject future operations. + pub async fn close(&self) -> CacheResult<()> { + if self.closed.swap(true, Ordering::AcqRel) { + return Ok(()); + } + self.provider.close().await + } + + fn ensure_open(&self) -> CacheResult<()> { + if self.closed.load(Ordering::Acquire) { + Err(CacheError::Closed) + } else { + Ok(()) + } + } +} + +#[async_trait] +impl AsyncCacheRuntime for CacheRuntime { + async fn get(&self, key: &str) -> CacheResult> { + self.ensure_open()?; + self.provider.get(key).await + } + + async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { + self.ensure_open()?; + self.provider.put(key, value, options).await + } + + async fn delete(&self, key: &str) -> CacheResult<()> { + self.ensure_open()?; + self.provider.delete(key).await + } + + async fn exists(&self, key: &str) -> CacheResult { + self.ensure_open()?; + self.provider.exists(key).await + } + + async fn batch_get(&self, keys: &[String]) -> CacheResult>> { + self.ensure_open()?; + self.provider.batch_get(keys).await + } + + async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + self.ensure_open()?; + self.provider.batch_put(entries).await + } + + async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + self.ensure_open()?; + self.provider.batch_delete(keys).await + } + + async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + self.ensure_open()?; + self.provider.execute_script(request).await + } +} + +/// Stateless synchronous facade over one CacheRuntime. +pub struct SyncCacheRuntimeFacade { + runtime: Arc, + executor: Arc, +} + +impl SyncCacheRuntime for SyncCacheRuntimeFacade { + fn get(&self, key: &str) -> CacheResult> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { AsyncCacheRuntime::get(runtime.as_ref(), &key).await }) + } + + fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor.run(async move { + AsyncCacheRuntime::put(runtime.as_ref(), &key, value, options).await + }) + } + + fn delete(&self, key: &str) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { AsyncCacheRuntime::delete(runtime.as_ref(), &key).await }) + } + + fn exists(&self, key: &str) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { AsyncCacheRuntime::exists(runtime.as_ref(), &key).await }) + } + + fn batch_get(&self, keys: &[String]) -> CacheResult>> { + let runtime = Arc::clone(&self.runtime); + let keys = keys.to_vec(); + self.executor + .run(async move { AsyncCacheRuntime::batch_get(runtime.as_ref(), &keys).await }) + } + + fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + self.executor + .run(async move { AsyncCacheRuntime::batch_put(runtime.as_ref(), entries).await }) + } + + fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + let keys = keys.to_vec(); + self.executor + .run(async move { AsyncCacheRuntime::batch_delete(runtime.as_ref(), &keys).await }) + } + + fn execute_script(&self, request: ScriptRequest) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + self.executor + .run(async move { AsyncCacheRuntime::execute_script(runtime.as_ref(), request).await }) + } +} diff --git a/crates/ragfs/src/cache_runtime/provider.rs b/crates/ragfs/src/cache_runtime/provider.rs new file mode 100644 index 0000000000..71422f2a46 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/provider.rs @@ -0,0 +1,46 @@ +//! Internal provider seam used by CacheRuntime. + +use super::{CacheError, CacheResult, PutOptions, ScriptRequest, ScriptResult}; +use async_trait::async_trait; +use bytes::Bytes; + +#[async_trait] +pub(crate) trait CacheProvider: Send + Sync { + async fn get(&self, key: &str) -> CacheResult>; + async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()>; + async fn delete(&self, key: &str) -> CacheResult<()>; + + async fn exists(&self, key: &str) -> CacheResult { + Ok(self.get(key).await?.is_some()) + } + + async fn batch_get(&self, keys: &[String]) -> CacheResult>> { + let mut values = Vec::with_capacity(keys.len()); + for key in keys { + values.push(self.get(key).await?); + } + Ok(values) + } + + async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + for (key, value) in entries { + self.put(&key, value, PutOptions::default()).await?; + } + Ok(()) + } + + async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + for key in keys { + self.delete(key).await?; + } + Ok(()) + } + + async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + Err(CacheError::UnsupportedScript(request.script_id)) + } + + async fn close(&self) -> CacheResult<()> { + Ok(()) + } +} diff --git a/crates/ragfs-cache-redis/src/client.rs b/crates/ragfs/src/cache_runtime/redis/client.rs similarity index 70% rename from crates/ragfs-cache-redis/src/client.rs rename to crates/ragfs/src/cache_runtime/redis/client.rs index 051ff9159c..d5db5b2308 100644 --- a/crates/ragfs-cache-redis/src/client.rs +++ b/crates/ragfs/src/cache_runtime/redis/client.rs @@ -1,5 +1,5 @@ -use crate::RedisConfig; -use ragfs::cache::{CacheError, CacheResult}; +use super::RedisProviderConfig; +use crate::cache_runtime::{CacheError, CacheResult, ScriptValue}; use redis::aio::MultiplexedConnection; use redis::{AsyncCommands, RedisError}; use std::env; @@ -9,7 +9,7 @@ use std::time::Duration; use tokio::sync::{RwLock, Semaphore}; use url::Url; -pub(crate) struct RedisClient { +pub(super) struct RedisClient { connection: RwLock>, concurrency: Arc, concurrency_limit: u32, @@ -18,17 +18,15 @@ pub(crate) struct RedisClient { } impl RedisClient { - pub(crate) async fn connect(config: &RedisConfig) -> CacheResult { + pub(super) async fn connect(config: &RedisProviderConfig) -> CacheResult { config.validate()?; - let endpoint = endpoint_url(config)?; - let redis_client = - redis::Client::open(endpoint).map_err(|error| map_redis_error("open", error))?; + let redis_client = redis::Client::open(endpoint_url(config)?) + .map_err(|error| map_redis_error("open", error))?; let connect = async { - let connection = redis_client + redis_client .get_multiplexed_async_connection() .await - .map_err(|error| map_redis_error("connect", error))?; - Ok::<_, CacheError>(connection) + .map_err(|error| map_redis_error("connect", error)) }; let connection = tokio::time::timeout(Duration::from_millis(config.connect_timeout_ms), connect) @@ -59,15 +57,11 @@ impl RedisClient { if self.closed.load(Ordering::Acquire) { return Err(CacheError::Unavailable("Redis provider is closed".into())); } - let work = async { let _permit = Arc::clone(&self.concurrency) .acquire_owned() .await .map_err(|_| CacheError::Unavailable("Redis client is closing".into()))?; - if self.closed.load(Ordering::Acquire) { - return Err(CacheError::Unavailable("Redis provider is closed".into())); - } let connection = self.connection.read().await.clone().ok_or_else(|| { CacheError::Unavailable("Redis connection has been released".into()) })?; @@ -75,7 +69,6 @@ impl RedisClient { .await .map_err(|error| map_redis_error(operation, error)) }; - tokio::time::timeout(self.command_timeout, work) .await .map_err(|_| { @@ -86,14 +79,14 @@ impl RedisClient { })? } - pub(crate) async fn health_check(&self) -> CacheResult<()> { + pub(super) async fn health_check(&self) -> CacheResult<()> { self.execute("PING", |mut connection| async move { redis::cmd("PING").query_async(&mut connection).await }) .await } - pub(crate) async fn get(&self, key: String) -> CacheResult>> { + pub(super) async fn get(&self, key: String) -> CacheResult>> { self.execute( "GET", |mut connection| async move { connection.get(key).await }, @@ -101,7 +94,7 @@ impl RedisClient { .await } - pub(crate) async fn set( + pub(super) async fn set( &self, key: String, value: Vec, @@ -123,7 +116,7 @@ impl RedisClient { .await } - pub(crate) async fn delete(&self, key: String) -> CacheResult<()> { + pub(super) async fn delete(&self, key: String) -> CacheResult<()> { self.execute("DEL", |mut connection| async move { let _: u64 = connection.del(key).await?; Ok(()) @@ -131,24 +124,27 @@ impl RedisClient { .await } - pub(crate) async fn exists(&self, key: String) -> CacheResult { + pub(super) async fn exists(&self, key: String) -> CacheResult { self.execute("EXISTS", |mut connection| async move { connection.exists(key).await }) .await } - pub(crate) async fn batch_get(&self, keys: Vec) -> CacheResult>>> { + pub(super) async fn batch_get(&self, keys: Vec) -> CacheResult>>> { if keys.is_empty() { return Ok(Vec::new()); } self.execute("MGET", |mut connection| async move { - connection.get(keys).await + redis::cmd("MGET") + .arg(keys) + .query_async(&mut connection) + .await }) .await } - pub(crate) async fn batch_set( + pub(super) async fn batch_set( &self, entries: Vec<(String, Vec)>, ttl_ms: Option, @@ -175,7 +171,7 @@ impl RedisClient { .await } - pub(crate) async fn batch_delete(&self, keys: Vec) -> CacheResult<()> { + pub(super) async fn batch_delete(&self, keys: Vec) -> CacheResult<()> { if keys.is_empty() { return Ok(()); } @@ -186,7 +182,41 @@ impl RedisClient { .await } - pub(crate) async fn close(&self) -> CacheResult<()> { + pub(super) async fn execute_script( + &self, + lua: &'static str, + keys: Vec, + args: Vec>, + ) -> CacheResult { + self.execute("EVALSHA", |mut connection| async move { + let script = redis::Script::new(lua); + let mut invocation = script.prepare_invoke(); + for key in keys { + invocation.key(key); + } + for arg in args { + invocation.arg(arg); + } + invocation + .invoke_async::(&mut connection) + .await + }) + .await + .and_then(redis_value_to_script_value) + } + + #[cfg(test)] + pub(super) async fn script_flush(&self) -> CacheResult<()> { + self.execute("SCRIPT FLUSH", |mut connection| async move { + redis::cmd("SCRIPT") + .arg("FLUSH") + .query_async(&mut connection) + .await + }) + .await + } + + pub(super) async fn close(&self) -> CacheResult<()> { if self.closed.swap(true, Ordering::AcqRel) { return Ok(()); } @@ -200,12 +230,30 @@ impl RedisClient { } } -fn endpoint_url(config: &RedisConfig) -> CacheResult { +fn redis_value_to_script_value(value: redis::Value) -> CacheResult { + match value { + redis::Value::Nil => Ok(ScriptValue::Null), + redis::Value::Int(value) => Ok(ScriptValue::Integer(value)), + redis::Value::BulkString(value) => Ok(ScriptValue::Bytes(value)), + redis::Value::Array(values) | redis::Value::Set(values) => values + .into_iter() + .map(redis_value_to_script_value) + .collect::>>() + .map(ScriptValue::Array), + redis::Value::SimpleString(value) => Ok(ScriptValue::Bytes(value.into_bytes())), + redis::Value::Okay => Ok(ScriptValue::Bytes(b"OK".to_vec())), + redis::Value::Boolean(value) => Ok(ScriptValue::Boolean(value)), + other => Err(CacheError::InvalidData(format!( + "unsupported Redis script result: {other:?}" + ))), + } +} + +fn endpoint_url(config: &RedisProviderConfig) -> CacheResult { let endpoint = config.endpoints[0].clone(); if config.username.is_empty() && config.password_env.is_empty() { return Ok(endpoint); } - let mut url = Url::parse(&endpoint).map_err(|error| { CacheError::InvalidArgument(format!("Redis endpoint URL is invalid: {error}")) })?; @@ -239,23 +287,3 @@ fn map_redis_error(operation: &str, error: RedisError) -> CacheError { } CacheError::Internal(format!("Redis {operation} failed: {error}")) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn endpoint_uses_password_from_environment() { - std::env::set_var("RAGFS_REDIS_TEST_PASSWORD", "secret"); - let config = RedisConfig { - username: "user".into(), - password_env: "RAGFS_REDIS_TEST_PASSWORD".into(), - ..RedisConfig::default() - }; - - let endpoint = endpoint_url(&config).unwrap(); - - assert_eq!(endpoint, "redis://user:secret@127.0.0.1:6379"); - std::env::remove_var("RAGFS_REDIS_TEST_PASSWORD"); - } -} diff --git a/crates/ragfs/src/cache_runtime/redis/config.rs b/crates/ragfs/src/cache_runtime/redis/config.rs new file mode 100644 index 0000000000..f2145b2aa4 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/redis/config.rs @@ -0,0 +1,123 @@ +//! Redis provider configuration. + +use crate::cache_runtime::{CacheError, CacheResult}; + +/// Connection and execution settings for the built-in Redis provider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RedisProviderConfig { + /// Redis deployment mode. + pub mode: String, + /// Redis endpoints. + pub endpoints: Vec, + /// Optional ACL username. + pub username: String, + /// Environment variable containing the Redis password. + pub password_env: String, + /// Maximum concurrent commands. + pub pool_size: usize, + /// Connection timeout in milliseconds. + pub connect_timeout_ms: u64, + /// Command timeout in milliseconds. + pub command_timeout_ms: u64, + /// Reserved compatibility field; unified Runtime keys require this to be empty. + pub key_prefix: String, + /// Default TTL in seconds; zero disables expiration. + pub default_ttl_seconds: u64, + /// Whether reads may use replicas. + pub read_from_replica: bool, +} + +impl Default for RedisProviderConfig { + fn default() -> Self { + Self { + mode: "standalone".into(), + endpoints: vec!["redis://127.0.0.1:6379".into()], + username: String::new(), + password_env: String::new(), + pool_size: 32, + connect_timeout_ms: 1_000, + command_timeout_ms: 20, + key_prefix: String::new(), + default_ttl_seconds: 3_600, + read_from_replica: false, + } + } +} + +impl RedisProviderConfig { + pub(super) fn validate(&self) -> CacheResult<()> { + if self.mode != "standalone" { + return Err(CacheError::InvalidArgument( + "Redis mode must be standalone in this adapter stage".into(), + )); + } + if self.endpoints.is_empty() + || self + .endpoints + .iter() + .any(|endpoint| endpoint.trim().is_empty()) + { + return Err(CacheError::InvalidArgument( + "Redis endpoints must not be empty".into(), + )); + } + if self.pool_size == 0 { + return Err(CacheError::InvalidArgument( + "Redis pool_size must be greater than zero".into(), + )); + } + if self.connect_timeout_ms == 0 || self.command_timeout_ms == 0 { + return Err(CacheError::InvalidArgument( + "Redis timeouts must be greater than zero".into(), + )); + } + if !self.key_prefix.is_empty() { + return Err(CacheError::InvalidArgument( + "Redis provider key_prefix must be empty because Runtime keys are fully qualified" + .into(), + )); + } + if self.default_ttl_seconds.checked_mul(1_000).is_none() { + return Err(CacheError::InvalidArgument( + "Redis default TTL is too large".into(), + )); + } + if self.read_from_replica { + return Err(CacheError::InvalidArgument( + "Redis read_from_replica is not supported in standalone mode".into(), + )); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_default_ttl_that_cannot_be_sent_as_milliseconds() { + let config = RedisProviderConfig { + default_ttl_seconds: u64::MAX, + ..RedisProviderConfig::default() + }; + + assert!(matches!( + config.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("TTL") + )); + } + + #[test] + fn rejects_provider_key_prefix_because_runtime_keys_are_fully_qualified() { + let config = RedisProviderConfig { + key_prefix: "provider-prefix".into(), + ..RedisProviderConfig::default() + }; + + assert!(matches!( + config.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("key_prefix") + )); + } +} diff --git a/crates/ragfs/src/cache_runtime/redis/mod.rs b/crates/ragfs/src/cache_runtime/redis/mod.rs new file mode 100644 index 0000000000..3c424a0935 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/redis/mod.rs @@ -0,0 +1,9 @@ +//! Built-in Redis CacheRuntime provider. + +mod client; +mod config; +mod provider; + +use client::RedisClient; +pub use config::RedisProviderConfig; +pub(crate) use provider::RedisProvider; diff --git a/crates/ragfs/src/cache_runtime/redis/provider.rs b/crates/ragfs/src/cache_runtime/redis/provider.rs new file mode 100644 index 0000000000..a2d273bf89 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/redis/provider.rs @@ -0,0 +1,162 @@ +use super::{RedisClient, RedisProviderConfig}; +use crate::cache_runtime::provider::CacheProvider; +use crate::cache_runtime::{ + CacheError, CacheResult, PutOptions, ScriptRegistry, ScriptRequest, ScriptResult, +}; +use async_trait::async_trait; +use bytes::Bytes; +use std::sync::Arc; +use std::time::Duration; + +pub(crate) struct RedisProvider { + client: Arc, + scripts: Arc, + default_ttl: Option, +} + +impl RedisProvider { + pub(crate) async fn connect( + config: RedisProviderConfig, + scripts: Arc, + ) -> CacheResult { + config.validate()?; + let default_ttl = if config.default_ttl_seconds == 0 { + None + } else { + Some(Duration::from_secs(config.default_ttl_seconds)) + }; + let client = Arc::new(RedisClient::connect(&config).await?); + Ok(Self { + client, + scripts, + default_ttl, + }) + } + + fn ttl_ms(&self, options: PutOptions) -> CacheResult> { + options + .ttl + .or(self.default_ttl) + .map(|ttl| { + u64::try_from(ttl.as_millis()) + .map_err(|_| CacheError::InvalidArgument("Redis TTL is too large".to_string())) + }) + .transpose() + } +} + +#[async_trait] +impl CacheProvider for RedisProvider { + async fn get(&self, key: &str) -> CacheResult> { + Ok(self.client.get(key.to_string()).await?.map(Bytes::from)) + } + + async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { + self.client + .set(key.to_string(), value.to_vec(), self.ttl_ms(options)?) + .await + } + + async fn delete(&self, key: &str) -> CacheResult<()> { + self.client.delete(key.to_string()).await + } + + async fn exists(&self, key: &str) -> CacheResult { + self.client.exists(key.to_string()).await + } + + async fn batch_get(&self, keys: &[String]) -> CacheResult>> { + Ok(self + .client + .batch_get(keys.to_vec()) + .await? + .into_iter() + .map(|value| value.map(Bytes::from)) + .collect()) + } + + async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + let ttl_ms = self.ttl_ms(PutOptions::default())?; + self.client + .batch_set( + entries + .into_iter() + .map(|(key, value)| (key, value.to_vec())) + .collect(), + ttl_ms, + ) + .await + } + + async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + self.client.batch_delete(keys.to_vec()).await + } + + async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + let lua = self.scripts.resolve(&request.script_id)?; + let value = self + .client + .execute_script( + lua, + request.keys, + request.args.into_iter().map(|arg| arg.to_vec()).collect(), + ) + .await?; + ScriptResult::encode(&value) + } + + async fn close(&self) -> CacheResult<()> { + self.client.close().await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cache_runtime::{AsyncCacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue}; + + #[tokio::test] + async fn executes_registered_script_and_recovers_after_script_flush() { + let Ok(endpoint) = std::env::var("REDIS_URL") else { + return; + }; + let scripts = Arc::new(crate::cache_runtime::ScriptRegistry::default()); + scripts + .register(ScriptDefinition { + id: "runtime.test.echo.v1", + redis_lua: "return {KEYS[1], ARGV[1]}", + }) + .unwrap(); + let config = RedisProviderConfig { + endpoints: vec![endpoint], + key_prefix: String::new(), + command_timeout_ms: 1_000, + ..RedisProviderConfig::default() + }; + let expected_key = format!("ragfs-script-test:{}:key", std::process::id()); + let provider = Arc::new( + RedisProvider::connect(config, Arc::clone(&scripts)) + .await + .unwrap(), + ); + let runtime = crate::cache_runtime::CacheRuntime::from_provider(provider.clone()); + let request = ScriptRequest { + script_id: "runtime.test.echo.v1".into(), + keys: vec![expected_key.clone()], + args: vec![Bytes::from_static(b"value")], + }; + + let first = runtime.execute_script(request.clone()).await.unwrap(); + assert_eq!( + first.decode().unwrap(), + ScriptValue::Array(vec![ + ScriptValue::Bytes(expected_key.into_bytes()), + ScriptValue::Bytes(b"value".to_vec()), + ]) + ); + + provider.client.script_flush().await.unwrap(); + let second = runtime.execute_script(request).await.unwrap(); + assert_eq!(second.decode().unwrap(), first.decode().unwrap()); + } +} diff --git a/crates/ragfs/src/cache_runtime/script.rs b/crates/ragfs/src/cache_runtime/script.rs new file mode 100644 index 0000000000..52105ad286 --- /dev/null +++ b/crates/ragfs/src/cache_runtime/script.rs @@ -0,0 +1,72 @@ +//! Internal named-script registration and result encoding. + +use super::{CacheError, CacheResult, ScriptResult}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::RwLock; + +/// One stable script identifier and its Redis implementation. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ScriptDefinition { + pub(crate) id: &'static str, + pub(crate) redis_lua: &'static str, +} + +/// Scripts registered by business modules during Runtime initialization. +#[derive(Default)] +pub(crate) struct ScriptRegistry { + definitions: RwLock>, +} + +impl ScriptRegistry { + pub(crate) fn register(&self, definition: ScriptDefinition) -> CacheResult<()> { + let mut definitions = self + .definitions + .write() + .map_err(|_| CacheError::Internal("script registry lock poisoned".into()))?; + if let Some(existing) = definitions.get(definition.id) { + if *existing != definition.redis_lua { + return Err(CacheError::InvalidArgument(format!( + "script {} is already registered with different content", + definition.id + ))); + } + return Ok(()); + } + definitions.insert(definition.id, definition.redis_lua); + Ok(()) + } + + pub(crate) fn resolve(&self, script_id: &str) -> CacheResult<&'static str> { + self.definitions + .read() + .map_err(|_| CacheError::Internal("script registry lock poisoned".into()))? + .get(script_id) + .copied() + .ok_or_else(|| CacheError::UnsupportedScript(script_id.to_string())) + } +} + +/// Provider-neutral representation of an atomic script result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(crate) enum ScriptValue { + Null, + Integer(i64), + Bytes(Vec), + Array(Vec), + Boolean(bool), +} + +impl ScriptResult { + pub(crate) fn encode(value: &ScriptValue) -> CacheResult { + serde_json::to_vec(value) + .map(bytes::Bytes::from) + .map(|payload| Self { payload }) + .map_err(|error| CacheError::InvalidData(error.to_string())) + } + + pub(crate) fn decode(&self) -> CacheResult { + serde_json::from_slice(&self.payload) + .map_err(|error| CacheError::InvalidData(error.to_string())) + } +} diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rs index c0ea5b04dd..ea46f7c881 100644 --- a/crates/ragfs/src/core/builder.rs +++ b/crates/ragfs/src/core/builder.rs @@ -14,6 +14,15 @@ use super::filesystem::FileSystem; use super::mountable::MountableFS; use super::stats_wrapper::StatsWrappedFS; +#[cfg(feature = "cache")] +use super::errors::{Error, Result}; +#[cfg(feature = "cache")] +use crate::cache::{CacheNamespace, CachePolicy}; +#[cfg(feature = "cache")] +use crate::cache_runtime::{ + CacheRuntime, DynamicProviderConfig, RedisProviderConfig, +}; + use crate::lock::{ FilesystemPathLockProvider, MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider, PathLockWrappedFS, @@ -53,6 +62,49 @@ pub struct EncryptionConfig { pub provider_type: u8, } +/// Production providers supported by the shared CacheRuntime. +#[cfg(feature = "cache")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CacheRuntimeProviderConfig { + /// Built-in Redis provider. + Redis(RedisProviderConfig), + /// Provider loaded from a versioned dynamic library. + Dynamic(DynamicProviderConfig), +} + +/// CacheFS-specific behavior layered over one shared Runtime. +#[cfg(feature = "cache")] +#[derive(Clone)] +pub struct CacheFsConfig { + /// Whether mounted data filesystems are wrapped by CachedFileSystem. + pub enabled: bool, + /// CacheFS key namespace. + pub namespace: String, + /// Existing CacheFS policy and traversal settings. + pub policy: CachePolicy, +} + +#[cfg(feature = "cache")] +impl Default for CacheFsConfig { + fn default() -> Self { + Self { + enabled: false, + namespace: "openviking".into(), + policy: CachePolicy::default(), + } + } +} + +/// Shared Runtime plus the business modules that opt into it. +#[cfg(feature = "cache")] +#[derive(Clone, Default)] +pub struct CacheStackConfig { + /// One global provider. `None` means no Runtime is initialized. + pub provider: Option, + /// CacheFS enablement and policy. + pub cachefs: CacheFsConfig, +} + /// The assembled stack handles returned by the builder. pub struct RagfsStack { /// Mount manager (mount/unmount/list/stats/register_plugin live here). @@ -61,6 +113,9 @@ pub struct RagfsStack { pub top: Arc, /// PathLock manager. OpenViking always constructs the stack with PathLock enabled. pub pathlock_manager: Arc, + /// Shared cache runtime, present only when a provider was configured. + #[cfg(feature = "cache")] + pub cache_runtime: Option>, } /// Build the standard RAGFS stack. @@ -81,6 +136,60 @@ pub async fn build_stack_with_mountable( config: RagfsConfig, mountable: Arc, ) -> RagfsStack { + build_stack_with_mountable_and_runtime(config, mountable, None).await +} + +#[cfg(feature = "cache")] +/// Build the standard stack and initialize at most one shared CacheRuntime. +pub async fn build_configured_stack( + config: RagfsConfig, + cache: Option, +) -> Result { + let Some(cache) = cache else { + return Ok(build_default_stack(config).await); + }; + if cache.cachefs.enabled && cache.provider.is_none() { + return Err(Error::config( + "cache provider is required when CacheFS is enabled".to_string(), + )); + } + let runtime = match cache.provider { + Some(CacheRuntimeProviderConfig::Redis(provider)) => Some( + CacheRuntime::redis(provider) + .await + .map_err(|error| Error::config(format!("cache provider init failed: {error}")))?, + ), + Some(CacheRuntimeProviderConfig::Dynamic(provider)) => Some( + CacheRuntime::dynamic(provider) + .await + .map_err(|error| Error::config(format!("cache provider init failed: {error}")))?, + ), + None => None, + }; + let mountable = if cache.cachefs.enabled { + Arc::new(MountableFS::with_cache_runtime( + runtime + .as_ref() + .expect("cache provider is validated") + .clone(), + CacheNamespace::new(&cache.cachefs.namespace), + cache.cachefs.policy, + )) + } else { + Arc::new(MountableFS::new()) + }; + Ok(build_stack_with_mountable_and_runtime(config, mountable, runtime).await) +} + +async fn build_stack_with_mountable_and_runtime( + config: RagfsConfig, + mountable: Arc, + #[cfg(feature = "cache")] runtime: Option>, + #[cfg(not(feature = "cache"))] _runtime: Option<()>, +) -> RagfsStack { + #[cfg(feature = "cache")] + register_builtin_plugins_with_runtime(&mountable, runtime.clone()).await; + #[cfg(not(feature = "cache"))] register_builtin_plugins(&mountable).await; // Forward encryption config to MountableFS for per-backend wrapping. @@ -111,11 +220,45 @@ pub async fn build_stack_with_mountable( )); let top: Arc = Arc::new(StatsWrappedFS::with_arc(inner)); - RagfsStack { mountable, top, pathlock_manager } + RagfsStack { + mountable, + top, + pathlock_manager, + #[cfg(feature = "cache")] + cache_runtime: runtime, + } } /// The single built-in plugin registration sequence (eliminates drift across call sites). pub async fn register_builtin_plugins(fs: &MountableFS) { + #[cfg(feature = "cache")] + return register_builtin_plugins_with_runtime(fs, None).await; + + #[cfg(not(feature = "cache"))] + register_builtin_plugins_without_runtime(fs).await; +} + +#[cfg(feature = "cache")] +async fn register_builtin_plugins_with_runtime( + fs: &MountableFS, + runtime: Option>, +) { + fs.register_plugin(MemFSPlugin).await; + fs.register_plugin(KVFSPlugin).await; + fs.register_plugin(match runtime { + Some(runtime) => QueueFSPlugin::with_cache_runtime(runtime), + None => QueueFSPlugin::new(), + }) + .await; + fs.register_plugin(SQLFSPlugin::new()).await; + fs.register_plugin(LocalFSPlugin::new()).await; + fs.register_plugin(ServerInfoFSPlugin::new()).await; + #[cfg(feature = "s3")] + fs.register_plugin(S3FSPlugin::new()).await; +} + +#[cfg(not(feature = "cache"))] +async fn register_builtin_plugins_without_runtime(fs: &MountableFS) { fs.register_plugin(MemFSPlugin).await; fs.register_plugin(KVFSPlugin).await; fs.register_plugin(QueueFSPlugin::new()).await; @@ -327,6 +470,82 @@ mod tests { assert_eq!(raw, b"hello", "plaintext stack stores raw bytes"); } + #[cfg(feature = "cache")] + #[tokio::test] + async fn configured_stack_does_not_create_runtime_when_cache_is_unused() { + let stack = build_configured_stack(RagfsConfig::default(), None) + .await + .unwrap(); + + assert!(stack.cache_runtime.is_none()); + } + + #[cfg(feature = "cache")] + #[tokio::test] + async fn configured_stack_rejects_cachefs_without_global_provider() { + let result = build_configured_stack( + RagfsConfig::default(), + Some(CacheStackConfig { + provider: None, + cachefs: CacheFsConfig { + enabled: true, + ..CacheFsConfig::default() + }, + }), + ) + .await; + let error = match result { + Ok(_) => panic!("CacheFS unexpectedly started without a provider"), + Err(error) => error, + }; + + assert!(error.to_string().contains("cache provider")); + } + + #[cfg(feature = "cache")] + #[tokio::test] + async fn configured_runtime_can_power_queuefs_without_enabling_cachefs() { + let Ok(endpoint) = std::env::var("QUEUEFS_REDIS_TEST_URL") else { + return; + }; + let stack = build_configured_stack( + RagfsConfig::default(), + Some(CacheStackConfig { + provider: Some(CacheRuntimeProviderConfig::Redis(RedisProviderConfig { + endpoints: vec![endpoint], + key_prefix: String::new(), + command_timeout_ms: 1_000, + ..RedisProviderConfig::default() + })), + cachefs: CacheFsConfig::default(), + }), + ) + .await + .unwrap(); + let mut params = HashMap::new(); + params.insert( + "backend".into(), + ConfigValue::String("cache".into()), + ); + params.insert( + "cache_key_prefix".into(), + ConfigValue::String(format!("builder-test-{}", uuid::Uuid::new_v4())), + ); + + stack + .mountable + .mount(plugin_config("queuefs", "/queue", params)) + .await + .unwrap(); + stack.top.mkdir("/queue/jobs", 0o755).await.unwrap(); + stack + .top + .write("/queue/jobs/enqueue", b"job", 0, WriteFlag::None) + .await + .unwrap(); + assert_eq!(stack.top.read("/queue/jobs/size", 0, 0).await.unwrap(), b"1"); + } + #[tokio::test] async fn encrypted_stack_preserves_queuefs_control_semantics() { let stack = build_default_stack(enc_config()).await; diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs index d2d72a9899..0d8bdcdcca 100644 --- a/crates/ragfs/src/core/mod.rs +++ b/crates/ragfs/src/core/mod.rs @@ -27,6 +27,10 @@ pub use builder::{ build_default_stack, build_stack_with_mountable, register_builtin_plugins, EncryptionConfig as BuilderEncryptionConfig, RagfsConfig, RagfsStack, }; +#[cfg(feature = "cache")] +pub use builder::{ + build_configured_stack, CacheFsConfig, CacheRuntimeProviderConfig, CacheStackConfig, +}; pub use context::{FsContext, FsContextInner, FsContextView, PathLockContext, FS_CTX}; pub use encryption_wrapper::EncryptionWrappedFS; pub use errors::{Error, Result}; diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs index 9f6bb74905..a7b52e371b 100644 --- a/crates/ragfs/src/core/mountable.rs +++ b/crates/ragfs/src/core/mountable.rs @@ -13,9 +13,7 @@ use std::time::Duration; use tokio::sync::RwLock; use tracing::warn; -use crate::lock::{ - AutoPathLockAction, PathLockKind, PathLockManager, PathLockRequest, -}; +use crate::lock::{AutoPathLockAction, PathLockKind, PathLockManager, PathLockRequest}; use crate::multibackend::factory::build_multi_write_fs; use crate::multibackend::types::MultiBackendBuildContext; use crate::shape::validate::ensure_backend_shape; @@ -33,9 +31,9 @@ use super::types::{ BackendsConfig, FileInfo, GlobPage, GrepResult, PluginConfig, TreeEntry, WriteFlag, }; #[cfg(feature = "cache")] -use crate::cache::{ - CacheNamespace, CachePolicy, CacheProvider, CacheTraversalMode, CachedFileSystem, -}; +use crate::cache::{CacheNamespace, CachePolicy, CacheTraversalMode, CachedFileSystem}; +#[cfg(feature = "cache")] +use crate::cache_runtime::CacheRuntime; /// Information about a mounted filesystem #[derive(Clone)] @@ -88,7 +86,7 @@ pub struct MountableFS { #[cfg(feature = "cache")] #[derive(Clone)] struct MountCacheConfig { - provider: Arc, + runtime: Arc, namespace: CacheNamespace, policy: CachePolicy, } @@ -223,13 +221,10 @@ impl MountableFS { } } - /// Create a new MountableFS that transparently wraps mounted backends with cache. - /// - /// Encrypted multi-write mounts skip the mount-level cache because their encryption boundary - /// lives inside `MultiWriteWrappedFS`; caching outside it would store plaintext. + /// Create a MountableFS backed by the unified CacheRuntime. #[cfg(feature = "cache")] - pub fn with_cache( - provider: Arc, + pub fn with_cache_runtime( + runtime: Arc, namespace: CacheNamespace, policy: CachePolicy, ) -> Self { @@ -240,7 +235,7 @@ impl MountableFS { encryption_provider_type: RwLock::new(None), pathlock_manager: OnceLock::new(), cache: Some(MountCacheConfig { - provider, + runtime, namespace, policy, }), @@ -357,9 +352,9 @@ impl MountableFS { (Some(rk), Some(pt)) => { if !Self::supports_encrypted_publish(&config.name) { return Err(Error::config(format!( - "encrypted backend '{}' must support replace() semantics", - config.name - ))); + "encrypted backend '{}' must support replace() semantics", + config.name + ))); } let pl_mgr = self .pathlock_manager @@ -397,9 +392,9 @@ impl MountableFS { arc } else { match &self.cache { - Some(cache) => Arc::new(CachedFileSystem::new( + Some(cache) => Arc::new(CachedFileSystem::with_runtime( Box::new(ArcFileSystem(arc)), - cache.provider.clone(), + cache.runtime.clone(), mount_namespace(&cache.namespace, &normalized_path), cache .policy @@ -445,13 +440,9 @@ impl MountableFS { bc: &BackendsConfig, ) -> Result { let (enc_root_key, enc_provider_type) = self.get_encryption_config().await; - let pathlock_manager = self - .pathlock_manager - .get() - .cloned() - .ok_or_else(|| { - Error::config("pathlock manager must be initialized before multi-write mount") - })?; + let pathlock_manager = self.pathlock_manager.get().cloned().ok_or_else(|| { + Error::config("pathlock manager must be initialized before multi-write mount") + })?; build_multi_write_fs( &self.registry, config, @@ -480,9 +471,9 @@ impl MountableFS { } else { cache.policy.clone() }; - Arc::new(CachedFileSystem::new( + Arc::new(CachedFileSystem::with_runtime( Box::new(ArcFileSystem(fs)), - cache.provider.clone(), + cache.runtime.clone(), mount_namespace(&cache.namespace, mount_path), policy, )) @@ -603,11 +594,8 @@ impl MountableFS { return Ok(false); } - let manager = self - .pathlock_manager - .get() - .cloned() - .ok_or_else(|| { + let manager = + self.pathlock_manager.get().cloned().ok_or_else(|| { Error::config("pathlock manager must be initialized before raw copy") })?; let request = PathLockRequest { @@ -655,9 +643,9 @@ impl MountableFS { match (result, release) { (Err(error), _) => Err(error), (Ok(performed), Ok(())) => Ok(performed), - (Ok(_), Err(error)) => Err(Error::internal(format!( - "copy lock release error: {error}" - ))), + (Ok(_), Err(error)) => { + Err(Error::internal(format!("copy lock release error: {error}"))) + } } } @@ -1455,11 +1443,12 @@ mod tests { /// Create a cache-enabled MountableFS backed by the real in-memory plugin. #[cfg(feature = "cache")] async fn mounted_cached_memfs(namespace: &str, mount_path: &str) -> MountableFS { - use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::CacheRuntime; use crate::plugins::MemFSPlugin; - let mfs = MountableFS::with_cache( - Arc::new(MemoryCacheProvider::new()), + let mfs = MountableFS::with_cache_runtime( + CacheRuntime::memory(), CacheNamespace::new(namespace), CachePolicy::default(), ); @@ -1539,11 +1528,12 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn mount_wraps_backend_with_cache_when_configured() { - use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::CacheRuntime; let reads = Arc::new(AtomicU64::new(0)); - let mfs = MountableFS::with_cache( - Arc::new(MemoryCacheProvider::new()), + let mfs = MountableFS::with_cache_runtime( + CacheRuntime::memory(), CacheNamespace::new("mount-test"), CachePolicy::default(), ); @@ -1571,12 +1561,13 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn queuefs_mount_bypasses_cache_even_when_cache_is_configured() { - use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::{CacheRuntime, MemoryMockProvider}; use crate::plugins::QueueFSPlugin; - let provider = Arc::new(MemoryCacheProvider::new()); - let mfs = MountableFS::with_cache( - provider.clone(), + let provider = Arc::new(MemoryMockProvider::new()); + let mfs = MountableFS::with_cache_runtime( + CacheRuntime::memory_with_provider(provider.clone()), CacheNamespace::new("queue-cache-test"), CachePolicy::default().with_bypass_prefix("/queue"), ); @@ -1590,10 +1581,7 @@ mod tests { .unwrap(); mfs.mkdir("/queue/Embedding", 0o755).await.unwrap(); - assert_eq!( - mfs.read("/queue/Embedding/size", 0, 0).await.unwrap(), - b"0" - ); + assert_eq!(mfs.read("/queue/Embedding/size", 0, 0).await.unwrap(), b"0"); mfs.write( "/queue/Embedding/enqueue", br#"{"id":"one"}"#, @@ -1666,11 +1654,12 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn copy_within_mount_overwrite_invalidates_cached_destination() { - use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::CacheRuntime; use crate::plugins::MemFSPlugin; - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache( - Arc::new(MemoryCacheProvider::new()), + let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache_runtime( + CacheRuntime::memory(), CacheNamespace::new("copy-cache-test"), CachePolicy::default(), ))) @@ -1718,23 +1707,24 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn encrypted_mount_caches_ciphertext_below_account_validation() { - use crate::cache::{CacheNamespace, CachePolicy, CacheProvider, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::{AsyncCacheRuntime, CacheRuntime, MemoryMockProvider}; use crate::core::{FsContextInner, FS_CTX}; use crate::lock::{ MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider, }; use crate::plugins::MemFSPlugin; - let cache_provider = Arc::new(MemoryCacheProvider::new()); - let mfs = Arc::new(MountableFS::with_cache( - cache_provider.clone(), + let cache_provider = Arc::new(MemoryMockProvider::new()); + let cache_runtime = CacheRuntime::memory_with_provider(cache_provider.clone()); + let mfs = Arc::new(MountableFS::with_cache_runtime( + cache_runtime.clone(), CacheNamespace::new("encrypted-mount-test"), CachePolicy::default(), )); mfs.register_plugin(MemFSPlugin).await; mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await; - let pathlock_provider: Arc = - Arc::new(MemoryPathLockProvider::new()); + let pathlock_provider: Arc = Arc::new(MemoryPathLockProvider::new()); let manager = Arc::new(PathLockManager::new( mfs.clone() as Arc, pathlock_provider, @@ -1784,7 +1774,7 @@ mod tests { .into_iter() .find(|key| key.contains(":file:")) .expect("encrypted read should populate one file cache object"); - let encoded = cache_provider + let encoded = cache_runtime .get(&file_key) .await .unwrap() @@ -1805,14 +1795,15 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn encrypted_multiwrite_mount_does_not_install_plaintext_cache() { - use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::CacheRuntime; use crate::lock::{ MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider, }; use crate::plugins::MemFSPlugin; - let mfs = Arc::new(MountableFS::with_cache( - Arc::new(MemoryCacheProvider::new()), + let mfs = Arc::new(MountableFS::with_cache_runtime( + CacheRuntime::memory(), CacheNamespace::new("encrypted-multiwrite-test"), CachePolicy::default(), )); @@ -1847,12 +1838,13 @@ mod tests { #[cfg(feature = "cache")] #[tokio::test] async fn cached_unencrypted_multiwrite_keeps_admin_and_copy_fast_paths() { - use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider}; + use crate::cache::{CacheNamespace, CachePolicy}; + use crate::cache_runtime::CacheRuntime; use crate::core::{FsContextInner, FS_CTX}; use crate::plugins::MemFSPlugin; - let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache( - Arc::new(MemoryCacheProvider::new()), + let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache_runtime( + CacheRuntime::memory(), CacheNamespace::new("cached-multiwrite-test"), CachePolicy::default(), ))) @@ -2248,13 +2240,8 @@ mod tests { )); FS_CTX .scope(ctx, async { - mfs.write( - "/local/tenant/file.txt", - b"content", - 0, - WriteFlag::Create, - ) - .await + mfs.write("/local/tenant/file.txt", b"content", 0, WriteFlag::Create) + .await }) .await .unwrap(); diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rs index 06fad00b83..3fa37e7354 100644 --- a/crates/ragfs/src/lib.rs +++ b/crates/ragfs/src/lib.rs @@ -31,6 +31,8 @@ #[cfg(feature = "cache")] pub mod cache; +#[cfg(feature = "cache")] +pub mod cache_runtime; pub mod core; pub mod crypto; pub mod lock; diff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rs new file mode 100644 index 0000000000..f8bddf87b0 --- /dev/null +++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs @@ -0,0 +1,645 @@ +use super::backend::{Message, StoredMessage}; +use super::redis_backend::{ + heartbeat_key, instance_key_prefix, last_enqueue_time_from_pending_payloads, queue_names_key, + unix_secs, QueueKeys, ACK_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT, DEQUEUE_SCRIPT, + ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS, LIST_UNACKED_SCRIPT, PEEK_SCRIPT, + RECOVER_STALE_SCRIPT, REMOVE_QUEUE_SCRIPT, STARTUP_RECOVERY_SWEEPS, +}; +use crate::cache_runtime::{ + AsyncCacheRuntime, CacheError, CacheRuntime, PutOptions, ScriptDefinition, ScriptRequest, + ScriptValue, SyncCacheRuntime, +}; +use crate::core::errors::{Error, Result}; +use bytes::Bytes; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tokio::time::Instant; +use uuid::Uuid; + +const QUEUE_EXISTS_SCRIPT: &str = "return redis.call('SISMEMBER', KEYS[1], ARGV[1])"; +const LIST_QUEUES_SCRIPT: &str = "return redis.call('SMEMBERS', KEYS[1])"; +const SIZE_SCRIPT: &str = "return redis.call('LLEN', KEYS[1])"; +const LIST_PENDING_SCRIPT: &str = "return redis.call('LRANGE', KEYS[1], 0, -1)"; + +const CREATE_QUEUE_ID: &str = "queuefs.create_queue.v1"; +const REMOVE_QUEUE_ID: &str = "queuefs.remove_queue.v1"; +const ENQUEUE_ID: &str = "queuefs.enqueue.v1"; +const DEQUEUE_ID: &str = "queuefs.dequeue.v1"; +const PEEK_ID: &str = "queuefs.peek.v1"; +const LIST_UNACKED_ID: &str = "queuefs.list_unacked.v1"; +const ACK_ID: &str = "queuefs.ack.v1"; +const CLEAR_ID: &str = "queuefs.clear.v1"; +const RECOVER_STALE_ID: &str = "queuefs.recover_stale.v1"; +const QUEUE_EXISTS_ID: &str = "queuefs.queue_exists.v1"; +const LIST_QUEUES_ID: &str = "queuefs.list_queues.v1"; +const SIZE_ID: &str = "queuefs.size.v1"; +const LIST_PENDING_ID: &str = "queuefs.list_pending.v1"; + +const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[ + ScriptDefinition { + id: CREATE_QUEUE_ID, + redis_lua: CREATE_QUEUE_SCRIPT, + }, + ScriptDefinition { + id: REMOVE_QUEUE_ID, + redis_lua: REMOVE_QUEUE_SCRIPT, + }, + ScriptDefinition { + id: ENQUEUE_ID, + redis_lua: ENQUEUE_SCRIPT, + }, + ScriptDefinition { + id: DEQUEUE_ID, + redis_lua: DEQUEUE_SCRIPT, + }, + ScriptDefinition { + id: PEEK_ID, + redis_lua: PEEK_SCRIPT, + }, + ScriptDefinition { + id: LIST_UNACKED_ID, + redis_lua: LIST_UNACKED_SCRIPT, + }, + ScriptDefinition { + id: ACK_ID, + redis_lua: ACK_SCRIPT, + }, + ScriptDefinition { + id: CLEAR_ID, + redis_lua: CLEAR_SCRIPT, + }, + ScriptDefinition { + id: RECOVER_STALE_ID, + redis_lua: RECOVER_STALE_SCRIPT, + }, + ScriptDefinition { + id: QUEUE_EXISTS_ID, + redis_lua: QUEUE_EXISTS_SCRIPT, + }, + ScriptDefinition { + id: LIST_QUEUES_ID, + redis_lua: LIST_QUEUES_SCRIPT, + }, + ScriptDefinition { + id: SIZE_ID, + redis_lua: SIZE_SCRIPT, + }, + ScriptDefinition { + id: LIST_PENDING_ID, + redis_lua: LIST_PENDING_SCRIPT, + }, +]; + +pub(super) struct CacheQueueStorage { + runtime: Arc, + key_prefix: String, + instance_id: String, + heartbeat_stop: watch::Sender, + heartbeat_task: JoinHandle<()>, + recovery_stop: watch::Sender, + recovery_task: JoinHandle<()>, +} + +impl CacheQueueStorage { + pub(super) async fn open(runtime: Arc, key_prefix: String) -> Result { + for definition in SCRIPT_DEFINITIONS { + runtime + .register_script(*definition) + .map_err(|error| cache_error("register script", error))?; + } + let instance_id = Uuid::new_v4().to_string(); + let heartbeat = heartbeat_key(&key_prefix, &instance_id); + refresh_heartbeat(&runtime, &heartbeat).await?; + + let (heartbeat_stop, heartbeat_receiver) = watch::channel(false); + let heartbeat_task = tokio::spawn(run_heartbeat( + Arc::clone(&runtime), + heartbeat, + heartbeat_receiver, + )); + let (recovery_stop, recovery_receiver) = watch::channel(false); + let recovery_task = tokio::spawn(run_startup_recovery( + Arc::clone(&runtime), + key_prefix.clone(), + recovery_receiver, + )); + + Ok(Self { + runtime, + key_prefix, + instance_id, + heartbeat_stop, + heartbeat_task, + recovery_stop, + recovery_task, + }) + } + + async fn execute( + &self, + operation: &str, + script_id: &str, + keys: Vec, + args: Vec, + ) -> Result { + self.runtime + .execute_script(ScriptRequest { + script_id: script_id.to_string(), + keys, + args, + }) + .await + .and_then(|result| result.decode()) + .map_err(|error| cache_error(operation, error)) + } + + pub(super) async fn create_queue(&self, name: &str) -> Result<()> { + let keys = QueueKeys::new(&self.key_prefix, name); + let created = integer( + self.execute( + "create_queue", + CREATE_QUEUE_ID, + vec![queue_names_key(&self.key_prefix), keys.meta], + vec![bytes(name), bytes(unix_secs(SystemTime::now()).to_string())], + ) + .await?, + )?; + if created == 0 { + return Err(Error::AlreadyExists(format!( + "queue '{}' already exists", + name + ))); + } + Ok(()) + } + + pub(super) async fn remove_queue(&self, name: &str) -> Result<()> { + let removed = integer( + self.execute( + "remove_queue", + REMOVE_QUEUE_ID, + vec![queue_names_key(&self.key_prefix)], + vec![ + bytes(name), + bytes(super::redis_backend::queue_key_prefix(&self.key_prefix)), + ], + ) + .await?, + )?; + if removed == 0 { + return Err(Error::NotFound(format!("queue '{}' not found", name))); + } + Ok(()) + } + + async fn queue_exists_result(&self, name: &str) -> Result { + boolean( + self.execute( + "queue_exists", + QUEUE_EXISTS_ID, + vec![queue_names_key(&self.key_prefix)], + vec![bytes(name)], + ) + .await?, + ) + } + + pub(super) async fn queue_exists(&self, name: &str) -> bool { + match self.queue_exists_result(name).await { + Ok(exists) => exists, + Err(error) => { + tracing::error!(queue = name, error = %error, "queuefs cache queue_exists failed; returning false"); + false + } + } + } + + pub(super) async fn list_queues(&self, prefix: &str) -> Vec { + let result = self + .execute( + "list_queues", + LIST_QUEUES_ID, + vec![queue_names_key(&self.key_prefix)], + Vec::new(), + ) + .await + .and_then(string_array); + let mut queues = match result { + Ok(queues) => queues, + Err(error) => { + tracing::error!(prefix, error = %error, "queuefs cache list_queues failed; returning an empty list"); + return Vec::new(); + } + }; + queues.retain(|queue| queue.starts_with(prefix)); + queues.sort(); + queues + } + + async fn require_queue(&self, queue_name: &str) -> Result<()> { + if self.queue_exists_result(queue_name).await? { + Ok(()) + } else { + Err(Error::NotFound(format!("queue '{}' not found", queue_name))) + } + } + + pub(super) async fn enqueue(&self, queue_name: &str, msg: Message) -> Result<()> { + let keys = QueueKeys::new(&self.key_prefix, queue_name); + let payload = serde_json::to_string(&StoredMessage::from_message(&msg))?; + let enqueued = integer( + self.execute( + "enqueue", + ENQUEUE_ID, + vec![ + queue_names_key(&self.key_prefix), + keys.message(&msg.id), + keys.pending, + keys.meta, + ], + vec![ + bytes(queue_name), + bytes(&msg.id), + bytes(payload), + bytes(unix_secs(SystemTime::now()).to_string()), + ], + ) + .await?, + )?; + if enqueued == 0 { + return Err(Error::NotFound(format!("queue '{}' not found", queue_name))); + } + Ok(()) + } + + pub(super) async fn dequeue(&self, queue_name: &str) -> Result> { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + match self + .execute( + "dequeue", + DEQUEUE_ID, + vec![keys.pending, keys.processing], + vec![ + bytes(keys.message_prefix), + bytes(&self.instance_id), + bytes(unix_secs(SystemTime::now()).to_string()), + ], + ) + .await? + { + ScriptValue::Null => Ok(None), + ScriptValue::Array(values) => values + .get(1) + .ok_or_else(|| Error::internal("redis dequeue returned no payload")) + .and_then(string_value) + .and_then(Self::decode_message) + .map(Some), + other => Err(invalid_result("dequeue", other)), + } + } + + pub(super) async fn peek(&self, queue_name: &str) -> Result> { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + match self + .execute( + "peek", + PEEK_ID, + vec![keys.pending], + vec![bytes(keys.message_prefix)], + ) + .await? + { + ScriptValue::Null => Ok(None), + value => string_value(&value) + .and_then(Self::decode_message) + .map(Some), + } + } + + pub(super) async fn size(&self, queue_name: &str) -> Result { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + usize::try_from(integer( + self.execute("size", SIZE_ID, vec![keys.pending], Vec::new()) + .await?, + )?) + .map_err(|_| Error::internal("redis size returned an invalid value")) + } + + pub(super) async fn list_unacked(&self, queue_name: &str) -> Result> { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + string_array( + self.execute( + "list_unacked", + LIST_UNACKED_ID, + vec![keys.pending, keys.processing], + vec![bytes(keys.message_prefix)], + ) + .await?, + )? + .iter() + .map(|payload| Self::decode_message(payload)) + .collect() + } + + pub(super) async fn clear(&self, queue_name: &str) -> Result<()> { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + self.execute( + "clear", + CLEAR_ID, + vec![keys.pending, keys.processing], + vec![bytes(keys.message_prefix)], + ) + .await?; + Ok(()) + } + + pub(super) async fn ack(&self, queue_name: &str, msg_id: &str) -> Result { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + let message_key = keys.message(msg_id); + boolean( + self.execute( + "ack", + ACK_ID, + vec![keys.processing, message_key], + vec![bytes(msg_id)], + ) + .await?, + ) + } + + #[allow(dead_code)] + pub(super) async fn get_last_enqueue_time(&self, queue_name: &str) -> Result { + self.require_queue(queue_name).await?; + let keys = QueueKeys::new(&self.key_prefix, queue_name); + let pending_key = keys.pending.clone(); + let pending_ids = string_array( + self.execute( + "get_last_enqueue_time list pending", + LIST_PENDING_ID, + vec![pending_key], + Vec::new(), + ) + .await?, + )?; + if pending_ids.is_empty() { + return Ok(UNIX_EPOCH); + } + let message_keys = pending_ids + .iter() + .map(|id| keys.message(id)) + .collect::>(); + let payloads = self + .runtime + .batch_get(&message_keys) + .await + .map_err(|error| cache_error("get_last_enqueue_time load payloads", error))? + .into_iter() + .enumerate() + .map(|(index, payload)| { + payload + .ok_or_else(|| { + Error::internal(format!( + "redis get_last_enqueue_time missing payload for message {}", + pending_ids[index] + )) + }) + .and_then(|payload| { + String::from_utf8(payload.to_vec()).map_err(|error| { + Error::Serialization(format!("invalid queue payload: {error}")) + }) + }) + }) + .collect::>>()?; + last_enqueue_time_from_pending_payloads(&payloads) + } + + fn decode_message(payload: &str) -> Result { + serde_json::from_str::(payload) + .map(StoredMessage::into_message) + .map_err(|error| Error::Serialization(format!("invalid queue payload: {error}"))) + } +} + +impl Drop for CacheQueueStorage { + fn drop(&mut self) { + let _ = self.heartbeat_stop.send(true); + let _ = self.recovery_stop.send(true); + self.heartbeat_task.abort(); + self.recovery_task.abort(); + let key = heartbeat_key(&self.key_prefix, &self.instance_id); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + let runtime = Arc::clone(&self.runtime); + handle.spawn(async move { + let _ = runtime.delete(&key).await; + }); + } else if let Err(error) = self.runtime.sync_facade().delete(&key) { + tracing::warn!("queuefs cache heartbeat cleanup failed: {error}"); + } + } +} + +async fn run_heartbeat(runtime: Arc, key: String, mut stop: watch::Receiver) { + let mut interval = tokio::time::interval(Duration::from_secs(HEARTBEAT_INTERVAL_SECS)); + interval.tick().await; + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(error) = refresh_heartbeat(&runtime, &key).await { + tracing::warn!("queuefs cache heartbeat failed: {error}"); + } + } + changed = stop.changed() => { + if changed.is_err() || *stop.borrow() { + break; + } + } + } + } +} + +async fn refresh_heartbeat(runtime: &CacheRuntime, key: &str) -> Result<()> { + runtime + .put( + key, + Bytes::from_static(b"1"), + PutOptions { + ttl: Some(Duration::from_secs(HEARTBEAT_TTL_SECS)), + }, + ) + .await + .map_err(|error| cache_error("heartbeat", error)) +} + +async fn run_startup_recovery( + runtime: Arc, + key_prefix: String, + mut stop: watch::Receiver, +) { + let started_at = Instant::now(); + for sweep_index in 0..STARTUP_RECOVERY_SWEEPS { + let deadline = started_at + startup_recovery_delay_before_sweep(sweep_index); + if deadline > Instant::now() { + tokio::select! { + _ = tokio::time::sleep_until(deadline) => {} + changed = stop.changed() => { + if changed.is_err() || *stop.borrow() { + return; + } + } + } + } + if let Err(error) = recover_stale(&runtime, &key_prefix).await { + tracing::warn!("queuefs cache startup recover_stale failed: {error}"); + } + } +} + +fn startup_recovery_delay_before_sweep(sweep_index: usize) -> Duration { + Duration::from_secs(HEARTBEAT_TTL_SECS * sweep_index as u64) +} + +async fn recover_stale(runtime: &CacheRuntime, key_prefix: &str) -> Result { + let queues = execute_runtime( + runtime, + "recover_stale list queues", + LIST_QUEUES_ID, + vec![queue_names_key(key_prefix)], + Vec::new(), + ) + .await + .and_then(string_array)?; + let mut recovered = 0; + for queue in queues { + let keys = QueueKeys::new(key_prefix, &queue); + recovered += usize::try_from(integer( + execute_runtime( + runtime, + "recover_stale", + RECOVER_STALE_ID, + vec![keys.processing, keys.pending], + vec![bytes(instance_key_prefix(key_prefix))], + ) + .await?, + )?) + .map_err(|_| Error::internal("redis recover_stale returned an invalid value"))?; + } + Ok(recovered) +} + +async fn execute_runtime( + runtime: &CacheRuntime, + operation: &str, + script_id: &str, + keys: Vec, + args: Vec, +) -> Result { + runtime + .execute_script(ScriptRequest { + script_id: script_id.to_string(), + keys, + args, + }) + .await + .and_then(|result| result.decode()) + .map_err(|error| cache_error(operation, error)) +} + +fn bytes(value: impl AsRef<[u8]>) -> Bytes { + Bytes::copy_from_slice(value.as_ref()) +} + +fn integer(value: ScriptValue) -> Result { + match value { + ScriptValue::Integer(value) => Ok(value), + other => Err(invalid_result("integer", other)), + } +} + +fn boolean(value: ScriptValue) -> Result { + match value { + ScriptValue::Boolean(value) => Ok(value), + ScriptValue::Integer(value) => Ok(value != 0), + other => Err(invalid_result("boolean", other)), + } +} + +fn string_value(value: &ScriptValue) -> Result<&str> { + match value { + ScriptValue::Bytes(value) => std::str::from_utf8(value) + .map_err(|error| Error::Serialization(format!("invalid queue payload: {error}"))), + other => Err(invalid_result("string", other.clone())), + } +} + +fn string_array(value: ScriptValue) -> Result> { + match value { + ScriptValue::Array(values) => values + .iter() + .map(string_value) + .map(|value| value.map(str::to_string)) + .collect(), + other => Err(invalid_result("array", other)), + } +} + +fn invalid_result(expected: &str, value: ScriptValue) -> Error { + Error::internal(format!( + "cache script returned {value:?}, expected {expected}" + )) +} + +fn cache_error(operation: &str, error: CacheError) -> Error { + match error { + CacheError::Timeout(message) => Error::Timeout(format!("cache {operation}: {message}")), + CacheError::Unavailable(message) => Error::Network(format!("cache {operation}: {message}")), + CacheError::Closed => Error::Network(format!("cache {operation}: runtime is closed")), + other => Error::internal(format!("cache {operation}: {other}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn startup_recovery_uses_original_zero_thirty_sixty_second_offsets() { + assert_eq!( + startup_recovery_delay_before_sweep(0), + Duration::from_secs(0) + ); + assert_eq!( + startup_recovery_delay_before_sweep(1), + Duration::from_secs(30) + ); + assert_eq!( + startup_recovery_delay_before_sweep(2), + Duration::from_secs(60) + ); + } + + #[test] + fn drop_outside_tokio_runtime_removes_heartbeat() { + let runtime = CacheRuntime::memory(); + let tokio_runtime = tokio::runtime::Runtime::new().unwrap(); + let storage = tokio_runtime + .block_on(CacheQueueStorage::open( + Arc::clone(&runtime), + "drop-test".to_string(), + )) + .unwrap(); + let heartbeat = heartbeat_key("drop-test", &storage.instance_id); + let sync = runtime.sync_facade(); + + assert!(sync.exists(&heartbeat).unwrap()); + drop(storage); + assert!(!sync.exists(&heartbeat).unwrap()); + } +} diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs index 9f8c3eb94a..a8f8f1617a 100644 --- a/crates/ragfs/src/plugins/queuefs/mod.rs +++ b/crates/ragfs/src/plugins/queuefs/mod.rs @@ -11,6 +11,8 @@ //! - `/queue_name/ack` - Write message ID to this file to acknowledge and delete it mod backend; +#[cfg(feature = "cache")] +mod cache_backend; mod redis_backend; use crate::core::{ @@ -21,6 +23,8 @@ use crate::core::{ }; use async_trait::async_trait; use backend::{MemoryBackend, Message, QueueBackend, SQLiteQueueBackend, SQLiteQueueOptions}; +#[cfg(feature = "cache")] +use cache_backend::CacheQueueStorage; use redis_backend::RedisQueueBackend; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -77,6 +81,8 @@ enum BackendKind { Memory, Sqlite, Redis, + #[cfg(feature = "cache")] + Cache, } #[derive(Debug, Clone)] @@ -85,6 +91,8 @@ struct ParsedBackendConfig { sqlite_db_path: Option, sqlite_options: SQLiteQueueOptions, redis_options: Option, + #[cfg(feature = "cache")] + cache_key_prefix: Option, } #[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] @@ -263,10 +271,105 @@ struct ParsedPath { is_dir: bool, } +enum QueueStorage { + Local(Arc>>), + #[cfg(feature = "cache")] + Cache(Arc), +} + +impl QueueStorage { + async fn create_queue(&self, name: &str) -> Result<()> { + match self { + Self::Local(backend) => backend.lock().await.create_queue(name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.create_queue(name).await, + } + } + + async fn remove_queue(&self, name: &str) -> Result<()> { + match self { + Self::Local(backend) => backend.lock().await.remove_queue(name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.remove_queue(name).await, + } + } + + async fn queue_exists(&self, name: &str) -> bool { + match self { + Self::Local(backend) => backend.lock().await.queue_exists(name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.queue_exists(name).await, + } + } + + async fn list_queues(&self, prefix: &str) -> Vec { + match self { + Self::Local(backend) => backend.lock().await.list_queues(prefix), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.list_queues(prefix).await, + } + } + + async fn enqueue(&self, queue_name: &str, message: Message) -> Result<()> { + match self { + Self::Local(backend) => backend.lock().await.enqueue(queue_name, message), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.enqueue(queue_name, message).await, + } + } + + async fn dequeue(&self, queue_name: &str) -> Result> { + match self { + Self::Local(backend) => backend.lock().await.dequeue(queue_name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.dequeue(queue_name).await, + } + } + + async fn peek(&self, queue_name: &str) -> Result> { + match self { + Self::Local(backend) => backend.lock().await.peek(queue_name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.peek(queue_name).await, + } + } + + async fn size(&self, queue_name: &str) -> Result { + match self { + Self::Local(backend) => backend.lock().await.size(queue_name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.size(queue_name).await, + } + } + + async fn list_unacked(&self, queue_name: &str) -> Result> { + match self { + Self::Local(backend) => backend.lock().await.list_unacked(queue_name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.list_unacked(queue_name).await, + } + } + + async fn clear(&self, queue_name: &str) -> Result<()> { + match self { + Self::Local(backend) => backend.lock().await.clear(queue_name), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.clear(queue_name).await, + } + } + + async fn ack(&self, queue_name: &str, message_id: &str) -> Result { + match self { + Self::Local(backend) => backend.lock().await.ack(queue_name, message_id), + #[cfg(feature = "cache")] + Self::Cache(storage) => storage.ack(queue_name, message_id).await, + } + } +} + /// QueueFS - A filesystem-based message queue with multi-queue support pub struct QueueFileSystem { - /// The queue backend - backend: Arc>>, + storage: QueueStorage, } impl QueueFileSystem { @@ -278,10 +381,22 @@ impl QueueFileSystem { /// Create a QueueFileSystem with a specific backend implementation. pub fn with_backend(backend: Box) -> Self { Self { - backend: Arc::new(Mutex::new(backend)), + storage: QueueStorage::Local(Arc::new(Mutex::new(backend))), } } + #[cfg(feature = "cache")] + pub(crate) async fn with_cache_runtime( + runtime: Arc, + key_prefix: String, + ) -> Result { + Ok(Self { + storage: QueueStorage::Cache(Arc::new( + CacheQueueStorage::open(runtime, key_prefix).await?, + )), + }) + } + /// Check if a name is a control operation fn is_control_operation(name: &str) -> bool { CONTROL_FILES.iter().any(|spec| spec.name == name) @@ -380,7 +495,7 @@ impl FileSystem for QueueFileSystem { return Err(Error::InvalidOperation("not a directory path".to_string())); } if let Some(queue_name) = parsed.queue_name { - self.backend.lock().await.create_queue(&queue_name)?; + self.storage.create_queue(&queue_name).await?; Ok(()) } else { // Root directory always exists @@ -398,11 +513,9 @@ impl FileSystem for QueueFileSystem { .operation .ok_or_else(|| Error::InvalidOperation("no operation specified".to_string()))?; - let mut backend = self.backend.lock().await; - match operation.as_str() { "dequeue" => { - let Some(msg) = backend.dequeue(&queue_name)? else { + let Some(msg) = self.storage.dequeue(&queue_name).await? else { return Ok(b"{}".to_vec()); }; // Return in Go libagfsbinding format: {"id": "...", "data": "..."} @@ -414,7 +527,7 @@ impl FileSystem for QueueFileSystem { Ok(serde_json::to_vec(&response)?) } "peek" => { - let Some(msg) = backend.peek(&queue_name)? else { + let Some(msg) = self.storage.peek(&queue_name).await? else { return Ok(b"{}".to_vec()); }; // Return in Go libagfsbinding format: {"id": "...", "data": "..."} @@ -426,12 +539,14 @@ impl FileSystem for QueueFileSystem { Ok(serde_json::to_vec(&response)?) } "size" => { - let size = backend.size(&queue_name)?; + let size = self.storage.size(&queue_name).await?; Ok(size.to_string().into_bytes()) } "messages" => { - let messages = backend - .list_unacked(&queue_name)? + let messages = self + .storage + .list_unacked(&queue_name) + .await? .into_iter() .map(|msg| QueueMessage { id: msg.id, @@ -457,22 +572,20 @@ impl FileSystem for QueueFileSystem { .operation .ok_or_else(|| Error::InvalidOperation("no operation specified".to_string()))?; - let mut backend = self.backend.lock().await; - match operation.as_str() { "enqueue" => { let msg = Message::new(data.to_vec()); let len = data.len() as u64; - backend.enqueue(&queue_name, msg)?; + self.storage.enqueue(&queue_name, msg).await?; Ok(len) } "clear" => { - backend.clear(&queue_name)?; + self.storage.clear(&queue_name).await?; Ok(0) } "ack" => { let msg_id = String::from_utf8_lossy(data).trim().to_string(); - backend.ack(&queue_name, &msg_id)?; + self.storage.ack(&queue_name, &msg_id).await?; Ok(0) } _ => Err(Error::InvalidOperation(format!( @@ -489,12 +602,11 @@ impl FileSystem for QueueFileSystem { return Err(Error::NotADirectory(path.to_string())); } - let backend = self.backend.lock().await; let now = SystemTime::now(); // Root directory: list all top-level queues if parsed.queue_name.is_none() { - let queues = backend.list_queues(""); + let queues = self.storage.list_queues("").await; let mut top_level = std::collections::HashSet::new(); for q in queues { @@ -517,7 +629,7 @@ impl FileSystem for QueueFileSystem { // Queue directory: check if it has nested queues let queue_name = parsed.queue_name.unwrap(); - let all_queues = backend.list_queues(&queue_name); + let all_queues = self.storage.list_queues(&queue_name).await; let has_nested = all_queues .iter() @@ -549,7 +661,7 @@ impl FileSystem for QueueFileSystem { } // Leaf queue: return control files - if !backend.queue_exists(&queue_name) { + if !self.storage.queue_exists(&queue_name).await { return Err(Error::NotFound(format!("queue not found: {}", queue_name))); } @@ -570,12 +682,10 @@ impl FileSystem for QueueFileSystem { }); } - let backend = self.backend.lock().await; - if parsed.is_dir { // Queue directory let queue_name = parsed.queue_name.unwrap(); - if backend.queue_exists(&queue_name) { + if self.storage.queue_exists(&queue_name).await { Ok(FileInfo { name: queue_name .split('/') @@ -631,7 +741,7 @@ impl FileSystem for QueueFileSystem { } if let Some(queue_name) = parsed.queue_name { - self.backend.lock().await.remove_queue(&queue_name)?; + self.storage.remove_queue(&queue_name).await?; Ok(()) } else { Err(Error::InvalidOperation( @@ -650,6 +760,8 @@ impl FileSystem for QueueFileSystem { /// QueueFS Plugin pub struct QueueFSPlugin { config_params: Vec, + #[cfg(feature = "cache")] + cache_runtime: Option>, } impl QueueFSPlugin { @@ -687,10 +799,28 @@ impl QueueFSPlugin { "{}", "Redis connection settings when backend=redis", ), + #[cfg(feature = "cache")] + ConfigParameter::optional( + "cache_key_prefix", + "string", + "default", + "Queue key namespace when backend=cache", + ), ], + #[cfg(feature = "cache")] + cache_runtime: None, } } + #[cfg(feature = "cache")] + pub(crate) fn with_cache_runtime( + runtime: Arc, + ) -> Self { + let mut plugin = Self::new(); + plugin.cache_runtime = Some(runtime); + plugin + } + fn get_string_param<'a>(config: &'a PluginConfig, key: &str) -> Option<&'a str> { config.params.get(key).and_then(|v| v.as_string()) } @@ -701,6 +831,9 @@ impl QueueFSPlugin { fn parse_backend_config(config: &PluginConfig) -> Result { let backend_name = Self::get_string_param(config, "backend").unwrap_or("memory"); + #[cfg(feature = "cache")] + let valid_backends = ["memory", "sqlite", "sqlite3", "redis", "cache"]; + #[cfg(not(feature = "cache"))] let valid_backends = ["memory", "sqlite", "sqlite3", "redis"]; if !valid_backends.contains(&backend_name) { return Err(Error::config(format!( @@ -714,6 +847,8 @@ impl QueueFSPlugin { "memory" => BackendKind::Memory, "sqlite" | "sqlite3" => BackendKind::Sqlite, "redis" => BackendKind::Redis, + #[cfg(feature = "cache")] + "cache" => BackendKind::Cache, _ => { return Err(Error::config(format!( "unsupported queue backend: {}", @@ -741,6 +876,8 @@ impl QueueFSPlugin { let sqlite_db_path = match kind { BackendKind::Memory | BackendKind::Redis => None, + #[cfg(feature = "cache")] + BackendKind::Cache => None, BackendKind::Sqlite => { let db_path = Self::get_string_param(config, "db_path").unwrap_or(""); if db_path.trim().is_empty() { @@ -771,6 +908,25 @@ impl QueueFSPlugin { Some(options) } BackendKind::Memory | BackendKind::Sqlite => None, + #[cfg(feature = "cache")] + BackendKind::Cache => None, + }; + + #[cfg(feature = "cache")] + let cache_key_prefix = match kind { + BackendKind::Cache => { + let prefix = Self::get_string_param(config, "cache_key_prefix") + .unwrap_or("default") + .to_string(); + if prefix.trim().is_empty() || prefix.contains(['{', '}']) { + return Err(Error::config( + "queuefs cache_key_prefix must be non-empty and must not contain '{' or '}'" + .to_string(), + )); + } + Some(prefix) + } + _ => None, }; Ok(ParsedBackendConfig { @@ -781,6 +937,8 @@ impl QueueFSPlugin { busy_timeout_ms, }, redis_options, + #[cfg(feature = "cache")] + cache_key_prefix, }) } } @@ -836,7 +994,15 @@ impl ServicePlugin for QueueFSPlugin { } async fn validate(&self, config: &PluginConfig) -> Result<()> { - Self::parse_backend_config(config)?; + let parsed = Self::parse_backend_config(config)?; + #[cfg(feature = "cache")] + if matches!(parsed.kind, BackendKind::Cache) && self.cache_runtime.is_none() { + return Err(Error::config( + "queuefs backend=cache requires a configured CacheRuntime provider".to_string(), + )); + } + #[cfg(not(feature = "cache"))] + let _ = parsed; Ok(()) } @@ -859,6 +1025,21 @@ impl ServicePlugin for QueueFSPlugin { .expect("redis options are validated for redis backend"), )?) } + #[cfg(feature = "cache")] + BackendKind::Cache => { + return Ok(Box::new( + QueueFileSystem::with_cache_runtime( + self.cache_runtime + .as_ref() + .expect("cache runtime is validated") + .clone(), + parsed + .cache_key_prefix + .expect("cache key prefix is validated"), + ) + .await?, + )); + } }; Ok(Box::new(QueueFileSystem::with_backend(backend))) @@ -1117,7 +1298,14 @@ mod tests { assert_eq!(plugin.name(), "queuefs"); assert!(!plugin.readme().is_empty()); - assert_eq!(plugin.config_params().len(), 5); + let config_params = plugin.config_params(); + assert_eq!(config_params.len(), 5 + usize::from(cfg!(feature = "cache"))); + #[cfg(feature = "cache")] + assert!( + config_params + .iter() + .any(|parameter| parameter.name == "cache_key_prefix") + ); let config = PluginConfig::single_backend("queuefs", "/queue", std::collections::HashMap::new()); @@ -1385,6 +1573,80 @@ mod tests { assert!(result.is_err()); } + #[cfg(feature = "cache")] + #[tokio::test] + async fn cache_runtime_redis_storage_preserves_queuefs_file_flow() { + let Ok(endpoint) = std::env::var("QUEUEFS_REDIS_TEST_URL") else { + return; + }; + let runtime = crate::cache_runtime::CacheRuntime::redis( + crate::cache_runtime::RedisProviderConfig { + endpoints: vec![endpoint], + key_prefix: String::new(), + command_timeout_ms: 1_000, + ..crate::cache_runtime::RedisProviderConfig::default() + }, + ) + .await + .unwrap(); + let queue_prefix = format!("queuefs-runtime-test-{}", uuid::Uuid::new_v4()); + let fs = QueueFileSystem::with_cache_runtime(runtime.clone(), queue_prefix) + .await + .unwrap(); + + fs.mkdir("/Semantic", 0o755).await.unwrap(); + fs.write("/Semantic/enqueue", b"payload", 0, WriteFlag::None) + .await + .unwrap(); + assert_eq!(fs.read("/Semantic/size", 0, 0).await.unwrap(), b"1"); + let dequeued: TestQueueMessage = + serde_json::from_slice(&fs.read("/Semantic/dequeue", 0, 0).await.unwrap()).unwrap(); + assert_eq!(dequeued.data, "payload"); + fs.write( + "/Semantic/ack", + dequeued.id.as_bytes(), + 0, + WriteFlag::None, + ) + .await + .unwrap(); + assert_eq!(fs.read("/Semantic/size", 0, 0).await.unwrap(), b"0"); + fs.remove_all("/Semantic").await.unwrap(); + drop(fs); + runtime.close().await.unwrap(); + } + + #[cfg(feature = "cache")] + #[tokio::test] + async fn cache_runtime_failure_is_returned_without_sqlite_fallback() { + let Ok(endpoint) = std::env::var("QUEUEFS_REDIS_TEST_URL") else { + return; + }; + let runtime = crate::cache_runtime::CacheRuntime::redis( + crate::cache_runtime::RedisProviderConfig { + endpoints: vec![endpoint], + key_prefix: String::new(), + command_timeout_ms: 1_000, + ..crate::cache_runtime::RedisProviderConfig::default() + }, + ) + .await + .unwrap(); + let fs = QueueFileSystem::with_cache_runtime( + runtime.clone(), + format!("queuefs-runtime-failure-{}", uuid::Uuid::new_v4()), + ) + .await + .unwrap(); + + runtime.close().await.unwrap(); + + assert!(matches!( + fs.mkdir("/must-not-fallback", 0o755).await, + Err(Error::Network(_)) + )); + } + #[tokio::test] async fn test_path_parsing() { let fs = QueueFileSystem::new(); diff --git a/crates/ragfs/src/plugins/queuefs/redis_backend.rs b/crates/ragfs/src/plugins/queuefs/redis_backend.rs index 6852241013..607082f7bd 100644 --- a/crates/ragfs/src/plugins/queuefs/redis_backend.rs +++ b/crates/ragfs/src/plugins/queuefs/redis_backend.rs @@ -15,14 +15,14 @@ use std::thread::JoinHandle; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use uuid::Uuid; -const HEARTBEAT_TTL_SECS: u64 = 30; -const HEARTBEAT_INTERVAL_SECS: u64 = 10; -const STARTUP_RECOVERY_SWEEPS: usize = 3; +pub(super) const HEARTBEAT_TTL_SECS: u64 = 30; +pub(super) const HEARTBEAT_INTERVAL_SECS: u64 = 10; +pub(super) const STARTUP_RECOVERY_SWEEPS: usize = 3; const REDIS_POOL_MAX_SIZE: u32 = 2; const REDIS_POOL_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(5); const SENTINEL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); -const CREATE_QUEUE_SCRIPT: &str = r#" +pub(super) const CREATE_QUEUE_SCRIPT: &str = r#" if redis.call('SADD', KEYS[1], ARGV[1]) == 0 then return 0 end @@ -30,7 +30,7 @@ redis.call('HSET', KEYS[2], 'created_at', ARGV[2], 'last_updated', ARGV[2]) return 1 "#; -const REMOVE_QUEUE_SCRIPT: &str = r#" +pub(super) const REMOVE_QUEUE_SCRIPT: &str = r#" local removed = 0 local queues = redis.call('SMEMBERS', KEYS[1]) for _, queue in ipairs(queues) do @@ -58,7 +58,7 @@ end return removed "#; -const ENQUEUE_SCRIPT: &str = r#" +pub(super) const ENQUEUE_SCRIPT: &str = r#" if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then return 0 end @@ -68,7 +68,7 @@ redis.call('HSET', KEYS[4], 'last_updated', ARGV[4]) return 1 "#; -const DEQUEUE_SCRIPT: &str = r#" +pub(super) const DEQUEUE_SCRIPT: &str = r#" local id = redis.call('LPOP', KEYS[1]) if not id then return nil @@ -82,7 +82,7 @@ redis.call('ZADD', KEYS[2], ARGV[3], id .. '|' .. ARGV[2]) return {id, payload} "#; -const PEEK_SCRIPT: &str = r#" +pub(super) const PEEK_SCRIPT: &str = r#" local id = redis.call('LINDEX', KEYS[1], 0) if not id then return nil @@ -94,7 +94,7 @@ end return payload "#; -const LIST_UNACKED_SCRIPT: &str = r#" +pub(super) const LIST_UNACKED_SCRIPT: &str = r#" local result = {} local pending = redis.call('LRANGE', KEYS[1], 0, -1) for _, id in ipairs(pending) do @@ -119,7 +119,7 @@ end return result "#; -const ACK_SCRIPT: &str = r#" +pub(super) const ACK_SCRIPT: &str = r#" local members = redis.call('ZRANGE', KEYS[1], 0, -1) for _, member in ipairs(members) do if string.sub(member, 1, string.len(ARGV[1]) + 1) == ARGV[1] .. '|' then @@ -131,7 +131,7 @@ end return 0 "#; -const CLEAR_SCRIPT: &str = r#" +pub(super) const CLEAR_SCRIPT: &str = r#" local pending = redis.call('LRANGE', KEYS[1], 0, -1) for _, id in ipairs(pending) do redis.call('DEL', ARGV[1] .. id) @@ -147,7 +147,7 @@ redis.call('DEL', KEYS[1], KEYS[2]) return #pending + #processing "#; -const RECOVER_STALE_SCRIPT: &str = r#" +pub(super) const RECOVER_STALE_SCRIPT: &str = r#" local recovered = 0 local members = redis.call('ZRANGE', KEYS[1], 0, -1) for _, member in ipairs(members) do @@ -165,16 +165,16 @@ end return recovered "#; -struct QueueKeys { - meta: String, - pending: String, - processing: String, - message_prefix: String, +pub(super) struct QueueKeys { + pub(super) meta: String, + pub(super) pending: String, + pub(super) processing: String, + pub(super) message_prefix: String, } impl QueueKeys { /// Build all Redis keys owned by one queue. - fn new(key_prefix: &str, queue: &str) -> Self { + pub(super) fn new(key_prefix: &str, queue: &str) -> Self { let prefix = format!("{}{queue}", queue_key_prefix(key_prefix)); Self { meta: format!("{prefix}:meta"), @@ -185,7 +185,7 @@ impl QueueKeys { } /// Build the payload key for one message. - fn message(&self, message_id: &str) -> String { + pub(super) fn message(&self, message_id: &str) -> String { format!("{}{message_id}", self.message_prefix) } } @@ -937,32 +937,32 @@ impl QueueBackend for RedisQueueBackend { } /// Return the queue registry key for one namespace. -fn queue_names_key(key_prefix: &str) -> String { +pub(super) fn queue_names_key(key_prefix: &str) -> String { format!("{}names", queue_key_prefix(key_prefix)) } /// Return the queue key prefix for one namespace. -fn queue_key_prefix(key_prefix: &str) -> String { +pub(super) fn queue_key_prefix(key_prefix: &str) -> String { format!("{{{key_prefix}}}:ov:queue:") } /// Return the instance key prefix for one namespace. -fn instance_key_prefix(key_prefix: &str) -> String { +pub(super) fn instance_key_prefix(key_prefix: &str) -> String { format!("{}instance:", queue_key_prefix(key_prefix)) } /// Return the heartbeat key for one instance. -fn heartbeat_key(key_prefix: &str, instance_id: &str) -> String { +pub(super) fn heartbeat_key(key_prefix: &str, instance_id: &str) -> String { format!("{}{instance_id}:alive", instance_key_prefix(key_prefix)) } /// Return Unix seconds for Redis scores and metadata. -fn unix_secs(time: SystemTime) -> u64 { +pub(super) fn unix_secs(time: SystemTime) -> u64 { time.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() } /// Return the latest timestamp among the current pending payloads. -fn last_enqueue_time_from_pending_payloads(payloads: &[String]) -> Result { +pub(super) fn last_enqueue_time_from_pending_payloads(payloads: &[String]) -> Result { payloads.iter().try_fold(UNIX_EPOCH, |latest, payload| { let timestamp = serde_json::from_str::(payload) .map(StoredMessage::into_message) diff --git a/crates/ragfs/tests/cache_runtime.rs b/crates/ragfs/tests/cache_runtime.rs new file mode 100644 index 0000000000..b26c9633ac --- /dev/null +++ b/crates/ragfs/tests/cache_runtime.rs @@ -0,0 +1,127 @@ +use bytes::Bytes; +use ragfs::cache_runtime::{ + AsyncCacheRuntime, CacheError, CacheRuntime, MemoryMockProvider, PutOptions, ScriptRequest, + SyncCacheRuntime, +}; +use std::sync::Arc; + +#[tokio::test] +async fn async_runtime_supports_the_primitive_contract() { + let runtime = CacheRuntime::memory(); + + assert_eq!(runtime.get("missing").await.unwrap(), None); + assert!(!runtime.exists("missing").await.unwrap()); + + runtime + .put("a", Bytes::from_static(b"one"), PutOptions::default()) + .await + .unwrap(); + assert_eq!( + runtime.get("a").await.unwrap(), + Some(Bytes::from_static(b"one")) + ); + assert!(runtime.exists("a").await.unwrap()); + + runtime + .batch_put(vec![ + ("b".to_string(), Bytes::from_static(b"two")), + ("c".to_string(), Bytes::from_static(b"three")), + ]) + .await + .unwrap(); + assert_eq!( + runtime + .batch_get(&["c".to_string(), "missing".to_string(), "b".to_string()]) + .await + .unwrap(), + vec![ + Some(Bytes::from_static(b"three")), + None, + Some(Bytes::from_static(b"two")), + ] + ); + + runtime + .batch_delete(&["a".to_string(), "c".to_string()]) + .await + .unwrap(); + runtime.delete("b").await.unwrap(); + assert_eq!( + runtime + .batch_get(&["a".to_string(), "b".to_string(), "c".to_string()]) + .await + .unwrap(), + vec![None, None, None] + ); +} + +#[tokio::test] +async fn unknown_script_returns_an_explicit_error() { + let runtime = CacheRuntime::memory(); + let error = runtime + .execute_script(ScriptRequest { + script_id: "queuefs.unknown.v1".to_string(), + keys: Vec::new(), + args: Vec::new(), + }) + .await + .unwrap_err(); + + assert!(matches!(error, CacheError::UnsupportedScript(_))); +} + +#[test] +fn sync_and_async_facades_share_one_provider_instance() { + let runtime = CacheRuntime::memory(); + let sync = runtime.sync_facade(); + + sync.put( + "shared", + Bytes::from_static(b"value"), + PutOptions::default(), + ) + .unwrap(); + + let async_runtime = runtime.clone(); + let value = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { async_runtime.get("shared").await.unwrap() }) + }) + .join() + .unwrap(); + + assert_eq!(value, Some(Bytes::from_static(b"value"))); +} + +#[tokio::test] +async fn close_rejects_new_operations() { + let runtime = CacheRuntime::memory(); + runtime.close().await.unwrap(); + + assert!(matches!(runtime.get("key").await, Err(CacheError::Closed))); +} + +#[tokio::test] +async fn controlled_memory_provider_is_only_accessed_through_runtime() { + let provider = Arc::new(MemoryMockProvider::new()); + let runtime = CacheRuntime::memory_with_provider(Arc::clone(&provider)); + + runtime + .put( + "observed", + Bytes::from_static(b"value"), + PutOptions::default(), + ) + .await + .unwrap(); + assert_eq!(provider.keys().await, vec!["observed".to_string()]); + + provider.set_unavailable(true); + assert!(matches!( + runtime.get("observed").await, + Err(CacheError::Unavailable(_)) + )); +} diff --git a/crates/ragfs/tests/cache_wrapper.rs b/crates/ragfs/tests/cache_wrapper.rs index 3c4354847d..907d883017 100644 --- a/crates/ragfs/tests/cache_wrapper.rs +++ b/crates/ragfs/tests/cache_wrapper.rs @@ -1,14 +1,13 @@ use async_trait::async_trait; -use bytes::Bytes; use ragfs::cache::{ - CacheDecision, CacheError, CacheNamespace, CachePolicy, CacheProvider, CacheResult, - CacheTraversalMode, CachedFileSystem, MemoryCacheProvider, MemoryMockProvider, + CacheDecision, CacheNamespace, CachePolicy, CacheTraversalMode, CachedFileSystem, }; +use ragfs::cache_runtime::{AsyncCacheRuntime, CacheRuntime, MemoryMockProvider}; use ragfs::core::{FsContextInner, GrepResult, MultiWriteWrappedFS, TreeEntry, FS_CTX}; use ragfs::plugins::MemFileSystem; use ragfs::{Error, FileInfo, FileSystem, Result, WriteFlag}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; #[derive(Clone)] @@ -23,185 +22,6 @@ struct CountingFileSystem { partial_remove_path: Option, } -struct DeleteFailingProvider { - inner: MemoryCacheProvider, -} - -struct TrackingProvider { - inner: MemoryCacheProvider, - gets: AtomicU64, - batch_gets: AtomicU64, - active_gets: AtomicU64, - max_active_gets: AtomicU64, - seen_get_keys: Mutex>, - seen_batch_get_keys: Mutex>>, - get_delay: Duration, -} - -struct UnavailableProvider; - -impl DeleteFailingProvider { - fn new() -> Self { - Self { - inner: MemoryCacheProvider::new(), - } - } -} - -impl TrackingProvider { - fn new() -> Self { - Self { - inner: MemoryCacheProvider::new(), - gets: AtomicU64::new(0), - batch_gets: AtomicU64::new(0), - active_gets: AtomicU64::new(0), - max_active_gets: AtomicU64::new(0), - seen_get_keys: Mutex::new(Vec::new()), - seen_batch_get_keys: Mutex::new(Vec::new()), - get_delay: Duration::ZERO, - } - } - - fn with_get_delay(mut self, delay: Duration) -> Self { - self.get_delay = delay; - self - } - - fn reset_observed_reads(&self) { - self.gets.store(0, Ordering::Relaxed); - self.batch_gets.store(0, Ordering::Relaxed); - self.active_gets.store(0, Ordering::Relaxed); - self.max_active_gets.store(0, Ordering::Relaxed); - self.seen_get_keys.lock().unwrap().clear(); - self.seen_batch_get_keys.lock().unwrap().clear(); - } - - fn batch_get_count(&self) -> u64 { - self.batch_gets.load(Ordering::Relaxed) - } - - fn observed_read_keys(&self) -> Vec { - let mut keys = self.seen_get_keys.lock().unwrap().clone(); - keys.extend( - self.seen_batch_get_keys - .lock() - .unwrap() - .iter() - .flat_map(|batch| batch.iter().cloned()), - ); - keys - } - - fn max_concurrent_gets(&self) -> u64 { - self.max_active_gets.load(Ordering::Relaxed) - } - - fn enter_get(&self) { - let active = self.active_gets.fetch_add(1, Ordering::Relaxed) + 1; - let mut current = self.max_active_gets.load(Ordering::Relaxed); - while active > current { - match self.max_active_gets.compare_exchange_weak( - current, - active, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(observed) => current = observed, - } - } - } - - fn exit_get(&self) { - self.active_gets.fetch_sub(1, Ordering::Relaxed); - } -} - -#[async_trait] -impl CacheProvider for DeleteFailingProvider { - fn name(&self) -> &'static str { - "delete-failing" - } - - async fn get(&self, key: &str) -> CacheResult> { - self.inner.get(key).await - } - - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> { - self.inner.put(key, value).await - } - - async fn delete(&self, _key: &str) -> CacheResult<()> { - Err(CacheError::Unavailable( - "delete intentionally failed".to_string(), - )) - } -} - -#[async_trait] -impl CacheProvider for TrackingProvider { - fn name(&self) -> &'static str { - "tracking" - } - - fn capabilities(&self) -> ragfs::cache::ProviderCapabilities { - self.inner.capabilities() - } - - async fn get(&self, key: &str) -> CacheResult> { - self.gets.fetch_add(1, Ordering::Relaxed); - self.seen_get_keys.lock().unwrap().push(key.to_string()); - self.enter_get(); - if !self.get_delay.is_zero() { - tokio::time::sleep(self.get_delay).await; - } - let result = self.inner.get(key).await; - self.exit_get(); - result - } - - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> { - self.inner.put(key, value).await - } - - async fn delete(&self, key: &str) -> CacheResult<()> { - self.inner.delete(key).await - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - self.batch_gets.fetch_add(1, Ordering::Relaxed); - self.seen_batch_get_keys.lock().unwrap().push(keys.to_vec()); - self.inner.batch_get(keys).await - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - self.inner.batch_put(entries).await - } - - async fn invalidate(&self, keys: &[String]) -> CacheResult<()> { - self.inner.invalidate(keys).await - } -} - -#[async_trait] -impl CacheProvider for UnavailableProvider { - fn name(&self) -> &'static str { - "unavailable" - } - - async fn get(&self, _key: &str) -> CacheResult> { - Err(CacheError::Unavailable("provider is down".to_string())) - } - - async fn put(&self, _key: &str, _value: Bytes) -> CacheResult<()> { - Err(CacheError::Unavailable("provider is down".to_string())) - } - - async fn delete(&self, _key: &str) -> CacheResult<()> { - Err(CacheError::Unavailable("provider is down".to_string())) - } -} - impl CountingFileSystem { fn new() -> Self { Self { @@ -343,18 +163,18 @@ impl FileSystem for CountingFileSystem { } } -fn cached_fs(backend: CountingFileSystem) -> (Arc, Arc) { +fn cached_fs(backend: CountingFileSystem) -> (Arc, Arc) { cached_fs_with_policy(backend, CachePolicy::default()) } fn cached_fs_with_policy( backend: CountingFileSystem, policy: CachePolicy, -) -> (Arc, Arc) { - let provider = Arc::new(MemoryCacheProvider::new()); - let fs = Arc::new(CachedFileSystem::new( +) -> (Arc, Arc) { + let provider = Arc::new(MemoryMockProvider::new()); + let fs = Arc::new(CachedFileSystem::with_runtime( Box::new(backend), - provider.clone(), + CacheRuntime::memory_with_provider(provider.clone()), CacheNamespace::new("test"), policy, )); @@ -364,18 +184,18 @@ fn cached_fs_with_policy( fn cached_fs_with_tracking_provider( backend: CountingFileSystem, policy: CachePolicy, -) -> (Arc, Arc) { - cached_fs_with_tracking_provider_instance(backend, policy, Arc::new(TrackingProvider::new())) +) -> (Arc, Arc) { + cached_fs_with_tracking_provider_instance(backend, policy, Arc::new(MemoryMockProvider::new())) } fn cached_fs_with_tracking_provider_instance( backend: CountingFileSystem, policy: CachePolicy, - provider: Arc, -) -> (Arc, Arc) { - let fs = Arc::new(CachedFileSystem::new( + provider: Arc, +) -> (Arc, Arc) { + let fs = Arc::new(CachedFileSystem::with_runtime( Box::new(backend), - provider.clone(), + CacheRuntime::memory_with_provider(provider.clone()), CacheNamespace::new("tracking"), policy, )); @@ -390,6 +210,48 @@ fn cache_policy_traversal_mode_defaults_to_backend() { ); } +#[tokio::test] +async fn cached_filesystem_accepts_the_unified_runtime_without_changing_read_through() { + let backend = CountingFileSystem::new(); + backend + .write("/runtime.md", b"runtime", 0, WriteFlag::Create) + .await + .unwrap(); + let probe = backend.clone(); + let fs = CachedFileSystem::with_runtime( + Box::new(backend), + CacheRuntime::memory(), + CacheNamespace::new("runtime"), + CachePolicy::default(), + ); + + assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime"); + assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime"); + assert_eq!(probe.read_count(), 1); +} + +#[tokio::test] +async fn unified_runtime_failure_still_falls_back_to_backend_reads() { + let backend = CountingFileSystem::new(); + backend + .write("/fallback.txt", b"backend", 0, WriteFlag::Create) + .await + .unwrap(); + let runtime = CacheRuntime::memory(); + runtime.close().await.unwrap(); + let cached = CachedFileSystem::with_runtime( + Box::new(backend), + runtime, + CacheNamespace::new("runtime-fail-open"), + CachePolicy::default(), + ); + + assert_eq!( + cached.read("/fallback.txt", 0, 0).await.unwrap(), + b"backend" + ); +} + #[tokio::test] async fn default_tree_directory_delegates_to_backend() { let backend = CountingFileSystem::new(); @@ -608,7 +470,7 @@ async fn cached_grep_scans_cached_files_with_bounded_concurrency() { .await .unwrap(); } - let provider = Arc::new(TrackingProvider::new().with_get_delay(Duration::from_millis(30))); + let provider = Arc::new(MemoryMockProvider::new().with_get_delay(Duration::from_millis(30))); let (fs, provider) = cached_fs_with_tracking_provider_instance( backend, CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), @@ -708,9 +570,9 @@ async fn cached_grep_traversal_falls_back_for_multiwrite_backend() { let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary)) .build() .unwrap(); - let fs = CachedFileSystem::new( + let fs = CachedFileSystem::with_runtime( Box::new(multiwrite), - Arc::new(MemoryCacheProvider::new()), + CacheRuntime::memory(), CacheNamespace::new("grep-multiwrite"), CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), ); @@ -839,9 +701,11 @@ async fn cached_tree_traversal_falls_back_when_provider_is_unavailable() { .await .unwrap(); let probe = backend.clone(); - let fs = CachedFileSystem::new( + let provider = Arc::new(MemoryMockProvider::new()); + provider.set_unavailable(true); + let fs = CachedFileSystem::with_runtime( Box::new(backend), - Arc::new(UnavailableProvider), + CacheRuntime::memory_with_provider(provider), CacheNamespace::new("tree-unavailable"), CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), ); @@ -876,9 +740,9 @@ async fn cached_tree_traversal_falls_back_for_multiwrite_backend() { let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary)) .build() .unwrap(); - let fs = CachedFileSystem::new( + let fs = CachedFileSystem::with_runtime( Box::new(multiwrite), - Arc::new(MemoryCacheProvider::new()), + CacheRuntime::memory(), CacheNamespace::new("tree-multiwrite"), CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal), ); @@ -931,44 +795,6 @@ fn cache_policy_marks_high_value_objects_as_preferred() { assert_eq!(policy.directory_decision("/docs"), CacheDecision::Prefer); } -#[tokio::test] -async fn memory_provider_satisfies_the_common_contract() { - let provider = MemoryMockProvider::new(); - - provider.put("one", Bytes::from_static(b"1")).await.unwrap(); - assert!(provider.exists("one").await.unwrap()); - assert_eq!( - provider.get("one").await.unwrap(), - Some(Bytes::from_static(b"1")) - ); - - provider - .batch_put(vec![ - ("two".to_string(), Bytes::from_static(b"2")), - ("three".to_string(), Bytes::from_static(b"3")), - ]) - .await - .unwrap(); - assert_eq!( - provider - .batch_get(&["one".to_string(), "missing".to_string()]) - .await - .unwrap(), - vec![Some(Bytes::from_static(b"1")), None] - ); - - provider - .invalidate(&["one".to_string(), "two".to_string()]) - .await - .unwrap(); - assert!(!provider.exists("one").await.unwrap()); - assert!(!provider.exists("two").await.unwrap()); - provider.flush().await.unwrap(); - assert!(!provider.exists("three").await.unwrap()); - provider.close().await.unwrap(); - assert!(provider.get("three").await.is_err()); -} - #[tokio::test] async fn full_file_reads_are_read_through_cached_but_range_reads_bypass() { let backend = CountingFileSystem::new(); @@ -1260,17 +1086,17 @@ async fn shared_provider_generation_bump_invalidates_other_wrappers() { .unwrap(); let direct = backend.clone(); let probe = backend.clone(); - let provider = Arc::new(MemoryCacheProvider::new()); + let runtime = CacheRuntime::memory(); - let first = CachedFileSystem::new( + let first = CachedFileSystem::with_runtime( Box::new(backend.clone()), - provider.clone(), + runtime.clone(), CacheNamespace::new("shared"), CachePolicy::default(), ); - let second = CachedFileSystem::new( + let second = CachedFileSystem::with_runtime( Box::new(backend), - provider, + runtime, CacheNamespace::new("shared"), CachePolicy::default(), ); @@ -1298,11 +1124,12 @@ async fn provider_generation_eviction_after_restart_cannot_revive_old_descendant .unwrap(); let direct = backend.clone(); let probe = backend.clone(); - let provider = Arc::new(MemoryCacheProvider::new()); + let provider = Arc::new(MemoryMockProvider::new()); + let first_runtime = CacheRuntime::memory_with_provider(provider.clone()); - let first = CachedFileSystem::new( + let first = CachedFileSystem::with_runtime( Box::new(backend.clone()), - provider.clone(), + first_runtime.clone(), CacheNamespace::new("restart"), CachePolicy::default(), ); @@ -1317,14 +1144,15 @@ async fn provider_generation_eviction_after_restart_cannot_revive_old_descendant for key in provider.keys().await { if key.contains(":subtree:") { - provider.delete(&key).await.unwrap(); + first_runtime.delete(&key).await.unwrap(); } } drop(first); + drop(first_runtime); - let restarted = CachedFileSystem::new( + let restarted = CachedFileSystem::with_runtime( Box::new(backend), - provider, + CacheRuntime::memory_with_provider(provider), CacheNamespace::new("restart"), CachePolicy::default(), ); @@ -1423,9 +1251,11 @@ async fn failed_invalidation_bypasses_cache_instead_of_serving_stale_data() { .await .unwrap(); let probe = backend.clone(); - let fs = CachedFileSystem::new( + let provider = Arc::new(MemoryMockProvider::new()); + provider.set_delete_failure(true); + let fs = CachedFileSystem::with_runtime( Box::new(backend), - Arc::new(DeleteFailingProvider::new()), + CacheRuntime::memory_with_provider(provider), CacheNamespace::new("delete-failure"), CachePolicy::default(), ); @@ -1450,9 +1280,11 @@ async fn unavailable_provider_falls_back_to_backend_and_enters_bypass() { .await .unwrap(); let probe = backend.clone(); - let fs = CachedFileSystem::new( + let provider = Arc::new(MemoryMockProvider::new()); + provider.set_unavailable(true); + let fs = CachedFileSystem::with_runtime( Box::new(backend), - Arc::new(UnavailableProvider), + CacheRuntime::memory_with_provider(provider), CacheNamespace::new("unavailable"), CachePolicy::default(), ); diff --git a/crates/ragfs/tests/dynamic_provider.rs b/crates/ragfs/tests/dynamic_provider.rs new file mode 100644 index 0000000000..231437b5f7 --- /dev/null +++ b/crates/ragfs/tests/dynamic_provider.rs @@ -0,0 +1,131 @@ +use bytes::Bytes; +use ragfs::cache_runtime::{ + AsyncCacheRuntime, CacheError, CacheRuntime, DynamicProviderConfig, PutOptions, ScriptRequest, +}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +fn compile_fixture(name: &str, definitions: &[&str]) -> PathBuf { + let output_dir = tempfile::tempdir().unwrap().keep(); + let library = output_dir.join(format!( + "{}{}{}", + std::env::consts::DLL_PREFIX, + name, + std::env::consts::DLL_SUFFIX + )); + let source = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/dynamic_provider/provider.c"); + let mut command = Command::new("cc"); + if cfg!(target_os = "macos") { + command.arg("-dynamiclib"); + } else { + command.args(["-shared", "-fPIC"]); + } + for definition in definitions { + command.arg(format!("-D{definition}")); + } + let status = command + .arg(source) + .arg("-o") + .arg(&library) + .status() + .unwrap(); + assert!(status.success()); + library +} + +async fn runtime(library_path: PathBuf) -> std::sync::Arc { + CacheRuntime::dynamic(DynamicProviderConfig { + library_path, + params_json: "{}".into(), + }) + .await + .unwrap() +} + +#[tokio::test] +async fn dynamic_provider_runs_primitive_batch_and_script_contract() { + let runtime = runtime(compile_fixture("provider", &[])).await; + + assert_eq!(runtime.get("missing").await.unwrap(), None); + runtime + .put("one", Bytes::from_static(b"1"), PutOptions::default()) + .await + .unwrap(); + assert!(runtime.exists("one").await.unwrap()); + runtime + .batch_put(vec![ + ("two".into(), Bytes::from_static(b"2")), + ("three".into(), Bytes::from_static(b"3")), + ]) + .await + .unwrap(); + assert_eq!( + runtime + .batch_get(&["three".into(), "missing".into(), "one".into()]) + .await + .unwrap(), + vec![ + Some(Bytes::from_static(b"3")), + None, + Some(Bytes::from_static(b"1")), + ] + ); + let result = runtime + .execute_script(ScriptRequest { + script_id: "runtime.test.echo.v1".into(), + keys: vec!["one".into()], + args: vec![Bytes::from_static(b"argument")], + }) + .await + .unwrap(); + assert_eq!(result.payload, Bytes::from_static(b"argument")); + runtime + .batch_delete(&["one".into(), "two".into(), "three".into()]) + .await + .unwrap(); + assert!(!runtime.exists("one").await.unwrap()); + runtime.close().await.unwrap(); + assert!(matches!(runtime.get("one").await, Err(CacheError::Closed))); +} + +#[tokio::test] +async fn dynamic_provider_rejects_missing_symbol_and_abi_mismatch() { + let missing = CacheRuntime::dynamic(DynamicProviderConfig { + library_path: compile_fixture("missing", &["OMIT_ENTRY"]), + params_json: "{}".into(), + }) + .await; + let missing = match missing { + Ok(_) => panic!("missing entry unexpectedly loaded"), + Err(error) => error, + }; + assert!(matches!(missing, CacheError::AbiMismatch(_))); + + let mismatch = CacheRuntime::dynamic(DynamicProviderConfig { + library_path: compile_fixture("mismatch", &["ABI_VERSION=99"]), + params_json: "{}".into(), + }) + .await; + let mismatch = match mismatch { + Ok(_) => panic!("ABI mismatch unexpectedly loaded"), + Err(error) => error, + }; + assert!(matches!(mismatch, CacheError::AbiMismatch(_))); +} + +#[tokio::test] +async fn dynamic_provider_close_waits_for_inflight_blocking_call() { + let runtime = runtime(compile_fixture("slow", &[])).await; + let reader = { + let runtime = runtime.clone(); + tokio::spawn(async move { runtime.get("slow").await.unwrap() }) + }; + tokio::time::sleep(Duration::from_millis(25)).await; + let started = Instant::now(); + runtime.close().await.unwrap(); + + assert_eq!(reader.await.unwrap(), Some(Bytes::from_static(b"slow"))); + assert!(started.elapsed() >= Duration::from_millis(100)); +} diff --git a/crates/ragfs/tests/fixtures/dynamic_provider/provider.c b/crates/ragfs/tests/fixtures/dynamic_provider/provider.c new file mode 100644 index 0000000000..f209957d36 --- /dev/null +++ b/crates/ragfs/tests/fixtures/dynamic_provider/provider.c @@ -0,0 +1,174 @@ +#include +#include +#include +#include +#include + +#ifndef ABI_VERSION +#define ABI_VERSION 1 +#endif + +typedef struct { const uint8_t *ptr; size_t len; } OvSlice; +typedef struct { uint8_t *ptr; size_t len; } OvBuffer; +typedef struct { OvSlice key; OvSlice value; uint64_t ttl_ms; uint8_t has_ttl; } OvEntry; +typedef struct { uint64_t ttl_ms; uint8_t has_ttl; } OvPutOptions; +typedef struct { + OvSlice script_id; + const OvSlice *keys; + size_t key_count; + const OvSlice *args; + size_t arg_count; +} OvScriptRequest; + +typedef struct OvCacheProviderV1 { + uint32_t abi_version; + uint32_t struct_size; + int32_t (*init)(OvSlice, void **); + int32_t (*get)(void *, OvSlice, OvBuffer *); + int32_t (*put)(void *, OvSlice, OvSlice, const OvPutOptions *); + int32_t (*delete_key)(void *, OvSlice); + int32_t (*exists)(void *, OvSlice, uint8_t *); + int32_t (*batch_get)(void *, const OvSlice *, size_t, OvBuffer *); + int32_t (*batch_put)(void *, const OvEntry *, size_t); + int32_t (*batch_delete)(void *, const OvSlice *, size_t); + int32_t (*execute_script)(void *, const OvScriptRequest *, OvBuffer *); + int32_t (*health)(void *); + void (*free_buffer)(OvBuffer *); + void (*close)(void *); + const char *(*last_error)(void *); +} OvCacheProviderV1; + +typedef struct { char key[64]; uint8_t value[256]; size_t len; uint8_t used; } Item; +typedef struct { Item items[32]; char error[128]; } Provider; + +static int equal(OvSlice value, const char *text) { + size_t len = strlen(text); + return value.len == len && memcmp(value.ptr, text, len) == 0; +} + +static Item *find(Provider *provider, OvSlice key) { + for (size_t i = 0; i < 32; i++) { + if (provider->items[i].used && strlen(provider->items[i].key) == key.len && + memcmp(provider->items[i].key, key.ptr, key.len) == 0) return &provider->items[i]; + } + return NULL; +} + +static int32_t init_provider(OvSlice config, void **handle) { + (void)config; + *handle = calloc(1, sizeof(Provider)); + return *handle ? 0 : -1; +} + +static int32_t get_value(void *handle, OvSlice key, OvBuffer *out) { + Provider *provider = handle; + if (equal(key, "slow")) { + struct timespec delay = {0, 150000000}; + nanosleep(&delay, NULL); + out->ptr = malloc(4); + memcpy(out->ptr, "slow", 4); + out->len = 4; + return 0; + } + Item *item = find(provider, key); + if (!item) return 1; + out->ptr = malloc(item->len); + memcpy(out->ptr, item->value, item->len); + out->len = item->len; + return 0; +} + +static int32_t put_value(void *handle, OvSlice key, OvSlice value, const OvPutOptions *options) { + (void)options; + Provider *provider = handle; + Item *item = find(provider, key); + if (!item) { + for (size_t i = 0; i < 32; i++) if (!provider->items[i].used) { item = &provider->items[i]; break; } + } + if (!item || key.len >= sizeof(item->key) || value.len > sizeof(item->value)) return -1; + memset(item, 0, sizeof(*item)); + memcpy(item->key, key.ptr, key.len); + memcpy(item->value, value.ptr, value.len); + item->len = value.len; + item->used = 1; + return 0; +} + +static int32_t delete_value(void *handle, OvSlice key) { + Item *item = find(handle, key); + if (item) item->used = 0; + return 0; +} + +static int32_t exists_value(void *handle, OvSlice key, uint8_t *out) { + *out = find(handle, key) != NULL; + return 0; +} + +static void append(char **cursor, const char *text) { + size_t len = strlen(text); + memcpy(*cursor, text, len); + *cursor += len; +} + +static int32_t batch_get_value(void *handle, const OvSlice *keys, size_t count, OvBuffer *out) { + char *json = malloc(8192), *cursor = json; + append(&cursor, "["); + for (size_t i = 0; i < count; i++) { + if (i) append(&cursor, ","); + Item *item = find(handle, keys[i]); + if (!item) { append(&cursor, "null"); continue; } + append(&cursor, "["); + for (size_t j = 0; j < item->len; j++) { + if (j) append(&cursor, ","); + cursor += sprintf(cursor, "%u", item->value[j]); + } + append(&cursor, "]"); + } + append(&cursor, "]"); + out->ptr = (uint8_t *)json; + out->len = (size_t)(cursor - json); + return 0; +} + +static int32_t batch_put_value(void *handle, const OvEntry *entries, size_t count) { + for (size_t i = 0; i < count; i++) { + OvPutOptions options = { entries[i].ttl_ms, entries[i].has_ttl }; + if (put_value(handle, entries[i].key, entries[i].value, &options) != 0) return -1; + } + return 0; +} + +static int32_t batch_delete_value(void *handle, const OvSlice *keys, size_t count) { + for (size_t i = 0; i < count; i++) delete_value(handle, keys[i]); + return 0; +} + +static int32_t execute_script_value(void *handle, const OvScriptRequest *request, OvBuffer *out) { + (void)handle; + if (!equal(request->script_id, "runtime.test.echo.v1") || request->arg_count == 0) return -1; + out->ptr = malloc(request->args[0].len); + memcpy(out->ptr, request->args[0].ptr, request->args[0].len); + out->len = request->args[0].len; + return 0; +} + +static int32_t health_provider(void *handle) { return handle ? 0 : -1; } +static void free_buffer_value(OvBuffer *buffer) { free(buffer->ptr); buffer->ptr = NULL; buffer->len = 0; } +static void close_provider(void *handle) { free(handle); } +static const char *last_error_provider(void *handle) { return handle ? ((Provider *)handle)->error : "provider init failed"; } + +#ifndef OMIT_ENTRY +static const OvCacheProviderV1 API = { + ABI_VERSION, sizeof(OvCacheProviderV1), init_provider, get_value, put_value, + delete_value, exists_value, batch_get_value, batch_put_value, batch_delete_value, + execute_script_value, health_provider, free_buffer_value, close_provider, last_error_provider +}; + +#ifdef _WIN32 +__declspec(dllexport) +#endif +const OvCacheProviderV1 *openviking_cache_provider_v1(void) { return &API; } +#else +int fixture_without_provider_entry(void) { return 0; } +#endif diff --git a/crates/ragfs/tests/redis_runtime.rs b/crates/ragfs/tests/redis_runtime.rs new file mode 100644 index 0000000000..a878512eba --- /dev/null +++ b/crates/ragfs/tests/redis_runtime.rs @@ -0,0 +1,80 @@ +use bytes::Bytes; +use ragfs::cache_runtime::{AsyncCacheRuntime, CacheRuntime, PutOptions, RedisProviderConfig}; +use std::time::Duration; + +fn config() -> Option { + let endpoint = std::env::var("REDIS_URL").ok()?; + Some(RedisProviderConfig { + endpoints: vec![endpoint], + key_prefix: String::new(), + connect_timeout_ms: 30_000, + command_timeout_ms: 1_000, + default_ttl_seconds: 60, + ..RedisProviderConfig::default() + }) +} + +fn key(test_name: &str, value: &str) -> String { + format!( + "ragfs-runtime-test:{}:{test_name}:{value}", + std::process::id() + ) +} + +#[tokio::test] +async fn redis_runtime_preserves_primitive_and_batch_semantics() { + let Some(config) = config() else { + return; + }; + let runtime = CacheRuntime::redis(config).await.unwrap(); + let missing = key("contract", "missing"); + let one = key("contract", "one"); + let two = key("contract", "two"); + + assert_eq!(runtime.get(&missing).await.unwrap(), None); + runtime + .batch_put(vec![ + (one.clone(), Bytes::from_static(b"1")), + (two.clone(), Bytes::from_static(b"2")), + ]) + .await + .unwrap(); + assert_eq!( + runtime + .batch_get(&[two.clone(), missing, one.clone()]) + .await + .unwrap(), + vec![ + Some(Bytes::from_static(b"2")), + None, + Some(Bytes::from_static(b"1")), + ] + ); + runtime.batch_delete(&[one.clone(), two]).await.unwrap(); + assert!(!runtime.exists(&one).await.unwrap()); + runtime.close().await.unwrap(); +} + +#[tokio::test] +async fn redis_runtime_preserves_default_ttl_and_per_write_override() { + let Some(mut config) = config() else { + return; + }; + config.default_ttl_seconds = 30; + let runtime = CacheRuntime::redis(config).await.unwrap(); + let ttl_key = key("ttl", "ttl"); + + runtime + .put( + &ttl_key, + Bytes::from_static(b"short"), + PutOptions { + ttl: Some(Duration::from_millis(100)), + }, + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(runtime.get(&ttl_key).await.unwrap(), None); + runtime.close().await.unwrap(); +} diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py index 0a8afb3aff..9cf2dc33a9 100644 --- a/openviking/utils/agfs_utils.py +++ b/openviking/utils/agfs_utils.py @@ -34,8 +34,11 @@ def encryption_enabled(self) -> bool: def to_binding_dict(self) -> Dict[str, Any]: """Convert the runtime config into the sectioned dict consumed by `RAGFSBindingClient`.""" + cache_config = self.agfs.cache.model_dump(mode="json") + if getattr(getattr(self.agfs, "queuefs", None), "backend", None) == "cache": + cache_config["runtime_enabled"] = True binding_config: Dict[str, Any] = { - "cache": self.agfs.cache.model_dump(mode="json"), + "cache": cache_config, "pathlock": self.agfs.pathlock.model_dump(mode="json"), } @@ -196,6 +199,9 @@ def _build_queuefs_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, if backend == "redis": plugin_config["redis"] = queuefs_config.redis.model_dump() + if backend == "cache": + plugin_config["cache_key_prefix"] = queuefs_config.cache_key_prefix + return plugin_config diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py index 5bb01ef92a..e37299ca4f 100644 --- a/openviking_cli/utils/config/agfs_config.py +++ b/openviking_cli/utils/config/agfs_config.py @@ -193,7 +193,7 @@ class QueueFSConfig(BaseModel): backend: str = Field( default="sqlite", - description="QueueFS backend: 'memory' | 'sqlite' | 'sqlite3' | 'redis'", + description="QueueFS backend: 'memory' | 'sqlite' | 'sqlite3' | 'redis' | 'cache'", ) db_path: Optional[str] = Field( @@ -213,6 +213,11 @@ class QueueFSConfig(BaseModel): redis: QueueFSRedisConfig = Field(default_factory=QueueFSRedisConfig) + cache_key_prefix: str = Field( + default="default", + description="Queue key namespace when backend is 'cache'.", + ) + model_config = {"extra": "forbid"} @model_validator(mode="after") @@ -221,25 +226,32 @@ def validate_config(self): if self.mode not in valid_modes: raise ValueError("queuefs mode must be one of: 'shared', 'worker'") - valid_backends = {"memory", "sqlite", "sqlite3", "redis"} + valid_backends = {"memory", "sqlite", "sqlite3", "redis", "cache"} if self.backend not in valid_backends: raise ValueError( - "queuefs backend must be one of: 'memory', 'sqlite', 'sqlite3', 'redis'" + "queuefs backend must be one of: 'memory', 'sqlite', 'sqlite3', 'redis', 'cache'" ) if self.recover_stale_sec < 0: raise ValueError("queuefs recover_stale_sec must be >= 0") if self.busy_timeout_ms < 0: raise ValueError("queuefs busy_timeout_ms must be >= 0") + if not self.cache_key_prefix.strip() or any( + marker in self.cache_key_prefix for marker in ("{", "}") + ): + raise ValueError( + "queuefs cache_key_prefix must be non-empty and must not contain '{' or '}'" + ) return self class AGFSCacheProvider(str, Enum): """Cache providers supported by RAGFS.""" + REDIS = "redis" + DYNAMIC = "dynamic" MEMORY = "memory" YUANRONG = "yuanrong" MOONCAKE = "mooncake" - REDIS = "redis" class AGFSCacheTraversalMode(str, Enum): @@ -249,73 +261,14 @@ class AGFSCacheTraversalMode(str, Enum): CACHED_TRAVERSAL = "cached_traversal" -class YuanrongCacheConfig(BaseModel): - """Configuration for Yuanrong cache provider.""" +class DynamicCacheConfig(BaseModel): + """Configuration passed to a versioned dynamic cache provider.""" - host: str = Field(default="127.0.0.1", description="Yuanrong worker host") - port: int = Field(default=31501, description="Yuanrong worker port") - connect_timeout_ms: int = Field(default=5000, description="Yuanrong connect timeout") - request_timeout_ms: int = Field(default=5000, description="Yuanrong request timeout") - sdk_concurrency: int = Field(default=4, description="Yuanrong SDK concurrency") + library: str = Field(default="", description="Provider dynamic library path") + params: dict[str, Any] = Field(default_factory=dict, description="Provider-owned parameters") model_config = {"extra": "forbid"} - @model_validator(mode="after") - def validate_config(self): - if not self.host.strip(): - raise ValueError("yuanrong host must not be empty") - if self.port <= 0 or self.port > 65535: - raise ValueError("yuanrong port must be between 1 and 65535") - if self.connect_timeout_ms <= 0: - raise ValueError("yuanrong connect_timeout_ms must be > 0") - if self.request_timeout_ms <= 0: - raise ValueError("yuanrong request_timeout_ms must be > 0") - if self.sdk_concurrency <= 0: - raise ValueError("yuanrong sdk_concurrency must be > 0") - return self - - -class MooncakeCacheConfig(BaseModel): - """Configuration for Mooncake cache provider.""" - - local_hostname: str = Field(default="127.0.0.1", description="Mooncake local hostname") - metadata_server: str = Field( - default="http://127.0.0.1:8080/metadata", - description="Mooncake metadata server", - ) - master_server_addr: str = Field( - default="127.0.0.1:50051", - description="Mooncake master server address", - ) - protocol: str = Field(default="tcp", description="Mooncake transfer protocol") - device_name: str = Field(default="", description="Mooncake transport device name") - global_segment_size: int = Field(default=512 << 20, description="Mooncake global segment size") - local_buffer_size: int = Field(default=128 << 20, description="Mooncake local buffer size") - replica_num: int = Field(default=2, description="Mooncake replica count") - sdk_concurrency: int = Field(default=4, description="Mooncake SDK concurrency") - operation_timeout_ms: int = Field(default=5000, description="Mooncake operation timeout") - - model_config = {"extra": "forbid"} - - @model_validator(mode="after") - def validate_config(self): - for name in ("local_hostname", "metadata_server", "master_server_addr", "protocol"): - if not getattr(self, name).strip(): - raise ValueError(f"mooncake {name} must not be empty") - if self.protocol not in {"tcp", "rdma", "ascend", "cxl", "nvlink", "barex"}: - raise ValueError("mooncake protocol is unsupported") - if self.global_segment_size <= 0: - raise ValueError("mooncake global_segment_size must be > 0") - if self.local_buffer_size <= 0: - raise ValueError("mooncake local_buffer_size must be > 0") - if self.replica_num <= 0: - raise ValueError("mooncake replica_num must be > 0") - if self.sdk_concurrency <= 0: - raise ValueError("mooncake sdk_concurrency must be > 0") - if self.operation_timeout_ms <= 0: - raise ValueError("mooncake operation_timeout_ms must be > 0") - return self - class RedisCacheConfig(BaseModel): """Configuration for Redis cache provider.""" @@ -330,7 +283,10 @@ class RedisCacheConfig(BaseModel): pool_size: int = Field(default=32, description="Redis command concurrency") connect_timeout_ms: int = Field(default=1000, description="Redis connect timeout") command_timeout_ms: int = Field(default=20, description="Redis command timeout") - key_prefix: str = Field(default="ragfs-cache", description="Redis cache key prefix") + key_prefix: str = Field( + default="", + description="Reserved compatibility field; unified Runtime requires an empty value", + ) default_ttl_seconds: int = Field(default=3600, description="Redis default cache TTL") read_from_replica: bool = Field(default=False, description="Read from Redis replicas") @@ -350,8 +306,6 @@ def validate_config(self): raise ValueError("redis connect_timeout_ms must be > 0") if self.command_timeout_ms <= 0: raise ValueError("redis command_timeout_ms must be > 0") - if not self.key_prefix.strip(): - raise ValueError("redis key_prefix must not be empty") if self.default_ttl_seconds < 0: raise ValueError("redis default_ttl_seconds must be >= 0") if self.read_from_replica: @@ -364,7 +318,7 @@ class AGFSCacheConfig(BaseModel): enabled: bool = Field(default=False, description="Enable RAGFS cache") provider: AGFSCacheProvider = Field( - default=AGFSCacheProvider.MEMORY, + default=AGFSCacheProvider.REDIS, description="RAGFS cache provider", ) namespace: str = Field(default="openviking", description="RAGFS cache namespace") @@ -380,9 +334,8 @@ class AGFSCacheConfig(BaseModel): default_factory=list, description="Path prefixes that bypass cache", ) - yuanrong: YuanrongCacheConfig = Field(default_factory=YuanrongCacheConfig) - mooncake: MooncakeCacheConfig = Field(default_factory=MooncakeCacheConfig) redis: RedisCacheConfig = Field(default_factory=RedisCacheConfig) + dynamic: DynamicCacheConfig = Field(default_factory=DynamicCacheConfig) model_config = {"extra": "forbid"} @@ -392,6 +345,19 @@ def validate_config(self): raise ValueError("cache namespace must not be empty") if self.max_file_size_bytes <= 0: raise ValueError("cache max_file_size_bytes must be > 0") + if self.enabled and self.provider not in { + AGFSCacheProvider.REDIS, + AGFSCacheProvider.DYNAMIC, + }: + raise ValueError("enabled cache provider must be 'redis' or 'dynamic'") + if self.enabled and self.provider == AGFSCacheProvider.DYNAMIC: + if not self.dynamic.library.strip(): + raise ValueError("dynamic cache library must not be empty") + if self.enabled and self.provider == AGFSCacheProvider.REDIS: + if self.redis.key_prefix: + raise ValueError( + "redis cache key_prefix must be empty because Runtime keys are fully qualified" + ) return self diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py index cd685f80f9..d14c291514 100644 --- a/tests/misc/test_config_validation.py +++ b/tests/misc/test_config_validation.py @@ -188,44 +188,41 @@ def test_agfs_queuefs_validation_rejects_invalid_shapes(queuefs, match): AGFSConfig(path="/tmp/ov-test", backend="local", queuefs=queuefs) -def test_agfs_cache_defaults_to_disabled_memory_provider(): +def test_agfs_cache_defaults_to_disabled_redis_provider(): config = AGFSConfig(path="/tmp/ov-test", backend="local") assert config.cache.enabled is False - assert config.cache.provider == "memory" + assert config.cache.provider == "redis" assert config.cache.namespace == "openviking" assert config.cache.traversal_mode == "backend" -def test_agfs_cache_accepts_yuanrong_provider_config(): +def test_agfs_cache_accepts_dynamic_provider_config(): config = AGFSConfig( path="/tmp/ov-test", backend="local", cache={ "enabled": True, - "provider": "yuanrong", + "provider": "dynamic", "namespace": "ov-test", "max_file_size_bytes": 4096, "traversal_mode": "cached_traversal", "bypass_prefixes": ["/queue"], - "yuanrong": { - "host": "10.0.0.1", - "port": 31501, - "connect_timeout_ms": 1000, - "request_timeout_ms": 2000, - "sdk_concurrency": 2, + "dynamic": { + "library": "/opt/openviking/libprovider.so", + "params": {"endpoint": "provider:1234"}, }, }, ) assert config.cache.enabled is True - assert config.cache.provider == "yuanrong" + assert config.cache.provider == "dynamic" assert config.cache.namespace == "ov-test" assert config.cache.max_file_size_bytes == 4096 assert config.cache.traversal_mode == "cached_traversal" assert config.cache.bypass_prefixes == ["/queue"] - assert config.cache.yuanrong.host == "10.0.0.1" - assert config.cache.yuanrong.sdk_concurrency == 2 + assert config.cache.dynamic.library == "/opt/openviking/libprovider.so" + assert config.cache.dynamic.params == {"endpoint": "provider:1234"} def test_agfs_cache_accepts_redis_provider_config(): @@ -242,7 +239,7 @@ def test_agfs_cache_accepts_redis_provider_config(): "pool_size": 8, "connect_timeout_ms": 1000, "command_timeout_ms": 20, - "key_prefix": "ragfs-cache", + "key_prefix": "", "default_ttl_seconds": 3600, "read_from_replica": False, }, @@ -257,6 +254,19 @@ def test_agfs_cache_accepts_redis_provider_config(): assert config.cache.redis.default_ttl_seconds == 3600 +def test_agfs_cache_rejects_redis_provider_key_prefix_when_enabled(): + with pytest.raises(ValueError, match="key_prefix"): + AGFSConfig( + path="/tmp/ov-test", + backend="local", + cache={ + "enabled": True, + "provider": "redis", + "redis": {"key_prefix": "provider-prefix"}, + }, + ) + + def test_agfs_cache_rejects_invalid_provider(): with pytest.raises(ValueError, match="provider"): AGFSConfig( @@ -711,7 +721,7 @@ def test_ragfs_binding_config_builds_single_binding_dict_for_local_backend(tmp_p backend="local", cache={ "enabled": True, - "provider": "memory", + "provider": "redis", "namespace": "runtime-cache", }, ) @@ -736,6 +746,20 @@ def test_ragfs_binding_config_builds_single_binding_dict_for_local_backend(tmp_p } +def test_ragfs_binding_enables_runtime_for_queuefs_cache_backend(tmp_path): + agfs_config = AGFSConfig( + path=str(tmp_path), + backend="local", + cache={"enabled": False, "provider": "redis"}, + queuefs={"backend": "cache", "cache_key_prefix": "queue-runtime"}, + ) + + binding = RagfsBindingConfig(agfs=agfs_config).to_binding_dict() + + assert binding["cache"]["enabled"] is False + assert binding["cache"]["runtime_enabled"] is True + + def test_agfs_pathlock_config_validates_provider_and_expiry(tmp_path): """PathLock config accepts built-ins and rejects unsafe expiry values.""" config = AGFSConfig( @@ -757,7 +781,7 @@ def test_create_agfs_client_uses_single_binding_config_object(monkeypatch, tmp_p agfs_config = AGFSConfig( path=str(tmp_path), backend="memory", - cache={"enabled": True, "provider": "memory", "namespace": "runtime-cache"}, + cache={"enabled": True, "provider": "redis", "namespace": "runtime-cache"}, ) def _fake_get_binding_client(): diff --git a/tests/misc/test_ragfs_python_manifest_isolation.py b/tests/misc/test_ragfs_python_manifest_isolation.py index 794213f426..1e957651ee 100644 --- a/tests/misc/test_ragfs_python_manifest_isolation.py +++ b/tests/misc/test_ragfs_python_manifest_isolation.py @@ -22,13 +22,16 @@ def _section(text: str, name: str) -> str: return match.group(1) -def test_default_workspace_excludes_native_cache_providers(): +def test_workspace_uses_ragfs_runtime_and_one_python_binding_crate(): manifest = _read(ROOT / "Cargo.toml") workspace = _section(manifest, "workspace") members = _array_items(workspace, "members") excludes = _array_items(workspace, "exclude") - assert "crates/ragfs-cache-redis" in members + assert "crates/ragfs" in members + assert "crates/ragfs-python" in members + assert "crates/ragfs-cache-redis" not in members + assert "crates/ragfs-python-native" not in members assert "crates/ragfs-cache-mooncake" not in members assert "crates/ragfs-cache-yuanrong" not in members assert "crates/ragfs-cache-yuanrong-sys" not in members @@ -36,34 +39,32 @@ def test_default_workspace_excludes_native_cache_providers(): assert "crates/ragfs-cache-mooncake" in excludes assert "crates/ragfs-cache-yuanrong" in excludes assert "crates/ragfs-cache-yuanrong-sys" in excludes - assert "crates/ragfs-python-native" in excludes -def test_default_ragfs_python_manifest_depends_only_on_redis_provider(): +def test_ragfs_python_uses_the_runtime_embedded_in_ragfs(): manifest = _read(ROOT / "crates/ragfs-python/Cargo.toml") features = _section(manifest, "features") dependencies = _section(manifest, "dependencies") - assert "cache-redis" in features + assert "cache-redis" not in features assert "mooncake-native" not in features assert "yuanrong-native" not in features - assert "ragfs-cache-redis" in dependencies + assert 'ragfs = { path = "../ragfs", features = ["cache"] }' in dependencies + assert "ragfs-cache-redis" not in dependencies assert "ragfs-cache-mooncake" not in dependencies assert "ragfs-cache-yuanrong" not in dependencies -def test_native_ragfs_python_manifest_is_explicit_provider_entrypoint(): - manifest = _read(ROOT / "crates/ragfs-python-native/Cargo.toml") - lib = _section(manifest, "lib") - features = _section(manifest, "features") - dependencies = _section(manifest, "dependencies") +def test_legacy_provider_and_binding_manifests_are_removed(): + assert not (ROOT / "crates/ragfs-cache-redis/Cargo.toml").exists() + assert not (ROOT / "crates/ragfs-python-native/Cargo.toml").exists() + - assert 'path = "../ragfs-python/src/lib.rs"' in lib - assert "cache-redis" in features - assert "mooncake-native" in features - assert "yuanrong-native" in features +def test_source_distribution_contains_only_active_ragfs_crates(): + manifest = _read(ROOT / "MANIFEST.in") - assert "ragfs-cache-redis" in dependencies - assert "ragfs-cache-mooncake" in dependencies - assert "ragfs-cache-yuanrong" in dependencies + assert "graft crates/ragfs\n" in manifest + assert "graft crates/ragfs-python\n" in manifest + assert "graft crates/ragfs-cache-redis" not in manifest + assert "graft crates/ragfs-python-native" not in manifest From df2cf9536d770f19fbef24c8912cb55f8154c691 Mon Sep 17 00:00:00 2001 From: chrisfang Date: Mon, 24 Aug 2026 16:24:19 +0800 Subject: [PATCH 2/7] refactor(ragfs): remove bundled external cache providers --- CONTRIBUTING.md | 7 +- Cargo.toml | 5 - crates/ragfs-cache-mooncake/Cargo.toml | 20 - crates/ragfs-cache-mooncake/README.md | 82 --- crates/ragfs-cache-mooncake/src/client.rs | 136 ----- crates/ragfs-cache-mooncake/src/config.rs | 73 --- crates/ragfs-cache-mooncake/src/error.rs | 16 - crates/ragfs-cache-mooncake/src/lib.rs | 14 - crates/ragfs-cache-mooncake/src/native.rs | 121 ---- crates/ragfs-cache-mooncake/src/provider.rs | 209 ------- crates/ragfs-cache-mooncake/src/store.rs | 35 -- .../tests/cached_filesystem.rs | 335 ----------- .../tests/native_smoke.rs | 43 -- .../tests/provider_contract.rs | 357 ------------ crates/ragfs-cache-yuanrong-sys/Cargo.toml | 16 - crates/ragfs-cache-yuanrong-sys/build.rs | 41 -- .../native/yuanrong_bridge.cpp | 530 ------------------ .../native/yuanrong_bridge.h | 55 -- crates/ragfs-cache-yuanrong-sys/src/lib.rs | 113 ---- crates/ragfs-cache-yuanrong/Cargo.toml | 24 - crates/ragfs-cache-yuanrong/src/client.rs | 164 ------ crates/ragfs-cache-yuanrong/src/config.rs | 52 -- crates/ragfs-cache-yuanrong/src/error.rs | 16 - crates/ragfs-cache-yuanrong/src/lib.rs | 14 - crates/ragfs-cache-yuanrong/src/native.rs | 344 ------------ crates/ragfs-cache-yuanrong/src/provider.rs | 167 ------ crates/ragfs-cache-yuanrong/src/store.rs | 26 - .../tests/cached_filesystem.rs | 381 ------------- .../tests/native_smoke.rs | 76 --- .../tests/provider_contract.rs | 274 --------- docker/mooncake-test/Dockerfile | 92 --- docker/mooncake-test/run-smoke.sh | 81 --- docs/en/guides/14-ragfs-cache.md | 169 +----- docs/zh/guides/14-ragfs-cache.md | 164 +----- .../test_ragfs_python_manifest_isolation.py | 11 +- 35 files changed, 67 insertions(+), 4196 deletions(-) delete mode 100644 crates/ragfs-cache-mooncake/Cargo.toml delete mode 100644 crates/ragfs-cache-mooncake/README.md delete mode 100644 crates/ragfs-cache-mooncake/src/client.rs delete mode 100644 crates/ragfs-cache-mooncake/src/config.rs delete mode 100644 crates/ragfs-cache-mooncake/src/error.rs delete mode 100644 crates/ragfs-cache-mooncake/src/lib.rs delete mode 100644 crates/ragfs-cache-mooncake/src/native.rs delete mode 100644 crates/ragfs-cache-mooncake/src/provider.rs delete mode 100644 crates/ragfs-cache-mooncake/src/store.rs delete mode 100644 crates/ragfs-cache-mooncake/tests/cached_filesystem.rs delete mode 100644 crates/ragfs-cache-mooncake/tests/native_smoke.rs delete mode 100644 crates/ragfs-cache-mooncake/tests/provider_contract.rs delete mode 100644 crates/ragfs-cache-yuanrong-sys/Cargo.toml delete mode 100644 crates/ragfs-cache-yuanrong-sys/build.rs delete mode 100644 crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cpp delete mode 100644 crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.h delete mode 100644 crates/ragfs-cache-yuanrong-sys/src/lib.rs delete mode 100644 crates/ragfs-cache-yuanrong/Cargo.toml delete mode 100644 crates/ragfs-cache-yuanrong/src/client.rs delete mode 100644 crates/ragfs-cache-yuanrong/src/config.rs delete mode 100644 crates/ragfs-cache-yuanrong/src/error.rs delete mode 100644 crates/ragfs-cache-yuanrong/src/lib.rs delete mode 100644 crates/ragfs-cache-yuanrong/src/native.rs delete mode 100644 crates/ragfs-cache-yuanrong/src/provider.rs delete mode 100644 crates/ragfs-cache-yuanrong/src/store.rs delete mode 100644 crates/ragfs-cache-yuanrong/tests/cached_filesystem.rs delete mode 100644 crates/ragfs-cache-yuanrong/tests/native_smoke.rs delete mode 100644 crates/ragfs-cache-yuanrong/tests/provider_contract.rs delete mode 100644 docker/mooncake-test/Dockerfile delete mode 100644 docker/mooncake-test/run-smoke.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63b2c62fed..32204bd9a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,12 +131,7 @@ openviking/ ├── crates/ # Rust components │ ├── ov_cli/ # Rust CLI client │ ├── ragfs/ # Rust implementation of AGFS -│ ├── ragfs-python/ # Python binding for RAGFS -│ ├── ragfs-python-native/ # Native Python binding package -│ ├── ragfs-cache-redis/ # Redis cache backend -│ ├── ragfs-cache-mooncake/ # Mooncake cache backend -│ ├── ragfs-cache-yuanrong/ # YuanRong cache backend -│ └── ragfs-cache-yuanrong-sys/ # YuanRong FFI bindings +│ └── ragfs-python/ # Python binding for RAGFS ├── src/ # C++ extension sources (Python abi3) ├── third_party/ # Native third-party dependencies ├── examples/ # Usage and integration examples diff --git a/Cargo.toml b/Cargo.toml index b8789f2aea..f44b1edc3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,11 +4,6 @@ members = [ "crates/ragfs", "crates/ragfs-python", ] -exclude = [ - "crates/ragfs-cache-mooncake", - "crates/ragfs-cache-yuanrong", - "crates/ragfs-cache-yuanrong-sys", -] resolver = "2" [profile.release] diff --git a/crates/ragfs-cache-mooncake/Cargo.toml b/crates/ragfs-cache-mooncake/Cargo.toml deleted file mode 100644 index 2f90001b86..0000000000 --- a/crates/ragfs-cache-mooncake/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "ragfs-cache-mooncake" -version = "0.1.0" -edition = "2021" -rust-version = "1.91.1" -license = "Apache-2.0" -description = "Mooncake cache provider adapter for RAGFS" - -[dependencies] -async-trait = "0.1" -bytes = "1.5" -futures = "0.3" -mooncake_store = { git = "https://github.com/kvcache-ai/Mooncake.git", rev = "1352bbec43081e461356aaecf6c70cddd826b455", optional = true } -ragfs = { path = "../ragfs", features = ["cache"] } -thiserror = "1.0" -tokio = { version = "1.38", features = ["rt-multi-thread", "sync", "time", "macros"] } - -[features] -default = [] -mooncake-native = ["dep:mooncake_store"] diff --git a/crates/ragfs-cache-mooncake/README.md b/crates/ragfs-cache-mooncake/README.md deleted file mode 100644 index 0695fd727c..0000000000 --- a/crates/ragfs-cache-mooncake/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# RAGFS Mooncake Cache Provider - -This crate maps the RAGFS `CacheProvider` contract to Mooncake Store without -changing `CachedFileSystem`. - -The optional official binding is pinned to Mooncake commit: - -```text -1352bbec43081e461356aaecf6c70cddd826b455 -``` - -Default tests use the same synchronous object boundary as the native binding, -so they run without Mooncake services or C++ libraries: - -```bash -cargo test -p ragfs-cache-mooncake -``` - -## Native Build - -Build the matching Mooncake checkout with Store Rust support: - -```bash -cmake -S /path/to/Mooncake -B /path/to/Mooncake/build \ - -DWITH_STORE=ON \ - -DWITH_STORE_RUST=ON -cmake --build /path/to/Mooncake/build \ - --target build_mooncake_store_rust -j -``` - -Before building OpenViking, provide the paths expected by the official crate: - -```bash -export MOONCAKE_BUILD_DIR=/path/to/Mooncake/build -export MOONCAKE_STORE_LIB_DIR=$MOONCAKE_BUILD_DIR/mooncake-store/src -export MOONCAKE_STORE_INCLUDE_DIR=/path/to/Mooncake/mooncake-store/include -export LD_LIBRARY_PATH="$MOONCAKE_STORE_LIB_DIR:$LD_LIBRARY_PATH" -``` - -Compile the adapter against the official binding: - -```bash -cargo check -p ragfs-cache-mooncake --features mooncake-native -``` - -## TCP Smoke Test - -With Mooncake Master and its metadata service running: - -```bash -OPENVIKING_RUN_MOONCAKE_INTEGRATION=true \ -MOONCAKE_LOCAL_HOSTNAME=127.0.0.1 \ -MOONCAKE_METADATA_SERVER=http://127.0.0.1:8080/metadata \ -MOONCAKE_MASTER_SERVER_ADDR=127.0.0.1:50051 \ -MOONCAKE_PROTOCOL=tcp \ -cargo test -p ragfs-cache-mooncake \ - --features mooncake-native \ - --test native_smoke -- --nocapture -``` - -The test writes one complete object, reads it back, removes it, verifies the -miss result, and closes the provider. - -## Docker Smoke Test - -The repository includes a Linux ARM64 image that builds the pinned Mooncake -commit, starts its HTTP metadata service and Master, runs Mooncake's official -Rust smoke test, and then runs the OpenViking provider smoke test: - -```bash -docker build \ - -t openviking-mooncake-test:arm64 \ - docker/mooncake-test - -docker run --rm --shm-size=4g \ - -v "$PWD:/workspace/OpenViking" \ - -v openviking-cargo-registry:/root/.cargo/registry \ - -v openviking-cargo-git:/root/.cargo/git \ - -v openviking-cargo-target:/workspace/OpenViking/target-docker \ - -e CARGO_TARGET_DIR=/workspace/OpenViking/target-docker \ - openviking-mooncake-test:arm64 -``` diff --git a/crates/ragfs-cache-mooncake/src/client.rs b/crates/ragfs-cache-mooncake/src/client.rs deleted file mode 100644 index 1893c87209..0000000000 --- a/crates/ragfs-cache-mooncake/src/client.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::{MooncakeObjectStore, MooncakeReplicateConfig, MooncakeStoreError}; -use ragfs::cache::{CacheError, CacheResult}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{RwLock, Semaphore}; - -pub(crate) struct MooncakeClient { - store: RwLock>>, - concurrency: Arc, - concurrency_limit: u32, - timeout: Duration, - closed: AtomicBool, -} - -impl MooncakeClient { - pub(crate) fn new( - store: Arc, - concurrency_limit: usize, - timeout: Duration, - ) -> Self { - Self { - store: RwLock::new(Some(store)), - concurrency: Arc::new(Semaphore::new(concurrency_limit)), - concurrency_limit: concurrency_limit as u32, - timeout, - closed: AtomicBool::new(false), - } - } - - async fn execute(&self, operation: &'static str, call: F) -> CacheResult - where - T: Send + 'static, - F: FnOnce(Arc) -> Result + Send + 'static, - { - if self.closed.load(Ordering::Acquire) { - return Err(CacheError::Unavailable( - "Mooncake provider is closed".into(), - )); - } - - let work = async { - let permit = Arc::clone(&self.concurrency) - .acquire_owned() - .await - .map_err(|_| CacheError::Unavailable("Mooncake client is closing".into()))?; - if self.closed.load(Ordering::Acquire) { - return Err(CacheError::Unavailable( - "Mooncake provider is closed".into(), - )); - } - let store = self.store.read().await.clone().ok_or_else(|| { - CacheError::Unavailable("Mooncake store has been released".into()) - })?; - tokio::task::spawn_blocking(move || { - let _permit = permit; - call(store) - }) - .await - .map_err(|error| { - CacheError::Internal(format!( - "Mooncake {operation} blocking task failed: {error}" - )) - })? - .map_err(map_store_error) - }; - - tokio::time::timeout(self.timeout, work) - .await - .map_err(|_| { - CacheError::Timeout(format!( - "Mooncake {operation} exceeded {} ms", - self.timeout.as_millis() - )) - })? - } - - pub(crate) async fn health_check(&self) -> CacheResult<()> { - self.execute("health_check", |store| store.health_check()) - .await - } - - pub(crate) async fn is_exist(&self, key: &str) -> CacheResult { - let key = key.to_owned(); - self.execute("is_exist", move |store| store.is_exist(&key)) - .await - } - - pub(crate) async fn get(&self, key: &str) -> CacheResult> { - let key = key.to_owned(); - self.execute("get", move |store| store.get(&key)).await - } - - pub(crate) async fn put( - &self, - key: &str, - value: &[u8], - replicate: &MooncakeReplicateConfig, - ) -> CacheResult<()> { - let key = key.to_owned(); - let value = value.to_vec(); - let replicate = replicate.clone(); - self.execute("put", move |store| store.put(&key, &value, &replicate)) - .await - } - - pub(crate) async fn remove(&self, key: &str) -> CacheResult<()> { - let key = key.to_owned(); - self.execute("remove", move |store| store.remove(&key, true)) - .await - } - - pub(crate) async fn close(&self) -> CacheResult<()> { - if self.closed.swap(true, Ordering::AcqRel) { - return Ok(()); - } - let permits = Arc::clone(&self.concurrency) - .acquire_many_owned(self.concurrency_limit) - .await - .map_err(|_| CacheError::Unavailable("Mooncake client is closing".into()))?; - self.store.write().await.take(); - drop(permits); - Ok(()) - } -} - -fn map_store_error(error: MooncakeStoreError) -> CacheError { - match error { - MooncakeStoreError::NotFound => { - CacheError::Unavailable("Mooncake object disappeared".into()) - } - MooncakeStoreError::Unavailable(message) => CacheError::Unavailable(message), - MooncakeStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message), - MooncakeStoreError::Internal(message) => CacheError::Internal(message), - } -} diff --git a/crates/ragfs-cache-mooncake/src/config.rs b/crates/ragfs-cache-mooncake/src/config.rs deleted file mode 100644 index c5dddf0067..0000000000 --- a/crates/ragfs-cache-mooncake/src/config.rs +++ /dev/null @@ -1,73 +0,0 @@ -use ragfs::cache::{CacheError, CacheResult}; - -/// Connection and execution settings for a Mooncake provider. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MooncakeConfig { - /// Hostname or IP address registered for the local Mooncake client. - pub local_hostname: String, - /// Metadata service URL. - pub metadata_server: String, - /// Mooncake Master service address. - pub master_server_addr: String, - /// Transfer protocol, such as `tcp` or `rdma`. - pub protocol: String, - /// Optional transport device name. - pub device_name: String, - /// Size of the global Mooncake segment in bytes. - pub global_segment_size: u64, - /// Size of the local transfer buffer in bytes. - pub local_buffer_size: u64, - /// Number of object replicas requested for writes. - pub replica_num: usize, - /// Maximum number of concurrent synchronous SDK operations. - pub sdk_concurrency: usize, - /// Timeout applied while waiting for each SDK operation. - pub operation_timeout_ms: u64, -} - -impl MooncakeConfig { - pub(crate) fn validate(&self) -> CacheResult<()> { - for (name, value) in [ - ("local_hostname", self.local_hostname.as_str()), - ("metadata_server", self.metadata_server.as_str()), - ("master_server_addr", self.master_server_addr.as_str()), - ("protocol", self.protocol.as_str()), - ] { - if value.trim().is_empty() { - return Err(CacheError::InvalidArgument(format!( - "Mooncake {name} must not be empty" - ))); - } - } - if !matches!( - self.protocol.as_str(), - "tcp" | "rdma" | "ascend" | "cxl" | "nvlink" | "barex" - ) { - return Err(CacheError::InvalidArgument(format!( - "unsupported Mooncake protocol: {}", - self.protocol - ))); - } - if self.global_segment_size == 0 { - return Err(CacheError::InvalidArgument( - "Mooncake global_segment_size must be greater than zero".into(), - )); - } - if self.local_buffer_size == 0 { - return Err(CacheError::InvalidArgument( - "Mooncake local_buffer_size must be greater than zero".into(), - )); - } - if self.sdk_concurrency == 0 || self.sdk_concurrency > u32::MAX as usize { - return Err(CacheError::InvalidArgument( - "Mooncake sdk_concurrency must be between 1 and u32::MAX".into(), - )); - } - if self.operation_timeout_ms == 0 { - return Err(CacheError::InvalidArgument( - "Mooncake operation_timeout_ms must be greater than zero".into(), - )); - } - Ok(()) - } -} diff --git a/crates/ragfs-cache-mooncake/src/error.rs b/crates/ragfs-cache-mooncake/src/error.rs deleted file mode 100644 index 273a5ed83f..0000000000 --- a/crates/ragfs-cache-mooncake/src/error.rs +++ /dev/null @@ -1,16 +0,0 @@ -/// Errors returned by the synchronous Mooncake object API. -#[derive(Debug, thiserror::Error)] -pub enum MooncakeStoreError { - /// The requested object does not exist. - #[error("object not found")] - NotFound, - /// Mooncake or one of its required services is unavailable. - #[error("Mooncake unavailable: {0}")] - Unavailable(String), - /// Mooncake rejected an argument. - #[error("invalid Mooncake argument: {0}")] - InvalidArgument(String), - /// Mooncake returned an unspecified operation failure. - #[error("Mooncake operation failed: {0}")] - Internal(String), -} diff --git a/crates/ragfs-cache-mooncake/src/lib.rs b/crates/ragfs-cache-mooncake/src/lib.rs deleted file mode 100644 index 7ba5a1c1f6..0000000000 --- a/crates/ragfs-cache-mooncake/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Mooncake cache provider adapter for RAGFS. - -mod client; -mod config; -mod error; -#[cfg(feature = "mooncake-native")] -mod native; -mod provider; -mod store; - -pub use config::MooncakeConfig; -pub use error::MooncakeStoreError; -pub use provider::MooncakeProvider; -pub use store::{MooncakeObjectStore, MooncakeReplicateConfig}; diff --git a/crates/ragfs-cache-mooncake/src/native.rs b/crates/ragfs-cache-mooncake/src/native.rs deleted file mode 100644 index 48e4b626b6..0000000000 --- a/crates/ragfs-cache-mooncake/src/native.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::{ - MooncakeConfig, MooncakeObjectStore, MooncakeProvider, MooncakeReplicateConfig, - MooncakeStoreError, -}; -use mooncake_store::{MooncakeStore, ReplicateConfig as NativeReplicateConfig, StoreError}; -use ragfs::cache::{CacheError, CacheResult}; -use std::sync::Arc; -use std::time::Duration; - -struct NativeMooncakeStore { - store: MooncakeStore, -} - -impl NativeMooncakeStore { - fn connect(config: &MooncakeConfig) -> Result { - let store = MooncakeStore::new().map_err(map_initialization_error)?; - store - .setup( - &config.local_hostname, - &config.metadata_server, - config.global_segment_size, - config.local_buffer_size, - &config.protocol, - &config.device_name, - &config.master_server_addr, - ) - .map_err(map_initialization_error)?; - Ok(Self { store }) - } -} - -impl MooncakeObjectStore for NativeMooncakeStore { - fn health_check(&self) -> Result<(), MooncakeStoreError> { - self.store.health_check().map_err(map_initialization_error) - } - - fn is_exist(&self, key: &str) -> Result { - self.store.is_exist(key).map_err(map_operation_error) - } - - fn get(&self, key: &str) -> Result, MooncakeStoreError> { - self.store.get(key).map_err(map_operation_error) - } - - fn put( - &self, - key: &str, - value: &[u8], - replicate: &MooncakeReplicateConfig, - ) -> Result<(), MooncakeStoreError> { - let native = NativeReplicateConfig { - replica_num: replicate.replica_num, - with_soft_pin: replicate.with_soft_pin, - with_hard_pin: replicate.with_hard_pin, - preferred_segments: replicate.preferred_segments.clone(), - }; - self.store - .put(key, value, Some(&native)) - .map_err(map_operation_error) - } - - fn remove(&self, key: &str, force: bool) -> Result<(), MooncakeStoreError> { - self.store.remove(key, force).map_err(map_operation_error) - } -} - -impl MooncakeProvider { - /// Connect to Mooncake through the pinned official Rust binding. - pub async fn connect(config: MooncakeConfig) -> CacheResult { - config.validate()?; - let setup_config = config.clone(); - let setup_timeout = Duration::from_millis(config.operation_timeout_ms); - let store = tokio::time::timeout( - setup_timeout, - tokio::task::spawn_blocking(move || NativeMooncakeStore::connect(&setup_config)), - ) - .await - .map_err(|_| { - CacheError::Timeout(format!( - "Mooncake setup exceeded {} ms", - setup_timeout.as_millis() - )) - })? - .map_err(|error| CacheError::Internal(format!("Mooncake setup task failed: {error}")))? - .map_err(map_cache_error)?; - - Self::from_store(config, Arc::new(store)).await - } -} - -fn map_initialization_error(error: StoreError) -> MooncakeStoreError { - match error { - StoreError::InvalidString(error) => MooncakeStoreError::InvalidArgument(error.to_string()), - StoreError::InvalidArgument(message) => MooncakeStoreError::InvalidArgument(message), - StoreError::NullHandle | StoreError::OperationFailed(_) | StoreError::NotFound => { - MooncakeStoreError::Unavailable(error.to_string()) - } - } -} - -fn map_operation_error(error: StoreError) -> MooncakeStoreError { - match error { - StoreError::NotFound => MooncakeStoreError::NotFound, - StoreError::InvalidString(error) => MooncakeStoreError::InvalidArgument(error.to_string()), - StoreError::InvalidArgument(message) => MooncakeStoreError::InvalidArgument(message), - StoreError::NullHandle | StoreError::OperationFailed(_) => { - MooncakeStoreError::Internal(error.to_string()) - } - } -} - -fn map_cache_error(error: MooncakeStoreError) -> CacheError { - match error { - MooncakeStoreError::NotFound => { - CacheError::Unavailable("Mooncake setup object not found".into()) - } - MooncakeStoreError::Unavailable(message) => CacheError::Unavailable(message), - MooncakeStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message), - MooncakeStoreError::Internal(message) => CacheError::Internal(message), - } -} diff --git a/crates/ragfs-cache-mooncake/src/provider.rs b/crates/ragfs-cache-mooncake/src/provider.rs deleted file mode 100644 index 6ae2f2590d..0000000000 --- a/crates/ragfs-cache-mooncake/src/provider.rs +++ /dev/null @@ -1,209 +0,0 @@ -use crate::client::MooncakeClient; -use crate::{MooncakeConfig, MooncakeObjectStore, MooncakeReplicateConfig}; -use async_trait::async_trait; -use bytes::Bytes; -use futures::stream::{self, StreamExt}; -use ragfs::cache::{CacheError, CacheProvider, CacheResult, ProviderCapabilities}; -use std::collections::HashSet; -use std::fmt; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; - -/// Mooncake implementation of the common RAGFS cache provider contract. -pub struct MooncakeProvider { - client: Arc, - replicate: MooncakeReplicateConfig, - batch_concurrency: usize, - known_keys: Mutex>, -} - -impl fmt::Debug for MooncakeProvider { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("MooncakeProvider") - .field("batch_concurrency", &self.batch_concurrency) - .finish_non_exhaustive() - } -} - -impl MooncakeProvider { - /// Construct a provider over a Mooncake-compatible object store. - /// - /// Construction validates configuration and performs a health check. - pub async fn from_store( - config: MooncakeConfig, - store: Arc, - ) -> CacheResult { - config.validate()?; - let client = Arc::new(MooncakeClient::new( - store, - config.sdk_concurrency, - Duration::from_millis(config.operation_timeout_ms), - )); - client.health_check().await?; - Ok(Self { - client, - replicate: MooncakeReplicateConfig { - replica_num: config.replica_num, - ..MooncakeReplicateConfig::default() - }, - batch_concurrency: config.sdk_concurrency, - known_keys: Mutex::new(HashSet::new()), - }) - } - - /// Return a startup error when the official native binding is not compiled. - #[cfg(not(feature = "mooncake-native"))] - pub async fn connect(config: MooncakeConfig) -> CacheResult { - config.validate()?; - Err(CacheError::Unavailable( - "Mooncake support requires the mooncake-native feature".into(), - )) - } - - /// Check whether the connected Mooncake services are healthy. - pub async fn health_check(&self) -> CacheResult<()> { - self.client.health_check().await - } - - async fn get_object(&self, key: &str) -> CacheResult> { - if !self.client.is_exist(key).await? { - return Ok(None); - } - match self.client.get(key).await { - Ok(value) => Ok(Some(Bytes::from(value))), - Err(error) => match self.client.is_exist(key).await { - Ok(false) => Ok(None), - Ok(true) | Err(_) => Err(error), - }, - } - } - - async fn delete_object(&self, key: &str) -> CacheResult<()> { - if !self.client.is_exist(key).await? { - lock_known_keys(&self.known_keys)?.remove(key); - return Ok(()); - } - match self.client.remove(key).await { - Ok(()) => { - lock_known_keys(&self.known_keys)?.remove(key); - Ok(()) - } - Err(error) => match self.client.is_exist(key).await { - Ok(false) => { - lock_known_keys(&self.known_keys)?.remove(key); - Ok(()) - } - Ok(true) | Err(_) => Err(error), - }, - } - } -} - -fn lock_known_keys( - known_keys: &Mutex>, -) -> CacheResult>> { - known_keys - .lock() - .map_err(|_| CacheError::Internal("Mooncake key tracker is poisoned".into())) -} - -#[async_trait] -impl CacheProvider for MooncakeProvider { - fn name(&self) -> &'static str { - "mooncake" - } - - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities { - batch_get: true, - batch_put: true, - native_ttl: false, - } - } - - async fn get(&self, key: &str) -> CacheResult> { - self.get_object(key).await - } - - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> { - self.client.put(key, &value, &self.replicate).await?; - lock_known_keys(&self.known_keys)?.insert(key.to_owned()); - Ok(()) - } - - async fn delete(&self, key: &str) -> CacheResult<()> { - self.delete_object(key).await - } - - async fn exists(&self, key: &str) -> CacheResult { - self.client.is_exist(key).await - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - let mut values = stream::iter(keys.iter().cloned().enumerate()) - .map(|(index, key)| async move { (index, self.get_object(&key).await) }) - .buffer_unordered(self.batch_concurrency) - .collect::>() - .await; - values.sort_by_key(|(index, _)| *index); - values - .into_iter() - .map(|(_, value)| value) - .collect::>>() - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - let results = stream::iter(entries) - .map(|(key, value)| async move { self.put(&key, value).await }) - .buffer_unordered(self.batch_concurrency) - .collect::>() - .await; - results.into_iter().collect() - } - - async fn invalidate(&self, keys: &[String]) -> CacheResult<()> { - let results = stream::iter(keys.iter().cloned()) - .map(|key| async move { self.delete_object(&key).await }) - .buffer_unordered(self.batch_concurrency) - .collect::>() - .await; - results.into_iter().collect() - } - - async fn flush(&self) -> CacheResult<()> { - let keys = lock_known_keys(&self.known_keys)? - .iter() - .cloned() - .collect::>(); - self.invalidate(&keys).await - } - - async fn close(&self) -> CacheResult<()> { - self.client.close().await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::panic::{self, AssertUnwindSafe}; - - #[test] - fn lock_known_keys_returns_internal_error_when_poisoned() { - let known_keys = Mutex::new(HashSet::new()); - - let _ = panic::catch_unwind(AssertUnwindSafe(|| { - let _guard = known_keys.lock().unwrap(); - panic!("poison known_keys"); - })); - - let error = lock_known_keys(&known_keys).unwrap_err(); - - assert!(matches!( - error, - CacheError::Internal(message) - if message == "Mooncake key tracker is poisoned" - )); - } -} diff --git a/crates/ragfs-cache-mooncake/src/store.rs b/crates/ragfs-cache-mooncake/src/store.rs deleted file mode 100644 index c3fdde08a9..0000000000 --- a/crates/ragfs-cache-mooncake/src/store.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::MooncakeStoreError; - -/// Replication options applied to each Mooncake object write. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct MooncakeReplicateConfig { - /// Number of replicas to create. - pub replica_num: usize, - /// Prefer a replica local to the client. - pub with_soft_pin: bool, - /// Prevent eviction of the object. - pub with_hard_pin: bool, - /// Preferred Mooncake segment names. - pub preferred_segments: Vec, -} - -/// Synchronous object operations exposed by Mooncake's Rust binding. -/// -/// The asynchronous client executes every method on Tokio's blocking pool. -pub trait MooncakeObjectStore: Send + Sync + 'static { - /// Check that Mooncake services are reachable. - fn health_check(&self) -> Result<(), MooncakeStoreError>; - /// Return whether an object exists. - fn is_exist(&self, key: &str) -> Result; - /// Read a complete object. - fn get(&self, key: &str) -> Result, MooncakeStoreError>; - /// Store a complete object. - fn put( - &self, - key: &str, - value: &[u8], - replicate: &MooncakeReplicateConfig, - ) -> Result<(), MooncakeStoreError>; - /// Remove an object. - fn remove(&self, key: &str, force: bool) -> Result<(), MooncakeStoreError>; -} diff --git a/crates/ragfs-cache-mooncake/tests/cached_filesystem.rs b/crates/ragfs-cache-mooncake/tests/cached_filesystem.rs deleted file mode 100644 index eeb73d6ca3..0000000000 --- a/crates/ragfs-cache-mooncake/tests/cached_filesystem.rs +++ /dev/null @@ -1,335 +0,0 @@ -use async_trait::async_trait; -use ragfs::cache::{CacheNamespace, CachePolicy, CacheProvider, CachedFileSystem}; -use ragfs::core::{GrepResult, TreeEntry}; -use ragfs::plugins::MemFileSystem; -use ragfs::{FileInfo, FileSystem, Result, WriteFlag}; -use ragfs_cache_mooncake::{ - MooncakeConfig, MooncakeObjectStore, MooncakeProvider, MooncakeReplicateConfig, - MooncakeStoreError, -}; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; - -#[derive(Default)] -struct SharedStore { - values: Mutex>>, - available: AtomicBool, -} - -impl SharedStore { - fn available() -> Self { - Self { - available: AtomicBool::new(true), - ..Self::default() - } - } - - fn check(&self) -> std::result::Result<(), MooncakeStoreError> { - if self.available.load(Ordering::SeqCst) { - Ok(()) - } else { - Err(MooncakeStoreError::Unavailable("store unavailable".into())) - } - } -} - -impl MooncakeObjectStore for SharedStore { - fn health_check(&self) -> std::result::Result<(), MooncakeStoreError> { - self.check() - } - - fn is_exist(&self, key: &str) -> std::result::Result { - self.check()?; - Ok(self.values.lock().unwrap().contains_key(key)) - } - - fn get(&self, key: &str) -> std::result::Result, MooncakeStoreError> { - self.check()?; - self.values - .lock() - .unwrap() - .get(key) - .cloned() - .ok_or(MooncakeStoreError::NotFound) - } - - fn put( - &self, - key: &str, - value: &[u8], - _replicate: &MooncakeReplicateConfig, - ) -> std::result::Result<(), MooncakeStoreError> { - self.check()?; - self.values - .lock() - .unwrap() - .insert(key.to_owned(), value.to_vec()); - Ok(()) - } - - fn remove(&self, key: &str, _force: bool) -> std::result::Result<(), MooncakeStoreError> { - self.check()?; - self.values.lock().unwrap().remove(key); - Ok(()) - } -} - -#[derive(Clone)] -struct CountingFileSystem { - inner: Arc, - reads: Arc, -} - -impl CountingFileSystem { - fn new() -> Self { - Self { - inner: Arc::new(MemFileSystem::new()), - reads: Arc::new(AtomicU64::new(0)), - } - } - - fn read_count(&self) -> u64 { - self.reads.load(Ordering::SeqCst) - } -} - -#[async_trait] -impl FileSystem for CountingFileSystem { - async fn create(&self, path: &str) -> Result<()> { - self.inner.create(path).await - } - - async fn mkdir(&self, path: &str, mode: u32) -> Result<()> { - self.inner.mkdir(path, mode).await - } - - async fn remove(&self, path: &str) -> Result<()> { - self.inner.remove(path).await - } - - async fn remove_all(&self, path: &str) -> Result<()> { - self.inner.remove_all(path).await - } - - async fn read(&self, path: &str, offset: u64, size: u64) -> Result> { - self.reads.fetch_add(1, Ordering::SeqCst); - self.inner.read(path, offset, size).await - } - - async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result { - self.inner.write(path, data, offset, flags).await - } - - async fn read_dir(&self, path: &str) -> Result> { - self.inner.read_dir(path).await - } - - async fn stat(&self, path: &str) -> Result { - self.inner.stat(path).await - } - - async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> { - self.inner.rename(old_path, new_path).await - } - - async fn chmod(&self, path: &str, mode: u32) -> Result<()> { - self.inner.chmod(path, mode).await - } - - async fn truncate(&self, path: &str, size: u64) -> Result<()> { - self.inner.truncate(path, size).await - } - - async fn grep( - &self, - path: &str, - pattern: &str, - recursive: bool, - case_insensitive: bool, - node_limit: Option, - exclude_path: Option<&str>, - level_limit: Option, - ) -> Result { - self.inner - .grep( - path, - pattern, - recursive, - case_insensitive, - node_limit, - exclude_path, - level_limit, - ) - .await - } - - async fn tree_directory( - &self, - path: &str, - show_hidden: bool, - node_limit: Option, - level_limit: Option, - ) -> Result> { - self.inner - .tree_directory(path, show_hidden, node_limit, level_limit) - .await - } -} - -fn config() -> MooncakeConfig { - MooncakeConfig { - local_hostname: "127.0.0.1".into(), - metadata_server: "http://127.0.0.1:8080/metadata".into(), - master_server_addr: "127.0.0.1:50051".into(), - protocol: "tcp".into(), - device_name: String::new(), - global_segment_size: 512 << 20, - local_buffer_size: 128 << 20, - replica_num: 2, - sdk_concurrency: 4, - operation_timeout_ms: 100, - } -} - -async fn cached_fs( - backend: CountingFileSystem, - store: Arc, - namespace: &str, -) -> CachedFileSystem { - let provider: Arc = - Arc::new(MooncakeProvider::from_store(config(), store).await.unwrap()); - CachedFileSystem::new( - Box::new(backend), - provider, - CacheNamespace::new(namespace), - CachePolicy::default(), - ) -} - -#[tokio::test] -async fn mooncake_provider_supports_miss_fill_hit_and_write_after_read() { - let backend = CountingFileSystem::new(); - backend - .write("/value.md", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let fs = cached_fs(backend, Arc::new(SharedStore::available()), "read-write").await; - - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old"); - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old"); - assert_eq!(probe.read_count(), 1); - - fs.write("/value.md", b"new", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"new"); - assert_eq!(probe.read_count(), 1); -} - -#[tokio::test] -async fn mooncake_provider_preserves_invalidation_and_subtree_generation_rules() { - let backend = CountingFileSystem::new(); - backend.mkdir("/tree", 0o755).await.unwrap(); - backend - .write("/tree/leaf", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let fs = cached_fs(backend, Arc::new(SharedStore::available()), "invalidation").await; - - assert_eq!(fs.read("/tree/leaf", 0, 0).await.unwrap(), b"old"); - fs.rename("/tree/leaf", "/tree/moved").await.unwrap(); - assert!(fs.read("/tree/leaf", 0, 0).await.is_err()); - assert_eq!(fs.read("/tree/moved", 0, 0).await.unwrap(), b"old"); - fs.remove("/tree/moved").await.unwrap(); - assert!(fs.read("/tree/moved", 0, 0).await.is_err()); - - direct - .write("/tree/leaf", b"stale", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/tree/leaf", 0, 0).await.unwrap(), b"stale"); - fs.remove_all("/tree").await.unwrap(); - direct.mkdir("/tree", 0o755).await.unwrap(); - direct - .write("/tree/leaf", b"fresh", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/tree/leaf", 0, 0).await.unwrap(), b"fresh"); -} - -#[tokio::test] -async fn mooncake_provider_caches_directory_entries_and_invalidates_directory_renames() { - let backend = CountingFileSystem::new(); - backend.mkdir("/root", 0o755).await.unwrap(); - backend.mkdir("/root/old", 0o755).await.unwrap(); - backend - .write("/root/old/leaf", b"value", 0, WriteFlag::Create) - .await - .unwrap(); - let fs = cached_fs(backend, Arc::new(SharedStore::available()), "directories").await; - - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - fs.mkdir("/root/created", 0o755).await.unwrap(); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2); - - assert_eq!(fs.read("/root/old/leaf", 0, 0).await.unwrap(), b"value"); - fs.rename("/root/old", "/root/moved").await.unwrap(); - assert!(fs.read("/root/old/leaf", 0, 0).await.is_err()); - assert_eq!(fs.read("/root/moved/leaf", 0, 0).await.unwrap(), b"value"); -} - -#[tokio::test] -async fn unavailable_mooncake_falls_back_without_breaking_backend_reads() { - let backend = CountingFileSystem::new(); - backend - .write("/available.md", b"backend", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let store = Arc::new(SharedStore::available()); - let fs = cached_fs(backend, store.clone(), "fallback").await; - store.available.store(false, Ordering::SeqCst); - - assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend"); - assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend"); - assert_eq!(probe.read_count(), 2); - assert!(fs.metrics().snapshot().errors >= 1); -} - -#[tokio::test] -async fn multiple_wrappers_share_one_mooncake_provider_without_key_collisions() { - let store = Arc::new(SharedStore::available()); - let provider: Arc = - Arc::new(MooncakeProvider::from_store(config(), store).await.unwrap()); - let first_backend = CountingFileSystem::new(); - first_backend - .write("/same.md", b"first", 0, WriteFlag::Create) - .await - .unwrap(); - let second_backend = CountingFileSystem::new(); - second_backend - .write("/same.md", b"second", 0, WriteFlag::Create) - .await - .unwrap(); - let first = CachedFileSystem::new( - Box::new(first_backend), - provider.clone(), - CacheNamespace::new("mount-one"), - CachePolicy::default(), - ); - let second = CachedFileSystem::new( - Box::new(second_backend), - provider, - CacheNamespace::new("mount-two"), - CachePolicy::default(), - ); - - assert_eq!(first.read("/same.md", 0, 0).await.unwrap(), b"first"); - assert_eq!(second.read("/same.md", 0, 0).await.unwrap(), b"second"); - assert_eq!(first.read("/same.md", 0, 0).await.unwrap(), b"first"); - assert_eq!(second.read("/same.md", 0, 0).await.unwrap(), b"second"); -} diff --git a/crates/ragfs-cache-mooncake/tests/native_smoke.rs b/crates/ragfs-cache-mooncake/tests/native_smoke.rs deleted file mode 100644 index 6419ab8613..0000000000 --- a/crates/ragfs-cache-mooncake/tests/native_smoke.rs +++ /dev/null @@ -1,43 +0,0 @@ -#![cfg(feature = "mooncake-native")] - -use bytes::Bytes; -use ragfs::cache::CacheProvider; -use ragfs_cache_mooncake::{MooncakeConfig, MooncakeProvider}; - -fn required_env(name: &str) -> String { - std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set for native smoke test")) -} - -#[tokio::test] -async fn native_mooncake_round_trips_complete_objects_over_tcp() { - if std::env::var("OPENVIKING_RUN_MOONCAKE_INTEGRATION").as_deref() != Ok("true") { - return; - } - - let config = MooncakeConfig { - local_hostname: required_env("MOONCAKE_LOCAL_HOSTNAME"), - metadata_server: required_env("MOONCAKE_METADATA_SERVER"), - master_server_addr: required_env("MOONCAKE_MASTER_SERVER_ADDR"), - protocol: std::env::var("MOONCAKE_PROTOCOL").unwrap_or_else(|_| "tcp".into()), - device_name: std::env::var("MOONCAKE_DEVICE_NAME").unwrap_or_default(), - global_segment_size: 512 << 20, - local_buffer_size: 128 << 20, - replica_num: 1, - sdk_concurrency: 4, - operation_timeout_ms: 5_000, - }; - let provider = MooncakeProvider::connect(config).await.unwrap(); - let key = format!("openviking:smoke:{}", std::process::id()); - - provider - .put(&key, Bytes::from_static(b"mooncake-object")) - .await - .unwrap(); - assert_eq!( - provider.get(&key).await.unwrap(), - Some(Bytes::from_static(b"mooncake-object")) - ); - provider.delete(&key).await.unwrap(); - assert_eq!(provider.get(&key).await.unwrap(), None); - provider.close().await.unwrap(); -} diff --git a/crates/ragfs-cache-mooncake/tests/provider_contract.rs b/crates/ragfs-cache-mooncake/tests/provider_contract.rs deleted file mode 100644 index 9fb3464472..0000000000 --- a/crates/ragfs-cache-mooncake/tests/provider_contract.rs +++ /dev/null @@ -1,357 +0,0 @@ -use bytes::Bytes; -use ragfs::cache::{CacheError, CacheProvider}; -use ragfs_cache_mooncake::{ - MooncakeConfig, MooncakeObjectStore, MooncakeProvider, MooncakeReplicateConfig, - MooncakeStoreError, -}; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -#[derive(Default)] -struct FakeStore { - values: Mutex>>, - remove_force_flags: Mutex>, - healthy: AtomicBool, - fail_get: AtomicBool, - remove_during_get: AtomicBool, - delay_ms: AtomicUsize, - active: AtomicUsize, - max_active: AtomicUsize, -} - -impl FakeStore { - fn healthy() -> Self { - Self { - healthy: AtomicBool::new(true), - ..Self::default() - } - } - - fn enter(&self) -> ActiveGuard<'_> { - let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; - self.max_active.fetch_max(active, Ordering::SeqCst); - let delay = self.delay_ms.load(Ordering::SeqCst); - if delay > 0 { - std::thread::sleep(Duration::from_millis(delay as u64)); - } - ActiveGuard { store: self } - } -} - -struct ActiveGuard<'a> { - store: &'a FakeStore, -} - -impl Drop for ActiveGuard<'_> { - fn drop(&mut self) { - self.store.active.fetch_sub(1, Ordering::SeqCst); - } -} - -impl MooncakeObjectStore for FakeStore { - fn health_check(&self) -> Result<(), MooncakeStoreError> { - if self.healthy.load(Ordering::SeqCst) { - Ok(()) - } else { - Err(MooncakeStoreError::Unavailable("not healthy".into())) - } - } - - fn is_exist(&self, key: &str) -> Result { - let _guard = self.enter(); - Ok(self.values.lock().unwrap().contains_key(key)) - } - - fn get(&self, key: &str) -> Result, MooncakeStoreError> { - let _guard = self.enter(); - if self.remove_during_get.load(Ordering::SeqCst) { - self.values.lock().unwrap().remove(key); - return Err(MooncakeStoreError::NotFound); - } - if self.fail_get.load(Ordering::SeqCst) { - return Err(MooncakeStoreError::Internal("get failed".into())); - } - self.values - .lock() - .unwrap() - .get(key) - .cloned() - .ok_or(MooncakeStoreError::NotFound) - } - - fn put( - &self, - key: &str, - value: &[u8], - _replicate: &MooncakeReplicateConfig, - ) -> Result<(), MooncakeStoreError> { - let _guard = self.enter(); - self.values - .lock() - .unwrap() - .insert(key.to_string(), value.to_vec()); - Ok(()) - } - - fn remove(&self, key: &str, force: bool) -> Result<(), MooncakeStoreError> { - let _guard = self.enter(); - self.remove_force_flags.lock().unwrap().push(force); - self.values.lock().unwrap().remove(key); - Ok(()) - } -} - -fn config() -> MooncakeConfig { - MooncakeConfig { - local_hostname: "127.0.0.1".into(), - metadata_server: "http://127.0.0.1:8080/metadata".into(), - master_server_addr: "127.0.0.1:50051".into(), - protocol: "tcp".into(), - device_name: String::new(), - global_segment_size: 512 << 20, - local_buffer_size: 128 << 20, - replica_num: 2, - sdk_concurrency: 2, - operation_timeout_ms: 100, - } -} - -async fn provider(store: Arc) -> MooncakeProvider { - MooncakeProvider::from_store(config(), store).await.unwrap() -} - -#[tokio::test] -async fn initialization_requires_valid_config_and_health_check() { - let mut invalid = config(); - invalid.sdk_concurrency = 0; - let error = MooncakeProvider::from_store(invalid, Arc::new(FakeStore::healthy())) - .await - .unwrap_err(); - assert!(matches!(error, CacheError::InvalidArgument(_))); - - let mut invalid_protocol = config(); - invalid_protocol.protocol = "unknown".into(); - let error = MooncakeProvider::from_store(invalid_protocol, Arc::new(FakeStore::healthy())) - .await - .unwrap_err(); - assert!(matches!(error, CacheError::InvalidArgument(_))); - - let error = MooncakeProvider::from_store(config(), Arc::new(FakeStore::default())) - .await - .unwrap_err(); - assert!(matches!(error, CacheError::Unavailable(_))); -} - -#[tokio::test] -async fn get_distinguishes_hit_miss_and_internal_error() { - let store = Arc::new(FakeStore::healthy()); - store - .values - .lock() - .unwrap() - .insert("hit".into(), b"value".to_vec()); - let provider = provider(store.clone()).await; - - assert_eq!( - provider.get("hit").await.unwrap(), - Some(Bytes::from_static(b"value")) - ); - assert_eq!(provider.get("missing").await.unwrap(), None); - - store.fail_get.store(true, Ordering::SeqCst); - let error = provider.get("hit").await.unwrap_err(); - assert!(matches!(error, CacheError::Internal(_))); -} - -#[tokio::test] -async fn object_removed_between_exist_and_get_is_a_miss() { - let store = Arc::new(FakeStore::healthy()); - store - .values - .lock() - .unwrap() - .insert("racy".into(), b"value".to_vec()); - store.remove_during_get.store(true, Ordering::SeqCst); - let provider = provider(store).await; - - assert_eq!(provider.get("racy").await.unwrap(), None); -} - -#[tokio::test] -async fn put_delete_and_batch_operations_preserve_object_semantics() { - let store = Arc::new(FakeStore::healthy()); - let provider = provider(store.clone()).await; - - provider.put("one", Bytes::from_static(b"1")).await.unwrap(); - provider - .batch_put(vec![ - ("two".into(), Bytes::from_static(b"2")), - ("three".into(), Bytes::from_static(b"3")), - ]) - .await - .unwrap(); - assert_eq!( - provider - .batch_get(&["three".into(), "missing".into(), "one".into()]) - .await - .unwrap(), - vec![ - Some(Bytes::from_static(b"3")), - None, - Some(Bytes::from_static(b"1")) - ] - ); - - provider.delete("one").await.unwrap(); - provider.delete("one").await.unwrap(); - assert_eq!(provider.get("one").await.unwrap(), None); - - provider.invalidate(&["two".into()]).await.unwrap(); - assert_eq!(provider.get("two").await.unwrap(), None); - provider.flush().await.unwrap(); - assert_eq!(provider.get("three").await.unwrap(), None); - assert!( - store - .remove_force_flags - .lock() - .unwrap() - .iter() - .all(|force| *force), - "cache invalidation must override active Mooncake leases" - ); - - assert!(provider.capabilities().batch_get); - assert!(provider.capabilities().batch_put); - assert!(!provider.capabilities().native_ttl); -} - -#[tokio::test] -async fn blocking_calls_are_bounded_and_time_out_without_becoming_misses() { - let store = Arc::new(FakeStore::healthy()); - store - .values - .lock() - .unwrap() - .insert("slow".into(), b"value".to_vec()); - store.delay_ms.store(40, Ordering::SeqCst); - let mut bounded_config = config(); - bounded_config.operation_timeout_ms = 500; - let provider = Arc::new( - MooncakeProvider::from_store(bounded_config, store.clone()) - .await - .unwrap(), - ); - - let mut tasks = Vec::new(); - for _ in 0..8 { - let provider = provider.clone(); - tasks.push(tokio::spawn( - async move { provider.get("slow").await.unwrap() }, - )); - } - for task in tasks { - assert_eq!(task.await.unwrap(), Some(Bytes::from_static(b"value"))); - } - assert!(store.max_active.load(Ordering::SeqCst) <= 2); - - let timeout_store = Arc::new(FakeStore::healthy()); - timeout_store.delay_ms.store(80, Ordering::SeqCst); - let mut timeout_config = config(); - timeout_config.operation_timeout_ms = 10; - let timeout_provider = MooncakeProvider::from_store(timeout_config, timeout_store) - .await - .unwrap(); - let error = timeout_provider.get("slow").await.unwrap_err(); - assert!(matches!(error, CacheError::Timeout(_))); -} - -#[tokio::test] -async fn close_rejects_new_operations() { - let provider = provider(Arc::new(FakeStore::healthy())).await; - provider.close().await.unwrap(); - - let error = provider.get("key").await.unwrap_err(); - assert!(matches!(error, CacheError::Unavailable(_))); -} - -struct SlowDropStore { - started: Arc, - dropped: Arc, -} - -impl Drop for SlowDropStore { - fn drop(&mut self) { - self.dropped.store(true, Ordering::SeqCst); - } -} - -impl MooncakeObjectStore for SlowDropStore { - fn health_check(&self) -> Result<(), MooncakeStoreError> { - Ok(()) - } - - fn is_exist(&self, _key: &str) -> Result { - self.started.store(true, Ordering::SeqCst); - std::thread::sleep(Duration::from_millis(80)); - Ok(false) - } - - fn get(&self, _key: &str) -> Result, MooncakeStoreError> { - unreachable!("missing object must not be read") - } - - fn put( - &self, - _key: &str, - _value: &[u8], - _replicate: &MooncakeReplicateConfig, - ) -> Result<(), MooncakeStoreError> { - Ok(()) - } - - fn remove(&self, _key: &str, _force: bool) -> Result<(), MooncakeStoreError> { - Ok(()) - } -} - -#[tokio::test] -async fn close_waits_for_inflight_calls_before_releasing_store() { - let started = Arc::new(AtomicBool::new(false)); - let dropped = Arc::new(AtomicBool::new(false)); - let store = Arc::new(SlowDropStore { - started: started.clone(), - dropped: dropped.clone(), - }); - let provider = Arc::new( - MooncakeProvider::from_store(config(), store.clone()) - .await - .unwrap(), - ); - drop(store); - - let reader = { - let provider = provider.clone(); - tokio::spawn(async move { provider.get("slow").await }) - }; - while !started.load(Ordering::SeqCst) { - tokio::task::yield_now().await; - } - - let close_started = Instant::now(); - provider.close().await.unwrap(); - assert!(close_started.elapsed() >= Duration::from_millis(50)); - assert_eq!(reader.await.unwrap().unwrap(), None); - assert!(dropped.load(Ordering::SeqCst)); -} - -#[cfg(not(feature = "mooncake-native"))] -#[tokio::test] -async fn connect_without_native_feature_returns_startup_error() { - let error = MooncakeProvider::connect(config()).await.unwrap_err(); - assert!(matches!( - error, - CacheError::Unavailable(message) if message.contains("mooncake-native") - )); -} diff --git a/crates/ragfs-cache-yuanrong-sys/Cargo.toml b/crates/ragfs-cache-yuanrong-sys/Cargo.toml deleted file mode 100644 index 76d03847d1..0000000000 --- a/crates/ragfs-cache-yuanrong-sys/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "ragfs-cache-yuanrong-sys" -version = "0.1.0" -edition = "2021" -rust-version = "1.91.1" -license = "Apache-2.0" -description = "Native C ABI bridge for the Yuanrong DataSystem C++ SDK" -links = "openviking_yuanrong_bridge" -build = "build.rs" - -[build-dependencies] -cc = "1" - -[features] -default = [] -native = [] diff --git a/crates/ragfs-cache-yuanrong-sys/build.rs b/crates/ragfs-cache-yuanrong-sys/build.rs deleted file mode 100644 index c0d2315efb..0000000000 --- a/crates/ragfs-cache-yuanrong-sys/build.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::env; -use std::path::PathBuf; - -fn main() { - println!("cargo:rerun-if-changed=native/yuanrong_bridge.cpp"); - println!("cargo:rerun-if-changed=native/yuanrong_bridge.h"); - println!("cargo:rerun-if-env-changed=YUANRONG_SDK_INCLUDE"); - println!("cargo:rerun-if-env-changed=YUANRONG_SDK_LIB_DIR"); - println!("cargo:rerun-if-env-changed=YUANRONG_SDK_LIB_NAME"); - - if env::var_os("CARGO_FEATURE_NATIVE").is_none() { - return; - } - - let include = required_path("YUANRONG_SDK_INCLUDE"); - let lib_dir = required_path("YUANRONG_SDK_LIB_DIR"); - let lib_name = env::var("YUANRONG_SDK_LIB_NAME").unwrap_or_else(|_| "datasystem".into()); - - cc::Build::new() - .cpp(true) - .std("c++17") - .warnings(true) - .include("native") - .include(include) - .file("native/yuanrong_bridge.cpp") - .compile("openviking_yuanrong_bridge"); - - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - println!("cargo:rustc-link-lib=dylib={lib_name}"); -} - -fn required_path(name: &str) -> PathBuf { - let value = env::var_os(name).unwrap_or_else(|| { - panic!("{name} must point to the Yuanrong SDK directory when feature `native` is enabled") - }); - let path = PathBuf::from(value); - if !path.exists() { - panic!("{name} does not exist: {}", path.display()); - } - path -} diff --git a/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cpp b/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cpp deleted file mode 100644 index 7e69dcd178..0000000000 --- a/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cpp +++ /dev/null @@ -1,530 +0,0 @@ -#include "yuanrong_bridge.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "datasystem/datasystem.h" - -using datasystem::ConnectOptions; -using datasystem::DsClient; -using datasystem::MSetParam; -using datasystem::Optional; -using datasystem::ReadOnlyBuffer; -using datasystem::Status; -using datasystem::StatusCode; -using datasystem::StringView; - -struct YrClientHandle { - std::shared_ptr client; - // Serialize all SDK calls made through this native client handle. A single - // native YuanrongProvider owns one handle today, so sdk_concurrency > 1 in - // Rust does not create true backend concurrency for that provider. - // - // If the Rust layer creates multiple native clients to provide concurrent - // channels, those channels must not be treated as a global ordering - // guarantee for concurrent conflicting writes. - std::mutex call_mutex; - std::atomic shutdown { false }; -}; - -namespace { -thread_local std::string last_error; - -void set_error(const Status &status) -{ - last_error = status.ToString(); -} - -void set_error(const char *message) -{ - last_error = message; -} - -int map_status(const Status &status) -{ - if (status.IsOk()) { - last_error.clear(); - return YR_OK; - } - set_error(status); - switch (status.GetCode()) { - case StatusCode::K_NOT_FOUND: - case StatusCode::K_NOT_FOUND_IN_L2CACHE: - return YR_NOT_FOUND; - case StatusCode::K_INVALID: - case StatusCode::K_OUT_OF_RANGE: - case StatusCode::K_FILE_NAME_TOO_LONG: - return YR_INVALID_ARGUMENT; - case StatusCode::K_MASTER_TIMEOUT: - case StatusCode::K_RPC_DEADLINE_EXCEEDED: - case StatusCode::K_FUTURE_TIMEOUT: - return YR_TIMEOUT; - case StatusCode::K_NOT_READY: - case StatusCode::K_SHUTTING_DOWN: - case StatusCode::K_WORKER_ABNORMAL: - case StatusCode::K_CLIENT_WORKER_DISCONNECT: - case StatusCode::K_RPC_UNAVAILABLE: - case StatusCode::K_URMA_ERROR: - case StatusCode::K_RDMA_ERROR: - return YR_UNAVAILABLE; - default: - return YR_INTERNAL; - } -} - -bool valid_client(YrClientHandle *client) -{ - if (client == nullptr || client->client == nullptr) { - set_error("Yuanrong client handle is null"); - return false; - } - if (client->shutdown.load()) { - set_error("Yuanrong client is shut down"); - return false; - } - return true; -} - -bool valid_bytes(const uint8_t *data, size_t len, const char *name) -{ - if (data == nullptr || len == 0) { - last_error = std::string(name) + " must not be empty"; - return false; - } - return true; -} - -std::string to_string(const uint8_t *data, size_t len) -{ - return std::string(reinterpret_cast(data), len); -} - -int copy_value(const void *source, size_t size, uint8_t **out) -{ - auto *copy = static_cast(std::malloc(size)); - if (copy == nullptr) { - set_error("failed to allocate Yuanrong result buffer"); - return YR_INTERNAL; - } - std::memcpy(copy, source, size); - *out = copy; - return YR_OK; -} - -template -int protect(Function &&function) noexcept -{ - try { - return function(); - } catch (const std::exception &error) { - last_error = std::string("Yuanrong bridge exception: ") + error.what(); - return YR_INTERNAL; - } catch (...) { - set_error("Yuanrong bridge caught an unknown exception"); - return YR_INTERNAL; - } -} - -template -void protect_void(Function &&function) noexcept -{ - try { - function(); - } catch (const std::exception &error) { - last_error = std::string("Yuanrong bridge exception: ") + error.what(); - } catch (...) { - set_error("Yuanrong bridge caught an unknown exception"); - } -} -} // namespace - -int client_create_impl(const char *host, uint16_t port, int32_t connect_timeout_ms, - int32_t request_timeout_ms, YrClientHandle **out) -{ - if (host == nullptr || host[0] == '\0' || port == 0 || connect_timeout_ms <= 0 - || request_timeout_ms <= 0 || out == nullptr) { - set_error("invalid Yuanrong connection options"); - return YR_INVALID_ARGUMENT; - } - *out = nullptr; - ConnectOptions options; - options.host = host; - options.port = port; - options.connectTimeoutMs = connect_timeout_ms; - options.requestTimeoutMs = request_timeout_ms; - auto handle = std::make_unique(); - handle->client = std::make_shared(options); - int code = map_status(handle->client->Init()); - if (code != YR_OK) { - return code; - } - *out = handle.release(); - return YR_OK; -} - -int client_health_check_impl(YrClientHandle *client) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - std::lock_guard guard(client->call_mutex); - return map_status(client->client->KV()->HealthCheck()); -} - -int client_get_impl(YrClientHandle *client, const uint8_t *key, size_t key_len, - uint8_t **data, size_t *size) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (!valid_bytes(key, key_len, "key") || data == nullptr || size == nullptr) { - return YR_INVALID_ARGUMENT; - } - *data = nullptr; - *size = 0; - std::lock_guard guard(client->call_mutex); - std::string value; - int code = map_status(client->client->KV()->Get(to_string(key, key_len), value)); - if (code != YR_OK) { - return code; - } - if (value.empty()) { - set_error("Yuanrong returned an empty cache value"); - return YR_INTERNAL; - } - code = copy_value(value.data(), value.size(), data); - if (code == YR_OK) { - *size = value.size(); - } - return code; -} - -int client_set_impl(YrClientHandle *client, const uint8_t *key, size_t key_len, - const uint8_t *data, size_t size) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (!valid_bytes(key, key_len, "key") || !valid_bytes(data, size, "value")) { - return YR_INVALID_ARGUMENT; - } - std::lock_guard guard(client->call_mutex); - StringView value(reinterpret_cast(data), size); - return map_status(client->client->KV()->Set(to_string(key, key_len), value)); -} - -int client_delete_impl(YrClientHandle *client, const uint8_t *key, size_t key_len) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (!valid_bytes(key, key_len, "key")) { - return YR_INVALID_ARGUMENT; - } - std::lock_guard guard(client->call_mutex); - int code = map_status(client->client->KV()->Del(to_string(key, key_len))); - return code == YR_NOT_FOUND ? YR_OK : code; -} - -int client_exists_impl(YrClientHandle *client, const uint8_t *key, size_t key_len, - uint8_t *exists) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (!valid_bytes(key, key_len, "key") || exists == nullptr) { - return YR_INVALID_ARGUMENT; - } - std::lock_guard guard(client->call_mutex); - std::vector results; - int code = map_status(client->client->KV()->Exist({ to_string(key, key_len) }, results)); - if (code == YR_NOT_FOUND) { - *exists = 0; - return YR_OK; - } - if (code != YR_OK) { - return code; - } - if (results.size() != 1) { - set_error("Yuanrong Exist returned an unexpected result count"); - return YR_INTERNAL; - } - *exists = results[0] ? 1 : 0; - return YR_OK; -} - -int client_mget_impl(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, size_t count, YrBuffer **values) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (keys == nullptr || key_lens == nullptr || count == 0 || values == nullptr) { - set_error("invalid Yuanrong mget arguments"); - return YR_INVALID_ARGUMENT; - } - *values = nullptr; - std::vector native_keys; - native_keys.reserve(count); - for (size_t i = 0; i < count; ++i) { - if (!valid_bytes(keys[i], key_lens[i], "key")) { - return YR_INVALID_ARGUMENT; - } - native_keys.push_back(to_string(keys[i], key_lens[i])); - } - - std::lock_guard guard(client->call_mutex); - std::vector> buffers; - int code = map_status(client->client->KV()->Get(native_keys, buffers)); - if (code == YR_NOT_FOUND) { - buffers.resize(count); - code = YR_OK; - } - if (code != YR_OK) { - return code; - } - if (buffers.size() != count) { - set_error("Yuanrong MGet returned an unexpected result count"); - return YR_INTERNAL; - } - - auto *results = static_cast(std::calloc(count, sizeof(YrBuffer))); - if (results == nullptr) { - set_error("failed to allocate Yuanrong mget result array"); - return YR_INTERNAL; - } - for (size_t i = 0; i < count; ++i) { - if (!buffers[i]) { - continue; - } - Status latch = buffers[i]->RLatch(); - if (latch.IsError()) { - set_error(latch); - yr_buffers_free(results, count); - return map_status(latch); - } - const auto size = static_cast(buffers[i]->GetSize()); - code = size == 0 ? YR_INTERNAL - : copy_value(buffers[i]->ImmutableData(), size, &results[i].data); - Status unlatch = buffers[i]->UnRLatch(); - if (code != YR_OK) { - yr_buffers_free(results, count); - return code; - } - if (unlatch.IsError()) { - set_error(unlatch); - yr_buffers_free(results, count); - return map_status(unlatch); - } - results[i].len = size; - results[i].found = 1; - } - *values = results; - last_error.clear(); - return YR_OK; -} - -int client_mset_impl(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, const uint8_t *const *values, - const size_t *value_lens, size_t count) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (keys == nullptr || key_lens == nullptr || values == nullptr || value_lens == nullptr - || count == 0) { - set_error("invalid Yuanrong mset arguments"); - return YR_INVALID_ARGUMENT; - } - std::vector native_keys; - std::vector native_values; - native_keys.reserve(count); - native_values.reserve(count); - for (size_t i = 0; i < count; ++i) { - if (!valid_bytes(keys[i], key_lens[i], "key") - || !valid_bytes(values[i], value_lens[i], "value")) { - return YR_INVALID_ARGUMENT; - } - native_keys.push_back(to_string(keys[i], key_lens[i])); - native_values.emplace_back(reinterpret_cast(values[i]), value_lens[i]); - } - std::lock_guard guard(client->call_mutex); - std::vector failed_keys; - int code = map_status(client->client->KV()->MSet(native_keys, native_values, failed_keys, - MSetParam {})); - if (code != YR_OK) { - return code; - } - for (const auto &failed_key : failed_keys) { - auto position = std::find(native_keys.begin(), native_keys.end(), failed_key); - if (position == native_keys.end()) { - set_error("Yuanrong MSet returned an unknown failed key"); - return YR_INTERNAL; - } - const auto index = static_cast(std::distance(native_keys.begin(), position)); - code = map_status(client->client->KV()->Set(failed_key, native_values[index])); - if (code != YR_OK) { - return code; - } - } - return YR_OK; -} - -int client_mdelete_impl(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, size_t count) -{ - if (!valid_client(client)) { - return YR_UNAVAILABLE; - } - if (keys == nullptr || key_lens == nullptr || count == 0) { - set_error("invalid Yuanrong mdelete arguments"); - return YR_INVALID_ARGUMENT; - } - std::vector native_keys; - native_keys.reserve(count); - for (size_t i = 0; i < count; ++i) { - if (!valid_bytes(keys[i], key_lens[i], "key")) { - return YR_INVALID_ARGUMENT; - } - native_keys.push_back(to_string(keys[i], key_lens[i])); - } - std::lock_guard guard(client->call_mutex); - std::vector failed_keys; - int code = map_status(client->client->KV()->Del(native_keys, failed_keys)); - if (code == YR_NOT_FOUND) { - return YR_OK; - } - if (code != YR_OK) { - return code; - } - for (const auto &failed_key : failed_keys) { - code = map_status(client->client->KV()->Del(failed_key)); - if (code != YR_OK && code != YR_NOT_FOUND) { - return code; - } - } - return YR_OK; -} - -int client_shutdown_impl(YrClientHandle *client) -{ - if (client == nullptr || client->client == nullptr) { - set_error("Yuanrong client handle is null"); - return YR_INVALID_ARGUMENT; - } - std::lock_guard guard(client->call_mutex); - if (client->shutdown.load()) { - return YR_OK; - } - int code = map_status(client->client->ShutDown()); - if (code == YR_OK) { - client->shutdown.store(true); - } - return code; -} - -void client_destroy_impl(YrClientHandle *client) -{ - if (client == nullptr) { - return; - } - if (!client->shutdown.load() && client->client != nullptr) { - std::lock_guard guard(client->call_mutex); - (void)client->client->ShutDown(); - client->shutdown.store(true); - } - delete client; -} - -extern "C" int yr_client_create(const char *host, uint16_t port, int32_t connect_timeout_ms, - int32_t request_timeout_ms, YrClientHandle **out) -{ - return protect( - [&] { return client_create_impl(host, port, connect_timeout_ms, request_timeout_ms, out); }); -} - -extern "C" int yr_client_health_check(YrClientHandle *client) -{ - return protect([&] { return client_health_check_impl(client); }); -} - -extern "C" int yr_client_get(YrClientHandle *client, const uint8_t *key, size_t key_len, - uint8_t **data, size_t *size) -{ - return protect([&] { return client_get_impl(client, key, key_len, data, size); }); -} - -extern "C" int yr_client_set(YrClientHandle *client, const uint8_t *key, size_t key_len, - const uint8_t *data, size_t size) -{ - return protect([&] { return client_set_impl(client, key, key_len, data, size); }); -} - -extern "C" int yr_client_delete(YrClientHandle *client, const uint8_t *key, size_t key_len) -{ - return protect([&] { return client_delete_impl(client, key, key_len); }); -} - -extern "C" int yr_client_exists(YrClientHandle *client, const uint8_t *key, size_t key_len, - uint8_t *exists) -{ - return protect([&] { return client_exists_impl(client, key, key_len, exists); }); -} - -extern "C" int yr_client_mget(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, size_t count, YrBuffer **values) -{ - return protect([&] { return client_mget_impl(client, keys, key_lens, count, values); }); -} - -extern "C" int yr_client_mset(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, const uint8_t *const *values, - const size_t *value_lens, size_t count) -{ - return protect( - [&] { return client_mset_impl(client, keys, key_lens, values, value_lens, count); }); -} - -extern "C" int yr_client_mdelete(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, size_t count) -{ - return protect([&] { return client_mdelete_impl(client, keys, key_lens, count); }); -} - -extern "C" int yr_client_shutdown(YrClientHandle *client) -{ - return protect([&] { return client_shutdown_impl(client); }); -} - -extern "C" void yr_client_destroy(YrClientHandle *client) -{ - protect_void([&] { client_destroy_impl(client); }); -} - -extern "C" void yr_buffer_free(void *data) -{ - std::free(data); -} - -extern "C" void yr_buffers_free(YrBuffer *values, size_t count) -{ - if (values == nullptr) { - return; - } - for (size_t i = 0; i < count; ++i) { - std::free(values[i].data); - } - std::free(values); -} - -extern "C" const char *yr_last_error(YrClientHandle *) -{ - return last_error.c_str(); -} diff --git a/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.h b/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.h deleted file mode 100644 index 68632f008a..0000000000 --- a/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef OPENVIKING_YUANRONG_BRIDGE_H -#define OPENVIKING_YUANRONG_BRIDGE_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct YrClientHandle YrClientHandle; - -typedef struct YrBuffer { - uint8_t *data; - size_t len; - uint8_t found; -} YrBuffer; - -enum YrStatus { - YR_OK = 0, - YR_NOT_FOUND = 1, - YR_INVALID_ARGUMENT = 2, - YR_UNAVAILABLE = 3, - YR_TIMEOUT = 4, - YR_INTERNAL = 5, -}; - -int yr_client_create(const char *host, uint16_t port, int32_t connect_timeout_ms, - int32_t request_timeout_ms, YrClientHandle **out); -int yr_client_health_check(YrClientHandle *client); -int yr_client_get(YrClientHandle *client, const uint8_t *key, size_t key_len, - uint8_t **data, size_t *size); -int yr_client_set(YrClientHandle *client, const uint8_t *key, size_t key_len, - const uint8_t *data, size_t size); -int yr_client_delete(YrClientHandle *client, const uint8_t *key, size_t key_len); -int yr_client_exists(YrClientHandle *client, const uint8_t *key, size_t key_len, - uint8_t *exists); -int yr_client_mget(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, size_t count, YrBuffer **values); -int yr_client_mset(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, const uint8_t *const *values, - const size_t *value_lens, size_t count); -int yr_client_mdelete(YrClientHandle *client, const uint8_t *const *keys, - const size_t *key_lens, size_t count); -int yr_client_shutdown(YrClientHandle *client); -void yr_client_destroy(YrClientHandle *client); -void yr_buffer_free(void *data); -void yr_buffers_free(YrBuffer *values, size_t count); -const char *yr_last_error(YrClientHandle *client); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/crates/ragfs-cache-yuanrong-sys/src/lib.rs b/crates/ragfs-cache-yuanrong-sys/src/lib.rs deleted file mode 100644 index 5f9aca8a20..0000000000 --- a/crates/ragfs-cache-yuanrong-sys/src/lib.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! Unsafe bindings for OpenViking's stable Yuanrong C ABI bridge. - -#[cfg(feature = "native")] -use std::ffi::{c_char, c_void}; -use std::ffi::{c_int, c_uchar}; - -/// Operation completed successfully. -pub const YR_OK: c_int = 0; -/// Requested key does not exist. -pub const YR_NOT_FOUND: c_int = 1; -/// Yuanrong rejected an argument. -pub const YR_INVALID_ARGUMENT: c_int = 2; -/// Worker or client connection is unavailable. -pub const YR_UNAVAILABLE: c_int = 3; -/// Yuanrong operation timed out. -pub const YR_TIMEOUT: c_int = 4; -/// Yuanrong returned another internal error. -pub const YR_INTERNAL: c_int = 5; - -/// Opaque Yuanrong client owned by the C++ bridge. -#[repr(C)] -pub struct YrClientHandle { - _private: [u8; 0], -} - -/// Buffer allocated by the C++ bridge. -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct YrBuffer { - /// Buffer address, or null when the corresponding key is missing. - pub data: *mut c_uchar, - /// Buffer length. - pub len: usize, - /// One when the key exists, zero when it is missing. - pub found: c_uchar, -} - -#[cfg(feature = "native")] -unsafe extern "C" { - /// Create and initialize a Yuanrong `DsClient`. - pub fn yr_client_create( - host: *const c_char, - port: u16, - connect_timeout_ms: i32, - request_timeout_ms: i32, - out: *mut *mut YrClientHandle, - ) -> c_int; - /// Check the local worker connection. - pub fn yr_client_health_check(client: *mut YrClientHandle) -> c_int; - /// Read one complete KV value. - pub fn yr_client_get( - client: *mut YrClientHandle, - key: *const c_uchar, - key_len: usize, - data: *mut *mut c_uchar, - size: *mut usize, - ) -> c_int; - /// Store one complete KV value. - pub fn yr_client_set( - client: *mut YrClientHandle, - key: *const c_uchar, - key_len: usize, - data: *const c_uchar, - size: usize, - ) -> c_int; - /// Delete one KV value. - pub fn yr_client_delete( - client: *mut YrClientHandle, - key: *const c_uchar, - key_len: usize, - ) -> c_int; - /// Check whether one KV value exists. - pub fn yr_client_exists( - client: *mut YrClientHandle, - key: *const c_uchar, - key_len: usize, - exists: *mut c_uchar, - ) -> c_int; - /// Read multiple values, preserving key order. - pub fn yr_client_mget( - client: *mut YrClientHandle, - keys: *const *const c_uchar, - key_lens: *const usize, - count: usize, - values: *mut *mut YrBuffer, - ) -> c_int; - /// Store multiple values. - pub fn yr_client_mset( - client: *mut YrClientHandle, - keys: *const *const c_uchar, - key_lens: *const usize, - values: *const *const c_uchar, - value_lens: *const usize, - count: usize, - ) -> c_int; - /// Delete multiple values. - pub fn yr_client_mdelete( - client: *mut YrClientHandle, - keys: *const *const c_uchar, - key_lens: *const usize, - count: usize, - ) -> c_int; - /// Shut down the SDK client. - pub fn yr_client_shutdown(client: *mut YrClientHandle) -> c_int; - /// Destroy the bridge handle. - pub fn yr_client_destroy(client: *mut YrClientHandle); - /// Free a single value returned by `yr_client_get`. - pub fn yr_buffer_free(data: *mut c_void); - /// Free values returned by `yr_client_mget`. - pub fn yr_buffers_free(values: *mut YrBuffer, count: usize); - /// Return diagnostics for the last bridge call on the current thread. - pub fn yr_last_error(client: *mut YrClientHandle) -> *const c_char; -} diff --git a/crates/ragfs-cache-yuanrong/Cargo.toml b/crates/ragfs-cache-yuanrong/Cargo.toml deleted file mode 100644 index ca45f88d9e..0000000000 --- a/crates/ragfs-cache-yuanrong/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "ragfs-cache-yuanrong" -version = "0.1.0" -edition = "2021" -rust-version = "1.91.1" -license = "Apache-2.0" -description = "Yuanrong DataSystem cache provider adapter for RAGFS" - -[dependencies] -async-trait = "0.1" -bytes = "1.5" -ragfs = { path = "../ragfs", features = ["cache"] } -ragfs-cache-yuanrong-sys = { path = "../ragfs-cache-yuanrong-sys", optional = true } -sha2 = { version = "0.10", optional = true } -thiserror = "1.0" -tokio = { version = "1.38", features = ["rt-multi-thread", "sync", "time", "macros"] } - -[features] -default = [] -yuanrong-native = [ - "dep:ragfs-cache-yuanrong-sys", - "dep:sha2", - "ragfs-cache-yuanrong-sys/native", -] diff --git a/crates/ragfs-cache-yuanrong/src/client.rs b/crates/ragfs-cache-yuanrong/src/client.rs deleted file mode 100644 index e4a6aac828..0000000000 --- a/crates/ragfs-cache-yuanrong/src/client.rs +++ /dev/null @@ -1,164 +0,0 @@ -use crate::{YuanrongKvStore, YuanrongStoreError}; -use ragfs::cache::{CacheError, CacheResult}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{RwLock, Semaphore}; - -pub(crate) struct YuanrongClient { - store: RwLock>>, - concurrency: Arc, - concurrency_limit: u32, - timeout: Duration, - closed: AtomicBool, -} - -impl YuanrongClient { - pub(crate) fn new( - store: Arc, - concurrency_limit: usize, - timeout: Duration, - ) -> Self { - Self { - store: RwLock::new(Some(store)), - concurrency: Arc::new(Semaphore::new(concurrency_limit)), - concurrency_limit: concurrency_limit as u32, - timeout, - closed: AtomicBool::new(false), - } - } - - async fn execute(&self, operation: &'static str, call: F) -> CacheResult - where - T: Send + 'static, - F: FnOnce(Arc) -> Result + Send + 'static, - { - if self.closed.load(Ordering::Acquire) { - return Err(CacheError::Unavailable( - "Yuanrong provider is closed".into(), - )); - } - - let work = async { - let permit = Arc::clone(&self.concurrency) - .acquire_owned() - .await - .map_err(|_| CacheError::Unavailable("Yuanrong client is closing".into()))?; - if self.closed.load(Ordering::Acquire) { - return Err(CacheError::Unavailable( - "Yuanrong provider is closed".into(), - )); - } - let store = self.store.read().await.clone().ok_or_else(|| { - CacheError::Unavailable("Yuanrong KV client has been released".into()) - })?; - tokio::task::spawn_blocking(move || { - let _permit = permit; - call(store) - }) - .await - .map_err(|error| { - CacheError::Internal(format!( - "Yuanrong {operation} blocking task failed: {error}" - )) - })? - .map_err(map_store_error) - }; - - tokio::time::timeout(self.timeout, work) - .await - .map_err(|_| { - CacheError::Timeout(format!( - "Yuanrong {operation} exceeded {} ms", - self.timeout.as_millis() - )) - })? - } - - pub(crate) async fn health_check(&self) -> CacheResult<()> { - self.execute("health_check", |store| store.health_check()) - .await - } - - pub(crate) async fn get(&self, key: &str) -> CacheResult>> { - let key = key.to_owned(); - self.execute("get", move |store| store.get(&key)).await - } - - pub(crate) async fn set(&self, key: &str, value: &[u8]) -> CacheResult<()> { - let key = key.to_owned(); - let value = value.to_vec(); - self.execute("set", move |store| store.set(&key, &value)) - .await - } - - pub(crate) async fn delete(&self, key: &str) -> CacheResult<()> { - let key = key.to_owned(); - self.execute("delete", move |store| store.delete(&key)) - .await - } - - pub(crate) async fn exists(&self, key: &str) -> CacheResult { - let key = key.to_owned(); - self.execute("exists", move |store| store.exists(&key)) - .await - } - - pub(crate) async fn batch_get(&self, keys: &[String]) -> CacheResult>>> { - let keys = keys.to_vec(); - self.execute("batch_get", move |store| store.batch_get(&keys)) - .await - } - - pub(crate) async fn batch_set(&self, entries: Vec<(String, Vec)>) -> CacheResult<()> { - self.execute("batch_set", move |store| store.batch_set(&entries)) - .await - } - - pub(crate) async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { - let keys = keys.to_vec(); - self.execute("batch_delete", move |store| store.batch_delete(&keys)) - .await - } - - pub(crate) async fn close(&self) -> CacheResult<()> { - if self.closed.swap(true, Ordering::AcqRel) { - return Ok(()); - } - let permits = Arc::clone(&self.concurrency) - .acquire_many_owned(self.concurrency_limit) - .await - .map_err(|_| CacheError::Unavailable("Yuanrong client is closing".into()))?; - let store = self.store.write().await.take(); - let result = if let Some(store) = store { - tokio::time::timeout( - self.timeout, - tokio::task::spawn_blocking(move || store.shutdown()), - ) - .await - .map_err(|_| { - CacheError::Timeout(format!( - "Yuanrong shutdown exceeded {} ms", - self.timeout.as_millis() - )) - })? - .map_err(|error| { - CacheError::Internal(format!("Yuanrong shutdown task failed: {error}")) - })? - .map_err(map_store_error) - } else { - Ok(()) - }; - drop(permits); - result - } -} - -fn map_store_error(error: YuanrongStoreError) -> CacheError { - match error { - YuanrongStoreError::Unavailable(message) => CacheError::Unavailable(message), - YuanrongStoreError::Timeout(message) => CacheError::Timeout(message), - YuanrongStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message), - YuanrongStoreError::Internal(message) => CacheError::Internal(message), - } -} diff --git a/crates/ragfs-cache-yuanrong/src/config.rs b/crates/ragfs-cache-yuanrong/src/config.rs deleted file mode 100644 index 0acbf46c95..0000000000 --- a/crates/ragfs-cache-yuanrong/src/config.rs +++ /dev/null @@ -1,52 +0,0 @@ -use ragfs::cache::{CacheError, CacheResult}; - -/// Connection and execution settings for a Yuanrong provider. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct YuanrongConfig { - /// Yuanrong worker host. - pub host: String, - /// Yuanrong worker port. - pub port: u16, - /// Timeout used while connecting to the worker. - pub connect_timeout_ms: u64, - /// Timeout applied to each KV operation. - pub request_timeout_ms: u64, - /// Maximum number of concurrent blocking tasks allowed by the Rust client. - /// - /// The current native bridge creates one native client handle per - /// `YuanrongProvider`; calls through that handle are serialized in C++. - /// Therefore `sdk_concurrency > 1` does not provide true Yuanrong backend - /// concurrency for a single native provider instance. - pub sdk_concurrency: usize, -} - -impl YuanrongConfig { - pub(crate) fn validate(&self) -> CacheResult<()> { - if self.host.trim().is_empty() { - return Err(CacheError::InvalidArgument( - "Yuanrong host must not be empty".into(), - )); - } - if self.port == 0 { - return Err(CacheError::InvalidArgument( - "Yuanrong port must be greater than zero".into(), - )); - } - if self.connect_timeout_ms == 0 || self.connect_timeout_ms > i32::MAX as u64 { - return Err(CacheError::InvalidArgument( - "Yuanrong connect_timeout_ms must be between 1 and i32::MAX".into(), - )); - } - if self.request_timeout_ms == 0 || self.request_timeout_ms > i32::MAX as u64 { - return Err(CacheError::InvalidArgument( - "Yuanrong request_timeout_ms must be between 1 and i32::MAX".into(), - )); - } - if self.sdk_concurrency == 0 || self.sdk_concurrency > u32::MAX as usize { - return Err(CacheError::InvalidArgument( - "Yuanrong sdk_concurrency must be between 1 and u32::MAX".into(), - )); - } - Ok(()) - } -} diff --git a/crates/ragfs-cache-yuanrong/src/error.rs b/crates/ragfs-cache-yuanrong/src/error.rs deleted file mode 100644 index 07d0e62f8a..0000000000 --- a/crates/ragfs-cache-yuanrong/src/error.rs +++ /dev/null @@ -1,16 +0,0 @@ -/// Errors returned by the synchronous Yuanrong KV boundary. -#[derive(Debug, thiserror::Error)] -pub enum YuanrongStoreError { - /// The worker or client connection is unavailable. - #[error("Yuanrong unavailable: {0}")] - Unavailable(String), - /// A Yuanrong SDK operation timed out. - #[error("Yuanrong operation timed out: {0}")] - Timeout(String), - /// Yuanrong rejected an argument. - #[error("invalid Yuanrong argument: {0}")] - InvalidArgument(String), - /// Yuanrong returned an unspecified operation failure. - #[error("Yuanrong operation failed: {0}")] - Internal(String), -} diff --git a/crates/ragfs-cache-yuanrong/src/lib.rs b/crates/ragfs-cache-yuanrong/src/lib.rs deleted file mode 100644 index bf604e1b3f..0000000000 --- a/crates/ragfs-cache-yuanrong/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Yuanrong DataSystem cache provider adapter for RAGFS. - -mod client; -mod config; -mod error; -#[cfg(feature = "yuanrong-native")] -mod native; -mod provider; -mod store; - -pub use config::YuanrongConfig; -pub use error::YuanrongStoreError; -pub use provider::YuanrongProvider; -pub use store::YuanrongKvStore; diff --git a/crates/ragfs-cache-yuanrong/src/native.rs b/crates/ragfs-cache-yuanrong/src/native.rs deleted file mode 100644 index d55a132da4..0000000000 --- a/crates/ragfs-cache-yuanrong/src/native.rs +++ /dev/null @@ -1,344 +0,0 @@ -use crate::{YuanrongConfig, YuanrongKvStore, YuanrongProvider, YuanrongStoreError}; -use ragfs::cache::{CacheError, CacheResult}; -use ragfs_cache_yuanrong_sys as sys; -use sha2::{Digest, Sha256}; -use std::ffi::{CStr, CString}; -use std::ptr::{self, NonNull}; -use std::slice; -use std::sync::Arc; -use std::time::Duration; - -struct NativeYuanrongStore { - handle: NonNull, -} - -unsafe impl Send for NativeYuanrongStore {} -unsafe impl Sync for NativeYuanrongStore {} - -impl NativeYuanrongStore { - const VALUE_FORMAT_V1: u8 = 1; - - fn connect(config: &YuanrongConfig) -> Result { - let host = CString::new(config.host.as_str()).map_err(|_| { - YuanrongStoreError::InvalidArgument("host contains an embedded NUL byte".into()) - })?; - let mut handle = ptr::null_mut(); - let code = unsafe { - sys::yr_client_create( - host.as_ptr(), - config.port, - config.connect_timeout_ms as i32, - config.request_timeout_ms as i32, - &mut handle, - ) - }; - let handle = NonNull::new(handle); - if code != sys::YR_OK { - return Err(map_native_error(code, handle)); - } - Ok(Self { - handle: handle.ok_or_else(|| { - YuanrongStoreError::Internal( - "Yuanrong create succeeded without returning a handle".into(), - ) - })?, - }) - } - - fn call(&self, code: i32) -> Result<(), YuanrongStoreError> { - if code == sys::YR_OK { - Ok(()) - } else { - Err(map_native_error(code, Some(self.handle))) - } - } - - fn native_key(key: &str) -> Result { - if key.is_empty() { - return Err(YuanrongStoreError::InvalidArgument( - "Yuanrong cache key must not be empty".into(), - )); - } - let digest = Sha256::digest(key.as_bytes()); - let mut encoded = String::with_capacity(67); - encoded.push_str("ov_"); - for byte in digest { - use std::fmt::Write; - write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail"); - } - Ok(encoded) - } - - fn native_keys(keys: &[String]) -> Result, YuanrongStoreError> { - keys.iter().map(|key| Self::native_key(key)).collect() - } - - fn key_parts(keys: &[String]) -> (Vec<*const u8>, Vec) { - ( - keys.iter().map(|key| key.as_ptr()).collect(), - keys.iter().map(String::len).collect(), - ) - } - - fn encode_value(value: &[u8]) -> Vec { - let mut encoded = Vec::with_capacity(value.len() + 1); - encoded.push(Self::VALUE_FORMAT_V1); - encoded.extend_from_slice(value); - encoded - } - - fn decode_value(value: &[u8]) -> Result, YuanrongStoreError> { - match value.split_first() { - Some((&Self::VALUE_FORMAT_V1, data)) => Ok(data.to_vec()), - _ => Err(YuanrongStoreError::Internal( - "Yuanrong cache value has an unsupported format".into(), - )), - } - } -} - -impl Drop for NativeYuanrongStore { - fn drop(&mut self) { - unsafe { - sys::yr_client_destroy(self.handle.as_ptr()); - } - } -} - -impl YuanrongKvStore for NativeYuanrongStore { - fn health_check(&self) -> Result<(), YuanrongStoreError> { - self.call(unsafe { sys::yr_client_health_check(self.handle.as_ptr()) }) - } - - fn get(&self, key: &str) -> Result>, YuanrongStoreError> { - let key = Self::native_key(key)?; - let mut data = ptr::null_mut(); - let mut size = 0; - let code = unsafe { - sys::yr_client_get( - self.handle.as_ptr(), - key.as_ptr(), - key.len(), - &mut data, - &mut size, - ) - }; - if code == sys::YR_NOT_FOUND { - return Ok(None); - } - self.call(code)?; - let data = NonNull::new(data).ok_or_else(|| { - YuanrongStoreError::Internal("Yuanrong get succeeded without returning a buffer".into()) - })?; - let value = Self::decode_value(unsafe { slice::from_raw_parts(data.as_ptr(), size) }); - unsafe { - sys::yr_buffer_free(data.as_ptr().cast()); - } - Ok(Some(value?)) - } - - fn set(&self, key: &str, value: &[u8]) -> Result<(), YuanrongStoreError> { - let key = Self::native_key(key)?; - let value = Self::encode_value(value); - self.call(unsafe { - sys::yr_client_set( - self.handle.as_ptr(), - key.as_ptr(), - key.len(), - value.as_ptr(), - value.len(), - ) - }) - } - - fn delete(&self, key: &str) -> Result<(), YuanrongStoreError> { - let key = Self::native_key(key)?; - self.call(unsafe { sys::yr_client_delete(self.handle.as_ptr(), key.as_ptr(), key.len()) }) - } - - fn exists(&self, key: &str) -> Result { - let key = Self::native_key(key)?; - let mut exists = 0; - self.call(unsafe { - sys::yr_client_exists(self.handle.as_ptr(), key.as_ptr(), key.len(), &mut exists) - })?; - Ok(exists != 0) - } - - fn batch_get(&self, keys: &[String]) -> Result>>, YuanrongStoreError> { - if keys.is_empty() { - return Ok(Vec::new()); - } - let native_keys = Self::native_keys(keys)?; - let (key_ptrs, key_lens) = Self::key_parts(&native_keys); - let mut values = ptr::null_mut(); - self.call(unsafe { - sys::yr_client_mget( - self.handle.as_ptr(), - key_ptrs.as_ptr(), - key_lens.as_ptr(), - keys.len(), - &mut values, - ) - })?; - let values = NonNull::new(values).ok_or_else(|| { - YuanrongStoreError::Internal("Yuanrong mget succeeded without returning results".into()) - })?; - let native = unsafe { slice::from_raw_parts(values.as_ptr(), keys.len()) }; - let result = native - .iter() - .map(|value| { - if value.found == 0 { - Ok(None) - } else { - let data = NonNull::new(value.data).ok_or_else(|| { - YuanrongStoreError::Internal( - "Yuanrong mget returned a null value buffer".into(), - ) - })?; - Self::decode_value(unsafe { slice::from_raw_parts(data.as_ptr(), value.len) }) - .map(Some) - } - }) - .collect(); - unsafe { - sys::yr_buffers_free(values.as_ptr(), keys.len()); - } - result - } - - fn batch_set(&self, entries: &[(String, Vec)]) -> Result<(), YuanrongStoreError> { - const MAX_MSET_KEYS: usize = 1_999; - const MAX_MSET_ENCODED_VALUE_SIZE: usize = 500 * 1024; - - for chunk in entries.chunks(MAX_MSET_KEYS) { - if chunk - .iter() - .any(|(_, value)| value.len() + 1 >= MAX_MSET_ENCODED_VALUE_SIZE) - { - for (key, value) in chunk { - self.set(key, value)?; - } - continue; - } - let logical_keys = chunk.iter().map(|(key, _)| key.clone()).collect::>(); - let keys = Self::native_keys(&logical_keys)?; - let (key_ptrs, key_lens) = Self::key_parts(&keys); - let values = chunk - .iter() - .map(|(_, value)| Self::encode_value(value)) - .collect::>(); - let value_ptrs = values.iter().map(Vec::as_ptr).collect::>(); - let value_lens = values.iter().map(Vec::len).collect::>(); - self.call(unsafe { - sys::yr_client_mset( - self.handle.as_ptr(), - key_ptrs.as_ptr(), - key_lens.as_ptr(), - value_ptrs.as_ptr(), - value_lens.as_ptr(), - chunk.len(), - ) - })?; - } - Ok(()) - } - - fn batch_delete(&self, keys: &[String]) -> Result<(), YuanrongStoreError> { - const MAX_BATCH_KEYS: usize = 10_000; - - for chunk in keys.chunks(MAX_BATCH_KEYS) { - if chunk.is_empty() { - continue; - } - let native_keys = Self::native_keys(chunk)?; - let (key_ptrs, key_lens) = Self::key_parts(&native_keys); - self.call(unsafe { - sys::yr_client_mdelete( - self.handle.as_ptr(), - key_ptrs.as_ptr(), - key_lens.as_ptr(), - chunk.len(), - ) - })?; - } - Ok(()) - } - - fn shutdown(&self) -> Result<(), YuanrongStoreError> { - self.call(unsafe { sys::yr_client_shutdown(self.handle.as_ptr()) }) - } -} - -impl YuanrongProvider { - /// Connect to Yuanrong through the native C ABI bridge. - pub async fn connect(config: YuanrongConfig) -> CacheResult { - config.validate()?; - let setup_config = config.clone(); - let setup_timeout = Duration::from_millis(config.connect_timeout_ms); - let store = tokio::time::timeout( - setup_timeout, - tokio::task::spawn_blocking(move || NativeYuanrongStore::connect(&setup_config)), - ) - .await - .map_err(|_| { - CacheError::Timeout(format!( - "Yuanrong connection exceeded {} ms", - setup_timeout.as_millis() - )) - })? - .map_err(|error| CacheError::Internal(format!("Yuanrong connection task failed: {error}")))? - .map_err(map_cache_error)?; - - // A native provider currently owns one native client handle. The C++ - // bridge serializes SDK calls on that handle, so sdk_concurrency only - // bounds Rust blocking tasks here; it is not backend concurrency. - Self::from_store(config, Arc::new(store)).await - } -} - -fn map_native_error(code: i32, handle: Option>) -> YuanrongStoreError { - let raw_handle = handle.map_or(ptr::null_mut(), NonNull::as_ptr); - let value = unsafe { sys::yr_last_error(raw_handle) }; - let message = (!value.is_null()) - .then(|| { - unsafe { CStr::from_ptr(value) } - .to_string_lossy() - .into_owned() - }) - .filter(|message| !message.is_empty()) - .unwrap_or_else(|| format!("Yuanrong bridge returned status {code}")); - - match code { - sys::YR_INVALID_ARGUMENT => YuanrongStoreError::InvalidArgument(message), - sys::YR_UNAVAILABLE => YuanrongStoreError::Unavailable(message), - sys::YR_TIMEOUT => YuanrongStoreError::Timeout(message), - sys::YR_NOT_FOUND => { - YuanrongStoreError::Internal(format!("unexpected not-found status: {message}")) - } - _ => YuanrongStoreError::Internal(message), - } -} - -fn map_cache_error(error: YuanrongStoreError) -> CacheError { - match error { - YuanrongStoreError::Unavailable(message) => CacheError::Unavailable(message), - YuanrongStoreError::Timeout(message) => CacheError::Timeout(message), - YuanrongStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message), - YuanrongStoreError::Internal(message) => CacheError::Internal(message), - } -} - -#[cfg(test)] -mod tests { - use super::NativeYuanrongStore; - - #[test] - fn native_value_codec_round_trips_empty_and_binary_values() { - for value in [Vec::new(), vec![0, 1, 2, 255]] { - let encoded = NativeYuanrongStore::encode_value(&value); - assert!(!encoded.is_empty()); - assert_eq!(NativeYuanrongStore::decode_value(&encoded).unwrap(), value); - } - } -} diff --git a/crates/ragfs-cache-yuanrong/src/provider.rs b/crates/ragfs-cache-yuanrong/src/provider.rs deleted file mode 100644 index 9b6e2a43fe..0000000000 --- a/crates/ragfs-cache-yuanrong/src/provider.rs +++ /dev/null @@ -1,167 +0,0 @@ -use crate::client::YuanrongClient; -use crate::{YuanrongConfig, YuanrongKvStore}; -use async_trait::async_trait; -use bytes::Bytes; -use ragfs::cache::{CacheError, CacheProvider, CacheResult, ProviderCapabilities}; -use std::collections::HashSet; -use std::fmt; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; - -/// Yuanrong implementation of the common RAGFS cache provider contract. -pub struct YuanrongProvider { - client: Arc, - known_keys: Mutex>, -} - -impl fmt::Debug for YuanrongProvider { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("YuanrongProvider") - .finish_non_exhaustive() - } -} - -impl YuanrongProvider { - /// Construct a provider over a Yuanrong-compatible synchronous KV store. - /// - /// Construction validates configuration and performs a health check. - pub async fn from_store( - config: YuanrongConfig, - store: Arc, - ) -> CacheResult { - config.validate()?; - let client = Arc::new(YuanrongClient::new( - store, - config.sdk_concurrency, - Duration::from_millis(config.request_timeout_ms), - )); - client.health_check().await?; - Ok(Self { - client, - known_keys: Mutex::new(HashSet::new()), - }) - } - - /// Return a startup error when the native Yuanrong bridge is not compiled. - #[cfg(not(feature = "yuanrong-native"))] - pub async fn connect(config: YuanrongConfig) -> CacheResult { - config.validate()?; - Err(CacheError::Unavailable( - "Yuanrong support requires the yuanrong-native feature".into(), - )) - } - - /// Check whether the connected Yuanrong worker is healthy. - pub async fn health_check(&self) -> CacheResult<()> { - self.client.health_check().await - } -} - -fn lock_known_keys( - known_keys: &Mutex>, -) -> CacheResult>> { - known_keys - .lock() - .map_err(|_| CacheError::Internal("Yuanrong key tracker is poisoned".into())) -} - -#[async_trait] -impl CacheProvider for YuanrongProvider { - fn name(&self) -> &'static str { - "yuanrong" - } - - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities { - batch_get: true, - batch_put: true, - native_ttl: false, - } - } - - async fn get(&self, key: &str) -> CacheResult> { - Ok(self.client.get(key).await?.map(Bytes::from)) - } - - async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> { - self.client.set(key, &value).await?; - lock_known_keys(&self.known_keys)?.insert(key.to_owned()); - Ok(()) - } - - async fn delete(&self, key: &str) -> CacheResult<()> { - self.client.delete(key).await?; - lock_known_keys(&self.known_keys)?.remove(key); - Ok(()) - } - - async fn exists(&self, key: &str) -> CacheResult { - self.client.exists(key).await - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - Ok(self - .client - .batch_get(keys) - .await? - .into_iter() - .map(|value| value.map(Bytes::from)) - .collect()) - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - let native_entries = entries - .iter() - .map(|(key, value)| (key.clone(), value.to_vec())) - .collect(); - self.client.batch_set(native_entries).await?; - lock_known_keys(&self.known_keys)?.extend(entries.into_iter().map(|(key, _)| key)); - Ok(()) - } - - async fn invalidate(&self, keys: &[String]) -> CacheResult<()> { - self.client.batch_delete(keys).await?; - let mut known_keys = lock_known_keys(&self.known_keys)?; - for key in keys { - known_keys.remove(key); - } - Ok(()) - } - - async fn flush(&self) -> CacheResult<()> { - let keys = lock_known_keys(&self.known_keys)? - .iter() - .cloned() - .collect::>(); - self.invalidate(&keys).await - } - - async fn close(&self) -> CacheResult<()> { - self.client.close().await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::panic::{self, AssertUnwindSafe}; - - #[test] - fn lock_known_keys_returns_internal_error_when_poisoned() { - let known_keys = Mutex::new(HashSet::new()); - - let _ = panic::catch_unwind(AssertUnwindSafe(|| { - let _guard = known_keys.lock().unwrap(); - panic!("poison known_keys"); - })); - - let error = lock_known_keys(&known_keys).unwrap_err(); - - assert!(matches!( - error, - CacheError::Internal(message) - if message == "Yuanrong key tracker is poisoned" - )); - } -} diff --git a/crates/ragfs-cache-yuanrong/src/store.rs b/crates/ragfs-cache-yuanrong/src/store.rs deleted file mode 100644 index 8d62b2ff72..0000000000 --- a/crates/ragfs-cache-yuanrong/src/store.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::YuanrongStoreError; - -/// Synchronous Yuanrong KV operations used by the async provider. -/// -/// The production implementation delegates to the Yuanrong C++ SDK. Tests use -/// this boundary to exercise the provider without requiring a worker process. -pub trait YuanrongKvStore: Send + Sync + 'static { - /// Check that the connected worker is healthy. - fn health_check(&self) -> Result<(), YuanrongStoreError>; - /// Read one complete value. A missing key returns `Ok(None)`. - fn get(&self, key: &str) -> Result>, YuanrongStoreError>; - /// Store one complete value. - fn set(&self, key: &str, value: &[u8]) -> Result<(), YuanrongStoreError>; - /// Delete one key. Missing keys are treated as success. - fn delete(&self, key: &str) -> Result<(), YuanrongStoreError>; - /// Check whether one key exists. - fn exists(&self, key: &str) -> Result; - /// Read multiple values while preserving input order. - fn batch_get(&self, keys: &[String]) -> Result>>, YuanrongStoreError>; - /// Store multiple values. - fn batch_set(&self, entries: &[(String, Vec)]) -> Result<(), YuanrongStoreError>; - /// Delete multiple keys. - fn batch_delete(&self, keys: &[String]) -> Result<(), YuanrongStoreError>; - /// Shut down the SDK client. - fn shutdown(&self) -> Result<(), YuanrongStoreError>; -} diff --git a/crates/ragfs-cache-yuanrong/tests/cached_filesystem.rs b/crates/ragfs-cache-yuanrong/tests/cached_filesystem.rs deleted file mode 100644 index c0a8539b75..0000000000 --- a/crates/ragfs-cache-yuanrong/tests/cached_filesystem.rs +++ /dev/null @@ -1,381 +0,0 @@ -use async_trait::async_trait; -use ragfs::cache::{CacheNamespace, CachePolicy, CacheProvider, CachedFileSystem}; -use ragfs::core::{GrepResult, TreeEntry}; -use ragfs::plugins::MemFileSystem; -use ragfs::{FileInfo, FileSystem, Result as FsResult, WriteFlag}; -use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongKvStore, YuanrongProvider, YuanrongStoreError}; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; - -#[derive(Default)] -struct SharedKvStore { - values: Mutex>>, - unavailable: AtomicBool, -} - -impl SharedKvStore { - fn check(&self) -> std::result::Result<(), YuanrongStoreError> { - if self.unavailable.load(Ordering::SeqCst) { - Err(YuanrongStoreError::Unavailable("worker unavailable".into())) - } else { - Ok(()) - } - } -} - -impl YuanrongKvStore for SharedKvStore { - fn health_check(&self) -> std::result::Result<(), YuanrongStoreError> { - self.check() - } - - fn get(&self, key: &str) -> std::result::Result>, YuanrongStoreError> { - self.check()?; - Ok(self.values.lock().unwrap().get(key).cloned()) - } - - fn set(&self, key: &str, value: &[u8]) -> std::result::Result<(), YuanrongStoreError> { - self.check()?; - self.values - .lock() - .unwrap() - .insert(key.to_owned(), value.to_vec()); - Ok(()) - } - - fn delete(&self, key: &str) -> std::result::Result<(), YuanrongStoreError> { - self.check()?; - self.values.lock().unwrap().remove(key); - Ok(()) - } - - fn exists(&self, key: &str) -> std::result::Result { - self.check()?; - Ok(self.values.lock().unwrap().contains_key(key)) - } - - fn batch_get( - &self, - keys: &[String], - ) -> std::result::Result>>, YuanrongStoreError> { - self.check()?; - let values = self.values.lock().unwrap(); - Ok(keys.iter().map(|key| values.get(key).cloned()).collect()) - } - - fn batch_set( - &self, - entries: &[(String, Vec)], - ) -> std::result::Result<(), YuanrongStoreError> { - self.check()?; - self.values.lock().unwrap().extend(entries.iter().cloned()); - Ok(()) - } - - fn batch_delete(&self, keys: &[String]) -> std::result::Result<(), YuanrongStoreError> { - self.check()?; - let mut values = self.values.lock().unwrap(); - for key in keys { - values.remove(key); - } - Ok(()) - } - - fn shutdown(&self) -> std::result::Result<(), YuanrongStoreError> { - Ok(()) - } -} - -#[derive(Clone)] -struct CountingFileSystem { - inner: Arc, - reads: Arc, - read_dirs: Arc, -} - -impl CountingFileSystem { - fn new() -> Self { - Self { - inner: Arc::new(MemFileSystem::new()), - reads: Arc::new(AtomicU64::new(0)), - read_dirs: Arc::new(AtomicU64::new(0)), - } - } -} - -#[async_trait] -impl FileSystem for CountingFileSystem { - async fn create(&self, path: &str) -> FsResult<()> { - self.inner.create(path).await - } - - async fn mkdir(&self, path: &str, mode: u32) -> FsResult<()> { - self.inner.mkdir(path, mode).await - } - - async fn remove(&self, path: &str) -> FsResult<()> { - self.inner.remove(path).await - } - - async fn remove_all(&self, path: &str) -> FsResult<()> { - self.inner.remove_all(path).await - } - - async fn read(&self, path: &str, offset: u64, size: u64) -> FsResult> { - self.reads.fetch_add(1, Ordering::SeqCst); - self.inner.read(path, offset, size).await - } - - async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> FsResult { - self.inner.write(path, data, offset, flags).await - } - - async fn read_dir(&self, path: &str) -> FsResult> { - self.read_dirs.fetch_add(1, Ordering::SeqCst); - self.inner.read_dir(path).await - } - - async fn stat(&self, path: &str) -> FsResult { - self.inner.stat(path).await - } - - async fn rename(&self, old_path: &str, new_path: &str) -> FsResult<()> { - self.inner.rename(old_path, new_path).await - } - - async fn chmod(&self, path: &str, mode: u32) -> FsResult<()> { - self.inner.chmod(path, mode).await - } - - async fn truncate(&self, path: &str, size: u64) -> FsResult<()> { - self.inner.truncate(path, size).await - } - - async fn grep( - &self, - path: &str, - pattern: &str, - recursive: bool, - case_insensitive: bool, - node_limit: Option, - exclude_path: Option<&str>, - level_limit: Option, - ) -> FsResult { - self.inner - .grep( - path, - pattern, - recursive, - case_insensitive, - node_limit, - exclude_path, - level_limit, - ) - .await - } - - async fn tree_directory( - &self, - path: &str, - show_hidden: bool, - node_limit: Option, - level_limit: Option, - ) -> FsResult> { - self.inner - .tree_directory(path, show_hidden, node_limit, level_limit) - .await - } -} - -fn config() -> YuanrongConfig { - YuanrongConfig { - host: "127.0.0.1".into(), - port: 9088, - connect_timeout_ms: 1_000, - request_timeout_ms: 100, - sdk_concurrency: 4, - } -} - -async fn cached_fs( - backend: CountingFileSystem, - store: Arc, - namespace: &str, -) -> CachedFileSystem { - let provider: Arc = - Arc::new(YuanrongProvider::from_store(config(), store).await.unwrap()); - CachedFileSystem::new( - Box::new(backend), - provider, - CacheNamespace::new(namespace), - CachePolicy::default(), - ) -} - -#[tokio::test] -async fn yuanrong_hit_miss_fill_and_write_after_read_are_consistent() { - let backend = CountingFileSystem::new(); - backend - .write("/value.md", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let fs = cached_fs(backend, Arc::new(SharedKvStore::default()), "read-write").await; - - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old"); - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old"); - assert_eq!(probe.reads.load(Ordering::SeqCst), 1); - - fs.write("/value.md", b"new", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"new"); - assert_eq!(probe.reads.load(Ordering::SeqCst), 1); -} - -#[tokio::test] -async fn delete_rename_remove_all_and_directory_changes_invalidate_yuanrong_keys() { - let backend = CountingFileSystem::new(); - backend.mkdir("/root", 0o755).await.unwrap(); - backend.mkdir("/root/tree", 0o755).await.unwrap(); - backend - .write("/root/tree/leaf", b"old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let fs = cached_fs(backend, Arc::new(SharedKvStore::default()), "invalidation").await; - - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1); - assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 1); - fs.mkdir("/root/created", 0o755).await.unwrap(); - assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2); - assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 2); - - assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"old"); - fs.rename("/root/tree/leaf", "/root/tree/moved") - .await - .unwrap(); - assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err()); - assert_eq!(fs.read("/root/tree/moved", 0, 0).await.unwrap(), b"old"); - fs.remove("/root/tree/moved").await.unwrap(); - assert!(fs.read("/root/tree/moved", 0, 0).await.is_err()); - - direct - .write("/root/tree/leaf", b"stale", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"stale"); - fs.remove_all("/root/tree").await.unwrap(); - direct.mkdir("/root/tree", 0o755).await.unwrap(); - direct - .write("/root/tree/leaf", b"fresh", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"fresh"); - - fs.rename("/root/tree", "/root/renamed").await.unwrap(); - assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err()); - assert_eq!(fs.read("/root/renamed/leaf", 0, 0).await.unwrap(), b"fresh"); -} - -#[tokio::test] -async fn unavailable_yuanrong_falls_back_without_breaking_backend_reads() { - let backend = CountingFileSystem::new(); - backend - .write("/available.md", b"backend", 0, WriteFlag::Create) - .await - .unwrap(); - let probe = backend.clone(); - let store = Arc::new(SharedKvStore::default()); - let fs = cached_fs(backend, store.clone(), "fallback").await; - store.unavailable.store(true, Ordering::SeqCst); - - assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend"); - assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend"); - assert_eq!(probe.reads.load(Ordering::SeqCst), 2); - assert!(fs.metrics().snapshot().errors >= 1); -} - -#[cfg(feature = "yuanrong-native")] -#[tokio::test] -async fn native_yuanrong_cached_filesystem_hits_fills_writes_and_invalidates() { - if std::env::var("OPENVIKING_RUN_YUANRONG_INTEGRATION").as_deref() != Ok("true") { - return; - } - - let backend = CountingFileSystem::new(); - backend.mkdir("/native", 0o755).await.unwrap(); - backend - .write("/native/value", b"backend-old", 0, WriteFlag::Create) - .await - .unwrap(); - let direct = backend.clone(); - let native_config = YuanrongConfig { - host: std::env::var("YUANRONG_WORKER_HOST").unwrap_or_else(|_| "127.0.0.1".into()), - port: std::env::var("YUANRONG_WORKER_PORT") - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(31501), - connect_timeout_ms: 5_000, - request_timeout_ms: 5_000, - sdk_concurrency: 4, - }; - let provider: Arc = - Arc::new(YuanrongProvider::connect(native_config).await.unwrap()); - let namespace = format!("native-fs-{}", std::process::id()); - let fs = CachedFileSystem::new( - Box::new(backend), - provider.clone(), - CacheNamespace::new(namespace), - CachePolicy::default(), - ); - - assert_eq!( - fs.read("/native/value", 0, 0).await.unwrap(), - b"backend-old" - ); - direct - .write("/native/value", b"backend-mutated", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!( - fs.read("/native/value", 0, 0).await.unwrap(), - b"backend-old", - "second read must come from Yuanrong rather than the mutated backend" - ); - - fs.write("/native/value", b"write-through", 0, WriteFlag::Truncate) - .await - .unwrap(); - assert_eq!( - fs.read("/native/value", 0, 0).await.unwrap(), - b"write-through" - ); - fs.write("/native/empty", b"", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/native/empty", 0, 0).await.unwrap(), b""); - fs.rename("/native/value", "/native/moved").await.unwrap(); - assert!(fs.read("/native/value", 0, 0).await.is_err()); - assert_eq!( - fs.read("/native/moved", 0, 0).await.unwrap(), - b"write-through" - ); - fs.remove("/native/moved").await.unwrap(); - assert!(fs.read("/native/moved", 0, 0).await.is_err()); - - direct - .write("/native/leaf", b"stale", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/native/leaf", 0, 0).await.unwrap(), b"stale"); - fs.remove_all("/native").await.unwrap(); - direct.mkdir("/native", 0o755).await.unwrap(); - direct - .write("/native/leaf", b"fresh", 0, WriteFlag::Create) - .await - .unwrap(); - assert_eq!(fs.read("/native/leaf", 0, 0).await.unwrap(), b"fresh"); - provider.close().await.unwrap(); -} diff --git a/crates/ragfs-cache-yuanrong/tests/native_smoke.rs b/crates/ragfs-cache-yuanrong/tests/native_smoke.rs deleted file mode 100644 index 0531d3cf4d..0000000000 --- a/crates/ragfs-cache-yuanrong/tests/native_smoke.rs +++ /dev/null @@ -1,76 +0,0 @@ -#![cfg(feature = "yuanrong-native")] - -use bytes::Bytes; -use ragfs::cache::CacheProvider; -use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongProvider}; - -fn integration_enabled() -> bool { - std::env::var("OPENVIKING_RUN_YUANRONG_INTEGRATION").as_deref() == Ok("true") -} - -fn config() -> YuanrongConfig { - YuanrongConfig { - host: std::env::var("YUANRONG_WORKER_HOST").unwrap_or_else(|_| "127.0.0.1".into()), - port: std::env::var("YUANRONG_WORKER_PORT") - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(31501), - connect_timeout_ms: 5_000, - request_timeout_ms: 5_000, - sdk_concurrency: 4, - } -} - -#[tokio::test] -async fn native_yuanrong_provider_round_trips_kv_and_batch_operations() { - if !integration_enabled() { - return; - } - - let provider = YuanrongProvider::connect(config()).await.unwrap(); - provider.health_check().await.unwrap(); - let prefix = format!("openviking_native_{}", std::process::id()); - let first = format!("{prefix}_first"); - let second = format!("{prefix}_second"); - - assert_eq!(provider.get(&first).await.unwrap(), None); - provider - .put(&first, Bytes::from_static(b"first-value")) - .await - .unwrap(); - assert_eq!( - provider.get(&first).await.unwrap(), - Some(Bytes::from_static(b"first-value")) - ); - let empty = format!("{prefix}_empty"); - provider.put(&empty, Bytes::new()).await.unwrap(); - assert_eq!(provider.get(&empty).await.unwrap(), Some(Bytes::new())); - - provider - .batch_put(vec![ - (first.clone(), Bytes::from_static(b"updated")), - (second.clone(), Bytes::from_static(b"second-value")), - ]) - .await - .unwrap(); - assert_eq!( - provider - .batch_get(&[second.clone(), format!("{prefix}_missing"), first.clone()]) - .await - .unwrap(), - vec![ - Some(Bytes::from_static(b"second-value")), - None, - Some(Bytes::from_static(b"updated")), - ] - ); - - provider - .invalidate(&[first.clone(), second.clone(), empty.clone()]) - .await - .unwrap(); - assert_eq!(provider.get(&first).await.unwrap(), None); - assert_eq!(provider.get(&second).await.unwrap(), None); - assert_eq!(provider.get(&empty).await.unwrap(), None); - provider.close().await.unwrap(); -} diff --git a/crates/ragfs-cache-yuanrong/tests/provider_contract.rs b/crates/ragfs-cache-yuanrong/tests/provider_contract.rs deleted file mode 100644 index fb9b40ab6c..0000000000 --- a/crates/ragfs-cache-yuanrong/tests/provider_contract.rs +++ /dev/null @@ -1,274 +0,0 @@ -use bytes::Bytes; -use ragfs::cache::{CacheError, CacheProvider}; -use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongKvStore, YuanrongProvider, YuanrongStoreError}; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -#[derive(Default)] -struct FakeKvStore { - values: Mutex>>, - healthy: AtomicBool, - available: AtomicBool, - delay_ms: AtomicUsize, - active: AtomicUsize, - max_active: AtomicUsize, - batch_get_calls: AtomicUsize, - batch_set_calls: AtomicUsize, - batch_delete_calls: AtomicUsize, - shutdown_calls: AtomicUsize, -} - -impl FakeKvStore { - fn available() -> Self { - Self { - healthy: AtomicBool::new(true), - available: AtomicBool::new(true), - ..Self::default() - } - } - - fn enter(&self) -> Result, YuanrongStoreError> { - if !self.available.load(Ordering::SeqCst) { - return Err(YuanrongStoreError::Unavailable("worker unavailable".into())); - } - let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; - self.max_active.fetch_max(active, Ordering::SeqCst); - let delay = self.delay_ms.load(Ordering::SeqCst); - if delay > 0 { - std::thread::sleep(Duration::from_millis(delay as u64)); - } - Ok(ActiveGuard { store: self }) - } -} - -struct ActiveGuard<'a> { - store: &'a FakeKvStore, -} - -impl Drop for ActiveGuard<'_> { - fn drop(&mut self) { - self.store.active.fetch_sub(1, Ordering::SeqCst); - } -} - -impl YuanrongKvStore for FakeKvStore { - fn health_check(&self) -> Result<(), YuanrongStoreError> { - let _guard = self.enter()?; - if self.healthy.load(Ordering::SeqCst) { - Ok(()) - } else { - Err(YuanrongStoreError::Unavailable("unhealthy worker".into())) - } - } - - fn get(&self, key: &str) -> Result>, YuanrongStoreError> { - let _guard = self.enter()?; - Ok(self.values.lock().unwrap().get(key).cloned()) - } - - fn set(&self, key: &str, value: &[u8]) -> Result<(), YuanrongStoreError> { - let _guard = self.enter()?; - self.values - .lock() - .unwrap() - .insert(key.to_owned(), value.to_vec()); - Ok(()) - } - - fn delete(&self, key: &str) -> Result<(), YuanrongStoreError> { - let _guard = self.enter()?; - self.values.lock().unwrap().remove(key); - Ok(()) - } - - fn exists(&self, key: &str) -> Result { - let _guard = self.enter()?; - Ok(self.values.lock().unwrap().contains_key(key)) - } - - fn batch_get(&self, keys: &[String]) -> Result>>, YuanrongStoreError> { - let _guard = self.enter()?; - self.batch_get_calls.fetch_add(1, Ordering::SeqCst); - let values = self.values.lock().unwrap(); - Ok(keys.iter().map(|key| values.get(key).cloned()).collect()) - } - - fn batch_set(&self, entries: &[(String, Vec)]) -> Result<(), YuanrongStoreError> { - let _guard = self.enter()?; - self.batch_set_calls.fetch_add(1, Ordering::SeqCst); - self.values.lock().unwrap().extend(entries.iter().cloned()); - Ok(()) - } - - fn batch_delete(&self, keys: &[String]) -> Result<(), YuanrongStoreError> { - let _guard = self.enter()?; - self.batch_delete_calls.fetch_add(1, Ordering::SeqCst); - let mut values = self.values.lock().unwrap(); - for key in keys { - values.remove(key); - } - Ok(()) - } - - fn shutdown(&self) -> Result<(), YuanrongStoreError> { - self.shutdown_calls.fetch_add(1, Ordering::SeqCst); - Ok(()) - } -} - -fn config() -> YuanrongConfig { - YuanrongConfig { - host: "127.0.0.1".into(), - port: 9088, - connect_timeout_ms: 1_000, - request_timeout_ms: 100, - sdk_concurrency: 2, - } -} - -async fn provider(store: Arc) -> YuanrongProvider { - YuanrongProvider::from_store(config(), store).await.unwrap() -} - -#[tokio::test] -async fn initialization_validates_config_and_health() { - let mut invalid = config(); - invalid.host.clear(); - let error = YuanrongProvider::from_store(invalid, Arc::new(FakeKvStore::available())) - .await - .unwrap_err(); - assert!(matches!(error, CacheError::InvalidArgument(_))); - - let unhealthy = Arc::new(FakeKvStore::available()); - unhealthy.healthy.store(false, Ordering::SeqCst); - let error = YuanrongProvider::from_store(config(), unhealthy) - .await - .unwrap_err(); - assert!(matches!(error, CacheError::Unavailable(_))); -} - -#[tokio::test] -async fn hit_miss_write_delete_and_exists_map_to_kv_operations() { - let store = Arc::new(FakeKvStore::available()); - store - .values - .lock() - .unwrap() - .insert("hit".into(), b"value".to_vec()); - let provider = provider(store).await; - - assert_eq!( - provider.get("hit").await.unwrap(), - Some(Bytes::from_static(b"value")) - ); - assert_eq!(provider.get("missing").await.unwrap(), None); - provider - .put("written", Bytes::from_static(b"new")) - .await - .unwrap(); - assert!(provider.exists("written").await.unwrap()); - assert_eq!( - provider.get("written").await.unwrap(), - Some(Bytes::from_static(b"new")) - ); - provider.delete("written").await.unwrap(); - provider.delete("written").await.unwrap(); - assert_eq!(provider.get("written").await.unwrap(), None); -} - -#[tokio::test] -async fn batch_operations_use_native_kv_batch_calls_and_preserve_order() { - let store = Arc::new(FakeKvStore::available()); - let provider = provider(store.clone()).await; - - provider - .batch_put(vec![ - ("one".into(), Bytes::from_static(b"1")), - ("two".into(), Bytes::from_static(b"2")), - ]) - .await - .unwrap(); - assert_eq!( - provider - .batch_get(&["two".into(), "missing".into(), "one".into()]) - .await - .unwrap(), - vec![ - Some(Bytes::from_static(b"2")), - None, - Some(Bytes::from_static(b"1")) - ] - ); - provider - .invalidate(&["one".into(), "two".into()]) - .await - .unwrap(); - - assert_eq!(store.batch_set_calls.load(Ordering::SeqCst), 1); - assert_eq!(store.batch_get_calls.load(Ordering::SeqCst), 1); - assert_eq!(store.batch_delete_calls.load(Ordering::SeqCst), 1); - assert!(provider.capabilities().batch_get); - assert!(provider.capabilities().batch_put); -} - -#[tokio::test] -async fn synchronous_calls_are_bounded_and_timeout_is_not_a_miss() { - let store = Arc::new(FakeKvStore::available()); - store - .values - .lock() - .unwrap() - .insert("slow".into(), b"value".to_vec()); - store.delay_ms.store(30, Ordering::SeqCst); - let mut bounded = config(); - bounded.request_timeout_ms = 500; - let provider = Arc::new( - YuanrongProvider::from_store(bounded, store.clone()) - .await - .unwrap(), - ); - - let tasks = (0..8) - .map(|_| { - let provider = provider.clone(); - tokio::spawn(async move { provider.get("slow").await.unwrap() }) - }) - .collect::>(); - for task in tasks { - assert_eq!(task.await.unwrap(), Some(Bytes::from_static(b"value"))); - } - assert!(store.max_active.load(Ordering::SeqCst) <= 2); - - let timeout_store = Arc::new(FakeKvStore::available()); - let mut timeout_config = config(); - timeout_config.request_timeout_ms = 10; - let provider = YuanrongProvider::from_store(timeout_config, timeout_store.clone()) - .await - .unwrap(); - timeout_store.delay_ms.store(80, Ordering::SeqCst); - let error = provider.get("slow").await.unwrap_err(); - assert!(matches!(error, CacheError::Timeout(_))); -} - -#[tokio::test] -async fn close_shuts_down_store_and_rejects_new_operations() { - let store = Arc::new(FakeKvStore::available()); - let provider = provider(store.clone()).await; - - provider.close().await.unwrap(); - assert_eq!(store.shutdown_calls.load(Ordering::SeqCst), 1); - let error = provider.get("key").await.unwrap_err(); - assert!(matches!(error, CacheError::Unavailable(_))); -} - -#[cfg(not(feature = "yuanrong-native"))] -#[tokio::test] -async fn connect_without_native_feature_returns_startup_error() { - let error = YuanrongProvider::connect(config()).await.unwrap_err(); - assert!(matches!( - error, - CacheError::Unavailable(message) if message.contains("yuanrong-native") - )); -} diff --git a/docker/mooncake-test/Dockerfile b/docker/mooncake-test/Dockerfile deleted file mode 100644 index 9ffa8b8576..0000000000 --- a/docker/mooncake-test/Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -FROM public.ecr.aws/ubuntu/ubuntu:22.04 - -ARG DEBIAN_FRONTEND=noninteractive -ARG MOONCAKE_COMMIT=1352bbec43081e461356aaecf6c70cddd826b455 -ARG RUST_TOOLCHAIN=1.91.1 - -ENV PATH=/root/.cargo/bin:${PATH} \ - CARGO_NET_GIT_FETCH_WITH_CLI=true - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - clang \ - cmake \ - curl \ - git \ - libasio-dev \ - libboost-all-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - libgrpc-dev \ - libgtest-dev \ - libhiredis-dev \ - libibverbs-dev \ - libjemalloc-dev \ - libjsoncpp-dev \ - libmsgpack-dev \ - libnuma-dev \ - libprotobuf-dev \ - libpython3-dev \ - libssl-dev \ - libunwind-dev \ - liburing-dev \ - libxxhash-dev \ - libyaml-cpp-dev \ - libzstd-dev \ - ninja-build \ - pkg-config \ - protobuf-compiler-grpc \ - python3 \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" - -RUN git clone --filter=blob:none https://github.com/kvcache-ai/Mooncake.git /opt/Mooncake \ - && cd /opt/Mooncake \ - && git checkout "${MOONCAKE_COMMIT}" \ - && git submodule update --init --recursive - -RUN cmake -S /opt/Mooncake/extern/yalantinglibs \ - -B /opt/Mooncake/extern/yalantinglibs/build \ - -G Ninja \ - -DBUILD_EXAMPLES=OFF \ - -DBUILD_BENCHMARK=OFF \ - -DBUILD_UNIT_TESTS=OFF \ - && cmake --build /opt/Mooncake/extern/yalantinglibs/build --parallel 2 \ - && cmake --install /opt/Mooncake/extern/yalantinglibs/build - -RUN cmake -S /opt/Mooncake -B /opt/Mooncake/build -G Ninja \ - -DWITH_STORE=ON \ - -DWITH_STORE_RUST=ON \ - -DWITH_P2P_STORE=OFF \ - -DUSE_TCP=ON \ - -DUSE_HTTP=ON \ - -DUSE_ETCD=OFF \ - -DSTORE_USE_ETCD=OFF \ - -DUSE_REDIS=OFF \ - -DSTORE_USE_REDIS=OFF \ - -DUSE_CUDA=OFF \ - -DUSE_MNNVL=OFF \ - -DUSE_UB=OFF \ - -DUSE_CXL=OFF \ - -DBUILD_UNIT_TESTS=OFF \ - -DBUILD_EXAMPLES=OFF \ - -DBUILD_BENCHMARK=OFF \ - -DWITH_METRICS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - && cmake --build /opt/Mooncake/build \ - --target build_mooncake_store_rust mooncake_master \ - --parallel 2 - -RUN python3 -m pip install --no-cache-dir aiohttp - -COPY run-smoke.sh /usr/local/bin/run-openviking-mooncake-smoke -RUN chmod +x /usr/local/bin/run-openviking-mooncake-smoke - -WORKDIR /workspace/OpenViking -ENTRYPOINT ["/usr/local/bin/run-openviking-mooncake-smoke"] diff --git a/docker/mooncake-test/run-smoke.sh b/docker/mooncake-test/run-smoke.sh deleted file mode 100644 index 656a1f2cf4..0000000000 --- a/docker/mooncake-test/run-smoke.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -MOONCAKE_ROOT=/opt/Mooncake -MOONCAKE_BUILD_DIR=${MOONCAKE_ROOT}/build - -export MOONCAKE_BUILD_DIR -export MOONCAKE_STORE_LIB_DIR=${MOONCAKE_BUILD_DIR}/mooncake-store/src -export MOONCAKE_STORE_INCLUDE_DIR=${MOONCAKE_ROOT}/mooncake-store/include -export LD_LIBRARY_PATH="${MOONCAKE_BUILD_DIR}/mooncake-asio:\ -${MOONCAKE_BUILD_DIR}/mooncake-common:\ -${MOONCAKE_BUILD_DIR}/mooncake-common/src:\ -${MOONCAKE_BUILD_DIR}/mooncake-common/etcd:\ -${MOONCAKE_BUILD_DIR}/mooncake-store/src:\ -${MOONCAKE_BUILD_DIR}/mooncake-store/src/cachelib_memory_allocator:\ -${MOONCAKE_BUILD_DIR}/mooncake-transfer-engine/src:\ -${MOONCAKE_BUILD_DIR}/mooncake-transfer-engine/src/common/base:\ -${LD_LIBRARY_PATH:-}" - -cleanup() { - local exit_code=$? - if [[ -n "${MASTER_PID:-}" ]]; then - kill "${MASTER_PID}" 2>/dev/null || true - fi - if [[ -n "${METADATA_PID:-}" ]]; then - kill "${METADATA_PID}" 2>/dev/null || true - fi - if [[ ${exit_code} -ne 0 ]]; then - echo "=== Mooncake metadata log ===" - cat /tmp/mooncake-metadata.log 2>/dev/null || true - echo "=== Mooncake master log ===" - cat /tmp/mooncake-master.log 2>/dev/null || true - fi - exit "${exit_code}" -} -trap cleanup EXIT - -cd "${MOONCAKE_ROOT}/mooncake-transfer-engine/example/http-metadata-server-python" -python3 bootstrap_server.py >/tmp/mooncake-metadata.log 2>&1 & -METADATA_PID=$! - -"${MOONCAKE_BUILD_DIR}/mooncake-store/src/mooncake_master" \ - --eviction_high_watermark_ratio=0.95 \ - --cluster_id=openviking_smoke \ - --port=50051 \ - >/tmp/mooncake-master.log 2>&1 & -MASTER_PID=$! - -for _ in $(seq 1 60); do - if curl -fsS http://127.0.0.1:8080/health >/dev/null 2>&1 \ - && kill -0 "${MASTER_PID}" 2>/dev/null; then - break - fi - sleep 1 -done - -if ! kill -0 "${METADATA_PID}" 2>/dev/null || ! kill -0 "${MASTER_PID}" 2>/dev/null; then - echo "Mooncake services failed to start" >&2 - exit 1 -fi - -echo "=== Official Mooncake Rust smoke ===" -cd "${MOONCAKE_ROOT}/mooncake-store/rust" -MC_RUST_STORE_RUN_INTEGRATION=true \ -MC_METADATA_SERVER=http://127.0.0.1:8080/metadata \ -MC_RUST_STORE_MASTER_ADDR=127.0.0.1:50051 \ -MC_RUST_STORE_LOCAL_HOSTNAME=127.0.0.1 \ -MC_RUST_STORE_PROTOCOL=tcp \ -MC_RUST_STORE_DEVICE_NAME= \ -cargo test --test minimal_smoke -- --nocapture - -echo "=== OpenViking MooncakeProvider smoke ===" -cd /workspace/OpenViking -OPENVIKING_RUN_MOONCAKE_INTEGRATION=true \ -MOONCAKE_LOCAL_HOSTNAME=127.0.0.1 \ -MOONCAKE_METADATA_SERVER=http://127.0.0.1:8080/metadata \ -MOONCAKE_MASTER_SERVER_ADDR=127.0.0.1:50051 \ -MOONCAKE_PROTOCOL=tcp \ -cargo test --locked -p ragfs-cache-mooncake \ - --features mooncake-native \ - --test native_smoke -- --nocapture diff --git a/docs/en/guides/14-ragfs-cache.md b/docs/en/guides/14-ragfs-cache.md index 6ce0795bda..d9fe972651 100644 --- a/docs/en/guides/14-ragfs-cache.md +++ b/docs/en/guides/14-ragfs-cache.md @@ -38,7 +38,7 @@ Then enable the cache under `storage.agfs.cache` in `~/.openviking/ov.conf`. The "pool_size": 32, "connect_timeout_ms": 1000, "command_timeout_ms": 20, - "key_prefix": "ragfs-cache", + "key_prefix": "", "default_ttl_seconds": 3600, "read_from_replica": false } @@ -65,110 +65,24 @@ Available Providers: | Provider | Best for | Notes | |----------|----------|-------| -| `memory` | Local validation and tests | In-process cache; lost after restart | -| `redis` | Fast rollout on standard networks | Currently supports standalone; read from primary only | -| `yuanrong` | Near-compute cache, shared memory, or heterogeneous multi-tier cache | Requires Yuanrong worker and native feature | -| `mooncake` | Remote memory pool, RDMA/TCP data plane | Requires Mooncake services and native feature | +| `redis` | Default delivery on standard networks | Built into RAGFS; currently supports standalone and reads from primary only | +| `dynamic` | YuanRong, Mooncake, or closed-source cache systems | Loads an external Provider `.so` through a versioned C ABI | -If the runtime package was not compiled with the selected Provider, startup returns an error similar to "requires the ... feature". +`MemoryMockProvider` is only used by unit and smoke tests; it is not a production configuration option. -## Native Provider Builds +## Dynamic Provider Delivery -The standard OpenViking wheel is suitable for the `memory` and `redis` -Providers. The `yuanrong` and `mooncake` Providers depend on platform-specific -native SDKs and must be built for the target deployment environment. +The standard OpenViking wheel contains only the built-in Redis Provider and the DynamicProvider loader. Source code and SDKs for external Providers such as YuanRong and Mooncake are not distributed in the OpenViking repository. The Provider publisher builds and releases the `.so` independently. -Install the wheel builder first: +A dynamic library must export this versioned entry point: -```bash -python -m pip install "maturin[patchelf]" -``` - -### Yuanrong - -Install the Yuanrong DataSystem C++ SDK and export its header and library -locations: - -```bash -export YUANRONG_SDK_INCLUDE=/path/to/yuanrong/include -export YUANRONG_SDK_LIB_DIR=/path/to/yuanrong/lib -# Optional; defaults to "datasystem". -export YUANRONG_SDK_LIB_NAME=datasystem -export LD_LIBRARY_PATH="$YUANRONG_SDK_LIB_DIR:${LD_LIBRARY_PATH:-}" -``` - -Build and install the wheel: - -```bash -maturin build --release \ - --manifest-path crates/ragfs-python-native/Cargo.toml \ - --features yuanrong-native - -python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl -``` - -The Yuanrong worker configured by `storage.agfs.cache.yuanrong` must be -available when OpenViking starts. - -### Mooncake - -Check out the Mooncake revision used by -`crates/ragfs-cache-mooncake/Cargo.toml`, then build Mooncake Store with Rust -support: - -```bash -cmake -S /path/to/Mooncake -B /path/to/Mooncake/build \ - -DWITH_STORE=ON \ - -DWITH_STORE_RUST=ON \ - -DCMAKE_BUILD_TYPE=Release - -cmake --build /path/to/Mooncake/build \ - --target build_mooncake_store_rust mooncake_master -j -``` - -Export the paths required by the official Mooncake Rust binding: - -```bash -export MOONCAKE_BUILD_DIR=/path/to/Mooncake/build -export MOONCAKE_STORE_LIB_DIR="$MOONCAKE_BUILD_DIR/mooncake-store/src" -export MOONCAKE_STORE_INCLUDE_DIR=/path/to/Mooncake/mooncake-store/include -export LD_LIBRARY_PATH="$MOONCAKE_BUILD_DIR/mooncake-common:\ -$MOONCAKE_BUILD_DIR/mooncake-common/src:\ -$MOONCAKE_BUILD_DIR/mooncake-store/src:\ -$MOONCAKE_BUILD_DIR/mooncake-store/src/cachelib_memory_allocator:\ -$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src:\ -$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src/common/base:\ -${LD_LIBRARY_PATH:-}" -``` - -Build and install the wheel: - -```bash -maturin build --release \ - --manifest-path crates/ragfs-python-native/Cargo.toml \ - --features mooncake-native - -python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl +```text +openviking_cache_provider_v1 ``` -The Mooncake metadata service and Master configured by -`storage.agfs.cache.mooncake` must be available when OpenViking starts. -Native wheels are platform-specific and should be built on a system compatible -with the target deployment environment. +Provider artifacts should declare the ABI version, target OS and CPU, minimum glibc version, external SDK version, dynamic dependencies, and SHA256. When a Provider depends on native libraries, its publisher must make them discoverable through RPATH, `LD_LIBRARY_PATH`, or deployment instructions. -For production wheels, use a Mooncake revision whose Rust `build.rs` links -`libasan` only when ASan is explicitly enabled. Verify that the release wheel -does not contain or depend on `libasan`: - -```bash -rm -rf /tmp/ragfs-python-wheel -python -m zipfile -e target/wheels/ragfs_python-*.whl /tmp/ragfs-python-wheel -readelf -d /tmp/ragfs-python-wheel/ragfs_python/ragfs_python.abi3.so \ - | grep libasan -find /tmp/ragfs-python-wheel -name 'libasan*' -``` - -Both checks should produce no output. +External Providers can be upgraded independently without rebuilding the default OpenViking wheel. OpenViking only needs a coordinated upgrade when the DynamicProvider ABI becomes incompatible. ## Configuration @@ -177,9 +91,10 @@ Both checks should produce no output. | Option | Type | Default | Description | |--------|------|---------|-------------| | `enabled` | bool | `false` | Enable the RAGFS cache | -| `provider` | str | `"memory"` | `memory`, `redis`, `yuanrong`, or `mooncake` | +| `provider` | str | `"redis"` | `redis` or `dynamic` | | `namespace` | str | `"openviking"` | Cache namespace for isolating deployments or tenants | | `max_file_size_bytes` | int | `1048576` | Maximum full-file object size admitted to cache | +| `traversal_mode` | str | `"backend"` | Use backend traversal or `cached_traversal` for recursive APIs | | `bypass_prefixes` | list[str] | `[]` | Path prefixes that always bypass cache | Redis configuration: @@ -193,33 +108,13 @@ Redis configuration: | `pool_size` | `32` | Command concurrency | | `connect_timeout_ms` | `1000` | Connection timeout | | `command_timeout_ms` | `20` | Command timeout | -| `key_prefix` | `"ragfs-cache"` | Redis-side key prefix | +| `key_prefix` | `""` | Reserved compatibility field; the unified Runtime requires an empty value | | `default_ttl_seconds` | `3600` | Default TTL; `0` means no TTL | | `read_from_replica` | `false` | Must be `false` in standalone mode | -Yuanrong configuration: - -```json -{ - "storage": { - "agfs": { - "cache": { - "enabled": true, - "provider": "yuanrong", - "yuanrong": { - "host": "127.0.0.1", - "port": 31501, - "connect_timeout_ms": 5000, - "request_timeout_ms": 5000, - "sdk_concurrency": 4 - } - } - } - } -} -``` +DynamicProvider configuration example: -Mooncake configuration: +`dynamic.params` is entirely Provider-owned. The fields below only illustrate configuration forwarding; use the schema documented by the Provider publisher. ```json { @@ -227,18 +122,13 @@ Mooncake configuration: "agfs": { "cache": { "enabled": true, - "provider": "mooncake", - "mooncake": { - "local_hostname": "127.0.0.1", - "metadata_server": "http://127.0.0.1:8080/metadata", - "master_server_addr": "127.0.0.1:50051", - "protocol": "tcp", - "device_name": "", - "global_segment_size": 536870912, - "local_buffer_size": 134217728, - "replica_num": 2, - "sdk_concurrency": 4, - "operation_timeout_ms": 5000 + "provider": "dynamic", + "dynamic": { + "library": "/opt/openviking/providers/libopenviking_cache_provider.so", + "params": { + "endpoint": "127.0.0.1:31501", + "request_timeout_ms": 5000 + } } } } @@ -251,7 +141,7 @@ Mooncake configuration: RAGFS splits caching into two layers: - `CachedFileSystem`: implements filesystem semantics, including cache hit/miss handling, backend fallback, cache fill, invalidation, generation checks, and metrics. -- `CacheProvider`: only stores cache objects through `get`, `put`, `delete`, batch reads/writes, and close operations. +- `CacheRuntime`: exposes common primitive operations and binds either the built-in RedisProvider or an external DynamicProvider at startup. Call flow: @@ -259,11 +149,12 @@ Call flow: OpenViking -> RAGFS / MountableFS -> CachedFileSystem - |-> CacheProvider -> Memory / Redis / Yuanrong / Mooncake + |-> CacheRuntime -> RedisProvider + | `-> DynamicProvider -> external provider .so `-> Backend FileSystem ``` -With this boundary, file, directory, rename, recursive delete, and write-after-invalidation logic live only in the common layer. A Provider does not need to understand path semantics; it only needs to store stable key-value objects. +With this boundary, file, directory, rename, recursive delete, and write-after-invalidation logic live only in the common layer. An external Provider does not need to understand path semantics; it only supplies primitive key-value operations through the stable C ABI. ## Cache Objects @@ -366,9 +257,9 @@ Recommended signals to watch: ## Recommended Rollout -1. Use `memory` locally to validate the configuration shape. -2. Use `redis` to validate real remote-cache benefits. -3. Move to `yuanrong` or `mooncake` for high-performance environments. +1. Disable caching to validate baseline backend behavior. +2. Use the built-in `redis` Provider to validate remote-cache benefits. +3. Use an independently released DynamicProvider `.so` for high-performance or closed-source cache systems. 4. Cache summary files and raw `read_dir` first, then expand to more regular small files. 5. Add lock, control-plane, and permission-sensitive paths to `bypass_prefixes`. diff --git a/docs/zh/guides/14-ragfs-cache.md b/docs/zh/guides/14-ragfs-cache.md index 4450c32ee1..75d4e7af7b 100644 --- a/docs/zh/guides/14-ragfs-cache.md +++ b/docs/zh/guides/14-ragfs-cache.md @@ -38,7 +38,7 @@ openviking-server doctor "pool_size": 32, "connect_timeout_ms": 1000, "command_timeout_ms": 20, - "key_prefix": "ragfs-cache", + "key_prefix": "", "default_ttl_seconds": 3600, "read_from_replica": false } @@ -65,105 +65,24 @@ openviking-server | Provider | 适用场景 | 备注 | |----------|----------|------| -| `memory` | 本地验证、测试 | 进程内缓存，重启后丢失 | -| `redis` | 快速落地、普通网络环境 | 当前支持 standalone；建议只从 primary 读取 | -| `yuanrong` | 近计算缓存、共享内存或异构多级缓存 | 需要 Yuanrong worker 和 native feature | -| `mooncake` | 远程内存池、RDMA/TCP 数据面 | 需要 Mooncake 服务和 native feature | +| `redis` | 默认交付、普通网络环境 | 内置于 RAGFS，当前支持 standalone，只从 primary 读取 | +| `dynamic` | YuanRong、Mooncake 或闭源缓存系统 | 通过版本化 C ABI 加载外部 Provider `.so` | -如果运行包没有编译对应 Provider，启动时会返回类似 “requires the ... feature” 的错误。 +`MemoryMockProvider` 只用于单元测试和 smoke test，不是生产配置项。 -## 原生 Provider 构建 +## 动态 Provider 发布 -标准 OpenViking wheel 适用于 `memory` 和 `redis` Provider。`yuanrong` 和 -`mooncake` Provider 依赖平台相关的原生 SDK，需要针对目标部署环境单独构建。 +标准 OpenViking wheel 只内置 Redis Provider 和 DynamicProvider 加载能力。YuanRong、Mooncake 等外部 Provider 的构建源码和 SDK 不随 OpenViking 主仓发布，由 Provider 发布方在独立环境中构建并交付 `.so`。 -先安装 wheel 构建工具： +动态库必须导出以下版本化入口： -```bash -python -m pip install "maturin[patchelf]" -``` - -### Yuanrong - -安装 Yuanrong DataSystem C++ SDK，并导出头文件和库目录： - -```bash -export YUANRONG_SDK_INCLUDE=/path/to/yuanrong/include -export YUANRONG_SDK_LIB_DIR=/path/to/yuanrong/lib -# 可选，默认值为 "datasystem"。 -export YUANRONG_SDK_LIB_NAME=datasystem -export LD_LIBRARY_PATH="$YUANRONG_SDK_LIB_DIR:${LD_LIBRARY_PATH:-}" -``` - -构建并安装 wheel： - -```bash -maturin build --release \ - --manifest-path crates/ragfs-python-native/Cargo.toml \ - --features yuanrong-native - -python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl -``` - -OpenViking 启动时，`storage.agfs.cache.yuanrong` 配置的 Yuanrong worker -必须可用。 - -### Mooncake - -检出 `crates/ragfs-cache-mooncake/Cargo.toml` 使用的 Mooncake revision， -然后构建启用 Rust 支持的 Mooncake Store： - -```bash -cmake -S /path/to/Mooncake -B /path/to/Mooncake/build \ - -DWITH_STORE=ON \ - -DWITH_STORE_RUST=ON \ - -DCMAKE_BUILD_TYPE=Release - -cmake --build /path/to/Mooncake/build \ - --target build_mooncake_store_rust mooncake_master -j -``` - -导出 Mooncake 官方 Rust binding 所需路径： - -```bash -export MOONCAKE_BUILD_DIR=/path/to/Mooncake/build -export MOONCAKE_STORE_LIB_DIR="$MOONCAKE_BUILD_DIR/mooncake-store/src" -export MOONCAKE_STORE_INCLUDE_DIR=/path/to/Mooncake/mooncake-store/include -export LD_LIBRARY_PATH="$MOONCAKE_BUILD_DIR/mooncake-common:\ -$MOONCAKE_BUILD_DIR/mooncake-common/src:\ -$MOONCAKE_BUILD_DIR/mooncake-store/src:\ -$MOONCAKE_BUILD_DIR/mooncake-store/src/cachelib_memory_allocator:\ -$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src:\ -$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src/common/base:\ -${LD_LIBRARY_PATH:-}" -``` - -构建并安装 wheel： - -```bash -maturin build --release \ - --manifest-path crates/ragfs-python-native/Cargo.toml \ - --features mooncake-native - -python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl +```text +openviking_cache_provider_v1 ``` -OpenViking 启动时，`storage.agfs.cache.mooncake` 配置的 Mooncake metadata -service 和 Master 必须可用。原生 wheel 与平台相关，应在与目标部署环境兼容的 -系统中构建。 +Provider 发布物应注明 ABI 版本、目标 OS/CPU、最低 glibc 版本、外部 SDK 版本、动态依赖和 SHA256。依赖外部原生库时，由 Provider 发布方通过 RPATH、`LD_LIBRARY_PATH` 或部署说明保证动态链接器能够找到依赖。 -生产 wheel 应使用仅在显式启用 ASan 时才链接 `libasan` 的 Mooncake revision。 -构建后检查 release wheel 不包含且不依赖 `libasan`： - -```bash -rm -rf /tmp/ragfs-python-wheel -python -m zipfile -e target/wheels/ragfs_python-*.whl /tmp/ragfs-python-wheel -readelf -d /tmp/ragfs-python-wheel/ragfs_python/ragfs_python.abi3.so \ - | grep libasan -find /tmp/ragfs-python-wheel -name 'libasan*' -``` - -两项检查都应无输出。 +外部 Provider 可以独立升级，不需要重新构建默认 OpenViking wheel；只有 DynamicProvider ABI 不兼容时，才需要同步升级 OpenViking。 ## 配置项 @@ -172,9 +91,10 @@ find /tmp/ragfs-python-wheel -name 'libasan*' | 参数 | 类型 | 默认值 | 说明 | |------|------|--------|------| | `enabled` | bool | `false` | 是否启用 RAGFS 缓存 | -| `provider` | str | `"memory"` | `memory`、`redis`、`yuanrong` 或 `mooncake` | +| `provider` | str | `"redis"` | `redis` 或 `dynamic` | | `namespace` | str | `"openviking"` | 缓存命名空间，用于隔离不同部署或租户 | | `max_file_size_bytes` | int | `1048576` | 允许进入缓存的最大完整文件大小 | +| `traversal_mode` | str | `"backend"` | 递归 API 使用 backend 遍历或 `cached_traversal` | | `bypass_prefixes` | list[str] | `[]` | 强制绕过缓存的路径前缀 | Redis 配置： @@ -188,33 +108,13 @@ Redis 配置： | `pool_size` | `32` | 命令并发数 | | `connect_timeout_ms` | `1000` | 连接超时 | | `command_timeout_ms` | `20` | 命令超时 | -| `key_prefix` | `"ragfs-cache"` | Redis 侧 key 前缀 | +| `key_prefix` | `""` | 保留兼容字段；统一 Runtime 要求为空 | | `default_ttl_seconds` | `3600` | 默认 TTL；`0` 表示不设置 TTL | | `read_from_replica` | `false` | standalone 模式下必须为 `false` | -Yuanrong 配置： - -```json -{ - "storage": { - "agfs": { - "cache": { - "enabled": true, - "provider": "yuanrong", - "yuanrong": { - "host": "127.0.0.1", - "port": 31501, - "connect_timeout_ms": 5000, - "request_timeout_ms": 5000, - "sdk_concurrency": 4 - } - } - } - } -} -``` +DynamicProvider 配置示例： -Mooncake 配置： +`dynamic.params` 完全由外部 Provider 定义，下面的字段只用于说明配置传递方式，实际配置以 Provider 发布说明为准。 ```json { @@ -222,18 +122,13 @@ Mooncake 配置： "agfs": { "cache": { "enabled": true, - "provider": "mooncake", - "mooncake": { - "local_hostname": "127.0.0.1", - "metadata_server": "http://127.0.0.1:8080/metadata", - "master_server_addr": "127.0.0.1:50051", - "protocol": "tcp", - "device_name": "", - "global_segment_size": 536870912, - "local_buffer_size": 134217728, - "replica_num": 2, - "sdk_concurrency": 4, - "operation_timeout_ms": 5000 + "provider": "dynamic", + "dynamic": { + "library": "/opt/openviking/providers/libopenviking_cache_provider.so", + "params": { + "endpoint": "127.0.0.1:31501", + "request_timeout_ms": 5000 + } } } } @@ -246,7 +141,7 @@ Mooncake 配置： RAGFS 将缓存拆成两层： - `CachedFileSystem`：实现文件系统语义，包括 cache hit/miss、backend 回源、回填、失效、generation 校验和指标。 -- `CacheProvider`：只负责缓存对象的 `get`、`put`、`delete`、批量读写和关闭。 +- `CacheRuntime`：向业务层提供统一基础操作，并在启动时绑定内置 RedisProvider 或外部 DynamicProvider。 调用关系： @@ -254,11 +149,12 @@ RAGFS 将缓存拆成两层： OpenViking -> RAGFS / MountableFS -> CachedFileSystem - |-> CacheProvider -> Memory / Redis / Yuanrong / Mooncake + |-> CacheRuntime -> RedisProvider + | `-> DynamicProvider -> external provider .so `-> Backend FileSystem ``` -这种边界让文件、目录、rename、递归删除和写后失效逻辑只在公共层实现。Provider 不需要理解路径语义，只需要稳定存取 key-value 对象。 +这种边界让文件、目录、rename、递归删除和写后失效逻辑只在公共层实现。外部 Provider 不需要理解路径语义，只需要通过稳定 C ABI 提供 key-value 基础操作。 ## 缓存对象 @@ -361,9 +257,9 @@ RAGFS 会自动绕过不适合缓存的路径： ## 推荐使用顺序 -1. 本地用 `memory` 验证配置形态。 -2. 用 `redis` 验证真实远程缓存收益。 -3. 对高性能环境再接入 `yuanrong` 或 `mooncake`。 +1. 关闭缓存验证 backend 基线行为。 +2. 用内置 `redis` 验证真实远程缓存收益。 +3. 对高性能或闭源缓存系统使用独立发布的 DynamicProvider `.so`。 4. 先缓存摘要文件和 raw `read_dir`，再扩展到更多普通小文件。 5. 将锁、控制面和权限敏感路径加入 `bypass_prefixes`。 diff --git a/tests/misc/test_ragfs_python_manifest_isolation.py b/tests/misc/test_ragfs_python_manifest_isolation.py index 1e957651ee..7d0a613c63 100644 --- a/tests/misc/test_ragfs_python_manifest_isolation.py +++ b/tests/misc/test_ragfs_python_manifest_isolation.py @@ -26,7 +26,6 @@ def test_workspace_uses_ragfs_runtime_and_one_python_binding_crate(): manifest = _read(ROOT / "Cargo.toml") workspace = _section(manifest, "workspace") members = _array_items(workspace, "members") - excludes = _array_items(workspace, "exclude") assert "crates/ragfs" in members assert "crates/ragfs-python" in members @@ -36,10 +35,6 @@ def test_workspace_uses_ragfs_runtime_and_one_python_binding_crate(): assert "crates/ragfs-cache-yuanrong" not in members assert "crates/ragfs-cache-yuanrong-sys" not in members - assert "crates/ragfs-cache-mooncake" in excludes - assert "crates/ragfs-cache-yuanrong" in excludes - assert "crates/ragfs-cache-yuanrong-sys" in excludes - def test_ragfs_python_uses_the_runtime_embedded_in_ragfs(): manifest = _read(ROOT / "crates/ragfs-python/Cargo.toml") @@ -58,6 +53,9 @@ def test_ragfs_python_uses_the_runtime_embedded_in_ragfs(): def test_legacy_provider_and_binding_manifests_are_removed(): assert not (ROOT / "crates/ragfs-cache-redis/Cargo.toml").exists() + assert not (ROOT / "crates/ragfs-cache-mooncake/Cargo.toml").exists() + assert not (ROOT / "crates/ragfs-cache-yuanrong/Cargo.toml").exists() + assert not (ROOT / "crates/ragfs-cache-yuanrong-sys/Cargo.toml").exists() assert not (ROOT / "crates/ragfs-python-native/Cargo.toml").exists() @@ -67,4 +65,7 @@ def test_source_distribution_contains_only_active_ragfs_crates(): assert "graft crates/ragfs\n" in manifest assert "graft crates/ragfs-python\n" in manifest assert "graft crates/ragfs-cache-redis" not in manifest + assert "graft crates/ragfs-cache-mooncake" not in manifest + assert "graft crates/ragfs-cache-yuanrong" not in manifest + assert "graft crates/ragfs-cache-yuanrong-sys" not in manifest assert "graft crates/ragfs-python-native" not in manifest From 33b50791e03636e55fe0621d1ca52b318e6b9d62 Mon Sep 17 00:00:00 2001 From: chrisfang Date: Wed, 26 Aug 2026 17:22:12 +0800 Subject: [PATCH 3/7] refactor(ragfs): unify cache runtime and redis backend --- Cargo.lock | 334 ++-- crates/ragfs-python/src/lib.rs | 340 +++- crates/ragfs/Cargo.toml | 7 +- crates/ragfs/src/cache/wrapper.rs | 68 +- crates/ragfs/src/cache_runtime/api.rs | 129 +- crates/ragfs/src/cache_runtime/dynamic/abi.rs | 93 - crates/ragfs/src/cache_runtime/dynamic/mod.rs | 3 - .../src/cache_runtime/dynamic/provider.rs | 368 ---- crates/ragfs/src/cache_runtime/error.rs | 18 + crates/ragfs/src/cache_runtime/memory.rs | 501 +++++- crates/ragfs/src/cache_runtime/mod.rs | 412 ++++- crates/ragfs/src/cache_runtime/provider.rs | 57 +- .../ragfs/src/cache_runtime/redis/client.rs | 677 ++++++-- .../ragfs/src/cache_runtime/redis/config.rs | 326 +++- crates/ragfs/src/cache_runtime/redis/mod.rs | 2 +- .../ragfs/src/cache_runtime/redis/provider.rs | 178 +- crates/ragfs/src/core/mountable.rs | 2 +- .../src/plugins/queuefs/cache_backend.rs | 137 +- .../src/plugins/queuefs/cache_protocol.rs | 209 +++ crates/ragfs/src/plugins/queuefs/mod.rs | 494 +----- .../src/plugins/queuefs/redis_backend.rs | 1519 ----------------- crates/ragfs/tests/cache_runtime.rs | 305 +++- crates/ragfs/tests/cache_wrapper.rs | 48 +- crates/ragfs/tests/dynamic_provider.rs | 132 +- crates/ragfs/tests/redis_runtime.rs | 255 ++- docs/en/guides/01-configuration.md | 58 +- docs/en/guides/14-ragfs-cache.md | 80 +- docs/zh/guides/01-configuration.md | 73 +- docs/zh/guides/14-ragfs-cache.md | 80 +- openviking/utils/agfs_utils.py | 158 +- openviking_cli/utils/config/__init__.py | 2 + openviking_cli/utils/config/agfs_config.py | 109 +- openviking_cli/utils/config/cache_config.py | 24 + .../utils/config/open_viking_config.py | 102 ++ tests/misc/test_config_validation.py | 308 ++++ 35 files changed, 4266 insertions(+), 3342 deletions(-) delete mode 100644 crates/ragfs/src/cache_runtime/dynamic/abi.rs delete mode 100644 crates/ragfs/src/cache_runtime/dynamic/provider.rs create mode 100644 crates/ragfs/src/plugins/queuefs/cache_protocol.rs delete mode 100644 crates/ragfs/src/plugins/queuefs/redis_backend.rs create mode 100644 openviking_cli/utils/config/cache_config.py diff --git a/Cargo.lock b/Cargo.lock index f01f14253e..028e06eebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,7 +26,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures 0.2.17", + "cpufeatures", ] [[package]] @@ -178,10 +178,13 @@ dependencies = [ ] [[package]] -name = "arcstr" -version = "1.2.0" +name = "arc-swap" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] [[package]] name = "arg_enum_proc_macro" @@ -221,17 +224,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-lock" -version = "3.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - [[package]] name = "async-trait" version = "0.1.89" @@ -293,7 +285,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -965,17 +957,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - [[package]] name = "chrono" version = "0.4.44" @@ -1107,20 +1088,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "combine" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" -dependencies = [ - "bytes", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "tokio-util", -] - [[package]] name = "compact_str" version = "0.8.1" @@ -1206,6 +1173,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie-factory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" + [[package]] name = "coolor" version = "1.1.0" @@ -1240,15 +1213,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "crc" version = "3.4.0" @@ -1817,16 +1781,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - [[package]] name = "exr" version = "1.74.0" @@ -1928,6 +1882,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "flume" version = "0.11.1" @@ -1951,6 +1914,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1960,6 +1938,46 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fred" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a7b2fd0f08b23315c13b6156f971aeedb6f75fb16a29ac1872d2eabccc1490e" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "bytes-utils", + "float-cmp", + "fred-macros", + "futures", + "log", + "native-tls", + "parking_lot", + "rand 0.8.5", + "redis-protocol", + "semver", + "sha-1", + "socket2 0.5.10", + "tokio", + "tokio-native-tls", + "tokio-stream", + "tokio-util", + "url", + "urlencoding", +] + +[[package]] +name = "fred-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1458c6e22d36d61507034d5afecc64f105c1d39712b7ac6ec3b352c423f715cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2111,7 +2129,6 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3049,16 +3066,6 @@ dependencies = [ "cc", ] -[[package]] -name = "libloading" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" -dependencies = [ - "cfg-if", - "windows-link", -] - [[package]] name = "libm" version = "0.2.16" @@ -3252,6 +3259,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3284,6 +3297,23 @@ dependencies = [ "pxfm", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -3320,6 +3350,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -3461,12 +3501,59 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -3702,7 +3789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "opaque-debug", "universal-hash", ] @@ -3939,17 +4026,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - [[package]] name = "radix_trie" version = "0.2.1" @@ -3976,6 +4052,7 @@ dependencies = [ "dashmap", "filelocks", "flate2", + "fred", "futures", "gix-actor", "gix-date", @@ -3989,14 +4066,12 @@ dependencies = [ "grep-searcher", "hkdf", "ignore", - "libloading", "lru", "mime_guess", + "native-tls", "path-clean", - "r2d2", "radix_trie", "rand 0.8.5", - "redis", "regex", "rusqlite", "serde", @@ -4053,17 +4128,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.2", - "rand_core 0.10.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -4102,12 +4166,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - [[package]] name = "ratatui" version = "0.29.0" @@ -4200,35 +4258,17 @@ dependencies = [ ] [[package]] -name = "redis" -version = "1.5.0" +name = "redis-protocol" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84" +checksum = "9cdba59219406899220fc4cdfd17a95191ba9c9afb719b5fa5a083d63109a9f1" dependencies = [ - "arcstr", - "async-lock", "bytes", - "cfg-if", - "combine", + "bytes-utils", + "cookie-factory", "crc16", - "futures-util", - "itoa", "log", - "num-bigint", - "percent-encoding", - "pin-project-lite", - "r2d2", - "rand 0.10.2", - "rustls 0.23.37", - "rustls-native-certs", - "ryu", - "sha1_smol", - "socket2 0.6.3", - "tokio", - "tokio-rustls 0.26.4", - "tokio-util", - "url", - "xxhash-rust", + "nom 7.1.3", ] [[package]] @@ -4567,15 +4607,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -4713,6 +4744,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha-1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4720,7 +4762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] @@ -4737,7 +4779,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] @@ -5397,6 +5439,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -6447,12 +6499,6 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" -[[package]] -name = "xxhash-rust" -version = "0.8.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" - [[package]] name = "xz2" version = "0.1.7" diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs index 0ba9fe0fbc..35ba404441 100644 --- a/crates/ragfs-python/src/lib.rs +++ b/crates/ragfs-python/src/lib.rs @@ -286,14 +286,22 @@ struct DynamicCacheConfig { struct RedisCacheConfig { mode: String, endpoints: Vec, + master_name: Option, username: String, password_env: String, + password: String, + sentinel_username: String, + sentinel_password_env: String, + sentinel_password: String, + db: i64, pool_size: usize, connect_timeout_ms: u64, command_timeout_ms: u64, key_prefix: String, default_ttl_seconds: u64, read_from_replica: bool, + tls_enabled: bool, + tls_insecure_skip_verify: bool, } impl Default for RagfsCacheConfig { @@ -326,14 +334,22 @@ impl Default for RedisCacheConfig { Self { mode: "standalone".to_string(), endpoints: vec!["redis://127.0.0.1:6379".to_string()], + master_name: None, username: String::new(), password_env: String::new(), + password: String::new(), + sentinel_username: String::new(), + sentinel_password_env: String::new(), + sentinel_password: String::new(), + db: 0, pool_size: 32, connect_timeout_ms: 1_000, command_timeout_ms: 20, key_prefix: String::new(), default_ttl_seconds: 3_600, read_from_replica: false, + tls_enabled: false, + tls_insecure_skip_verify: false, } } } @@ -347,14 +363,22 @@ impl RagfsCacheConfig { CacheProviderKind::Redis => CacheRuntimeProviderConfig::Redis(RedisProviderConfig { mode: self.redis.mode.clone(), endpoints: self.redis.endpoints.clone(), + master_name: self.redis.master_name.clone(), username: self.redis.username.clone(), password_env: self.redis.password_env.clone(), + password: self.redis.password.clone(), + sentinel_username: self.redis.sentinel_username.clone(), + sentinel_password_env: self.redis.sentinel_password_env.clone(), + sentinel_password: self.redis.sentinel_password.clone(), + db: self.redis.db, pool_size: self.redis.pool_size, connect_timeout_ms: self.redis.connect_timeout_ms, command_timeout_ms: self.redis.command_timeout_ms, key_prefix: self.redis.key_prefix.clone(), default_ttl_seconds: self.redis.default_ttl_seconds, read_from_replica: self.redis.read_from_replica, + tls_enabled: self.redis.tls_enabled, + tls_insecure_skip_verify: self.redis.tls_insecure_skip_verify, }), CacheProviderKind::Dynamic => { CacheRuntimeProviderConfig::Dynamic(DynamicProviderConfig { @@ -381,37 +405,199 @@ fn cache_config_from_ov_conf(path: &str) -> Result { let json: serde_json::Value = serde_json::from_str(&raw) .map_err(|error| format!("failed to parse OpenViking config {path}: {error}"))?; - let configured_provider = json - .get("storage") - .and_then(|storage| storage.get("agfs")) + let agfs = json.get("storage").and_then(|storage| storage.get("agfs")); + let configured_provider = agfs .and_then(|agfs| agfs.get("cache")) .and_then(|cache| cache.get("provider")) .and_then(serde_json::Value::as_str) .unwrap_or("redis"); - let mut config = match json - .get("storage") - .and_then(|storage| storage.get("agfs")) - .and_then(|agfs| agfs.get("cache")) - { + let mut config = match agfs.and_then(|agfs| agfs.get("cache")) { Some(cache) => cache_config_from_value(cache), None => Ok(RagfsCacheConfig::default()), }?; - if json - .get("storage") - .and_then(|storage| storage.get("agfs")) + let queuefs_backend = agfs .and_then(|agfs| agfs.get("queuefs")) .and_then(|queuefs| queuefs.get("backend")) - .and_then(serde_json::Value::as_str) - == Some("cache") - { - config.runtime_enabled = true; - config.provider = provider_kind(configured_provider.to_string())?; + .and_then(serde_json::Value::as_str); + match queuefs_backend { + Some("cache") => { + config.runtime_enabled = true; + config.provider = provider_kind(configured_provider.to_string())?; + } + Some("redis") => { + if configured_provider != "redis" + && agfs + .and_then(|agfs| agfs.get("cache")) + .and_then(serde_json::Value::as_object) + .is_some_and(|cache| cache.contains_key("provider")) + { + return Err( + "queuefs backend=redis conflicts with the global cache provider".to_string(), + ); + } + config.runtime_enabled = true; + config.provider = CacheProviderKind::Redis; + merge_legacy_queuefs_redis_config(&mut config, agfs)?; + } + _ => {} } validate_cache_runtime_config(&config)?; Ok(config) } +fn merge_legacy_queuefs_redis_config( + config: &mut RagfsCacheConfig, + agfs: Option<&serde_json::Value>, +) -> Result<(), String> { + let cache_redis = agfs + .and_then(|agfs| agfs.get("cache")) + .and_then(|cache| cache.get("redis")) + .and_then(serde_json::Value::as_object); + let legacy_redis = agfs + .and_then(|agfs| agfs.get("queuefs")) + .and_then(|queuefs| queuefs.get("redis")) + .map(|redis| { + redis + .as_object() + .ok_or_else(|| "storage.agfs.queuefs.redis must be an object".to_string()) + }) + .transpose()?; + let empty = serde_json::Map::new(); + let legacy_redis = legacy_redis.unwrap_or(&empty); + + let legacy_mode = match string_field(legacy_redis, "mode", "singleton")?.as_str() { + "singleton" => "standalone".to_string(), + value => value.to_string(), + }; + let current_mode = match config.redis.mode.as_str() { + "singleton" => "standalone", + value => value, + }; + if merge_legacy_field( + "mode", + cache_redis, + legacy_redis, + current_mode == legacy_mode, + )? { + config.redis.mode = legacy_mode; + } + + let legacy_endpoints = string_array_field_or_default( + legacy_redis, + "endpoints", + &["redis://127.0.0.1:6379".to_string()], + )?; + if merge_legacy_field( + "endpoints", + cache_redis, + legacy_redis, + config.redis.endpoints == legacy_endpoints, + )? { + config.redis.endpoints = legacy_endpoints; + } + + let legacy_master_name = optional_string_field(legacy_redis, "master_name")?; + if merge_legacy_field( + "master_name", + cache_redis, + legacy_redis, + config.redis.master_name == legacy_master_name, + )? { + config.redis.master_name = legacy_master_name; + } + + macro_rules! merge_string { + ($field:ident) => {{ + let value = + optional_string_field(legacy_redis, stringify!($field))?.unwrap_or_default(); + if merge_legacy_field( + stringify!($field), + cache_redis, + legacy_redis, + config.redis.$field == value, + )? { + config.redis.$field = value; + } + }}; + } + merge_string!(username); + merge_string!(password); + merge_string!(sentinel_username); + merge_string!(sentinel_password); + + macro_rules! merge_i64 { + ($field:ident, $default:expr) => {{ + let value = i64_field(legacy_redis, stringify!($field), $default)?; + if merge_legacy_field( + stringify!($field), + cache_redis, + legacy_redis, + config.redis.$field == value, + )? { + config.redis.$field = value; + } + }}; + } + merge_i64!(db, 0); + + macro_rules! merge_u64 { + ($field:ident, $default:expr) => {{ + let value = u64_field(legacy_redis, stringify!($field), $default)?; + if merge_legacy_field( + stringify!($field), + cache_redis, + legacy_redis, + config.redis.$field == value, + )? { + config.redis.$field = value; + } + }}; + } + merge_u64!(connect_timeout_ms, 3_000); + merge_u64!(command_timeout_ms, 3_000); + + macro_rules! merge_bool { + ($field:ident) => {{ + let value = bool_field(legacy_redis, stringify!($field), false)?; + if merge_legacy_field( + stringify!($field), + cache_redis, + legacy_redis, + config.redis.$field == value, + )? { + config.redis.$field = value; + } + }}; + } + merge_bool!(tls_enabled); + merge_bool!(tls_insecure_skip_verify); + + if !config.redis.password.is_empty() && !config.redis.password_env.is_empty() { + return Err("conflicting Redis setting: password".to_string()); + } + if !config.redis.sentinel_password.is_empty() && !config.redis.sentinel_password_env.is_empty() + { + return Err("conflicting Redis setting: sentinel_password".to_string()); + } + Ok(()) +} + +fn merge_legacy_field( + field: &str, + global: Option<&serde_json::Map>, + legacy: &serde_json::Map, + values_match: bool, +) -> Result { + let global_explicit = global.is_some_and(|global| global.contains_key(field)); + let legacy_explicit = legacy.contains_key(field); + if global_explicit && legacy_explicit && !values_match { + Err(format!("conflicting Redis setting: {field}")) + } else { + Ok(legacy_explicit || !global_explicit) + } +} + fn cache_config_from_value(cache: &serde_json::Value) -> Result { if cache.is_null() { return Ok(RagfsCacheConfig::default()); @@ -448,9 +634,21 @@ fn cache_config_from_value(cache: &serde_json::Value) -> Result Result, + key: &str, +) -> Result, String> { + match object.get(key) { + Some(serde_json::Value::Null) | None => Ok(None), + Some(value) => value + .as_str() + .map(|value| Some(value.to_string())) + .ok_or_else(|| format!("{key} must be a string or null")), + } +} + +fn i64_field( + object: &serde_json::Map, + key: &str, + default: i64, +) -> Result { + match object.get(key) { + Some(value) => value + .as_i64() + .ok_or_else(|| format!("{key} must be an integer")), + None => Ok(default), + } +} + fn u64_field( object: &serde_json::Map, key: &str, @@ -2843,6 +3073,84 @@ mod tests { fs::remove_file(path).unwrap(); } + #[test] + fn legacy_queuefs_redis_config_initializes_the_shared_runtime() { + let path = std::env::temp_dir().join(format!( + "openviking-legacy-queue-redis-config-{}.json", + std::process::id() + )); + fs::write( + &path, + r#"{ + "storage": { + "agfs": { + "queuefs": { + "backend": "redis", + "redis": { + "mode": "singleton", + "endpoints": ["redis://redis.example.com:6379"], + "username": "queue-user", + "password": "legacy-secret", + "db": 2, + "connect_timeout_ms": 1500, + "command_timeout_ms": 2500, + "key_prefix": "tenant-a" + } + } + } + } + }"#, + ) + .unwrap(); + + let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap(); + + assert!(cache_config.runtime_enabled); + assert!(!cache_config.enabled); + assert_eq!(cache_config.provider, CacheProviderKind::Redis); + assert_eq!(cache_config.redis.mode, "standalone"); + assert_eq!( + cache_config.redis.endpoints, + vec!["redis://redis.example.com:6379"] + ); + assert_eq!(cache_config.redis.username, "queue-user"); + assert_eq!(cache_config.redis.password, "legacy-secret"); + assert_eq!(cache_config.redis.db, 2); + + fs::remove_file(path).unwrap(); + } + + #[test] + fn legacy_queuefs_redis_rejects_conflicting_global_settings() { + let path = std::env::temp_dir().join(format!( + "openviking-conflicting-queue-redis-config-{}.json", + std::process::id() + )); + fs::write( + &path, + r#"{ + "storage": { + "agfs": { + "cache": { + "provider": "redis", + "redis": {"endpoints": ["redis://global.example.com:6379"]} + }, + "queuefs": { + "backend": "redis", + "redis": {"endpoints": ["redis://legacy.example.com:6379"]} + } + } + } + }"#, + ) + .unwrap(); + + let error = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap_err(); + assert!(error.contains("conflicting Redis setting: endpoints")); + + fs::remove_file(path).unwrap(); + } + #[test] fn queuefs_cache_runtime_rejects_redis_provider_key_prefix() { let path = std::env::temp_dir().join(format!( diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml index 7f6a0de073..f422c39b63 100644 --- a/crates/ragfs/Cargo.toml +++ b/crates/ragfs/Cargo.toml @@ -83,8 +83,8 @@ bytes = "1.5" # Database rusqlite = { version = "0.32", features = ["bundled"] } -redis = { version = "1.5.0", features = ["cluster", "r2d2", "sentinel", "tls-rustls-insecure", "tokio-rustls-comp"] } -r2d2 = "0.8" +fred = { version = "10.1.0", default-features = false, features = ["i-keys", "i-lists", "i-scripts", "i-server", "i-sets", "replicas", "sentinel-auth", "sha-1", "vendored-openssl"], optional = true } +native-tls = { version = "0.2", optional = true } sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite", "mysql"], optional = true } # AWS S3 @@ -100,7 +100,6 @@ regex = "1.10" mime_guess = "2.0" globset = "0.4" url = "2.5" -libloading = { version = "0.8", optional = true } # Encryption (envelope encryption: AES-256-GCM + HKDF-SHA256) aes-gcm = "0.10" @@ -137,6 +136,6 @@ uuid = { version = "1.0", features = ["v4"] } [features] default = [] -cache = ["dep:libloading"] +cache = ["dep:fred", "dep:native-tls"] s3 = ["aws-sdk-s3", "aws-config", "aws-types"] full = ["s3", "cache"] diff --git a/crates/ragfs/src/cache/wrapper.rs b/crates/ragfs/src/cache/wrapper.rs index 3e93b0eca5..1e91a61ccc 100644 --- a/crates/ragfs/src/cache/wrapper.rs +++ b/crates/ragfs/src/cache/wrapper.rs @@ -2,7 +2,7 @@ use super::envelope::{CacheEnvelope, CacheObjectKind, GenerationSnapshot}; use super::{CacheMetrics, CachePolicy, CacheTraversalMode}; -use crate::cache_runtime::{AsyncCacheRuntime, CacheError, CacheResult, CacheRuntime, PutOptions}; +use crate::cache_runtime::{CacheError, CacheResult, CacheRuntime, SetOptions, SetResult}; use crate::core::filesystem::{ compile_grep_regex, is_excluded_path, normalize_prefix_path, relative_depth, relative_match_file, sort_directory_entries, @@ -22,6 +22,7 @@ use tokio::sync::{Mutex, RwLock}; use uuid::Uuid; const GREP_CACHE_FILE_CONCURRENCY: usize = 8; +const GENERATION_PUT_CONCURRENCY: usize = 8; /// Namespace prepended to every provider key owned by one wrapper. #[derive(Debug, Clone, PartialEq, Eq)] @@ -396,7 +397,7 @@ impl CachedFileSystem { } let started = Instant::now(); - let result = self.runtime.batch_get(keys).await; + let result = self.runtime.mget(keys).await; self.metrics.get(started.elapsed()); let values = result?; if values.len() != keys.len() { @@ -411,10 +412,15 @@ impl CachedFileSystem { async fn cache_put(&self, key: &str, value: Bytes, affected_path: &str) -> bool { let started = Instant::now(); - let result = self.runtime.put(key, value, PutOptions::default()).await; + let result = self.runtime.set(key, value, SetOptions::default()).await; self.metrics.put(started.elapsed()); match result { - Ok(()) => true, + Ok(SetResult::Applied) => true, + Ok(SetResult::ConditionNotMet) => { + self.metrics.error(); + self.mark_bypass(affected_path).await; + false + } Err(_) => { self.metrics.error(); self.mark_bypass(affected_path).await; @@ -425,10 +431,10 @@ impl CachedFileSystem { async fn cache_delete(&self, key: &str, affected_path: &str) { let started = Instant::now(); - let result = self.runtime.delete(key).await; + let result = self.runtime.del(&[key.to_string()]).await; self.metrics.delete(started.elapsed()); match result { - Ok(()) => self.metrics.invalidation(), + Ok(_) => self.metrics.invalidation(), Err(_) => { self.metrics.error(); self.mark_bypass(affected_path).await; @@ -545,39 +551,23 @@ impl CachedFileSystem { } async fn put_missing_generations(&self, missing: Vec<(String, u64)>) { - if missing.is_empty() { - return; - } - - if missing.len() > 1 { - let entries = missing - .into_iter() - .map(|(key, value)| (key, Bytes::copy_from_slice(&value.to_be_bytes()))) - .collect(); - let started = Instant::now(); - if self.runtime.batch_put(entries).await.is_err() { - self.metrics.error(); - } - self.metrics.put(started.elapsed()); - return; - } - - for (key, value) in missing { - let started = Instant::now(); - if self - .runtime - .put( - &key, - Bytes::copy_from_slice(&value.to_be_bytes()), - PutOptions::default(), - ) - .await - .is_err() - { - self.metrics.error(); - } - self.metrics.put(started.elapsed()); - } + stream::iter(missing) + .for_each_concurrent(GENERATION_PUT_CONCURRENCY, |(key, value)| async move { + let started = Instant::now(); + let result = self + .runtime + .set( + &key, + Bytes::copy_from_slice(&value.to_be_bytes()), + SetOptions::default(), + ) + .await; + self.metrics.put(started.elapsed()); + if !matches!(result, Ok(SetResult::Applied)) { + self.metrics.error(); + } + }) + .await; } async fn generation_snapshots(&self, path: &str) -> CacheResult> { diff --git a/crates/ragfs/src/cache_runtime/api.rs b/crates/ragfs/src/cache_runtime/api.rs index 35a91175b8..f1f5131706 100644 --- a/crates/ragfs/src/cache_runtime/api.rs +++ b/crates/ragfs/src/cache_runtime/api.rs @@ -1,54 +1,87 @@ //! Public CacheRuntime operation types and interfaces. -use super::{CacheResult, ScriptRequest, ScriptResult}; -use async_trait::async_trait; use bytes::Bytes; use std::time::Duration; -/// Options applied to one cache write. -#[derive(Debug, Clone, Copy, Default)] -pub struct PutOptions { - /// Optional provider-side expiration. - pub ttl: Option, -} - -/// Asynchronous primitive cache interface. -#[async_trait] -pub trait AsyncCacheRuntime: Send + Sync { - /// Read one value. - async fn get(&self, key: &str) -> CacheResult>; - /// Write one value. - async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()>; - /// Delete one value. - async fn delete(&self, key: &str) -> CacheResult<()>; - /// Check whether one value exists. - async fn exists(&self, key: &str) -> CacheResult; - /// Read multiple values while preserving input order. - async fn batch_get(&self, keys: &[String]) -> CacheResult>>; - /// Write multiple values. - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()>; - /// Delete multiple values. - async fn batch_delete(&self, keys: &[String]) -> CacheResult<()>; - /// Execute one provider-specific named atomic program. - async fn execute_script(&self, request: ScriptRequest) -> CacheResult; -} - -/// Synchronous facade over the same CacheRuntime instance. -pub trait SyncCacheRuntime: Send + Sync { - /// Read one value. - fn get(&self, key: &str) -> CacheResult>; - /// Write one value. - fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()>; - /// Delete one value. - fn delete(&self, key: &str) -> CacheResult<()>; - /// Check whether one value exists. - fn exists(&self, key: &str) -> CacheResult; - /// Read multiple values while preserving input order. - fn batch_get(&self, keys: &[String]) -> CacheResult>>; - /// Write multiple values. - fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()>; - /// Delete multiple values. - fn batch_delete(&self, keys: &[String]) -> CacheResult<()>; - /// Execute one provider-specific named atomic program. - fn execute_script(&self, request: ScriptRequest) -> CacheResult; +/// Expiration applied by Redis SET. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Expiration { + /// Expire after the given duration. + After(Duration), +} + +/// Conditional behavior for Redis SET. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum SetCondition { + /// Always store the value. + #[default] + None, + /// Store only when the key does not exist. + Nx, + /// Store only when the key already exists. + Xx, +} + +/// Options applied to Redis-style SET operations. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SetOptions { + /// Optional value expiration. + pub expiration: Option, + /// Optional existence condition. + pub condition: SetCondition, + /// Preserve an existing expiration when replacing a value. + pub keep_ttl: bool, +} + +/// Outcome of a Redis-style SET operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SetResult { + /// The value was stored. + Applied, + /// NX or XX prevented the write. + ConditionNotMet, +} + +/// Position used by LINSERT. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ListInsertPosition { + /// Insert before the pivot. + Before, + /// Insert after the pivot. + After, +} + +/// List end used by LMOVE. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ListDirection { + /// The head of a list. + Left, + /// The tail of a list. + Right, +} + +/// Arguments for LINSERT. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListInsertRequest { + /// List key. + pub key: String, + /// Insert before or after the pivot. + pub position: ListInsertPosition, + /// Existing pivot value. + pub pivot: Bytes, + /// Value to insert. + pub value: Bytes, +} + +/// Arguments for LMOVE. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListMoveRequest { + /// Source list key. + pub source: String, + /// Destination list key. + pub destination: String, + /// Source list end. + pub source_direction: ListDirection, + /// Destination list end. + pub destination_direction: ListDirection, } diff --git a/crates/ragfs/src/cache_runtime/dynamic/abi.rs b/crates/ragfs/src/cache_runtime/dynamic/abi.rs deleted file mode 100644 index 07721f4e5f..0000000000 --- a/crates/ragfs/src/cache_runtime/dynamic/abi.rs +++ /dev/null @@ -1,93 +0,0 @@ -use std::ffi::{c_char, c_void}; - -pub(super) const ABI_VERSION_V1: u32 = 1; -pub(super) const STATUS_OK: i32 = 0; -pub(super) const STATUS_NOT_FOUND: i32 = 1; - -#[repr(C)] -#[derive(Clone, Copy)] -pub(super) struct OvSlice { - pub(super) ptr: *const u8, - pub(super) len: usize, -} - -impl OvSlice { - pub(super) fn new(value: &[u8]) -> Self { - Self { - ptr: value.as_ptr(), - len: value.len(), - } - } -} - -#[repr(C)] -#[derive(Clone, Copy, Default)] -pub(super) struct OvBuffer { - pub(super) ptr: *mut u8, - pub(super) len: usize, -} - -#[repr(C)] -#[derive(Clone, Copy)] -pub(super) struct OvPutOptions { - pub(super) ttl_ms: u64, - pub(super) has_ttl: u8, -} - -#[repr(C)] -#[derive(Clone, Copy)] -pub(super) struct OvEntry { - pub(super) key: OvSlice, - pub(super) value: OvSlice, - pub(super) ttl_ms: u64, - pub(super) has_ttl: u8, -} - -#[repr(C)] -#[derive(Clone, Copy)] -pub(super) struct OvScriptRequest { - pub(super) script_id: OvSlice, - pub(super) keys: *const OvSlice, - pub(super) key_count: usize, - pub(super) args: *const OvSlice, - pub(super) arg_count: usize, -} - -pub(super) type InitFn = unsafe extern "C" fn(OvSlice, *mut *mut c_void) -> i32; -pub(super) type GetFn = unsafe extern "C" fn(*mut c_void, OvSlice, *mut OvBuffer) -> i32; -pub(super) type PutFn = - unsafe extern "C" fn(*mut c_void, OvSlice, OvSlice, *const OvPutOptions) -> i32; -pub(super) type DeleteFn = unsafe extern "C" fn(*mut c_void, OvSlice) -> i32; -pub(super) type ExistsFn = unsafe extern "C" fn(*mut c_void, OvSlice, *mut u8) -> i32; -pub(super) type BatchGetFn = - unsafe extern "C" fn(*mut c_void, *const OvSlice, usize, *mut OvBuffer) -> i32; -pub(super) type BatchPutFn = unsafe extern "C" fn(*mut c_void, *const OvEntry, usize) -> i32; -pub(super) type BatchDeleteFn = unsafe extern "C" fn(*mut c_void, *const OvSlice, usize) -> i32; -pub(super) type ExecuteScriptFn = - unsafe extern "C" fn(*mut c_void, *const OvScriptRequest, *mut OvBuffer) -> i32; -pub(super) type HealthFn = unsafe extern "C" fn(*mut c_void) -> i32; -pub(super) type FreeBufferFn = unsafe extern "C" fn(*mut OvBuffer); -pub(super) type CloseFn = unsafe extern "C" fn(*mut c_void); -pub(super) type LastErrorFn = unsafe extern "C" fn(*mut c_void) -> *const c_char; - -#[repr(C)] -#[derive(Clone, Copy)] -pub(super) struct OvCacheProviderV1 { - pub(super) abi_version: u32, - pub(super) struct_size: u32, - pub(super) init: Option, - pub(super) get: Option, - pub(super) put: Option, - pub(super) delete_key: Option, - pub(super) exists: Option, - pub(super) batch_get: Option, - pub(super) batch_put: Option, - pub(super) batch_delete: Option, - pub(super) execute_script: Option, - pub(super) health: Option, - pub(super) free_buffer: Option, - pub(super) close: Option, - pub(super) last_error: Option, -} - -pub(super) type ProviderEntryV1 = unsafe extern "C" fn() -> *const OvCacheProviderV1; diff --git a/crates/ragfs/src/cache_runtime/dynamic/mod.rs b/crates/ragfs/src/cache_runtime/dynamic/mod.rs index 24e850c56a..7e9f1b5d74 100644 --- a/crates/ragfs/src/cache_runtime/dynamic/mod.rs +++ b/crates/ragfs/src/cache_runtime/dynamic/mod.rs @@ -1,6 +1,3 @@ -mod abi; mod config; -mod provider; pub use config::DynamicProviderConfig; -pub(crate) use provider::DynamicProvider; diff --git a/crates/ragfs/src/cache_runtime/dynamic/provider.rs b/crates/ragfs/src/cache_runtime/dynamic/provider.rs deleted file mode 100644 index 15f2e89ebf..0000000000 --- a/crates/ragfs/src/cache_runtime/dynamic/provider.rs +++ /dev/null @@ -1,368 +0,0 @@ -use super::abi::*; -use super::DynamicProviderConfig; -use crate::cache_runtime::provider::CacheProvider; -use crate::cache_runtime::{CacheError, CacheResult, PutOptions, ScriptRequest, ScriptResult}; -use async_trait::async_trait; -use bytes::Bytes; -use libloading::Library; -use std::ffi::{c_void, CStr}; -use std::mem::size_of; -use std::ptr::NonNull; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use tokio::sync::Semaphore; - -const MAX_INFLIGHT: u32 = 1024; - -pub(crate) struct DynamicProvider { - inner: Arc, -} - -struct DynamicProviderInner { - _library: Library, - api: OvCacheProviderV1, - handle: NonNull, - closed: AtomicBool, - inflight: Arc, -} - -unsafe impl Send for DynamicProviderInner {} -unsafe impl Sync for DynamicProviderInner {} - -impl DynamicProvider { - pub(crate) async fn connect(config: DynamicProviderConfig) -> CacheResult { - let inner = tokio::task::spawn_blocking(move || unsafe { load(config) }) - .await - .map_err(|error| { - CacheError::Internal(format!("dynamic provider init failed: {error}")) - })??; - Ok(Self { - inner: Arc::new(inner), - }) - } - - async fn call(&self, operation: &'static str, call: F) -> CacheResult - where - T: Send + 'static, - F: FnOnce(Arc) -> CacheResult + Send + 'static, - { - if self.inner.closed.load(Ordering::Acquire) { - return Err(CacheError::Closed); - } - let permit = Arc::clone(&self.inner.inflight) - .acquire_owned() - .await - .map_err(|_| CacheError::Closed)?; - if self.inner.closed.load(Ordering::Acquire) { - return Err(CacheError::Closed); - } - let inner = Arc::clone(&self.inner); - tokio::task::spawn_blocking(move || { - let _permit = permit; - call(inner) - }) - .await - .map_err(|error| CacheError::Internal(format!("dynamic {operation} failed: {error}")))? - } -} - -#[async_trait] -impl CacheProvider for DynamicProvider { - async fn get(&self, key: &str) -> CacheResult> { - let key = key.as_bytes().to_vec(); - self.call("get", move |inner| unsafe { - let mut buffer = OvBuffer::default(); - let status = required(inner.api.get, "get")?( - inner.handle.as_ptr(), - OvSlice::new(&key), - &mut buffer, - ); - if status == STATUS_NOT_FOUND { - return Ok(None); - } - status_ok(&inner, "get", status)?; - take_buffer(&inner, buffer).map(Some) - }) - .await - } - - async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { - let key = key.as_bytes().to_vec(); - self.call("put", move |inner| unsafe { - let ttl_ms = options - .ttl - .map(|ttl| { - u64::try_from(ttl.as_millis()).map_err(|_| { - CacheError::InvalidArgument("dynamic provider TTL is too large".into()) - }) - }) - .transpose()?; - let options = OvPutOptions { - ttl_ms: ttl_ms.unwrap_or_default(), - has_ttl: u8::from(ttl_ms.is_some()), - }; - let status = required(inner.api.put, "put")?( - inner.handle.as_ptr(), - OvSlice::new(&key), - OvSlice::new(&value), - &options, - ); - status_ok(&inner, "put", status) - }) - .await - } - - async fn delete(&self, key: &str) -> CacheResult<()> { - let key = key.as_bytes().to_vec(); - self.call("delete", move |inner| unsafe { - let status = required(inner.api.delete_key, "delete")?( - inner.handle.as_ptr(), - OvSlice::new(&key), - ); - status_ok(&inner, "delete", status) - }) - .await - } - - async fn exists(&self, key: &str) -> CacheResult { - let key = key.as_bytes().to_vec(); - self.call("exists", move |inner| unsafe { - let mut exists = 0; - let status = required(inner.api.exists, "exists")?( - inner.handle.as_ptr(), - OvSlice::new(&key), - &mut exists, - ); - status_ok(&inner, "exists", status)?; - Ok(exists != 0) - }) - .await - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - let keys = keys.to_vec(); - self.call("batch_get", move |inner| unsafe { - let slices = keys - .iter() - .map(|key| OvSlice::new(key.as_bytes())) - .collect::>(); - let mut buffer = OvBuffer::default(); - let status = required(inner.api.batch_get, "batch_get")?( - inner.handle.as_ptr(), - slices.as_ptr(), - slices.len(), - &mut buffer, - ); - status_ok(&inner, "batch_get", status)?; - let payload = take_buffer(&inner, buffer)?; - serde_json::from_slice::>>>(&payload) - .map(|values| { - values - .into_iter() - .map(|value| value.map(Bytes::from)) - .collect() - }) - .map_err(|error| CacheError::InvalidData(error.to_string())) - }) - .await - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - self.call("batch_put", move |inner| unsafe { - let ffi_entries = entries - .iter() - .map(|(key, value)| OvEntry { - key: OvSlice::new(key.as_bytes()), - value: OvSlice::new(value), - ttl_ms: 0, - has_ttl: 0, - }) - .collect::>(); - let status = required(inner.api.batch_put, "batch_put")?( - inner.handle.as_ptr(), - ffi_entries.as_ptr(), - ffi_entries.len(), - ); - status_ok(&inner, "batch_put", status) - }) - .await - } - - async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { - let keys = keys.to_vec(); - self.call("batch_delete", move |inner| unsafe { - let slices = keys - .iter() - .map(|key| OvSlice::new(key.as_bytes())) - .collect::>(); - let status = required(inner.api.batch_delete, "batch_delete")?( - inner.handle.as_ptr(), - slices.as_ptr(), - slices.len(), - ); - status_ok(&inner, "batch_delete", status) - }) - .await - } - - async fn execute_script(&self, request: ScriptRequest) -> CacheResult { - self.call("execute_script", move |inner| unsafe { - let key_slices = request - .keys - .iter() - .map(|key| OvSlice::new(key.as_bytes())) - .collect::>(); - let arg_slices = request - .args - .iter() - .map(|arg| OvSlice::new(arg)) - .collect::>(); - let ffi_request = OvScriptRequest { - script_id: OvSlice::new(request.script_id.as_bytes()), - keys: key_slices.as_ptr(), - key_count: key_slices.len(), - args: arg_slices.as_ptr(), - arg_count: arg_slices.len(), - }; - let mut buffer = OvBuffer::default(); - let status = required(inner.api.execute_script, "execute_script")?( - inner.handle.as_ptr(), - &ffi_request, - &mut buffer, - ); - status_ok(&inner, "execute_script", status)?; - take_buffer(&inner, buffer).map(|payload| ScriptResult { payload }) - }) - .await - } - - async fn close(&self) -> CacheResult<()> { - if self.inner.closed.swap(true, Ordering::AcqRel) { - return Ok(()); - } - let permits = Arc::clone(&self.inner.inflight) - .acquire_many_owned(MAX_INFLIGHT) - .await - .map_err(|_| CacheError::Closed)?; - let inner = Arc::clone(&self.inner); - tokio::task::spawn_blocking(move || unsafe { - required(inner.api.close, "close").map(|close| close(inner.handle.as_ptr())) - }) - .await - .map_err(|error| CacheError::Internal(format!("dynamic close failed: {error}")))??; - drop(permits); - Ok(()) - } -} - -impl Drop for DynamicProviderInner { - fn drop(&mut self) { - if !self.closed.swap(true, Ordering::AcqRel) { - if let Some(close) = self.api.close { - unsafe { close(self.handle.as_ptr()) }; - } - } - } -} - -unsafe fn load(config: DynamicProviderConfig) -> CacheResult { - let library = Library::new(&config.library_path).map_err(|error| { - CacheError::AbiMismatch(format!( - "failed to load {}: {error}", - config.library_path.display() - )) - })?; - let entry = library - .get::(b"openviking_cache_provider_v1\0") - .map_err(|error| CacheError::AbiMismatch(format!("missing provider entry: {error}")))?; - let api_ptr = entry(); - let api = api_ptr - .as_ref() - .copied() - .ok_or_else(|| CacheError::AbiMismatch("provider entry returned null".into()))?; - if api.abi_version != ABI_VERSION_V1 { - return Err(CacheError::AbiMismatch(format!( - "expected ABI {}, got {}", - ABI_VERSION_V1, api.abi_version - ))); - } - if api.struct_size as usize != size_of::() { - return Err(CacheError::AbiMismatch(format!( - "expected struct size {}, got {}", - size_of::(), - api.struct_size - ))); - } - validate_api(&api)?; - let mut handle = std::ptr::null_mut(); - let status = - required(api.init, "init")?(OvSlice::new(config.params_json.as_bytes()), &mut handle); - let handle = NonNull::new(handle) - .ok_or_else(|| CacheError::Unavailable("dynamic provider init returned null".into()))?; - let inner = DynamicProviderInner { - _library: library, - api, - handle, - closed: AtomicBool::new(false), - inflight: Arc::new(Semaphore::new(MAX_INFLIGHT as usize)), - }; - status_ok(&inner, "init", status)?; - let health = required(inner.api.health, "health")?(inner.handle.as_ptr()); - status_ok(&inner, "health", health)?; - Ok(inner) -} - -fn validate_api(api: &OvCacheProviderV1) -> CacheResult<()> { - required(api.init, "init")?; - required(api.get, "get")?; - required(api.put, "put")?; - required(api.delete_key, "delete")?; - required(api.exists, "exists")?; - required(api.batch_get, "batch_get")?; - required(api.batch_put, "batch_put")?; - required(api.batch_delete, "batch_delete")?; - required(api.execute_script, "execute_script")?; - required(api.health, "health")?; - required(api.free_buffer, "free_buffer")?; - required(api.close, "close")?; - required(api.last_error, "last_error")?; - Ok(()) -} - -fn required(function: Option, name: &str) -> CacheResult { - function.ok_or_else(|| CacheError::AbiMismatch(format!("missing function {name}"))) -} - -fn status_ok(inner: &DynamicProviderInner, operation: &str, status: i32) -> CacheResult<()> { - if status == STATUS_OK { - return Ok(()); - } - let last_error = required(inner.api.last_error, "last_error")?; - let error_ptr = unsafe { last_error(inner.handle.as_ptr()) }; - let message = if error_ptr.is_null() { - format!("status {status}") - } else { - unsafe { CStr::from_ptr(error_ptr) } - .to_string_lossy() - .into_owned() - }; - Err(CacheError::Unavailable(format!( - "dynamic provider {operation} failed: {message}" - ))) -} - -unsafe fn take_buffer(inner: &DynamicProviderInner, mut buffer: OvBuffer) -> CacheResult { - let result = if buffer.len == 0 { - Ok(Bytes::new()) - } else if buffer.ptr.is_null() { - Err(CacheError::InvalidData( - "dynamic provider returned a null buffer".into(), - )) - } else { - Ok(Bytes::copy_from_slice(std::slice::from_raw_parts( - buffer.ptr, buffer.len, - ))) - }; - required(inner.api.free_buffer, "free_buffer")?(&mut buffer); - result -} diff --git a/crates/ragfs/src/cache_runtime/error.rs b/crates/ragfs/src/cache_runtime/error.rs index 3adf664605..d4ba510b15 100644 --- a/crates/ragfs/src/cache_runtime/error.rs +++ b/crates/ragfs/src/cache_runtime/error.rs @@ -12,18 +12,36 @@ pub enum CacheError { /// A provider operation exceeded its deadline. #[error("cache provider operation timed out: {0}")] Timeout(String), + /// Redis authentication failed. + #[error("cache provider authentication failed: {0}")] + Authentication(String), + /// The authenticated identity cannot execute an operation. + #[error("cache provider permission denied: {0}")] + PermissionDenied(String), /// A provider returned malformed data. #[error("invalid cache data: {0}")] InvalidData(String), /// A caller supplied an invalid argument. #[error("invalid cache argument: {0}")] InvalidArgument(String), + /// A multi-key Redis operation spans multiple cluster slots. + #[error("cache provider cross-slot operation: {0}")] + CrossSlot(String), + /// A write was sent to a read-only Redis node. + #[error("cache provider is read-only: {0}")] + ReadOnly(String), + /// A registered script is not loaded on the target Redis node. + #[error("cache provider script is not loaded: {0}")] + NoScript(String), /// The selected provider does not implement a named script. #[error("unsupported cache script: {0}")] UnsupportedScript(String), /// A dynamic provider uses an incompatible ABI. #[error("cache provider ABI mismatch: {0}")] AbiMismatch(String), + /// The selected provider is intentionally unavailable in this build. + #[error("unsupported cache provider: {0}")] + UnsupportedProvider(String), /// The runtime has already been closed. #[error("cache runtime is closed")] Closed, diff --git a/crates/ragfs/src/cache_runtime/memory.rs b/crates/ragfs/src/cache_runtime/memory.rs index ea1634c665..86a177b489 100644 --- a/crates/ragfs/src/cache_runtime/memory.rs +++ b/crates/ragfs/src/cache_runtime/memory.rs @@ -1,18 +1,24 @@ //! In-process provider used by Runtime tests and smoke validation. use super::provider::CacheProvider; -use super::{CacheError, CacheResult, PutOptions}; +use super::{ + CacheError, CacheResult, Expiration, ListDirection, ListInsertPosition, ListInsertRequest, + ListMoveRequest, SetCondition, SetOptions, SetResult, +}; use async_trait::async_trait; use bytes::Bytes; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Mutex; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::sync::RwLock; /// Controllable in-memory provider for tests and smoke validation. pub struct MemoryMockProvider { values: RwLock>, + expirations: RwLock>, + sets: RwLock>>, + lists: RwLock>>, closed: AtomicBool, unavailable: AtomicBool, delete_failure: AtomicBool, @@ -23,6 +29,20 @@ pub struct MemoryMockProvider { seen_get_keys: Mutex>, seen_batch_get_keys: Mutex>>, get_delay: Duration, + set_delay: Duration, + active_sets: AtomicU64, + max_active_sets: AtomicU64, + next_set_failure: Mutex>, +} + +struct ActiveSetGuard<'a> { + active_sets: &'a AtomicU64, +} + +impl Drop for ActiveSetGuard<'_> { + fn drop(&mut self) { + self.active_sets.fetch_sub(1, Ordering::Relaxed); + } } impl MemoryMockProvider { @@ -30,6 +50,9 @@ impl MemoryMockProvider { pub fn new() -> Self { Self { values: RwLock::new(HashMap::new()), + expirations: RwLock::new(HashMap::new()), + sets: RwLock::new(HashMap::new()), + lists: RwLock::new(HashMap::new()), closed: AtomicBool::new(false), unavailable: AtomicBool::new(false), delete_failure: AtomicBool::new(false), @@ -40,6 +63,10 @@ impl MemoryMockProvider { seen_get_keys: Mutex::new(Vec::new()), seen_batch_get_keys: Mutex::new(Vec::new()), get_delay: Duration::ZERO, + set_delay: Duration::ZERO, + active_sets: AtomicU64::new(0), + max_active_sets: AtomicU64::new(0), + next_set_failure: Mutex::new(None), } } @@ -49,6 +76,17 @@ impl MemoryMockProvider { self } + /// Delay individual set calls to exercise bounded write concurrency. + pub fn with_set_delay(mut self, delay: Duration) -> Self { + self.set_delay = delay; + self + } + + /// Fail the next set whose key contains the provided text. + pub fn fail_next_set_matching(&self, key_fragment: impl Into) { + *self.next_set_failure.lock().unwrap() = Some(key_fragment.into()); + } + /// Make all provider operations fail or recover them again. pub fn set_unavailable(&self, unavailable: bool) { self.unavailable.store(unavailable, Ordering::Release); @@ -74,6 +112,16 @@ impl MemoryMockProvider { self.values.read().await.keys().cloned().collect() } + /// Seed one set for Set query tests. + pub async fn insert_set_members(&self, key: &str, members: Vec) { + self.sets + .write() + .await + .entry(key.to_string()) + .or_default() + .extend(members); + } + /// Reset observed read calls and concurrency counters. pub fn reset_observed_reads(&self) { self.gets.store(0, Ordering::Relaxed); @@ -107,6 +155,11 @@ impl MemoryMockProvider { self.max_active_gets.load(Ordering::Relaxed) } + /// Return the maximum number of concurrent set calls. + pub fn max_concurrent_sets(&self) -> u64 { + self.max_active_sets.load(Ordering::Relaxed) + } + fn ensure_open(&self) -> CacheResult<()> { if self.closed.load(Ordering::Acquire) { Err(CacheError::Unavailable( @@ -136,6 +189,38 @@ impl MemoryMockProvider { } } } + + fn enter_set(&self) -> ActiveSetGuard<'_> { + let active = self.active_sets.fetch_add(1, Ordering::Relaxed) + 1; + let mut current = self.max_active_sets.load(Ordering::Relaxed); + while active > current { + match self.max_active_sets.compare_exchange_weak( + current, + active, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(observed) => current = observed, + } + } + ActiveSetGuard { + active_sets: &self.active_sets, + } + } + + fn should_fail_set(&self, key: &str) -> bool { + let mut failure = self.next_set_failure.lock().unwrap(); + if failure + .as_deref() + .is_some_and(|key_fragment| key.contains(key_fragment)) + { + *failure = None; + true + } else { + false + } + } } impl Default for MemoryMockProvider { @@ -154,59 +239,431 @@ impl CacheProvider for MemoryMockProvider { if !self.get_delay.is_zero() { tokio::time::sleep(self.get_delay).await; } - let value = self.values.read().await.get(key).cloned(); + let mut values = self.values.write().await; + let mut expirations = self.expirations.write().await; + purge_expired_value(&mut values, &mut expirations, key); + let value = values.get(key).cloned(); self.active_gets.fetch_sub(1, Ordering::Relaxed); Ok(value) } - async fn put(&self, key: &str, value: Bytes, _options: PutOptions) -> CacheResult<()> { + async fn set(&self, key: &str, value: Bytes, options: SetOptions) -> CacheResult { self.ensure_open()?; - self.values.write().await.insert(key.to_string(), value); - Ok(()) + if options.keep_ttl && options.expiration.is_some() { + return Err(CacheError::InvalidArgument( + "SET cannot combine expiration with keep_ttl".into(), + )); + } + let _active_set = self.enter_set(); + if !self.set_delay.is_zero() { + tokio::time::sleep(self.set_delay).await; + } + if self.should_fail_set(key) { + return Err(CacheError::Unavailable(format!( + "memory provider set intentionally failed for {key}" + ))); + } + let deadline = options + .expiration + .map(|expiration| match expiration { + Expiration::After(duration) => expiration_deadline(duration), + }) + .transpose()?; + let mut values = self.values.write().await; + let mut expirations = self.expirations.write().await; + purge_expired_value(&mut values, &mut expirations, key); + let exists = values.contains_key(key); + let applies = match options.condition { + SetCondition::None => true, + SetCondition::Nx => !exists, + SetCondition::Xx => exists, + }; + if !applies { + return Ok(SetResult::ConditionNotMet); + } + self.sets.write().await.remove(key); + self.lists.write().await.remove(key); + values.insert(key.to_string(), value); + if !options.keep_ttl { + if let Some(deadline) = deadline { + expirations.insert(key.to_string(), deadline); + } else { + expirations.remove(key); + } + } + Ok(SetResult::Applied) } - async fn delete(&self, key: &str) -> CacheResult<()> { + async fn del(&self, keys: &[String]) -> CacheResult { self.ensure_open()?; if self.delete_failure.load(Ordering::Acquire) { return Err(CacheError::Unavailable( "memory provider delete intentionally failed".to_string(), )); } - self.values.write().await.remove(key); - Ok(()) + let mut values = self.values.write().await; + let mut expirations = self.expirations.write().await; + let mut sets = self.sets.write().await; + let mut lists = self.lists.write().await; + let mut removed = 0; + for key in keys { + let existed = values.remove(key).is_some() + | sets.remove(key).is_some() + | lists.remove(key).is_some(); + expirations.remove(key); + if existed { + removed += 1; + } + } + Ok(removed) } - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { + async fn mget(&self, keys: &[String]) -> CacheResult>> { self.ensure_open()?; self.batch_gets.fetch_add(1, Ordering::Relaxed); self.seen_batch_get_keys.lock().unwrap().push(keys.to_vec()); - let values = self.values.read().await; + let mut values = self.values.write().await; + let mut expirations = self.expirations.write().await; + for key in keys { + purge_expired_value(&mut values, &mut expirations, key); + } Ok(keys.iter().map(|key| values.get(key).cloned()).collect()) } - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + async fn mset(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { self.ensure_open()?; - self.values.write().await.extend(entries); + let keys = entries.iter().map(|(key, _)| key).collect::>(); + let mut values = self.values.write().await; + let mut expirations = self.expirations.write().await; + let mut sets = self.sets.write().await; + let mut lists = self.lists.write().await; + for key in keys { + expirations.remove(key); + sets.remove(key); + lists.remove(key); + } + values.extend(entries); Ok(()) } - async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + async fn incr_by(&self, key: &str, delta: i64) -> CacheResult { self.ensure_open()?; - if self.delete_failure.load(Ordering::Acquire) { - return Err(CacheError::Unavailable( - "memory provider delete intentionally failed".to_string(), - )); - } let mut values = self.values.write().await; - for key in keys { - values.remove(key); + let current = match values.get(key) { + None => 0, + Some(value) => std::str::from_utf8(value) + .ok() + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| { + CacheError::InvalidData(format!("value at {key} is not an integer")) + })?, + }; + let next = current.checked_add(delta).ok_or_else(|| { + CacheError::InvalidData(format!("integer operation at {key} overflowed")) + })?; + values.insert(key.to_string(), Bytes::from(next.to_string())); + Ok(next) + } + + async fn sismember(&self, key: &str, member: &[u8]) -> CacheResult { + self.ensure_open()?; + Ok(self + .sets + .read() + .await + .get(key) + .is_some_and(|members| members.contains(member))) + } + + async fn smembers(&self, key: &str) -> CacheResult> { + self.ensure_open()?; + Ok(self + .sets + .read() + .await + .get(key) + .map(|members| members.iter().cloned().collect()) + .unwrap_or_default()) + } + + async fn scard(&self, key: &str) -> CacheResult { + self.ensure_open()?; + Ok(self + .sets + .read() + .await + .get(key) + .map_or(0, |members| members.len() as u64)) + } + + async fn lpush(&self, key: &str, values: Vec) -> CacheResult { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let list = lists.entry(key.to_string()).or_default(); + for value in values { + list.push_front(value); + } + Ok(list.len() as u64) + } + + async fn rpush(&self, key: &str, values: Vec) -> CacheResult { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let list = lists.entry(key.to_string()).or_default(); + list.extend(values); + Ok(list.len() as u64) + } + + async fn lpop(&self, key: &str, count: Option) -> CacheResult> { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let Some(list) = lists.get_mut(key) else { + return Ok(Vec::new()); + }; + let count = count.unwrap_or(1) as usize; + let mut values = Vec::with_capacity(count.min(list.len())); + for _ in 0..count { + let Some(value) = list.pop_front() else { + break; + }; + values.push(value); + } + if list.is_empty() { + lists.remove(key); + } + Ok(values) + } + + async fn rpop(&self, key: &str, count: Option) -> CacheResult> { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let Some(list) = lists.get_mut(key) else { + return Ok(Vec::new()); + }; + let count = count.unwrap_or(1) as usize; + let mut values = Vec::with_capacity(count.min(list.len())); + for _ in 0..count { + let Some(value) = list.pop_back() else { + break; + }; + values.push(value); + } + if list.is_empty() { + lists.remove(key); + } + Ok(values) + } + + async fn llen(&self, key: &str) -> CacheResult { + self.ensure_open()?; + Ok(self + .lists + .read() + .await + .get(key) + .map_or(0, |list| list.len() as u64)) + } + + async fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult> { + self.ensure_open()?; + let lists = self.lists.read().await; + let Some(list) = lists.get(key) else { + return Ok(Vec::new()); + }; + let len = list.len() as i64; + let start = normalize_list_index(len, start).max(0); + let stop = normalize_list_index(len, stop).min(len - 1); + if len == 0 || start >= len || start > stop { + return Ok(Vec::new()); + } + Ok(list + .iter() + .skip(start as usize) + .take((stop - start + 1) as usize) + .cloned() + .collect()) + } + + async fn lindex(&self, key: &str, index: i64) -> CacheResult> { + self.ensure_open()?; + let lists = self.lists.read().await; + let Some(list) = lists.get(key) else { + return Ok(None); + }; + let index = normalize_list_index(list.len() as i64, index); + if index < 0 { + return Ok(None); + } + Ok(list.get(index as usize).cloned()) + } + + async fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()> { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let list = lists + .get_mut(key) + .ok_or_else(|| CacheError::InvalidArgument(format!("list {key} does not exist")))?; + let index = normalize_list_index(list.len() as i64, index); + let item = usize::try_from(index) + .ok() + .and_then(|index| list.get_mut(index)) + .ok_or_else(|| CacheError::InvalidArgument("list index is out of range".into()))?; + *item = value; + Ok(()) + } + + async fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()> { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let Some(list) = lists.get(key) else { + return Ok(()); + }; + let retained = list_range(list, start, stop); + if retained.is_empty() { + lists.remove(key); + } else { + lists.insert(key.to_string(), retained.into()); } Ok(()) } + async fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let Some(list) = lists.get_mut(key) else { + return Ok(0); + }; + let limit = count.unsigned_abs() as usize; + let mut removed = 0_u64; + if count >= 0 { + let mut retained = VecDeque::with_capacity(list.len()); + while let Some(item) = list.pop_front() { + if item == value && (count == 0 || removed < limit as u64) { + removed += 1; + } else { + retained.push_back(item); + } + } + *list = retained; + } else { + let mut retained = VecDeque::with_capacity(list.len()); + while let Some(item) = list.pop_back() { + if item == value && removed < limit as u64 { + removed += 1; + } else { + retained.push_front(item); + } + } + *list = retained; + } + if list.is_empty() { + lists.remove(key); + } + Ok(removed) + } + + async fn linsert(&self, request: ListInsertRequest) -> CacheResult { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let Some(list) = lists.get_mut(&request.key) else { + return Ok(0); + }; + let Some(pivot) = list.iter().position(|item| item == &request.pivot) else { + return Ok(-1); + }; + let index = match request.position { + ListInsertPosition::Before => pivot, + ListInsertPosition::After => pivot + 1, + }; + list.insert(index, request.value); + Ok(list.len() as i64) + } + + async fn lmove(&self, request: ListMoveRequest) -> CacheResult> { + self.ensure_open()?; + let mut lists = self.lists.write().await; + let value = { + let Some(source) = lists.get_mut(&request.source) else { + return Ok(None); + }; + match request.source_direction { + ListDirection::Left => source.pop_front(), + ListDirection::Right => source.pop_back(), + } + }; + let Some(value) = value else { + return Ok(None); + }; + let destination = lists.entry(request.destination).or_default(); + match request.destination_direction { + ListDirection::Left => destination.push_front(value.clone()), + ListDirection::Right => destination.push_back(value.clone()), + } + if lists + .get(&request.source) + .is_some_and(|source| source.is_empty()) + { + lists.remove(&request.source); + } + Ok(Some(value)) + } + async fn close(&self) -> CacheResult<()> { self.closed.store(true, Ordering::Release); self.values.write().await.clear(); + self.expirations.write().await.clear(); + self.sets.write().await.clear(); + self.lists.write().await.clear(); Ok(()) } } + +fn expiration_deadline(duration: Duration) -> CacheResult { + if duration.is_zero() { + return Err(CacheError::InvalidArgument( + "expiration must be greater than zero".into(), + )); + } + Instant::now() + .checked_add(duration) + .ok_or_else(|| CacheError::InvalidArgument("expiration is too large".into())) +} + +fn purge_expired_value( + values: &mut HashMap, + expirations: &mut HashMap, + key: &str, +) { + if expirations + .get(key) + .is_some_and(|deadline| *deadline <= Instant::now()) + { + expirations.remove(key); + values.remove(key); + } +} + +fn normalize_list_index(len: i64, index: i64) -> i64 { + if index < 0 { + len.saturating_add(index) + } else { + index + } +} + +fn list_range(list: &VecDeque, start: i64, stop: i64) -> Vec { + let len = list.len() as i64; + if len == 0 { + return Vec::new(); + } + let start = normalize_list_index(len, start).max(0); + let stop = normalize_list_index(len, stop).min(len - 1); + if start >= len || start > stop { + return Vec::new(); + } + list.iter() + .skip(start as usize) + .take((stop - start + 1) as usize) + .cloned() + .collect() +} diff --git a/crates/ragfs/src/cache_runtime/mod.rs b/crates/ragfs/src/cache_runtime/mod.rs index e1db4028fd..6fcc5c7add 100644 --- a/crates/ragfs/src/cache_runtime/mod.rs +++ b/crates/ragfs/src/cache_runtime/mod.rs @@ -9,13 +9,15 @@ mod provider; mod redis; mod script; -pub use api::{AsyncCacheRuntime, PutOptions, SyncCacheRuntime}; +pub use api::{ + Expiration, ListDirection, ListInsertPosition, ListInsertRequest, ListMoveRequest, + SetCondition, SetOptions, SetResult, +}; pub use dynamic::DynamicProviderConfig; pub use error::{CacheError, CacheResult}; pub use memory::MemoryMockProvider; -pub use redis::RedisProviderConfig; +pub use redis::{RedisDeploymentMode, RedisProviderConfig}; -use async_trait::async_trait; use bytes::Bytes; use executor::RuntimeExecutor; use provider::CacheProvider; @@ -57,13 +59,21 @@ impl CacheRuntime { fn from_provider_with_scripts( provider: Arc, scripts: Arc, + ) -> Arc { + let executor = + Arc::new(RuntimeExecutor::new().expect("CacheRuntime executor must initialize")); + Self::from_provider_with_parts(provider, scripts, executor) + } + + fn from_provider_with_parts( + provider: Arc, + scripts: Arc, + executor: Arc, ) -> Arc { Arc::new(Self { provider, scripts, - executor: Arc::new( - RuntimeExecutor::new().expect("CacheRuntime executor must initialize"), - ), + executor, closed: AtomicBool::new(false), }) } @@ -88,16 +98,194 @@ impl CacheRuntime { )) } + /// Connect Redis on the dedicated RuntimeExecutor for synchronous callers. + pub fn connect_sync(config: RedisProviderConfig) -> CacheResult> { + if tokio::runtime::Handle::try_current().is_ok() { + return Err(CacheError::InvalidExecutionContext); + } + let executor = Arc::new(RuntimeExecutor::new()?); + let scripts = Arc::new(ScriptRegistry::default()); + let provider_scripts = Arc::clone(&scripts); + let provider = executor.run(async move { + redis::RedisProvider::connect(config, provider_scripts) + .await + .map(|provider| Arc::new(provider) as Arc) + })?; + Ok(Self::from_provider_with_parts(provider, scripts, executor)) + } + /// Load one provider through the versioned dynamic C ABI. - pub async fn dynamic(config: DynamicProviderConfig) -> CacheResult> { - let provider = dynamic::DynamicProvider::connect(config).await?; - Ok(Self::from_provider(Arc::new(provider))) + pub async fn dynamic(_config: DynamicProviderConfig) -> CacheResult> { + Err(CacheError::UnsupportedProvider( + "DynamicProvider is planned for a later release".into(), + )) } pub(crate) fn register_script(&self, definition: ScriptDefinition) -> CacheResult<()> { self.scripts.register(definition) } + /// Read one value. + pub async fn get(&self, key: &str) -> CacheResult> { + self.ensure_open()?; + self.provider.get(key).await + } + + /// Store one value using Redis SET semantics. + pub async fn set( + &self, + key: &str, + value: Bytes, + options: SetOptions, + ) -> CacheResult { + self.ensure_open()?; + self.provider.set(key, value, options).await + } + + /// Delete multiple keys and return the number removed. + pub async fn del(&self, keys: &[String]) -> CacheResult { + self.ensure_open()?; + self.provider.del(keys).await + } + + /// Read multiple keys while preserving input order. + pub async fn mget(&self, keys: &[String]) -> CacheResult>> { + self.ensure_open()?; + self.provider.mget(keys).await + } + + /// Store multiple values. + pub async fn mset(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + self.ensure_open()?; + self.provider.mset(entries).await + } + + /// Increment one integer value by one. + pub async fn incr(&self, key: &str) -> CacheResult { + self.incr_by(key, 1).await + } + + /// Increment one integer value by `delta`. + pub async fn incr_by(&self, key: &str, delta: i64) -> CacheResult { + self.ensure_open()?; + self.provider.incr_by(key, delta).await + } + + /// Decrement one integer value by one. + pub async fn decr(&self, key: &str) -> CacheResult { + self.incr_by(key, -1).await + } + + /// Decrement one integer value by `delta`. + pub async fn decr_by(&self, key: &str, delta: i64) -> CacheResult { + let increment = delta.checked_neg().ok_or_else(|| { + CacheError::InvalidArgument("decrement delta is too large".to_string()) + })?; + self.incr_by(key, increment).await + } + + /// Check whether a set contains one member. + pub async fn sismember(&self, key: &str, member: &[u8]) -> CacheResult { + self.ensure_open()?; + self.provider.sismember(key, member).await + } + + /// Return all members of one set in unspecified order. + pub async fn smembers(&self, key: &str) -> CacheResult> { + self.ensure_open()?; + self.provider.smembers(key).await + } + + /// Return the cardinality of one set. + pub async fn scard(&self, key: &str) -> CacheResult { + self.ensure_open()?; + self.provider.scard(key).await + } + + /// Push values to the head of one list. + pub async fn lpush(&self, key: &str, values: Vec) -> CacheResult { + self.ensure_open()?; + self.provider.lpush(key, values).await + } + + /// Push values to the tail of one list. + pub async fn rpush(&self, key: &str, values: Vec) -> CacheResult { + self.ensure_open()?; + self.provider.rpush(key, values).await + } + + /// Pop values from the head of one list. + pub async fn lpop(&self, key: &str, count: Option) -> CacheResult> { + self.ensure_open()?; + self.provider.lpop(key, count).await + } + + /// Pop values from the tail of one list. + pub async fn rpop(&self, key: &str, count: Option) -> CacheResult> { + self.ensure_open()?; + self.provider.rpop(key, count).await + } + + /// Return the length of one list. + pub async fn llen(&self, key: &str) -> CacheResult { + self.ensure_open()?; + self.provider.llen(key).await + } + + /// Return an inclusive range from one list. + pub async fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult> { + self.ensure_open()?; + self.provider.lrange(key, start, stop).await + } + + /// Return one list element by index. + pub async fn lindex(&self, key: &str, index: i64) -> CacheResult> { + self.ensure_open()?; + self.provider.lindex(key, index).await + } + + /// Replace one list element by index. + pub async fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()> { + self.ensure_open()?; + self.provider.lset(key, index, value).await + } + + /// Trim one list to an inclusive range. + pub async fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()> { + self.ensure_open()?; + self.provider.ltrim(key, start, stop).await + } + + /// Remove matching elements from one list. + pub async fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult { + self.ensure_open()?; + self.provider.lrem(key, count, value).await + } + + /// Insert one element relative to a pivot. + pub async fn linsert(&self, request: ListInsertRequest) -> CacheResult { + self.ensure_open()?; + self.provider.linsert(request).await + } + + /// Atomically move one element between lists. + pub async fn lmove(&self, request: ListMoveRequest) -> CacheResult> { + self.ensure_open()?; + self.provider.lmove(request).await + } + + /// Execute one registered script. + pub async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + self.ensure_open()?; + self.provider.execute_script(request).await + } + + /// Check whether the provider is healthy. + pub async fn ping(&self) -> CacheResult<()> { + self.ensure_open()?; + self.provider.ping().await + } + /// Wrap the current Runtime with synchronous primitive operations. pub fn sync_facade(self: &Arc) -> SyncCacheRuntimeFacade { SyncCacheRuntimeFacade { @@ -123,108 +311,208 @@ impl CacheRuntime { } } -#[async_trait] -impl AsyncCacheRuntime for CacheRuntime { - async fn get(&self, key: &str) -> CacheResult> { - self.ensure_open()?; - self.provider.get(key).await +/// Stateless synchronous facade over one CacheRuntime. +pub struct SyncCacheRuntimeFacade { + runtime: Arc, + executor: Arc, +} + +impl SyncCacheRuntimeFacade { + /// Read one value. + pub fn get(&self, key: &str) -> CacheResult> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor.run(async move { runtime.get(&key).await }) } - async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { - self.ensure_open()?; - self.provider.put(key, value, options).await + /// Store one value using Redis SET semantics. + pub fn set(&self, key: &str, value: Bytes, options: SetOptions) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { runtime.set(&key, value, options).await }) } - async fn delete(&self, key: &str) -> CacheResult<()> { - self.ensure_open()?; - self.provider.delete(key).await + /// Delete multiple keys and return the number removed. + pub fn del(&self, keys: &[String]) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + let keys = keys.to_vec(); + self.executor.run(async move { runtime.del(&keys).await }) } - async fn exists(&self, key: &str) -> CacheResult { - self.ensure_open()?; - self.provider.exists(key).await + /// Read multiple keys while preserving input order. + pub fn mget(&self, keys: &[String]) -> CacheResult>> { + let runtime = Arc::clone(&self.runtime); + let keys = keys.to_vec(); + self.executor.run(async move { runtime.mget(&keys).await }) } - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - self.ensure_open()?; - self.provider.batch_get(keys).await + /// Store multiple values. + pub fn mset(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + self.executor + .run(async move { runtime.mset(entries).await }) } - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - self.ensure_open()?; - self.provider.batch_put(entries).await + /// Increment one integer value by one. + pub fn incr(&self, key: &str) -> CacheResult { + self.incr_by(key, 1) } - async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { - self.ensure_open()?; - self.provider.batch_delete(keys).await + /// Increment one integer value by `delta`. + pub fn incr_by(&self, key: &str, delta: i64) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { runtime.incr_by(&key, delta).await }) } - async fn execute_script(&self, request: ScriptRequest) -> CacheResult { - self.ensure_open()?; - self.provider.execute_script(request).await + /// Decrement one integer value by one. + pub fn decr(&self, key: &str) -> CacheResult { + self.incr_by(key, -1) } -} -/// Stateless synchronous facade over one CacheRuntime. -pub struct SyncCacheRuntimeFacade { - runtime: Arc, - executor: Arc, -} + /// Decrement one integer value by `delta`. + pub fn decr_by(&self, key: &str, delta: i64) -> CacheResult { + let increment = delta.checked_neg().ok_or_else(|| { + CacheError::InvalidArgument("decrement delta is too large".to_string()) + })?; + self.incr_by(key, increment) + } -impl SyncCacheRuntime for SyncCacheRuntimeFacade { - fn get(&self, key: &str) -> CacheResult> { + /// Check whether a set contains one member. + pub fn sismember(&self, key: &str, member: &[u8]) -> CacheResult { let runtime = Arc::clone(&self.runtime); let key = key.to_string(); + let member = member.to_vec(); self.executor - .run(async move { AsyncCacheRuntime::get(runtime.as_ref(), &key).await }) + .run(async move { runtime.sismember(&key, &member).await }) } - fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { + /// Return all members of one set in unspecified order. + pub fn smembers(&self, key: &str) -> CacheResult> { let runtime = Arc::clone(&self.runtime); let key = key.to_string(); - self.executor.run(async move { - AsyncCacheRuntime::put(runtime.as_ref(), &key, value, options).await - }) + self.executor + .run(async move { runtime.smembers(&key).await }) + } + + /// Return the cardinality of one set. + pub fn scard(&self, key: &str) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor.run(async move { runtime.scard(&key).await }) } - fn delete(&self, key: &str) -> CacheResult<()> { + /// Push values to the head of one list. + pub fn lpush(&self, key: &str, values: Vec) -> CacheResult { let runtime = Arc::clone(&self.runtime); let key = key.to_string(); self.executor - .run(async move { AsyncCacheRuntime::delete(runtime.as_ref(), &key).await }) + .run(async move { runtime.lpush(&key, values).await }) } - fn exists(&self, key: &str) -> CacheResult { + /// Push values to the tail of one list. + pub fn rpush(&self, key: &str, values: Vec) -> CacheResult { let runtime = Arc::clone(&self.runtime); let key = key.to_string(); self.executor - .run(async move { AsyncCacheRuntime::exists(runtime.as_ref(), &key).await }) + .run(async move { runtime.rpush(&key, values).await }) } - fn batch_get(&self, keys: &[String]) -> CacheResult>> { + /// Pop values from the head of one list. + pub fn lpop(&self, key: &str, count: Option) -> CacheResult> { let runtime = Arc::clone(&self.runtime); - let keys = keys.to_vec(); + let key = key.to_string(); self.executor - .run(async move { AsyncCacheRuntime::batch_get(runtime.as_ref(), &keys).await }) + .run(async move { runtime.lpop(&key, count).await }) } - fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + /// Pop values from the tail of one list. + pub fn rpop(&self, key: &str, count: Option) -> CacheResult> { let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); self.executor - .run(async move { AsyncCacheRuntime::batch_put(runtime.as_ref(), entries).await }) + .run(async move { runtime.rpop(&key, count).await }) } - fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { + /// Return the length of one list. + pub fn llen(&self, key: &str) -> CacheResult { let runtime = Arc::clone(&self.runtime); - let keys = keys.to_vec(); + let key = key.to_string(); + self.executor.run(async move { runtime.llen(&key).await }) + } + + /// Return an inclusive range from one list. + pub fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { runtime.lrange(&key, start, stop).await }) + } + + /// Return one list element by index. + pub fn lindex(&self, key: &str, index: i64) -> CacheResult> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); self.executor - .run(async move { AsyncCacheRuntime::batch_delete(runtime.as_ref(), &keys).await }) + .run(async move { runtime.lindex(&key, index).await }) } - fn execute_script(&self, request: ScriptRequest) -> CacheResult { + /// Replace one list element by index. + pub fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()> { let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); self.executor - .run(async move { AsyncCacheRuntime::execute_script(runtime.as_ref(), request).await }) + .run(async move { runtime.lset(&key, index, value).await }) + } + + /// Trim one list to an inclusive range. + pub fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { runtime.ltrim(&key, start, stop).await }) + } + + /// Remove matching elements from one list. + pub fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + let key = key.to_string(); + self.executor + .run(async move { runtime.lrem(&key, count, value).await }) + } + + /// Insert one element relative to a pivot. + pub fn linsert(&self, request: ListInsertRequest) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + self.executor + .run(async move { runtime.linsert(request).await }) + } + + /// Atomically move one element between lists. + pub fn lmove(&self, request: ListMoveRequest) -> CacheResult> { + let runtime = Arc::clone(&self.runtime); + self.executor + .run(async move { runtime.lmove(request).await }) + } + + /// Execute one registered script. + pub fn execute_script(&self, request: ScriptRequest) -> CacheResult { + let runtime = Arc::clone(&self.runtime); + self.executor + .run(async move { runtime.execute_script(request).await }) + } + + /// Check whether the provider is healthy. + pub fn ping(&self) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + self.executor.run(async move { runtime.ping().await }) + } + + /// Close the shared runtime. + pub fn close(&self) -> CacheResult<()> { + let runtime = Arc::clone(&self.runtime); + self.executor.run(async move { runtime.close().await }) } } diff --git a/crates/ragfs/src/cache_runtime/provider.rs b/crates/ragfs/src/cache_runtime/provider.rs index 71422f2a46..3b0fd50e6f 100644 --- a/crates/ragfs/src/cache_runtime/provider.rs +++ b/crates/ragfs/src/cache_runtime/provider.rs @@ -1,45 +1,44 @@ -//! Internal provider seam used by CacheRuntime. +//! Internal provider interface used by CacheRuntime. -use super::{CacheError, CacheResult, PutOptions, ScriptRequest, ScriptResult}; +use super::{ + CacheError, CacheResult, ListInsertRequest, ListMoveRequest, ScriptRequest, ScriptResult, + SetOptions, SetResult, +}; use async_trait::async_trait; use bytes::Bytes; #[async_trait] pub(crate) trait CacheProvider: Send + Sync { async fn get(&self, key: &str) -> CacheResult>; - async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()>; - async fn delete(&self, key: &str) -> CacheResult<()>; + async fn set(&self, key: &str, value: Bytes, options: SetOptions) -> CacheResult; + async fn del(&self, keys: &[String]) -> CacheResult; + async fn mget(&self, keys: &[String]) -> CacheResult>>; + async fn mset(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()>; + async fn incr_by(&self, key: &str, delta: i64) -> CacheResult; + async fn sismember(&self, key: &str, member: &[u8]) -> CacheResult; + async fn smembers(&self, key: &str) -> CacheResult>; + async fn scard(&self, key: &str) -> CacheResult; + async fn lpush(&self, key: &str, values: Vec) -> CacheResult; + async fn rpush(&self, key: &str, values: Vec) -> CacheResult; + async fn lpop(&self, key: &str, count: Option) -> CacheResult>; + async fn rpop(&self, key: &str, count: Option) -> CacheResult>; + async fn llen(&self, key: &str) -> CacheResult; + async fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult>; + async fn lindex(&self, key: &str, index: i64) -> CacheResult>; + async fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()>; + async fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()>; + async fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult; + async fn linsert(&self, request: ListInsertRequest) -> CacheResult; + async fn lmove(&self, request: ListMoveRequest) -> CacheResult>; - async fn exists(&self, key: &str) -> CacheResult { - Ok(self.get(key).await?.is_some()) - } - - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - let mut values = Vec::with_capacity(keys.len()); - for key in keys { - values.push(self.get(key).await?); - } - Ok(values) - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - for (key, value) in entries { - self.put(&key, value, PutOptions::default()).await?; - } - Ok(()) + async fn execute_script(&self, request: ScriptRequest) -> CacheResult { + Err(CacheError::UnsupportedScript(request.script_id)) } - async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { - for key in keys { - self.delete(key).await?; - } + async fn ping(&self) -> CacheResult<()> { Ok(()) } - async fn execute_script(&self, request: ScriptRequest) -> CacheResult { - Err(CacheError::UnsupportedScript(request.script_id)) - } - async fn close(&self) -> CacheResult<()> { Ok(()) } diff --git a/crates/ragfs/src/cache_runtime/redis/client.rs b/crates/ragfs/src/cache_runtime/redis/client.rs index d5db5b2308..8b2d61abb4 100644 --- a/crates/ragfs/src/cache_runtime/redis/client.rs +++ b/crates/ragfs/src/cache_runtime/redis/client.rs @@ -1,73 +1,106 @@ +use super::config::{parse_endpoint, RedisDeploymentMode}; use super::RedisProviderConfig; -use crate::cache_runtime::{CacheError, CacheResult, ScriptValue}; -use redis::aio::MultiplexedConnection; -use redis::{AsyncCommands, RedisError}; +use crate::cache_runtime::{ + CacheError, CacheResult, Expiration, ListDirection, ListInsertPosition, ListInsertRequest, + ListMoveRequest, ScriptValue, SetCondition, SetOptions, SetResult, +}; +use bytes::Bytes; +use fred::error::{Error as FredError, ErrorKind}; +use fred::prelude::*; +use fred::types::{ + config::{ClusterDiscoveryPolicy, Server, TlsConfig, TlsConnector, TlsHostMapping}, + lists::{LMoveDirection, ListLocation}, + ConnectHandle, Expiration as FredExpiration, Map, SetOptions as FredSetOptions, Value, +}; +use futures::Future; +use std::collections::HashMap; use std::env; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{RwLock, Semaphore}; -use url::Url; +use tokio::sync::{Mutex, RwLock, Semaphore}; pub(super) struct RedisClient { - connection: RwLock>, + client: Client, + connection_task: Mutex>, + scripts: RwLock>, concurrency: Arc, concurrency_limit: u32, command_timeout: Duration, + deployment_mode: RedisDeploymentMode, + read_from_replica: bool, closed: AtomicBool, } impl RedisClient { pub(super) async fn connect(config: &RedisProviderConfig) -> CacheResult { config.validate()?; - let redis_client = redis::Client::open(endpoint_url(config)?) - .map_err(|error| map_redis_error("open", error))?; - let connect = async { - redis_client - .get_multiplexed_async_connection() - .await - .map_err(|error| map_redis_error("connect", error)) - }; - let connection = - tokio::time::timeout(Duration::from_millis(config.connect_timeout_ms), connect) - .await - .map_err(|_| { - CacheError::Timeout(format!( - "Redis connect exceeded {} ms", - config.connect_timeout_ms - )) - })??; - let client = Self { - connection: RwLock::new(Some(connection)), + let deployment_mode = config.deployment_mode()?; + let fred_config = fred_config(config, deployment_mode)?; + let mut builder = Builder::from_config(fred_config); + builder + .with_connection_config(|connection| { + connection.connection_timeout = Duration::from_millis(config.connect_timeout_ms); + connection.internal_command_timeout = + Duration::from_millis(config.command_timeout_ms); + connection.max_command_attempts = 1; + connection.max_command_buffer_len = config.pool_size; + if config.read_from_replica { + connection.replica.primary_fallback = true; + } + }) + .with_performance_config(|performance| { + performance.default_command_timeout = + Duration::from_millis(config.command_timeout_ms); + }) + .set_policy(ReconnectPolicy::new_exponential(0, 100, 30_000, 2)); + let client = builder + .build() + .map_err(|error| map_fred_error("build", error))?; + let connect_timeout = Duration::from_millis(config.connect_timeout_ms); + let connection_task = tokio::time::timeout(connect_timeout, client.init()) + .await + .map_err(|_| { + CacheError::Timeout(format!( + "Redis connect exceeded {} ms", + config.connect_timeout_ms + )) + })? + .map_err(|error| map_fred_error("connect", error))?; + let concurrency_limit = u32::try_from(config.pool_size).map_err(|_| { + CacheError::InvalidArgument("Redis pool_size exceeds the supported limit".into()) + })?; + let result = Self { + client, + connection_task: Mutex::new(Some(connection_task)), + scripts: RwLock::new(HashMap::new()), concurrency: Arc::new(Semaphore::new(config.pool_size)), - concurrency_limit: config.pool_size as u32, + concurrency_limit, command_timeout: Duration::from_millis(config.command_timeout_ms), + deployment_mode, + read_from_replica: config.read_from_replica, closed: AtomicBool::new(false), }; - client.health_check().await?; - Ok(client) + result.health_check().await?; + Ok(result) } - async fn execute(&self, operation: &'static str, call: F) -> CacheResult + async fn execute(&self, operation: &'static str, future: F) -> CacheResult where T: Send, - F: FnOnce(MultiplexedConnection) -> Fut + Send, - Fut: std::future::Future> + Send, + F: Future> + Send, { if self.closed.load(Ordering::Acquire) { - return Err(CacheError::Unavailable("Redis provider is closed".into())); + return Err(CacheError::Closed); } let work = async { let _permit = Arc::clone(&self.concurrency) .acquire_owned() .await - .map_err(|_| CacheError::Unavailable("Redis client is closing".into()))?; - let connection = self.connection.read().await.clone().ok_or_else(|| { - CacheError::Unavailable("Redis connection has been released".into()) - })?; - call(connection) + .map_err(|_| CacheError::Closed)?; + future .await - .map_err(|error| map_redis_error(operation, error)) + .map_err(|error| map_fred_error(operation, error)) }; tokio::time::timeout(self.command_timeout, work) .await @@ -80,140 +113,315 @@ impl RedisClient { } pub(super) async fn health_check(&self) -> CacheResult<()> { - self.execute("PING", |mut connection| async move { - redis::cmd("PING").query_async(&mut connection).await - }) - .await + let _: String = self + .execute("PING", self.client.ping::(None)) + .await?; + Ok(()) } - pub(super) async fn get(&self, key: String) -> CacheResult>> { - self.execute( - "GET", - |mut connection| async move { connection.get(key).await }, - ) - .await + pub(super) async fn get(&self, key: &str) -> CacheResult> { + if self.read_from_replica { + self.execute("GET", self.client.replicas().get(key)).await + } else { + self.execute("GET", self.client.get(key)).await + } } pub(super) async fn set( &self, - key: String, - value: Vec, - ttl_ms: Option, - ) -> CacheResult<()> { - self.execute("SET", |mut connection| async move { - if let Some(ttl_ms) = ttl_ms { - redis::cmd("SET") - .arg(key) - .arg(value) - .arg("PX") - .arg(ttl_ms) - .query_async(&mut connection) - .await - } else { - connection.set(key, value).await - } - }) - .await - } - - pub(super) async fn delete(&self, key: String) -> CacheResult<()> { - self.execute("DEL", |mut connection| async move { - let _: u64 = connection.del(key).await?; - Ok(()) + key: &str, + value: Bytes, + options: SetOptions, + default_ttl: Option, + ) -> CacheResult { + if options.keep_ttl && options.expiration.is_some() { + return Err(CacheError::InvalidArgument( + "Redis SET cannot combine expiration with keep_ttl".into(), + )); + } + let expiration = fred_expiration(options, default_ttl)?; + let condition = match options.condition { + SetCondition::None => None, + SetCondition::Nx => Some(FredSetOptions::NX), + SetCondition::Xx => Some(FredSetOptions::XX), + }; + let response: Option = self + .execute( + "SET", + self.client.set(key, value, expiration, condition, false), + ) + .await?; + Ok(if response.is_some() { + SetResult::Applied + } else { + SetResult::ConditionNotMet }) - .await } - pub(super) async fn exists(&self, key: String) -> CacheResult { - self.execute("EXISTS", |mut connection| async move { - connection.exists(key).await - }) - .await + pub(super) async fn del(&self, keys: &[String]) -> CacheResult { + if keys.is_empty() { + return Ok(0); + } + self.execute("DEL", self.client.del(keys.to_vec())).await } - pub(super) async fn batch_get(&self, keys: Vec) -> CacheResult>>> { + pub(super) async fn mget(&self, keys: &[String]) -> CacheResult>> { if keys.is_empty() { return Ok(Vec::new()); } - self.execute("MGET", |mut connection| async move { - redis::cmd("MGET") - .arg(keys) - .query_async(&mut connection) + if self.deployment_mode == RedisDeploymentMode::Cluster && !keys_share_slot(keys) { + let mut values = Vec::with_capacity(keys.len()); + for key in keys { + values.push(self.get(key).await?); + } + return Ok(values); + } + if self.read_from_replica { + self.execute("MGET", self.client.replicas().mget(keys.to_vec())) .await - }) - .await + } else { + self.execute("MGET", self.client.mget(keys.to_vec())).await + } } - pub(super) async fn batch_set( + pub(super) async fn mset( &self, - entries: Vec<(String, Vec)>, - ttl_ms: Option, + entries: Vec<(String, Bytes)>, + default_ttl: Option, ) -> CacheResult<()> { if entries.is_empty() { return Ok(()); } - self.execute("pipeline SET", |mut connection| async move { - let mut pipe = redis::pipe(); + let keys = entries + .iter() + .map(|(key, _)| key.clone()) + .collect::>(); + self.require_same_slot(&keys, "MSET")?; + if let Some(default_ttl) = default_ttl { + let expiration = duration_to_expiration(default_ttl)?; + let pipeline = self.client.pipeline(); for (key, value) in entries { - if let Some(ttl_ms) = ttl_ms { - pipe.cmd("SET") - .arg(key) - .arg(value) - .arg("PX") - .arg(ttl_ms) - .ignore(); - } else { - pipe.cmd("SET").arg(key).arg(value).ignore(); - } + let _: () = pipeline + .set(key, value, Some(expiration.clone()), None, false) + .await + .map_err(|error| map_fred_error("MSET pipeline", error))?; } - pipe.query_async(&mut connection).await - }) - .await + let _: () = self.execute("MSET pipeline", pipeline.all()).await?; + } else { + let values = entries.into_iter().collect::(); + self.execute("MSET", self.client.mset(values)).await?; + } + Ok(()) } - pub(super) async fn batch_delete(&self, keys: Vec) -> CacheResult<()> { - if keys.is_empty() { - return Ok(()); + pub(super) async fn incr_by(&self, key: &str, delta: i64) -> CacheResult { + self.execute("INCRBY", self.client.incr_by(key, delta)) + .await + } + + pub(super) async fn sismember(&self, key: &str, member: &[u8]) -> CacheResult { + if self.read_from_replica { + self.execute("SISMEMBER", self.client.replicas().sismember(key, member)) + .await + } else { + self.execute("SISMEMBER", self.client.sismember(key, member)) + .await } - self.execute("DEL", |mut connection| async move { - let _: u64 = connection.del(keys).await?; - Ok(()) - }) + } + + pub(super) async fn smembers(&self, key: &str) -> CacheResult> { + if self.read_from_replica { + self.execute("SMEMBERS", self.client.replicas().smembers(key)) + .await + } else { + self.execute("SMEMBERS", self.client.smembers(key)).await + } + } + + pub(super) async fn scard(&self, key: &str) -> CacheResult { + if self.read_from_replica { + self.execute("SCARD", self.client.replicas().scard(key)) + .await + } else { + self.execute("SCARD", self.client.scard(key)).await + } + } + + pub(super) async fn lpush(&self, key: &str, values: Vec) -> CacheResult { + if values.is_empty() { + return Err(CacheError::InvalidArgument( + "Redis LPUSH requires at least one value".into(), + )); + } + self.execute("LPUSH", self.client.lpush(key, values)).await + } + + pub(super) async fn rpush(&self, key: &str, values: Vec) -> CacheResult { + if values.is_empty() { + return Err(CacheError::InvalidArgument( + "Redis RPUSH requires at least one value".into(), + )); + } + self.execute("RPUSH", self.client.rpush(key, values)).await + } + + pub(super) async fn lpop(&self, key: &str, count: Option) -> CacheResult> { + if let Some(count) = count { + let count = usize::try_from(count) + .map_err(|_| CacheError::InvalidArgument("Redis LPOP count is too large".into()))?; + self.execute("LPOP", self.client.lpop(key, Some(count))) + .await + } else { + let value: Option = self.execute("LPOP", self.client.lpop(key, None)).await?; + Ok(value.into_iter().collect()) + } + } + + pub(super) async fn rpop(&self, key: &str, count: Option) -> CacheResult> { + if let Some(count) = count { + let count = usize::try_from(count) + .map_err(|_| CacheError::InvalidArgument("Redis RPOP count is too large".into()))?; + self.execute("RPOP", self.client.rpop(key, Some(count))) + .await + } else { + let value: Option = self.execute("RPOP", self.client.rpop(key, None)).await?; + Ok(value.into_iter().collect()) + } + } + + pub(super) async fn llen(&self, key: &str) -> CacheResult { + if self.read_from_replica { + self.execute("LLEN", self.client.replicas().llen(key)).await + } else { + self.execute("LLEN", self.client.llen(key)).await + } + } + + pub(super) async fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult> { + if self.read_from_replica { + self.execute("LRANGE", self.client.replicas().lrange(key, start, stop)) + .await + } else { + self.execute("LRANGE", self.client.lrange(key, start, stop)) + .await + } + } + + pub(super) async fn lindex(&self, key: &str, index: i64) -> CacheResult> { + if self.read_from_replica { + self.execute("LINDEX", self.client.replicas().lindex(key, index)) + .await + } else { + self.execute("LINDEX", self.client.lindex(key, index)).await + } + } + + pub(super) async fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()> { + self.execute("LSET", self.client.lset(key, index, value)) + .await + } + + pub(super) async fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()> { + self.execute("LTRIM", self.client.ltrim(key, start, stop)) + .await + } + + pub(super) async fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult { + self.execute("LREM", self.client.lrem(key, count, value)) + .await + } + + pub(super) async fn linsert(&self, request: ListInsertRequest) -> CacheResult { + let position = match request.position { + ListInsertPosition::Before => ListLocation::Before, + ListInsertPosition::After => ListLocation::After, + }; + self.execute( + "LINSERT", + self.client + .linsert(request.key, position, request.pivot, request.value), + ) + .await + } + + pub(super) async fn lmove(&self, request: ListMoveRequest) -> CacheResult> { + self.require_same_slot( + &[request.source.clone(), request.destination.clone()], + "LMOVE", + )?; + self.execute( + "LMOVE", + self.client.lmove( + request.source, + request.destination, + list_direction(request.source_direction), + list_direction(request.destination_direction), + ), + ) .await } pub(super) async fn execute_script( &self, + script_id: &str, lua: &'static str, keys: Vec, - args: Vec>, + args: Vec, ) -> CacheResult { - self.execute("EVALSHA", |mut connection| async move { - let script = redis::Script::new(lua); - let mut invocation = script.prepare_invoke(); - for key in keys { - invocation.key(key); - } - for arg in args { - invocation.arg(arg); + self.require_same_slot(&keys, "EVALSHA")?; + let cached_sha = { + let scripts = self.scripts.read().await; + scripts.get(script_id).cloned() + }; + let sha = match cached_sha { + Some(sha) => sha, + None => self.load_script(script_id, lua).await?, + }; + match self.evalsha(&sha, keys.clone(), args.clone()).await { + Err(CacheError::NoScript(_)) => { + let sha = self.load_script(script_id, lua).await?; + self.evalsha(&sha, keys, args).await } - invocation - .invoke_async::(&mut connection) - .await - }) - .await - .and_then(redis_value_to_script_value) + result => result, + } + } + + async fn load_script(&self, script_id: &str, lua: &'static str) -> CacheResult { + let sha: String = if self.deployment_mode == RedisDeploymentMode::Cluster { + self.execute("SCRIPT LOAD", self.client.script_load_cluster(lua)) + .await? + } else { + self.execute("SCRIPT LOAD", self.client.script_load(lua)) + .await? + }; + self.scripts + .write() + .await + .insert(script_id.to_string(), sha.clone()); + Ok(sha) + } + + async fn evalsha( + &self, + sha: &str, + keys: Vec, + args: Vec, + ) -> CacheResult { + let value: Value = self + .execute("EVALSHA", self.client.evalsha(sha, keys, args)) + .await?; + fred_value_to_script_value(value) } #[cfg(test)] pub(super) async fn script_flush(&self) -> CacheResult<()> { - self.execute("SCRIPT FLUSH", |mut connection| async move { - redis::cmd("SCRIPT") - .arg("FLUSH") - .query_async(&mut connection) + if self.deployment_mode == RedisDeploymentMode::Cluster { + self.execute("SCRIPT FLUSH", self.client.script_flush_cluster(false)) .await - }) - .await + } else { + self.execute("SCRIPT FLUSH", self.client.script_flush(false)) + .await + } } pub(super) async fn close(&self) -> CacheResult<()> { @@ -223,67 +431,184 @@ impl RedisClient { let permits = Arc::clone(&self.concurrency) .acquire_many_owned(self.concurrency_limit) .await - .map_err(|_| CacheError::Unavailable("Redis client is closing".into()))?; - self.connection.write().await.take(); + .map_err(|_| CacheError::Closed)?; + self.client + .quit() + .await + .map_err(|error| map_fred_error("QUIT", error))?; + if let Some(task) = self.connection_task.lock().await.take() { + task.await + .map_err(|error| CacheError::Internal(format!("Redis task failed: {error}")))? + .map_err(|error| map_fred_error("connection task", error))?; + } drop(permits); Ok(()) } + + fn require_same_slot(&self, keys: &[String], operation: &str) -> CacheResult<()> { + if self.deployment_mode == RedisDeploymentMode::Cluster && !keys_share_slot(keys) { + Err(CacheError::CrossSlot(format!( + "Redis {operation} keys do not share one cluster slot" + ))) + } else { + Ok(()) + } + } } -fn redis_value_to_script_value(value: redis::Value) -> CacheResult { +fn fred_config( + config: &RedisProviderConfig, + deployment_mode: RedisDeploymentMode, +) -> CacheResult { + let hosts = config + .endpoints + .iter() + .map(|endpoint| parse_endpoint(endpoint).map(|(host, port)| Server::new(host, port))) + .collect::>>()?; + let server = match deployment_mode { + RedisDeploymentMode::Standalone => ServerConfig::Centralized { + server: hosts[0].clone(), + }, + RedisDeploymentMode::Cluster => ServerConfig::Clustered { + hosts, + policy: ClusterDiscoveryPolicy::ConfigEndpoint, + }, + RedisDeploymentMode::Sentinel => ServerConfig::Sentinel { + hosts, + service_name: config.master_name.clone().expect("validated master_name"), + username: non_empty(&config.sentinel_username), + password: resolve_secret(&config.sentinel_password_env, &config.sentinel_password)?, + }, + }; + let mut result = Config { + server, + username: non_empty(&config.username), + password: resolve_secret(&config.password_env, &config.password)?, + database: Some(config.db as u8), + ..Config::default() + }; + if config.tls_enabled { + let mut connector = native_tls::TlsConnector::builder(); + if config.tls_insecure_skip_verify { + connector.danger_accept_invalid_certs(true); + connector.danger_accept_invalid_hostnames(true); + } + let connector = TlsConnector::try_from(connector) + .map_err(|error| map_fred_error("TLS configuration", error))?; + result.tls = Some(TlsConfig { + connector, + hostnames: TlsHostMapping::None, + }); + } + Ok(result) +} + +fn non_empty(value: &str) -> Option { + (!value.trim().is_empty()).then(|| value.to_string()) +} + +fn resolve_secret(variable: &str, plaintext: &str) -> CacheResult> { + if variable.trim().is_empty() { + return Ok(non_empty(plaintext)); + } + env::var(variable).map(Some).map_err(|_| { + CacheError::InvalidArgument(format!( + "Redis secret environment variable {variable} is not set" + )) + }) +} + +fn fred_expiration( + options: SetOptions, + default_ttl: Option, +) -> CacheResult> { + if options.keep_ttl { + return Ok(Some(FredExpiration::KEEPTTL)); + } + options + .expiration + .map(|expiration| match expiration { + Expiration::After(duration) => duration_to_expiration(duration), + }) + .transpose() + .and_then(|expiration| match expiration { + Some(expiration) => Ok(Some(expiration)), + None => default_ttl.map(duration_to_expiration).transpose(), + }) +} + +fn duration_to_expiration(duration: Duration) -> CacheResult { + let millis = i64::try_from(duration.as_millis()) + .map_err(|_| CacheError::InvalidArgument("Redis TTL is too large".into()))?; + if millis <= 0 { + return Err(CacheError::InvalidArgument( + "Redis TTL must be greater than zero".into(), + )); + } + Ok(FredExpiration::PX(millis)) +} + +fn list_direction(direction: ListDirection) -> LMoveDirection { + match direction { + ListDirection::Left => LMoveDirection::Left, + ListDirection::Right => LMoveDirection::Right, + } +} + +fn keys_share_slot(keys: &[String]) -> bool { + let Some(first) = keys.first() else { + return true; + }; + let slot = fred::util::redis_keyslot(first.as_bytes()); + keys.iter() + .skip(1) + .all(|key| fred::util::redis_keyslot(key.as_bytes()) == slot) +} + +fn fred_value_to_script_value(value: Value) -> CacheResult { match value { - redis::Value::Nil => Ok(ScriptValue::Null), - redis::Value::Int(value) => Ok(ScriptValue::Integer(value)), - redis::Value::BulkString(value) => Ok(ScriptValue::Bytes(value)), - redis::Value::Array(values) | redis::Value::Set(values) => values + Value::Null => Ok(ScriptValue::Null), + Value::Integer(value) => Ok(ScriptValue::Integer(value)), + Value::Bytes(value) => Ok(ScriptValue::Bytes(value.to_vec())), + Value::String(value) => Ok(ScriptValue::Bytes(value.as_bytes().to_vec())), + Value::Array(values) => values .into_iter() - .map(redis_value_to_script_value) + .map(fred_value_to_script_value) .collect::>>() .map(ScriptValue::Array), - redis::Value::SimpleString(value) => Ok(ScriptValue::Bytes(value.into_bytes())), - redis::Value::Okay => Ok(ScriptValue::Bytes(b"OK".to_vec())), - redis::Value::Boolean(value) => Ok(ScriptValue::Boolean(value)), + Value::Boolean(value) => Ok(ScriptValue::Boolean(value)), other => Err(CacheError::InvalidData(format!( "unsupported Redis script result: {other:?}" ))), } } -fn endpoint_url(config: &RedisProviderConfig) -> CacheResult { - let endpoint = config.endpoints[0].clone(); - if config.username.is_empty() && config.password_env.is_empty() { - return Ok(endpoint); - } - let mut url = Url::parse(&endpoint).map_err(|error| { - CacheError::InvalidArgument(format!("Redis endpoint URL is invalid: {error}")) - })?; - if !config.username.is_empty() { - url.set_username(&config.username) - .map_err(|_| CacheError::InvalidArgument("Redis username is invalid".into()))?; - } - if !config.password_env.is_empty() { - let password = env::var(&config.password_env).map_err(|_| { - CacheError::InvalidArgument(format!( - "Redis password_env {} is not set", - config.password_env - )) - })?; - url.set_password(Some(&password)) - .map_err(|_| CacheError::InvalidArgument("Redis password is invalid".into()))?; +fn map_fred_error(operation: &str, error: FredError) -> CacheError { + let details = error.details().to_ascii_uppercase(); + if details.contains("NOSCRIPT") { + return CacheError::NoScript(format!("Redis {operation}: {error}")); } - Ok(url.to_string()) -} - -fn map_redis_error(operation: &str, error: RedisError) -> CacheError { - if error.is_timeout() { - return CacheError::Timeout(format!("Redis {operation} timed out: {error}")); + if details.contains("CROSSSLOT") { + return CacheError::CrossSlot(format!("Redis {operation}: {error}")); } - if error.is_connection_refusal() - || error.is_connection_dropped() - || error.is_cluster_error() - || error.is_io_error() - { - return CacheError::Unavailable(format!("Redis {operation} unavailable: {error}")); + if details.contains("READONLY") { + return CacheError::ReadOnly(format!("Redis {operation}: {error}")); + } + if details.contains("NOPERM") { + return CacheError::PermissionDenied(format!("Redis {operation}: {error}")); + } + match error.kind() { + ErrorKind::Timeout => CacheError::Timeout(format!("Redis {operation}: {error}")), + ErrorKind::Auth => CacheError::Authentication(format!("Redis {operation}: {error}")), + ErrorKind::IO | ErrorKind::Routing | ErrorKind::Cluster | ErrorKind::Sentinel => { + CacheError::Unavailable(format!("Redis {operation}: {error}")) + } + ErrorKind::InvalidArgument | ErrorKind::Config | ErrorKind::Url => { + CacheError::InvalidArgument(format!("Redis {operation}: {error}")) + } + ErrorKind::Protocol | ErrorKind::Parse | ErrorKind::NotFound => { + CacheError::InvalidData(format!("Redis {operation}: {error}")) + } + _ => CacheError::Internal(format!("Redis {operation}: {error}")), } - CacheError::Internal(format!("Redis {operation} failed: {error}")) } diff --git a/crates/ragfs/src/cache_runtime/redis/config.rs b/crates/ragfs/src/cache_runtime/redis/config.rs index f2145b2aa4..9e106a06c0 100644 --- a/crates/ragfs/src/cache_runtime/redis/config.rs +++ b/crates/ragfs/src/cache_runtime/redis/config.rs @@ -1,18 +1,43 @@ //! Redis provider configuration. use crate::cache_runtime::{CacheError, CacheResult}; +use std::fmt; +use url::Url; + +/// Redis deployment topology managed by Fred. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RedisDeploymentMode { + /// One Redis server. + Standalone, + /// Redis Cluster with slot discovery. + Cluster, + /// Redis primary discovered through Sentinel. + Sentinel, +} /// Connection and execution settings for the built-in Redis provider. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct RedisProviderConfig { /// Redis deployment mode. pub mode: String, /// Redis endpoints. pub endpoints: Vec, + /// Sentinel service name when `mode=sentinel`. + pub master_name: Option, /// Optional ACL username. pub username: String, /// Environment variable containing the Redis password. pub password_env: String, + /// Legacy plaintext Redis password. Prefer `password_env` for new configurations. + pub password: String, + /// Optional Sentinel ACL username. + pub sentinel_username: String, + /// Environment variable containing the Sentinel password. + pub sentinel_password_env: String, + /// Legacy plaintext Sentinel password. Prefer `sentinel_password_env` for new configurations. + pub sentinel_password: String, + /// Redis database number. Cluster requires database zero. + pub db: i64, /// Maximum concurrent commands. pub pool_size: usize, /// Connection timeout in milliseconds. @@ -25,6 +50,10 @@ pub struct RedisProviderConfig { pub default_ttl_seconds: u64, /// Whether reads may use replicas. pub read_from_replica: bool, + /// Enable TLS for Redis and Sentinel connections. + pub tls_enabled: bool, + /// Disable certificate verification. Requires `tls_enabled=true`. + pub tls_insecure_skip_verify: bool, } impl Default for RedisProviderConfig { @@ -32,25 +61,41 @@ impl Default for RedisProviderConfig { Self { mode: "standalone".into(), endpoints: vec!["redis://127.0.0.1:6379".into()], + master_name: None, username: String::new(), password_env: String::new(), + password: String::new(), + sentinel_username: String::new(), + sentinel_password_env: String::new(), + sentinel_password: String::new(), + db: 0, pool_size: 32, connect_timeout_ms: 1_000, command_timeout_ms: 20, key_prefix: String::new(), default_ttl_seconds: 3_600, read_from_replica: false, + tls_enabled: false, + tls_insecure_skip_verify: false, } } } impl RedisProviderConfig { - pub(super) fn validate(&self) -> CacheResult<()> { - if self.mode != "standalone" { - return Err(CacheError::InvalidArgument( - "Redis mode must be standalone in this adapter stage".into(), - )); + /// Normalize the configured deployment mode. + pub fn deployment_mode(&self) -> CacheResult { + match self.mode.trim().to_ascii_lowercase().as_str() { + "standalone" | "singleton" => Ok(RedisDeploymentMode::Standalone), + "cluster" => Ok(RedisDeploymentMode::Cluster), + "sentinel" => Ok(RedisDeploymentMode::Sentinel), + other => Err(CacheError::InvalidArgument(format!( + "unsupported Redis mode {other}; expected standalone, cluster, or sentinel" + ))), } + } + + pub(super) fn validate(&self) -> CacheResult<()> { + let mode = self.deployment_mode()?; if self.endpoints.is_empty() || self .endpoints @@ -61,6 +106,34 @@ impl RedisProviderConfig { "Redis endpoints must not be empty".into(), )); } + for endpoint in &self.endpoints { + validate_endpoint(endpoint)?; + } + if mode == RedisDeploymentMode::Standalone && self.endpoints.len() != 1 { + return Err(CacheError::InvalidArgument( + "Redis standalone mode requires exactly one endpoint".into(), + )); + } + if mode == RedisDeploymentMode::Cluster && self.db != 0 { + return Err(CacheError::InvalidArgument( + "Redis cluster mode requires db=0".into(), + )); + } + if mode == RedisDeploymentMode::Sentinel + && self + .master_name + .as_deref() + .is_none_or(|value| value.trim().is_empty()) + { + return Err(CacheError::InvalidArgument( + "Redis sentinel mode requires master_name".into(), + )); + } + if self.db < 0 || self.db > u8::MAX as i64 { + return Err(CacheError::InvalidArgument( + "Redis db must be between 0 and 255".into(), + )); + } if self.pool_size == 0 { return Err(CacheError::InvalidArgument( "Redis pool_size must be greater than zero".into(), @@ -82,19 +155,232 @@ impl RedisProviderConfig { "Redis default TTL is too large".into(), )); } - if self.read_from_replica { + if !self.password_env.trim().is_empty() && !self.password.is_empty() { return Err(CacheError::InvalidArgument( - "Redis read_from_replica is not supported in standalone mode".into(), + "Redis password and password_env cannot both be configured".into(), + )); + } + if !self.sentinel_password_env.trim().is_empty() && !self.sentinel_password.is_empty() { + return Err(CacheError::InvalidArgument( + "Redis sentinel_password and sentinel_password_env cannot both be configured" + .into(), + )); + } + if self.read_from_replica && mode != RedisDeploymentMode::Cluster { + return Err(CacheError::InvalidArgument( + "Redis read_from_replica is only supported in cluster mode".into(), + )); + } + if self.tls_insecure_skip_verify && !self.tls_enabled { + return Err(CacheError::InvalidArgument( + "Redis tls_insecure_skip_verify requires tls_enabled=true".into(), )); } Ok(()) } } +impl fmt::Debug for RedisProviderConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RedisProviderConfig") + .field("mode", &self.mode) + .field("endpoints", &self.endpoints) + .field("master_name", &self.master_name) + .field("username", &self.username) + .field("password_env", &self.password_env) + .field("password_configured", &!self.password.is_empty()) + .field("sentinel_username", &self.sentinel_username) + .field("sentinel_password_env", &self.sentinel_password_env) + .field( + "sentinel_password_configured", + &!self.sentinel_password.is_empty(), + ) + .field("db", &self.db) + .field("pool_size", &self.pool_size) + .field("connect_timeout_ms", &self.connect_timeout_ms) + .field("command_timeout_ms", &self.command_timeout_ms) + .field("key_prefix", &self.key_prefix) + .field("default_ttl_seconds", &self.default_ttl_seconds) + .field("read_from_replica", &self.read_from_replica) + .field("tls_enabled", &self.tls_enabled) + .field("tls_insecure_skip_verify", &self.tls_insecure_skip_verify) + .finish() + } +} + +pub(super) fn parse_endpoint(endpoint: &str) -> CacheResult<(String, u16)> { + let url = Url::parse(endpoint).map_err(|_| { + CacheError::InvalidArgument( + "Redis endpoints must use valid redis:// or rediss:// URLs".into(), + ) + })?; + if !matches!(url.scheme(), "redis" | "rediss") { + return Err(CacheError::InvalidArgument( + "Redis endpoints must use valid redis:// or rediss:// URLs".into(), + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(CacheError::InvalidArgument( + "Redis endpoints must not include credentials; use dedicated Redis fields".into(), + )); + } + if !matches!(url.path(), "" | "/") || url.query().is_some() || url.fragment().is_some() { + return Err(CacheError::InvalidArgument( + "Redis endpoints must not include database paths, query parameters, or fragments" + .into(), + )); + } + let host = url + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(|| CacheError::InvalidArgument("Redis endpoint host is missing".into()))?; + let port = url + .port() + .unwrap_or(if url.scheme() == "rediss" { 6380 } else { 6379 }); + if port == 0 { + return Err(CacheError::InvalidArgument( + "Redis endpoint port is invalid".into(), + )); + } + Ok((host.to_string(), port)) +} + +fn validate_endpoint(endpoint: &str) -> CacheResult<()> { + parse_endpoint(endpoint).map(|_| ()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn accepts_standalone_cluster_and_sentinel_modes() { + let standalone = RedisProviderConfig::default(); + assert_eq!( + standalone.deployment_mode().unwrap(), + RedisDeploymentMode::Standalone + ); + + let cluster = RedisProviderConfig { + mode: "cluster".into(), + endpoints: vec![ + "redis://127.0.0.1:7000".into(), + "redis://127.0.0.1:7001".into(), + ], + db: 0, + ..RedisProviderConfig::default() + }; + assert_eq!( + cluster.deployment_mode().unwrap(), + RedisDeploymentMode::Cluster + ); + cluster.validate().unwrap(); + + let sentinel = RedisProviderConfig { + mode: "sentinel".into(), + endpoints: vec!["redis://127.0.0.1:26379".into()], + master_name: Some("mymaster".into()), + ..RedisProviderConfig::default() + }; + assert_eq!( + sentinel.deployment_mode().unwrap(), + RedisDeploymentMode::Sentinel + ); + sentinel.validate().unwrap(); + } + + #[test] + fn singleton_is_a_legacy_alias_for_standalone() { + let config = RedisProviderConfig { + mode: "singleton".into(), + ..RedisProviderConfig::default() + }; + + assert_eq!( + config.deployment_mode().unwrap(), + RedisDeploymentMode::Standalone + ); + config.validate().unwrap(); + } + + #[test] + fn validates_topology_specific_settings() { + let standalone = RedisProviderConfig { + endpoints: vec![ + "redis://127.0.0.1:6379".into(), + "redis://127.0.0.1:6380".into(), + ], + ..RedisProviderConfig::default() + }; + assert!(matches!( + standalone.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("exactly one") + )); + + let cluster = RedisProviderConfig { + mode: "cluster".into(), + db: 1, + ..RedisProviderConfig::default() + }; + assert!(matches!( + cluster.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("db=0") + )); + + let sentinel = RedisProviderConfig { + mode: "sentinel".into(), + master_name: None, + ..RedisProviderConfig::default() + }; + assert!(matches!( + sentinel.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("master_name") + )); + + let sentinel_replica_reads = RedisProviderConfig { + mode: "sentinel".into(), + endpoints: vec!["redis://127.0.0.1:26379".into()], + master_name: Some("mymaster".into()), + read_from_replica: true, + ..RedisProviderConfig::default() + }; + assert!(matches!( + sentinel_replica_reads.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("only supported in cluster") + )); + + let cluster_replica_reads = RedisProviderConfig { + mode: "cluster".into(), + endpoints: vec!["redis://127.0.0.1:7000".into()], + read_from_replica: true, + ..RedisProviderConfig::default() + }; + cluster_replica_reads.validate().unwrap(); + } + + #[test] + fn rejects_endpoint_credentials_and_invalid_tls_settings() { + let credentials = RedisProviderConfig { + endpoints: vec!["redis://user:secret@127.0.0.1:6379".into()], + ..RedisProviderConfig::default() + }; + assert!(matches!( + credentials.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("credentials") + )); + + let insecure_without_tls = RedisProviderConfig { + tls_insecure_skip_verify: true, + tls_enabled: false, + ..RedisProviderConfig::default() + }; + assert!(matches!( + insecure_without_tls.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("tls_enabled") + )); + } + #[test] fn rejects_default_ttl_that_cannot_be_sent_as_milliseconds() { let config = RedisProviderConfig { @@ -120,4 +406,28 @@ mod tests { Err(CacheError::InvalidArgument(message)) if message.contains("key_prefix") )); } + + #[test] + fn rejects_ambiguous_secrets_and_redacts_plaintext_values() { + let config = RedisProviderConfig { + password_env: "OV_REDIS_PASSWORD".into(), + password: "plain-secret".into(), + ..RedisProviderConfig::default() + }; + assert!(matches!( + config.validate(), + Err(CacheError::InvalidArgument(message)) if message.contains("cannot both") + )); + + let debug = format!( + "{:?}", + RedisProviderConfig { + password: "plain-secret".into(), + sentinel_password: "sentinel-secret".into(), + ..RedisProviderConfig::default() + } + ); + assert!(!debug.contains("plain-secret")); + assert!(!debug.contains("sentinel-secret")); + } } diff --git a/crates/ragfs/src/cache_runtime/redis/mod.rs b/crates/ragfs/src/cache_runtime/redis/mod.rs index 3c424a0935..0b4a713fb1 100644 --- a/crates/ragfs/src/cache_runtime/redis/mod.rs +++ b/crates/ragfs/src/cache_runtime/redis/mod.rs @@ -5,5 +5,5 @@ mod config; mod provider; use client::RedisClient; -pub use config::RedisProviderConfig; +pub use config::{RedisDeploymentMode, RedisProviderConfig}; pub(crate) use provider::RedisProvider; diff --git a/crates/ragfs/src/cache_runtime/redis/provider.rs b/crates/ragfs/src/cache_runtime/redis/provider.rs index a2d273bf89..ad559808fa 100644 --- a/crates/ragfs/src/cache_runtime/redis/provider.rs +++ b/crates/ragfs/src/cache_runtime/redis/provider.rs @@ -1,7 +1,8 @@ use super::{RedisClient, RedisProviderConfig}; use crate::cache_runtime::provider::CacheProvider; use crate::cache_runtime::{ - CacheError, CacheResult, PutOptions, ScriptRegistry, ScriptRequest, ScriptResult, + CacheResult, ListInsertRequest, ListMoveRequest, ScriptRegistry, ScriptRequest, ScriptResult, + SetOptions, SetResult, }; use async_trait::async_trait; use bytes::Bytes; @@ -32,75 +33,99 @@ impl RedisProvider { default_ttl, }) } - - fn ttl_ms(&self, options: PutOptions) -> CacheResult> { - options - .ttl - .or(self.default_ttl) - .map(|ttl| { - u64::try_from(ttl.as_millis()) - .map_err(|_| CacheError::InvalidArgument("Redis TTL is too large".to_string())) - }) - .transpose() - } } #[async_trait] impl CacheProvider for RedisProvider { async fn get(&self, key: &str) -> CacheResult> { - Ok(self.client.get(key.to_string()).await?.map(Bytes::from)) + self.client.get(key).await } - async fn put(&self, key: &str, value: Bytes, options: PutOptions) -> CacheResult<()> { - self.client - .set(key.to_string(), value.to_vec(), self.ttl_ms(options)?) - .await + async fn set(&self, key: &str, value: Bytes, options: SetOptions) -> CacheResult { + self.client.set(key, value, options, self.default_ttl).await } - async fn delete(&self, key: &str) -> CacheResult<()> { - self.client.delete(key.to_string()).await + async fn del(&self, keys: &[String]) -> CacheResult { + self.client.del(keys).await } - async fn exists(&self, key: &str) -> CacheResult { - self.client.exists(key.to_string()).await + async fn mget(&self, keys: &[String]) -> CacheResult>> { + self.client.mget(keys).await } - async fn batch_get(&self, keys: &[String]) -> CacheResult>> { - Ok(self - .client - .batch_get(keys.to_vec()) - .await? - .into_iter() - .map(|value| value.map(Bytes::from)) - .collect()) - } - - async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { - let ttl_ms = self.ttl_ms(PutOptions::default())?; - self.client - .batch_set( - entries - .into_iter() - .map(|(key, value)| (key, value.to_vec())) - .collect(), - ttl_ms, - ) - .await + async fn mset(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> { + self.client.mset(entries, self.default_ttl).await + } + + async fn incr_by(&self, key: &str, delta: i64) -> CacheResult { + self.client.incr_by(key, delta).await + } + + async fn sismember(&self, key: &str, member: &[u8]) -> CacheResult { + self.client.sismember(key, member).await + } + + async fn smembers(&self, key: &str) -> CacheResult> { + self.client.smembers(key).await + } + + async fn scard(&self, key: &str) -> CacheResult { + self.client.scard(key).await + } + + async fn lpush(&self, key: &str, values: Vec) -> CacheResult { + self.client.lpush(key, values).await + } + + async fn rpush(&self, key: &str, values: Vec) -> CacheResult { + self.client.rpush(key, values).await + } + + async fn lpop(&self, key: &str, count: Option) -> CacheResult> { + self.client.lpop(key, count).await + } + + async fn rpop(&self, key: &str, count: Option) -> CacheResult> { + self.client.rpop(key, count).await + } + + async fn llen(&self, key: &str) -> CacheResult { + self.client.llen(key).await + } + + async fn lrange(&self, key: &str, start: i64, stop: i64) -> CacheResult> { + self.client.lrange(key, start, stop).await + } + + async fn lindex(&self, key: &str, index: i64) -> CacheResult> { + self.client.lindex(key, index).await + } + + async fn lset(&self, key: &str, index: i64, value: Bytes) -> CacheResult<()> { + self.client.lset(key, index, value).await + } + + async fn ltrim(&self, key: &str, start: i64, stop: i64) -> CacheResult<()> { + self.client.ltrim(key, start, stop).await } - async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> { - self.client.batch_delete(keys.to_vec()).await + async fn lrem(&self, key: &str, count: i64, value: Bytes) -> CacheResult { + self.client.lrem(key, count, value).await + } + + async fn linsert(&self, request: ListInsertRequest) -> CacheResult { + self.client.linsert(request).await + } + + async fn lmove(&self, request: ListMoveRequest) -> CacheResult> { + self.client.lmove(request).await } async fn execute_script(&self, request: ScriptRequest) -> CacheResult { let lua = self.scripts.resolve(&request.script_id)?; let value = self .client - .execute_script( - lua, - request.keys, - request.args.into_iter().map(|arg| arg.to_vec()).collect(), - ) + .execute_script(&request.script_id, lua, request.keys, request.args) .await?; ScriptResult::encode(&value) } @@ -113,7 +138,7 @@ impl CacheProvider for RedisProvider { #[cfg(test)] mod tests { use super::*; - use crate::cache_runtime::{AsyncCacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue}; + use crate::cache_runtime::{ScriptDefinition, ScriptRequest, ScriptValue}; #[tokio::test] async fn executes_registered_script_and_recovers_after_script_flush() { @@ -159,4 +184,57 @@ mod tests { let second = runtime.execute_script(request).await.unwrap(); assert_eq!(second.decode().unwrap(), first.decode().unwrap()); } + + #[tokio::test] + async fn set_queries_preserve_binary_members_on_real_redis() { + let Ok(endpoint) = std::env::var("REDIS_URL") else { + return; + }; + let scripts = Arc::new(crate::cache_runtime::ScriptRegistry::default()); + scripts + .register(ScriptDefinition { + id: "runtime.test.sadd.v1", + redis_lua: "return redis.call('SADD', KEYS[1], ARGV[1], ARGV[2])", + }) + .unwrap(); + let config = RedisProviderConfig { + endpoints: vec![endpoint], + key_prefix: String::new(), + command_timeout_ms: 1_000, + ..RedisProviderConfig::default() + }; + let provider = Arc::new( + RedisProvider::connect(config, Arc::clone(&scripts)) + .await + .unwrap(), + ); + let runtime = crate::cache_runtime::CacheRuntime::from_provider(provider); + let set_key = format!("ragfs-set-test:{}:members", std::process::id()); + let binary_member = Bytes::from_static(b"binary\0member"); + let text_member = Bytes::from_static(b"text-member"); + + runtime + .execute_script(ScriptRequest { + script_id: "runtime.test.sadd.v1".into(), + keys: vec![set_key.clone()], + args: vec![binary_member.clone(), text_member.clone()], + }) + .await + .unwrap(); + + assert!(runtime + .sismember(&set_key, binary_member.as_ref()) + .await + .unwrap()); + assert!(!runtime.sismember(&set_key, b"missing").await.unwrap()); + assert_eq!(runtime.scard(&set_key).await.unwrap(), 2); + let mut members = runtime.smembers(&set_key).await.unwrap(); + members.sort(); + let mut expected = vec![binary_member, text_member]; + expected.sort(); + assert_eq!(members, expected); + + runtime.del(&[set_key]).await.unwrap(); + runtime.close().await.unwrap(); + } } diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs index a7b52e371b..d07574a364 100644 --- a/crates/ragfs/src/core/mountable.rs +++ b/crates/ragfs/src/core/mountable.rs @@ -1708,7 +1708,7 @@ mod tests { #[tokio::test] async fn encrypted_mount_caches_ciphertext_below_account_validation() { use crate::cache::{CacheNamespace, CachePolicy}; - use crate::cache_runtime::{AsyncCacheRuntime, CacheRuntime, MemoryMockProvider}; + use crate::cache_runtime::{CacheRuntime, MemoryMockProvider}; use crate::core::{FsContextInner, FS_CTX}; use crate::lock::{ MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider, diff --git a/crates/ragfs/src/plugins/queuefs/cache_backend.rs b/crates/ragfs/src/plugins/queuefs/cache_backend.rs index f8bddf87b0..3b317d1dfa 100644 --- a/crates/ragfs/src/plugins/queuefs/cache_backend.rs +++ b/crates/ragfs/src/plugins/queuefs/cache_backend.rs @@ -1,13 +1,13 @@ use super::backend::{Message, StoredMessage}; -use super::redis_backend::{ - heartbeat_key, instance_key_prefix, last_enqueue_time_from_pending_payloads, queue_names_key, - unix_secs, QueueKeys, ACK_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT, DEQUEUE_SCRIPT, - ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS, LIST_UNACKED_SCRIPT, PEEK_SCRIPT, - RECOVER_STALE_SCRIPT, REMOVE_QUEUE_SCRIPT, STARTUP_RECOVERY_SWEEPS, +use super::cache_protocol::{ + heartbeat_key, instance_key_prefix, last_enqueue_time_from_pending_payloads, queue_key_prefix, + queue_names_key, unix_secs, QueueKeys, ACK_SCRIPT, CLEAR_SCRIPT, CREATE_QUEUE_SCRIPT, + DEQUEUE_SCRIPT, ENQUEUE_SCRIPT, HEARTBEAT_INTERVAL_SECS, HEARTBEAT_TTL_SECS, + LIST_UNACKED_SCRIPT, PEEK_SCRIPT, RECOVER_STALE_SCRIPT, REMOVE_QUEUE_SCRIPT, + STARTUP_RECOVERY_SWEEPS, }; use crate::cache_runtime::{ - AsyncCacheRuntime, CacheError, CacheRuntime, PutOptions, ScriptDefinition, ScriptRequest, - ScriptValue, SyncCacheRuntime, + CacheError, CacheRuntime, Expiration, ScriptDefinition, ScriptRequest, ScriptValue, SetOptions, }; use crate::core::errors::{Error, Result}; use bytes::Bytes; @@ -18,11 +18,6 @@ use tokio::task::JoinHandle; use tokio::time::Instant; use uuid::Uuid; -const QUEUE_EXISTS_SCRIPT: &str = "return redis.call('SISMEMBER', KEYS[1], ARGV[1])"; -const LIST_QUEUES_SCRIPT: &str = "return redis.call('SMEMBERS', KEYS[1])"; -const SIZE_SCRIPT: &str = "return redis.call('LLEN', KEYS[1])"; -const LIST_PENDING_SCRIPT: &str = "return redis.call('LRANGE', KEYS[1], 0, -1)"; - const CREATE_QUEUE_ID: &str = "queuefs.create_queue.v1"; const REMOVE_QUEUE_ID: &str = "queuefs.remove_queue.v1"; const ENQUEUE_ID: &str = "queuefs.enqueue.v1"; @@ -32,10 +27,6 @@ const LIST_UNACKED_ID: &str = "queuefs.list_unacked.v1"; const ACK_ID: &str = "queuefs.ack.v1"; const CLEAR_ID: &str = "queuefs.clear.v1"; const RECOVER_STALE_ID: &str = "queuefs.recover_stale.v1"; -const QUEUE_EXISTS_ID: &str = "queuefs.queue_exists.v1"; -const LIST_QUEUES_ID: &str = "queuefs.list_queues.v1"; -const SIZE_ID: &str = "queuefs.size.v1"; -const LIST_PENDING_ID: &str = "queuefs.list_pending.v1"; const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[ ScriptDefinition { @@ -74,22 +65,6 @@ const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[ id: RECOVER_STALE_ID, redis_lua: RECOVER_STALE_SCRIPT, }, - ScriptDefinition { - id: QUEUE_EXISTS_ID, - redis_lua: QUEUE_EXISTS_SCRIPT, - }, - ScriptDefinition { - id: LIST_QUEUES_ID, - redis_lua: LIST_QUEUES_SCRIPT, - }, - ScriptDefinition { - id: SIZE_ID, - redis_lua: SIZE_SCRIPT, - }, - ScriptDefinition { - id: LIST_PENDING_ID, - redis_lua: LIST_PENDING_SCRIPT, - }, ]; pub(super) struct CacheQueueStorage { @@ -181,10 +156,7 @@ impl CacheQueueStorage { "remove_queue", REMOVE_QUEUE_ID, vec![queue_names_key(&self.key_prefix)], - vec![ - bytes(name), - bytes(super::redis_backend::queue_key_prefix(&self.key_prefix)), - ], + vec![bytes(name), bytes(queue_key_prefix(&self.key_prefix))], ) .await?, )?; @@ -195,15 +167,10 @@ impl CacheQueueStorage { } async fn queue_exists_result(&self, name: &str) -> Result { - boolean( - self.execute( - "queue_exists", - QUEUE_EXISTS_ID, - vec![queue_names_key(&self.key_prefix)], - vec![bytes(name)], - ) - .await?, - ) + self.runtime + .sismember(&queue_names_key(&self.key_prefix), name.as_bytes()) + .await + .map_err(|error| cache_error("queue_exists", error)) } pub(super) async fn queue_exists(&self, name: &str) -> bool { @@ -218,16 +185,13 @@ impl CacheQueueStorage { pub(super) async fn list_queues(&self, prefix: &str) -> Vec { let result = self - .execute( - "list_queues", - LIST_QUEUES_ID, - vec![queue_names_key(&self.key_prefix)], - Vec::new(), - ) + .runtime + .smembers(&queue_names_key(&self.key_prefix)) .await - .and_then(string_array); + .map_err(|error| cache_error("list_queues", error)) + .and_then(bytes_array_to_strings); let mut queues = match result { - Ok(queues) => queues, + Ok(values) => values, Err(error) => { tracing::error!(prefix, error = %error, "queuefs cache list_queues failed; returning an empty list"); return Vec::new(); @@ -323,10 +287,12 @@ impl CacheQueueStorage { pub(super) async fn size(&self, queue_name: &str) -> Result { self.require_queue(queue_name).await?; let keys = QueueKeys::new(&self.key_prefix, queue_name); - usize::try_from(integer( - self.execute("size", SIZE_ID, vec![keys.pending], Vec::new()) - .await?, - )?) + usize::try_from( + self.runtime + .llen(&keys.pending) + .await + .map_err(|error| cache_error("size", error))?, + ) .map_err(|_| Error::internal("redis size returned an invalid value")) } @@ -380,15 +346,12 @@ impl CacheQueueStorage { self.require_queue(queue_name).await?; let keys = QueueKeys::new(&self.key_prefix, queue_name); let pending_key = keys.pending.clone(); - let pending_ids = string_array( - self.execute( - "get_last_enqueue_time list pending", - LIST_PENDING_ID, - vec![pending_key], - Vec::new(), - ) - .await?, - )?; + let pending_ids = self + .runtime + .lrange(&pending_key, 0, -1) + .await + .map_err(|error| cache_error("get_last_enqueue_time list pending", error)) + .and_then(bytes_array_to_strings)?; if pending_ids.is_empty() { return Ok(UNIX_EPOCH); } @@ -398,7 +361,7 @@ impl CacheQueueStorage { .collect::>(); let payloads = self .runtime - .batch_get(&message_keys) + .mget(&message_keys) .await .map_err(|error| cache_error("get_last_enqueue_time load payloads", error))? .into_iter() @@ -438,9 +401,9 @@ impl Drop for CacheQueueStorage { if let Ok(handle) = tokio::runtime::Handle::try_current() { let runtime = Arc::clone(&self.runtime); handle.spawn(async move { - let _ = runtime.delete(&key).await; + let _ = runtime.del(&[key]).await; }); - } else if let Err(error) = self.runtime.sync_facade().delete(&key) { + } else if let Err(error) = self.runtime.sync_facade().del(&[key]) { tracing::warn!("queuefs cache heartbeat cleanup failed: {error}"); } } @@ -467,14 +430,16 @@ async fn run_heartbeat(runtime: Arc, key: String, mut stop: watch: async fn refresh_heartbeat(runtime: &CacheRuntime, key: &str) -> Result<()> { runtime - .put( + .set( key, Bytes::from_static(b"1"), - PutOptions { - ttl: Some(Duration::from_secs(HEARTBEAT_TTL_SECS)), + SetOptions { + expiration: Some(Expiration::After(Duration::from_secs(HEARTBEAT_TTL_SECS))), + ..SetOptions::default() }, ) .await + .map(|_| ()) .map_err(|error| cache_error("heartbeat", error)) } @@ -507,15 +472,11 @@ fn startup_recovery_delay_before_sweep(sweep_index: usize) -> Duration { } async fn recover_stale(runtime: &CacheRuntime, key_prefix: &str) -> Result { - let queues = execute_runtime( - runtime, - "recover_stale list queues", - LIST_QUEUES_ID, - vec![queue_names_key(key_prefix)], - Vec::new(), - ) - .await - .and_then(string_array)?; + let queues = runtime + .smembers(&queue_names_key(key_prefix)) + .await + .map_err(|error| cache_error("recover_stale list queues", error)) + .and_then(bytes_array_to_strings)?; let mut recovered = 0; for queue in queues { let keys = QueueKeys::new(key_prefix, &queue); @@ -590,6 +551,16 @@ fn string_array(value: ScriptValue) -> Result> { } } +fn bytes_array_to_strings(values: Vec) -> Result> { + values + .into_iter() + .map(|value| { + String::from_utf8(value.to_vec()) + .map_err(|error| Error::Serialization(format!("invalid queue payload: {error}"))) + }) + .collect() +} + fn invalid_result(expected: &str, value: ScriptValue) -> Error { Error::internal(format!( "cache script returned {value:?}, expected {expected}" @@ -638,8 +609,8 @@ mod tests { let heartbeat = heartbeat_key("drop-test", &storage.instance_id); let sync = runtime.sync_facade(); - assert!(sync.exists(&heartbeat).unwrap()); + assert!(sync.get(&heartbeat).unwrap().is_some()); drop(storage); - assert!(!sync.exists(&heartbeat).unwrap()); + assert!(sync.get(&heartbeat).unwrap().is_none()); } } diff --git a/crates/ragfs/src/plugins/queuefs/cache_protocol.rs b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs new file mode 100644 index 0000000000..28963f943b --- /dev/null +++ b/crates/ragfs/src/plugins/queuefs/cache_protocol.rs @@ -0,0 +1,209 @@ +use super::backend::StoredMessage; +use crate::core::errors::{Error, Result}; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub(super) const HEARTBEAT_TTL_SECS: u64 = 30; +pub(super) const HEARTBEAT_INTERVAL_SECS: u64 = 10; +pub(super) const STARTUP_RECOVERY_SWEEPS: usize = 3; + +pub(super) const CREATE_QUEUE_SCRIPT: &str = r#" +if redis.call('SADD', KEYS[1], ARGV[1]) == 0 then + return 0 +end +redis.call('HSET', KEYS[2], 'created_at', ARGV[2], 'last_updated', ARGV[2]) +return 1 +"#; + +pub(super) const REMOVE_QUEUE_SCRIPT: &str = r#" +local removed = 0 +local queues = redis.call('SMEMBERS', KEYS[1]) +for _, queue in ipairs(queues) do + if queue == ARGV[1] or string.sub(queue, 1, string.len(ARGV[1]) + 1) == ARGV[1] .. '/' then + local prefix = ARGV[2] .. queue + local pending_key = prefix .. ':pending' + local processing_key = prefix .. ':processing' + local message_prefix = prefix .. ':msg:' + local pending = redis.call('LRANGE', pending_key, 0, -1) + for _, id in ipairs(pending) do + redis.call('DEL', message_prefix .. id) + end + local processing = redis.call('ZRANGE', processing_key, 0, -1) + for _, member in ipairs(processing) do + local separator = string.find(member, '|', 1, true) + if separator then + redis.call('DEL', message_prefix .. string.sub(member, 1, separator - 1)) + end + end + redis.call('DEL', prefix .. ':meta', pending_key, processing_key) + redis.call('SREM', KEYS[1], queue) + removed = removed + 1 + end +end +return removed +"#; + +pub(super) const ENQUEUE_SCRIPT: &str = r#" +if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then + return 0 +end +redis.call('SET', KEYS[2], ARGV[3]) +redis.call('RPUSH', KEYS[3], ARGV[2]) +redis.call('HSET', KEYS[4], 'last_updated', ARGV[4]) +return 1 +"#; + +pub(super) const DEQUEUE_SCRIPT: &str = r#" +local id = redis.call('LPOP', KEYS[1]) +if not id then + return nil +end +local payload = redis.call('GET', ARGV[1] .. id) +if not payload then + redis.call('LPUSH', KEYS[1], id) + return redis.error_reply('queuefs payload missing for message ' .. id) +end +redis.call('ZADD', KEYS[2], ARGV[3], id .. '|' .. ARGV[2]) +return {id, payload} +"#; + +pub(super) const PEEK_SCRIPT: &str = r#" +local id = redis.call('LINDEX', KEYS[1], 0) +if not id then + return nil +end +local payload = redis.call('GET', ARGV[1] .. id) +if not payload then + return redis.error_reply('queuefs payload missing for message ' .. id) +end +return payload +"#; + +pub(super) const LIST_UNACKED_SCRIPT: &str = r#" +local result = {} +local pending = redis.call('LRANGE', KEYS[1], 0, -1) +for _, id in ipairs(pending) do + local payload = redis.call('GET', ARGV[1] .. id) + if not payload then + return redis.error_reply('queuefs payload missing for message ' .. id) + end + table.insert(result, payload) +end +local processing = redis.call('ZRANGE', KEYS[2], 0, -1) +for _, member in ipairs(processing) do + local separator = string.find(member, '|', 1, true) + if separator then + local id = string.sub(member, 1, separator - 1) + local payload = redis.call('GET', ARGV[1] .. id) + if not payload then + return redis.error_reply('queuefs payload missing for message ' .. id) + end + table.insert(result, payload) + end +end +return result +"#; + +pub(super) const ACK_SCRIPT: &str = r#" +local members = redis.call('ZRANGE', KEYS[1], 0, -1) +for _, member in ipairs(members) do + if string.sub(member, 1, string.len(ARGV[1]) + 1) == ARGV[1] .. '|' then + redis.call('ZREM', KEYS[1], member) + redis.call('DEL', KEYS[2]) + return 1 + end +end +return 0 +"#; + +pub(super) const CLEAR_SCRIPT: &str = r#" +local pending = redis.call('LRANGE', KEYS[1], 0, -1) +for _, id in ipairs(pending) do + redis.call('DEL', ARGV[1] .. id) +end +local processing = redis.call('ZRANGE', KEYS[2], 0, -1) +for _, member in ipairs(processing) do + local separator = string.find(member, '|', 1, true) + if separator then + redis.call('DEL', ARGV[1] .. string.sub(member, 1, separator - 1)) + end +end +redis.call('DEL', KEYS[1], KEYS[2]) +return #pending + #processing +"#; + +pub(super) const RECOVER_STALE_SCRIPT: &str = r#" +local recovered = 0 +local members = redis.call('ZRANGE', KEYS[1], 0, -1) +for _, member in ipairs(members) do + local separator = string.find(member, '|', 1, true) + if separator then + local id = string.sub(member, 1, separator - 1) + local instance = string.sub(member, separator + 1) + if redis.call('EXISTS', ARGV[1] .. instance .. ':alive') == 0 then + redis.call('ZREM', KEYS[1], member) + redis.call('RPUSH', KEYS[2], id) + recovered = recovered + 1 + end + end +end +return recovered +"#; + +pub(super) struct QueueKeys { + pub(super) meta: String, + pub(super) pending: String, + pub(super) processing: String, + pub(super) message_prefix: String, +} + +impl QueueKeys { + pub(super) fn new(key_prefix: &str, queue: &str) -> Self { + let prefix = format!("{}{queue}", queue_key_prefix(key_prefix)); + Self { + meta: format!("{prefix}:meta"), + pending: format!("{prefix}:pending"), + processing: format!("{prefix}:processing"), + message_prefix: format!("{prefix}:msg:"), + } + } + + pub(super) fn message(&self, message_id: &str) -> String { + format!("{}{message_id}", self.message_prefix) + } +} + +pub(super) fn queue_names_key(key_prefix: &str) -> String { + format!("{}names", queue_key_prefix(key_prefix)) +} + +pub(super) fn queue_key_prefix(key_prefix: &str) -> String { + format!("{{{key_prefix}}}:ov:queue:") +} + +pub(super) fn instance_key_prefix(key_prefix: &str) -> String { + format!("{}instance:", queue_key_prefix(key_prefix)) +} + +pub(super) fn heartbeat_key(key_prefix: &str, instance_id: &str) -> String { + format!("{}{instance_id}:alive", instance_key_prefix(key_prefix)) +} + +pub(super) fn unix_secs(time: SystemTime) -> u64 { + time.duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +pub(super) fn last_enqueue_time_from_pending_payloads(payloads: &[String]) -> Result { + payloads.iter().try_fold(UNIX_EPOCH, |latest, payload| { + let timestamp = serde_json::from_str::(payload) + .map(StoredMessage::into_message) + .map(|message| message.timestamp) + .map_err(|error| Error::Serialization(format!("invalid queue payload: {error}")))?; + Ok(if timestamp > latest { + timestamp + } else { + latest + }) + }) +} diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs index a8f8f1617a..c089f836b9 100644 --- a/crates/ragfs/src/plugins/queuefs/mod.rs +++ b/crates/ragfs/src/plugins/queuefs/mod.rs @@ -13,7 +13,8 @@ mod backend; #[cfg(feature = "cache")] mod cache_backend; -mod redis_backend; +#[cfg(feature = "cache")] +mod cache_protocol; use crate::core::{ errors::{Error, Result}, @@ -25,8 +26,7 @@ use async_trait::async_trait; use backend::{MemoryBackend, Message, QueueBackend, SQLiteQueueBackend, SQLiteQueueOptions}; #[cfg(feature = "cache")] use cache_backend::CacheQueueStorage; -use redis_backend::RedisQueueBackend; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use std::sync::Arc; use std::time::SystemTime; use tokio::sync::Mutex; @@ -80,7 +80,6 @@ struct QueueMessage { enum BackendKind { Memory, Sqlite, - Redis, #[cfg(feature = "cache")] Cache, } @@ -90,180 +89,10 @@ struct ParsedBackendConfig { kind: BackendKind, sqlite_db_path: Option, sqlite_options: SQLiteQueueOptions, - redis_options: Option, #[cfg(feature = "cache")] cache_key_prefix: Option, } -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "lowercase")] -enum RedisMode { - #[default] - Singleton, - Cluster, - Sentinel, -} - -#[derive(Clone, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct RedisQueueOptions { - mode: RedisMode, - endpoints: Vec, - master_name: Option, - username: Option, - password: Option, - sentinel_username: Option, - sentinel_password: Option, - db: i64, - connect_timeout_ms: u64, - command_timeout_ms: u64, - key_prefix: String, - tls_enabled: bool, - tls_insecure_skip_verify: bool, -} - -impl std::fmt::Debug for RedisQueueOptions { - /// Format Redis settings without exposing the configured password. - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RedisQueueOptions") - .field("mode", &self.mode) - .field("endpoints", &self.endpoints) - .field("master_name", &self.master_name) - .field("username", &self.username) - .field("password_configured", &self.password.is_some()) - .field( - "sentinel_username_configured", - &self.sentinel_username.is_some(), - ) - .field( - "sentinel_password_configured", - &self.sentinel_password.is_some(), - ) - .field("db", &self.db) - .field("connect_timeout_ms", &self.connect_timeout_ms) - .field("command_timeout_ms", &self.command_timeout_ms) - .field("key_prefix", &self.key_prefix) - .field("tls_enabled", &self.tls_enabled) - .field( - "tls_insecure_skip_verify", - &self.tls_insecure_skip_verify, - ) - .finish() - } -} - -impl Default for RedisQueueOptions { - /// Return the default standalone Redis connection settings. - fn default() -> Self { - Self { - mode: RedisMode::Singleton, - endpoints: vec!["redis://127.0.0.1:6379".to_string()], - master_name: None, - username: None, - password: None, - sentinel_username: None, - sentinel_password: None, - db: 0, - connect_timeout_ms: 3_000, - command_timeout_ms: 3_000, - key_prefix: "default".to_string(), - tls_enabled: false, - tls_insecure_skip_verify: false, - } - } -} - -impl RedisQueueOptions { - /// Validate Redis endpoint and timeout settings. - fn validate(&self) -> Result<()> { - if self.endpoints.is_empty() || self.endpoints.iter().any(|value| value.trim().is_empty()) { - return Err(Error::config( - "queuefs redis endpoints must not be empty".to_string(), - )); - } - for value in &self.endpoints { - if !value.starts_with("redis://") && !value.starts_with("rediss://") { - return Err(Error::config( - "queuefs redis endpoints must use valid redis:// or rediss:// URLs".to_string(), - )); - } - let info = redis::IntoConnectionInfo::into_connection_info(value.as_str()).map_err(|_| { - Error::config("queuefs redis endpoints must use valid redis:// or rediss:// URLs".to_string()) - })?; - let endpoint = value - .split_once("://") - .map(|(_, endpoint)| endpoint) - .unwrap_or_default(); - let options_start = endpoint - .find(|character| matches!(character, '/' | '?' | '#')) - .unwrap_or(endpoint.len()); - if endpoint[..options_start].contains('@') - || !matches!(&endpoint[options_start..], "" | "/") - { - return Err(Error::config( - "queuefs redis endpoints must not include credentials, database paths, query parameters, or fragments; use dedicated redis fields".to_string(), - )); - } - if matches!( - info.addr(), - redis::ConnectionAddr::Tcp(_, 0) | redis::ConnectionAddr::TcpTls { port: 0, .. } - ) { - return Err(Error::config("queuefs redis endpoint port is invalid".to_string())); - } - } - if self.mode == RedisMode::Singleton && self.endpoints.len() != 1 { - return Err(Error::config( - "queuefs redis singleton mode requires exactly one endpoint".to_string(), - )); - } - if self.mode == RedisMode::Cluster && self.db != 0 { - return Err(Error::config( - "queuefs redis cluster mode requires db=0".to_string(), - )); - } - if self.mode == RedisMode::Sentinel - && self - .master_name - .as_deref() - .is_none_or(|value| value.trim().is_empty()) - { - return Err(Error::config( - "queuefs redis sentinel mode requires master_name".to_string(), - )); - } - if self.db < 0 { - return Err(Error::config("queuefs redis db must be >= 0".to_string())); - } - if self.connect_timeout_ms == 0 { - return Err(Error::config( - "queuefs redis connect_timeout_ms must be > 0".to_string(), - )); - } - if self.command_timeout_ms == 0 { - return Err(Error::config( - "queuefs redis command_timeout_ms must be > 0".to_string(), - )); - } - if self.key_prefix.trim().is_empty() { - return Err(Error::config( - "queuefs redis key_prefix must not be empty".to_string(), - )); - } - if self.key_prefix.contains(['{', '}']) { - return Err(Error::config( - "queuefs redis key_prefix must not contain '{' or '}'".to_string(), - )); - } - if self.tls_insecure_skip_verify && !self.tls_enabled { - return Err(Error::config( - "queuefs redis tls_insecure_skip_verify requires tls_enabled=true".to_string(), - )); - } - Ok(()) - } -} - /// Parsed path information struct ParsedPath { queue_name: Option, @@ -813,9 +642,7 @@ impl QueueFSPlugin { } #[cfg(feature = "cache")] - pub(crate) fn with_cache_runtime( - runtime: Arc, - ) -> Self { + pub(crate) fn with_cache_runtime(runtime: Arc) -> Self { let mut plugin = Self::new(); plugin.cache_runtime = Some(runtime); plugin @@ -834,7 +661,7 @@ impl QueueFSPlugin { #[cfg(feature = "cache")] let valid_backends = ["memory", "sqlite", "sqlite3", "redis", "cache"]; #[cfg(not(feature = "cache"))] - let valid_backends = ["memory", "sqlite", "sqlite3", "redis"]; + let valid_backends = ["memory", "sqlite", "sqlite3"]; if !valid_backends.contains(&backend_name) { return Err(Error::config(format!( "unsupported queue backend: {} (valid: {})", @@ -846,9 +673,8 @@ impl QueueFSPlugin { let kind = match backend_name { "memory" => BackendKind::Memory, "sqlite" | "sqlite3" => BackendKind::Sqlite, - "redis" => BackendKind::Redis, #[cfg(feature = "cache")] - "cache" => BackendKind::Cache, + "cache" | "redis" => BackendKind::Cache, _ => { return Err(Error::config(format!( "unsupported queue backend: {}", @@ -875,7 +701,7 @@ impl QueueFSPlugin { } let sqlite_db_path = match kind { - BackendKind::Memory | BackendKind::Redis => None, + BackendKind::Memory => None, #[cfg(feature = "cache")] BackendKind::Cache => None, BackendKind::Sqlite => { @@ -889,35 +715,16 @@ impl QueueFSPlugin { Some(db_path.to_string()) } }; - let redis_options = match kind { - BackendKind::Redis => { - let options = match config.params.get("redis") { - Some(crate::core::types::ConfigValue::Json(value)) => { - serde_json::from_value(value.clone()).map_err(|error| { - Error::config(format!("invalid queuefs redis config: {error}")) - })? - } - Some(_) => { - return Err(Error::config( - "queuefs redis config must be a JSON object".to_string(), - )) - } - None => RedisQueueOptions::default(), - }; - options.validate()?; - Some(options) - } - BackendKind::Memory | BackendKind::Sqlite => None, - #[cfg(feature = "cache")] - BackendKind::Cache => None, - }; - #[cfg(feature = "cache")] let cache_key_prefix = match kind { BackendKind::Cache => { - let prefix = Self::get_string_param(config, "cache_key_prefix") - .unwrap_or("default") - .to_string(); + let prefix = if backend_name == "redis" { + legacy_redis_key_prefix(config)? + } else { + Self::get_string_param(config, "cache_key_prefix") + .unwrap_or("default") + .to_string() + }; if prefix.trim().is_empty() || prefix.contains(['{', '}']) { return Err(Error::config( "queuefs cache_key_prefix must be non-empty and must not contain '{' or '}'" @@ -936,13 +743,36 @@ impl QueueFSPlugin { recover_stale_sec, busy_timeout_ms, }, - redis_options, #[cfg(feature = "cache")] cache_key_prefix, }) } } +#[cfg(feature = "cache")] +fn legacy_redis_key_prefix(config: &PluginConfig) -> Result { + let redis = match config.params.get("redis") { + Some(crate::core::types::ConfigValue::Json(value)) => { + value.as_object().ok_or_else(|| { + Error::config("queuefs redis config must be a JSON object".to_string()) + })? + } + Some(_) => { + return Err(Error::config( + "queuefs redis config must be a JSON object".to_string(), + )) + } + None => return Ok("default".to_string()), + }; + match redis.get("key_prefix") { + Some(serde_json::Value::String(value)) => Ok(value.clone()), + Some(_) => Err(Error::config( + "queuefs redis key_prefix must be a string".to_string(), + )), + None => Ok("default".to_string()), + } +} + impl Default for QueueFSPlugin { fn default() -> Self { Self::new() @@ -1018,13 +848,6 @@ impl ServicePlugin for QueueFSPlugin { .expect("sqlite db_path is validated"), parsed.sqlite_options, )?), - BackendKind::Redis => { - Box::new(RedisQueueBackend::open( - parsed - .redis_options - .expect("redis options are validated for redis backend"), - )?) - } #[cfg(feature = "cache")] BackendKind::Cache => { return Ok(Box::new( @@ -1063,6 +886,7 @@ mod tests { } /// Build a QueueFS plugin config containing a nested Redis object. + #[cfg(feature = "cache")] fn redis_plugin_config(redis: serde_json::Value) -> PluginConfig { let mut params = std::collections::HashMap::new(); params.insert( @@ -1299,13 +1123,14 @@ mod tests { assert_eq!(plugin.name(), "queuefs"); assert!(!plugin.readme().is_empty()); let config_params = plugin.config_params(); - assert_eq!(config_params.len(), 5 + usize::from(cfg!(feature = "cache"))); - #[cfg(feature = "cache")] - assert!( - config_params - .iter() - .any(|parameter| parameter.name == "cache_key_prefix") + assert_eq!( + config_params.len(), + 5 + usize::from(cfg!(feature = "cache")) ); + #[cfg(feature = "cache")] + assert!(config_params + .iter() + .any(|parameter| parameter.name == "cache_key_prefix")); let config = PluginConfig::single_backend("queuefs", "/queue", std::collections::HashMap::new()); @@ -1322,190 +1147,30 @@ mod tests { assert_eq!(msg.data, "test"); } + #[cfg(feature = "cache")] #[test] - fn test_parse_redis_backend_config() { - let parsed = QueueFSPlugin::parse_backend_config(&redis_plugin_config( - serde_json::json!({ - "mode": "singleton", - "endpoints": ["redis://127.0.0.1:6379"], - "master_name": null, - "username": "queue-user", - "password": "secret", - "sentinel_username": null, - "sentinel_password": null, - "db": 2, - "connect_timeout_ms": 1500, - "command_timeout_ms": 2500, - "key_prefix": "tenant-a", - "tls_enabled": false, - "tls_insecure_skip_verify": false - }), - )) + fn legacy_redis_backend_is_normalized_to_cache_runtime() { + let parsed = QueueFSPlugin::parse_backend_config(&redis_plugin_config(serde_json::json!({ + "mode": "singleton", + "endpoints": ["redis://127.0.0.1:6379"], + "key_prefix": "tenant-a" + }))) .unwrap(); - assert!(matches!(parsed.kind, BackendKind::Redis)); - let options = parsed.redis_options.unwrap(); - assert_eq!(options.endpoints, vec!["redis://127.0.0.1:6379"]); - assert_eq!(options.username.as_deref(), Some("queue-user")); - assert_eq!(options.password.as_deref(), Some("secret")); - assert_eq!(options.db, 2); - assert_eq!(options.connect_timeout_ms, 1500); - assert_eq!(options.command_timeout_ms, 2500); - assert_eq!(options.key_prefix, "tenant-a"); - } - - #[test] - /// Accept valid Cluster and Sentinel topology configurations. - fn test_parse_redis_backend_config_accepts_high_availability_modes() { - let cluster = QueueFSPlugin::parse_backend_config(&redis_plugin_config( - serde_json::json!({ - "mode": "cluster", - "endpoints": ["redis://cluster-1:6379", "redis://cluster-2:6379"], - "db": 0 - }), - )); - assert!(cluster.is_ok()); - - let sentinel = QueueFSPlugin::parse_backend_config(&redis_plugin_config( - serde_json::json!({ - "mode": "sentinel", - "endpoints": ["redis://sentinel-1:26379", "redis://sentinel-2:26379"], - "master_name": "mymaster" - }), - )); - assert!(sentinel.is_ok()); - } - - #[test] - /// Reject topology settings that violate the selected Redis mode. - fn test_parse_redis_backend_config_rejects_invalid_mode_settings() { - let cases = [ - (serde_json::json!({"mode": "invalid"}), "invalid"), - ( - serde_json::json!({ - "mode": "singleton", - "endpoints": ["redis://redis-1:6379", "redis://redis-2:6379"] - }), - "singleton", - ), - ( - serde_json::json!({"mode": "cluster", "endpoints": []}), - "endpoints", - ), - (serde_json::json!({"mode": "cluster", "db": 1}), "db"), - ( - serde_json::json!({ - "mode": "sentinel", - "endpoints": [], - "master_name": "mymaster" - }), - "endpoints", - ), - ( - serde_json::json!({"mode": "sentinel", "master_name": ""}), - "master_name", - ), - ( - serde_json::json!({"key_prefix": "invalid{tag}"}), - "key_prefix", - ), - ]; - - for (redis, expected) in cases { - let error = QueueFSPlugin::parse_backend_config(&redis_plugin_config(redis)) - .unwrap_err() - .to_string(); - assert!( - error.contains(expected), - "expected {expected:?} in {error:?}" - ); - } - } - - #[test] - fn test_parse_redis_backend_config_rejects_empty_endpoints() { - let mut params = std::collections::HashMap::new(); - params.insert( - "backend".to_string(), - crate::core::types::ConfigValue::String("redis".to_string()), - ); - params.insert( - "redis".to_string(), - crate::core::types::ConfigValue::Json(serde_json::json!({ - "endpoints": [] - })), - ); - let config = PluginConfig::single_backend("queuefs", "/queue", params); - - assert!(QueueFSPlugin::parse_backend_config(&config) - .unwrap_err() - .to_string() - .contains("endpoints")); - } - - #[test] - fn test_parse_redis_backend_config_rejects_invalid_endpoint() { - let mut params = std::collections::HashMap::new(); - params.insert( - "backend".to_string(), - crate::core::types::ConfigValue::String("redis".to_string()), - ); - params.insert( - "redis".to_string(), - crate::core::types::ConfigValue::Json(serde_json::json!({ - "endpoints": ["http://127.0.0.1:6379"] - })), - ); - let config = PluginConfig::single_backend("queuefs", "/queue", params); - - assert!(QueueFSPlugin::parse_backend_config(&config) - .unwrap_err() - .to_string() - .contains("redis:// or rediss://")); - } - - #[test] - /// Reject an empty Redis namespace prefix. - fn test_parse_redis_backend_config_rejects_empty_key_prefix() { - let mut params = std::collections::HashMap::new(); - params.insert( - "backend".to_string(), - crate::core::types::ConfigValue::String("redis".to_string()), - ); - params.insert( - "redis".to_string(), - crate::core::types::ConfigValue::Json(serde_json::json!({ - "key_prefix": "" - })), - ); - let config = PluginConfig::single_backend("queuefs", "/queue", params); - - assert!(QueueFSPlugin::parse_backend_config(&config) - .unwrap_err() - .to_string() - .contains("key_prefix")); + assert!(matches!(parsed.kind, BackendKind::Cache)); + assert_eq!(parsed.cache_key_prefix.as_deref(), Some("tenant-a")); } + #[cfg(feature = "cache")] #[test] - /// Reject the removed Redis connection pool setting. - fn test_parse_redis_backend_config_rejects_removed_pool_max_size() { - let mut params = std::collections::HashMap::new(); - params.insert( - "backend".to_string(), - crate::core::types::ConfigValue::String("redis".to_string()), - ); - params.insert( - "redis".to_string(), - crate::core::types::ConfigValue::Json(serde_json::json!({ - "pool_max_size": 16 - })), - ); - let config = PluginConfig::single_backend("queuefs", "/queue", params); + fn legacy_redis_backend_rejects_invalid_namespace() { + let error = QueueFSPlugin::parse_backend_config(&redis_plugin_config( + serde_json::json!({"key_prefix": "invalid{tag}"}), + )) + .unwrap_err() + .to_string(); - assert!(QueueFSPlugin::parse_backend_config(&config) - .unwrap_err() - .to_string() - .contains("pool_max_size")); + assert!(error.contains("key_prefix")); } #[tokio::test] @@ -1579,16 +1244,15 @@ mod tests { let Ok(endpoint) = std::env::var("QUEUEFS_REDIS_TEST_URL") else { return; }; - let runtime = crate::cache_runtime::CacheRuntime::redis( - crate::cache_runtime::RedisProviderConfig { + let runtime = + crate::cache_runtime::CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig { endpoints: vec![endpoint], key_prefix: String::new(), command_timeout_ms: 1_000, ..crate::cache_runtime::RedisProviderConfig::default() - }, - ) - .await - .unwrap(); + }) + .await + .unwrap(); let queue_prefix = format!("queuefs-runtime-test-{}", uuid::Uuid::new_v4()); let fs = QueueFileSystem::with_cache_runtime(runtime.clone(), queue_prefix) .await @@ -1602,14 +1266,9 @@ mod tests { let dequeued: TestQueueMessage = serde_json::from_slice(&fs.read("/Semantic/dequeue", 0, 0).await.unwrap()).unwrap(); assert_eq!(dequeued.data, "payload"); - fs.write( - "/Semantic/ack", - dequeued.id.as_bytes(), - 0, - WriteFlag::None, - ) - .await - .unwrap(); + fs.write("/Semantic/ack", dequeued.id.as_bytes(), 0, WriteFlag::None) + .await + .unwrap(); assert_eq!(fs.read("/Semantic/size", 0, 0).await.unwrap(), b"0"); fs.remove_all("/Semantic").await.unwrap(); drop(fs); @@ -1622,16 +1281,15 @@ mod tests { let Ok(endpoint) = std::env::var("QUEUEFS_REDIS_TEST_URL") else { return; }; - let runtime = crate::cache_runtime::CacheRuntime::redis( - crate::cache_runtime::RedisProviderConfig { + let runtime = + crate::cache_runtime::CacheRuntime::redis(crate::cache_runtime::RedisProviderConfig { endpoints: vec![endpoint], key_prefix: String::new(), command_timeout_ms: 1_000, ..crate::cache_runtime::RedisProviderConfig::default() - }, - ) - .await - .unwrap(); + }) + .await + .unwrap(); let fs = QueueFileSystem::with_cache_runtime( runtime.clone(), format!("queuefs-runtime-failure-{}", uuid::Uuid::new_v4()), diff --git a/crates/ragfs/src/plugins/queuefs/redis_backend.rs b/crates/ragfs/src/plugins/queuefs/redis_backend.rs deleted file mode 100644 index 607082f7bd..0000000000 --- a/crates/ragfs/src/plugins/queuefs/redis_backend.rs +++ /dev/null @@ -1,1519 +0,0 @@ -use super::backend::{Message, QueueBackend, StoredMessage}; -use super::{RedisMode, RedisQueueOptions}; -use crate::core::errors::{Error, Result}; -use redis::cluster::ClusterClient; -use redis::sentinel::{SentinelClient, SentinelClientBuilder, SentinelServerType}; -use redis::{ - Connection, ConnectionAddr, ConnectionInfo, ConnectionLike, IntoConnectionInfo, RedisError, - RedisConnectionInfo, RedisResult, ServerErrorKind, TlsMode, -}; -use r2d2::{ManageConnection, Pool}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::mpsc::{self, RecvTimeoutError, Sender}; -use std::sync::{Arc, Mutex}; -use std::thread::JoinHandle; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use uuid::Uuid; - -pub(super) const HEARTBEAT_TTL_SECS: u64 = 30; -pub(super) const HEARTBEAT_INTERVAL_SECS: u64 = 10; -pub(super) const STARTUP_RECOVERY_SWEEPS: usize = 3; -const REDIS_POOL_MAX_SIZE: u32 = 2; -const REDIS_POOL_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(5); -const SENTINEL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); - -pub(super) const CREATE_QUEUE_SCRIPT: &str = r#" -if redis.call('SADD', KEYS[1], ARGV[1]) == 0 then - return 0 -end -redis.call('HSET', KEYS[2], 'created_at', ARGV[2], 'last_updated', ARGV[2]) -return 1 -"#; - -pub(super) const REMOVE_QUEUE_SCRIPT: &str = r#" -local removed = 0 -local queues = redis.call('SMEMBERS', KEYS[1]) -for _, queue in ipairs(queues) do - if queue == ARGV[1] or string.sub(queue, 1, string.len(ARGV[1]) + 1) == ARGV[1] .. '/' then - local prefix = ARGV[2] .. queue - local pending_key = prefix .. ':pending' - local processing_key = prefix .. ':processing' - local message_prefix = prefix .. ':msg:' - local pending = redis.call('LRANGE', pending_key, 0, -1) - for _, id in ipairs(pending) do - redis.call('DEL', message_prefix .. id) - end - local processing = redis.call('ZRANGE', processing_key, 0, -1) - for _, member in ipairs(processing) do - local separator = string.find(member, '|', 1, true) - if separator then - redis.call('DEL', message_prefix .. string.sub(member, 1, separator - 1)) - end - end - redis.call('DEL', prefix .. ':meta', pending_key, processing_key) - redis.call('SREM', KEYS[1], queue) - removed = removed + 1 - end -end -return removed -"#; - -pub(super) const ENQUEUE_SCRIPT: &str = r#" -if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then - return 0 -end -redis.call('SET', KEYS[2], ARGV[3]) -redis.call('RPUSH', KEYS[3], ARGV[2]) -redis.call('HSET', KEYS[4], 'last_updated', ARGV[4]) -return 1 -"#; - -pub(super) const DEQUEUE_SCRIPT: &str = r#" -local id = redis.call('LPOP', KEYS[1]) -if not id then - return nil -end -local payload = redis.call('GET', ARGV[1] .. id) -if not payload then - redis.call('LPUSH', KEYS[1], id) - return redis.error_reply('queuefs payload missing for message ' .. id) -end -redis.call('ZADD', KEYS[2], ARGV[3], id .. '|' .. ARGV[2]) -return {id, payload} -"#; - -pub(super) const PEEK_SCRIPT: &str = r#" -local id = redis.call('LINDEX', KEYS[1], 0) -if not id then - return nil -end -local payload = redis.call('GET', ARGV[1] .. id) -if not payload then - return redis.error_reply('queuefs payload missing for message ' .. id) -end -return payload -"#; - -pub(super) const LIST_UNACKED_SCRIPT: &str = r#" -local result = {} -local pending = redis.call('LRANGE', KEYS[1], 0, -1) -for _, id in ipairs(pending) do - local payload = redis.call('GET', ARGV[1] .. id) - if not payload then - return redis.error_reply('queuefs payload missing for message ' .. id) - end - table.insert(result, payload) -end -local processing = redis.call('ZRANGE', KEYS[2], 0, -1) -for _, member in ipairs(processing) do - local separator = string.find(member, '|', 1, true) - if separator then - local id = string.sub(member, 1, separator - 1) - local payload = redis.call('GET', ARGV[1] .. id) - if not payload then - return redis.error_reply('queuefs payload missing for message ' .. id) - end - table.insert(result, payload) - end -end -return result -"#; - -pub(super) const ACK_SCRIPT: &str = r#" -local members = redis.call('ZRANGE', KEYS[1], 0, -1) -for _, member in ipairs(members) do - if string.sub(member, 1, string.len(ARGV[1]) + 1) == ARGV[1] .. '|' then - redis.call('ZREM', KEYS[1], member) - redis.call('DEL', KEYS[2]) - return 1 - end -end -return 0 -"#; - -pub(super) const CLEAR_SCRIPT: &str = r#" -local pending = redis.call('LRANGE', KEYS[1], 0, -1) -for _, id in ipairs(pending) do - redis.call('DEL', ARGV[1] .. id) -end -local processing = redis.call('ZRANGE', KEYS[2], 0, -1) -for _, member in ipairs(processing) do - local separator = string.find(member, '|', 1, true) - if separator then - redis.call('DEL', ARGV[1] .. string.sub(member, 1, separator - 1)) - end -end -redis.call('DEL', KEYS[1], KEYS[2]) -return #pending + #processing -"#; - -pub(super) const RECOVER_STALE_SCRIPT: &str = r#" -local recovered = 0 -local members = redis.call('ZRANGE', KEYS[1], 0, -1) -for _, member in ipairs(members) do - local separator = string.find(member, '|', 1, true) - if separator then - local id = string.sub(member, 1, separator - 1) - local instance = string.sub(member, separator + 1) - if redis.call('EXISTS', ARGV[1] .. instance .. ':alive') == 0 then - redis.call('ZREM', KEYS[1], member) - redis.call('RPUSH', KEYS[2], id) - recovered = recovered + 1 - end - end -end -return recovered -"#; - -pub(super) struct QueueKeys { - pub(super) meta: String, - pub(super) pending: String, - pub(super) processing: String, - pub(super) message_prefix: String, -} - -impl QueueKeys { - /// Build all Redis keys owned by one queue. - pub(super) fn new(key_prefix: &str, queue: &str) -> Self { - let prefix = format!("{}{queue}", queue_key_prefix(key_prefix)); - Self { - meta: format!("{prefix}:meta"), - pending: format!("{prefix}:pending"), - processing: format!("{prefix}:processing"), - message_prefix: format!("{prefix}:msg:"), - } - } - - /// Build the payload key for one message. - pub(super) fn message(&self, message_id: &str) -> String { - format!("{}{message_id}", self.message_prefix) - } -} - -struct SentinelConnection { - connection: Connection, - generation: ConnectionGeneration, - snapshot: u64, -} - -impl SentinelConnection { - /// Wrap one Sentinel connection with the current topology generation. - fn new(connection: Connection, generation: ConnectionGeneration) -> Self { - let snapshot = generation.snapshot(); - Self { - connection, - generation, - snapshot, - } - } - - /// Invalidate this and every other connection from the same Sentinel generation. - fn invalidate_topology(&self) { - self.generation.invalidate(); - } - - /// Return whether this connection belongs to the current Sentinel topology. - fn is_current(&self) -> bool { - self.generation.is_current(self.snapshot) - } -} - -#[derive(Clone, Default)] -struct ConnectionGeneration(Arc); - -impl ConnectionGeneration { - /// Capture the generation assigned to a newly created connection. - fn snapshot(&self) -> u64 { - self.0.load(Ordering::SeqCst) - } - - /// Advance the generation after a Sentinel topology error. - fn invalidate(&self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - - /// Return whether a connection belongs to the current generation. - fn is_current(&self, snapshot: u64) -> bool { - self.snapshot() == snapshot - } -} - -struct SingletonPoolManager { - client: redis::Client, - connect_timeout: Duration, - command_timeout: Duration, -} - -impl SingletonPoolManager { - /// Build a Singleton manager with physical connect and command timeouts. - fn new(client: redis::Client, options: &RedisQueueOptions) -> Self { - Self { - client, - connect_timeout: Duration::from_millis(options.connect_timeout_ms), - command_timeout: Duration::from_millis(options.command_timeout_ms), - } - } -} - -impl ManageConnection for SingletonPoolManager { - type Connection = Connection; - type Error = RedisError; - - /// Open one Singleton connection with the configured physical connect timeout. - fn connect(&self) -> RedisResult { - let connection = self - .client - .get_connection_with_timeout(self.connect_timeout)?; - configure_connection(&connection, self.command_timeout)?; - Ok(connection) - } - - /// Validate one checked-out Singleton connection. - fn is_valid(&self, connection: &mut Self::Connection) -> RedisResult<()> { - validate_connection(connection) - } - - /// Return whether r2d2 must discard this Singleton connection. - fn has_broken(&self, connection: &mut Self::Connection) -> bool { - !connection.is_open() - } -} - -struct SentinelPoolManager { - client: Mutex, - discovery_runtime: tokio::runtime::Runtime, - connect_timeout: Duration, - command_timeout: Duration, - generation: ConnectionGeneration, -} - -impl SentinelPoolManager { - /// Build a Sentinel manager from `client` and `options`, returning an initialization error. - fn new(client: SentinelClient, options: &RedisQueueOptions) -> RedisResult { - let discovery_runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(RedisError::from)?; - Ok(Self { - client: Mutex::new(client), - discovery_runtime, - connect_timeout: Duration::from_millis(options.connect_timeout_ms), - command_timeout: Duration::from_millis(options.command_timeout_ms), - generation: ConnectionGeneration::default(), - }) - } -} - -impl ManageConnection for SentinelPoolManager { - type Connection = SentinelConnection; - type Error = RedisError; - - /// Discover the current Sentinel master and open one data connection. - fn connect(&self) -> RedisResult { - let client = { - let mut sentinel = self.client.lock().map_err(|_| { - RedisError::from(( - redis::ErrorKind::Client, - "redis sentinel client lock poisoned", - )) - })?; - self.discovery_runtime.block_on(async { - match tokio::time::timeout( - SENTINEL_DISCOVERY_TIMEOUT, - sentinel.async_get_client(), - ) - .await - { - Ok(result) => result, - Err(_) => Err(RedisError::from(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "redis sentinel discovery timed out", - ))), - } - }) - }?; - let connection = client.get_connection_with_timeout(self.connect_timeout)?; - configure_connection(&connection, self.command_timeout)?; - Ok(SentinelConnection::new( - connection, - self.generation.clone(), - )) - } - - /// Validate one checked-out Sentinel data connection. - fn is_valid(&self, connection: &mut Self::Connection) -> RedisResult<()> { - if !connection.is_current() { - return Err(RedisError::from(( - redis::ErrorKind::Io, - "redis sentinel topology changed", - ))); - } - validate_connection(&mut connection.connection) - } - - /// Return whether r2d2 must discard this Sentinel data connection. - fn has_broken(&self, connection: &mut Self::Connection) -> bool { - !connection.is_current() || !connection.connection.is_open() - } -} - -#[derive(Clone)] -enum RedisPool { - Singleton(Pool), - Cluster(Pool), - Sentinel(Pool), -} - -impl RedisPool { - /// Build and initialize the configured topology pool. - fn open(options: &RedisQueueOptions) -> Result { - let config_error = |error| { - Error::config(format!("invalid queuefs redis client configuration: {error}")) - }; - match options.mode { - RedisMode::Singleton => { - let info = endpoint_connection_info(&options.endpoints[0], options) - .map_err(config_error)?; - let client = redis::Client::open(info).map_err(config_error)?; - build_pool(SingletonPoolManager::new(client, options)).map(Self::Singleton) - } - RedisMode::Cluster => { - let nodes = options - .endpoints - .iter() - .map(|endpoint| endpoint_connection_info(endpoint, options)) - .collect::>>() - .map_err(config_error)?; - let mut builder = ClusterClient::builder(nodes) - .retries(0) - .connection_timeout(Duration::from_millis(options.connect_timeout_ms)) - .response_timeout(Duration::from_millis(options.command_timeout_ms)); - if let Some(username) = &options.username { - builder = builder.username(username.clone()); - } - if let Some(password) = &options.password { - builder = builder.password(password.clone()); - } - if options.tls_enabled { - builder = builder - .tls(tls_mode(options)) - .danger_accept_invalid_hostnames(options.tls_insecure_skip_verify); - } - build_pool(builder.build().map_err(config_error)?).map(Self::Cluster) - } - RedisMode::Sentinel => { - let sentinels = options - .endpoints - .iter() - .map(|endpoint| sentinel_addr(endpoint, options)) - .collect::>>() - .map_err(config_error)?; - let mut builder = SentinelClientBuilder::new( - sentinels, - options - .master_name - .clone() - .expect("sentinel master_name is validated"), - SentinelServerType::Master, - ) - .map_err(config_error)? - .set_client_to_redis_db(options.db); - if let Some(username) = &options.username { - builder = builder.set_client_to_redis_username(username.clone()); - } - if let Some(password) = &options.password { - builder = builder.set_client_to_redis_password(password.clone()); - } - if let Some(username) = &options.sentinel_username { - builder = builder.set_client_to_sentinel_username(username.clone()); - } - if let Some(password) = &options.sentinel_password { - builder = builder.set_client_to_sentinel_password(password.clone()); - } - if options.tls_enabled { - let mode = tls_mode(options); - builder = builder - .set_client_to_redis_tls_mode(mode) - .set_client_to_sentinel_tls_mode(mode); - } - let client = builder.build().map_err(config_error)?; - let manager = SentinelPoolManager::new(client, options).map_err(config_error)?; - build_pool(manager).map(Self::Sentinel) - } - } - } - - /// Checkout one pooled connection and execute one command without replaying failures. - fn execute( - &self, - operation: &str, - call: impl FnOnce(&mut dyn ConnectionLike) -> RedisResult, - ) -> Result { - match self { - Self::Singleton(pool) => execute_pool(pool, operation, call), - Self::Cluster(pool) => execute_pool(pool, operation, call), - Self::Sentinel(pool) => { - let mut connection = pool.get().map_err(|error| pool_error(operation, error))?; - let result = call(&mut connection.connection); - if let Err(error) = &result { - if is_sentinel_topology_error(error) { - connection.invalidate_topology(); - } - } - result.map_err(|error| redis_error(operation, error)) - } - } - } - - /// Execute one best-effort command only when an idle pooled connection is available. - fn try_execute( - &self, - call: impl FnOnce(&mut dyn ConnectionLike) -> RedisResult, - ) -> Option> { - match self { - Self::Singleton(pool) => try_execute_pool(pool, call), - Self::Cluster(pool) => try_execute_pool(pool, call), - Self::Sentinel(pool) => { - let mut connection = pool.try_get()?; - let result = call(&mut connection.connection); - if let Err(error) = &result { - if is_sentinel_topology_error(error) { - connection.invalidate_topology(); - } - } - Some(result) - } - } - } -} - -/// Execute one command through a pool backed by a redis-rs native manager. -fn execute_pool( - pool: &Pool, - operation: &str, - call: impl FnOnce(&mut dyn ConnectionLike) -> RedisResult, -) -> Result -where - M: ManageConnection, - M::Connection: ConnectionLike, -{ - let mut connection = pool.get().map_err(|error| pool_error(operation, error))?; - call(&mut *connection).map_err(|error| redis_error(operation, error)) -} - -/// Execute one best-effort command through an available native pooled connection. -fn try_execute_pool( - pool: &Pool, - call: impl FnOnce(&mut dyn ConnectionLike) -> RedisResult, -) -> Option> -where - M: ManageConnection, - M::Connection: ConnectionLike, -{ - let mut connection = pool.try_get()?; - Some(call(&mut *connection)) -} - -/// Redis-backed QueueFS implementation. -pub(super) struct RedisQueueBackend { - pool: RedisPool, - key_prefix: String, - instance_id: String, - heartbeat_stop: Option>, - heartbeat_thread: Option>, - startup_recovery_stop: Option>, - startup_recovery_thread: Option>, -} - -impl RedisQueueBackend { - /// Connect to Redis, register this instance, recover stale work, and start heartbeats. - pub(super) fn open(options: RedisQueueOptions) -> Result { - let pool = RedisPool::open(&options)?; - let key_prefix = options.key_prefix; - let instance_id = Uuid::new_v4().to_string(); - let heartbeat_key = heartbeat_key(&key_prefix, &instance_id); - refresh_heartbeat(&pool, &heartbeat_key)?; - let mut backend = Self { - pool, - key_prefix, - instance_id, - heartbeat_stop: None, - heartbeat_thread: None, - startup_recovery_stop: None, - startup_recovery_thread: None, - }; - backend.start_heartbeat(heartbeat_key); - backend.start_startup_recovery(); - Ok(backend) - } - - /// Execute a Redis operation using one checked-out pooled connection. - fn with_connection( - &self, - operation: &str, - call: impl FnOnce(&mut dyn ConnectionLike) -> redis::RedisResult, - ) -> Result { - self.pool.execute(operation, call) - } - - /// Start the heartbeat thread using a separate pool checkout for every renewal. - /// `key` is this instance's heartbeat key in Redis. - /// Returns no value; the spawned thread is stored on `self`. - fn start_heartbeat(&mut self, key: String) { - let (sender, receiver) = mpsc::channel(); - let pool = self.pool.clone(); - self.heartbeat_stop = Some(sender); - self.heartbeat_thread = Some(std::thread::spawn(move || { - loop { - if let Err(error) = refresh_heartbeat(&pool, &key) { - tracing::warn!("queuefs redis heartbeat failed: {error}"); - } - match receiver.recv_timeout(Duration::from_secs(HEARTBEAT_INTERVAL_SECS)) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => break, - Err(RecvTimeoutError::Timeout) => {} - } - } - })); - } - - /// Start a bounded startup recovery thread. - /// The thread runs at offsets 0, 30, and 60 seconds from startup unless stopped early. - /// Returns no value; the spawned thread is stored on `self`. - fn start_startup_recovery(&mut self) { - let (sender, receiver) = mpsc::channel(); - let pool = self.pool.clone(); - let key_prefix = self.key_prefix.clone(); - self.startup_recovery_stop = Some(sender); - self.startup_recovery_thread = Some(std::thread::spawn(move || { - // ponytail: keep startup recovery bounded to the restart TTL window; - // upgrade to steady-state scanning only if deployments need long-lived handoff. - let started_at = Instant::now(); - for sweep_index in 0..STARTUP_RECOVERY_SWEEPS { - let deadline = started_at + Self::startup_recovery_delay_before_sweep(sweep_index); - loop { - let now = Instant::now(); - if now >= deadline { - break; - } - match receiver.recv_timeout(deadline.saturating_duration_since(now)) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => return, - Err(RecvTimeoutError::Timeout) => {} - } - } - Self::run_startup_recovery_sweep(&pool, &key_prefix); - } - })); - } - - /// Return the delay before one bounded startup recovery sweep. - /// `sweep_index` is zero-based and maps to the documented 0/30/60-second schedule. - /// Returns the sweep offset from startup. - fn startup_recovery_delay_before_sweep(sweep_index: usize) -> Duration { - Duration::from_secs(HEARTBEAT_TTL_SECS * sweep_index as u64) - } - - /// Run one startup recovery sweep. - /// `pool` provides Redis connections and `key_prefix` selects the queue namespace. - /// Returns no value; recovery results are emitted through logs. - fn run_startup_recovery_sweep(pool: &RedisPool, key_prefix: &str) { - match Self::recover_stale(pool, key_prefix) { - Ok(recovered) => { - if recovered > 0 { - tracing::info!("queuefs redis recovered {recovered} stale message(s)"); - } - } - Err(error) => tracing::warn!("queuefs redis startup recover_stale failed: {error}"), - } - } - - /// Recover processing messages whose owning instance heartbeat has expired. - fn recover_stale(pool: &RedisPool, key_prefix: &str) -> Result { - let queue_names_key = queue_names_key(key_prefix); - let instance_key_prefix = instance_key_prefix(key_prefix); - let queues = pool.execute("recover_stale list queues", |connection| { - redis::cmd("SMEMBERS") - .arg(&queue_names_key) - .query::>(connection) - })?; - let mut recovered = 0; - for queue in queues { - let keys = QueueKeys::new(key_prefix, &queue); - recovered += pool.execute("recover_stale", |connection| { - redis::Script::new(RECOVER_STALE_SCRIPT) - .key(&keys.processing) - .key(&keys.pending) - .arg(&instance_key_prefix) - .invoke::(connection) - })?; - } - Ok(recovered) - } - - /// Deserialize one stored Redis payload. - fn decode_message(payload: &str) -> Result { - serde_json::from_str::(payload) - .map(StoredMessage::into_message) - .map_err(|error| Error::Serialization(format!("invalid queue payload: {error}"))) - } - - /// Require that a queue is registered. - fn require_queue(&self, queue_name: &str) -> Result<()> { - if self.queue_exists_result(queue_name)? { - Ok(()) - } else { - Err(Error::NotFound(format!( - "queue '{}' not found", - queue_name - ))) - } - } - - /// Return whether a queue is registered while preserving Redis failures. - fn queue_exists_result(&self, queue_name: &str) -> Result { - let queue_names_key = queue_names_key(&self.key_prefix); - self.with_connection("queue_exists", |connection| { - redis::cmd("SISMEMBER") - .arg(&queue_names_key) - .arg(queue_name) - .query::(connection) - }) - } -} - -impl Drop for RedisQueueBackend { - /// Stop and join the heartbeat thread without waiting for its next interval. - fn drop(&mut self) { - if let Some(sender) = self.heartbeat_stop.take() { - let _ = sender.send(()); - } - if let Some(sender) = self.startup_recovery_stop.take() { - let _ = sender.send(()); - } - if let Some(thread) = self.heartbeat_thread.take() { - let _ = thread.join(); - } - if let Some(thread) = self.startup_recovery_thread.take() { - let _ = thread.join(); - } - let _ = self.pool.try_execute(|connection| { - redis::cmd("DEL") - .arg(heartbeat_key(&self.key_prefix, &self.instance_id)) - .query::<()>(connection) - }); - } -} - -impl QueueBackend for RedisQueueBackend { - /// Create a queue and its metadata atomically. - fn create_queue(&mut self, name: &str) -> Result<()> { - let queue_names_key = queue_names_key(&self.key_prefix); - let keys = QueueKeys::new(&self.key_prefix, name); - let created = self.with_connection("create_queue", |connection| { - redis::Script::new(CREATE_QUEUE_SCRIPT) - .key(&queue_names_key) - .key(&keys.meta) - .arg(name) - .arg(unix_secs(SystemTime::now())) - .invoke::(connection) - })?; - if created == 0 { - return Err(Error::AlreadyExists(format!( - "queue '{}' already exists", - name - ))); - } - Ok(()) - } - - /// Remove a queue and all of its Redis keys atomically. - fn remove_queue(&mut self, name: &str) -> Result<()> { - let queue_names_key = queue_names_key(&self.key_prefix); - let queue_key_prefix = queue_key_prefix(&self.key_prefix); - let removed = self.with_connection("remove_queue", |connection| { - redis::Script::new(REMOVE_QUEUE_SCRIPT) - .key(&queue_names_key) - .arg(name) - .arg(&queue_key_prefix) - .invoke::(connection) - })?; - if removed == 0 { - return Err(Error::NotFound(format!("queue '{}' not found", name))); - } - Ok(()) - } - - /// Return whether a queue is registered. - fn queue_exists(&self, name: &str) -> bool { - match self.queue_exists_result(name) { - Ok(exists) => exists, - Err(error) => { - tracing::error!( - queue = name, - error = %error, - "queuefs redis queue_exists failed; returning false" - ); - false - } - } - } - - /// List registered queues matching the requested prefix. - fn list_queues(&self, prefix: &str) -> Vec { - let queue_names_key = queue_names_key(&self.key_prefix); - let mut queues = match self.with_connection("list_queues", |connection| { - redis::cmd("SMEMBERS") - .arg(&queue_names_key) - .query::>(connection) - }) { - Ok(queues) => queues, - Err(error) => { - tracing::error!( - prefix, - error = %error, - "queuefs redis list_queues failed; returning an empty list" - ); - return Vec::new(); - } - }; - queues.retain(|queue| queue.starts_with(prefix)); - queues.sort(); - queues - } - - /// Store a payload and append its ID to the pending list atomically. - fn enqueue(&mut self, queue_name: &str, msg: Message) -> Result<()> { - let queue_names_key = queue_names_key(&self.key_prefix); - let keys = QueueKeys::new(&self.key_prefix, queue_name); - let payload = serde_json::to_string(&StoredMessage::from_message(&msg))?; - let enqueued = self.with_connection("enqueue", |connection| { - redis::Script::new(ENQUEUE_SCRIPT) - .key(&queue_names_key) - .key(keys.message(&msg.id)) - .key(&keys.pending) - .key(&keys.meta) - .arg(queue_name) - .arg(&msg.id) - .arg(&payload) - .arg(unix_secs(SystemTime::now())) - .invoke::(connection) - })?; - if enqueued == 0 { - return Err(Error::NotFound(format!( - "queue '{}' not found", - queue_name - ))); - } - Ok(()) - } - - /// Move the oldest pending message to processing and return its payload atomically. - fn dequeue(&mut self, queue_name: &str) -> Result> { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - let result = self.with_connection("dequeue", |connection| { - redis::Script::new(DEQUEUE_SCRIPT) - .key(&keys.pending) - .key(&keys.processing) - .arg(&keys.message_prefix) - .arg(&self.instance_id) - .arg(unix_secs(SystemTime::now())) - .invoke::>>(connection) - })?; - result - .map(|values| { - values - .get(1) - .ok_or_else(|| Error::internal("redis dequeue returned no payload")) - .and_then(|payload| Self::decode_message(payload)) - }) - .transpose() - } - - /// Read the oldest pending payload without changing queue state. - fn peek(&self, queue_name: &str) -> Result> { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - self.with_connection("peek", |connection| { - redis::Script::new(PEEK_SCRIPT) - .key(&keys.pending) - .arg(&keys.message_prefix) - .invoke::>(connection) - })? - .map(|payload| Self::decode_message(&payload)) - .transpose() - } - - /// Return the number of pending messages. - fn size(&self, queue_name: &str) -> Result { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - self.with_connection("size", |connection| { - redis::cmd("LLEN") - .arg(&keys.pending) - .query::(connection) - }) - } - - /// Return all pending and processing messages without changing their state. - fn list_unacked(&self, queue_name: &str) -> Result> { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - let payloads = self.with_connection("list_unacked", |connection| { - redis::Script::new(LIST_UNACKED_SCRIPT) - .key(&keys.pending) - .key(&keys.processing) - .arg(&keys.message_prefix) - .invoke::>(connection) - })?; - payloads - .iter() - .map(|payload| Self::decode_message(payload)) - .collect() - } - - /// Delete all pending, processing, and payload keys for one queue atomically. - fn clear(&mut self, queue_name: &str) -> Result<()> { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - self.with_connection("clear", |connection| { - redis::Script::new(CLEAR_SCRIPT) - .key(&keys.pending) - .key(&keys.processing) - .arg(&keys.message_prefix) - .invoke::(connection) - })?; - Ok(()) - } - - /// Return the latest timestamp among the current pending messages. - fn get_last_enqueue_time(&self, queue_name: &str) -> Result { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - let pending_ids = self.with_connection("get_last_enqueue_time list pending", |connection| { - redis::cmd("LRANGE") - .arg(&keys.pending) - .arg(0) - .arg(-1) - .query::>(connection) - })?; - if pending_ids.is_empty() { - return Ok(UNIX_EPOCH); - } - let payloads = self.with_connection("get_last_enqueue_time load payloads", |connection| { - let mut command = redis::cmd("MGET"); - for id in &pending_ids { - command.arg(keys.message(id)); - } - command.query::>>(connection) - })?; - let payloads = payloads - .into_iter() - .enumerate() - .map(|(index, payload)| { - payload.ok_or_else(|| { - Error::internal(format!( - "redis get_last_enqueue_time missing payload for message {}", - pending_ids[index] - )) - }) - }) - .collect::>>()?; - // ponytail: scan current pending payloads on demand; if this becomes hot, add a dedicated pending timestamp index. - last_enqueue_time_from_pending_payloads(&payloads) - } - - /// Remove a processing record and its payload atomically. - fn ack(&mut self, queue_name: &str, msg_id: &str) -> Result { - self.require_queue(queue_name)?; - let keys = QueueKeys::new(&self.key_prefix, queue_name); - self.with_connection("ack", |connection| { - redis::Script::new(ACK_SCRIPT) - .key(&keys.processing) - .key(keys.message(msg_id)) - .arg(msg_id) - .invoke::(connection) - }) - } -} - -/// Return the queue registry key for one namespace. -pub(super) fn queue_names_key(key_prefix: &str) -> String { - format!("{}names", queue_key_prefix(key_prefix)) -} - -/// Return the queue key prefix for one namespace. -pub(super) fn queue_key_prefix(key_prefix: &str) -> String { - format!("{{{key_prefix}}}:ov:queue:") -} - -/// Return the instance key prefix for one namespace. -pub(super) fn instance_key_prefix(key_prefix: &str) -> String { - format!("{}instance:", queue_key_prefix(key_prefix)) -} - -/// Return the heartbeat key for one instance. -pub(super) fn heartbeat_key(key_prefix: &str, instance_id: &str) -> String { - format!("{}{instance_id}:alive", instance_key_prefix(key_prefix)) -} - -/// Return Unix seconds for Redis scores and metadata. -pub(super) fn unix_secs(time: SystemTime) -> u64 { - time.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() -} - -/// Return the latest timestamp among the current pending payloads. -pub(super) fn last_enqueue_time_from_pending_payloads(payloads: &[String]) -> Result { - payloads.iter().try_fold(UNIX_EPOCH, |latest, payload| { - let timestamp = serde_json::from_str::(payload) - .map(StoredMessage::into_message) - .map(|message| message.timestamp) - .map_err(|error| Error::Serialization(format!("invalid queue payload: {error}")))?; - Ok(if timestamp > latest { timestamp } else { latest }) - }) -} - -/// Build one data-node connection descriptor with authentication, database, and TLS. -fn endpoint_connection_info( - endpoint: &str, - options: &RedisQueueOptions, -) -> redis::RedisResult { - let info = endpoint.into_connection_info()?; - let mut redis_settings = RedisConnectionInfo::default() - .set_db(options.db) - .set_protocol(info.redis_settings().protocol()); - if let Some(username) = &options.username { - redis_settings = redis_settings.set_username(username); - } - if let Some(password) = &options.password { - redis_settings = redis_settings.set_password(password); - } - let address = if options.tls_enabled { - force_tls(info.addr().clone(), options.tls_insecure_skip_verify) - } else { - info.addr().clone() - }; - Ok(info - .set_redis_settings(redis_settings) - .set_addr(address)) -} - -/// Build one Sentinel node address with the configured transport security. -fn sentinel_addr( - endpoint: &str, - options: &RedisQueueOptions, -) -> redis::RedisResult { - let info = endpoint.into_connection_info()?; - Ok(if options.tls_enabled { - force_tls(info.addr().clone(), options.tls_insecure_skip_verify) - } else { - info.addr().clone() - }) -} - -/// Convert a TCP address into its TLS form while preserving non-TCP transports. -fn force_tls(address: ConnectionAddr, insecure: bool) -> ConnectionAddr { - match address { - ConnectionAddr::Tcp(host, port) | ConnectionAddr::TcpTls { host, port, .. } => { - ConnectionAddr::TcpTls { - host, - port, - insecure, - tls_params: None, - } - } - address => address, - } -} - -/// Return the redis-rs TLS mode represented by QueueFS settings. -fn tls_mode(options: &RedisQueueOptions) -> TlsMode { - if options.tls_insecure_skip_verify { - TlsMode::Insecure - } else { - TlsMode::Secure - } -} - -/// Apply QueueFS command timeouts to one direct Redis connection. -fn configure_connection( - connection: &Connection, - command_timeout: Duration, -) -> redis::RedisResult<()> { - let timeout = Some(command_timeout); - connection.set_read_timeout(timeout)?; - connection.set_write_timeout(timeout)?; - Ok(()) -} - -/// Validate that one direct Redis connection is still usable. -fn validate_connection(connection: &mut Connection) -> RedisResult<()> { - if connection.check_connection() { - Ok(()) - } else { - Err(RedisError::from(( - redis::ErrorKind::Io, - "redis connection closed", - ))) - } -} - -/// Return whether one Sentinel response means the connected master is no longer writable. -fn is_sentinel_topology_error(error: &RedisError) -> bool { - matches!( - error.kind(), - redis::ErrorKind::Server(ServerErrorKind::ReadOnly | ServerErrorKind::MasterDown) - ) -} - -/// Build one initialized r2d2 pool with the QueueFS capacity and checkout timeout. -fn build_pool(manager: M) -> Result> -where - M: ManageConnection, -{ - Pool::builder() - .max_size(REDIS_POOL_MAX_SIZE) - .connection_timeout(REDIS_POOL_CHECKOUT_TIMEOUT) - .build(manager) - .map_err(|error| pool_error("connect", error)) -} - -/// Map r2d2 initialization and checkout failures into a QueueFS network error. -fn pool_error(operation: &str, error: r2d2::Error) -> Error { - Error::Network(format!("redis {operation} pool error: {error}")) -} - -/// Refresh one instance heartbeat through one pooled logical connection. -fn refresh_heartbeat(pool: &RedisPool, key: &str) -> Result<()> { - pool.execute("heartbeat", |connection| { - redis::cmd("SET") - .arg(key) - .arg("1") - .arg("EX") - .arg(HEARTBEAT_TTL_SECS) - .query::<()>(connection) - }) -} - -/// Map redis-rs failures into the QueueFS error categories. -fn redis_error(operation: &str, error: redis::RedisError) -> Error { - if error.is_timeout() { - Error::Timeout(format!("redis {operation} error: {error}")) - } else if error.is_io_error() || error.is_connection_dropped() { - Error::Network(format!("redis {operation} error: {error}")) - } else { - Error::internal(format!("redis {operation} error: {error}")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{Duration, Instant, UNIX_EPOCH}; - - /// Exercise one complete QueueFS message lifecycle. - fn assert_queue_flow(mut backend: RedisQueueBackend) { - let queue = format!("queuefs-test-{}", Uuid::new_v4()); - backend.create_queue(&queue).unwrap(); - let message = Message::new(b"topology".to_vec()); - let message_id = message.id.clone(); - backend.enqueue(&queue, message).unwrap(); - assert_eq!(backend.peek(&queue).unwrap().unwrap().id, message_id); - assert_eq!(backend.dequeue(&queue).unwrap().unwrap().id, message_id); - assert!(backend.ack(&queue, &message_id).unwrap()); - backend.remove_queue(&queue).unwrap(); - } - - #[test] - /// Build queue keys under the configured namespace. - fn queue_keys_use_the_documented_layout() { - let keys = QueueKeys::new("tenant-a", "Semantic"); - - assert_eq!(queue_names_key("tenant-a"), "{tenant-a}:ov:queue:names"); - assert_eq!(keys.meta, "{tenant-a}:ov:queue:Semantic:meta"); - assert_eq!(keys.pending, "{tenant-a}:ov:queue:Semantic:pending"); - assert_eq!( - keys.processing, - "{tenant-a}:ov:queue:Semantic:processing" - ); - assert_eq!( - keys.message("message-id"), - "{tenant-a}:ov:queue:Semantic:msg:message-id" - ); - assert_eq!( - heartbeat_key("tenant-a", "instance-id"), - "{tenant-a}:ov:queue:instance:instance-id:alive" - ); - } - - #[test] - /// Derive the last enqueue time from the latest pending message timestamp. - fn last_enqueue_time_comes_from_pending_messages() { - let mut first = Message::new(b"first".to_vec()); - first.timestamp = UNIX_EPOCH + Duration::from_secs(10); - let mut second = Message::new(b"second".to_vec()); - second.timestamp = UNIX_EPOCH + Duration::from_secs(20); - let payloads = vec![ - serde_json::to_string(&StoredMessage::from_message(&first)).unwrap(), - serde_json::to_string(&StoredMessage::from_message(&second)).unwrap(), - ]; - - assert_eq!( - last_enqueue_time_from_pending_payloads(&payloads).unwrap(), - UNIX_EPOCH + Duration::from_secs(20) - ); - } - - #[test] - /// Discard Sentinel connections only for errors that indicate a stale master. - fn sentinel_topology_errors_invalidate_the_connection() { - assert!(is_sentinel_topology_error(&redis::RedisError::from(( - redis::ErrorKind::Server(ServerErrorKind::ReadOnly), - "old master is read-only", - )))); - assert!(is_sentinel_topology_error(&redis::RedisError::from(( - redis::ErrorKind::Server(ServerErrorKind::MasterDown), - "master is down", - )))); - assert!(!is_sentinel_topology_error(&redis::RedisError::from(( - redis::ErrorKind::Server(ServerErrorKind::ResponseError), - "ordinary command error", - )))); - } - - #[test] - /// Invalidate every pooled Sentinel connection created before a topology change. - fn sentinel_topology_generation_invalidates_all_old_connections() { - let generation = ConnectionGeneration::default(); - let first = generation.snapshot(); - let second = generation.snapshot(); - assert!(generation.is_current(first)); - assert!(generation.is_current(second)); - - generation.invalidate(); - - assert!(!generation.is_current(first)); - assert!(!generation.is_current(second)); - assert!(generation.is_current(generation.snapshot())); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_TEST_URL"] - /// Replace a dead Singleton connection before use without replaying command failures. - fn redis_backend_reconnects_singleton_after_disconnect() { - let endpoint = - std::env::var("QUEUEFS_REDIS_TEST_URL").expect("QUEUEFS_REDIS_TEST_URL is required"); - let mut backend = RedisQueueBackend::open(RedisQueueOptions { - endpoints: vec![endpoint.clone()], - key_prefix: format!("queuefs-test-{}", Uuid::new_v4()), - ..RedisQueueOptions::default() - }) - .unwrap(); - backend.create_queue("reconnect").unwrap(); - let client_id = backend - .with_connection("test client id", |connection| { - redis::cmd("CLIENT").arg("ID").query::(connection) - }) - .unwrap(); - let mut control = redis::Client::open(endpoint) - .unwrap() - .get_connection() - .unwrap(); - redis::cmd("CLIENT") - .arg("KILL") - .arg("ID") - .arg(client_id) - .query::(&mut control) - .unwrap(); - - assert_eq!(backend.size("reconnect").unwrap(), 0); - - let mut command_calls = 0; - let result = backend.with_connection("test command failure", |_connection| { - command_calls += 1; - RedisResult::<()>::Err(redis::RedisError::from(( - redis::ErrorKind::Server(ServerErrorKind::ResponseError), - "test command failure", - ))) - }); - assert!(result.is_err()); - assert_eq!(command_calls, 1); - - backend.remove_queue("reconnect").unwrap(); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_CLUSTER_TEST_URLS"] - /// Execute QueueFS Lua scripts through a real Redis Cluster connection. - fn redis_backend_cluster_executes_queue_flow() { - let endpoints = std::env::var("QUEUEFS_REDIS_CLUSTER_TEST_URLS") - .expect("QUEUEFS_REDIS_CLUSTER_TEST_URLS is required"); - let options = RedisQueueOptions { - mode: RedisMode::Cluster, - endpoints: endpoints.split(',').map(str::to_string).collect(), - key_prefix: format!("queuefs-test-{}", Uuid::new_v4()), - ..RedisQueueOptions::default() - }; - - assert_queue_flow(RedisQueueBackend::open(options).unwrap()); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_SENTINEL_TEST_URLS"] - /// Execute QueueFS operations through Sentinel master discovery. - fn redis_backend_sentinel_executes_queue_flow() { - let endpoints = std::env::var("QUEUEFS_REDIS_SENTINEL_TEST_URLS") - .expect("QUEUEFS_REDIS_SENTINEL_TEST_URLS is required"); - let options = RedisQueueOptions { - mode: RedisMode::Sentinel, - endpoints: endpoints.split(',').map(str::to_string).collect(), - master_name: Some( - std::env::var("QUEUEFS_REDIS_SENTINEL_MASTER") - .unwrap_or_else(|_| "mymaster".to_string()), - ), - key_prefix: format!("queuefs-test-{}", Uuid::new_v4()), - ..RedisQueueOptions::default() - }; - - assert_queue_flow(RedisQueueBackend::open(options).unwrap()); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_SENTINEL_TEST_URLS and a replica"] - /// Rediscover the Sentinel master after the connected master is demoted. - fn redis_backend_sentinel_reconnects_after_failover() { - let endpoints = std::env::var("QUEUEFS_REDIS_SENTINEL_TEST_URLS") - .expect("QUEUEFS_REDIS_SENTINEL_TEST_URLS is required"); - let master_name = std::env::var("QUEUEFS_REDIS_SENTINEL_MASTER") - .unwrap_or_else(|_| "mymaster".to_string()); - let sentinel_endpoint = endpoints.split(',').next().unwrap(); - let mut sentinel = redis::Client::open(sentinel_endpoint) - .unwrap() - .get_connection() - .unwrap(); - let original_master = redis::cmd("SENTINEL") - .arg("GET-MASTER-ADDR-BY-NAME") - .arg(&master_name) - .query::>(&mut sentinel) - .unwrap(); - let original_master_client = redis::Client::open(format!( - "redis://{}:{}", - original_master[0], original_master[1] - )) - .unwrap(); - let mut backend = RedisQueueBackend::open(RedisQueueOptions { - mode: RedisMode::Sentinel, - endpoints: endpoints.split(',').map(str::to_string).collect(), - master_name: Some(master_name.clone()), - key_prefix: format!("queuefs-test-{}", Uuid::new_v4()), - ..RedisQueueOptions::default() - }) - .unwrap(); - backend.create_queue("failover").unwrap(); - - redis::cmd("SENTINEL") - .arg("FAILOVER") - .arg(&master_name) - .query::<()>(&mut sentinel) - .unwrap(); - let deadline = Instant::now() + Duration::from_secs(15); - loop { - let current_master = redis::cmd("SENTINEL") - .arg("GET-MASTER-ADDR-BY-NAME") - .arg(&master_name) - .query::>(&mut sentinel) - .unwrap(); - let original_is_replica = original_master_client - .get_connection() - .and_then(|mut connection| { - redis::cmd("INFO") - .arg("replication") - .query::(&mut connection) - }) - .is_ok_and(|replication| replication.contains("role:slave")); - if current_master != original_master && original_is_replica { - break; - } - assert!(Instant::now() < deadline, "Sentinel failover timed out"); - std::thread::sleep(Duration::from_millis(100)); - } - - if backend - .enqueue("failover", Message::new(b"first".to_vec())) - .is_err() - { - backend - .enqueue("failover", Message::new(b"second".to_vec())) - .unwrap(); - } - assert!(backend.size("failover").unwrap() > 0); - backend.remove_queue("failover").unwrap(); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_TEST_URL"] - /// Reject an unregistered queue inside the enqueue script before writing any keys. - fn enqueue_script_rejects_unregistered_queue_atomically() { - let endpoint = - std::env::var("QUEUEFS_REDIS_TEST_URL").expect("QUEUEFS_REDIS_TEST_URL is required"); - let options = RedisQueueOptions { - endpoints: vec![endpoint], - key_prefix: format!("queuefs-test-{}", Uuid::new_v4()), - ..RedisQueueOptions::default() - }; - let mut backend = RedisQueueBackend::open(options).unwrap(); - let queue = "removed"; - let queue_names_key = queue_names_key(&backend.key_prefix); - let keys = QueueKeys::new(&backend.key_prefix, queue); - let message = Message::new(b"orphan".to_vec()); - let message_key = keys.message(&message.id); - - assert!(backend.enqueue(queue, message).is_err()); - let existing_keys = backend - .with_connection("test_enqueue_missing_queue keys", |connection| { - redis::cmd("EXISTS") - .arg(&queue_names_key) - .arg(&message_key) - .arg(&keys.pending) - .arg(&keys.meta) - .query::(connection) - }) - .unwrap(); - assert_eq!(existing_keys, 0); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_TEST_URL"] - /// Remove a queue tree without deleting a same-prefix sibling queue. - fn redis_backend_removes_nested_queues_recursively() { - let endpoint = - std::env::var("QUEUEFS_REDIS_TEST_URL").expect("QUEUEFS_REDIS_TEST_URL is required"); - let key_prefix = format!("queuefs-test-{}", Uuid::new_v4()); - let options = RedisQueueOptions { - endpoints: vec![endpoint], - key_prefix, - ..RedisQueueOptions::default() - }; - let mut backend = RedisQueueBackend::open(options).unwrap(); - backend.create_queue("worker-0").unwrap(); - backend.create_queue("worker-0/Embedding").unwrap(); - backend.create_queue("worker-01").unwrap(); - backend - .enqueue("worker-0/Embedding", Message::new(b"nested".to_vec())) - .unwrap(); - backend - .enqueue("worker-01", Message::new(b"sibling".to_vec())) - .unwrap(); - - backend.remove_queue("worker-0").unwrap(); - - assert!(!backend.queue_exists("worker-0")); - assert!(!backend.queue_exists("worker-0/Embedding")); - assert!(backend.queue_exists("worker-01")); - assert_eq!(backend.size("worker-01").unwrap(), 1); - backend.remove_queue("worker-01").unwrap(); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_TEST_URL"] - /// Isolate queue state between independent Redis key prefixes. - fn redis_backend_isolates_key_prefixes() { - let endpoint = - std::env::var("QUEUEFS_REDIS_TEST_URL").expect("QUEUEFS_REDIS_TEST_URL is required"); - let suffix = Uuid::new_v4(); - let mut first = RedisQueueBackend::open(RedisQueueOptions { - endpoints: vec![endpoint.clone()], - key_prefix: format!("queuefs-test-a-{suffix}"), - ..RedisQueueOptions::default() - }) - .unwrap(); - let mut second = RedisQueueBackend::open(RedisQueueOptions { - endpoints: vec![endpoint], - key_prefix: format!("queuefs-test-b-{suffix}"), - ..RedisQueueOptions::default() - }) - .unwrap(); - - first.create_queue("Semantic").unwrap(); - assert!(!second.queue_exists("Semantic")); - second.create_queue("Semantic").unwrap(); - first - .enqueue("Semantic", Message::new(b"first".to_vec())) - .unwrap(); - - assert_eq!(first.size("Semantic").unwrap(), 1); - assert_eq!(second.size("Semantic").unwrap(), 0); - first.remove_queue("Semantic").unwrap(); - second.remove_queue("Semantic").unwrap(); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_TEST_URL"] - /// Recover processing messages owned by instances without a heartbeat. - fn redis_backend_recovers_messages_owned_by_dead_instances() { - let endpoint = - std::env::var("QUEUEFS_REDIS_TEST_URL").expect("QUEUEFS_REDIS_TEST_URL is required"); - let options = RedisQueueOptions { - endpoints: vec![endpoint], - ..RedisQueueOptions::default() - }; - let queue = format!("queuefs-test-{}", Uuid::new_v4()); - let mut backend = RedisQueueBackend::open(options.clone()).unwrap(); - backend.create_queue(&queue).unwrap(); - let message = Message::new(b"payload".to_vec()); - let message_id = message.id.clone(); - backend.enqueue(&queue, message).unwrap(); - - let keys = QueueKeys::new(&options.key_prefix, &queue); - backend - .with_connection("test_dead_owner", |connection| { - redis::Script::new( - r#" -local id = redis.call('LPOP', KEYS[1]) -redis.call('ZADD', KEYS[2], ARGV[1], id .. '|dead-instance') -return id -"#, - ) - .key(&keys.pending) - .key(&keys.processing) - .arg(unix_secs(SystemTime::now())) - .invoke::(connection) - }) - .unwrap(); - drop(backend); - - let mut recovered = RedisQueueBackend::open(options).unwrap(); - let dequeued = recovered.dequeue(&queue).unwrap().unwrap(); - assert_eq!(dequeued.id, message_id); - assert_eq!(dequeued.data, b"payload"); - assert!(recovered.ack(&queue, &message_id).unwrap()); - recovered.remove_queue(&queue).unwrap(); - } - - #[test] - #[ignore = "requires QUEUEFS_REDIS_TEST_URL"] - /// Recover this instance's processing messages after a graceful restart. - fn redis_backend_graceful_restart_recovers_processing_messages() { - let endpoint = - std::env::var("QUEUEFS_REDIS_TEST_URL").expect("QUEUEFS_REDIS_TEST_URL is required"); - let options = RedisQueueOptions { - endpoints: vec![endpoint], - ..RedisQueueOptions::default() - }; - let queue = format!("queuefs-test-{}", Uuid::new_v4()); - let mut backend = RedisQueueBackend::open(options.clone()).unwrap(); - backend.create_queue(&queue).unwrap(); - let message = Message::new(b"payload".to_vec()); - let message_id = message.id.clone(); - backend.enqueue(&queue, message).unwrap(); - assert_eq!(backend.dequeue(&queue).unwrap().unwrap().id, message_id); - drop(backend); - - let mut restarted = RedisQueueBackend::open(options).unwrap(); - assert_eq!( - restarted.dequeue(&queue).unwrap().unwrap().id, - message_id - ); - assert!(restarted.ack(&queue, &message_id).unwrap()); - restarted.remove_queue(&queue).unwrap(); - } -} diff --git a/crates/ragfs/tests/cache_runtime.rs b/crates/ragfs/tests/cache_runtime.rs index b26c9633ac..be96fac85a 100644 --- a/crates/ragfs/tests/cache_runtime.rs +++ b/crates/ragfs/tests/cache_runtime.rs @@ -1,29 +1,31 @@ use bytes::Bytes; use ragfs::cache_runtime::{ - AsyncCacheRuntime, CacheError, CacheRuntime, MemoryMockProvider, PutOptions, ScriptRequest, - SyncCacheRuntime, + CacheError, CacheRuntime, Expiration, ListDirection, ListInsertPosition, ListInsertRequest, + ListMoveRequest, MemoryMockProvider, ScriptRequest, SetCondition, SetOptions, SetResult, }; use std::sync::Arc; +use std::time::Duration; #[tokio::test] async fn async_runtime_supports_the_primitive_contract() { let runtime = CacheRuntime::memory(); assert_eq!(runtime.get("missing").await.unwrap(), None); - assert!(!runtime.exists("missing").await.unwrap()); - runtime - .put("a", Bytes::from_static(b"one"), PutOptions::default()) - .await - .unwrap(); + assert_eq!( + runtime + .set("a", Bytes::from_static(b"one"), SetOptions::default()) + .await + .unwrap(), + SetResult::Applied + ); assert_eq!( runtime.get("a").await.unwrap(), Some(Bytes::from_static(b"one")) ); - assert!(runtime.exists("a").await.unwrap()); runtime - .batch_put(vec![ + .mset(vec![ ("b".to_string(), Bytes::from_static(b"two")), ("c".to_string(), Bytes::from_static(b"three")), ]) @@ -31,7 +33,7 @@ async fn async_runtime_supports_the_primitive_contract() { .unwrap(); assert_eq!( runtime - .batch_get(&["c".to_string(), "missing".to_string(), "b".to_string()]) + .mget(&["c".to_string(), "missing".to_string(), "b".to_string()]) .await .unwrap(), vec![ @@ -41,17 +43,251 @@ async fn async_runtime_supports_the_primitive_contract() { ] ); + assert_eq!( + runtime + .del(&["a".to_string(), "c".to_string(), "missing".to_string()]) + .await + .unwrap(), + 2 + ); + assert_eq!(runtime.del(&["b".to_string()]).await.unwrap(), 1); + assert_eq!( + runtime + .mget(&["a".to_string(), "b".to_string(), "c".to_string()]) + .await + .unwrap(), + vec![None, None, None] + ); +} + +#[tokio::test] +async fn set_reports_nx_and_xx_outcomes() { + let runtime = CacheRuntime::memory(); + let nx = SetOptions { + condition: SetCondition::Nx, + ..SetOptions::default() + }; + let xx = SetOptions { + condition: SetCondition::Xx, + ..SetOptions::default() + }; + + assert_eq!( + runtime + .set("lock", Bytes::from_static(b"owner-1"), nx) + .await + .unwrap(), + SetResult::Applied + ); + assert_eq!( + runtime + .set("lock", Bytes::from_static(b"owner-2"), nx) + .await + .unwrap(), + SetResult::ConditionNotMet + ); + assert_eq!( + runtime + .set("missing", Bytes::from_static(b"owner"), xx) + .await + .unwrap(), + SetResult::ConditionNotMet + ); + assert_eq!( + runtime + .set("lock", Bytes::from_static(b"owner-2"), xx) + .await + .unwrap(), + SetResult::Applied + ); +} + +#[tokio::test] +async fn set_expiration_and_keep_ttl_match_redis_semantics() { + let runtime = CacheRuntime::memory(); + runtime + .set( + "ttl", + Bytes::from_static(b"first"), + SetOptions { + expiration: Some(Expiration::After(Duration::from_millis(40))), + ..SetOptions::default() + }, + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + runtime + .set( + "ttl", + Bytes::from_static(b"second"), + SetOptions { + keep_ttl: true, + ..SetOptions::default() + }, + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(30)).await; + + assert_eq!(runtime.get("ttl").await.unwrap(), None); +} + +#[tokio::test] +async fn atomic_integer_commands_follow_redis_string_semantics() { + let runtime = CacheRuntime::memory(); + + assert_eq!(runtime.incr("sequence").await.unwrap(), 1); + assert_eq!(runtime.incr_by("sequence", 4).await.unwrap(), 5); + assert_eq!(runtime.decr("sequence").await.unwrap(), 4); + assert_eq!(runtime.decr_by("sequence", 6).await.unwrap(), -2); + + runtime + .set( + "not-an-integer", + Bytes::from_static(b"value"), + SetOptions::default(), + ) + .await + .unwrap(); + assert!(matches!( + runtime.incr("not-an-integer").await, + Err(CacheError::InvalidData(_)) + )); +} + +#[tokio::test] +async fn set_queries_return_membership_members_and_cardinality() { + let provider = Arc::new(MemoryMockProvider::new()); + provider + .insert_set_members( + "queues", + vec![Bytes::from_static(b"beta"), Bytes::from_static(b"alpha")], + ) + .await; + let runtime = CacheRuntime::memory_with_provider(provider); + + assert!(runtime.sismember("queues", b"alpha").await.unwrap()); + assert!(!runtime.sismember("queues", b"missing").await.unwrap()); + assert_eq!(runtime.scard("queues").await.unwrap(), 2); + assert_eq!(runtime.scard("missing").await.unwrap(), 0); + + let mut members = runtime.smembers("queues").await.unwrap(); + members.sort(); + assert_eq!( + members, + vec![Bytes::from_static(b"alpha"), Bytes::from_static(b"beta")] + ); +} + +#[tokio::test] +async fn non_blocking_list_commands_preserve_redis_order() { + let runtime = CacheRuntime::memory(); + + assert_eq!( + runtime + .rpush( + "pending", + vec![Bytes::from_static(b"a"), Bytes::from_static(b"b")], + ) + .await + .unwrap(), + 2 + ); + assert_eq!( + runtime + .lpush("pending", vec![Bytes::from_static(b"zero")]) + .await + .unwrap(), + 3 + ); + assert_eq!( + runtime.lrange("pending", 0, -1).await.unwrap(), + vec![ + Bytes::from_static(b"zero"), + Bytes::from_static(b"a"), + Bytes::from_static(b"b"), + ] + ); + assert_eq!(runtime.llen("pending").await.unwrap(), 3); + assert_eq!( + runtime.lindex("pending", -1).await.unwrap(), + Some(Bytes::from_static(b"b")) + ); + assert_eq!( + runtime.lpop("pending", None).await.unwrap(), + vec![Bytes::from_static(b"zero")] + ); + assert_eq!( + runtime.rpop("pending", Some(1)).await.unwrap(), + vec![Bytes::from_static(b"b")] + ); + assert_eq!(runtime.llen("pending").await.unwrap(), 1); +} + +#[tokio::test] +async fn non_blocking_list_mutations_match_redis_semantics() { + let runtime = CacheRuntime::memory(); + runtime + .rpush( + "source", + vec![ + Bytes::from_static(b"a"), + Bytes::from_static(b"b"), + Bytes::from_static(b"a"), + Bytes::from_static(b"c"), + ], + ) + .await + .unwrap(); + runtime - .batch_delete(&["a".to_string(), "c".to_string()]) + .lset("source", 1, Bytes::from_static(b"B")) .await .unwrap(); - runtime.delete("b").await.unwrap(); assert_eq!( runtime - .batch_get(&["a".to_string(), "b".to_string(), "c".to_string()]) + .linsert(ListInsertRequest { + key: "source".to_string(), + position: ListInsertPosition::Before, + pivot: Bytes::from_static(b"B"), + value: Bytes::from_static(b"x"), + }) .await .unwrap(), - vec![None, None, None] + 5 + ); + assert_eq!( + runtime + .lrem("source", 0, Bytes::from_static(b"a")) + .await + .unwrap(), + 2 + ); + runtime.ltrim("source", 0, 1).await.unwrap(); + runtime + .rpush("destination", vec![Bytes::from_static(b"d")]) + .await + .unwrap(); + assert_eq!( + runtime + .lmove(ListMoveRequest { + source: "source".to_string(), + destination: "destination".to_string(), + source_direction: ListDirection::Right, + destination_direction: ListDirection::Left, + }) + .await + .unwrap(), + Some(Bytes::from_static(b"B")) + ); + assert_eq!( + runtime.lrange("source", 0, -1).await.unwrap(), + vec![Bytes::from_static(b"x")] + ); + assert_eq!( + runtime.lrange("destination", 0, -1).await.unwrap(), + vec![Bytes::from_static(b"B"), Bytes::from_static(b"d")] ); } @@ -75,12 +311,15 @@ fn sync_and_async_facades_share_one_provider_instance() { let runtime = CacheRuntime::memory(); let sync = runtime.sync_facade(); - sync.put( - "shared", - Bytes::from_static(b"value"), - PutOptions::default(), - ) - .unwrap(); + assert_eq!( + sync.set( + "shared", + Bytes::from_static(b"value"), + SetOptions::default(), + ) + .unwrap(), + SetResult::Applied + ); let async_runtime = runtime.clone(); let value = std::thread::spawn(move || { @@ -96,6 +335,28 @@ fn sync_and_async_facades_share_one_provider_instance() { assert_eq!(value, Some(Bytes::from_static(b"value"))); } +#[test] +fn sync_facade_exposes_lifecycle_operations() { + let runtime = CacheRuntime::memory(); + let sync = runtime.sync_facade(); + + sync.ping().unwrap(); + sync.close().unwrap(); + assert!(matches!(sync.get("closed"), Err(CacheError::Closed))); +} + +#[tokio::test] +async fn sync_facade_rejects_calls_from_tokio_runtime() { + let runtime = CacheRuntime::memory(); + let sync = runtime.sync_facade(); + + assert!(matches!( + sync.incr("counter"), + Err(CacheError::InvalidExecutionContext) + )); + assert_eq!(runtime.get("counter").await.unwrap(), None); +} + #[tokio::test] async fn close_rejects_new_operations() { let runtime = CacheRuntime::memory(); @@ -110,10 +371,10 @@ async fn controlled_memory_provider_is_only_accessed_through_runtime() { let runtime = CacheRuntime::memory_with_provider(Arc::clone(&provider)); runtime - .put( + .set( "observed", Bytes::from_static(b"value"), - PutOptions::default(), + SetOptions::default(), ) .await .unwrap(); diff --git a/crates/ragfs/tests/cache_wrapper.rs b/crates/ragfs/tests/cache_wrapper.rs index 907d883017..4f1f2380ca 100644 --- a/crates/ragfs/tests/cache_wrapper.rs +++ b/crates/ragfs/tests/cache_wrapper.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use ragfs::cache::{ CacheDecision, CacheNamespace, CachePolicy, CacheTraversalMode, CachedFileSystem, }; -use ragfs::cache_runtime::{AsyncCacheRuntime, CacheRuntime, MemoryMockProvider}; +use ragfs::cache_runtime::{CacheRuntime, MemoryMockProvider}; use ragfs::core::{FsContextInner, GrepResult, MultiWriteWrappedFS, TreeEntry, FS_CTX}; use ragfs::plugins::MemFileSystem; use ragfs::{Error, FileInfo, FileSystem, Result, WriteFlag}; @@ -1144,7 +1144,7 @@ async fn provider_generation_eviction_after_restart_cannot_revive_old_descendant for key in provider.keys().await { if key.contains(":subtree:") { - first_runtime.delete(&key).await.unwrap(); + first_runtime.del(&[key]).await.unwrap(); } } drop(first); @@ -1298,6 +1298,50 @@ async fn unavailable_provider_falls_back_to_backend_and_enters_bypass() { assert!(metrics.policy_bypasses >= 1); } +#[tokio::test] +async fn generation_backfill_is_bounded_and_continues_after_one_set_failure() { + let backend = CountingFileSystem::new(); + let mut current = String::new(); + for component in ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l"] { + current.push('/'); + current.push_str(component); + backend.mkdir(¤t, 0o755).await.unwrap(); + } + let path = "/a/b/c/d/e/f/g/h/i/j/k/l/file.md"; + backend + .write(path, b"generation", 0, WriteFlag::Create) + .await + .unwrap(); + + let provider = Arc::new(MemoryMockProvider::new().with_set_delay(Duration::from_millis(20))); + provider.fail_next_set_matching(":subtree:"); + let fs = CachedFileSystem::with_runtime( + Box::new(backend), + CacheRuntime::memory_with_provider(Arc::clone(&provider)), + CacheNamespace::new("generation-set-failure"), + CachePolicy::default(), + ); + + assert_eq!(fs.read(path, 0, 0).await.unwrap(), b"generation"); + + let keys = provider.keys().await; + assert_eq!( + keys.iter().filter(|key| key.contains(":subtree:")).count(), + 13, + "one of the fourteen ancestor generation writes should fail" + ); + assert_eq!( + keys.iter().filter(|key| key.contains(":file:")).count(), + 1, + "a generation write failure must not prevent the file cache fill" + ); + assert_eq!(fs.metrics().snapshot().errors, 1); + assert!( + (2..=8).contains(&provider.max_concurrent_sets()), + "generation writes should run concurrently with a fixed upper bound" + ); +} + #[tokio::test] async fn metrics_cover_operations_bytes_latency_and_errors() { let backend = CountingFileSystem::new(); diff --git a/crates/ragfs/tests/dynamic_provider.rs b/crates/ragfs/tests/dynamic_provider.rs index 231437b5f7..650ec0ddb0 100644 --- a/crates/ragfs/tests/dynamic_provider.rs +++ b/crates/ragfs/tests/dynamic_provider.rs @@ -1,131 +1,15 @@ -use bytes::Bytes; -use ragfs::cache_runtime::{ - AsyncCacheRuntime, CacheError, CacheRuntime, DynamicProviderConfig, PutOptions, ScriptRequest, -}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::time::{Duration, Instant}; - -fn compile_fixture(name: &str, definitions: &[&str]) -> PathBuf { - let output_dir = tempfile::tempdir().unwrap().keep(); - let library = output_dir.join(format!( - "{}{}{}", - std::env::consts::DLL_PREFIX, - name, - std::env::consts::DLL_SUFFIX - )); - let source = - Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/dynamic_provider/provider.c"); - let mut command = Command::new("cc"); - if cfg!(target_os = "macos") { - command.arg("-dynamiclib"); - } else { - command.args(["-shared", "-fPIC"]); - } - for definition in definitions { - command.arg(format!("-D{definition}")); - } - let status = command - .arg(source) - .arg("-o") - .arg(&library) - .status() - .unwrap(); - assert!(status.success()); - library -} - -async fn runtime(library_path: PathBuf) -> std::sync::Arc { - CacheRuntime::dynamic(DynamicProviderConfig { - library_path, - params_json: "{}".into(), - }) - .await - .unwrap() -} +use ragfs::cache_runtime::{CacheError, CacheRuntime, DynamicProviderConfig}; +use std::path::PathBuf; #[tokio::test] -async fn dynamic_provider_runs_primitive_batch_and_script_contract() { - let runtime = runtime(compile_fixture("provider", &[])).await; - - assert_eq!(runtime.get("missing").await.unwrap(), None); - runtime - .put("one", Bytes::from_static(b"1"), PutOptions::default()) - .await - .unwrap(); - assert!(runtime.exists("one").await.unwrap()); - runtime - .batch_put(vec![ - ("two".into(), Bytes::from_static(b"2")), - ("three".into(), Bytes::from_static(b"3")), - ]) - .await - .unwrap(); - assert_eq!( - runtime - .batch_get(&["three".into(), "missing".into(), "one".into()]) - .await - .unwrap(), - vec![ - Some(Bytes::from_static(b"3")), - None, - Some(Bytes::from_static(b"1")), - ] - ); - let result = runtime - .execute_script(ScriptRequest { - script_id: "runtime.test.echo.v1".into(), - keys: vec!["one".into()], - args: vec![Bytes::from_static(b"argument")], - }) - .await - .unwrap(); - assert_eq!(result.payload, Bytes::from_static(b"argument")); - runtime - .batch_delete(&["one".into(), "two".into(), "three".into()]) - .await - .unwrap(); - assert!(!runtime.exists("one").await.unwrap()); - runtime.close().await.unwrap(); - assert!(matches!(runtime.get("one").await, Err(CacheError::Closed))); -} - -#[tokio::test] -async fn dynamic_provider_rejects_missing_symbol_and_abi_mismatch() { - let missing = CacheRuntime::dynamic(DynamicProviderConfig { - library_path: compile_fixture("missing", &["OMIT_ENTRY"]), +async fn dynamic_provider_is_explicitly_unsupported_in_this_release() { + let result = CacheRuntime::dynamic(DynamicProviderConfig { + library_path: PathBuf::from("libcache_provider.so"), params_json: "{}".into(), }) .await; - let missing = match missing { - Ok(_) => panic!("missing entry unexpectedly loaded"), - Err(error) => error, - }; - assert!(matches!(missing, CacheError::AbiMismatch(_))); - let mismatch = CacheRuntime::dynamic(DynamicProviderConfig { - library_path: compile_fixture("mismatch", &["ABI_VERSION=99"]), - params_json: "{}".into(), - }) - .await; - let mismatch = match mismatch { - Ok(_) => panic!("ABI mismatch unexpectedly loaded"), - Err(error) => error, - }; - assert!(matches!(mismatch, CacheError::AbiMismatch(_))); -} - -#[tokio::test] -async fn dynamic_provider_close_waits_for_inflight_blocking_call() { - let runtime = runtime(compile_fixture("slow", &[])).await; - let reader = { - let runtime = runtime.clone(); - tokio::spawn(async move { runtime.get("slow").await.unwrap() }) - }; - tokio::time::sleep(Duration::from_millis(25)).await; - let started = Instant::now(); - runtime.close().await.unwrap(); - - assert_eq!(reader.await.unwrap(), Some(Bytes::from_static(b"slow"))); - assert!(started.elapsed() >= Duration::from_millis(100)); + assert!( + matches!(result, Err(CacheError::UnsupportedProvider(message)) if message.contains("DynamicProvider")) + ); } diff --git a/crates/ragfs/tests/redis_runtime.rs b/crates/ragfs/tests/redis_runtime.rs index a878512eba..6c86db3db8 100644 --- a/crates/ragfs/tests/redis_runtime.rs +++ b/crates/ragfs/tests/redis_runtime.rs @@ -1,5 +1,7 @@ use bytes::Bytes; -use ragfs::cache_runtime::{AsyncCacheRuntime, CacheRuntime, PutOptions, RedisProviderConfig}; +use ragfs::cache_runtime::{ + CacheError, CacheRuntime, Expiration, RedisProviderConfig, SetCondition, SetOptions, SetResult, +}; use std::time::Duration; fn config() -> Option { @@ -21,6 +23,17 @@ fn key(test_name: &str, value: &str) -> String { ) } +fn topology_endpoints(variable: &str) -> Option> { + let endpoints = std::env::var(variable).ok()?; + let endpoints = endpoints + .split(',') + .map(str::trim) + .filter(|endpoint| !endpoint.is_empty()) + .map(str::to_string) + .collect::>(); + (!endpoints.is_empty()).then_some(endpoints) +} + #[tokio::test] async fn redis_runtime_preserves_primitive_and_batch_semantics() { let Some(config) = config() else { @@ -33,7 +46,7 @@ async fn redis_runtime_preserves_primitive_and_batch_semantics() { assert_eq!(runtime.get(&missing).await.unwrap(), None); runtime - .batch_put(vec![ + .mset(vec![ (one.clone(), Bytes::from_static(b"1")), (two.clone(), Bytes::from_static(b"2")), ]) @@ -41,7 +54,7 @@ async fn redis_runtime_preserves_primitive_and_batch_semantics() { .unwrap(); assert_eq!( runtime - .batch_get(&[two.clone(), missing, one.clone()]) + .mget(&[two.clone(), missing, one.clone()]) .await .unwrap(), vec![ @@ -50,8 +63,8 @@ async fn redis_runtime_preserves_primitive_and_batch_semantics() { Some(Bytes::from_static(b"1")), ] ); - runtime.batch_delete(&[one.clone(), two]).await.unwrap(); - assert!(!runtime.exists(&one).await.unwrap()); + assert_eq!(runtime.del(&[one.clone(), two]).await.unwrap(), 2); + assert_eq!(runtime.get(&one).await.unwrap(), None); runtime.close().await.unwrap(); } @@ -65,11 +78,12 @@ async fn redis_runtime_preserves_default_ttl_and_per_write_override() { let ttl_key = key("ttl", "ttl"); runtime - .put( + .set( &ttl_key, Bytes::from_static(b"short"), - PutOptions { - ttl: Some(Duration::from_millis(100)), + SetOptions { + expiration: Some(Expiration::After(Duration::from_millis(100))), + ..SetOptions::default() }, ) .await @@ -78,3 +92,228 @@ async fn redis_runtime_preserves_default_ttl_and_per_write_override() { assert_eq!(runtime.get(&ttl_key).await.unwrap(), None); runtime.close().await.unwrap(); } + +#[tokio::test] +async fn redis_runtime_supports_conditions_counters_and_lists() { + let Some(config) = config() else { + return; + }; + let runtime = CacheRuntime::redis(config).await.unwrap(); + let string_key = key("commands", "string"); + let counter_key = key("commands", "counter"); + let list_key = key("commands", "list"); + + assert_eq!( + runtime + .set( + &string_key, + Bytes::from_static(b"first"), + SetOptions { + condition: SetCondition::Nx, + ..SetOptions::default() + }, + ) + .await + .unwrap(), + SetResult::Applied + ); + assert_eq!( + runtime + .set( + &string_key, + Bytes::from_static(b"ignored"), + SetOptions { + condition: SetCondition::Nx, + ..SetOptions::default() + }, + ) + .await + .unwrap(), + SetResult::ConditionNotMet + ); + + assert_eq!(runtime.incr_by(&counter_key, 4).await.unwrap(), 4); + assert_eq!(runtime.decr(&counter_key).await.unwrap(), 3); + + assert_eq!( + runtime + .rpush( + &list_key, + vec![Bytes::from_static(b"a"), Bytes::from_static(b"b")], + ) + .await + .unwrap(), + 2 + ); + assert_eq!( + runtime.lpop(&list_key, Some(2)).await.unwrap(), + vec![Bytes::from_static(b"a"), Bytes::from_static(b"b")] + ); + + runtime + .del(&[string_key, counter_key, list_key]) + .await + .unwrap(); + runtime.close().await.unwrap(); +} + +#[test] +fn sync_redis_runtime_is_initialized_on_the_runtime_executor() { + let Some(config) = config() else { + return; + }; + let runtime = CacheRuntime::connect_sync(config).unwrap(); + let sync = runtime.sync_facade(); + let sync_key = key("sync", "value"); + + sync.ping().unwrap(); + assert_eq!( + sync.set( + &sync_key, + Bytes::from_static(b"sync"), + SetOptions::default(), + ) + .unwrap(), + SetResult::Applied + ); + assert_eq!( + sync.get(&sync_key).unwrap(), + Some(Bytes::from_static(b"sync")) + ); + sync.del(&[sync_key]).unwrap(); + sync.close().unwrap(); +} + +#[tokio::test] +async fn redis_cluster_supports_same_slot_replica_reads_and_cross_slot_errors() { + let Some(endpoints) = topology_endpoints("REDIS_CLUSTER_TEST_URLS") else { + return; + }; + let runtime = CacheRuntime::redis(RedisProviderConfig { + mode: "cluster".into(), + endpoints, + read_from_replica: true, + connect_timeout_ms: 10_000, + command_timeout_ms: 3_000, + default_ttl_seconds: 60, + ..RedisProviderConfig::default() + }) + .await + .unwrap(); + let same_slot_one = key("cluster", "{runtime}:one"); + let same_slot_two = key("cluster", "{runtime}:two"); + + runtime + .mset(vec![ + (same_slot_one.clone(), Bytes::from_static(b"one")), + (same_slot_two.clone(), Bytes::from_static(b"two")), + ]) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + let values = runtime + .mget(&[same_slot_one.clone(), same_slot_two.clone()]) + .await + .unwrap(); + if values + == vec![ + Some(Bytes::from_static(b"one")), + Some(Bytes::from_static(b"two")), + ] + { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "replicas did not observe the writes before the deadline" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + + let cross_slot = runtime + .mset(vec![ + (key("cluster", "{slot-a}:one"), Bytes::from_static(b"one")), + (key("cluster", "{slot-b}:two"), Bytes::from_static(b"two")), + ]) + .await; + assert!(matches!(cross_slot, Err(CacheError::CrossSlot(_)))); + + runtime.del(&[same_slot_one, same_slot_two]).await.unwrap(); + runtime.close().await.unwrap(); +} + +#[tokio::test] +async fn redis_sentinel_discovers_the_configured_master() { + let Some(endpoints) = topology_endpoints("REDIS_SENTINEL_TEST_URLS") else { + return; + }; + let Ok(master_name) = std::env::var("REDIS_SENTINEL_TEST_MASTER") else { + return; + }; + let runtime = CacheRuntime::redis(RedisProviderConfig { + mode: "sentinel".into(), + endpoints, + master_name: Some(master_name), + connect_timeout_ms: 10_000, + command_timeout_ms: 3_000, + default_ttl_seconds: 60, + ..RedisProviderConfig::default() + }) + .await + .unwrap(); + let sentinel_key = key("sentinel", "value"); + + runtime + .set( + &sentinel_key, + Bytes::from_static(b"sentinel"), + SetOptions::default(), + ) + .await + .unwrap(); + assert_eq!( + runtime.get(&sentinel_key).await.unwrap(), + Some(Bytes::from_static(b"sentinel")) + ); + + runtime.del(&[sentinel_key]).await.unwrap(); + runtime.close().await.unwrap(); +} + +#[tokio::test] +async fn redis_runtime_accepts_the_legacy_plaintext_password() { + let Ok(endpoint) = std::env::var("REDIS_PASSWORD_TEST_URL") else { + return; + }; + let Ok(password) = std::env::var("REDIS_PASSWORD_TEST_SECRET") else { + return; + }; + let runtime = CacheRuntime::redis(RedisProviderConfig { + endpoints: vec![endpoint], + password, + connect_timeout_ms: 10_000, + command_timeout_ms: 3_000, + ..RedisProviderConfig::default() + }) + .await + .unwrap(); + let password_key = key("password", "value"); + + runtime + .set( + &password_key, + Bytes::from_static(b"authenticated"), + SetOptions::default(), + ) + .await + .unwrap(); + assert_eq!( + runtime.get(&password_key).await.unwrap(), + Some(Bytes::from_static(b"authenticated")) + ); + + runtime.del(&[password_key]).await.unwrap(); + runtime.close().await.unwrap(); +} diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index 54fe00144d..eea6ac5a79 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1142,21 +1142,77 @@ Notes: See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more examples. +##### Global Cache Provider and CacheFS Configuration + +The top-level `cache` section is a sibling of `storage`. Its public shape is Provider-neutral: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `provider` | str | Global Cache Provider. This release supports `redis` | required | +| `params` | object | Provider-owned parameters; parsed as Redis connection settings when `provider=redis` | `{}` | + +`storage.agfs.cachefs` only controls CacheFS behavior: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `backend` | str | `local` keeps the original filesystem path; `cache` enables the CacheFS wrapper | `local` | +| `namespace` | str | CacheFS key namespace | `openviking` | +| `max_file_size_bytes` | int | Maximum full-file object size admitted to cache | `1048576` | +| `traversal_mode` | str | `backend` or `cached_traversal` | `backend` | +| `bypass_prefixes` | array[str] | Path prefixes that bypass cache | `[]` | + +```json +{ + "cache": { + "provider": "redis", + "params": { + "mode": "sentinel", + "endpoints": [ + "redis://sentinel-1:26379", + "redis://sentinel-2:26379" + ], + "master_name": "mymaster", + "password_env": "OPENVIKING_REDIS_PASSWORD", + "connect_timeout_ms": 1000, + "command_timeout_ms": 1000 + } + }, + "storage": { + "agfs": { + "cachefs": { + "backend": "cache", + "namespace": "production" + }, + "queuefs": { + "backend": "cache", + "cache_key_prefix": "production" + } + } + } +} +``` + +The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`. When all modules use local backends, `cache.params` is not parsed and no Provider connection is opened. + ##### QueueFS Configuration | Parameter | Type | Description | Default | |-----------|------|-------------|---------| | `mode` | str | QueueFS namespace mode: `"shared"` uses `/queue`; `"worker"` isolates each worker under `/queue/worker-` | `"shared"` | -| `backend` | str | QueueFS backend: `"memory"`, `"sqlite"`, or `"sqlite3"` | `"sqlite"` | +| `backend` | str | QueueFS backend: `"memory"`, `"sqlite"`, `"sqlite3"`, or `"cache"`; `"redis"` is a legacy compatibility value | `"sqlite"` | | `db_path` | str (optional) | SQLite database path for QueueFS when backend is `"sqlite"` or `"sqlite3"` | `null` | | `recover_stale_sec` | int | Recover `processing` queue messages older than this many seconds on startup. `0` means recover all stale processing messages | `0` | | `busy_timeout_ms` | int | SQLite busy timeout for QueueFS in milliseconds | `5000` | +| `cache_key_prefix` | str | QueueFS key namespace when backend is `"cache"` | `"default"` | Notes: - QueueFS defaults to `sqlite` even if the main AGFS storage backend is `local`, `s3`, or `memory`. - `mode=shared` keeps the historical global queue namespace at `/queue`; `mode=worker` isolates each worker under `/queue/worker-`. - `db_path` is only used when QueueFS backend is `sqlite` or `sqlite3`. +- `backend=cache` automatically binds the global `cache.provider + cache.params` configuration. +- Redis Cluster slot routing, topology refresh, Sentinel discovery, and reconnects are handled by the Fred RedisProvider. +- QueueFS cache keys use `{cache_key_prefix}:ov:*`; use different prefixes for deployments or tenants sharing one Redis cluster. - Redis backend runs three bounded `recover_stale` sweeps in a dedicated startup recovery thread at startup, 30 seconds, and 60 seconds to cover the heartbeat-expiry window after a container restart; it does not run long-lived periodic recovery. - If both `storage.agfs.queuefs.db_path` and legacy `storage.agfs.queue_db_path` are set, `storage.agfs.queuefs.db_path` wins. - If QueueFS backend is `memory`, any `db_path` or legacy `queue_db_path` is ignored. diff --git a/docs/en/guides/14-ragfs-cache.md b/docs/en/guides/14-ragfs-cache.md index d9fe972651..9e944f4a5d 100644 --- a/docs/en/guides/14-ragfs-cache.md +++ b/docs/en/guides/14-ragfs-cache.md @@ -18,30 +18,31 @@ openviking-server init openviking-server doctor ``` -Then enable the cache under `storage.agfs.cache` in `~/.openviking/ov.conf`. The following Redis example is a good quick validation setup: +Configure the global top-level `cache` Provider, then select `backend=cache` under `storage.agfs.cachefs`: ```json { + "cache": { + "provider": "redis", + "params": { + "mode": "standalone", + "endpoints": ["redis://127.0.0.1:6379"], + "pool_size": 32, + "connect_timeout_ms": 1000, + "command_timeout_ms": 1000, + "default_ttl_seconds": 3600, + "read_from_replica": false + } + }, "storage": { "workspace": "./data", "agfs": { "backend": "local", - "cache": { - "enabled": true, - "provider": "redis", + "cachefs": { + "backend": "cache", "namespace": "openviking", "max_file_size_bytes": 1048576, - "bypass_prefixes": ["/queue", "/tmp"], - "redis": { - "mode": "standalone", - "endpoints": ["redis://127.0.0.1:6379"], - "pool_size": 32, - "connect_timeout_ms": 1000, - "command_timeout_ms": 20, - "key_prefix": "", - "default_ttl_seconds": 3600, - "read_from_replica": false - } + "bypass_prefixes": ["/queue", "/tmp"] } } } @@ -65,14 +66,14 @@ Available Providers: | Provider | Best for | Notes | |----------|----------|-------| -| `redis` | Default delivery on standard networks | Built into RAGFS; currently supports standalone and reads from primary only | -| `dynamic` | YuanRong, Mooncake, or closed-source cache systems | Loads an external Provider `.so` through a versioned C ABI | +| `redis` | Default delivery on standard networks | Built into RAGFS; supports standalone, Cluster, and Sentinel | +| `dynamic` | YuanRong, Mooncake, or closed-source cache systems | Not implemented in this release; returns UnsupportedProvider | `MemoryMockProvider` is only used by unit and smoke tests; it is not a production configuration option. -## Dynamic Provider Delivery +## Future DynamicProvider -The standard OpenViking wheel contains only the built-in Redis Provider and the DynamicProvider loader. Source code and SDKs for external Providers such as YuanRong and Mooncake are not distributed in the OpenViking repository. The Provider publisher builds and releases the `.so` independently. +This release only ships the built-in RedisProvider. DynamicProvider, `.so` loading, and the versioned C ABI are deferred; configuring `provider=dynamic` currently returns UnsupportedProvider during startup. A dynamic library must export this versioned entry point: @@ -86,12 +87,18 @@ External Providers can be upgraded independently without rebuilding the default ## Configuration -`storage.agfs.cache` supports these common options: +The top-level `cache` section is a sibling of `storage`: + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `provider` | str | none | Provider name; this release supports `redis` | +| `params` | object | `{}` | Provider-owned parameters | + +`storage.agfs.cachefs` controls CacheFS behavior: | Option | Type | Default | Description | |--------|------|---------|-------------| -| `enabled` | bool | `false` | Enable the RAGFS cache | -| `provider` | str | `"redis"` | `redis` or `dynamic` | +| `backend` | str | `"local"` | `local` preserves existing behavior; `cache` enables CachedFileSystem | | `namespace` | str | `"openviking"` | Cache namespace for isolating deployments or tenants | | `max_file_size_bytes` | int | `1048576` | Maximum full-file object size admitted to cache | | `traversal_mode` | str | `"backend"` | Use backend traversal or `cached_traversal` for recursive APIs | @@ -110,27 +117,20 @@ Redis configuration: | `command_timeout_ms` | `20` | Command timeout | | `key_prefix` | `""` | Reserved compatibility field; the unified Runtime requires an empty value | | `default_ttl_seconds` | `3600` | Default TTL; `0` means no TTL | -| `read_from_replica` | `false` | Must be `false` in standalone mode | +| `read_from_replica` | `false` | Supported only in Cluster mode; enable according to CacheFS consistency requirements | -DynamicProvider configuration example: +Future DynamicProvider configuration shape: -`dynamic.params` is entirely Provider-owned. The fields below only illustrate configuration forwarding; use the schema documented by the Provider publisher. +`cache.params` is entirely Provider-owned. The fields below only illustrate configuration forwarding; use the schema documented by the Provider publisher. ```json { - "storage": { - "agfs": { - "cache": { - "enabled": true, - "provider": "dynamic", - "dynamic": { - "library": "/opt/openviking/providers/libopenviking_cache_provider.so", - "params": { - "endpoint": "127.0.0.1:31501", - "request_timeout_ms": 5000 - } - } - } + "cache": { + "provider": "dynamic", + "params": { + "library": "/opt/openviking/providers/libopenviking_cache_provider.so", + "endpoint": "127.0.0.1:31501", + "request_timeout_ms": 5000 } } } @@ -141,7 +141,7 @@ DynamicProvider configuration example: RAGFS splits caching into two layers: - `CachedFileSystem`: implements filesystem semantics, including cache hit/miss handling, backend fallback, cache fill, invalidation, generation checks, and metrics. -- `CacheRuntime`: exposes common primitive operations and binds either the built-in RedisProvider or an external DynamicProvider at startup. +- `CacheRuntime`: exposes common primitive operations and binds the built-in RedisProvider in this release; DynamicProvider remains a future extension. Call flow: @@ -150,7 +150,7 @@ OpenViking -> RAGFS / MountableFS -> CachedFileSystem |-> CacheRuntime -> RedisProvider - | `-> DynamicProvider -> external provider .so + | `-> DynamicProvider (future) `-> Backend FileSystem ``` diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index 4c589392ad..6ab34e2740 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1111,18 +1111,71 @@ RAGFS 默认使用 Rust binding 模式，通过 Rust 实现直接访问文件系 更多配置示例见 [多写存储指南](./13-multi-write-storage.md)。 +##### 全局 Cache Provider 与 CacheFS 配置 + +全局 `cache` 与 `storage` 并列，标准配置只包含 Provider 名称和 Provider 自有参数： + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `provider` | str | 全局 Cache Provider；本期支持 `redis` | 必填 | +| `params` | object | Provider 自有参数；当 `provider=redis` 时解析为 Redis 连接参数 | `{}` | + +`storage.agfs.cachefs` 只控制 CacheFS 业务行为： + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `backend` | str | `local` 完全沿用原文件系统；`cache` 启用 CacheFS wrapper | `local` | +| `namespace` | str | CacheFS key 命名空间 | `openviking` | +| `max_file_size_bytes` | int | 允许缓存的单文件最大字节数 | `1048576` | +| `traversal_mode` | str | `backend` 或 `cached_traversal` | `backend` | +| `bypass_prefixes` | array[str] | 绕过缓存的路径前缀 | `[]` | + +```json +{ + "cache": { + "provider": "redis", + "params": { + "mode": "sentinel", + "endpoints": [ + "redis://sentinel-1:26379", + "redis://sentinel-2:26379" + ], + "master_name": "mymaster", + "password_env": "OPENVIKING_REDIS_PASSWORD", + "connect_timeout_ms": 1000, + "command_timeout_ms": 1000 + } + }, + "storage": { + "agfs": { + "cachefs": { + "backend": "cache", + "namespace": "production" + }, + "queuefs": { + "backend": "cache", + "cache_key_prefix": "production" + } + } + } +} +``` + +标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache` 时初始化 CacheRuntime；全部模块使用本地 backend 时不解析 `cache.params`，也不连接 Provider。 + ##### QueueFS 配置 | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| | `mode` | str | QueueFS 命名空间模式：`"shared"` 使用 `/queue`；`"worker"` 为每个 worker 隔离到 `/queue/worker-` | `"shared"` | -| `backend` | str | QueueFS 后端：`"memory"`、`"sqlite"`、`"sqlite3"` 或 `"redis"` | `"sqlite"` | +| `backend` | str | QueueFS 后端：`"memory"`、`"sqlite"`、`"sqlite3"` 或 `"cache"`；`"redis"` 为旧配置兼容值 | `"sqlite"` | | `db_path` | str（可选） | 当 backend 为 `"sqlite"` 或 `"sqlite3"` 时使用的 QueueFS sqlite 数据库路径 | `null` | | `recover_stale_sec` | int | 启动时恢复超过该秒数的 `processing` 队列消息；`0` 表示恢复全部 stale processing 消息 | `0` | | `busy_timeout_ms` | int | QueueFS sqlite 的 busy timeout，单位毫秒 | `5000` | -| `redis` | object | 当 backend 为 `"redis"` 时使用的连接参数 | 见下表 | +| `cache_key_prefix` | str | 当 backend 为 `"cache"` 时使用的 QueueFS key 命名空间 | `"default"` | +| `redis` | object | 旧 `backend=redis` 的兼容连接参数；新配置使用顶层 `cache.params` | 见下表 | -QueueFS Redis 参数： +旧 QueueFS Redis 参数（仅兼容）： | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| @@ -1147,11 +1200,11 @@ QueueFS Redis 参数： - `db_path` 仅在 QueueFS backend 为 `sqlite` 或 `sqlite3` 时生效。 - `recover_stale_sec` 和 `busy_timeout_ms` 仅在 QueueFS backend 为 `sqlite` 或 `sqlite3` 时生效。 - Redis Singleton 模式必须且只能配置一个 endpoint。 -- Redis Cluster 模式的 endpoints 是初始节点，且必须配置 `db=0`；slot 路由、`MOVED`/`ASK` 处理和节点重连由 redis-rs 完成。 -- Redis Sentinel 模式的 endpoints 是 Sentinel 节点，并且必须配置非空 `master_name`；master 发现和故障切换后的重连由 redis-rs 完成。 -- Redis Sentinel 模式下，`connect_timeout_ms` 作用于发现 Master 后的数据节点连接；redis-rs 同步 Sentinel discovery 不暴露物理建连 timeout，该阶段由内部固定 5 秒的 pool checkout timeout 限制调用方等待。 +- 新配置使用 `queuefs.backend=cache`，并自动绑定顶层 `cache.provider + cache.params`。 +- Redis Cluster 模式的 endpoints 是初始节点，且必须配置 `db=0`；slot 路由、`MOVED`/`ASK`、拓扑更新和重连由 Fred RedisProvider 处理。 +- Redis Sentinel 模式的 endpoints 是 Sentinel 节点，并且必须配置非空 `master_name`；master 发现和故障切换后的重连由 Fred RedisProvider 处理。 - `username` 和 `password` 用于 Redis 数据节点；`sentinel_username` 和 `sentinel_password` 仅用于 Sentinel 节点。 -- Redis backend 使用 `{key_prefix}:ov:*` key；连接同一 Redis database 的不同业务必须配置不同的 `key_prefix`。 +- Cache backend 使用 `{cache_key_prefix}:ov:*` key；连接同一 Redis 集群的不同环境或租户必须配置不同的 `cache_key_prefix`。 - Redis backend 的实例心跳 TTL 为 30 秒，每 10 秒续约一次。 - Redis backend 会在独立的 startup recovery 线程中按实例心跳状态执行三次有界 `recover_stale` 扫描，时间点分别为启动后立即、30 秒和 60 秒，用于覆盖容器异常退出后旧实例心跳尚未过期的恢复窗口；运行期间不做长期周期恢复。 - `tls_insecure_skip_verify=true` 时必须同时设置 `tls_enabled=true`。 @@ -1176,7 +1229,7 @@ QueueFS Redis 参数： } ``` -Redis QueueFS 配置示例： +旧 Redis QueueFS 兼容配置示例（新配置不推荐使用）： ```json { @@ -1208,7 +1261,7 @@ Redis QueueFS 配置示例： } ``` -Redis Cluster 只需配置可用于发现拓扑的初始节点： +旧 Redis QueueFS Cluster 兼容配置： ```json { @@ -1234,7 +1287,7 @@ Redis Cluster 只需配置可用于发现拓扑的初始节点： } ``` -Redis Sentinel 分别配置数据节点和 Sentinel 的 ACL： +旧 Redis QueueFS Sentinel 兼容配置： ```json { diff --git a/docs/zh/guides/14-ragfs-cache.md b/docs/zh/guides/14-ragfs-cache.md index 75d4e7af7b..4ee3a55040 100644 --- a/docs/zh/guides/14-ragfs-cache.md +++ b/docs/zh/guides/14-ragfs-cache.md @@ -18,30 +18,31 @@ openviking-server init openviking-server doctor ``` -然后在 `~/.openviking/ov.conf` 的 `storage.agfs.cache` 中启用缓存。下面是 Redis 示例，适合快速验证： +然后在 `~/.openviking/ov.conf` 中配置顶层 `cache` Provider，并在 `storage.agfs.cachefs` 选择 `backend=cache`： ```json { + "cache": { + "provider": "redis", + "params": { + "mode": "standalone", + "endpoints": ["redis://127.0.0.1:6379"], + "pool_size": 32, + "connect_timeout_ms": 1000, + "command_timeout_ms": 1000, + "default_ttl_seconds": 3600, + "read_from_replica": false + } + }, "storage": { "workspace": "./data", "agfs": { "backend": "local", - "cache": { - "enabled": true, - "provider": "redis", + "cachefs": { + "backend": "cache", "namespace": "openviking", "max_file_size_bytes": 1048576, - "bypass_prefixes": ["/queue", "/tmp"], - "redis": { - "mode": "standalone", - "endpoints": ["redis://127.0.0.1:6379"], - "pool_size": 32, - "connect_timeout_ms": 1000, - "command_timeout_ms": 20, - "key_prefix": "", - "default_ttl_seconds": 3600, - "read_from_replica": false - } + "bypass_prefixes": ["/queue", "/tmp"] } } } @@ -65,14 +66,14 @@ openviking-server | Provider | 适用场景 | 备注 | |----------|----------|------| -| `redis` | 默认交付、普通网络环境 | 内置于 RAGFS，当前支持 standalone，只从 primary 读取 | -| `dynamic` | YuanRong、Mooncake 或闭源缓存系统 | 通过版本化 C ABI 加载外部 Provider `.so` | +| `redis` | 默认交付、普通网络环境 | 内置于 RAGFS，支持 standalone、Cluster 和 Sentinel | +| `dynamic` | YuanRong、Mooncake 或闭源缓存系统 | 本期未实现，配置后返回 UnsupportedProvider | `MemoryMockProvider` 只用于单元测试和 smoke test，不是生产配置项。 -## 动态 Provider 发布 +## 后续 DynamicProvider -标准 OpenViking wheel 只内置 Redis Provider 和 DynamicProvider 加载能力。YuanRong、Mooncake 等外部 Provider 的构建源码和 SDK 不随 OpenViking 主仓发布，由 Provider 发布方在独立环境中构建并交付 `.so`。 +本期标准 OpenViking wheel 只内置 RedisProvider。DynamicProvider、`.so` 加载器和版本化 C ABI 放在后续阶段实现；当前配置 `provider=dynamic` 会在启动阶段返回 UnsupportedProvider。 动态库必须导出以下版本化入口： @@ -86,12 +87,18 @@ Provider 发布物应注明 ABI 版本、目标 OS/CPU、最低 glibc 版本、 ## 配置项 -`storage.agfs.cache` 支持以下通用配置： +顶层 `cache` 与 `storage` 并列： + +| 参数 | 类型 | 默认值 | 说明 | +|------|------|--------|------| +| `provider` | str | 无 | Provider 名称，本期支持 `redis` | +| `params` | object | `{}` | Provider 自有参数 | + +`storage.agfs.cachefs` 支持以下业务配置： | 参数 | 类型 | 默认值 | 说明 | |------|------|--------|------| -| `enabled` | bool | `false` | 是否启用 RAGFS 缓存 | -| `provider` | str | `"redis"` | `redis` 或 `dynamic` | +| `backend` | str | `"local"` | `local` 沿用原逻辑；`cache` 启用 CachedFileSystem | | `namespace` | str | `"openviking"` | 缓存命名空间，用于隔离不同部署或租户 | | `max_file_size_bytes` | int | `1048576` | 允许进入缓存的最大完整文件大小 | | `traversal_mode` | str | `"backend"` | 递归 API 使用 backend 遍历或 `cached_traversal` | @@ -110,27 +117,20 @@ Redis 配置： | `command_timeout_ms` | `20` | 命令超时 | | `key_prefix` | `""` | 保留兼容字段；统一 Runtime 要求为空 | | `default_ttl_seconds` | `3600` | 默认 TTL；`0` 表示不设置 TTL | -| `read_from_replica` | `false` | standalone 模式下必须为 `false` | +| `read_from_replica` | `false` | 仅 Cluster 模式支持；CacheFS 可按一致性要求选择是否开启 | -DynamicProvider 配置示例： +未来 DynamicProvider 配置结构： -`dynamic.params` 完全由外部 Provider 定义，下面的字段只用于说明配置传递方式，实际配置以 Provider 发布说明为准。 +`cache.params` 完全由外部 Provider 定义，下面的字段只用于说明配置传递方式，实际配置以 Provider 发布说明为准。 ```json { - "storage": { - "agfs": { - "cache": { - "enabled": true, - "provider": "dynamic", - "dynamic": { - "library": "/opt/openviking/providers/libopenviking_cache_provider.so", - "params": { - "endpoint": "127.0.0.1:31501", - "request_timeout_ms": 5000 - } - } - } + "cache": { + "provider": "dynamic", + "params": { + "library": "/opt/openviking/providers/libopenviking_cache_provider.so", + "endpoint": "127.0.0.1:31501", + "request_timeout_ms": 5000 } } } @@ -141,7 +141,7 @@ DynamicProvider 配置示例： RAGFS 将缓存拆成两层： - `CachedFileSystem`：实现文件系统语义，包括 cache hit/miss、backend 回源、回填、失效、generation 校验和指标。 -- `CacheRuntime`：向业务层提供统一基础操作，并在启动时绑定内置 RedisProvider 或外部 DynamicProvider。 +- `CacheRuntime`：向业务层提供统一基础操作，本期在启动时绑定内置 RedisProvider；DynamicProvider 为后续扩展。 调用关系： @@ -150,7 +150,7 @@ OpenViking -> RAGFS / MountableFS -> CachedFileSystem |-> CacheRuntime -> RedisProvider - | `-> DynamicProvider -> external provider .so + | `-> DynamicProvider（后续） `-> Backend FileSystem ``` diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py index 9cf2dc33a9..d10dc8123e 100644 --- a/openviking/utils/agfs_utils.py +++ b/openviking/utils/agfs_utils.py @@ -24,6 +24,7 @@ class RagfsBindingConfig: """Single binding config object for both stack construction and backend mount setup.""" agfs: Any + cache: Any | None = None root_key: bytes | None = None provider_type: int | None = None log: Dict[str, Any] | None = None @@ -34,9 +35,57 @@ def encryption_enabled(self) -> bool: def to_binding_dict(self) -> Dict[str, Any]: """Convert the runtime config into the sectioned dict consumed by `RAGFSBindingClient`.""" - cache_config = self.agfs.cache.model_dump(mode="json") - if getattr(getattr(self.agfs, "queuefs", None), "backend", None) == "cache": - cache_config["runtime_enabled"] = True + queuefs = getattr(self.agfs, "queuefs", None) + queue_backend = getattr(queuefs, "backend", None) + cachefs = getattr(self.agfs, "cachefs", None) + cachefs_backend = getattr(cachefs, "backend", "local") + uses_runtime = cachefs_backend == "cache" or queue_backend in {"cache", "redis"} + legacy_cache = getattr(self.agfs, "cache", None) + legacy_fields_set = set(getattr(legacy_cache, "model_fields_set", set())) + + provider_explicit = False + redis_explicit: set[str] = set() + if self.cache is not None and uses_runtime: + if legacy_fields_set: + raise ValueError( + "top-level cache config conflicts with deprecated storage.agfs.cache" + ) + cache_config, redis_explicit = _build_provider_cache_config( + self.cache, + cachefs, + cachefs_backend == "cache", + uses_runtime, + ) + provider_explicit = True + elif self.cache is not None: + cache_config = _disabled_cache_config(cachefs) + else: + cache_config = ( + legacy_cache.model_dump(mode="json") + if legacy_cache is not None + else _disabled_cache_config(cachefs) + ) + if cachefs_backend == "cache": + if legacy_cache is None or not legacy_fields_set: + raise ValueError( + "top-level cache config is required when cachefs backend=cache" + ) + cache_config["enabled"] = True + if uses_runtime: + cache_config["runtime_enabled"] = True + if legacy_cache is not None: + provider_explicit = "provider" in legacy_fields_set + legacy_redis = getattr(legacy_cache, "redis", None) + redis_explicit = set(getattr(legacy_redis, "model_fields_set", set())) + + if queue_backend == "redis": + _merge_legacy_queuefs_redis_config( + cache_config, + provider=cache_config.get("provider", "redis"), + provider_explicit=provider_explicit, + redis_explicit=redis_explicit, + queuefs_model=queuefs, + ) binding_config: Dict[str, Any] = { "cache": cache_config, "pathlock": self.agfs.pathlock.model_dump(mode="json"), @@ -58,6 +107,101 @@ def to_binding_dict(self) -> Dict[str, Any]: return binding_config +def _disabled_cache_config(cachefs_model: Any) -> Dict[str, Any]: + traversal_mode = getattr(cachefs_model, "traversal_mode", "backend") + return { + "enabled": False, + "runtime_enabled": False, + "provider": "redis", + "namespace": getattr(cachefs_model, "namespace", "openviking"), + "max_file_size_bytes": getattr(cachefs_model, "max_file_size_bytes", 1024 * 1024), + "traversal_mode": getattr(traversal_mode, "value", traversal_mode), + "bypass_prefixes": list(getattr(cachefs_model, "bypass_prefixes", [])), + } + + +def _build_provider_cache_config( + provider_config: Any, + cachefs_model: Any, + cachefs_enabled: bool, + runtime_enabled: bool, +) -> tuple[Dict[str, Any], set[str]]: + provider = provider_config.provider.strip() + params = dict(provider_config.params) + cache_config = _disabled_cache_config(cachefs_model) + cache_config.update( + { + "enabled": cachefs_enabled, + "runtime_enabled": runtime_enabled, + "provider": provider, + } + ) + if provider == "redis": + from openviking_cli.utils.config.agfs_config import RedisCacheConfig + + redis = RedisCacheConfig.model_validate(params) + cache_config["redis"] = redis.model_dump(mode="json") + return cache_config, set(redis.model_fields_set) + if provider == "dynamic": + cache_config["dynamic"] = params + return cache_config, set() + + +def _merge_legacy_queuefs_redis_config( + cache_config: Dict[str, Any], + *, + provider: str, + provider_explicit: bool, + redis_explicit: set[str], + queuefs_model: Any, +) -> None: + """Normalize the legacy QueueFS Redis connection into the shared Runtime config.""" + if provider_explicit and provider != "redis": + raise ValueError("queuefs backend=redis conflicts with the global cache provider") + + cache_config["provider"] = "redis" + target = cache_config.setdefault("redis", {}) + explicit = redis_explicit + legacy_explicit = queuefs_model.redis.model_fields_set + legacy = queuefs_model.redis.model_dump(mode="json") + legacy["mode"] = "standalone" if legacy["mode"] == "singleton" else legacy["mode"] + + nullable_strings = { + "master_name", + "username", + "password", + "sentinel_username", + "sentinel_password", + } + fields = ( + "mode", + "endpoints", + "master_name", + "username", + "password", + "sentinel_username", + "sentinel_password", + "db", + "connect_timeout_ms", + "command_timeout_ms", + "tls_enabled", + "tls_insecure_skip_verify", + ) + for field in fields: + value = legacy[field] + if field in nullable_strings and value is None: + value = "" if field != "master_name" else None + if field in explicit and field in legacy_explicit and target.get(field) != value: + raise ValueError(f"conflicting Redis setting: {field}") + if field in legacy_explicit or field not in explicit: + target[field] = value + + if legacy.get("password") and target.get("password_env"): + raise ValueError("conflicting Redis setting: password") + if legacy.get("sentinel_password") and target.get("sentinel_password_env"): + raise ValueError("conflicting Redis setting: sentinel_password") + + def _run_coro_blocking(coro: Any) -> Any: """Run an async coroutine from sync startup code, even if an event loop is already running.""" try: @@ -105,6 +249,7 @@ def build_runtime_ragfs_binding_config(config: Any) -> tuple[RagfsBindingConfig, agfs_config = _get_config_value(storage, "agfs") if storage is not None else None if agfs_config is None: raise ValueError("OpenViking config storage.agfs is required") + cache_config = _get_config_value(config, "cache") log_config = _get_config_value(config, "log") log_level = _get_config_value(log_config, "level", "INFO") @@ -122,7 +267,11 @@ def build_runtime_ragfs_binding_config(config: Any) -> tuple[RagfsBindingConfig, encryptor = _run_coro_blocking(bootstrap_encryption(_dump_openviking_config(config))) if encryptor is None: - return RagfsBindingConfig(agfs=agfs_config, log=binding_log), None + return RagfsBindingConfig( + agfs=agfs_config, + cache=cache_config, + log=binding_log, + ), None root_key = _run_coro_blocking(encryptor.provider.get_root_key()) if not isinstance(root_key, (bytes, bytearray)) or len(root_key) != 32: @@ -131,6 +280,7 @@ def build_runtime_ragfs_binding_config(config: Any) -> tuple[RagfsBindingConfig, return ( RagfsBindingConfig( agfs=agfs_config, + cache=cache_config, root_key=bytes(root_key), provider_type=encryptor.provider_type, log=binding_log, diff --git a/openviking_cli/utils/config/__init__.py b/openviking_cli/utils/config/__init__.py index b84173610a..7ca257f71e 100644 --- a/openviking_cli/utils/config/__init__.py +++ b/openviking_cli/utils/config/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0 from . import embedding_config from .agfs_config import AGFSConfig +from .cache_config import CacheConfig from .config_loader import ( load_json_config, require_config, @@ -88,6 +89,7 @@ __all__ = [ "AGFSConfig", + "CacheConfig", "SYSTEM_CONFIG_DIR", "DEFAULT_OV_CONF", "DEFAULT_OVCLI_CONF", diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py index e37299ca4f..10b668778a 100644 --- a/openviking_cli/utils/config/agfs_config.py +++ b/openviking_cli/utils/config/agfs_config.py @@ -261,6 +261,38 @@ class AGFSCacheTraversalMode(str, Enum): CACHED_TRAVERSAL = "cached_traversal" +class AGFSCacheFSConfig(BaseModel): + """CacheFS behavior independent from the selected global Provider.""" + + backend: Literal["local", "cache"] = Field( + default="local", + description="CacheFS backend: 'local' | 'cache'", + ) + namespace: str = Field(default="openviking", description="RAGFS cache namespace") + max_file_size_bytes: int = Field( + default=1024 * 1024, + description="Maximum full-file object size admitted to cache", + ) + traversal_mode: AGFSCacheTraversalMode = Field( + default=AGFSCacheTraversalMode.BACKEND, + description="Traversal strategy for tree, glob, and grep", + ) + bypass_prefixes: list[str] = Field( + default_factory=list, + description="Path prefixes that bypass cache", + ) + + model_config = {"extra": "forbid"} + + @model_validator(mode="after") + def validate_config(self): + if not self.namespace.strip(): + raise ValueError("cachefs namespace must not be empty") + if self.max_file_size_bytes <= 0: + raise ValueError("cachefs max_file_size_bytes must be > 0") + return self + + class DynamicCacheConfig(BaseModel): """Configuration passed to a versioned dynamic cache provider.""" @@ -280,6 +312,22 @@ class RedisCacheConfig(BaseModel): ) username: str = Field(default="", description="Redis ACL username") password_env: str = Field(default="", description="Environment variable containing password") + password: str = Field( + default="", + description="Legacy plaintext Redis password; prefer password_env", + repr=False, + ) + master_name: Optional[str] = Field(default=None, description="Sentinel master name") + sentinel_username: str = Field(default="", description="Sentinel ACL username") + sentinel_password_env: str = Field( + default="", description="Environment variable containing Sentinel password" + ) + sentinel_password: str = Field( + default="", + description="Legacy plaintext Sentinel password; prefer sentinel_password_env", + repr=False, + ) + db: int = Field(default=0, description="Redis database number") pool_size: int = Field(default=32, description="Redis command concurrency") connect_timeout_ms: int = Field(default=1000, description="Redis connect timeout") command_timeout_ms: int = Field(default=20, description="Redis command timeout") @@ -289,17 +337,50 @@ class RedisCacheConfig(BaseModel): ) default_ttl_seconds: int = Field(default=3600, description="Redis default cache TTL") read_from_replica: bool = Field(default=False, description="Read from Redis replicas") + tls_enabled: bool = Field(default=False, description="Enable Redis TLS") + tls_insecure_skip_verify: bool = Field( + default=False, description="Skip Redis TLS certificate verification" + ) model_config = {"extra": "forbid"} @model_validator(mode="after") def validate_config(self): - if self.mode != "standalone": - raise ValueError("redis mode must be standalone") + if self.mode == "singleton": + self.mode = "standalone" + if self.mode not in {"standalone", "cluster", "sentinel"}: + raise ValueError("redis mode must be standalone, cluster, or sentinel") if not self.endpoints: raise ValueError("redis endpoints must not be empty") - if any(not endpoint.strip() for endpoint in self.endpoints): - raise ValueError("redis endpoints must not contain empty values") + for endpoint in self.endpoints: + parsed = urlparse(endpoint) + if parsed.scheme not in {"redis", "rediss"} or not parsed.hostname: + raise ValueError("redis endpoints must use redis:// or rediss:// URLs") + try: + port = parsed.port + except ValueError as error: + raise ValueError("redis endpoint port is invalid") from error + if port == 0: + raise ValueError("redis endpoint port is invalid") + if ( + parsed.username is not None + or parsed.password is not None + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + ): + raise ValueError( + "redis endpoints must not include credentials, database paths, " + "query parameters, or fragments; use dedicated redis fields" + ) + if self.mode == "standalone" and len(self.endpoints) != 1: + raise ValueError("redis standalone mode requires exactly one endpoint") + if self.mode == "cluster" and self.db != 0: + raise ValueError("redis cluster mode requires db=0") + if self.mode == "sentinel" and not (self.master_name or "").strip(): + raise ValueError("redis sentinel mode requires master_name") + if self.db < 0 or self.db > 255: + raise ValueError("redis db must be between 0 and 255") if self.pool_size <= 0: raise ValueError("redis pool_size must be > 0") if self.connect_timeout_ms <= 0: @@ -308,8 +389,16 @@ def validate_config(self): raise ValueError("redis command_timeout_ms must be > 0") if self.default_ttl_seconds < 0: raise ValueError("redis default_ttl_seconds must be >= 0") - if self.read_from_replica: - raise ValueError("redis read_from_replica is not supported in standalone mode") + if self.read_from_replica and self.mode != "cluster": + raise ValueError("redis read_from_replica is only supported in cluster mode") + if self.password_env and self.password: + raise ValueError("redis password and password_env cannot both be configured") + if self.sentinel_password_env and self.sentinel_password: + raise ValueError( + "redis sentinel_password and sentinel_password_env cannot both be configured" + ) + if self.tls_insecure_skip_verify and not self.tls_enabled: + raise ValueError("redis tls_insecure_skip_verify requires tls_enabled=true") return self @@ -461,9 +550,15 @@ class AGFSConfig(BaseModel): description="QueueFS configuration.", ) + cachefs: AGFSCacheFSConfig = Field( + default_factory=AGFSCacheFSConfig, + description="CacheFS configuration.", + ) + cache: AGFSCacheConfig = Field( default_factory=AGFSCacheConfig, - description="RAGFS cache configuration.", + description="Deprecated nested cache and Provider configuration.", + exclude=True, ) pathlock: AGFSPathLockConfig = Field( diff --git a/openviking_cli/utils/config/cache_config.py b/openviking_cli/utils/config/cache_config.py new file mode 100644 index 0000000000..4030547576 --- /dev/null +++ b/openviking_cli/utils/config/cache_config.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +from typing import Any, Dict + +from pydantic import BaseModel, Field, model_validator + + +class CacheConfig(BaseModel): + """Global cache Provider configuration shared by RAGFS modules.""" + + provider: str = Field(description="Cache Provider name") + params: Dict[str, Any] = Field( + default_factory=dict, + description="Provider-owned configuration parameters", + repr=False, + ) + + model_config = {"extra": "forbid"} + + @model_validator(mode="after") + def validate_config(self): + if not self.provider.strip(): + raise ValueError("cache provider must not be empty") + return self diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py index 601c9eb422..e97706fe93 100644 --- a/openviking_cli/utils/config/open_viking_config.py +++ b/openviking_cli/utils/config/open_viking_config.py @@ -11,6 +11,7 @@ from openviking_cli.session.user_id import UserIdentifier +from .cache_config import CacheConfig from .config_loader import resolve_config_path from .config_utils import format_validation_error, raise_unknown_config_fields from .consts import ( @@ -147,6 +148,11 @@ class OpenVikingConfig(BaseModel): description="Deprecated and ignored. User is the only data-plane identity.", ) + cache: Optional[CacheConfig] = Field( + default=None, + description="Global cache Provider configuration", + ) + storage: StorageConfig = Field( default_factory=StorageConfig, description="Storage configuration" ) @@ -309,6 +315,15 @@ def _warn_on_deprecated_language_fallback(self) -> "OpenVikingConfig": ) return self + @model_validator(mode="after") + def _validate_cache_runtime_config(self) -> "OpenVikingConfig": + agfs = self.storage.agfs + uses_canonical_cache = agfs.cachefs.backend == "cache" or agfs.queuefs.backend == "cache" + has_legacy_cache = bool(agfs.cache.model_fields_set) + if uses_canonical_cache and self.cache is None and not has_legacy_cache: + raise ValueError("top-level cache config is required when an AGFS backend uses cache") + return self + @model_validator(mode="before") @classmethod def _inherit_git_defaults_from_agfs(cls, data: Any) -> Any: @@ -361,6 +376,93 @@ def _inherit_git_defaults_from_agfs(cls, data: Any) -> Any: data["git"] = git return data + @model_validator(mode="before") + @classmethod + def _normalize_legacy_cache_config(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + + normalized = dict(data) + storage_value = normalized.get("storage") + if not isinstance(storage_value, dict): + return normalized + storage = dict(storage_value) + agfs_value = storage.get("agfs") + if not isinstance(agfs_value, dict): + return normalized + agfs = dict(agfs_value) + + cache_value = normalized.get("cache") + canonical_cache = dict(cache_value) if isinstance(cache_value, dict) else None + + def merge_provider(provider: str, params: Dict[str, Any], source: str) -> None: + nonlocal canonical_cache + if canonical_cache is None: + canonical_cache = {"provider": provider, "params": dict(params)} + return + if canonical_cache.get("provider") != provider: + raise ValueError(f"{source} conflicts with top-level cache provider") + existing_params = canonical_cache.get("params", {}) + if not isinstance(existing_params, dict): + raise ValueError("top-level cache params must be an object") + merged_params = dict(existing_params) + for key, value in params.items(): + if key in merged_params and merged_params[key] != value: + raise ValueError(f"{source} conflicts with top-level cache params: {key}") + merged_params.setdefault(key, value) + canonical_cache["params"] = merged_params + + legacy_cache = agfs.pop("cache", None) + if isinstance(legacy_cache, dict): + cachefs_value = agfs.get("cachefs") + cachefs = dict(cachefs_value) if isinstance(cachefs_value, dict) else {} + cachefs.setdefault( + "backend", "cache" if legacy_cache.get("enabled", False) else "local" + ) + for field in ( + "namespace", + "max_file_size_bytes", + "traversal_mode", + "bypass_prefixes", + ): + if field in legacy_cache: + cachefs.setdefault(field, legacy_cache[field]) + agfs["cachefs"] = cachefs + + provider = str(legacy_cache.get("provider", "redis")) + provider_params = legacy_cache.get(provider, {}) + provider_was_configured = ( + bool(legacy_cache.get("enabled", False)) + or "provider" in legacy_cache + or provider in legacy_cache + ) + if provider_was_configured: + if not isinstance(provider_params, dict): + raise ValueError(f"storage.agfs.cache.{provider} must be an object") + merge_provider(provider, provider_params, "storage.agfs.cache") + + queuefs_value = agfs.get("queuefs") + if isinstance(queuefs_value, dict) and queuefs_value.get("backend") == "redis": + queuefs = dict(queuefs_value) + redis_value = queuefs.pop("redis", {}) + if not isinstance(redis_value, dict): + raise ValueError("storage.agfs.queuefs.redis must be an object") + redis_params = dict(redis_value) + key_prefix = redis_params.pop("key_prefix", None) + if redis_params.get("mode") == "singleton": + redis_params["mode"] = "standalone" + merge_provider("redis", redis_params, "storage.agfs.queuefs.redis") + queuefs["backend"] = "cache" + if key_prefix is not None: + queuefs.setdefault("cache_key_prefix", key_prefix) + agfs["queuefs"] = queuefs + + if canonical_cache is not None: + normalized["cache"] = canonical_cache + storage["agfs"] = agfs + normalized["storage"] = storage + return normalized + allow_private_networks: bool = Field( default=False, description=( diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py index d14c291514..6b8c6b6ff2 100644 --- a/tests/misc/test_config_validation.py +++ b/tests/misc/test_config_validation.py @@ -15,8 +15,10 @@ mount_agfs_backend, ) from openviking_cli.utils.config.agfs_config import AGFSConfig, S3Config +from openviking_cli.utils.config.cache_config import CacheConfig from openviking_cli.utils.config.consts import OPENVIKING_CONFIG_ENV from openviking_cli.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfig +from openviking_cli.utils.config.open_viking_config import OpenVikingConfig from openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig, VolcengineConfig from openviking_cli.utils.config.vlm_config import VLMConfig @@ -254,6 +256,312 @@ def test_agfs_cache_accepts_redis_provider_config(): assert config.cache.redis.default_ttl_seconds == 3600 +@pytest.mark.parametrize( + "redis_config", + [ + { + "mode": "cluster", + "endpoints": ["redis://cluster-1:6379", "redis://cluster-2:6379"], + "db": 0, + "read_from_replica": True, + }, + { + "mode": "sentinel", + "endpoints": ["redis://sentinel-1:26379", "redis://sentinel-2:26379"], + "master_name": "mymaster", + "sentinel_username": "sentinel-user", + "sentinel_password_env": "OV_SENTINEL_PASSWORD", + }, + ], +) +def test_agfs_cache_redis_accepts_high_availability_modes(redis_config): + config = AGFSConfig( + path="/tmp/ov-test", + backend="local", + cache={"enabled": True, "provider": "redis", "redis": redis_config}, + ) + + assert config.cache.redis.mode == redis_config["mode"] + assert config.cache.redis.endpoints == redis_config["endpoints"] + + +def test_top_level_cache_provider_params_build_redis_binding_config(): + config = OpenVikingConfig.from_dict( + { + "cache": { + "provider": "redis", + "params": { + "mode": "sentinel", + "endpoints": [ + "redis://sentinel-1:26379", + "redis://sentinel-2:26379", + ], + "master_name": "mymaster", + "command_timeout_ms": 1000, + }, + }, + "storage": { + "workspace": "/tmp/ov-test", + "agfs": {"cachefs": {"backend": "cache", "namespace": "tenant-a"}}, + }, + } + ) + + binding = RagfsBindingConfig( + agfs=config.storage.agfs, + cache=config.cache, + ).to_binding_dict() + + assert config.cache.provider == "redis" + assert config.cache.params["mode"] == "sentinel" + assert binding["cache"]["enabled"] is True + assert binding["cache"]["runtime_enabled"] is True + assert binding["cache"]["provider"] == "redis" + assert binding["cache"]["namespace"] == "tenant-a" + assert binding["cache"]["redis"]["mode"] == "sentinel" + assert binding["cache"]["redis"]["master_name"] == "mymaster" + + +def test_queuefs_cache_uses_top_level_provider_without_enabling_cachefs(): + config = OpenVikingConfig.model_validate( + { + "cache": { + "provider": "redis", + "params": {"endpoints": ["redis://redis:6379"]}, + }, + "storage": { + "workspace": "/tmp/ov-test", + "agfs": { + "cachefs": {"backend": "local"}, + "queuefs": { + "backend": "cache", + "cache_key_prefix": "queue-runtime", + }, + }, + }, + } + ) + + binding = RagfsBindingConfig( + agfs=config.storage.agfs, + cache=config.cache, + ).to_binding_dict() + + assert binding["cache"]["enabled"] is False + assert binding["cache"]["runtime_enabled"] is True + assert binding["cache"]["redis"]["endpoints"] == ["redis://redis:6379"] + + +def test_top_level_cache_params_preserve_cluster_replica_read_config(): + config = OpenVikingConfig.model_validate( + { + "cache": { + "provider": "redis", + "params": { + "mode": "cluster", + "endpoints": [ + "redis://cluster-1:6379", + "redis://cluster-2:6379", + ], + "db": 0, + "read_from_replica": True, + }, + }, + "storage": {"agfs": {"queuefs": {"backend": "cache"}}}, + } + ) + + binding = RagfsBindingConfig( + agfs=config.storage.agfs, + cache=config.cache, + ).to_binding_dict() + + assert binding["cache"]["redis"]["mode"] == "cluster" + assert binding["cache"]["redis"]["read_from_replica"] is True + + +def test_cache_backend_requires_top_level_cache_config(): + with pytest.raises(ValueError, match="top-level cache config"): + OpenVikingConfig.model_validate( + { + "storage": { + "workspace": "/tmp/ov-test", + "agfs": {"cachefs": {"backend": "cache"}}, + } + } + ) + + +def test_top_level_cache_rejects_global_enabled_flag(): + with pytest.raises(ValueError, match="enabled"): + OpenVikingConfig.model_validate( + { + "cache": { + "enabled": True, + "provider": "redis", + "params": {}, + } + } + ) + + +def test_top_level_cache_repr_hides_provider_params(): + config = CacheConfig( + provider="redis", + params={"password": "top-secret"}, + ) + + assert "top-secret" not in repr(config) + + +def test_unused_top_level_cache_does_not_parse_provider_params(): + config = OpenVikingConfig.model_validate( + { + "cache": { + "provider": "future-provider", + "params": {"provider_owned": True}, + } + } + ) + + binding = RagfsBindingConfig( + agfs=config.storage.agfs, + cache=config.cache, + ).to_binding_dict() + + assert binding["cache"]["enabled"] is False + assert binding["cache"]["runtime_enabled"] is False + assert binding["cache"]["provider"] == "redis" + + +def test_top_level_cache_rejects_deprecated_nested_cache_conflict(): + agfs = AGFSConfig( + cachefs={"backend": "cache"}, + cache={"enabled": True, "provider": "redis"}, + ) + cache = CacheConfig(provider="redis", params={}) + + with pytest.raises(ValueError, match="conflicts with deprecated storage.agfs.cache"): + RagfsBindingConfig( + agfs=agfs, + cache=cache, + ).to_binding_dict() + + +def test_openviking_config_dump_omits_deprecated_nested_cache(): + config = OpenVikingConfig.model_validate( + { + "cache": {"provider": "redis", "params": {}}, + "storage": {"agfs": {"cachefs": {"backend": "cache"}}}, + } + ) + + dumped = config.model_dump(mode="json") + + assert dumped["cache"] == {"provider": "redis", "params": {}} + assert dumped["storage"]["agfs"]["cachefs"]["backend"] == "cache" + assert "cache" not in dumped["storage"]["agfs"] + + +def test_openviking_config_migrates_legacy_nested_cache_to_canonical_shape(): + config = OpenVikingConfig.from_dict( + { + "storage": { + "agfs": { + "cache": { + "enabled": True, + "provider": "redis", + "namespace": "legacy-cache", + "redis": { + "mode": "sentinel", + "endpoints": ["redis://sentinel:26379"], + "master_name": "mymaster", + }, + } + } + } + } + ) + + assert config.cache.provider == "redis" + assert config.cache.params["mode"] == "sentinel" + assert config.storage.agfs.cachefs.backend == "cache" + assert config.storage.agfs.cachefs.namespace == "legacy-cache" + assert "cache" not in config.model_dump(mode="json")["storage"]["agfs"] + + +def test_openviking_config_migrates_legacy_queuefs_redis_to_canonical_shape(): + config = OpenVikingConfig.from_dict( + { + "storage": { + "agfs": { + "queuefs": { + "backend": "redis", + "redis": { + "mode": "singleton", + "endpoints": ["redis://redis:6379"], + "key_prefix": "legacy-queue", + }, + } + } + } + } + ) + + assert config.cache.provider == "redis" + assert config.cache.params["mode"] == "standalone" + assert config.storage.agfs.queuefs.backend == "cache" + assert config.storage.agfs.queuefs.cache_key_prefix == "legacy-queue" + + +def test_binding_config_migrates_legacy_queuefs_redis_to_cache_runtime(): + config = AGFSConfig( + path="/tmp/ov-test", + backend="local", + queuefs={ + "backend": "redis", + "redis": { + "mode": "singleton", + "endpoints": ["redis://redis.example.com:6379"], + "username": "queue-user", + "password": "legacy-secret", + "db": 2, + "connect_timeout_ms": 1500, + "command_timeout_ms": 2500, + "key_prefix": "tenant-a", + }, + }, + ) + + binding = RagfsBindingConfig(config).to_binding_dict() + + assert binding["cache"]["runtime_enabled"] is True + assert binding["cache"]["provider"] == "redis" + assert binding["cache"]["redis"]["mode"] == "standalone" + assert binding["cache"]["redis"]["endpoints"] == ["redis://redis.example.com:6379"] + assert binding["cache"]["redis"]["username"] == "queue-user" + assert binding["cache"]["redis"]["password"] == "legacy-secret" + assert binding["cache"]["redis"]["db"] == 2 + + +def test_binding_config_rejects_conflicting_legacy_and_global_redis_settings(): + config = AGFSConfig( + path="/tmp/ov-test", + backend="local", + cache={ + "provider": "redis", + "redis": {"endpoints": ["redis://global.example.com:6379"]}, + }, + queuefs={ + "backend": "redis", + "redis": {"endpoints": ["redis://legacy.example.com:6379"]}, + }, + ) + + with pytest.raises(ValueError, match="conflicting Redis setting: endpoints"): + RagfsBindingConfig(config).to_binding_dict() + + def test_agfs_cache_rejects_redis_provider_key_prefix_when_enabled(): with pytest.raises(ValueError, match="key_prefix"): AGFSConfig( From 29334f4da6533971a8d370557180a44bb7eafb76 Mon Sep 17 00:00:00 2001 From: chrisfang Date: Thu, 27 Aug 2026 16:20:35 +0800 Subject: [PATCH 4/7] chore: sync contributing guide with main --- CONTRIBUTING.md | 662 +++++++++++++----------------------------------- 1 file changed, 181 insertions(+), 481 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32204bd9a6..37a7c5040a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,568 +1,268 @@ -# Contributing Guide +# Contributing to OpenViking -Thank you for your interest in OpenViking! We welcome contributions of all kinds: +English / [中文](CONTRIBUTING_CN.md) / [日本語](CONTRIBUTING_JA.md) -- Bug reports -- Feature requests -- Documentation improvements -- Code contributions +Thank you for contributing to OpenViking. This guide exists to help contributors +submit changes that are clear, focused, and practical to review. ---- +We welcome bug reports, feature requests, documentation improvements, and code +contributions. -## Development Setup +## What We Value -### Prerequisites +OpenViking values focused, well-understood changes. Contributors are responsible +for understanding, explaining, and validating their changes, whether or not AI +tools were used. -- **Python**: 3.10+ -- **Go**: 1.22+ (Required only for Go SDK development under `sdk/go`) -- **Rust**: 1.91.1+ (Required for source builds because the bundled `ov` CLI is built during packaging) -- **C++ Compiler**: GCC 9+ or Clang 11+ (Required for building core extensions, must support C++17) -- **CMake**: 3.15+ +Prefer the smallest complete change. Concise code means fewer concepts, branches, +duplicated rules, and speculative abstractions—not fewer necessary lines. A good +change is direct, readable, and easy to explain from its entrypoint to its +observable behavior. -#### Platform-Specific Native Build Tools +In practice: -- **Linux**: Install `build-essential`; some environments may also require `pkg-config` -- **macOS**: Install Xcode Command Line Tools (`xcode-select --install`) -- **Windows**: Install CMake and MinGW for local native builds +- Solve one coherent problem per PR. Do not mix unrelated cleanup or refactoring. +- Reuse the existing owner of a rule instead of introducing a parallel mechanism. +- Avoid speculative fallbacks, flags, state fields, and abstractions. +- Remove code, tests, and compatibility paths that the new implementation replaces. +- Keep necessary structure when it makes ownership, lifecycle, or failure handling + clearer. -#### Supported Platforms (Pre-compiled Wheels) +### Review Priority -OpenViking provides pre-compiled **Wheel** packages for the following environments: +Maintainer time is limited, so focused PRs are reviewed first: -- **Windows**: x86_64 -- **macOS**: x86_64, arm64 (Apple Silicon) -- **Linux**: x86_64, arm64 (manylinux) +- PRs with **100 or fewer changed lines** are usually reviewed more promptly. +- PRs with **200 or fewer changed lines** are prioritized over larger PRs. -For other platforms (e.g., FreeBSD), the package will be automatically compiled from source during installation via `pip`. Ensure you have the [Prerequisites](#prerequisites) installed. +These are review priorities, not hard limits or response-time guarantees. Changed +lines mean additions plus deletions in hand-written source, tests, and documentation; +generated files, vendored code, and lockfiles are excluded when assessing size. -### 1. Fork and Clone +Do not omit necessary tests or documentation to stay below a threshold. Split a +large change only where each PR remains independently understandable and correct. +Small size does not override correctness, design quality, or compatibility. -```bash -git clone https://github.com/YOUR_USERNAME/openviking.git -cd openviking -``` +## Before You Start -### 2. Install Dependencies +1. Search existing issues, PRs, and code for the same behavior or domain rule. +2. For a bug, reproduce it through the real production entrypoint when possible. +3. Identify the owning module and trace where the value or state is created, + normalized, stored, and consumed. +4. For a feature, describe the problem and expected behavior before designing the + implementation. -We recommend using `uv` for Python environment management: +Open an issue or start a discussion before implementing a change that affects: -```bash -# Install uv (if not installed) -curl -LsSf https://astral.sh/uv/install.sh | sh +- public REST, SDK, CLI, MCP, or configuration semantics; +- persisted data, storage schemas, VFS/AGFS paths, or encrypted file behavior; +- asynchronous task ownership, queues, cancellation, cleanup, or result state; +- resource import/watch behavior, session lifecycle, or memory extraction; +- retrieval levels, directory scope, or ranking semantics; +- tenant, account, user, or peer identity boundaries; +- multiple owner modules or a large architectural refactor. -# Sync dependencies and create virtual environment -uv sync --all-extras -source .venv/bin/activate # Linux/macOS -# or .venv\Scripts\activate # Windows -``` +Include the current behavior, proposed behavior, a concrete request or configuration +example, and any compatibility impact. This lets maintainers confirm the design +boundary before implementation work begins. -#### Local Development & Native Rebuilds +Use the repository's GitHub templates for [bug reports](https://github.com/volcengine/OpenViking/issues/new?template=bug_report.yml), +[feature requests](https://github.com/volcengine/OpenViking/issues/new?template=feature_request.yml), +and [questions](https://github.com/volcengine/OpenViking/issues/new?template=question.yml). -OpenViking defaults to `binding-client` mode for AGFS/RAGFS, which requires pre-built native artifacts. If you modify the **RAGFS Rust binding**, the bundled **Rust CLI**, or the **C++ extensions**, or if the pre-built artifacts are not found, you need to re-compile and re-install them. Run the following command in the project root: +## Find the Right Area -```bash -uv pip install -e . --force-reinstall -``` +If you know the affected area, mention it in the issue or PR. If you are unsure, +describe the observable behavior and use case first; a maintainer will help route it. -This command ensures that `setup.py` is re-executed, triggering rebuilds for AGFS/RAGFS, the bundled `ov` CLI, and the C++ components. +This map reflects sustained authorship and review activity in PRs merged from June 24 +through August 24, 2026. It is routing guidance, not exclusive code ownership; mention +only the contacts relevant to the change. -### 3. Configure Environment +| Domain | Area | Representative paths or topics | Active maintainers / reviewers | +|---|---|---|---| +| Platform | Server, API, auth, identity, admin, tasks | `openviking/server`, `openviking/service` | `@qin-ctx` | +| Resource | Ingestion, watch, and task pipeline | `openviking/resource` | `@qin-ctx`, `@KCHENPENGFEI` | +| Resource | Resource parsing | `openviking/parse` | `@zihengli-bytedance`, `@KCHENPENGFEI` | +| Memory | Session, memory extraction, and compilation | `openviking/session`, memory extraction, `ov compile` | `@chenjw`, `@heaoxiang-ai`, `@fujiajie666` | +| Retrieval | Search and vector databases | `openviking/retrieve`, `openviking/storage/vectordb` | `@zhoujh01`, `@t0saki` | +| Storage | RAGFS, PathLock, QueueFS, and encryption | `openviking/storage`, `openviking/pyagfs`, `openviking/crypto`, `crates/ragfs*` | `@baojun-zhang` | +| Integration | Agent plugins and MCP | `agent-plugins`, memory plugin examples, server MCP | `@t0saki`, `@ZaynJarvis` | +| Integration | VikingBot and agent compilation | `bot`, `ov compile` | `@yeshion23333`, `@fujiajie666` | +| Client | SDKs, CLI, and LangChain | `sdk`, `crates/ov_cli`, `integrations/langchain` | `@zhoujh01`, `@t0saki`, `@ehz0ah` | +| Product | Web Studio | `web-studio` | `@yufeng201`, `@ZaynJarvis` | +| Project | Documentation, CI, and plugin releases | `docs`, `.github/workflows` | `@yufeng201`, `@ZaynJarvis` | -Run the interactive wizard to pick providers and write `~/.openviking/ov.conf`, then -validate the result: +For cross-module changes or areas without a clear owner, identify the primary affected +area first, then mention `@qin-ctx`, `@ZaynJarvis`, or `@zhoujh01`. -```console -openviking-server init -openviking-server doctor -``` +## Development Setup -Manual `ov.conf` templates, per-provider examples, and environment variables are in the -[Configuration guide](https://docs.openviking.ai/en/guides/01-configuration). The default -config file is loaded automatically; set `OPENVIKING_CONFIG_FILE` only when using a -non-default path. +### Prerequisites -### 4. Verify Installation +- Python 3.10+ +- Rust 1.91.1+ for source builds, Rust bindings, and the bundled `ov` CLI +- Go 1.22+ only for development under `sdk/go` +- A C++17 compiler: GCC 9+ or Clang 11+ +- CMake 3.15+ -```bash -python -c "import openviking; print(openviking.__version__)" -``` +On Linux, install `build-essential` and, where needed, `pkg-config`. On macOS, +install Xcode Command Line Tools. On Windows, install CMake and MinGW for local +native builds. -### 5. Build Rust CLI (Optional) +### Install -The Rust CLI (`ov`) provides a high-performance command-line client for interacting with OpenViking Server. - -Even if you do not plan to use `ov` directly, the Rust toolchain is still required when building OpenViking from source because packaging also builds the bundled CLI binary. +Fork the repository, then clone your fork: ```bash -# Build and install from source -cargo install --path crates/ov_cli - -# Or install the published npm CLI package (downloads pre-built binary) -npm i -g @openviking/cli +git clone https://github.com/YOUR_USERNAME/OpenViking.git +cd OpenViking ``` -After installation, run `ov --help` to see all available commands. CLI connection config goes in `~/.openviking/ovcli.conf`. - ---- - -## Project Structure +We recommend using `uv`: -``` -openviking/ -├── pyproject.toml # Python project and tooling configuration -├── Cargo.toml # Rust workspace configuration -├── openviking/ # Python SDK and server implementation -│ ├── client/ # HTTP client compatibility exports -│ ├── connector/ # Data connectors -│ ├── core/ # Core data models and directory abstractions -│ ├── ingest/ # Ingestion pipeline -│ ├── integrations/ # Agent integrations -│ ├── models/ # Embedding and VLM backends -│ ├── parse/ # Resource parsers and detectors -│ ├── resource/ # Resource processing and watch management -│ ├── retrieve/ # Retrieval system -│ ├── server/ # HTTP server -│ ├── session/ # Session management and compression -│ └── storage/ # Storage layer -├── openviking_cli/ # Server bootstrap and Python CLI support -├── bot/ # VikingBot agent framework -├── sdk/ # Go, Python, and TypeScript client SDKs -├── web-studio/ # Studio web frontend -├── crates/ # Rust components -│ ├── ov_cli/ # Rust CLI client -│ ├── ragfs/ # Rust implementation of AGFS -│ └── ragfs-python/ # Python binding for RAGFS -├── src/ # C++ extension sources (Python abi3) -├── third_party/ # Native third-party dependencies -├── examples/ # Usage and integration examples -├── benchmark/ # Benchmark suites -├── tests/ # Python and integration test suites -├── deploy/ # Deployment assets -├── docker/ # Docker build files -├── npm/ # npm CLI package -├── scripts/ # Development and maintenance scripts -└── docs/ # English, Chinese, and Japanese documentation +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +uv sync --all-extras ``` ---- - -## Code Style - -We use the following tools to maintain code consistency: - -| Tool | Purpose | Config | -|------|---------|--------| -| **Ruff** | Linting, Formatting, Import sorting | `pyproject.toml` | -| **mypy** | Type checking | `pyproject.toml` | - -### Running Checks +Verify the environment: ```bash -# Format code -ruff format openviking/ - -# Lint -ruff check openviking/ - -# Type check -mypy openviking/ +uv run python -c "import openviking; print(openviking.__version__)" ``` -### Style Guidelines - -1. **Line width**: 100 characters -2. **Indentation**: 4 spaces -3. **Strings**: Prefer double quotes -4. **Type hints**: Encouraged but not required -5. **Docstrings**: Required for public APIs (1-2 lines max) - ---- - -## Testing - -### Running Tests +To configure a local server, run: ```bash -# Run all tests -pytest - -# Run specific test module -pytest tests/client/ -v -pytest tests/server/ -v -pytest tests/parse/ -v - -# Run specific test file -pytest tests/client/test_http_client_config.py - -# Run specific test -pytest tests/client/test_http_client_config.py - -# Run by keyword -pytest -k "search" -v - -# Run with coverage -pytest --cov=openviking --cov-report=term-missing +uv run openviking-server init +uv run openviking-server doctor ``` -### Writing Tests - -Tests are organized in subdirectories under `tests/`. The project uses `asyncio_mode = "auto"`, so async tests do **not** need the `@pytest.mark.asyncio` decorator: - -```python -# tests/service/test_example.py -class TestResourceService: - async def test_add_resource(self, service, request_context, sample_markdown_file): - result = await service.resources.add_resource( - path=str(sample_markdown_file), - ctx=request_context, - reason="test document", - ) - assert "root_uri" in result - assert result["root_uri"].startswith("viking://") -``` - -Common fixtures are defined in `tests/conftest.py`, including the initialized `service`, `request_context`, `temp_dir`, and sample files. - ---- - -## Maintainer Routing and Contribution Entry - -### Contributor-Facing Module Map - -If you are not sure where your question, issue, or PR belongs, start with this table: - -| Domain | Area | Primary Contact | -|--------|------|-----------------| -| Integration | Bot | `@yeshion23333` | -| Integration | OpenClaw Plugin | `@Mijamind719`, `@wlff123` | -| Platform | Framework / Multi-tenant / Resources / Session | `@qin-ctx` | -| Platform | Incremental / Scheduled Update | `@myysy` | -| Knowledge | Memory | `@chenjw` | -| Knowledge | Retrieval / Directory Semantics | `@zhoujh01` | -| Storage & Security | Virtual FS / File Encryption | `@chuanbao666`, `@baojun-zhang` | +Configuration details and provider examples are in the +[configuration guide](https://docs.openviking.ai/en/guides/01-configuration). -If the area is still unclear, mention one of the cross-module maintainers listed below. +If you modify the RAGFS Rust binding, bundled Rust CLI, or C++ extensions, rebuild +the native components: -### Maintainer Routing Map - -Use this table when routing issues, PRs, or design questions to a more specific owner: - -| Domain | Subarea | Representative Paths or Topics | Primary Contact | Backup / Cross-Module | -|--------|---------|--------------------------------|-----------------|-----------------------| -| Integration | Bot Runtime | `bot/vikingbot`, `bot/bridge`, deployment scripts, bot docs | `@yeshion23333` | `@chenjw` | -| Integration | OpenClaw Plugin | `examples/openclaw-plugin`, installation, remote mode, compatibility | `@Mijamind719`, `@wlff123` | `@LinQiang391` | -| Platform | Server & Multi-tenant | `openviking/server`, `openviking/service`, auth, identity, admin, tenant boundary | `@qin-ctx` | `@MaojiaSheng` | -| Platform | Resource & Session Lifecycle | `openviking/resource`, `openviking/session`, resource ingestion, session lifecycle | `@qin-ctx` | `@MaojiaSheng` | -| Platform | Incremental & Scheduled Update | `openviking/resource/watch_manager.py`, `openviking/resource/watch_scheduler.py` | `@myysy` | `@qin-ctx` | -| Knowledge | Memory Engine | `openviking/session/memory`, `memory_extractor.py`, `memory_deduplicator.py` | `@chenjw` | `@qin-ctx` | -| Knowledge | Retrieval & Directory Semantics | `openviking/retrieve`, intent analysis, hierarchical retrieval, directory semantics | `@zhoujh01` | `@qin-ctx` | -| Storage & Security | VFS / AGFS Path Semantics | `openviking/storage`, `openviking/pyagfs`, filesystem behavior, path semantics | `@chuanbao666`, `@baojun-zhang` | `@zhoujh01` | -| Storage & Security | Encryption & Data Safety | `openviking/crypto`, file encryption, storage safety | `@chuanbao666`, `@baojun-zhang` | `@zhoujh01` | - -For areas without a stable owner yet, cross-module maintainers will help route the request first. - -### Cross-Module Maintainers - -- `@MaojiaSheng` -- `@qin-ctx` -- `@zhoujh01` - -Cross-module maintainers help with issue routing, cross-cutting design questions, and fallback review support. - -### How to Ask for Help - -- If you already know the affected module, mention it in the issue or PR description. -- If you are unsure about the module, describe the use case and affected behavior first. -- If you want to work on an issue, leave a comment before starting, especially for cross-module changes. -- If your PR spans multiple areas, call out the primary affected domain in the description. +```bash +uv pip install -e . --force-reinstall +``` -### Contribution Entry Labels +Component-specific SDKs, integrations, plugins, and benchmarks may have additional +setup instructions in their local README or package manifest. -Issue templates already classify reports such as `bug`, `enhancement`, and `question`. Maintainers may also use the following labels to make contribution entry clearer: +## Making a Change -| Label | Meaning | -|-------|---------| -| `good first issue` | Newcomer-friendly work with clear scope and acceptance criteria | -| `help wanted` | Tasks that benefit from contributors who already know the codebase or review style | -| `needs-design` | Work that needs maintainer clarification before implementation | -| `needs-review` | Pull requests waiting for the first review round | +### Ownership and Design -### Contributor Growth Path +- Put behavior in its owning module. Higher layers should transport or consume the + result, not reimplement the same rule. +- Convert external compatibility shapes into one canonical domain model at the + boundary. Keep inner business logic free of input-shape guessing. +- Preserve meaningful server, network, timeout, authentication, and conflict errors + at client-facing boundaries. +- Keep task state causally tied to the task that produced it. Do not infer completion + from global queue state or an unrelated callback. +- Prefer one authoritative source for every value and rule. -The project uses a practical contribution path so contributors can see what “next step” looks like: +If a local edge case starts changing task boundaries, public semantics, or the +overall architecture, stop and return to the design discussion instead of adding +special branches throughout the main path. -| Stage | Typical Signals | Common Next Step | -|-------|------------------|------------------| -| New Contributor | First issue or first PR, often docs, tests, or scoped fixes | Start with `good first issue` items and get familiar with local workflow | -| Active Contributor | One or more merged contributions | Pick up `help wanted` work in an area you already touched | -| Module Contributor | Repeated contributions in the same subarea | Help with triage, reproduction, docs, or review comments in that area | -| Backup Reviewer Candidate | Stable contribution record in one subarea | Help with first-pass review, routing, and contributor support | +### Code Style -## Contribution Workflow +Python uses Ruff for formatting and linting, and mypy for type checking. The +configured line width is 100 characters. -### 1. Create a Branch +Run checks on the paths you changed: ```bash -git checkout main -git pull origin main -git checkout -b feature/your-feature-name +uv run ruff format +uv run ruff check +uv run mypy ``` -Branch naming conventions: -- `feature/xxx` - New features -- `fix/xxx` - Bug fixes -- `docs/xxx` - Documentation updates -- `refactor/xxx` - Code refactoring +Public APIs should have short, useful docstrings. Prefer clear names and direct +control flow over comments that restate the code. -### 2. Make Changes +For Rust, Go, TypeScript, documentation, and plugin changes, use the formatter, +lint, type-check, and test commands defined by that component. -- Follow code style guidelines -- Add tests for new functionality -- Update documentation as needed +### Tests -### 3. Commit Changes +Validate the smallest meaningful public contract and major failure boundary affected +by the change. -```bash -git add . -git commit -m "feat: add new parser for xlsx files" -``` +- Prefer updating an existing high-value contract test. +- Do not add a new unit test or test file by default. +- Do not test private helper existence, mock call order, simple field forwarding, or + framework behavior unless it protects a lasting public contract. +- A small, clear fix does not automatically require a new test, but its validation + must be explained. +- Put temporary reproduction, diagnostic, stress, and validation scripts in + `test_scripts/`, not in source, benchmark, or maintenance script directories. -### 4. Push and Create PR +Run the relevant focused tests, for example: ```bash -git push origin feature/your-feature-name -``` - -Then create a Pull Request on GitHub. - ---- - -## Commit Convention - -We follow [Conventional Commits](https://www.conventionalcommits.org/): - +uv run pytest tests/client/test_http_client_config.py +uv run pytest tests/server/ -k "search" ``` -(): - - - -