Merge remote-tracking branch 'upstream/storage-service-dev' into merge-main

Author: sitaowang1998

Cargo.lock

@@ -30,12 +30,14 @@ tokio = {
   version = "1.50.0",
   features = ["macros", "rt-multi-thread", "sync", "time"]
 }
+tokio-util = { version = "0.7.18", features = ["rt"] }
 tracing = { version = "0.1.44", features = ["attributes"] }
+uuid = { version = "1.19.0", features = ["serde"] }
 
 [dev-dependencies]
 anyhow = "1.0.98"
 rand = "0.9.1"
 serial_test = { version = "3.2.0", features = ["file_locks"] }
 tabled = "0.20.0"
 tokio = { version = "1.50.0", features = ["macros", "rt-multi-thread", "sync"] }
-tokio-util = { version = "0.7", features = ["rt"] }
+uuid = { version = "1.19.0", features = ["v4"] }

components/spider-storage/Cargo.toml

@@ -78,11 +78,17 @@ pub enum InternalError {
     #[error("invalid config: {0}")]
     ReadyQueueInvalidConfig(&'static str),
 
+    #[error("invalid config: {0}")]
+    TaskInstancePoolInvalidConfig(&'static str),
+
     #[error("ready queue channel is closed")]
     ReadyQueueChannelClosed,
 
     #[error(transparent)]
     WireError(#[from] WireError),
+
+    #[error(transparent)]
+    Db(#[from] crate::db::DbError),
 }
 
 /// Enums for all errors representing operations that are rejected due to stale cache state.

components/spider-storage/src/cache/error.rs

@@ -296,7 +296,7 @@ impl ReadyQueueReceiverHandle {
 ///
 /// * Forwards [`ReadyQueueConfig::validate`]'s return values on failure.
 pub fn create_ready_queue(
-    config: ReadyQueueConfig,
+    config: &ReadyQueueConfig,
 ) -> Result<(ReadyQueueSenderHandle, ReadyQueueReceiverHandle), InternalError> {
     config.validate()?;
 
@@ -379,7 +379,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     async fn recv_returns_early_when_max_items_reached() -> anyhow::Result<()> {
-        let (sender, receiver) = create_ready_queue(ReadyQueueConfig {
+        let (sender, receiver) = create_ready_queue(&ReadyQueueConfig {
             task_capacity: 1,
             commit_capacity: 1,
             cleanup_capacity: 1,
@@ -425,7 +425,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     async fn recv_returns_empty_when_wait_elapses() -> anyhow::Result<()> {
-        let (_sender, receiver) = create_ready_queue(ReadyQueueConfig {
+        let (_sender, receiver) = create_ready_queue(&ReadyQueueConfig {
             task_capacity: 1,
             commit_capacity: 1,
             cleanup_capacity: 1,

components/spider-storage/src/ready_queue.rs

@@ -1,9 +1,11 @@
 pub mod error;
 pub mod job_cache;
+pub mod runtime;
 pub mod service;
 
 pub use error::StorageServerError;
 pub use job_cache::JobCache;
+pub use runtime::{Runtime, create_runtime};
 pub use service::ServiceState;
 
 #[cfg(test)]