From 136de3bb1034c2bb54b09fe96b0fc93a310ac410 Mon Sep 17 00:00:00 2001 From: Luke Kim <80174+lukekim@users.noreply.github.com> Date: Wed, 18 Feb 2026 19:44:10 -0600 Subject: [PATCH 1/5] refactor: Update setup and create_tables methods to use metadata and datasets parameters --- .../skills/system-adapter-builder/SKILL.md | 4 +- .../validate_system_adapter_templates.yml | 20 ++++---- README.md | 17 ++++--- crates/checkpointer/src/main.rs | 2 +- crates/etl/src/lib.rs | 10 ++-- crates/etl/src/main.rs | 2 +- crates/system-adapter-protocol/src/client.rs | 10 ++-- crates/system-adapter-protocol/src/lib.rs | 18 ++++--- crates/system-adapter-protocol/src/server.rs | 12 ++--- src/main.rs | 8 +-- system-adapters/databricks/src/main.rs | 13 ++--- system-adapters/templates/go/main.go | 3 +- .../spicebench/template/AdapterServer.java | 51 +++++++++++++++---- system-adapters/templates/nodejs/adapter.js | 4 +- system-adapters/templates/python/adapter.py | 4 +- system-adapters/templates/rust/src/main.rs | 3 +- 16 files changed, 106 insertions(+), 75 deletions(-) diff --git a/.claude/skills/system-adapter-builder/SKILL.md b/.claude/skills/system-adapter-builder/SKILL.md index 4ee0d274..a21182d1 100644 --- a/.claude/skills/system-adapter-builder/SKILL.md +++ b/.claude/skills/system-adapter-builder/SKILL.md @@ -16,8 +16,8 @@ A JSON-RPC 2.0 adapter that supports both transports: Required methods: -- `setup(run_id, datasets)` -- `create_tables(run_id)` +- `setup(run_id, metadata)` +- `create_tables(run_id, datasets)` - `teardown(run_id)` - `metrics(run_id)` - `rpc.methods` diff --git a/.github/workflows/validate_system_adapter_templates.yml b/.github/workflows/validate_system_adapter_templates.yml index 3b38bfdf..616c4532 100644 --- a/.github/workflows/validate_system_adapter_templates.yml +++ b/.github/workflows/validate_system_adapter_templates.yml @@ -46,7 +46,7 @@ jobs: for _ in {1..30}; do if curl -sS -o /tmp/python-resp.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ + -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ http://127.0.0.1:18080/jsonrpc; then break fi @@ -57,7 +57,7 @@ jobs: grep -q '"driver":"flightsql"' /tmp/python-resp.json grep -q '"db_kwargs"' /tmp/python-resp.json curl -sS -o /tmp/python-create.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ + -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ http://127.0.0.1:18080/jsonrpc grep -q '"ok":true' /tmp/python-create.json curl -sS -o /tmp/python-methods.json -H 'Content-Type: application/json' \ @@ -89,7 +89,7 @@ jobs: for _ in {1..30}; do if curl -sS -o /tmp/node-resp.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ + -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ http://127.0.0.1:18081/jsonrpc; then break fi @@ -100,7 +100,7 @@ jobs: grep -q '"driver":"flightsql"' /tmp/node-resp.json grep -q '"db_kwargs"' /tmp/node-resp.json curl -sS -o /tmp/node-create.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ + -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ http://127.0.0.1:18081/jsonrpc grep -q '"ok":true' /tmp/node-create.json curl -sS -o /tmp/node-methods.json -H 'Content-Type: application/json' \ @@ -134,7 +134,7 @@ jobs: for _ in {1..30}; do if curl -sS -o /tmp/rust-resp.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ + -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ http://127.0.0.1:18082/jsonrpc; then break fi @@ -145,7 +145,7 @@ jobs: grep -q '"driver":"flightsql"' /tmp/rust-resp.json grep -q '"db_kwargs"' /tmp/rust-resp.json curl -sS -o /tmp/rust-create.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ + -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ http://127.0.0.1:18082/jsonrpc grep -q '"ok":true' /tmp/rust-create.json curl -sS -o /tmp/rust-methods.json -H 'Content-Type: application/json' \ @@ -179,7 +179,7 @@ jobs: for _ in {1..30}; do if curl -sS -o /tmp/go-resp.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ + -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ http://127.0.0.1:18083/jsonrpc; then break fi @@ -190,7 +190,7 @@ jobs: grep -q '"driver":"flightsql"' /tmp/go-resp.json grep -q '"db_kwargs"' /tmp/go-resp.json curl -sS -o /tmp/go-create.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ + -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ http://127.0.0.1:18083/jsonrpc grep -q '"ok":true' /tmp/go-create.json curl -sS -o /tmp/go-methods.json -H 'Content-Type: application/json' \ @@ -223,7 +223,7 @@ jobs: for _ in {1..30}; do if curl -sS -o /tmp/java-resp.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ + -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ http://127.0.0.1:18084/jsonrpc; then break fi @@ -234,7 +234,7 @@ jobs: grep -q '"driver":"flightsql"' /tmp/java-resp.json grep -q '"db_kwargs"' /tmp/java-resp.json curl -sS -o /tmp/java-create.json -H 'Content-Type: application/json' \ - -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \ + -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \ http://127.0.0.1:18084/jsonrpc grep -q '"ok":true' /tmp/java-create.json curl -sS -o /tmp/java-methods.json -H 'Content-Type: application/json' \ diff --git a/README.md b/README.md index 272b45e2..4854b8b3 100644 --- a/README.md +++ b/README.md @@ -104,8 +104,8 @@ flowchart TB orchestrator -->|"start run"| run - adapter_iface -->|"setup(run_id, datasets)\n→ ADBC driver + kwargs"| executors - adapter_iface -->|"create_tables(run_id)"| sut + adapter_iface -->|"setup(run_id, metadata)\n→ ADBC driver + kwargs"| executors + adapter_iface -->|"create_tables(run_id, datasets)"| sut setup_phase -->|"system ready"| bench_phase bench_phase -->|"benchmark complete"| teardown_phase @@ -122,7 +122,7 @@ A **Run** is a single end-to-end execution of the benchmark for one system. Each | Phase | What happens | Timed? | | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | -| **1. Setup** | Connect to system adapter via JSON-RPC (stdio or HTTP). Call `setup(run_id, datasets)` to provision the SUT and return ADBC driver config, then `create_tables(run_id)`. | No | +| **1. Setup** | Connect to system adapter via JSON-RPC (stdio or HTTP). Call `setup(run_id, metadata)` to provision the SUT and return ADBC driver config, then `create_tables(run_id, datasets)`. | No | | **2. Benchmark (timed)** | Three sequential stages — warm-up (1× query set), baseline (10% of duration, 60s–600s), and load test (full duration with concurrent clients). | Yes | | **3. Teardown** | Call `teardown(run_id)` via the adapter to deprovision resources and clean up. | No | @@ -215,8 +215,8 @@ Results from every Run are published to [SpiceBench.com](https://spicebench.com) To benchmark a new platform, implement the JSON-RPC 2.0 adapter with these methods: -1. **`setup(run_id, datasets)`** — Provision infrastructure and configure the target system. -2. **`create_tables(run_id)`** — Create/register destination tables for the benchmark datasets. +1. **`setup(run_id, metadata)`** — Provision infrastructure and configure the target system. +2. **`create_tables(run_id, datasets)`** — Create/register destination tables for the benchmark datasets. 3. **`teardown(run_id)`** — Clean up provisioned resources. 4. **`metrics(run_id)`** *(optional)* — Return current resource usage (CPU, memory, disk, IOPS) and ingestion progress (rows, bytes, rows/s, active connections). @@ -243,8 +243,8 @@ The `spicebench` CLI connects to a system adapter using JSON-RPC 2.0 over either For each run, SpiceBench calls adapter JSON-RPC methods in this order: -1. `setup(run_id, datasets, metadata)` -2. `create_tables(run_id)` +1. `setup(run_id, metadata)` +2. `create_tables(run_id, datasets)` 3. benchmark execution and optional periodic `metrics(run_id)` scraping 4. `teardown(run_id)` @@ -256,7 +256,8 @@ Tiny `create_tables` request example: "id": 2, "method": "create_tables", "params": { - "run_id": "00000000-0000-0000-0000-000000000000" + "run_id": "00000000-0000-0000-0000-000000000000", + "datasets": {} } } ``` diff --git a/crates/checkpointer/src/main.rs b/crates/checkpointer/src/main.rs index 826523bd..5d8ef9fd 100644 --- a/crates/checkpointer/src/main.rs +++ b/crates/checkpointer/src/main.rs @@ -198,7 +198,7 @@ async fn main() -> anyhow::Result<()> { ); // Log the tables and schemas that will be processed. - let datasets = pipeline.setup_request_datasets(); + let datasets = pipeline.create_tables_request_datasets(); for (name, config) in &datasets { tracing::info!(table = %name, schema = ?config.schema, "Dataset table registered"); } diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index 3d4fd812..d4e9da92 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -183,7 +183,7 @@ struct PipelineWorkState { /// # Lifecycle /// /// 1. **[`NotStarted`](PipelineState::NotStarted)** — created via [`ETLPipeline::new`] -/// with a dataset, source, and target. Call [`setup_request_datasets`](ETLPipeline::setup_request_datasets) +/// with a dataset, source, and target. Call [`create_tables_request_datasets`](ETLPipeline::create_tables_request_datasets) /// to obtain the dataset configurations that a system adapter needs. /// 2. **[`Initialized`](PipelineState::Initialized)** — the first batch (batch 0) /// has been ETL'd into the target via [`initialize`](ETLPipeline::initialize). @@ -277,14 +277,14 @@ impl ETLPipeline { self.cancel_token.cancel(); } - /// Returns the dataset configurations required to set up the system adapter. + /// Returns the dataset configurations required for `create_tables`. /// /// Each entry maps a table name to its /// [`DatasetConfig`](system_adapter_protocol::DatasetConfig), which includes /// the rehydrated Arrow schema. This can be used to build a - /// [`SetupRequest`](system_adapter_protocol::SetupRequest) for the system - /// adapter. - pub fn setup_request_datasets(&self) -> HashMap { + /// [`CreateTablesRequest`](system_adapter_protocol::CreateTablesRequest) for + /// the system adapter. + pub fn create_tables_request_datasets(&self) -> HashMap { self.dataset .tables() .into_iter() diff --git a/crates/etl/src/main.rs b/crates/etl/src/main.rs index 98a229a6..2bc6820f 100644 --- a/crates/etl/src/main.rs +++ b/crates/etl/src/main.rs @@ -136,7 +136,7 @@ async fn main() -> anyhow::Result<()> { ); // Log the tables and schemas that will be processed. - let datasets = pipeline.setup_request_datasets(); + let datasets = pipeline.create_tables_request_datasets(); for (name, config) in &datasets { tracing::info!(table = %name, schema = ?config.schema, "Dataset table registered"); } diff --git a/crates/system-adapter-protocol/src/client.rs b/crates/system-adapter-protocol/src/client.rs index 0b926b3a..a58d78a1 100644 --- a/crates/system-adapter-protocol/src/client.rs +++ b/crates/system-adapter-protocol/src/client.rs @@ -179,14 +179,9 @@ impl Client { pub async fn setup( &mut self, run_id: uuid::Uuid, - datasets: std::collections::HashMap, metadata: std::collections::HashMap, ) -> Result { - let request = crate::SetupRequest { - run_id, - datasets, - metadata, - }; + let request = crate::SetupRequest { run_id, metadata }; let rpc_request = JsonRpcRequest::new(1, crate::methods::SETUP, request); let response = self.call_typed(rpc_request).await?; response @@ -198,8 +193,9 @@ impl Client { pub async fn create_tables( &mut self, run_id: uuid::Uuid, + datasets: std::collections::HashMap, ) -> Result { - let request = crate::CreateTablesRequest { run_id }; + let request = crate::CreateTablesRequest { run_id, datasets }; let rpc_request = JsonRpcRequest::new(1, crate::methods::CREATE_TABLES, request); let response = self.call_typed(rpc_request).await?; response diff --git a/crates/system-adapter-protocol/src/lib.rs b/crates/system-adapter-protocol/src/lib.rs index 29a30868..f45e419f 100644 --- a/crates/system-adapter-protocol/src/lib.rs +++ b/crates/system-adapter-protocol/src/lib.rs @@ -41,8 +41,8 @@ limitations under the License. //! //! // Setup a benchmark run //! let run_id = Uuid::new_v4(); -//! let setup_response = client.setup(run_id, HashMap::new(), HashMap::new()).await?; -//! let create_tables_response = client.create_tables(run_id).await?; +//! let setup_response = client.setup(run_id, HashMap::new()).await?; +//! let create_tables_response = client.create_tables(run_id, HashMap::new()).await?; //! //! println!("Driver: {:?}", setup_response.driver); //! @@ -72,7 +72,6 @@ limitations under the License. //! async fn setup( //! &mut self, //! run_id: Uuid, -//! datasets: HashMap, //! metadata: HashMap, //! ) -> Result { //! // Your setup logic here @@ -83,7 +82,12 @@ limitations under the License. //! }) //! } //! -//! async fn create_tables(&mut self, run_id: Uuid) -> Result { +//! async fn create_tables( +//! &mut self, +//! run_id: Uuid, +//! datasets: HashMap, +//! ) -> Result { +//! let _ = datasets; //! Ok(CreateTablesResponse { ok: true }) //! } //! @@ -146,11 +150,10 @@ pub struct DatasetConfig { /// /// JSON-RPC method: `setup` #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SetupRequest { /// Unique identifier for this benchmark run pub run_id: Uuid, - /// Map of dataset name to dataset definition - pub datasets: HashMap, /// Arbitrary run metadata propagated from spicebench to adapters #[serde(default)] pub metadata: HashMap, @@ -169,9 +172,12 @@ pub struct SetupResponse { /// /// JSON-RPC method: `create_tables` #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct CreateTablesRequest { /// Unique identifier for this benchmark run pub run_id: Uuid, + /// Map of dataset name to dataset definition + pub datasets: HashMap, } /// Response from create_tables request diff --git a/crates/system-adapter-protocol/src/server.rs b/crates/system-adapter-protocol/src/server.rs index 4249a890..338eb99d 100644 --- a/crates/system-adapter-protocol/src/server.rs +++ b/crates/system-adapter-protocol/src/server.rs @@ -75,7 +75,6 @@ pub trait Handler: Send + Sync { async fn setup( &mut self, run_id: Uuid, - datasets: HashMap, metadata: HashMap, ) -> std::result::Result; @@ -83,6 +82,7 @@ pub trait Handler: Send + Sync { async fn create_tables( &mut self, run_id: Uuid, + datasets: HashMap, ) -> std::result::Result; /// Teardown a benchmark run @@ -243,9 +243,7 @@ impl Server { Err(e) => return e, }; Self::handler_response( - self.handler - .setup(req.run_id, req.datasets, req.metadata) - .await, + self.handler.setup(req.run_id, req.metadata).await, id, ) } @@ -259,7 +257,7 @@ impl Server { Ok(r) => r, Err(e) => return e, }; - Self::handler_response(self.handler.create_tables(req.run_id).await, id) + Self::handler_response(self.handler.create_tables(req.run_id, req.datasets).await, id) } async fn handle_teardown( @@ -304,7 +302,6 @@ mod tests { async fn setup( &mut self, _run_id: Uuid, - _datasets: HashMap, _metadata: HashMap, ) -> std::result::Result { Ok(SetupResponse { @@ -316,6 +313,7 @@ mod tests { async fn create_tables( &mut self, _run_id: Uuid, + _datasets: HashMap, ) -> std::result::Result { Ok(CreateTablesResponse { ok: true }) } @@ -335,7 +333,7 @@ mod tests { #[tokio::test] async fn test_server_setup() { let mut server = Server::new(TestHandler); - let request = r#"{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{},"metadata":{}}}"#; + let request = r#"{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","metadata":{}}}"#; let response = server.handle_request(request).await; assert!(response.get("result").is_some()); diff --git a/src/main.rs b/src/main.rs index 60d71539..9ecb2273 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,7 +37,7 @@ mod scenario; use crate::commands::connect_system_adapter; use crate::scenario::Scenario; -fn setup_request_datasets( +fn create_tables_request_datasets( dataset: &Arc, ) -> HashMap { dataset @@ -122,7 +122,7 @@ async fn main() -> anyhow::Result<()> { let mutations = MutationConfig::new(0.1, 0.1); let setup_dataset = dataset_source.create(&generation_config, &mutations)?; - let datasets = setup_request_datasets(&setup_dataset); + let datasets = create_tables_request_datasets(&setup_dataset); let setup_metadata = std::collections::HashMap::from([ ( @@ -136,7 +136,7 @@ async fn main() -> anyhow::Result<()> { ]); let adbc_driver = match system_adapter_client - .setup(run_id, datasets, setup_metadata) + .setup(run_id, setup_metadata) .await { Ok(response) => response, @@ -182,7 +182,7 @@ async fn main() -> anyhow::Result<()> { &mutations, )?; - if let Err(e) = system_adapter_client.create_tables(run_id).await { + if let Err(e) = system_adapter_client.create_tables(run_id, datasets).await { pipeline.cancel(); return Err(anyhow::anyhow!( "Failed to create tables via system adapter: {e}" diff --git a/system-adapters/databricks/src/main.rs b/system-adapters/databricks/src/main.rs index 792d5864..933ad14b 100644 --- a/system-adapters/databricks/src/main.rs +++ b/system-adapters/databricks/src/main.rs @@ -183,7 +183,6 @@ impl TableFormat { #[derive(Debug, Clone)] struct RunState { - datasets: HashMap, table_format: TableFormat, created_tables: Vec, cluster_id: Option, @@ -1025,13 +1024,9 @@ impl Handler for DatabricksAdapter { async fn setup( &mut self, run_id: Uuid, - datasets: HashMap, metadata: HashMap, ) -> std::result::Result { - eprintln!( - "[databricks-adapter] setup: run_id={run_id}, datasets={}", - datasets.len() - ); + eprintln!("[databricks-adapter] setup: run_id={run_id}"); let (cluster_id, cluster_created_by_adapter) = match &self.config.compute_target { ComputeTarget::SparkCluster(_) => { @@ -1068,7 +1063,6 @@ impl Handler for DatabricksAdapter { self.runs.insert( run_id, RunState { - datasets, table_format, created_tables: Vec::new(), cluster_id, @@ -1094,13 +1088,14 @@ impl Handler for DatabricksAdapter { async fn create_tables( &mut self, run_id: Uuid, + datasets: HashMap, ) -> std::result::Result { - let (datasets, table_format) = { + let table_format = { let state = self .runs .get(&run_id) .ok_or_else(|| format!("Unknown run_id: {run_id}"))?; - (state.datasets.clone(), state.table_format) + state.table_format }; let mut created_tables = Vec::with_capacity(datasets.len()); diff --git a/system-adapters/templates/go/main.go b/system-adapters/templates/go/main.go index ffb1c318..82a7049a 100644 --- a/system-adapters/templates/go/main.go +++ b/system-adapters/templates/go/main.go @@ -68,7 +68,7 @@ func methodSetup(_ map[string]interface{}) interface{} { // query driver details SpiceBench should use. // Example: // - create run-scoped database/schema - // - configure ingestion resources for dataset list + // - configure ingestion resources needed before table creation // - resolve query endpoint and auth material from control plane host := getenvOr("SUT_HOST", "127.0.0.1") @@ -95,6 +95,7 @@ func methodCreateTables(_ map[string]interface{}) interface{} { // Stub: Create/register destination tables for benchmark datasets. // Example: // - create tables if they do not exist + // - iterate datasets passed in params to build table definitions // - apply expected schema/partitioning return map[string]interface{}{"ok": true} } diff --git a/system-adapters/templates/java/src/main/java/com/spicebench/template/AdapterServer.java b/system-adapters/templates/java/src/main/java/com/spicebench/template/AdapterServer.java index 637cc790..ac991e40 100644 --- a/system-adapters/templates/java/src/main/java/com/spicebench/template/AdapterServer.java +++ b/system-adapters/templates/java/src/main/java/com/spicebench/template/AdapterServer.java @@ -118,10 +118,10 @@ private static ObjectNode dispatch(JsonNode request) { } return switch (method) { - case "setup" -> jsonrpcSuccess(id, methodSetup()); - case "create_tables" -> jsonrpcSuccess(id, methodCreateTables()); - case "teardown" -> jsonrpcSuccess(id, methodTeardown()); - case "metrics" -> jsonrpcSuccess(id, methodMetrics()); + case "setup" -> jsonrpcSuccess(id, methodSetup(params)); + case "create_tables" -> jsonrpcSuccess(id, methodCreateTables(params)); + case "teardown" -> jsonrpcSuccess(id, methodTeardown(params)); + case "metrics" -> jsonrpcSuccess(id, methodMetrics(params)); case "rpc.methods" -> jsonrpcSuccess(id, methodRpcMethods()); default -> jsonrpcError(id, -32601, "Method not found", null); }; @@ -151,14 +151,20 @@ private static ObjectNode jsonrpcError(JsonNode id, int code, String message, Ob return response; } - private static JsonNode methodSetup() { + private static JsonNode methodSetup(JsonNode params) { + JsonNode runId = params.get("run_id"); + if (runId == null) { + runId = MAPPER.nullNode(); + } + // Stub: Provision or initialize your SUT for this run and return // query driver details SpiceBench should use. // Example: // - create run-scoped schema/database - // - configure ingestion resources for datasets + // - configure ingestion resources needed before table creation // - wait for readiness checks to pass // - resolve endpoint and credentials from your control plane + runId.asText(); String host = getenvOr("SUT_HOST", "127.0.0.1"); int port = getenvIntOr("SUT_PORT", 50051); boolean tls = "true".equalsIgnoreCase(getenvOr("SUT_TLS", "false")); @@ -176,32 +182,59 @@ private static JsonNode methodSetup() { return result; } - private static JsonNode methodCreateTables() { + private static JsonNode methodCreateTables(JsonNode params) { + JsonNode runId = params.get("run_id"); + JsonNode datasets = params.get("datasets"); + if (runId == null) { + runId = MAPPER.nullNode(); + } + if (datasets == null) { + datasets = MAPPER.createObjectNode(); + } + // Stub: Create/register destination tables for benchmark datasets. // Example: // - create tables if they do not exist + // - iterate datasets and map each schema to table DDL // - apply expected schema/partitioning + runId.asText(); + datasets.isObject(); + ObjectNode result = MAPPER.createObjectNode(); result.put("ok", true); return result; } - private static JsonNode methodTeardown() { + private static JsonNode methodTeardown(JsonNode params) { + JsonNode runId = params.get("run_id"); + if (runId == null) { + runId = MAPPER.nullNode(); + } + // Stub: Deprovision resources created in setup. // Example: // - drop run-scoped schema/database // - stop ingestion workers/jobs + runId.asText(); + ObjectNode result = MAPPER.createObjectNode(); result.put("ok", true); return result; } - private static JsonNode methodMetrics() { + private static JsonNode methodMetrics(JsonNode params) { + JsonNode runId = params.get("run_id"); + if (runId == null) { + runId = MAPPER.nullNode(); + } + // Stub: Poll live SUT telemetry and map to this metrics schema. // Example sources: // - CPU/memory/disk from host/cloud monitoring APIs // - ingestion throughput from ingestion status endpoint // - active connections from DB/service diagnostics endpoint + runId.asText(); + ObjectNode resource = MAPPER.createObjectNode(); resource.put("cpu_usage_percent", 0.0); resource.put("memory_usage_bytes", 0); diff --git a/system-adapters/templates/nodejs/adapter.js b/system-adapters/templates/nodejs/adapter.js index 3779b7f0..dea83e0b 100644 --- a/system-adapters/templates/nodejs/adapter.js +++ b/system-adapters/templates/nodejs/adapter.js @@ -28,13 +28,11 @@ function jsonrpcError(id, code, message, data) { function methodSetup(params) { void params.run_id; - void (params.datasets || {}); // Stub: Provision or initialize your SUT for this run and return // query driver details SpiceBench should use. // Example: // - create a test database or schema for this run_id - // - configure ingestion routes for provided datasets // - block until SUT readiness checks are healthy // - resolve endpoint + credentials from your control plane @@ -55,10 +53,12 @@ function methodSetup(params) { function methodCreateTables(params) { void params.run_id; + void (params.datasets || {}); // Stub: Create/register destination tables for benchmark datasets. // Example: // - create tables if they do not exist + // - iterate provided datasets and map each schema to target DDL // - apply expected schema/partitioning return { ok: true }; diff --git a/system-adapters/templates/python/adapter.py b/system-adapters/templates/python/adapter.py index 9657adad..9b2e92a7 100644 --- a/system-adapters/templates/python/adapter.py +++ b/system-adapters/templates/python/adapter.py @@ -34,13 +34,11 @@ def jsonrpc_error(request_id: Any, code: int, message: str, data: Any | None = N def method_setup(params: dict[str, Any]) -> dict[str, Any]: _run_id = params.get("run_id") - _datasets = params.get("datasets", {}) # Stub: Provision or initialize your SUT for this run and return # query driver details SpiceBench should use. # Example: # - create a test database / schema for _run_id - # - configure ingestion pipelines for _datasets # - wait for SUT readiness checks to pass # - resolve connection details from your control plane @@ -61,10 +59,12 @@ def method_setup(params: dict[str, Any]) -> dict[str, Any]: def method_create_tables(params: dict[str, Any]) -> dict[str, Any]: _run_id = params.get("run_id") + _datasets = params.get("datasets", {}) # Stub: Create/register destination tables for benchmark datasets. # Example: # - create tables if they do not exist + # - iterate _datasets to map each dataset to a destination table # - apply expected schema/partitioning return {"ok": True} diff --git a/system-adapters/templates/rust/src/main.rs b/system-adapters/templates/rust/src/main.rs index c38b32bf..6c70257f 100644 --- a/system-adapters/templates/rust/src/main.rs +++ b/system-adapters/templates/rust/src/main.rs @@ -52,7 +52,7 @@ fn method_setup(_params: &Value) -> Value { // query driver details SpiceBench should use. // Example: // - create run-scoped database/schema - // - configure ingestion resources for requested datasets + // - configure ingestion resources needed before table creation // - wait for service readiness checks // - resolve connection details from your control plane let host = std::env::var("SUT_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); @@ -79,6 +79,7 @@ fn method_create_tables(_params: &Value) -> Value { // Stub: Create/register destination tables for benchmark datasets. // Example: // - create tables if they do not exist + // - iterate datasets passed in params to map each schema to table DDL // - apply expected schema/partitioning json!({"ok": true}) } From 2201474b8c50558192c2bb9b2d3c54342c632efd Mon Sep 17 00:00:00 2001 From: Luke Kim <80174+lukekim@users.noreply.github.com> Date: Wed, 18 Feb 2026 19:44:14 -0600 Subject: [PATCH 2/5] fix: Correct formatting in README for Run phases table --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4854b8b3..3df257c5 100644 --- a/README.md +++ b/README.md @@ -120,11 +120,11 @@ flowchart TB A **Run** is a single end-to-end execution of the benchmark for one system. Each Run proceeds through three phases: -| Phase | What happens | Timed? | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | +| Phase | What happens | Timed? | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | | **1. Setup** | Connect to system adapter via JSON-RPC (stdio or HTTP). Call `setup(run_id, metadata)` to provision the SUT and return ADBC driver config, then `create_tables(run_id, datasets)`. | No | -| **2. Benchmark (timed)** | Three sequential stages — warm-up (1× query set), baseline (10% of duration, 60s–600s), and load test (full duration with concurrent clients). | Yes | -| **3. Teardown** | Call `teardown(run_id)` via the adapter to deprovision resources and clean up. | No | +| **2. Benchmark (timed)** | Three sequential stages — warm-up (1× query set), baseline (10% of duration, 60s–600s), and load test (full duration with concurrent clients). | Yes | +| **3. Teardown** | Call `teardown(run_id)` via the adapter to deprovision resources and clean up. | No | The **E2E benchmark duration** (phase 2, load test stage) is the primary ranking metric. After the load test, each query's p99 latency is compared against the baseline: >20% increase = FAIL, 10–20% = WARN, ≥3 WARNs = FAIL. From 8d39129ecbfcad05c94ccb3479d57d63fecfb54b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 18 Feb 2026 17:46:15 -0800 Subject: [PATCH 3/5] chore: auto-fix cargo fmt + clippy --- crates/system-adapter-protocol/src/server.rs | 10 +++++----- src/main.rs | 5 +---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/system-adapter-protocol/src/server.rs b/crates/system-adapter-protocol/src/server.rs index 338eb99d..b079f332 100644 --- a/crates/system-adapter-protocol/src/server.rs +++ b/crates/system-adapter-protocol/src/server.rs @@ -242,10 +242,7 @@ impl Server { Ok(r) => r, Err(e) => return e, }; - Self::handler_response( - self.handler.setup(req.run_id, req.metadata).await, - id, - ) + Self::handler_response(self.handler.setup(req.run_id, req.metadata).await, id) } async fn handle_create_tables( @@ -257,7 +254,10 @@ impl Server { Ok(r) => r, Err(e) => return e, }; - Self::handler_response(self.handler.create_tables(req.run_id, req.datasets).await, id) + Self::handler_response( + self.handler.create_tables(req.run_id, req.datasets).await, + id, + ) } async fn handle_teardown( diff --git a/src/main.rs b/src/main.rs index 9ecb2273..4f4cd62f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -135,10 +135,7 @@ async fn main() -> anyhow::Result<()> { ), ]); - let adbc_driver = match system_adapter_client - .setup(run_id, setup_metadata) - .await - { + let adbc_driver = match system_adapter_client.setup(run_id, setup_metadata).await { Ok(response) => response, Err(e) => { return Err(anyhow::anyhow!("Failed to setup system adapter: {e}")); From 3e8ac270ff38f46d85f499fc33b350e2e5434322 Mon Sep 17 00:00:00 2001 From: Sergei Grebnov Date: Wed, 18 Feb 2026 19:59:04 -0600 Subject: [PATCH 4/5] Trigger CI --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2d1b13f7..245e6e1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ -[workspace] +[workspace] exclude = ["system-adapters/databricks"] members = [ ".", From c3c4bf232208d048004cd9ff8a1cb072110281e2 Mon Sep 17 00:00:00 2001 From: Sergei Grebnov Date: Wed, 18 Feb 2026 19:59:15 -0600 Subject: [PATCH 5/5] Trigger CI --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 245e6e1f..2d1b13f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ -[workspace] +[workspace] exclude = ["system-adapters/databricks"] members = [ ".",