Update setup and create_tables methods

.claude/skills/system-adapter-builder/SKILL.md

@@ -16,8 +16,8 @@ A JSON-RPC 2.0 adapter that supports both transports:
 
 Required methods:
 
-- `setup(run_id, datasets)`
-- `create_tables(run_id)`
+- `setup(run_id, metadata)`
+- `create_tables(run_id, datasets)`
 - `teardown(run_id)`
 - `metrics(run_id)`
 - `rpc.methods`

.github/workflows/validate_system_adapter_templates.yml

@@ -46,7 +46,7 @@ jobs:
 
           for _ in {1..30}; do
             if curl -sS -o /tmp/python-resp.json -H 'Content-Type: application/json' \
-              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
+              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
               http://127.0.0.1:18080/jsonrpc; then
               break
             fi
@@ -57,7 +57,7 @@ jobs:
           grep -q '"driver":"flightsql"' /tmp/python-resp.json
           grep -q '"db_kwargs"' /tmp/python-resp.json
           curl -sS -o /tmp/python-create.json -H 'Content-Type: application/json' \
-            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
+            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
             http://127.0.0.1:18080/jsonrpc
           grep -q '"ok":true' /tmp/python-create.json
           curl -sS -o /tmp/python-methods.json -H 'Content-Type: application/json' \
@@ -89,7 +89,7 @@ jobs:
 
           for _ in {1..30}; do
             if curl -sS -o /tmp/node-resp.json -H 'Content-Type: application/json' \
-              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
+              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
               http://127.0.0.1:18081/jsonrpc; then
               break
             fi
@@ -100,7 +100,7 @@ jobs:
           grep -q '"driver":"flightsql"' /tmp/node-resp.json
           grep -q '"db_kwargs"' /tmp/node-resp.json
           curl -sS -o /tmp/node-create.json -H 'Content-Type: application/json' \
-            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
+            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
             http://127.0.0.1:18081/jsonrpc
           grep -q '"ok":true' /tmp/node-create.json
           curl -sS -o /tmp/node-methods.json -H 'Content-Type: application/json' \
@@ -134,7 +134,7 @@ jobs:
 
           for _ in {1..30}; do
             if curl -sS -o /tmp/rust-resp.json -H 'Content-Type: application/json' \
-              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
+              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
               http://127.0.0.1:18082/jsonrpc; then
               break
             fi
@@ -145,7 +145,7 @@ jobs:
           grep -q '"driver":"flightsql"' /tmp/rust-resp.json
           grep -q '"db_kwargs"' /tmp/rust-resp.json
           curl -sS -o /tmp/rust-create.json -H 'Content-Type: application/json' \
-            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
+            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
             http://127.0.0.1:18082/jsonrpc
           grep -q '"ok":true' /tmp/rust-create.json
           curl -sS -o /tmp/rust-methods.json -H 'Content-Type: application/json' \
@@ -179,7 +179,7 @@ jobs:
 
           for _ in {1..30}; do
             if curl -sS -o /tmp/go-resp.json -H 'Content-Type: application/json' \
-              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
+              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
               http://127.0.0.1:18083/jsonrpc; then
               break
             fi
@@ -190,7 +190,7 @@ jobs:
           grep -q '"driver":"flightsql"' /tmp/go-resp.json
           grep -q '"db_kwargs"' /tmp/go-resp.json
           curl -sS -o /tmp/go-create.json -H 'Content-Type: application/json' \
-            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
+            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
             http://127.0.0.1:18083/jsonrpc
           grep -q '"ok":true' /tmp/go-create.json
           curl -sS -o /tmp/go-methods.json -H 'Content-Type: application/json' \
@@ -223,7 +223,7 @@ jobs:
 
           for _ in {1..30}; do
             if curl -sS -o /tmp/java-resp.json -H 'Content-Type: application/json' \
-              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
+              -d '{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
               http://127.0.0.1:18084/jsonrpc; then
               break
             fi
@@ -234,7 +234,7 @@ jobs:
           grep -q '"driver":"flightsql"' /tmp/java-resp.json
           grep -q '"db_kwargs"' /tmp/java-resp.json
           curl -sS -o /tmp/java-create.json -H 'Content-Type: application/json' \
-            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000"}}' \
+            -d '{"jsonrpc":"2.0","id":3,"method":"create_tables","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{}}}' \
             http://127.0.0.1:18084/jsonrpc
           grep -q '"ok":true' /tmp/java-create.json
           curl -sS -o /tmp/java-methods.json -H 'Content-Type: application/json' \

README.md

@@ -104,8 +104,8 @@ flowchart TB
 
     orchestrator -->|"start run"| run
 
-    adapter_iface -->|"setup(run_id, datasets)\n→ ADBC driver + kwargs"| executors
-    adapter_iface -->|"create_tables(run_id)"| sut
+    adapter_iface -->|"setup(run_id, metadata)\n→ ADBC driver + kwargs"| executors
+    adapter_iface -->|"create_tables(run_id, datasets)"| sut
     setup_phase -->|"system ready"| bench_phase
     bench_phase -->|"benchmark complete"| teardown_phase
 
@@ -120,11 +120,11 @@ flowchart TB
 
 A **Run** is a single end-to-end execution of the benchmark for one system. Each Run proceeds through three phases:
 
-| Phase                    | What happens                                                                                                                                                                                      | Timed? |
-| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ |
-| **1. Setup**             | Connect to system adapter via JSON-RPC (stdio or HTTP). Call `setup(run_id, datasets)` to provision the SUT and return ADBC driver config, then `create_tables(run_id)`. | No     |
-| **2. Benchmark (timed)** | Three sequential stages — warm-up (1× query set), baseline (10% of duration, 60s–600s), and load test (full duration with concurrent clients).                                                    | Yes    |
-| **3. Teardown**          | Call `teardown(run_id)` via the adapter to deprovision resources and clean up.                                                                                                                    | No     |
+| Phase                    | What happens                                                                                                                                                                       | Timed? |
+| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ |
+| **1. Setup**             | Connect to system adapter via JSON-RPC (stdio or HTTP). Call `setup(run_id, metadata)` to provision the SUT and return ADBC driver config, then `create_tables(run_id, datasets)`. | No     |
+| **2. Benchmark (timed)** | Three sequential stages — warm-up (1× query set), baseline (10% of duration, 60s–600s), and load test (full duration with concurrent clients).                                     | Yes    |
+| **3. Teardown**          | Call `teardown(run_id)` via the adapter to deprovision resources and clean up.                                                                                                     | No     |
 
 The **E2E benchmark duration** (phase 2, load test stage) is the primary ranking metric. After the load test, each query's p99 latency is compared against the baseline: >20% increase = FAIL, 10–20% = WARN, ≥3 WARNs = FAIL.
 
@@ -215,8 +215,8 @@ Results from every Run are published to [SpiceBench.com](https://spicebench.com)
 
 To benchmark a new platform, implement the JSON-RPC 2.0 adapter with these methods:
 
-1. **`setup(run_id, datasets)`** — Provision infrastructure and configure the target system.
-2. **`create_tables(run_id)`** — Create/register destination tables for the benchmark datasets.
+1. **`setup(run_id, metadata)`** — Provision infrastructure and configure the target system.
+2. **`create_tables(run_id, datasets)`** — Create/register destination tables for the benchmark datasets.
 3. **`teardown(run_id)`** — Clean up provisioned resources.
 4. **`metrics(run_id)`** *(optional)* — Return current resource usage (CPU, memory, disk, IOPS) and ingestion progress (rows, bytes, rows/s, active connections).
 
@@ -243,8 +243,8 @@ The `spicebench` CLI connects to a system adapter using JSON-RPC 2.0 over either
 
 For each run, SpiceBench calls adapter JSON-RPC methods in this order:
 
-1. `setup(run_id, datasets, metadata)`
-2. `create_tables(run_id)`
+1. `setup(run_id, metadata)`
+2. `create_tables(run_id, datasets)`
 3. benchmark execution and optional periodic `metrics(run_id)` scraping
 4. `teardown(run_id)`
 
@@ -256,7 +256,8 @@ Tiny `create_tables` request example:
     "id": 2,
     "method": "create_tables",
     "params": {
-        "run_id": "00000000-0000-0000-0000-000000000000"
+        "run_id": "00000000-0000-0000-0000-000000000000",
+        "datasets": {}
     }
 }
 ```

crates/checkpointer/src/main.rs

@@ -198,7 +198,7 @@ async fn main() -> anyhow::Result<()> {
     );
 
     // Log the tables and schemas that will be processed.
-    let datasets = pipeline.setup_request_datasets();
+    let datasets = pipeline.create_tables_request_datasets();
     for (name, config) in &datasets {
         tracing::info!(table = %name, schema = ?config.schema, "Dataset table registered");
     }

crates/etl/src/lib.rs

@@ -183,7 +183,7 @@ struct PipelineWorkState {
 /// # Lifecycle
 ///
 /// 1. **[`NotStarted`](PipelineState::NotStarted)** — created via [`ETLPipeline::new`]
-///    with a dataset, source, and target. Call [`setup_request_datasets`](ETLPipeline::setup_request_datasets)
+///    with a dataset, source, and target. Call [`create_tables_request_datasets`](ETLPipeline::create_tables_request_datasets)
 ///    to obtain the dataset configurations that a system adapter needs.
 /// 2. **[`Initialized`](PipelineState::Initialized)** — the first batch (batch 0)
 ///    has been ETL'd into the target via [`initialize`](ETLPipeline::initialize).
@@ -277,14 +277,14 @@ impl ETLPipeline {
         self.cancel_token.cancel();
     }
 
-    /// Returns the dataset configurations required to set up the system adapter.
+    /// Returns the dataset configurations required for `create_tables`.
     ///
     /// Each entry maps a table name to its
     /// [`DatasetConfig`](system_adapter_protocol::DatasetConfig), which includes
     /// the rehydrated Arrow schema. This can be used to build a
-    /// [`SetupRequest`](system_adapter_protocol::SetupRequest) for the system
-    /// adapter.
-    pub fn setup_request_datasets(&self) -> HashMap<String, ProtocolDatasetConfig> {
+    /// [`CreateTablesRequest`](system_adapter_protocol::CreateTablesRequest) for
+    /// the system adapter.
+    pub fn create_tables_request_datasets(&self) -> HashMap<String, ProtocolDatasetConfig> {
         self.dataset
             .tables()
             .into_iter()

crates/etl/src/main.rs

@@ -136,7 +136,7 @@ async fn main() -> anyhow::Result<()> {
     );
 
     // Log the tables and schemas that will be processed.
-    let datasets = pipeline.setup_request_datasets();
+    let datasets = pipeline.create_tables_request_datasets();
     for (name, config) in &datasets {
         tracing::info!(table = %name, schema = ?config.schema, "Dataset table registered");
     }

crates/system-adapter-protocol/src/client.rs

@@ -179,14 +179,9 @@ impl Client {
     pub async fn setup(
         &mut self,
         run_id: uuid::Uuid,
-        datasets: std::collections::HashMap<String, crate::DatasetConfig>,
         metadata: std::collections::HashMap<String, serde_json::Value>,
     ) -> Result<crate::SetupResponse> {
-        let request = crate::SetupRequest {
-            run_id,
-            datasets,
-            metadata,
-        };
+        let request = crate::SetupRequest { run_id, metadata };
         let rpc_request = JsonRpcRequest::new(1, crate::methods::SETUP, request);
         let response = self.call_typed(rpc_request).await?;
         response
@@ -198,8 +193,9 @@ impl Client {
     pub async fn create_tables(
         &mut self,
         run_id: uuid::Uuid,
+        datasets: std::collections::HashMap<String, crate::DatasetConfig>,
     ) -> Result<crate::CreateTablesResponse> {
-        let request = crate::CreateTablesRequest { run_id };
+        let request = crate::CreateTablesRequest { run_id, datasets };
         let rpc_request = JsonRpcRequest::new(1, crate::methods::CREATE_TABLES, request);
         let response = self.call_typed(rpc_request).await?;
         response

crates/system-adapter-protocol/src/lib.rs

@@ -41,8 +41,8 @@ limitations under the License.
 //!
 //! // Setup a benchmark run
 //! let run_id = Uuid::new_v4();
-//! let setup_response = client.setup(run_id, HashMap::new(), HashMap::new()).await?;
-//! let create_tables_response = client.create_tables(run_id).await?;
+//! let setup_response = client.setup(run_id, HashMap::new()).await?;
+//! let create_tables_response = client.create_tables(run_id, HashMap::new()).await?;
 //!
 //! println!("Driver: {:?}", setup_response.driver);
 //!
@@ -72,7 +72,6 @@ limitations under the License.
 //!     async fn setup(
 //!         &mut self,
 //!         run_id: Uuid,
-//!         datasets: HashMap<String, DatasetConfig>,
 //!         metadata: HashMap<String, serde_json::Value>,
 //!     ) -> Result<SetupResponse, String> {
 //!         // Your setup logic here
@@ -83,7 +82,12 @@ limitations under the License.
 //!         })
 //!     }
 //!
-//!     async fn create_tables(&mut self, run_id: Uuid) -> Result<CreateTablesResponse, String> {
+//!     async fn create_tables(
+//!         &mut self,
+//!         run_id: Uuid,
+//!         datasets: HashMap<String, DatasetConfig>,
+//!     ) -> Result<CreateTablesResponse, String> {
+//!         let _ = datasets;
 //!         Ok(CreateTablesResponse { ok: true })
 //!     }
 //!
@@ -146,11 +150,10 @@ pub struct DatasetConfig {
 ///
 /// JSON-RPC method: `setup`
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct SetupRequest {
     /// Unique identifier for this benchmark run
     pub run_id: Uuid,
-    /// Map of dataset name to dataset definition
-    pub datasets: HashMap<String, DatasetConfig>,
     /// Arbitrary run metadata propagated from spicebench to adapters
     #[serde(default)]
     pub metadata: HashMap<String, serde_json::Value>,
@@ -169,9 +172,12 @@ pub struct SetupResponse {
 ///
 /// JSON-RPC method: `create_tables`
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct CreateTablesRequest {
     /// Unique identifier for this benchmark run
     pub run_id: Uuid,
+    /// Map of dataset name to dataset definition
+    pub datasets: HashMap<String, DatasetConfig>,
 }
 
 /// Response from create_tables request

crates/system-adapter-protocol/src/server.rs

@@ -75,14 +75,14 @@ pub trait Handler: Send + Sync {
     async fn setup(
         &mut self,
         run_id: Uuid,
-        datasets: HashMap<String, DatasetConfig>,
         metadata: HashMap<String, serde_json::Value>,
     ) -> std::result::Result<SetupResponse, String>;
 
     /// Create benchmark tables for a run
     async fn create_tables(
         &mut self,
         run_id: Uuid,
+        datasets: HashMap<String, DatasetConfig>,
     ) -> std::result::Result<CreateTablesResponse, String>;
 
     /// Teardown a benchmark run
@@ -243,9 +243,7 @@ impl<H: Handler> Server<H> {
             Err(e) => return e,
         };
         Self::handler_response(
-            self.handler
-                .setup(req.run_id, req.datasets, req.metadata)
-                .await,
+            self.handler.setup(req.run_id, req.metadata).await,
             id,
         )
     }
@@ -259,7 +257,7 @@ impl<H: Handler> Server<H> {
             Ok(r) => r,
             Err(e) => return e,
         };
-        Self::handler_response(self.handler.create_tables(req.run_id).await, id)
+        Self::handler_response(self.handler.create_tables(req.run_id, req.datasets).await, id)
     }
 
     async fn handle_teardown(
@@ -304,7 +302,6 @@ mod tests {
         async fn setup(
             &mut self,
             _run_id: Uuid,
-            _datasets: HashMap<String, DatasetConfig>,
             _metadata: HashMap<String, serde_json::Value>,
         ) -> std::result::Result<SetupResponse, String> {
             Ok(SetupResponse {
@@ -316,6 +313,7 @@ mod tests {
         async fn create_tables(
             &mut self,
             _run_id: Uuid,
+            _datasets: HashMap<String, DatasetConfig>,
         ) -> std::result::Result<CreateTablesResponse, String> {
             Ok(CreateTablesResponse { ok: true })
         }
@@ -335,7 +333,7 @@ mod tests {
     #[tokio::test]
     async fn test_server_setup() {
         let mut server = Server::new(TestHandler);
-        let request = r#"{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","datasets":{},"metadata":{}}}"#;
+        let request = r#"{"jsonrpc":"2.0","id":1,"method":"setup","params":{"run_id":"00000000-0000-0000-0000-000000000000","metadata":{}}}"#;
         let response = server.handle_request(request).await;
 
         assert!(response.get("result").is_some());

src/main.rs

@@ -37,7 +37,7 @@ mod scenario;
 use crate::commands::connect_system_adapter;
 use crate::scenario::Scenario;
 
-fn setup_request_datasets(
+fn create_tables_request_datasets(
     dataset: &Arc<dyn Dataset>,
 ) -> HashMap<String, system_adapter_protocol::DatasetConfig> {
     dataset
@@ -122,7 +122,7 @@ async fn main() -> anyhow::Result<()> {
     let mutations = MutationConfig::new(0.1, 0.1);
 
     let setup_dataset = dataset_source.create(&generation_config, &mutations)?;
-    let datasets = setup_request_datasets(&setup_dataset);
+    let datasets = create_tables_request_datasets(&setup_dataset);
 
     let setup_metadata = std::collections::HashMap::from([
         (
@@ -136,7 +136,7 @@ async fn main() -> anyhow::Result<()> {
     ]);
 
     let adbc_driver = match system_adapter_client
-        .setup(run_id, datasets, setup_metadata)
+        .setup(run_id, setup_metadata)
         .await
     {
         Ok(response) => response,
@@ -182,7 +182,7 @@ async fn main() -> anyhow::Result<()> {
         &mutations,
     )?;
 
-    if let Err(e) = system_adapter_client.create_tables(run_id).await {
+    if let Err(e) = system_adapter_client.create_tables(run_id, datasets).await {
         pipeline.cancel();
         return Err(anyhow::anyhow!(
             "Failed to create tables via system adapter: {e}"