fix: Move TableFormat into spicebench

Author: peasee

crates/data-generation/src/config.rs

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-use clap::{Parser, Subcommand, ValueEnum};
+use clap::{Parser, Subcommand};
 
 #[derive(Parser)]
 #[command(about = "Spice.ai data generation tool - generates Arrow data and writes to S3")]
@@ -63,14 +63,6 @@ pub struct CommonArgs {
     #[arg(long, default_value = "")]
     pub prefix: String,
 
-    /// Logical table format propagated to system adapters
-    #[arg(long, value_enum, default_value = "parquet")]
-    pub table_format: TableFormat,
-
-    /// Executor instance type label propagated to adapters for dashboarding
-    #[arg(long, default_value = "unknown")]
-    pub executor_instance_type: String,
-
     /// AWS region
     #[arg(long)]
     pub region: Option<String>,
@@ -93,31 +85,10 @@ pub struct DatasetConfig {
 pub struct TargetConfig {
     pub bucket: String,
     pub prefix: String,
-    pub table_format: TableFormat,
-    pub executor_instance_type: String,
     pub region: Option<String>,
     pub endpoint: Option<String>,
 }
 
-#[derive(Clone, Debug, ValueEnum)]
-#[value(rename_all = "lower")]
-pub enum TableFormat {
-    Iceberg,
-    Parquet,
-    Delta,
-}
-
-impl std::fmt::Display for TableFormat {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let value = match self {
-            Self::Iceberg => "iceberg",
-            Self::Parquet => "parquet",
-            Self::Delta => "delta",
-        };
-        write!(f, "{value}")
-    }
-}
-
 pub struct IngestorConfig {
     pub max_concurrency: usize,
 }
@@ -135,8 +106,6 @@ impl CommonArgs {
         TargetConfig {
             bucket: self.bucket.clone(),
             prefix: self.prefix.clone(),
-            table_format: self.table_format.clone(),
-            executor_instance_type: self.executor_instance_type.clone(),
             region: self.region.clone(),
             endpoint: self.endpoint.clone(),
         }

crates/etl/src/main.rs

@@ -17,7 +17,7 @@ limitations under the License.
 use std::sync::Arc;
 
 use clap::Parser;
-use data_generation::config::{DatasetConfig, TableFormat, TargetConfig};
+use data_generation::config::{DatasetConfig, TargetConfig};
 use data_generation::storage::s3::S3Storage;
 use etl::{DatasetSource, ETLPipeline, PipelineState, StopReason};
 use tracing_subscriber::EnvFilter;
@@ -50,14 +50,6 @@ struct Cli {
     #[arg(long, default_value = "")]
     target_base_prefix: String,
 
-    /// Logical table format propagated to system adapters
-    #[arg(long, value_enum, default_value = "parquet")]
-    table_format: TableFormat,
-
-    /// Executor instance type label propagated to adapters for dashboarding
-    #[arg(long, default_value = "unknown")]
-    executor_instance_type: String,
-
     /// AWS region
     #[arg(long)]
     region: Option<String>,
@@ -90,8 +82,6 @@ impl Cli {
         TargetConfig {
             bucket: self.bucket.clone(),
             prefix: self.source_prefix.clone(),
-            table_format: self.table_format.clone(),
-            executor_instance_type: self.executor_instance_type.clone(),
             region: self.region.clone(),
             endpoint: self.endpoint.clone(),
         }
@@ -107,8 +97,6 @@ impl Cli {
         TargetConfig {
             bucket: self.bucket.clone(),
             prefix,
-            table_format: self.table_format.clone(),
-            executor_instance_type: self.executor_instance_type.clone(),
             region: self.region.clone(),
             endpoint: self.endpoint.clone(),
         }

src/args/mod.rs

@@ -15,11 +15,29 @@ limitations under the License.
 */
 
 use clap::{ArgAction, Parser, ValueEnum};
-use data_generation::config::TableFormat;
 
 mod dataset;
 use crate::scenario::Scenario;
 
+#[derive(Clone, Debug, ValueEnum)]
+#[value(rename_all = "lower")]
+pub enum TableFormat {
+    Iceberg,
+    Parquet,
+    Delta,
+}
+
+impl std::fmt::Display for TableFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let value = match self {
+            Self::Iceberg => "iceberg",
+            Self::Parquet => "parquet",
+            Self::Delta => "delta",
+        };
+        write!(f, "{value}")
+    }
+}
+
 /// Arguments Common to all [`TestCommands`].
 #[derive(Parser, Debug, Clone)]
 pub struct CommonArgs {

src/main.rs

@@ -79,8 +79,6 @@ async fn main() -> anyhow::Result<()> {
         prefix: cli.common.etl_source_prefix.clone(),
         region: cli.common.etl_region.clone(),
         endpoint: cli.common.etl_endpoint.clone(),
-        table_format: cli.common.table_format.clone(),
-        executor_instance_type: cli.common.executor_instance_type.clone(),
     };
 
     let run_suffix = Uuid::new_v4().to_string();
@@ -96,8 +94,6 @@ async fn main() -> anyhow::Result<()> {
         prefix: target_prefix,
         region: cli.common.etl_region.clone(),
         endpoint: cli.common.etl_endpoint.clone(),
-        table_format: cli.common.table_format.clone(),
-        executor_instance_type: cli.common.executor_instance_type.clone(),
     };
 
     let source = Arc::new(S3Storage::new(&source_config)?);
@@ -124,7 +120,7 @@ async fn main() -> anyhow::Result<()> {
     let setup_metadata = std::collections::HashMap::from([(
         "executor_instance_type".to_string(),
         serde_json::Value::String(cli.common.executor_instance_type.clone()),
-    )]);
+    ), ("table_format".to_string(), serde_json::Value::String(cli.common.table_format.to_string()))]);
 
     if let Err(e) = system_adapter_client
         .setup(run_id, datasets, setup_metadata)