feat: Connect ETL to spicebench (#49)

* feat: Connect ETL pipeline in spicebench

* fix: Update run spicebench args

* fix

* fix

* chore: fmt, move pipeline start to load run

* fix log

* formatting

---------

Co-authored-by: jeadie <jack@spice.ai>

Author: 2 people

.github/workflows/run_spicebench.yml

@@ -219,7 +219,6 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           RUST_LOG: 'info'
-            RUST_LOG: 'info'
         run: |
           ETL_ARGS="--etl-bucket ${ETL_BUCKET} --etl-num-steps ${ETL_NUM_STEPS}"
           if [ -n "${ETL_SOURCE_PREFIX}" ]; then

src/main.rs

@@ -84,17 +84,18 @@ async fn main() -> anyhow::Result<()> {
         executor_instance_type: cli.common.executor_instance_type.clone(),
     };
 
-    let run_suffix = Uuid::new_v4().to_string();
-    let target_prefix = if cli.common.etl_target_base_prefix.is_empty() {
-        run_suffix.clone()
-    } else {
-        format!("{}/{run_suffix}", cli.common.etl_target_base_prefix)
-    };
-    tracing::info!(target_prefix = %target_prefix, "Generated unique ETL target prefix");
-
     let target_config = TargetConfig {
         bucket: cli.common.etl_bucket.clone(),
-        prefix: target_prefix,
+        prefix: {
+            let run_suffix = Uuid::new_v4().to_string();
+            let target_prefix = if cli.common.etl_target_base_prefix.is_empty() {
+                run_suffix.clone()
+            } else {
+                format!("{}/{run_suffix}", cli.common.etl_target_base_prefix)
+            };
+            tracing::info!(target_prefix = %target_prefix, "Generated unique ETL target prefix");
+            target_prefix
+        },
         region: cli.common.etl_region.clone(),
         endpoint: cli.common.etl_endpoint.clone(),
         table_format: cli.common.table_format.clone(),