fix: Remove more dead code, remove datafusion dependency (#104)

* fix: Remove more dead code, remove datafusion dependency

* fix: Remove unused dependencies

Author: peasee

Cargo.lock

@@ -31,37 +31,23 @@ arrow = { version = "57.0.0", features = ["prettyprint"] }
 arrow-array = "57.0.0"
 arrow-buffer = "57.0.0"
 arrow-flight = "57.0.0"
-arrow-json = "57.0.0"
 arrow-schema = "57.0.0"
-async-openai = { git = "https://github.com/spiceai/async-openai", rev = "7d5d31718143bf847cbc46286d4be26f035b58a0", features = [
-    "byot",
-    "embedding",
-    "chat-completion",
-    "responses",
-    "model",
-] }
 async-trait = "0.1.89"
-aws-config = "1.8.13"
-aws-credential-types = "1.2.11"
-aws-sdk-dynamodb = "1.104.0"
 base64 = "0.22.1"
 byte-unit = "5.1.4"
 bytes = "1.10.0"
 chrono = "0.4.42"
 clap = { version = "4.5.54", features = ["derive", "env"] }
-datafusion = "51.0.0"
 duckdb = "1.4.4"
 duration-parse = { path = "crates/duration-parse" }
 tpchgen = "2.0.2"
 tpchgen-arrow = "2.0.2"
 futures = "0.3.30"
-hf-hub = { version = "0.4", features = ["tokio"] }
 http = "1.3.1"
 humantime = "2.1.0"
-indexmap = "2"
+indexmap = { version = "2", features = ["serde"] }
 insta = { version = "1.46.1", features = ["filters"] }
-itertools = "0.14"
-nix = "0.30"
+nix = { version = "0.30", features = ["signal"] }
 object_store = { version = "0.12.4", features = [
     "aws",
     "gcp",
@@ -121,7 +107,6 @@ url = { version = "2.5", features = ["serde"] }
 util = { path = "crates/util" }
 uuid = "1"
 
-spiceai = { git = "https://github.com/spiceai/spice-rs.git", rev = "4c63970cf1d78c71eb69e2f4346d5d4795631b47" } # branch: trunk
 
 [package]
 edition.workspace = true
@@ -142,7 +127,6 @@ checkpointer = { path = "crates/checkpointer" }
 clap.workspace = true
 data-generation = { path = "crates/data-generation" }
 etl = { path = "crates/etl" }
-reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 system-adapter-protocol = { path = "crates/system-adapter-protocol" }

Cargo.toml

@@ -13,7 +13,7 @@ serde.workspace = true
 snafu.workspace = true
 spicepod = { path = "../spicepod" }
 tracing.workspace = true
-util = { path = "../util" }
+util.workspace = true
 
 [features]
 default = []

crates/app/Cargo.toml

@@ -11,18 +11,14 @@ version.workspace = true
 [dependencies]
 anyhow.workspace = true
 arrow.workspace = true
-async-trait.workspace = true
-chrono.workspace = true
 clap = { workspace = true, features = ["derive"] }
 data-generation = { path = "../data-generation" }
 etl = { path = "../etl", features = ["duckdb"] }
 test-framework = { path = "../test-framework" }
 object_store = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
-system-adapter-protocol = { path = "../system-adapter-protocol" }
 parquet.workspace = true
 tokio.workspace = true
-tokio-util.workspace = true
 tracing.workspace = true
 tracing-subscriber = { workspace = true, features = ["env-filter"] }

crates/checkpointer/Cargo.toml

@@ -21,7 +21,7 @@ snafu.workspace = true
 sysinfo.workspace = true
 tokio.workspace = true
 tracing.workspace = true
-util = { path = "../util" }
+util.workspace = true
 
 [features]
 anonymous_telemetry = []

crates/telemetry/Cargo.toml

@@ -15,7 +15,6 @@ async-trait.workspace = true
 chrono.workspace = true
 clap = { workspace = true, features = ["derive"] }
 dashmap = "6.1.0"
-datafusion.workspace = true
 flight_client = { path = "../flight_client" }
 futures.workspace = true
 indicatif = "0.18.0"
@@ -32,16 +31,17 @@ serde.workspace = true
 serde_json.workspace = true
 yaml = { path = "../yaml" }
 spicepod = { path = "../spicepod" }
-sysinfo = "0.37.2"
+sysinfo.workspace = true
 telemetry = { path = "../telemetry" }
 tempfile.workspace = true
 tokio.workspace = true
 tokio-util.workspace = true
 tonic.workspace = true
 uuid = { workspace = true, features = ["v4"] }
+sqlparser = { version = "0.61", features = ["visitor"] }
 
 [target.'cfg(not(windows))'.dependencies]
-nix = { version = "0.30", features = ["signal"] }
+nix = { workspace = true }
 
 [dev-dependencies]
 rstest = "0.26.1"

crates/test-framework/Cargo.toml

@@ -233,12 +233,12 @@ impl Query {
     /// - The SQL query cannot be parsed
     /// - The query contains multiple statements (only single statements are supported)
     pub fn rewrite_with_reference_schema(&self, reference_schema: &str) -> anyhow::Result<Self> {
-        use datafusion::sql::sqlparser::ast::{Ident, ObjectNamePart, visit_relations_mut};
-        use datafusion::sql::sqlparser::parser::Parser;
+        use sqlparser::ast::{Ident, ObjectNamePart, visit_relations_mut};
+        use sqlparser::parser::Parser;
         use std::ops::ControlFlow;
 
         // Parse the SQL query using sqlparser
-        let dialect = datafusion::sql::sqlparser::dialect::PostgreSqlDialect {};
+        let dialect = sqlparser::dialect::PostgreSqlDialect {};
         let mut statements = Parser::parse_sql(&dialect, &self.sql).map_err(|e| {
             anyhow::anyhow!(
                 "Failed to parse query '{}' for reference schema rewrite: {}",

crates/test-framework/src/queries/mod.rs

@@ -10,16 +10,8 @@ version.workspace = true
 
 [dependencies]
 backoff = { version = "0.4.0", features = ["futures", "tokio"] }
-chrono.workspace = true
 ctrlc = "3.5"
-futures.workspace = true
 humantime.workspace = true
-rand.workspace = true
-serde_json.workspace = true
 tokio = { workspace = true, features = ["signal", "sync", "time"] }
 tracing = { workspace = true }
 tracing-subscriber.workspace = true
-arrow.workspace = true
-arrow-schema.workspace = true
-datafusion.workspace = true
-parking_lot.workspace = true