Upgrade datafusion to version 49, datafusion-federation to version 0.4.7 (#407)

nuno-faria · web-flow · commit 2064c60b0683 · 2025-08-04T11:23:12.000-05:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -38,12 +38,12 @@ arrow-ipc = { version = "55.0.0" }
 arrow-schema = { version = "55.2.0", features = ["serde"] }
 arrow-json = "55.0.0"
 arrow-odbc = { version = "16.0.2" }
-datafusion = { version = "48", default-features = false }
-datafusion-expr = { version = "48" }
-datafusion-federation = { version = "0.4.6" }
-datafusion-ffi = { version = "48" }
-datafusion-proto = { version = "48" }
-datafusion-physical-expr = { version = "48" }
-datafusion-physical-plan = { version = "48" }
+datafusion = { version = "49", default-features = false }
+datafusion-expr = { version = "49" }
+datafusion-federation = { version = "0.4.7" }
+datafusion-ffi = { version = "49" }
+datafusion-proto = { version = "49" }
+datafusion-physical-expr = { version = "49" }
+datafusion-physical-plan = { version = "49" }
 datafusion-table-providers = { path = "core" }
 duckdb = { version = "=1.3.0", package = "spiceai_duckdb_fork" } # Forked to add support for duckdb_scan_arrow, pending: https://github.com/duckdb/duckdb-rs/pull/488
diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -63,7 +63,7 @@ num-bigint = "0.4"
 odbc-api = { version = "13.0", optional = true }
 pem = { version = "3.0.4", optional = true }
 postgres-native-tls = { version = "0.5.0", optional = true }
-prost = { version = "0.14", optional = true }
+prost = { version = "0.13.5", optional = true }
 rand = { version = "0.9" }
 regex = { version = "1" }
 r2d2 = { version = "0.8", optional = true }
@@ -105,7 +105,7 @@ anyhow = "1.0"
 bollard = "0.19"
 geozero = { version = "0.14.0", features = ["with-wkb"] }
 insta = { version = "1.43.1", features = ["filters"] }
-prost = { version = "0.14" }
+prost = { version = "0.13.5" }
 rand = "0.9"
 reqwest = "0.12"
 rstest = "0.25.0"
diff --git a/core/src/clickhouse.rs b/core/src/clickhouse.rs
@@ -53,7 +53,7 @@ impl ClickHouseTableFactory {
             args,
             self.pool.clone(),
             schema,
-            Constraints::empty(),
+            Constraints::default(),
         ));
 
         #[cfg(feature = "clickhouse-federation")]
diff --git a/core/src/duckdb.rs b/core/src/duckdb.rs
@@ -161,7 +161,9 @@ pub enum Error {
     #[snafu(display("Failed to register Arrow scan view for DuckDB ingestion: {source}"))]
     UnableToRegisterArrowScanView { source: duckdb::Error },
 
-    #[snafu(display("Failed to register Arrow scan view to build table creation statement: {source}"))]
+    #[snafu(display(
+        "Failed to register Arrow scan view to build table creation statement: {source}"
+    ))]
     UnableToRegisterArrowScanViewForTableCreation { source: duckdb::Error },
 
     #[snafu(display("Failed to drop Arrow scan view for DuckDB ingestion: {source}"))]
@@ -717,7 +719,7 @@ pub(crate) mod tests {
             order_exprs: vec![],
             unbounded: false,
             options,
-            constraints: Constraints::empty(),
+            constraints: Constraints::default(),
             column_defaults: HashMap::new(),
             temporary: false,
         };
@@ -778,7 +780,7 @@ pub(crate) mod tests {
             order_exprs: vec![],
             unbounded: false,
             options,
-            constraints: Constraints::empty(),
+            constraints: Constraints::default(),
             column_defaults: HashMap::new(),
             temporary: false,
         };
@@ -835,7 +837,7 @@ pub(crate) mod tests {
             order_exprs: vec![],
             unbounded: false,
             options,
-            constraints: Constraints::empty(),
+            constraints: Constraints::default(),
             column_defaults: HashMap::new(),
             temporary: false,
         };
@@ -890,7 +892,7 @@ pub(crate) mod tests {
             order_exprs: vec![],
             unbounded: false,
             options,
-            constraints: Constraints::empty(),
+            constraints: Constraints::default(),
             column_defaults: HashMap::new(),
             temporary: false,
         };
@@ -948,7 +950,7 @@ pub(crate) mod tests {
             order_exprs: vec![],
             unbounded: false,
             options,
-            constraints: Constraints::empty(),
+            constraints: Constraints::default(),
             column_defaults: HashMap::new(),
             temporary: false,
         };
diff --git a/core/src/duckdb/creator.rs b/core/src/duckdb/creator.rs
@@ -409,9 +409,8 @@ impl TableManager {
         tx.register_arrow_scan_view(&view_name, &stream)
             .context(super::UnableToRegisterArrowScanViewForTableCreationSnafu)?;
 
-        let sql = format!(
-            r#"CREATE TABLE IF NOT EXISTS "{table_name}" AS SELECT * FROM "{view_name}""#,
-        );
+        let sql =
+            format!(r#"CREATE TABLE IF NOT EXISTS "{table_name}" AS SELECT * FROM "{view_name}""#,);
         tracing::debug!("{sql}");
 
         tx.execute(&sql, [])
diff --git a/core/src/mysql.rs b/core/src/mysql.rs
@@ -152,7 +152,7 @@ impl MySQLTableFactory {
             table_name,
             Arc::clone(&self.pool),
             schema,
-            Constraints::empty(),
+            Constraints::default(),
         );
 
         Ok(MySQLTableWriter::create(read_provider, mysql, None))
diff --git a/core/src/postgres.rs b/core/src/postgres.rs
@@ -179,7 +179,7 @@ impl PostgresTableFactory {
             table_reference,
             Arc::clone(&self.pool),
             schema,
-            Constraints::empty(),
+            Constraints::default(),
         );
 
         Ok(PostgresTableWriter::create(read_provider, postgres, None))
diff --git a/core/src/sql/db_connection_pool/sqlitepool.rs b/core/src/sql/db_connection_pool/sqlitepool.rs
@@ -291,7 +291,7 @@ mod tests {
         assert!(pool.mode == Mode::File);
         assert_eq!(pool.path, db_name.clone().into());
 
-        drop(pool);
+        pool.conn.close().await.unwrap();
 
         // cleanup
         std::fs::remove_file(&db_name).unwrap();
@@ -324,7 +324,7 @@ mod tests {
         assert!(pool.mode == Mode::File);
         assert_eq!(pool.path, db_names[0].clone().into());
 
-        drop(pool);
+        pool.conn.close().await.unwrap();
 
         // cleanup
         for db in &db_names {
@@ -345,7 +345,7 @@ mod tests {
         assert!(pool.mode == Mode::File);
         assert_eq!(pool.path, db_name.clone().into());
 
-        drop(pool);
+        pool.conn.close().await.unwrap();
 
         // cleanup
         std::fs::remove_file(&db_name).unwrap();
@@ -365,7 +365,7 @@ mod tests {
         assert!(pool.mode == Mode::Memory);
         assert_eq!(pool.path, "./test.sqlite".into());
 
-        drop(pool);
+        pool.conn.close().await.unwrap();
 
         // in memory mode, attachments are not created and nothing happens
         assert!(std::fs::metadata("./test.sqlite").is_err());
diff --git a/core/src/sqlite/write.rs b/core/src/sqlite/write.rs
@@ -290,7 +290,7 @@ mod tests {
             order_exprs: vec![],
             unbounded: false,
             options: HashMap::new(),
-            constraints: Constraints::empty(),
+            constraints: Constraints::default(),
             column_defaults: HashMap::default(),
             temporary: false,
         };
diff --git a/core/tests/clickhouse/mod.rs b/core/tests/clickhouse/mod.rs
@@ -1,4 +1,3 @@
-use arrow::array::RecordBatch;
 use clickhouse::Client;
 use common::{get_clickhouse_params, start_clickhouse_docker_container};
 use datafusion::{prelude::SessionContext, sql::TableReference};
diff --git a/core/tests/duckdb/mod.rs b/core/tests/duckdb/mod.rs
@@ -32,7 +32,7 @@ async fn arrow_duckdb_round_trip(
         order_exprs: vec![],
         unbounded: false,
         options: HashMap::new(),
-        constraints: Constraints::empty(),
+        constraints: Constraints::default(),
         column_defaults: HashMap::new(),
         temporary: false,
     };
@@ -93,7 +93,8 @@ async fn arrow_duckdb_round_trip(
 #[case::struct_type(get_arrow_struct_record_batch(), "struct")]
 #[ignore] // DuckDB does not support Decimal256 / duckdb_arrow_scan failed to register view
 #[case::decimal(get_arrow_decimal_record_batch(), "decimal")]
-#[ignore] // Interval(DayTime) is not supported: / "Conversion Error: Could not convert Interval to Microsecond"
+#[ignore]
+// Interval(DayTime) is not supported: / "Conversion Error: Could not convert Interval to Microsecond"
 #[case::interval(get_arrow_interval_record_batch(), "interval")]
 #[ignore] // TimeUnit::Nanosecond is not correctly supported; written values are zeros
 #[case::duration(get_arrow_duration_record_batch(), "duration")]
diff --git a/core/tests/mysql/mod.rs b/core/tests/mysql/mod.rs
@@ -704,7 +704,7 @@ async fn arrow_mysql_round_trip(
             .into_iter()
             .map(|(k, v)| (k, v.expose_secret().to_string()))
             .collect(),
-        constraints: Constraints::empty(),
+        constraints: Constraints::default(),
         column_defaults: Default::default(),
     };
     let table_provider = factory
diff --git a/core/tests/postgres/mod.rs b/core/tests/postgres/mod.rs
@@ -47,7 +47,7 @@ async fn arrow_postgres_round_trip(
         order_exprs: vec![],
         unbounded: false,
         options: common::get_pg_params(port),
-        constraints: Constraints::empty(),
+        constraints: Constraints::default(),
         column_defaults: HashMap::new(),
         temporary: false,
     };
diff --git a/core/tests/postgres/schema.rs b/core/tests/postgres/schema.rs
@@ -63,7 +63,7 @@ async fn test_postgres_schema_inference() {
         order_exprs: vec![],
         unbounded: false,
         options: common::get_pg_params(port),
-        constraints: Constraints::empty(),
+        constraints: Constraints::default(),
         column_defaults: HashMap::new(),
         temporary: false,
     };
