Initial integration tests

Author: phillipleblanc

.gitignore

@@ -6,3 +6,4 @@ package.json
 .DS_Store
 .vscode/settings.json
 .cargo
+tpch_sf1.db

Cargo.toml

@@ -2,7 +2,8 @@
 resolver = "2"
 
 members = [
-    "datafusion-federation"
+    "datafusion-federation",
+    "integration-test"
 ]
 
 [workspace.package]
@@ -21,6 +22,12 @@ datafusion-federation = { path = "./datafusion-federation", version = "0.4.2" }
 futures = "0.3.31"
 tokio = { version = "1.41", features = ["full"] }
 
-#[patch.crates-io]
-#duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "2e24b958e44ec7419290249e27a15f1a19703fff" }
+[patch.crates-io]
+duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "2e24b958e44ec7419290249e27a15f1a19703fff" } # spiceai-1.1.3-backported-arrow-54
 
+datafusion-federation = { path = "./datafusion-federation" }
+
+datafusion = { git = "https://github.com/spiceai/datafusion.git", rev = "987d143985657065c8fc4d3d347a2010fbf36fe4" }               # spiceai-46
+datafusion-expr = { git = "https://github.com/spiceai/datafusion.git", rev = "987d143985657065c8fc4d3d347a2010fbf36fe4" }          # spiceai-46
+datafusion-physical-expr = { git = "https://github.com/spiceai/datafusion.git", rev = "987d143985657065c8fc4d3d347a2010fbf36fe4" } # spiceai-46
+datafusion-physical-plan = { git = "https://github.com/spiceai/datafusion.git", rev = "987d143985657065c8fc4d3d347a2010fbf36fe4" } # spiceai-46

datafusion-federation/src/analyzer/mod.rs

@@ -3,13 +3,8 @@ mod scan_result;
 use crate::FederationProvider;
 use crate::{FederatedTableProviderAdaptor, FederatedTableSource, FederationProviderRef};
 use datafusion::logical_expr::{col, expr::InSubquery, LogicalPlanBuilder};
-use datafusion::optimizer::decorrelate_predicate_subquery::DecorrelatePredicateSubquery;
-use datafusion::optimizer::eliminate_join::EliminateJoin;
 use datafusion::optimizer::eliminate_nested_union::EliminateNestedUnion;
 use datafusion::optimizer::push_down_filter::PushDownFilter;
-use datafusion::optimizer::replace_distinct_aggregate::ReplaceDistinctWithAggregate;
-use datafusion::optimizer::scalar_subquery_to_join::ScalarSubqueryToJoin;
-use datafusion::optimizer::simplify_expressions::SimplifyExpressions;
 use datafusion::optimizer::{Optimizer, OptimizerContext};
 use datafusion::{
     common::tree_node::{Transformed, TreeNode, TreeNodeRecursion},
@@ -24,7 +19,7 @@ use scan_result::ScanResult;
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use optimize_projections::OptimizeProjections;
+pub use optimize_projections::OptimizeProjections;
 
 mod optimize_projections;
 
@@ -72,11 +67,6 @@ impl Default for FederationAnalyzerRule {
         Self {
             optimizer: Optimizer::with_rules(vec![
                 Arc::new(EliminateNestedUnion::new()),
-                Arc::new(SimplifyExpressions::new()),
-                Arc::new(ReplaceDistinctWithAggregate::new()),
-                Arc::new(EliminateJoin::new()),
-                Arc::new(DecorrelatePredicateSubquery::new()),
-                Arc::new(ScalarSubqueryToJoin::new()),
                 Arc::new(PushDownFilter::new()),
                 Arc::new(OptimizeProjections::new()),
             ]),

datafusion-federation/src/sql/analyzer/rewrite_table_scan.rs

@@ -840,7 +840,7 @@ mod tests {
             ),
             (
                 "SELECT foo.df_table.a FROM foo.df_table",
-                r#"SELECT remote_table.a FROM "default".remote_table"#,
+                r#"SELECT a FROM "default".remote_table"#,
             ),
             (
                 "SELECT MIN(a) FROM foo.df_table",

datafusion-federation/src/sql/mod.rs

@@ -584,7 +584,7 @@ mod tests {
         });
 
         let expected = vec![
-            r#"SELECT "table".a, "table".b, "table".c FROM "default"."table" UNION ALL SELECT "Table".a, "Table".b, "Table".c FROM "default"."Table"(1) AS Table"#,
+            r#"SELECT a, b, c FROM "default"."table" UNION ALL SELECT a, b, c FROM "default"."Table"(1) AS Table"#,
         ];
 
         assert_eq!(

integration-test/Cargo.toml

@@ -10,12 +10,12 @@ publish = false
 [dependencies]
 anyhow = "1.0.98"
 duckdb = "1.1.3"
-datafusion-table-providers = { git = "https://github.com/datafusion-contrib/datafusion-table-providers.git", rev = "dddeb8cf1c0b12968cb599aa22497920b3eadfdc", features = ["duckdb", "duckdb-federation"] } #spiceai
+datafusion-table-providers = { git = "https://github.com/datafusion-contrib/datafusion-table-providers.git", rev = "11e6ee6ce966bed75bfdf75204193503f1593cef", features = ["duckdb", "duckdb-federation"] } # phillip/250527-federation-upgrade
 datafusion = { workspace = true }
 tokio = { version = "1.35", features = ["rt", "rt-multi-thread", "macros"] }
 async-trait.workspace = true
 tonic = "0.12.2"
 futures.workspace = true
 insta = { version = "1.42.0", features = ["filters"]}
 chrono = "0.4.31"
-datafusion-federation.path = "../datafusion-federation"
+datafusion-federation.workspace = true