Merge remote-tracking branch 'origin/main' into lukim/explain

Author: lukekim

.github/workflows/docs.yml

@@ -15,7 +15,7 @@ jobs:
     name: Rustdoc
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive

.github/workflows/release_plz.yml

@@ -17,13 +17,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
-      - name: Install dependencies 
-        run: sudo apt install -y protobuf-compiler libprotobuf-dev
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run release-plz
         uses: MarcoIeni/[email protected]
         with:
@@ -41,7 +42,7 @@ jobs:
       cancel-in-progress: false
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Install Rust toolchain

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     name: Check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive
@@ -25,7 +25,7 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive
@@ -39,7 +39,7 @@ jobs:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt
@@ -49,7 +49,7 @@ jobs:
     name: Clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive
@@ -65,7 +65,7 @@ jobs:
     name: Package
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive

Cargo.toml

@@ -1,36 +1,20 @@
 [workspace]
 resolver = "2"
 
-members = ["datafusion-federation", "integration-test"]
+members = ["datafusion-federation"]
 
 [workspace.package]
-version = "0.4.2"
+version = "0.5.1"
 edition = "2021"
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/datafusion-contrib/datafusion-federation"
 
 [workspace.dependencies]
-arrow-json = "56"
-async-stream = "0.3.5"
-async-trait = "0.1.83"
-datafusion = "50"
-datafusion-federation = { path = "./datafusion-federation", version = "0.4.2" }
-futures = "0.3.31"
+arrow-json = "57.2"
+async-stream = "0.3"
+async-trait = "0.1"
+datafusion = "52.0"
+datafusion-federation = { path = "./datafusion-federation", version = "0.5.1" }
+futures = "0.3"
 tokio = { version = "1.41", features = ["full"] }
-
-[patch.crates-io]
-duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "dd02045c3aa77895723e873222cbe30f5c8f77a9" } # spiceai-1.3.2
-
-datafusion = { git = "https://github.com/spiceai/datafusion.git", rev = "cd6b2f85c4b1d9d0b450aacd7f3498a1dea48ec4" }               # spiceai-50
-datafusion-expr = { git = "https://github.com/spiceai/datafusion.git", rev = "cd6b2f85c4b1d9d0b450aacd7f3498a1dea48ec4" }          # spiceai-50
-datafusion-physical-expr = { git = "https://github.com/spiceai/datafusion.git", rev = "cd6b2f85c4b1d9d0b450aacd7f3498a1dea48ec4" } # spiceai-50
-datafusion-physical-plan = { git = "https://github.com/spiceai/datafusion.git", rev = "cd6b2f85c4b1d9d0b450aacd7f3498a1dea48ec4" } # spiceai-50
-
-arrow = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" }        # spiceai-56
-arrow-buffer = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" } # spiceai-56
-arrow-json = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" }   # spiceai-56
-arrow-ipc = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" }    # spiceai-56
-arrow-ord = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" }    # spiceai-56
-arrow-schema = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" } # spiceai-56
-parquet = { git = "https://github.com/spiceai/arrow-rs.git", rev = "9f9c372ff5744488226462e3ea7d94ff47909833" }      # spiceai-56

datafusion-federation/CHANGELOG.md

@@ -7,6 +7,94 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.1](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.5.0...v0.5.1) - 2026-03-04
+
+### Added
+
+- *(sql)* add SQL query rewriter hook to SQLTable ([#163](https://github.com/datafusion-contrib/datafusion-federation/pull/163))
+
+### Fixed
+
+- *(sql)* apply table-scan rewrite before EXPLAIN display ([#161](https://github.com/datafusion-contrib/datafusion-federation/pull/161))
+
+## [0.5.0](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.14...v0.5.0) - 2026-02-26
+
+### Other
+
+- Pushdown physical filters to `SQLExecutor` ([#160](https://github.com/datafusion-contrib/datafusion-federation/pull/160))
+
+## [0.4.14](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.13...v0.4.14) - 2026-01-14
+
+### Other
+
+- update Cargo.toml dependencies
+
+## [0.4.13](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.12...v0.4.13) - 2026-01-13
+
+### Other
+
+- update Cargo.toml dependencies
+
+## [0.4.12](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.11...v0.4.12) - 2025-12-04
+
+### Other
+
+- Fix conversion of empty `RecordBatch` ([#154](https://github.com/datafusion-contrib/datafusion-federation/pull/154))
+
+## [0.4.11](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.10...v0.4.11) - 2025-11-24
+
+### Other
+
+- Upgrade datafusion to 51, arrow to 57 ([#151](https://github.com/datafusion-contrib/datafusion-federation/pull/151))
+
+## [0.4.10](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.9...v0.4.10) - 2025-09-18
+
+### Other
+
+- Upgrade datafusion to version 50 ([#149](https://github.com/datafusion-contrib/datafusion-federation/pull/149))
+
+## [0.4.9](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.8...v0.4.9) - 2025-08-19
+
+### Other
+
+- Cargo clippy 1.89 ([#146](https://github.com/datafusion-contrib/datafusion-federation/pull/146))
+
+## [0.4.8](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.7...v0.4.8) - 2025-08-05
+
+### Other
+
+- Add metrics to `SchemaCastScanExec` and `VirtualExecutionPlan` ([#143](https://github.com/datafusion-contrib/datafusion-federation/pull/143))
+
+## [0.4.7](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.6...v0.4.7) - 2025-07-29
+
+### Other
+
+- Upgrade datafusion to version 49 ([#140](https://github.com/datafusion-contrib/datafusion-federation/pull/140))
+
+## [0.4.6](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.5...v0.4.6) - 2025-07-21
+
+### Other
+
+- Make VirtualExecutionPlan public ([#138](https://github.com/datafusion-contrib/datafusion-federation/pull/138))
+
+## [0.4.5](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.4...v0.4.5) - 2025-07-09
+
+### Other
+
+- Add ability to set statistics in the SQLExecutor ([#134](https://github.com/datafusion-contrib/datafusion-federation/pull/134))
+
+## [0.4.4](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.3...v0.4.4) - 2025-06-30
+
+### Other
+
+- Minor clippy fixes introduced in rust 1.88 ([#132](https://github.com/datafusion-contrib/datafusion-federation/pull/132))
+
+## [0.4.3](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.2...v0.4.3) - 2025-06-22
+
+### Other
+
+- Update DataFusion to 48 ([#130](https://github.com/datafusion-contrib/datafusion-federation/pull/130))
+
 ## [0.4.2](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.1...v0.4.2) - 2025-04-21
 
 ### Other

datafusion-federation/Cargo.toml

@@ -26,12 +26,11 @@ async-trait.workspace = true
 datafusion.workspace = true
 async-stream.workspace = true
 arrow-json.workspace = true
-tracing = "0.1"
 
 [dev-dependencies]
 tokio.workspace = true
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
-insta = { version = "1.42.0", features = ["filters"] }
+tracing = "0.1.40"
 
 [[example]]
 name = "df-csv"

datafusion-federation/examples/df-csv-advanced.rs

@@ -6,12 +6,12 @@ use datafusion::{
     execution::{
         context::SessionContext, options::CsvReadOptions, session_state::SessionStateBuilder,
     },
-    optimizer::Analyzer,
+    optimizer::Optimizer,
 };
 
 use datafusion_federation::{
     sql::{MultiSchemaProvider, SQLFederationProvider, SQLSchemaProvider},
-    FederatedQueryPlanner, FederationAnalyzerRule,
+    FederatedQueryPlanner, FederationOptimizerRule,
 };
 
 use shared::{overwrite_default_schema, MockPostgresExecutor, MockSqliteExecutor};
@@ -115,15 +115,15 @@ async fn main() {
     /////////////////////
     //  Main(local)  DB
     /////////////////////
-    // Get the default analyzer rules
-    let mut rules = Analyzer::new().rules;
+    // Get the default optimizer rules
+    let mut rules = Optimizer::new().rules;
 
-    // Create a new federation analyzer rule and add it to the default rules
-    rules.push(Arc::new(FederationAnalyzerRule::new()));
+    // Create a new federation optimizer rule and add it to the default rules
+    rules.push(Arc::new(FederationOptimizerRule::new()));
 
-    // Create a new SessionState with the analyzer rule we created above
+    // Create a new SessionState with the optimizer rule we created above
     let state = SessionStateBuilder::new()
-        .with_analyzer_rules(rules)
+        .with_optimizer_rules(rules)
         .with_query_planner(Arc::new(FederatedQueryPlanner::new()))
         .build();
 

datafusion-federation/examples/shared/mod.rs

@@ -6,7 +6,7 @@ use datafusion::{
     catalog::SchemaProvider,
     error::{DataFusionError, Result},
     execution::context::{SessionContext, SessionState},
-    physical_plan::{stream::RecordBatchStreamAdapter, SendableRecordBatchStream},
+    physical_plan::{stream::RecordBatchStreamAdapter, PhysicalExpr, SendableRecordBatchStream},
     sql::unparser::dialect::{DefaultDialect, Dialect},
 };
 use futures::TryStreamExt;
@@ -50,7 +50,12 @@ impl SQLExecutor for MockSqliteExecutor {
         Some("sqlite_exec".to_string())
     }
 
-    fn execute(&self, sql: &str, schema: SchemaRef) -> Result<SendableRecordBatchStream> {
+    fn execute(
+        &self,
+        sql: &str,
+        schema: SchemaRef,
+        _filters: &[Arc<dyn PhysicalExpr>],
+    ) -> Result<SendableRecordBatchStream> {
         // Execute it using the remote datafusion session context
         let future_stream = _execute(self.session.clone(), sql.to_string());
         let stream = futures::stream::once(future_stream).try_flatten();
@@ -103,7 +108,12 @@ impl SQLExecutor for MockPostgresExecutor {
         Some("postgres_exec".to_string())
     }
 
-    fn execute(&self, sql: &str, schema: SchemaRef) -> Result<SendableRecordBatchStream> {
+    fn execute(
+        &self,
+        sql: &str,
+        schema: SchemaRef,
+        _filters: &[Arc<dyn PhysicalExpr>],
+    ) -> Result<SendableRecordBatchStream> {
         // Execute it using the remote datafusion session context
         let future_stream = _execute(self.session.clone(), sql.to_string());
         let stream = futures::stream::once(future_stream).try_flatten();