Skip to content

Commit 09379a9

Browse files
phillipleblancphillipleblanc
authored
Merge pull request #65 from spiceai/lukim/spiceai-52-upgrade
Merge main and upgrade to DataFusion v52.3.0
2 parents 43c04eb + 051719c commit 09379a9

16 files changed

Lines changed: 482 additions & 82 deletions

File tree

‎.github/workflows/docs.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ jobs:
1515
name: Rustdoc
1616
runs-on: ubuntu-latest
1717
steps:
18-
- uses: actions/checkout@v4
18+
- uses: actions/checkout@v5
1919
with:
2020
fetch-depth: 0
2121
submodules: recursive
2222
- uses: dtolnay/rust-toolchain@nightly
23-
- uses: arduino/setup-protoc@v3
23+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
2424
with:
2525
repo-token: ${{ secrets.GITHUB_TOKEN }}
2626
- run: cargo rustdoc -p datafusion-federation -- --cfg docsrs

‎.github/workflows/integration-test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
uses: dtolnay/rust-toolchain@stable
2222

2323
- name: Setup protoc
24-
uses: arduino/setup-protoc@v3
24+
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
2525
with:
2626
repo-token: ${{ secrets.GITHUB_TOKEN }}
2727

‎.github/workflows/release_plz.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@ jobs:
1717
runs-on: ubuntu-latest
1818
steps:
1919
- name: Checkout repository
20-
uses: actions/checkout@v4
20+
uses: actions/checkout@v5
2121
with:
2222
fetch-depth: 0
2323
- name: Install Rust toolchain
2424
uses: dtolnay/rust-toolchain@stable
25-
- name: Install dependencies
26-
run: sudo apt install -y protobuf-compiler libprotobuf-dev
25+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
26+
with:
27+
repo-token: ${{ secrets.GITHUB_TOKEN }}
2728
- name: Run release-plz
2829
uses: MarcoIeni/release-plz-action@v0.5
2930
with:
@@ -41,7 +42,7 @@ jobs:
4142
cancel-in-progress: false
4243
steps:
4344
- name: Checkout repository
44-
uses: actions/checkout@v4
45+
uses: actions/checkout@v5
4546
with:
4647
fetch-depth: 0
4748
- name: Install Rust toolchain

‎.github/workflows/test.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ jobs:
1111
name: Check
1212
runs-on: ubuntu-latest
1313
steps:
14-
- uses: actions/checkout@v4
14+
- uses: actions/checkout@v5
1515
with:
1616
fetch-depth: 0
1717
submodules: recursive
1818
- uses: dtolnay/rust-toolchain@stable
19-
- uses: arduino/setup-protoc@v3
19+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
2020
with:
2121
repo-token: ${{ secrets.GITHUB_TOKEN }}
2222
- run: cargo check --all-features
@@ -25,12 +25,12 @@ jobs:
2525
name: Test
2626
runs-on: ubuntu-latest
2727
steps:
28-
- uses: actions/checkout@v4
28+
- uses: actions/checkout@v5
2929
with:
3030
fetch-depth: 0
3131
submodules: recursive
3232
- uses: dtolnay/rust-toolchain@stable
33-
- uses: arduino/setup-protoc@v3
33+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
3434
with:
3535
repo-token: ${{ secrets.GITHUB_TOKEN }}
3636
- run: cargo test --all-features
@@ -39,7 +39,7 @@ jobs:
3939
name: Rustfmt
4040
runs-on: ubuntu-latest
4141
steps:
42-
- uses: actions/checkout@v4
42+
- uses: actions/checkout@v5
4343
- uses: dtolnay/rust-toolchain@stable
4444
with:
4545
components: rustfmt
@@ -49,14 +49,14 @@ jobs:
4949
name: Clippy
5050
runs-on: ubuntu-latest
5151
steps:
52-
- uses: actions/checkout@v4
52+
- uses: actions/checkout@v5
5353
with:
5454
fetch-depth: 0
5555
submodules: recursive
5656
- uses: dtolnay/rust-toolchain@stable
5757
with:
5858
components: clippy
59-
- uses: arduino/setup-protoc@v3
59+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
6060
with:
6161
repo-token: ${{ secrets.GITHUB_TOKEN }}
6262
- run: cargo clippy -- -D warnings
@@ -65,12 +65,12 @@ jobs:
6565
name: Package
6666
runs-on: ubuntu-latest
6767
steps:
68-
- uses: actions/checkout@v4
68+
- uses: actions/checkout@v5
6969
with:
7070
fetch-depth: 0
7171
submodules: recursive
7272
- uses: dtolnay/rust-toolchain@stable
73-
- uses: arduino/setup-protoc@v3
73+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
7474
with:
7575
repo-token: ${{ secrets.GITHUB_TOKEN }}
7676
- run: cargo build --all --all-features

‎Cargo.toml‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,25 +12,32 @@ repository = "https://github.com/datafusion-contrib/datafusion-federation"
1212

1313
[workspace.dependencies]
1414
arrow-json = "57"
15-
async-stream = "0.3.5"
16-
async-trait = "0.1.83"
17-
datafusion = "52"
15+
async-stream = "0.3"
16+
async-trait = "0.1"
17+
datafusion = "52.3"
1818
datafusion-federation = { path = "./datafusion-federation", version = "0.4.2" }
19-
futures = "0.3.31"
19+
futures = "0.3"
2020
tokio = { version = "1.41", features = ["full"] }
2121

2222
[patch.crates-io]
23-
duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "2b2d90826ccba84435174e10c551dbdf3dae41f3" } # spiceai-57
23+
duckdb = { git = "https://github.com/spiceai/duckdb-rs.git", rev = "f4096c7592ed46b9e68755a49252d69783dece96" } # spiceai-52
2424

25-
datafusion = { git = "https://github.com/apache/datafusion.git", rev = "bc043c3a5fe448fc5735d9fa173b69ac95a7e11c" } # upstream:v52
26-
datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "bc043c3a5fe448fc5735d9fa173b69ac95a7e11c" } # upstream:v52
27-
datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "bc043c3a5fe448fc5735d9fa173b69ac95a7e11c" } # upstream:v52
28-
datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "bc043c3a5fe448fc5735d9fa173b69ac95a7e11c" } # upstream:v52
25+
datafusion = { git = "https://github.com/apache/datafusion.git", rev = "28d012a41a3017b5f682ef6b01468a7ff9a48fb7" } # upstream:v52.3.0
26+
datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "28d012a41a3017b5f682ef6b01468a7ff9a48fb7" } # upstream:v52.3.0
27+
datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "28d012a41a3017b5f682ef6b01468a7ff9a48fb7" } # upstream:v52.3.0
28+
datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "28d012a41a3017b5f682ef6b01468a7ff9a48fb7" } # upstream:v52.3.0
2929

30-
arrow = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
31-
arrow-buffer = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
32-
arrow-json = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
33-
arrow-ipc = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
34-
arrow-ord = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
35-
arrow-schema = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
36-
parquet = { git = "https://github.com/spiceai/arrow-rs.git", rev = "122357d9232caa9d512e4dc58176c9152f19595b" } # spiceai-57
30+
arrow = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
31+
arrow-array = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
32+
arrow-buffer = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
33+
arrow-cast = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
34+
arrow-data = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
35+
arrow-flight = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
36+
arrow-ipc = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
37+
arrow-json = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
38+
arrow-ord = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
39+
arrow-row = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
40+
arrow-schema = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
41+
arrow-select = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
42+
arrow-string = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches
43+
parquet = { git = "https://github.com/spiceai/arrow-rs.git", rev = "ca671dd37d73b730938f77f7a7ad76545280a4a8" } # spiceai-57.2-patches

‎datafusion-federation/CHANGELOG.md‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,94 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
## [0.5.1](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.5.0...v0.5.1) - 2026-03-04
11+
12+
### Added
13+
14+
- *(sql)* add SQL query rewriter hook to SQLTable ([#163](https://github.com/datafusion-contrib/datafusion-federation/pull/163))
15+
16+
### Fixed
17+
18+
- *(sql)* apply table-scan rewrite before EXPLAIN display ([#161](https://github.com/datafusion-contrib/datafusion-federation/pull/161))
19+
20+
## [0.5.0](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.14...v0.5.0) - 2026-02-26
21+
22+
### Other
23+
24+
- Pushdown physical filters to `SQLExecutor` ([#160](https://github.com/datafusion-contrib/datafusion-federation/pull/160))
25+
26+
## [0.4.14](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.13...v0.4.14) - 2026-01-14
27+
28+
### Other
29+
30+
- update Cargo.toml dependencies
31+
32+
## [0.4.13](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.12...v0.4.13) - 2026-01-13
33+
34+
### Other
35+
36+
- update Cargo.toml dependencies
37+
38+
## [0.4.12](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.11...v0.4.12) - 2025-12-04
39+
40+
### Other
41+
42+
- Fix conversion of empty `RecordBatch` ([#154](https://github.com/datafusion-contrib/datafusion-federation/pull/154))
43+
44+
## [0.4.11](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.10...v0.4.11) - 2025-11-24
45+
46+
### Other
47+
48+
- Upgrade datafusion to 51, arrow to 57 ([#151](https://github.com/datafusion-contrib/datafusion-federation/pull/151))
49+
50+
## [0.4.10](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.9...v0.4.10) - 2025-09-18
51+
52+
### Other
53+
54+
- Upgrade datafusion to version 50 ([#149](https://github.com/datafusion-contrib/datafusion-federation/pull/149))
55+
56+
## [0.4.9](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.8...v0.4.9) - 2025-08-19
57+
58+
### Other
59+
60+
- Cargo clippy 1.89 ([#146](https://github.com/datafusion-contrib/datafusion-federation/pull/146))
61+
62+
## [0.4.8](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.7...v0.4.8) - 2025-08-05
63+
64+
### Other
65+
66+
- Add metrics to `SchemaCastScanExec` and `VirtualExecutionPlan` ([#143](https://github.com/datafusion-contrib/datafusion-federation/pull/143))
67+
68+
## [0.4.7](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.6...v0.4.7) - 2025-07-29
69+
70+
### Other
71+
72+
- Upgrade datafusion to version 49 ([#140](https://github.com/datafusion-contrib/datafusion-federation/pull/140))
73+
74+
## [0.4.6](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.5...v0.4.6) - 2025-07-21
75+
76+
### Other
77+
78+
- Make VirtualExecutionPlan public ([#138](https://github.com/datafusion-contrib/datafusion-federation/pull/138))
79+
80+
## [0.4.5](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.4...v0.4.5) - 2025-07-09
81+
82+
### Other
83+
84+
- Add ability to set statistics in the SQLExecutor ([#134](https://github.com/datafusion-contrib/datafusion-federation/pull/134))
85+
86+
## [0.4.4](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.3...v0.4.4) - 2025-06-30
87+
88+
### Other
89+
90+
- Minor clippy fixes introduced in rust 1.88 ([#132](https://github.com/datafusion-contrib/datafusion-federation/pull/132))
91+
92+
## [0.4.3](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.2...v0.4.3) - 2025-06-22
93+
94+
### Other
95+
96+
- Update DataFusion to 48 ([#130](https://github.com/datafusion-contrib/datafusion-federation/pull/130))
97+
1098
## [0.4.2](https://github.com/datafusion-contrib/datafusion-federation/compare/v0.4.1...v0.4.2) - 2025-04-21
1199

12100
### Other

‎datafusion-federation/examples/shared/mod.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use datafusion::{
66
catalog::SchemaProvider,
77
error::{DataFusionError, Result},
88
execution::context::{SessionContext, SessionState},
9-
physical_plan::{stream::RecordBatchStreamAdapter, SendableRecordBatchStream},
9+
physical_plan::{stream::RecordBatchStreamAdapter, PhysicalExpr, SendableRecordBatchStream},
1010
sql::unparser::dialect::{DefaultDialect, Dialect},
1111
};
1212
use futures::TryStreamExt;
@@ -50,7 +50,12 @@ impl SQLExecutor for MockSqliteExecutor {
5050
Some("sqlite_exec".to_string())
5151
}
5252

53-
fn execute(&self, sql: &str, schema: SchemaRef) -> Result<SendableRecordBatchStream> {
53+
fn execute(
54+
&self,
55+
sql: &str,
56+
schema: SchemaRef,
57+
_filters: &[Arc<dyn PhysicalExpr>],
58+
) -> Result<SendableRecordBatchStream> {
5459
// Execute it using the remote datafusion session context
5560
let future_stream = _execute(self.session.clone(), sql.to_string());
5661
let stream = futures::stream::once(future_stream).try_flatten();
@@ -103,7 +108,12 @@ impl SQLExecutor for MockPostgresExecutor {
103108
Some("postgres_exec".to_string())
104109
}
105110

106-
fn execute(&self, sql: &str, schema: SchemaRef) -> Result<SendableRecordBatchStream> {
111+
fn execute(
112+
&self,
113+
sql: &str,
114+
schema: SchemaRef,
115+
_filters: &[Arc<dyn PhysicalExpr>],
116+
) -> Result<SendableRecordBatchStream> {
107117
// Execute it using the remote datafusion session context
108118
let future_stream = _execute(self.session.clone(), sql.to_string());
109119
let stream = futures::stream::once(future_stream).try_flatten();

‎datafusion-federation/src/analyzer/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ mod scan_result;
33
use crate::{FederatedTableProviderAdaptor, FederatedTableSource, FederationProviderRef};
44
use crate::{FederationAnalyzerForLogicalPlan, FederationProvider};
55
use datafusion::logical_expr::{col, expr::InSubquery, LogicalPlanBuilder};
6-
use datafusion::optimizer::eliminate_nested_union::EliminateNestedUnion;
6+
use datafusion::optimizer::optimize_unions::OptimizeUnions;
77
use datafusion::optimizer::push_down_filter::PushDownFilter;
88
use datafusion::optimizer::{Optimizer, OptimizerContext, OptimizerRule};
99
use datafusion::{
@@ -74,7 +74,7 @@ impl FederationAnalyzerRule {
7474

7575
pub fn default_optimizer_rules() -> Vec<Arc<dyn OptimizerRule + Send + Sync>> {
7676
vec![
77-
Arc::new(EliminateNestedUnion::new()),
77+
Arc::new(OptimizeUnions::new()),
7878
Arc::new(PushDownFilter::new()),
7979
]
8080
}

‎datafusion-federation/src/plan_node.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ use datafusion::{
1616
};
1717

1818
pub struct FederatedPlanNode {
19-
plan: LogicalPlan,
20-
planner: Arc<dyn FederationPlanner>,
19+
pub(crate) plan: LogicalPlan,
20+
pub(crate) planner: Arc<dyn FederationPlanner>,
2121
}
2222

2323
impl FederatedPlanNode {
@@ -28,6 +28,10 @@ impl FederatedPlanNode {
2828
pub fn plan(&self) -> &LogicalPlan {
2929
&self.plan
3030
}
31+
32+
pub fn planner(&self) -> &Arc<dyn FederationPlanner> {
33+
&self.planner
34+
}
3135
}
3236

3337
impl Debug for FederatedPlanNode {

0 commit comments

Comments
 (0)