upgrade to datafusion 45 (#38)

* upgrade to datafusion 45

* fix test

Author: suremarc

Cargo.toml

@@ -28,19 +28,19 @@ keywords = ["arrow", "arrow-rs", "datafusion"]
 rust-version = "1.80"
 
 [dependencies]
-arrow = "53"
-arrow-schema = "53"
+arrow = "54"
+arrow-schema = "54"
 async-trait = "0.1"
 dashmap = "6"
-datafusion = "43"
-datafusion-common = "43"
-datafusion-expr = "43"
-datafusion-functions = "43"
-datafusion-functions-aggregate = "43"
-datafusion-optimizer = "43"
-datafusion-physical-expr = "43"
-datafusion-physical-plan = "43"
-datafusion-sql = "43"
+datafusion = "45"
+datafusion-common = "45"
+datafusion-expr = "45"
+datafusion-functions = "45"
+datafusion-functions-aggregate = "45"
+datafusion-optimizer = "45"
+datafusion-physical-expr = "45"
+datafusion-physical-plan = "45"
+datafusion-sql = "45"
 futures = "0.3"
 itertools = "0.13"
 log = "0.4"

src/materialized/dependencies.rs

@@ -16,9 +16,9 @@
 // under the License.
 
 use datafusion::{
-    catalog::CatalogProviderList,
+    catalog::{CatalogProviderList, TableFunctionImpl},
     config::{CatalogOptions, ConfigOptions},
-    datasource::{function::TableFunctionImpl, provider_as_source, TableProvider, ViewTable},
+    datasource::{provider_as_source, TableProvider, ViewTable},
     prelude::{flatten, get_field, make_array},
 };
 use datafusion_common::{
@@ -1549,9 +1549,9 @@ mod test {
                 name: "window & unnest",
                 query_to_analyze: "
                 SELECT
-                    \"unnest_placeholder(date).year\" AS year,
-                    \"unnest_placeholder(date).month\" AS month,
-                    \"unnest_placeholder(date).day\" AS day,
+                    \"__unnest_placeholder(date).year\" AS year,
+                    \"__unnest_placeholder(date).month\" AS month,
+                    \"__unnest_placeholder(date).day\" AS day,
                     arr
                 FROM (
                     SELECT
@@ -1568,14 +1568,14 @@ mod test {
                 )",
                 projection: &["year", "month", "day"],
                 expected_plan: vec![
-                    "+--------------+---------------------------------------------------------------------------------------------------------------------------------+",
-                    "| plan_type    | plan                                                                                                                            |",
-                    "+--------------+---------------------------------------------------------------------------------------------------------------------------------+",
-                    "| logical_plan | Projection: unnest_placeholder(date).year AS year, unnest_placeholder(date).month AS month, unnest_placeholder(date).day AS day |",
-                    "|              |   Unnest: lists[] structs[unnest_placeholder(date)]                                                                             |",
-                    "|              |     Projection: named_struct(Utf8(\"year\"), t2.year, Utf8(\"month\"), t2.month, Utf8(\"day\"), t2.day) AS unnest_placeholder(date)   |",
-                    "|              |       TableScan: t2 projection=[year, month, day]                                                                               |",
-                    "+--------------+---------------------------------------------------------------------------------------------------------------------------------+",
+                    "+--------------+---------------------------------------------------------------------------------------------------------------------------------------+",
+                    "| plan_type    | plan                                                                                                                                  |",
+                    "+--------------+---------------------------------------------------------------------------------------------------------------------------------------+",
+                    "| logical_plan | Projection: __unnest_placeholder(date).year AS year, __unnest_placeholder(date).month AS month, __unnest_placeholder(date).day AS day |",
+                    "|              |   Unnest: lists[] structs[__unnest_placeholder(date)]                                                                                 |",
+                    "|              |     Projection: named_struct(Utf8(\"year\"), t2.year, Utf8(\"month\"), t2.month, Utf8(\"day\"), t2.day) AS __unnest_placeholder(date)       |",
+                    "|              |       TableScan: t2 projection=[year, month, day]                                                                                     |",
+                    "+--------------+---------------------------------------------------------------------------------------------------------------------------------------+",
                 ],
                 expected_output: vec![
                     "+------+-------+-----+",

src/materialized/file_metadata.rs

@@ -30,14 +30,14 @@ use datafusion::physical_plan::limit::LimitStream;
 use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, Partitioning, PhysicalExpr,
-    PlanProperties,
+    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr, PlanProperties,
 };
 use datafusion::{
     catalog::CatalogProviderList, execution::TaskContext, physical_plan::SendableRecordBatchStream,
 };
 use datafusion_common::{DataFusionError, Result, ScalarValue, ToDFSchema};
 use datafusion_expr::{Expr, Operator, TableProviderFilterPushDown, TableType};
+use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
 use futures::stream::{self, BoxStream};
 use futures::{future, Future, FutureExt, StreamExt, TryStreamExt};
 use itertools::Itertools;
@@ -152,8 +152,12 @@ impl FileMetadataExec {
         };
         let eq_properties = EquivalenceProperties::new(projected_schema);
         let partitioning = Partitioning::UnknownPartitioning(1);
-        let execution_mode = ExecutionMode::Bounded;
-        let plan_properties = PlanProperties::new(eq_properties, partitioning, execution_mode);
+        let plan_properties = PlanProperties::new(
+            eq_properties,
+            partitioning,
+            EmissionType::Final,
+            Boundedness::Bounded,
+        );
 
         let exec = Self {
             table_schema,
@@ -724,7 +728,7 @@ mod test {
     use anyhow::{Context, Result};
     use datafusion::{
         assert_batches_sorted_eq,
-        catalog_common::{MemoryCatalogProvider, MemorySchemaProvider},
+        catalog::{MemoryCatalogProvider, MemorySchemaProvider},
         execution::{
             object_store::{DefaultObjectStoreRegistry, ObjectStoreRegistry},
             runtime_env::RuntimeEnvBuilder,