Upgrade to pre-release DataFusion 55 (#44)

As part of [this](paradedb/paradedb#5553)
switch to pre-release DataFusion 55.

This PR updates the DF rev used, fixes the compile errors, updates a
bunch of test outputs, and removes the unnecessary statistics wrapper.

---------

Co-authored-by: Mohammad Dashti <mdashti@gmail.com>

Author: barbarj

Cargo.lock

@@ -2,8 +2,10 @@
 members = ["benchmarks", "cli", "console"]
 
 [workspace.dependencies]
-datafusion = { version = "54.0.0", default-features = false }
-datafusion-proto = { version = "54.0.0" }
+datafusion = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false }
+datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false, features = [
+    "parquet",
+] }
 
 [package]
 name = "datafusion-distributed"
@@ -15,12 +17,12 @@ documentation = "https://datafusion-contrib.github.io/datafusion-distributed/"
 repository = "https://github.com/datafusion-contrib/datafusion-distributed"
 
 [dependencies]
-chrono = { version = "0.4.44" }
+chrono = { version = "0.4.45" }
 datafusion = { workspace = true, features = [
-  "parquet",
-  "sql",
-  "unicode_expressions",
-  "datetime_expressions",
+    "parquet",
+    "sql",
+    "unicode_expressions",
+    "datetime_expressions",
 ] }
 datafusion-proto = { workspace = true }
 async-trait = "0.1.89"
@@ -50,16 +52,16 @@ bincode = "1"
 tonic-prost = { version = "0.14.2", optional = true }
 
 # grpc-specific features
-arrow-flight = { version = "58", optional = true }
-arrow-select = { version = "58", optional = true }
-arrow-ipc = { version = "58", features = ["zstd"], optional = true }
+arrow-flight = { version = "59", optional = true }
+arrow-select = { version = "59", optional = true }
+arrow-ipc = { version = "59", features = ["zstd"], optional = true }
 tonic = { version = "0.14.1", features = ["transport"], optional = true }
 tower = { version = "0.5.2", optional = true }
 
 # integration_tests deps
 insta = { version = "1.46.0", features = ["filters"], optional = true }
-parquet = { version = "58", optional = true }
-arrow = { version = "58", optional = true, features = ["test_utils"] }
+parquet = { version = "59", optional = true }
+arrow = { version = "59", optional = true, features = ["test_utils"] }
 hyper-util = { version = "0.1.16", optional = true }
 
 [features]
@@ -71,7 +73,7 @@ grpc = [
     "arrow-ipc",
     "tonic",
     "tonic-prost",
-    "tower"
+    "tower",
 ]
 
 integration = ["insta", "parquet", "arrow", "hyper-util", "grpc"]
@@ -83,8 +85,8 @@ sysinfo = ["dep:sysinfo"]
 [dev-dependencies]
 structopt = "0.3"
 insta = { version = "1.46.0", features = ["filters"] }
-parquet = "58"
-arrow = { version = "58", features = ["test_utils"] }
+parquet = "59"
+arrow = { version = "59", features = ["test_utils"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
 hyper-util = "0.1.16"
 pretty_assertions = "1.4"

Cargo.toml

@@ -11,7 +11,7 @@ datafusion-distributed = { path = "..", features = [
   "system-metrics",
 ] }
 tokio = { version = "1.48", features = ["full"] }
-parquet = { version = "58" }
+parquet = { version = "59" }
 structopt = { version = "0.3.26" }
 log = "0.4.27"
 serde = "1.0.219"
@@ -20,10 +20,10 @@ env_logger = "0.11.8"
 async-trait = "0.1.89"
 futures = "0.3.31"
 url = "2.5.7"
-arrow = { version = "58", features = ["test_utils"] }
+arrow = { version = "59", features = ["test_utils"] }
 tonic = { version = "0.14.1", features = ["transport"] }
-tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
-tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
+tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "0f6747e592dbdf9dadd5cd3c5a43e867267bb62f" }
+tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "0f6747e592dbdf9dadd5cd3c5a43e867267bb62f" }
 reqwest = "0.12"
 zip = "6.0"
 axum = "0.7"

benchmarks/Cargo.toml

@@ -1,4 +1,5 @@
 use super::common;
+use arrow::error::ArrowError;
 use arrow::record_batch::RecordBatch;
 use datafusion::error::DataFusionError;
 use parquet::{arrow::arrow_writer::ArrowWriter, file::properties::WriterProperties};
@@ -27,19 +28,20 @@ fn generate_table<A>(
     data_dir: &Path,
 ) -> Result<(), Box<dyn std::error::Error>>
 where
-    A: Iterator<Item = RecordBatch>,
+    A: Iterator<Item = Result<RecordBatch, ArrowError>>,
 {
     let output_path = data_dir.join(format!("{table_name}.parquet"));
 
     if let Some(first_batch) = data_source.next() {
+        let first_batch = first_batch?;
         let file = fs::File::create(&output_path)?;
         let props = WriterProperties::builder().build();
         let mut writer = ArrowWriter::try_new(file, first_batch.schema(), Some(props))?;
 
         writer.write(&first_batch)?;
 
         for batch in data_source {
-            writer.write(&batch)?;
+            writer.write(&batch?)?;
         }
 
         writer.close()?;