Upgrade to datafusion 47 (#3016)

Getting ready for the [datafusion 47
release](apache/datafusion#15072).

## Current issues
- [x] apache/datafusion#15072
- [x] duckdb-rs depends on arrow 54, opened a PR to fix -
duckdb/duckdb-rs#496.
- [ ] object-store 0.12 has a regression on Azure, not sure what's the
priority here but shouldn't be too hard to find the root cause if we
care. apache/arrow-rs-object-store#320

Author: AdamGS

Cargo.lock

@@ -55,17 +55,17 @@ version = "0.32.0"
 anyhow = "1.0.95"
 arbitrary = "1.3.2"
 arrayref = "0.3.7"
-arrow = "54.3.1"
-arrow-arith = "54.3.1"
-arrow-array = "54.3.1"
-arrow-buffer = "54.3.1"
-arrow-cast = "54.3.1"
-arrow-ord = "54.3.1"
-arrow-schema = "54.3.1"
-arrow-select = "54.3.1"
-arrow-string = "54.3.1"
+arrow = "55"
+arrow-arith = "55"
+arrow-array = "55"
+arrow-buffer = "55"
+arrow-cast = "55"
+arrow-ord = "55"
+arrow-schema = "55"
+arrow-select = "55"
+arrow-string = "55"
 async-once-cell = "0.5.4"
-async-trait = "0.1.86"
+async-trait = "0.1.88"
 backtrace = "0.3.74"
 bit-vec = "0.8.0"
 bytes = "1.10"
@@ -76,12 +76,13 @@ clap = "4.5"
 compio = { version = "0.14", features = ["io-uring"], default-features = false }
 crossterm = "0.28"
 dashmap = "6.1.0"
-datafusion = { version = "46", default-features = false }
-datafusion-common = { version = "46" }
-datafusion-execution = { version = "46" }
-datafusion-expr = { version = "46" }
-datafusion-physical-expr = { version = "46" }
-datafusion-physical-plan = { version = "46" }
+datafusion = { version = "47", default-features = false }
+datafusion-common = { version = "47" }
+datafusion-datasource = { version = "47" }
+datafusion-execution = { version = "47" }
+datafusion-expr = { version = "47" }
+datafusion-physical-expr = { version = "47" }
+datafusion-physical-plan = { version = "47" }
 divan = { package = "codspeed-divan-compat", version = "2.8.0" }
 duckdb = { path = "duckdb-vortex/duckdb-rs/crates/duckdb", features = [
     "vtab-full",
@@ -90,16 +91,16 @@ duckdb = { path = "duckdb-vortex/duckdb-rs/crates/duckdb", features = [
 dyn-hash = "0.2.0"
 enum-iterator = "2.0.0"
 fastlanes = "0.1.8"
-flatbuffers = "25"
-flexbuffers = "25"
+flatbuffers = "25.2.10"
+flexbuffers = "25.2.10"
 flume = "0.11"
 fsst-rs = "0.5.2"
 futures = { version = "0.3.31", default-features = false }
 futures-util = "0.3.31"
 getrandom = "0.3"
 goldenfile = "1"
 governor = "0.10"
-half = { version = "2.2.1", features = ["std", "num-traits"] }
+half = { version = "2.5", features = ["std", "num-traits"] }
 hashbrown = "0.15.1"
 homedir = "0.3.3"
 humansize = "2.1.3"
@@ -116,20 +117,19 @@ mimalloc = "0.1.42"
 moka = { version = "0.12.10", default-features = false }
 num-traits = "0.2.19"
 num_enum = "0.7.2"
-object_store = "0.11.0"
+object_store = "0.12"
 opentelemetry = "0.29.0"
 opentelemetry-otlp = "0.29.0"
 opentelemetry_sdk = "0.29.0"
 parking_lot = "0.12.3"
-parquet = "54.3.1"
+parquet = "55"
 paste = "1.0.15"
 pin-project = "1.1.5"
 pin-project-lite = "0.2.15"
 prost = "0.13.4"
 prost-build = "0.13.4"
 prost-types = "0.13.4"
-# bump pyo3 to >0.24 on next arrow release, for RUSTSEC-2025-0020
-pyo3 = { version = "0.23.4", features = ["extension-module", "abi3-py310"] }
+pyo3 = { version = "0.24.1", features = ["extension-module", "abi3-py310"] }
 pyo3-log = "0.12.1"
 rancor = "0.1.0"
 rand = "0.9.0"
@@ -165,7 +165,7 @@ tracing-chrome = "0.7.2"
 tracing-futures = "0.2.5"
 tracing-subscriber = "0.3.19"
 url = "2.5.4"
-uuid = { version = "1.15", features = ["js"] }
+uuid = { version = "1.16", features = ["js"] }
 wasm-bindgen-futures = "0.4.39"
 witchcraft-metrics = "1.0.1"
 

Cargo.toml

@@ -58,8 +58,6 @@ struct Args {
     #[arg(long)]
     queries_file: Option<PathBuf>,
     #[arg(long, default_value_t = false)]
-    emulate_object_store: bool,
-    #[arg(long, default_value_t = false)]
     disable_datafusion_cache: bool,
     #[arg(long)]
     export_spans: bool,
@@ -218,10 +216,7 @@ fn main() -> anyhow::Result<()> {
 
         let mut engine_ctx = match engine {
             Engine::DataFusion => {
-                let session_ctx = df::get_session_context(
-                    args.emulate_object_store,
-                    args.disable_datafusion_cache,
-                );
+                let session_ctx = df::get_session_context(args.disable_datafusion_cache);
                 // Register object store to the session.
                 df::make_object_store(&session_ctx, &base_url)?;
 

bench-vortex/src/bin/clickbench.rs

@@ -39,8 +39,6 @@ struct Args {
     #[arg(long, default_value_t, value_enum)]
     display_format: DisplayFormat,
     #[arg(long, default_value_t = false)]
-    emulate_object_store: bool,
-    #[arg(long, default_value_t = false)]
     disable_datafusion_cache: bool,
     #[arg(short, long, value_delimiter = ',')]
     dataset: PBIDataset,
@@ -107,8 +105,7 @@ fn main() -> anyhow::Result<()> {
 
     for target in &args.targets {
         let format = target.format();
-        let session =
-            df::get_session_context(args.emulate_object_store, args.disable_datafusion_cache);
+        let session = df::get_session_context(args.disable_datafusion_cache);
 
         let file_type = match format {
             Format::Csv => FileType::Csv,

bench-vortex/src/bin/public_bi.rs

@@ -65,8 +65,6 @@ struct Args {
     #[arg(short, long, default_value_t, value_enum)]
     display_format: DisplayFormat,
     #[arg(long, default_value_t = false)]
-    emulate_object_store: bool,
-    #[arg(long, default_value_t = false)]
     disable_datafusion_cache: bool,
     #[arg(long, default_value_t, value_enum)]
     data_generator: DataGenerator,
@@ -176,7 +174,6 @@ fn main() -> anyhow::Result<()> {
         args.iterations,
         args.targets,
         args.display_format,
-        args.emulate_object_store,
         args.disable_datafusion_cache,
         args.scale_factor,
         url,
@@ -295,7 +292,6 @@ async fn bench_main(
     iterations: usize,
     targets: Vec<Target>,
     display_format: DisplayFormat,
-    emulate_object_store: bool,
     disable_datafusion_cache: bool,
     scale_factor: u8,
     url: Url,
@@ -351,9 +347,7 @@ async fn bench_main(
         let format = target.format();
         match engine {
             Engine::DataFusion => {
-                let ctx =
-                    load_datasets(&url, format, emulate_object_store, disable_datafusion_cache)
-                        .await?;
+                let ctx = load_datasets(&url, format, disable_datafusion_cache).await?;
 
                 let mut plans = Vec::new();
 

bench-vortex/src/bin/tpch.rs

@@ -7,7 +7,6 @@ use datafusion::datasource::provider::DefaultTableFactory;
 use datafusion::execution::SessionStateBuilder;
 use datafusion::execution::cache::cache_manager::CacheManagerConfig;
 use datafusion::execution::cache::cache_unit::{DefaultFileStatisticsCache, DefaultListFilesCache};
-use datafusion::execution::object_store::DefaultObjectStoreRegistry;
 use datafusion::execution::runtime_env::RuntimeEnvBuilder;
 use datafusion::physical_plan::collect;
 use datafusion::physical_plan::execution_plan::ExecutionPlan;
@@ -22,8 +21,6 @@ use url::Url;
 use vortex::error::VortexResult;
 use vortex_datafusion::persistent::VortexFormatFactory;
 
-use crate::blob::SlowObjectStoreRegistry;
-
 pub static GIT_COMMIT_ID: LazyLock<String> = LazyLock::new(|| {
     String::from_utf8(
         Command::new("git")
@@ -37,17 +34,8 @@ pub static GIT_COMMIT_ID: LazyLock<String> = LazyLock::new(|| {
     .to_string()
 });
 
-pub fn get_session_context(
-    emulate_object_store: bool,
-    disable_datafusion_cache: bool,
-) -> SessionContext {
-    let registry = if emulate_object_store {
-        Arc::new(SlowObjectStoreRegistry::default()) as _
-    } else {
-        Arc::new(DefaultObjectStoreRegistry::new()) as _
-    };
-
-    let mut rt_builder = RuntimeEnvBuilder::new().with_object_store_registry(registry);
+pub fn get_session_context(disable_datafusion_cache: bool) -> SessionContext {
+    let mut rt_builder = RuntimeEnvBuilder::new();
 
     if !disable_datafusion_cache {
         let file_static_cache = Arc::new(DefaultFileStatisticsCache::default());

bench-vortex/src/blob.rs

@@ -11,7 +11,6 @@ use itertools::Itertools;
 use serde::Serialize;
 
 pub mod bench_run;
-pub mod blob;
 pub mod clickbench;
 pub mod compress;
 pub mod conversions;

bench-vortex/src/engines/df/mod.rs

@@ -40,10 +40,9 @@ pub const EXPECTED_ROW_COUNTS_SF10: [usize; TPC_H_ROW_COUNT_ARRAY_LENGTH] = [
 pub async fn load_datasets(
     base_dir: &Url,
     format: Format,
-    emulate_object_store: bool,
     disable_datafusion_cache: bool,
 ) -> anyhow::Result<SessionContext> {
-    let context = get_session_context(emulate_object_store, disable_datafusion_cache);
+    let context = get_session_context(disable_datafusion_cache);
 
     let object_store = make_object_store(&context, base_dir)?;