diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 919f7f09..64cf2316 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -146,31 +146,6 @@ jobs: - name: Run Deltalake tests run: | cargo test --features=deltalake extension_cases::deltalake - # test-hudi: - # name: Extension / Hudi - # runs-on: ubuntu-latest - # strategy: - # matrix: - # arch: [amd64] - # steps: - # - uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Cache Cargo - # uses: actions/cache@v4 - # with: - # path: /home/runner/.cargo - # key: cargo-dft-cache- - # - name: Cache Rust dependencies - # uses: actions/cache@v4 - # with: - # path: target - # key: ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.lock') }} - # - name: Setup Rust Toolchain - # uses: ./.github/actions/setup-rust - # - name: Run Hudi tests - # run: | - # cargo test --features=hudi extension_cases::hudi test-udfs-wasm: name: Extension / UDFs-WASM runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index 565f375a..a49700f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,9 +101,7 @@ http = [ "dep:tower-http", "dep:uuid", ] -# hudi = ["datafusion-app/hudi"] huggingface = ["datafusion-app/huggingface"] -# iceberg = ["datafusion-app/iceberg"] s3 = ["datafusion-app/s3", "url"] udfs-wasm = ["datafusion-app/udfs-wasm"] diff --git a/crates/datafusion-app/Cargo.toml b/crates/datafusion-app/Cargo.toml index 181356dc..15f40a33 100644 --- a/crates/datafusion-app/Cargo.toml +++ b/crates/datafusion-app/Cargo.toml @@ -23,11 +23,6 @@ deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "d8995e64c305d ], optional = true } directories = "5.0.1" futures = "0.3.30" -# hudi = { git = "https://github.com/apache/hudi-rs", rev = "eab36e2d3ea39372c52d2bf6fe41c9aa406c0bea", features = [ -# "datafusion", -# ], optional = true } -# iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "210134573569be0b6e49464aca076cb122e33e24", optional = true } -# iceberg-datafusion = { git = "https://github.com/apache/iceberg-rust", rev = "210134573569be0b6e49464aca076cb122e33e24", optional = true } indexmap = { version = "2.8.0", features = ["serde"] } itertools = "0.13.0" log = "0.4.22" @@ -53,9 +48,7 @@ deltalake = ["dep:deltalake"] flightsql = ["dep:arrow-flight", "dep:base64", "dep:tonic"] functions-json = ["dep:datafusion-functions-json"] functions-parquet = ["dep:datafusion-functions-parquet"] -# hudi = ["dep:hudi"] huggingface = ["object_store_opendal", "opendal", "url"] -# iceberg = ["dep:iceberg-catalog-rest", "dep:iceberg-datafusion"] observability = [] s3 = ["object_store/aws", "url"] udfs-wasm = ["dep:datafusion-udfs-wasm"] diff --git a/crates/datafusion-app/src/extensions/hudi.rs b/crates/datafusion-app/src/extensions/hudi.rs deleted file mode 100644 index f03e0fad..00000000 --- a/crates/datafusion-app/src/extensions/hudi.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Hugi integration: [HudiExtension] - -use crate::config::ExecutionConfig; -use crate::extensions::{DftSessionStateBuilder, Extension}; -use hudi::HudiTableFactory; -use std::sync::Arc; - -#[derive(Debug, Default)] -pub struct HudiExtension {} - -impl HudiExtension { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait::async_trait] -impl Extension for HudiExtension { - async fn register( - &self, - _config: ExecutionConfig, - builder: &mut DftSessionStateBuilder, - ) -> datafusion::common::Result<()> { - builder.add_table_factory("HUDI", Arc::new(HudiTableFactory {})); - Ok(()) - } -} diff --git a/crates/datafusion-app/src/extensions/iceberg.rs b/crates/datafusion-app/src/extensions/iceberg.rs deleted file mode 100644 index c6d0bad6..00000000 --- a/crates/datafusion-app/src/extensions/iceberg.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! DeltaLake integration: [DeltaLakeExtension] - -use crate::config::ExecutionConfig; -use crate::extensions::{DftSessionStateBuilder, Extension}; -use datafusion::common::DataFusionError; -use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; -use iceberg_datafusion::{IcebergCatalogProvider, IcebergTableProviderFactory}; -use std::sync::Arc; - -#[derive(Debug, Default)] -pub struct IcebergExtension {} - -impl IcebergExtension { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait::async_trait] -impl Extension for IcebergExtension { - async fn register( - &self, - config: ExecutionConfig, - builder: &mut DftSessionStateBuilder, - ) -> datafusion::common::Result<()> { - for cfg in config.iceberg.rest_catalogs { - let rest_catalog_config = RestCatalogConfig::builder().uri(cfg.addr).build(); - let rest_catalog = RestCatalog::new(rest_catalog_config); - let catalog_provider = IcebergCatalogProvider::try_new(Arc::new(rest_catalog)) - .await - .map_err(|e| DataFusionError::External(Box::new(e)))?; - builder.add_catalog_provider(&cfg.name, Arc::new(catalog_provider)); - } - // TODO Add Iceberg Catalog - let factory = Arc::new(IcebergTableProviderFactory {}); - builder.add_table_factory("ICEBERG", factory); - Ok(()) - } -} diff --git a/crates/datafusion-app/src/extensions/mod.rs b/crates/datafusion-app/src/extensions/mod.rs index b25d4cca..f07f51a5 100644 --- a/crates/datafusion-app/src/extensions/mod.rs +++ b/crates/datafusion-app/src/extensions/mod.rs @@ -25,12 +25,8 @@ use std::{fmt::Debug, sync::Arc}; mod builder; #[cfg(feature = "deltalake")] mod deltalake; -// #[cfg(feature = "hudi")] -// mod hudi; #[cfg(feature = "huggingface")] mod huggingface; -// #[cfg(feature = "iceberg")] -// mod iceberg; #[cfg(feature = "s3")] mod s3; @@ -60,10 +56,6 @@ pub fn enabled_extensions() -> Vec> { Arc::new(s3::AwsS3Extension::new()), #[cfg(feature = "deltalake")] Arc::new(deltalake::DeltaLakeExtension::new()), - // #[cfg(feature = "hudi")] - // Arc::new(hudi::HudiExtension::new()), - // #[cfg(feature = "iceberg")] - // Arc::new(iceberg::IcebergExtension::new()), #[cfg(feature = "huggingface")] Arc::new(huggingface::HuggingFaceExtension::new()), ] diff --git a/tests/extension_cases/hudi.rs b/tests/extension_cases/hudi.rs deleted file mode 100644 index 60511589..00000000 --- a/tests/extension_cases/hudi.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use url::Url; - -use crate::extension_cases::TestExecution; - -#[tokio::test] -async fn test_hudi() { - let test_exec = TestExecution::new(); - - let cwd = std::env::current_dir().unwrap(); - let path = Url::from_file_path(cwd.join("data/hudi/v6_simplekeygen_nonhivestyle")).unwrap(); - - let test_exec = test_exec - .await - .with_setup(&format!( - "CREATE EXTERNAL TABLE h STORED AS HUDI LOCATION '{}';", - path - )) - .await; - - let output = test_exec - .run_and_format("SELECT id FROM h ORDER BY id") - .await; - assert_eq!( - output, - vec!["+----+", "| id |", "+----+", "| 1 |", "| 2 |", "| 3 |", "| 4 |", "+----+"] - ); -} diff --git a/tests/extension_cases/mod.rs b/tests/extension_cases/mod.rs index 4498a9d3..04c9f44c 100644 --- a/tests/extension_cases/mod.rs +++ b/tests/extension_cases/mod.rs @@ -17,18 +17,16 @@ //! Tests for extensions (stored in the `extension_cases` directory) +#[cfg(feature = "flightsql")] +mod auth_basic; +#[cfg(feature = "flightsql")] +mod auth_bearer; #[cfg(feature = "deltalake")] mod deltalake; #[cfg(feature = "flightsql")] mod flightsql; #[cfg(feature = "functions-json")] mod functions_json; -// #[cfg(feature = "hudi")] -// mod hudi; -#[cfg(feature = "flightsql")] -mod auth_basic; -#[cfg(feature = "flightsql")] -mod auth_bearer; #[cfg(feature = "huggingface")] mod huggingface; #[cfg(feature = "s3")]