From db3f910bd8a8e5627b5468a46856c03b6f47932b Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Wed, 9 Apr 2025 09:06:08 -0400 Subject: [PATCH 1/2] REmove iceberg --- Cargo.lock | 853 +------------------- Cargo.toml | 2 +- crates/datafusion-app/Cargo.toml | 6 +- crates/datafusion-app/src/extensions/mod.rs | 4 +- 4 files changed, 25 insertions(+), 840 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45e223c0..238e00ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,23 +32,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.15", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.11" @@ -170,42 +153,12 @@ version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" -[[package]] -name = "apache-avro" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" -dependencies = [ - "bigdecimal", - "digest", - "libflate", - "log", - "num-bigint", - "quad-rand", - "rand 0.8.5", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "strum", - "strum_macros", - "thiserror 1.0.69", - "typed-builder 0.19.1", - "uuid", -] - [[package]] name = "arbitrary" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" -[[package]] -name = "array-init" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" - [[package]] name = "arrayref" version = "0.3.9" @@ -259,7 +212,7 @@ version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow-buffer", "arrow-data", "arrow-schema", @@ -432,7 +385,7 @@ version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow-array", "arrow-buffer", "arrow-data", @@ -490,17 +443,6 @@ dependencies = [ "zstd-safe", ] -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -696,15 +638,8 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] -[[package]] -name = "bimap" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" - [[package]] name = "bindgen" version = "0.69.5" @@ -783,29 +718,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "borsh" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5430e3be710b68d984d1391c854eb431a9d548640711faa54eecb1df93db91cc" -dependencies = [ - "borsh-derive", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b668d39970baad5356d7c83a86fee3a539e6f93bf6764c97368243e17a0487" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "brotli" version = "7.0.0" @@ -847,28 +759,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytemuck" version = "1.21.0" @@ -1177,15 +1067,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console" version = "0.15.10" @@ -1198,12 +1079,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-random" version = "0.1.18" @@ -1256,15 +1131,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpp_demangle" version = "0.4.4" @@ -1390,15 +1256,6 @@ dependencies = [ "target-lexicon 0.13.2", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.4.2" @@ -1408,15 +1265,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1540,12 +1388,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" - [[package]] name = "dashmap" version = "6.1.0" @@ -1626,8 +1468,6 @@ dependencies = [ "deltalake", "directories", "futures", - "iceberg-catalog-rest", - "iceberg-datafusion", "indexmap 2.8.0", "itertools 0.13.0", "log", @@ -1670,7 +1510,7 @@ version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow", "arrow-array", "arrow-buffer", @@ -1846,7 +1686,7 @@ version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow", "arrow-buffer", "arrow-schema", @@ -1869,7 +1709,7 @@ version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2001,7 +1841,7 @@ version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow", "arrow-array", "arrow-buffer", @@ -2026,7 +1866,7 @@ version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow", "arrow-buffer", "datafusion-common", @@ -2063,7 +1903,7 @@ version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow", "arrow-array", "arrow-buffer", @@ -2277,47 +2117,6 @@ dependencies = [ "z85", ] -[[package]] -name = "deranged" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" -dependencies = [ - "powerfmt", - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.98", -] - [[package]] name = "difflib" version = "0.4.0" @@ -2331,7 +2130,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", "crypto-common", "subtle", ] @@ -2409,15 +2207,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - [[package]] name = "doc-comment" version = "0.3.3" @@ -2502,27 +2291,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "event-listener" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" -dependencies = [ - "event-listener", - "pin-project-lite", -] - [[package]] name = "eyre" version = "0.6.12" @@ -2787,19 +2555,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "generator" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" -dependencies = [ - "cfg-if", - "libc", - "log", - "rustversion", - "windows", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2905,9 +2660,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] [[package]] name = "hashbrown" @@ -2915,7 +2667,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.11", + "ahash", "allocator-api2", ] @@ -2949,15 +2701,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - [[package]] name = "home" version = "0.5.9" @@ -3118,7 +2861,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -3130,88 +2873,6 @@ dependencies = [ "cc", ] -[[package]] -name = "iceberg" -version = "0.4.0" -source = "git+https://github.com/apache/iceberg-rust?rev=210134573569be0b6e49464aca076cb122e33e24#210134573569be0b6e49464aca076cb122e33e24" -dependencies = [ - "anyhow", - "apache-avro", - "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", - "async-trait", - "bimap", - "bitvec", - "bytes", - "chrono", - "derive_builder", - "fnv", - "futures", - "itertools 0.13.0", - "moka", - "murmur3", - "num-bigint", - "once_cell", - "opendal", - "ordered-float 4.6.0", - "parquet", - "paste", - "rand 0.8.5", - "reqwest", - "rust_decimal", - "serde", - "serde_bytes", - "serde_derive", - "serde_json", - "serde_repr", - "serde_with", - "tokio", - "typed-builder 0.20.0", - "url", - "uuid", - "zstd", -] - -[[package]] -name = "iceberg-catalog-rest" -version = "0.4.0" -source = "git+https://github.com/apache/iceberg-rust?rev=210134573569be0b6e49464aca076cb122e33e24#210134573569be0b6e49464aca076cb122e33e24" -dependencies = [ - "async-trait", - "chrono", - "http", - "iceberg", - "itertools 0.13.0", - "log", - "reqwest", - "serde", - "serde_derive", - "serde_json", - "tokio", - "typed-builder 0.20.0", - "uuid", -] - -[[package]] -name = "iceberg-datafusion" -version = "0.4.0" -source = "git+https://github.com/apache/iceberg-rust?rev=210134573569be0b6e49464aca076cb122e33e24#210134573569be0b6e49464aca076cb122e33e24" -dependencies = [ - "anyhow", - "async-trait", - "datafusion", - "futures", - "iceberg", - "tokio", -] - [[package]] name = "icu_collections" version = "1.5.0" @@ -3377,7 +3038,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde", ] [[package]] @@ -3558,7 +3218,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8243cf2d026264056bfacf305e54f5bee8866fd46b4c1873adcaebf614a0d306" dependencies = [ - "ahash 0.8.11", + "ahash", "bitvec", "lexical-parse-float 0.8.5", "num-bigint", @@ -3710,30 +3370,6 @@ version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" -[[package]] -name = "libflate" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" -dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", -] - [[package]] name = "libloading" version = "0.8.6" @@ -3795,23 +3431,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] -name = "loom" -version = "0.7.2" +name = "lru" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ "hashbrown 0.15.2", ] @@ -3912,7 +3535,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3" dependencies = [ - "ahash 0.8.11", + "ahash", "portable-atomic", ] @@ -3995,34 +3618,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "moka" -version = "0.12.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "event-listener", - "futures-util", - "loom", - "parking_lot", - "portable-atomic", - "rustc_version", - "smallvec", - "tagptr", - "thiserror 1.0.69", - "uuid", -] - -[[package]] -name = "murmur3" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" - [[package]] name = "native-tls" version = "0.2.14" @@ -4088,7 +3683,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -4100,12 +3694,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - [[package]] name = "num-integer" version = "0.1.46" @@ -4243,7 +3831,6 @@ dependencies = [ "base64 0.22.1", "bytes", "chrono", - "crc32c", "futures", "getrandom 0.2.15", "http", @@ -4252,7 +3839,6 @@ dependencies = [ "once_cell", "percent-encoding", "quick-xml 0.36.2", - "reqsign", "reqwest", "serde", "serde_json", @@ -4319,25 +3905,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ordered-float" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - [[package]] name = "overload" version = "0.1.1" @@ -4350,12 +3917,6 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.3" @@ -4385,7 +3946,7 @@ version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" dependencies = [ - "ahash 0.8.11", + "ahash", "arrow-array", "arrow-buffer", "arrow-cast", @@ -4550,12 +4111,6 @@ dependencies = [ "serde", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.20" @@ -4605,15 +4160,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "proc-macro-crate" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" -dependencies = [ - "toml_edit", -] - [[package]] name = "proc-macro2" version = "1.0.93" @@ -4664,26 +4210,6 @@ dependencies = [ "cc", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "pulley-interpreter" version = "29.0.1" @@ -4760,12 +4286,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quanta" version = "0.12.5" @@ -4781,16 +4301,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "quick-xml" -version = "0.35.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e446ed58cef1bbfe847bc2fda0e2e4ea9f0e57b90c507d4781292590d72a4e" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.36.2" @@ -5084,12 +4594,6 @@ dependencies = [ "regex-syntax 0.8.5", ] -[[package]] -name = "regex-lite" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" - [[package]] name = "regex-syntax" version = "0.6.29" @@ -5102,43 +4606,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - -[[package]] -name = "reqsign" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb0075a66c8bfbf4cc8b70dca166e722e1f55a3ea9250ecbb85f4d92a5f64149" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", - "form_urlencoded", - "getrandom 0.2.15", - "hex", - "hmac", - "home", - "http", - "log", - "percent-encoding", - "quick-xml 0.35.0", - "rand 0.8.5", - "reqwest", - "rust-ini", - "serde", - "serde_json", - "sha1", - "sha2", -] - [[package]] name = "reqwest" version = "0.12.14" @@ -5205,41 +4672,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "roaring" version = "0.10.10" @@ -5250,33 +4682,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "rust-ini" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f" -dependencies = [ - "cfg-if", - "ordered-multimap", - "trim-in-place", -] - -[[package]] -name = "rust_decimal" -version = "1.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -5406,24 +4811,12 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "2.11.1" @@ -5484,15 +4877,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" version = "1.0.218" @@ -5526,17 +4910,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_repr" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "serde_spanned" version = "0.6.8" @@ -5558,47 +4931,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" -dependencies = [ - "base64 0.22.1", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.8.0", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.8" @@ -5908,12 +5240,6 @@ dependencies = [ "winx", ] -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - [[package]] name = "tap" version = "1.0.1" @@ -6019,38 +5345,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float 2.10.1", -] - -[[package]] -name = "time" -version = "0.3.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - -[[package]] -name = "time-macros" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" -dependencies = [ - "num-conv", - "time-core", + "ordered-float", ] [[package]] @@ -6374,12 +5669,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "trim-in-place" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" - [[package]] name = "try-lock" version = "0.2.5" @@ -6424,46 +5713,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "typed-builder" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro 0.19.1", -] - -[[package]] -name = "typed-builder" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14ed59dc8b7b26cacb2a92bad2e8b1f098806063898ab42a3bd121d7d45e75" -dependencies = [ - "typed-builder-macro 0.20.0", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "typed-builder-macro" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560b82d656506509d43abe30e0ba64c56b1953ab3d4fe7ba5902747a7a3cedd5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "typenum" version = "1.18.0" @@ -7221,16 +6470,6 @@ dependencies = [ "wasmtime-environ", ] -[[package]] -name = "windows" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" -dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" version = "0.52.0" @@ -7240,41 +6479,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-implement" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - -[[package]] -name = "windows-interface" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "windows-link" version = "0.1.0" @@ -7287,20 +6491,11 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result 0.3.1", - "windows-strings 0.3.1", + "windows-result", + "windows-strings", "windows-targets 0.53.0", ] -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-result" version = "0.3.1" @@ -7310,16 +6505,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-strings" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 1e909347..565f375a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,7 +103,7 @@ http = [ ] # hudi = ["datafusion-app/hudi"] huggingface = ["datafusion-app/huggingface"] -iceberg = ["datafusion-app/iceberg"] +# iceberg = ["datafusion-app/iceberg"] s3 = ["datafusion-app/s3", "url"] udfs-wasm = ["datafusion-app/udfs-wasm"] diff --git a/crates/datafusion-app/Cargo.toml b/crates/datafusion-app/Cargo.toml index 66479f11..cff7f0a8 100644 --- a/crates/datafusion-app/Cargo.toml +++ b/crates/datafusion-app/Cargo.toml @@ -24,8 +24,8 @@ futures = "0.3.30" # hudi = { git = "https://github.com/apache/hudi-rs", rev = "eab36e2d3ea39372c52d2bf6fe41c9aa406c0bea", features = [ # "datafusion", # ], optional = true } -iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "210134573569be0b6e49464aca076cb122e33e24", optional = true } -iceberg-datafusion = { git = "https://github.com/apache/iceberg-rust", rev = "210134573569be0b6e49464aca076cb122e33e24", optional = true } +# iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "210134573569be0b6e49464aca076cb122e33e24", optional = true } +# iceberg-datafusion = { git = "https://github.com/apache/iceberg-rust", rev = "210134573569be0b6e49464aca076cb122e33e24", optional = true } indexmap = { version = "2.8.0", features = ["serde"] } itertools = "0.13.0" log = "0.4.22" @@ -53,7 +53,7 @@ functions-json = ["dep:datafusion-functions-json"] functions-parquet = ["dep:datafusion-functions-parquet"] # hudi = ["dep:hudi"] huggingface = ["object_store_opendal", "opendal", "url"] -iceberg = ["dep:iceberg-catalog-rest", "dep:iceberg-datafusion"] +# iceberg = ["dep:iceberg-catalog-rest", "dep:iceberg-datafusion"] observability = [] s3 = ["object_store/aws", "url"] udfs-wasm = ["dep:datafusion-udfs-wasm"] diff --git a/crates/datafusion-app/src/extensions/mod.rs b/crates/datafusion-app/src/extensions/mod.rs index 41ddb6f0..5897c7d4 100644 --- a/crates/datafusion-app/src/extensions/mod.rs +++ b/crates/datafusion-app/src/extensions/mod.rs @@ -29,8 +29,8 @@ mod deltalake; // mod hudi; #[cfg(feature = "huggingface")] mod huggingface; -#[cfg(feature = "iceberg")] -mod iceberg; +// #[cfg(feature = "iceberg")] +// mod iceberg; #[cfg(feature = "s3")] mod s3; From ac64951998d0510b25a75c42a8acac779971be5c Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Wed, 9 Apr 2025 09:22:24 -0400 Subject: [PATCH 2/2] Remove iceberg --- README.md | 4 ++-- crates/datafusion-app/src/config.rs | 22 +++++++++++----------- docs/features.md | 17 ----------------- 3 files changed, 13 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 68574ae3..f92e7914 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Documentation is undergoing a significant revamp - the new documentation will be `dft` is a batteries-included suite of [DataFusion](https://github.com/apache/arrow-datafusion) applications that provides: - **Data Source Integration**: Query files from S3, local filesystems, or HuggingFace datasets -- **Table Format Support**: Native support for Delta Lake and Iceberg +- **Table Format Support**: Native support for Delta Lake - **Extensibility**: UDFs defined in WASM (and soon Python) - **Helper Functions**: Built-in functions for JSON and Parquet data processing @@ -45,7 +45,7 @@ Common feature combinations: cargo install datafusion-dft --features=s3 # Data lake formats -cargo install datafusion-dft --features=deltalake,iceberg +cargo install datafusion-dft --features=deltalake # With JSON and Parquet functions cargo install datafusion-dft --features=function-json,functions-parquet diff --git a/crates/datafusion-app/src/config.rs b/crates/datafusion-app/src/config.rs index ad11a43e..b1d38751 100644 --- a/crates/datafusion-app/src/config.rs +++ b/crates/datafusion-app/src/config.rs @@ -58,9 +58,9 @@ pub fn merge_configs(shared: ExecutionConfig, priority: ExecutionConfig) -> Exec if merged.dedicated_executor_threads != priority.dedicated_executor_threads { merged.dedicated_executor_threads = priority.dedicated_executor_threads } - if merged.iceberg != priority.iceberg { - merged.iceberg = priority.iceberg - } + // if merged.iceberg != priority.iceberg { + // merged.iceberg = priority.iceberg + // } #[cfg(feature = "udfs-wasm")] if merged.wasm_udf != priority.wasm_udf { @@ -84,8 +84,8 @@ pub struct ExecutionConfig { pub dedicated_executor_enabled: bool, #[serde(default = "default_dedicated_executor_threads")] pub dedicated_executor_threads: usize, - #[serde(default = "default_iceberg_config")] - pub iceberg: IcebergConfig, + // #[serde(default = "default_iceberg_config")] + // pub iceberg: IcebergConfig, #[cfg(feature = "udfs-wasm")] #[serde(default = "default_wasm_udf")] pub wasm_udf: WasmUdfConfig, @@ -105,7 +105,7 @@ impl Default for ExecutionConfig { datafusion: None, dedicated_executor_enabled: default_dedicated_executor_enabled(), dedicated_executor_threads: default_dedicated_executor_threads(), - iceberg: default_iceberg_config(), + // iceberg: default_iceberg_config(), #[cfg(feature = "udfs-wasm")] wasm_udf: default_wasm_udf(), catalog: default_catalog(), @@ -144,11 +144,11 @@ fn default_dedicated_executor_threads() -> usize { num_cpus::get() } -fn default_iceberg_config() -> IcebergConfig { - IcebergConfig { - rest_catalogs: Vec::new(), - } -} +// fn default_iceberg_config() -> IcebergConfig { +// IcebergConfig { +// rest_catalogs: Vec::new(), +// } +// } #[cfg(feature = "udfs-wasm")] fn default_wasm_udf() -> WasmUdfConfig { diff --git a/docs/features.md b/docs/features.md index 0e9f332e..16814ab5 100644 --- a/docs/features.md +++ b/docs/features.md @@ -108,23 +108,6 @@ Register deltalake tables. For example: CREATE EXTERNAL TABLE table_name STORED AS DELTATABLE LOCATION 's3://bucket/table' ``` -### Iceberg (`--features=iceberg`) - -Register iceberg tables. For example: - -```sql -CREATE EXTERNAL TABLE table_name STORED AS ICEBERG LOCATION 's3://bucket/table' -``` - -Register Iceberg Rest Catalog - -```toml -[[execution.iceberg.rest_catalog]] -name = "my_iceberg_catalog" -addr = "192.168.1.1:8181" -``` - - ### Json Functions (`--features=function-json`) Adds functions from [datafusion-function-json] for querying JSON strings in DataFusion in `dft`. For example: