From 3f047bcbf8fbb8fb5d1160ff164a77630588688a Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Fri, 11 Apr 2025 09:44:57 -0400 Subject: [PATCH 1/2] Upgrade deltalake and df --- Cargo.lock | 392 +++++++++--------- Cargo.toml | 6 +- crates/datafusion-app/Cargo.toml | 12 +- crates/datafusion-app/src/stats.rs | 20 +- .../datafusion-functions-parquet/Cargo.toml | 6 +- .../datafusion-functions-parquet/src/lib.rs | 10 +- crates/datafusion-udfs-wasm/Cargo.toml | 2 +- .../wasm-examples/Cargo.toml | 2 +- 8 files changed, 232 insertions(+), 218 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 238e00ba..76edd7b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,9 +173,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc" +checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1" dependencies = [ "arrow-arith", "arrow-array", @@ -194,9 +194,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248" +checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a" dependencies = [ "arrow-array", "arrow-buffer", @@ -208,9 +208,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" +checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5" dependencies = [ "ahash", "arrow-buffer", @@ -225,9 +225,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.1.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" dependencies = [ "bytes", "half", @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870" +checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6" +checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65" dependencies = [ "arrow-array", "arrow-cast", @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.1.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ "arrow-buffer", "arrow-schema", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e" +checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86" dependencies = [ "arrow-array", "arrow-buffer", @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c" +checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da" +checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160" dependencies = [ "arrow-array", "arrow-buffer", @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c" +checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd" dependencies = [ "arrow-array", "arrow-buffer", @@ -372,18 +372,19 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.1.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" dependencies = [ + "bitflags 2.9.0", "serde", ] [[package]] name = "arrow-select" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" +checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c" dependencies = [ "ahash", "arrow-array", @@ -395,9 +396,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0" +checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458" dependencies = [ "arrow-array", "arrow-buffer", @@ -1189,7 +1190,7 @@ dependencies = [ "rustc-hash 2.1.1", "serde", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", ] [[package]] @@ -1236,7 +1237,7 @@ dependencies = [ "cranelift-codegen", "log", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", ] [[package]] @@ -1253,7 +1254,7 @@ checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" dependencies = [ "cranelift-codegen", "libc", - "target-lexicon 0.13.2", + "target-lexicon", ] [[package]] @@ -1404,29 +1405,31 @@ dependencies = [ [[package]] name = "datafusion" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae420e7a5b0b7f1c39364cc76cbcd0f5fdc416b2514ae3847c2676bbd60702a" +checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3" dependencies = [ "arrow", - "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression", "async-trait", "bytes", "bzip2", "chrono", "datafusion-catalog", + "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", + "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", + "datafusion-macros", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -1435,7 +1438,6 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "glob", "itertools 0.14.0", "log", "object_store", @@ -1443,10 +1445,9 @@ dependencies = [ "parquet", "rand 0.8.5", "regex", - "sqlparser", + "sqlparser 0.54.0", "tempfile", "tokio", - "tokio-util", "url", "uuid", "xz2", @@ -1485,9 +1486,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f27987bc22b810939e8dfecc55571e9d50355d6ea8ec1c47af8383a76a6d0e1" +checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2" dependencies = [ "arrow", "async-trait", @@ -1501,21 +1502,39 @@ dependencies = [ "itertools 0.14.0", "log", "parking_lot", - "sqlparser", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "log", + "object_store", + "tokio", ] [[package]] name = "datafusion-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602" +checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" dependencies = [ "ahash", "arrow", - "arrow-array", - "arrow-buffer", "arrow-ipc", - "arrow-schema", "base64 0.22.1", "half", "hashbrown 0.14.5", @@ -1526,21 +1545,55 @@ dependencies = [ "parquet", "paste", "recursive", - "sqlparser", + "sqlparser 0.54.0", "tokio", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4603c8e8a4baf77660ab7074cc66fc15cc8a18f2ce9dfadb755fc6ee294e48" +checksum = "e0fcf41523b22e14cc349b01526e8b9f59206653037f2949a4adbfde5f8cb668" dependencies = [ "log", "tokio", ] +[[package]] +name = "datafusion-datasource" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand 0.8.5", + "tokio", + "tokio-util", + "url", + "xz2", + "zstd", +] + [[package]] name = "datafusion-dft" version = "0.2.2" @@ -1594,15 +1647,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bf4bc68623a5cf231eed601ed6eb41f46a37c4d15d11a0bff24cbc8396cd66" +checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256" [[package]] name = "datafusion-execution" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b491c012cdf8e051053426013429a76f74ee3c2db68496c79c323ca1084d27" +checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1" dependencies = [ "arrow", "dashmap", @@ -1619,9 +1672,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a181408d4fc5dc22f9252781a8f39f2d0e5d1b33ec9bde242844980a2689c1" +checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f" dependencies = [ "arrow", "chrono", @@ -1635,26 +1688,27 @@ dependencies = [ "paste", "recursive", "serde_json", - "sqlparser", + "sqlparser 0.54.0", ] [[package]] name = "datafusion-expr-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1129b48e8534d8c03c6543bcdccef0b55c8ac0c1272a15a56c67068b6eb1885" +checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" dependencies = [ "arrow", "datafusion-common", + "indexmap 2.8.0", "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-functions" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6125874e4856dfb09b59886784fcb74cde5cfc5930b3a80a1a728ef7a010df6b" +checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40" dependencies = [ "arrow", "arrow-buffer", @@ -1668,7 +1722,6 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", - "hashbrown 0.14.5", "hex", "itertools 0.14.0", "log", @@ -1682,14 +1735,12 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6" +checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" dependencies = [ "ahash", "arrow", - "arrow-buffer", - "arrow-schema", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1705,9 +1756,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c" +checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" dependencies = [ "ahash", "arrow", @@ -1718,9 +1769,9 @@ dependencies = [ [[package]] name = "datafusion-functions-json" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297b8bf3c213cbf1a0c9d0c5f0bc43f6d181591aeade02a189790d28d79c87d0" +checksum = "3f9658d1ad5c3ac21667d04d01222202cb644fd85b2c5ea9d82c4efa33153d90" dependencies = [ "datafusion", "jiter", @@ -1730,15 +1781,12 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec5ee8cecb0dc370291279673097ddabec03a011f73f30d7f1096457127e03e" +checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a" dependencies = [ "arrow", - "arrow-array", - "arrow-buffer", "arrow-ord", - "arrow-schema", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1764,9 +1812,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c403ddd473bbb0952ba880008428b3c7febf0ed3ce1eec35a205db20efb2a36" +checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5" dependencies = [ "arrow", "async-trait", @@ -1780,9 +1828,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab18c2fb835614d06a75f24a9e09136d3a8c12a92d97c95a6af316a1787a9c5" +checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6" dependencies = [ "datafusion-common", "datafusion-doc", @@ -1797,9 +1845,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77b73bc15e7d1967121fdc7a55d819bfb9d6c03766a6c322247dce9094a53a4" +checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1807,9 +1855,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09369b8d962291e808977cf94d495fd8b5b38647232d7ef562c27ac0f495b0af" +checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" dependencies = [ "datafusion-expr", "quote", @@ -1818,9 +1866,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2403a7e4a84637f3de7d8d4d7a9ccc0cc4be92d89b0161ba3ee5be82f0531c54" +checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b" dependencies = [ "arrow", "chrono", @@ -1837,15 +1885,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c" +checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" dependencies = [ "ahash", "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1862,13 +1907,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435" +checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" dependencies = [ "ahash", "arrow", - "arrow-buffer", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -1877,12 +1921,11 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5e85c189d5238a5cf181a624e450c4cd4c66ac77ca551d6f3ff9080bac90bb" +checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1890,23 +1933,19 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "futures", "itertools 0.14.0", "log", "recursive", - "url", ] [[package]] name = "datafusion-physical-plan" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f" +checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" dependencies = [ "ahash", "arrow", - "arrow-array", - "arrow-buffer", "arrow-ord", "arrow-schema", "async-trait", @@ -1931,9 +1970,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2db5d79f0c974041787b899d24dc91bdab2ff112d1942dd71356a4ce3b407e6c" +checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f" dependencies = [ "arrow", "chrono", @@ -1947,9 +1986,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de21bde1603aac0ff32cf478e47081be6e3583c6861fe8f57034da911efe7578" +checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f" dependencies = [ "arrow", "datafusion-common", @@ -1958,13 +1997,11 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13caa4daede211ecec53c78b13c503b592794d125f9a3cc3afe992edf9e7f43" +checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64" dependencies = [ "arrow", - "arrow-array", - "arrow-schema", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -1972,7 +2009,7 @@ dependencies = [ "log", "recursive", "regex", - "sqlparser", + "sqlparser 0.54.0", ] [[package]] @@ -1997,18 +2034,11 @@ dependencies = [ [[package]] name = "delta_kernel" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac29dfbfdf1d1105af0ed897a9af34f34833550a0ed81b14a2fb669e096dac5" +checksum = "aae7dc3012ad01882cd7669fd9524d7069cd5a6f12d69932a6f125d3bf503019" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-json", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow", "bytes", "chrono", "delta_kernel_derive", @@ -2036,9 +2066,9 @@ dependencies = [ [[package]] name = "delta_kernel_derive" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29fe0a0c1bd83cad5c92ab1ffa035ebb1dd341b35d2fde9d4cc9fd222706dd2" +checksum = "0c8e41236d5a9f04da3072d7186a76aba734e7bfd2cd05f7877fde172b65fb11" dependencies = [ "proc-macro2", "quote", @@ -2047,16 +2077,18 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.24.0" -source = "git+https://github.com/delta-io/delta-rs?rev=d8995e64c305dc795f62d33ce2a4614b335944ff#d8995e64c305dc795f62d33ce2a4614b335944ff" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78889f4005974b848f130fa5dedae81987f1bc93b107291ea87d900c93b6c3bb" dependencies = [ "deltalake-core", ] [[package]] name = "deltalake-core" -version = "0.24.0" -source = "git+https://github.com/delta-io/delta-rs?rev=d8995e64c305dc795f62d33ce2a4614b335944ff#d8995e64c305dc795f62d33ce2a4614b335944ff" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0e2d408fe4cb2c3a81c241c8128fdd359dca92a74367b8671fbac206483163" dependencies = [ "arrow", "arrow-arith", @@ -2106,7 +2138,7 @@ dependencies = [ "roaring", "serde", "serde_json", - "sqlparser", + "sqlparser 0.53.0", "strum", "thiserror 2.0.11", "tokio", @@ -3214,13 +3246,13 @@ dependencies = [ [[package]] name = "jiter" -version = "0.8.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8243cf2d026264056bfacf305e54f5bee8866fd46b4c1873adcaebf614a0d306" +checksum = "c024ccb0ed468a474efa325edea34d4198fb601d290c4d1bc24fe31ed11902fc" dependencies = [ "ahash", "bitvec", - "lexical-parse-float 0.8.5", + "lexical-parse-float", "num-bigint", "num-traits", "pyo3", @@ -3276,42 +3308,21 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" dependencies = [ - "lexical-parse-float 1.0.5", - "lexical-parse-integer 1.0.5", - "lexical-util 1.0.6", + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", "lexical-write-float", "lexical-write-integer", ] -[[package]] -name = "lexical-parse-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" -dependencies = [ - "lexical-parse-integer 0.8.6", - "lexical-util 0.8.5", - "static_assertions", -] - [[package]] name = "lexical-parse-float" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ - "lexical-parse-integer 1.0.5", - "lexical-util 1.0.6", - "static_assertions", -] - -[[package]] -name = "lexical-parse-integer" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" -dependencies = [ - "lexical-util 0.8.5", + "lexical-parse-integer", + "lexical-util", "static_assertions", ] @@ -3321,16 +3332,7 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ - "lexical-util 1.0.6", - "static_assertions", -] - -[[package]] -name = "lexical-util" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" -dependencies = [ + "lexical-util", "static_assertions", ] @@ -3349,7 +3351,7 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" dependencies = [ - "lexical-util 1.0.6", + "lexical-util", "lexical-write-integer", "static_assertions", ] @@ -3360,7 +3362,7 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" dependencies = [ - "lexical-util 1.0.6", + "lexical-util", "static_assertions", ] @@ -3942,9 +3944,9 @@ dependencies = [ [[package]] name = "parquet" -version = "54.1.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" +checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd" dependencies = [ "ahash", "arrow-array", @@ -4224,9 +4226,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.5" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" dependencies = [ "cfg-if", "indoc", @@ -4243,19 +4245,19 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.5" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" dependencies = [ "once_cell", - "target-lexicon 0.12.16", + "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.23.5" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" dependencies = [ "libc", "pyo3-build-config", @@ -4263,9 +4265,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.5" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4275,9 +4277,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.23.5" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" dependencies = [ "heck", "proc-macro2", @@ -5088,6 +5090,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" dependencies = [ "log", +] + +[[package]] +name = "sqlparser" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" +dependencies = [ + "log", + "recursive", "sqlparser_derive", ] @@ -5246,12 +5258,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "target-lexicon" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" - [[package]] name = "target-lexicon" version = "0.13.2" @@ -5814,8 +5820,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" dependencies = [ "getrandom 0.3.1", + "js-sys", "rand 0.9.0", "serde", + "wasm-bindgen", ] [[package]] @@ -6091,7 +6099,7 @@ dependencies = [ "serde_json", "smallvec", "sptr", - "target-lexicon 0.13.2", + "target-lexicon", "trait-variant", "wasm-encoder 0.221.3", "wasmparser 0.221.3", @@ -6180,7 +6188,7 @@ dependencies = [ "log", "object 0.36.7", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", "thiserror 1.0.69", "wasmparser 0.221.3", "wasmtime-environ", @@ -6207,7 +6215,7 @@ dependencies = [ "serde", "serde_derive", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", "wasm-encoder 0.221.3", "wasmparser 0.221.3", "wasmprinter", @@ -6288,7 +6296,7 @@ dependencies = [ "cranelift-codegen", "gimli 0.31.1", "object 0.36.7", - "target-lexicon 0.13.2", + "target-lexicon", "wasmparser 0.221.3", "wasmtime-cranelift", "wasmtime-environ", @@ -6463,7 +6471,7 @@ dependencies = [ "gimli 0.31.1", "regalloc2", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", "thiserror 1.0.69", "wasmparser 0.221.3", "wasmtime-cranelift", diff --git a/Cargo.toml b/Cargo.toml index a49700f3..d93f1606 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,14 +14,14 @@ version = "0.2.2" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow-flight = { version = "54.1.0", features = [ +arrow-flight = { version = "54", features = [ "flight-sql-experimental", ], optional = true } axum = { version = "0.7.9", features = ["macros"], optional = true } clap = { version = "4.5.27", features = ["derive"] } color-eyre = "0.6.3" crossterm = { version = "0.28.1", features = ["event-stream"] } -datafusion = { version = "45.0.0" } +datafusion = { version = "46" } datafusion-app = { version = "0.1.0", path = "crates/datafusion-app" } directories = "5.0.1" env_logger = "0.11.5" @@ -34,7 +34,7 @@ lazy_static = "1.4.0" log = "0.4.22" metrics = { version = "0.24.0", optional = true } metrics-exporter-prometheus = { version = "0.16.0", optional = true } -parquet = "54.1.0" +parquet = "54" pin-project-lite = { version = "0.2.14" } prost = "0.13.1" ratatui = "0.28.0" diff --git a/crates/datafusion-app/Cargo.toml b/crates/datafusion-app/Cargo.toml index 15f40a33..ebeb8df0 100644 --- a/crates/datafusion-app/Cargo.toml +++ b/crates/datafusion-app/Cargo.toml @@ -6,21 +6,19 @@ version = "0.1.0" # Leaving Hudi and Iceberg around in case traction on those picks up. It would be better if I just remove them from main and create PRs with the diff - maybe ill get around to that. [dependencies] -arrow-flight = { version = "54.1.0", features = [ +arrow-flight = { version = "54", features = [ "flight-sql-experimental", ], optional = true } async-trait = "0.1.80" base64 = { version = "0.22.1", optional = true } color-eyre = "0.6.3" -datafusion = "45.0.0" -datafusion-functions-json = { version = "0.45", optional = true } -datafusion-functions-parquet = { version = "0.1.0", path = "..//datafusion-functions-parquet", optional = true } +datafusion = "46" +datafusion-functions-json = { version = "0.46", optional = true } +datafusion-functions-parquet = { version = "0.1.0", path = "../datafusion-functions-parquet", optional = true } datafusion-udfs-wasm = { version = "0.1.0", path = "../datafusion-udfs-wasm", features = [ "serde", ], optional = true } -deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "d8995e64c305dc795f62d33ce2a4614b335944ff", features = [ - "datafusion", -], optional = true } +deltalake = { version = "0.25", features = ["datafusion"], optional = true } directories = "5.0.1" futures = "0.3.30" indexmap = { version = "2.8.0", features = ["serde"] } diff --git a/crates/datafusion-app/src/stats.rs b/crates/datafusion-app/src/stats.rs index 25707f67..7f267700 100644 --- a/crates/datafusion-app/src/stats.rs +++ b/crates/datafusion-app/src/stats.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::{ - datasource::physical_plan::ParquetExec, + datasource::{physical_plan::ParquetSource, source::DataSourceExec}, physical_plan::{ aggregates::AggregateExec, filter::FilterExec, @@ -354,11 +354,19 @@ impl PlanIOVisitor { self.time_opening = metrics.sum_by_name("time_elapsed_opening"); self.time_scanning = metrics.sum_by_name("time_elapsed_scanning_total"); - if plan.as_any().downcast_ref::().is_some() { - self.parquet_output_rows = metrics.output_rows(); - self.parquet_rg_pruned_stats = metrics.sum_by_name("row_groups_pruned_statistics"); - self.parquet_rg_matched_stats = - metrics.sum_by_name("row_groups_matched_statistics"); + if let Some(data_source_exec) = plan.as_any().downcast_ref::() { + if data_source_exec + .data_source() + .as_any() + .downcast_ref::() + .is_some() + { + self.parquet_output_rows = metrics.output_rows(); + self.parquet_rg_pruned_stats = + metrics.sum_by_name("row_groups_pruned_statistics"); + self.parquet_rg_matched_stats = + metrics.sum_by_name("row_groups_matched_statistics"); + } } } } diff --git a/crates/datafusion-functions-parquet/Cargo.toml b/crates/datafusion-functions-parquet/Cargo.toml index 8c35dd5b..86f9e71a 100644 --- a/crates/datafusion-functions-parquet/Cargo.toml +++ b/crates/datafusion-functions-parquet/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/cra version = "0.1.0" [dependencies] -arrow = { version = "54.1.0" } +arrow = { version = "54" } async-trait = "0.1.41" -datafusion = { version = "45.0.0" } -parquet = { version = "54.1.0", default-features = false } +datafusion = { version = "46" } +parquet = { version = "54", default-features = false } diff --git a/crates/datafusion-functions-parquet/src/lib.rs b/crates/datafusion-functions-parquet/src/lib.rs index f0f3efbb..91c48cab 100644 --- a/crates/datafusion-functions-parquet/src/lib.rs +++ b/crates/datafusion-functions-parquet/src/lib.rs @@ -23,10 +23,10 @@ use async_trait::async_trait; use datafusion::catalog::Session; use datafusion::catalog::TableFunctionImpl; use datafusion::common::{plan_err, Column}; +use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::TableProvider; use datafusion::error::Result; use datafusion::logical_expr::Expr; -use datafusion::physical_plan::memory::MemoryExec; use datafusion::physical_plan::ExecutionPlan; use datafusion::scalar::ScalarValue; use parquet::basic::ConvertedType; @@ -51,7 +51,7 @@ impl TableProvider for ParquetMetadataTable { self } - fn schema(&self) -> datafusion::arrow::datatypes::SchemaRef { + fn schema(&self) -> arrow::datatypes::SchemaRef { self.schema.clone() } @@ -66,11 +66,11 @@ impl TableProvider for ParquetMetadataTable { _filters: &[Expr], _limit: Option, ) -> Result> { - Ok(Arc::new(MemoryExec::try_new( + Ok(MemorySourceConfig::try_new_exec( &[vec![self.batch.clone()]], TableProvider::schema(self), projection.cloned(), - )?)) + )?) } } @@ -183,7 +183,7 @@ impl TableFunctionImpl for ParquetMetadataFunc { Field::new("total_uncompressed_size", DataType::Int64, true), ])); - // construct recordbatch from metadata + // construct record batch from metadata let mut filename_arr = vec![]; let mut row_group_id_arr = vec![]; let mut row_group_num_rows_arr = vec![]; diff --git a/crates/datafusion-udfs-wasm/Cargo.toml b/crates/datafusion-udfs-wasm/Cargo.toml index 300d4987..fe99f858 100644 --- a/crates/datafusion-udfs-wasm/Cargo.toml +++ b/crates/datafusion-udfs-wasm/Cargo.toml @@ -4,7 +4,7 @@ name = "datafusion-udfs-wasm" version = "0.1.0" [dependencies] -datafusion = { version = "45.0.0", default-features = false } +datafusion = { version = "46", default-features = false } log = "0.4.25" serde = { version = "1.0.217", optional = true } wasi-common = "29.0.1" diff --git a/crates/datafusion-udfs-wasm/wasm-examples/Cargo.toml b/crates/datafusion-udfs-wasm/wasm-examples/Cargo.toml index ddc0cc0e..95f9ee19 100644 --- a/crates/datafusion-udfs-wasm/wasm-examples/Cargo.toml +++ b/crates/datafusion-udfs-wasm/wasm-examples/Cargo.toml @@ -7,4 +7,4 @@ version = "0.1.0" crate-type = ["cdylib"] [dependencies] -arrow = "53" +arrow = "54" From ad46ced11bdb21023770bc22ac62bcf733c34a46 Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Fri, 11 Apr 2025 10:08:14 -0400 Subject: [PATCH 2/2] Fix tests --- .github/workflows/test.yml | 2 +- tests/cli_cases/basic.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea4fc4b6..7e8f2663 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: key: ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.lock') }} - name: Setup Rust Toolchain uses: ./.github/actions/setup-rust - - name: Run tests (ex-FlightSQL) + - name: Run core tests run: | cargo test test-flightsql: diff --git a/tests/cli_cases/basic.rs b/tests/cli_cases/basic.rs index e6fe35a2..7e3e1e23 100644 --- a/tests/cli_cases/basic.rs +++ b/tests/cli_cases/basic.rs @@ -176,7 +176,7 @@ fn test_sql_err_in_file() { .failure(); let expected_err = - "Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: not"; + "Expected: [NOT] NULL | TRUE | FALSE | DISTINCT | [form] NORMALIZED FROM after IS, found: valid at Line: 1, Column: 20"; assert.code(1).stderr(contains_str(expected_err)); } @@ -199,7 +199,7 @@ SELECT this is not valid SQL .failure(); let expected_err = - "Expected: [NOT] NULL or TRUE|FALSE or [NOT] DISTINCT FROM after IS, found: not"; + "Expected: [NOT] NULL | TRUE | FALSE | DISTINCT | [form] NORMALIZED FROM after IS, found: valid at Line: 5, Column: 20"; assert.code(1).stderr(contains_str(expected_err)); }