diff --git a/Cargo.lock b/Cargo.lock index 4f23782..e94ef5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,6 +194,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.1" @@ -831,6 +840,15 @@ dependencies = [ "serde", ] +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -1086,7 +1104,7 @@ version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.98", @@ -1223,6 +1241,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.0" @@ -1391,6 +1419,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.14" @@ -1419,6 +1460,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1543,6 +1593,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" + [[package]] name = "datafusion" version = "45.0.0" @@ -1606,6 +1662,7 @@ dependencies = [ "datafusion-auth", "datafusion-functions-json", "datafusion-functions-parquet", + "datafusion-table-providers", "datafusion-udfs-wasm", "deltalake", "directories", @@ -2133,6 +2190,37 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datafusion-table-providers" +version = "0.2.4" +source = "git+https://github.com/jychen7/datafusion-table-providers?branch=datafusion-45#92f471c3e52d9d0f4d639a88b3fb39198fb93dfe" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "byteorder", + "chrono", + "dashmap", + "datafusion", + "fallible-iterator", + "fundu", + "futures", + "geo-types", + "itertools 0.14.0", + "mysql_async", + "num-bigint", + "sea-query", + "secrecy", + "serde_json", + "sha2", + "snafu", + "time", + "tokio", + "tracing", + "trust-dns-resolver", + "url", +] + [[package]] name = "datafusion-udfs-wasm" version = "0.1.0" @@ -2461,6 +2549,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -2603,6 +2703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", + "libz-sys", "miniz_oxide 0.8.5", ] @@ -2627,6 +2728,21 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2653,6 +2769,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fundu" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce12752fc64f35be3d53e0a57017cd30970f0cffd73f62c791837d8845badbd" +dependencies = [ + "fundu-core", +] + +[[package]] +name = "fundu-core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e463452e2d8b7600d38dcea1ed819773a57f0d710691bfc78db3961bd3f4c3ba" + [[package]] name = "funty" version = "2.0.0" @@ -2793,6 +2924,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo-types" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1157f0f936bf0cd68dec91e8f7c311afe60295574d62b70d4861a1bfdf2d9" +dependencies = [ + "approx", + "num-traits", + "serde", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -2914,6 +3056,12 @@ dependencies = [ "serde", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2950,6 +3098,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "1.2.0" @@ -3384,6 +3543,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "1.0.3" @@ -3439,6 +3608,17 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "inherent" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c38228f24186d9cc68c729accb4d413be9eaed6ad07ff79e0270d9e56f3de13" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "insta" version = "1.42.1" @@ -3488,6 +3668,18 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -3587,6 +3779,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "keyed_priority_queue" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" +dependencies = [ + "indexmap 2.7.1", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -3761,6 +3962,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-sys" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -3817,13 +4029,22 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lz4_flex" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ - "twox-hash", + "twox-hash 1.6.3", ] [[package]] @@ -3852,6 +4073,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.1.0" @@ -4024,6 +4251,107 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "mysql-common-derive" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb6d9ff4094f6d58d3f892fc558e60048476213dd17dcf904b62202e9029da6" +dependencies = [ + "darling", + "heck 0.5.0", + "num-bigint", + "proc-macro-crate", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.98", + "termcolor", + "thiserror 1.0.69", +] + +[[package]] +name = "mysql_async" +version = "0.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d14cf024116ba8fef4a7fec5abf0bd5de89b9fb29a7e55818a119ac5ec745077" +dependencies = [ + "bytes", + "crossbeam", + "flate2", + "futures-core", + "futures-sink", + "futures-util", + "keyed_priority_queue", + "lru", + "mysql_common", + "native-tls", + "pem", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "serde", + "serde_json", + "socket2", + "thiserror 2.0.11", + "tokio", + "tokio-native-tls", + "tokio-util", + "twox-hash 2.1.0", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34a9141e735d5bb02414a7ac03add09522466d4db65bdd827069f76ae0850e58" +dependencies = [ + "base64 0.22.1", + "bigdecimal", + "bitflags 2.9.0", + "btoi", + "byteorder", + "bytes", + "cc", + "chrono", + "cmake", + "crc32fast", + "flate2", + "lazy_static", + "mysql-common-derive", + "num-bigint", + "num-traits", + "rand 0.8.5", + "regex", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2", + "subprocess", + "thiserror 1.0.69", + "time", + "uuid", + "zstd", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -4246,12 +4574,50 @@ dependencies = [ "uuid", ] +[[package]] +name = "openssl" +version = "0.10.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -4359,7 +4725,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash", + "twox-hash 1.6.3", "zstd", "zstd-sys", ] @@ -4379,6 +4745,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64 0.22.1", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -4553,6 +4929,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -4692,7 +5090,7 @@ version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", @@ -4720,6 +5118,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.35.0" @@ -5125,6 +5529,16 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + [[package]] name = "ring" version = "0.17.11" @@ -5277,7 +5691,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.2.0", ] [[package]] @@ -5331,6 +5745,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "schannel" version = "0.1.27" @@ -5352,12 +5772,62 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sea-query" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b731192738ebf56d20580fc8ba2d23940333befe900b04dd08a26a77cd056f02" +dependencies = [ + "bigdecimal", + "chrono", + "inherent", + "rust_decimal", + "sea-query-derive", + "time", +] + +[[package]] +name = "sea-query-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9834af2c4bd8c5162f00c89f1701fb6886119a88062cf76fe842ea9e232b9839" +dependencies = [ + "darling", + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.98", + "thiserror 1.0.69", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.2.0" @@ -5365,7 +5835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -5632,7 +6102,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.98", @@ -5736,7 +6206,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -5749,13 +6219,23 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", "syn 2.0.98", ] +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.6.1" @@ -6028,6 +6508,16 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.1" @@ -6296,6 +6786,52 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc" +[[package]] +name = "trust-dns-proto" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.4.0", + "ipnet", + "once_cell", + "rand 0.8.5", + "smallvec", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand 0.8.5", + "resolv-conf", + "smallvec", + "thiserror 1.0.69", + "tokio", + "tracing", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -6340,6 +6876,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "twox-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" + [[package]] name = "typed-builder" version = "0.19.1" @@ -6386,12 +6928,27 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -6446,7 +7003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna", + "idna 1.0.3", "percent-encoding", ] @@ -6491,6 +7048,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -6963,7 +7526,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8358319c2dd1e4db79e3c1c5d3a5af84956615343f9f89f4e4996a36816e06e6" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "indexmap 2.7.1", "wit-parser", ] @@ -7040,6 +7603,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "widestring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" + [[package]] name = "wiggle" version = "29.0.1" @@ -7062,7 +7631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cf267dd05673912c8138f4b54acabe6bd53407d9d1536f0fadb6520dd16e101" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "proc-macro2", "quote", "shellexpand", @@ -7372,6 +7941,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "winx" version = "0.36.4" diff --git a/Cargo.toml b/Cargo.toml index fb47c57..a45735a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ hudi = ["datafusion-app/hudi"] iceberg = ["datafusion-app/iceberg"] flightsql = ["datafusion-app/flightsql", "dep:tonic", "dep:arrow-flight", "dep:metrics", "dep:metrics-exporter-prometheus", "dep:tower-http"] s3 = ["datafusion-app/s3", "url"] +mysql = ["datafusion-app/mysql"] functions-json = ["datafusion-app/functions-json"] functions-parquet = ["datafusion-app/functions-parquet"] udfs-wasm = ["datafusion-app/udfs-wasm", "dep:datafusion-udfs-wasm"] diff --git a/crates/datafusion-app/Cargo.toml b/crates/datafusion-app/Cargo.toml index a198e2c..ac3a08d 100644 --- a/crates/datafusion-app/Cargo.toml +++ b/crates/datafusion-app/Cargo.toml @@ -16,6 +16,7 @@ datafusion-functions-parquet = { version = "0.1.0", path = "..//datafusion-funct datafusion-udfs-wasm = { version = "0.1.0", path = "../datafusion-udfs-wasm", features = [ "serde", ], optional = true } +datafusion-table-providers = { git = "https://github.com/jychen7/datafusion-table-providers", branch = "datafusion-45", features = ["mysql"], optional = true } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "d8995e64c305dc795f62d33ce2a4614b335944ff", features = [ "datafusion", ], optional = true } @@ -60,3 +61,4 @@ functions-json = ["dep:datafusion-functions-json"] functions-parquet = ["dep:datafusion-functions-parquet"] udfs-wasm = ["dep:datafusion-udfs-wasm"] huggingface = ["opendal", "object_store_opendal", "url"] +mysql = ["dep:datafusion-table-providers"] diff --git a/crates/datafusion-app/src/config.rs b/crates/datafusion-app/src/config.rs index 5861266..afd080e 100644 --- a/crates/datafusion-app/src/config.rs +++ b/crates/datafusion-app/src/config.rs @@ -34,6 +34,7 @@ use object_store::aws::{AmazonS3, AmazonS3Builder}; #[derive(Clone, Debug, Deserialize)] pub struct ExecutionConfig { + pub database: Option, pub object_store: Option, #[serde(default = "default_ddl_path")] pub ddl_path: Option, @@ -59,6 +60,7 @@ pub struct ExecutionConfig { impl Default for ExecutionConfig { fn default() -> Self { Self { + database: None, object_store: None, ddl_path: default_ddl_path(), benchmark_iterations: default_benchmark_iterations(), @@ -195,6 +197,81 @@ pub struct ObjectStoreConfig { pub huggingface: Option>, } +#[derive(Clone, Debug, Deserialize)] +pub struct DatabaseConfig { + #[cfg(feature = "mysql")] + pub mysql: Option>, +} + +#[cfg(feature = "mysql")] +#[derive(Clone, Debug, Deserialize)] +pub struct MySQLConfig { + #[serde(default = "default_mysql_name")] + pub name: String, + #[serde(default = "default_mysql_host")] + pub host: String, + #[serde(default = "default_mysql_port")] + pub port: u16, + #[serde(default = "default_mysql_user")] + pub user: String, + #[serde(default = "default_mysql_password")] + pub password: String, + #[serde(default = "default_mysql_database")] + pub database: String, + #[serde(default = "default_mysql_sslmode")] + pub sslmode: String, +} + +#[cfg(feature = "mysql")] +fn default_mysql_name() -> String { + "mysql".to_string() +} + +#[cfg(feature = "mysql")] +fn default_mysql_host() -> String { + "127.0.0.1".to_string() +} + +#[cfg(feature = "mysql")] +fn default_mysql_port() -> u16 { + 3306 +} + +#[cfg(feature = "mysql")] +fn default_mysql_user() -> String { + "root".to_string() +} + +#[cfg(feature = "mysql")] +fn default_mysql_password() -> String { + "".to_string() +} + +#[cfg(feature = "mysql")] +fn default_mysql_database() -> String { + "".to_string() +} + +#[cfg(feature = "mysql")] +fn default_mysql_sslmode() -> String { + "disabled".to_string() +} + +#[cfg(feature = "mysql")] +impl Default for MySQLConfig { + fn default() -> Self { + Self { + name: default_mysql_name(), + host: default_mysql_host(), + port: default_mysql_port(), + user: default_mysql_user(), + password: default_mysql_password(), + database: default_mysql_database(), + sslmode: default_mysql_sslmode(), + } + } +} + #[derive(Clone, Debug, Deserialize)] pub struct RestCatalogConfig { pub name: String, diff --git a/crates/datafusion-app/src/extensions/mod.rs b/crates/datafusion-app/src/extensions/mod.rs index d410223..217c2d2 100644 --- a/crates/datafusion-app/src/extensions/mod.rs +++ b/crates/datafusion-app/src/extensions/mod.rs @@ -33,7 +33,8 @@ mod huggingface; mod iceberg; #[cfg(feature = "s3")] mod s3; - +#[cfg(feature = "mysql")] +mod mysql; pub use builder::DftSessionStateBuilder; #[async_trait::async_trait] @@ -66,5 +67,7 @@ pub fn enabled_extensions() -> Vec> { Arc::new(iceberg::IcebergExtension::new()), #[cfg(feature = "huggingface")] Arc::new(huggingface::HuggingFaceExtension::new()), + #[cfg(feature = "mysql")] + Arc::new(mysql::MysqlExtension::new()), ] } diff --git a/crates/datafusion-app/src/extensions/mysql.rs b/crates/datafusion-app/src/extensions/mysql.rs new file mode 100644 index 0000000..85a680e --- /dev/null +++ b/crates/datafusion-app/src/extensions/mysql.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! MySQL Integration: [MysqlExtension] + +use std::{collections::HashMap, sync::Arc}; +use crate::config::ExecutionConfig; +use crate::extensions::{DftSessionStateBuilder, Extension}; +use datafusion_table_providers::{ + common::DatabaseCatalogProvider, mysql::MySQLTableFactory, + sql::db_connection_pool::mysqlpool::MySQLConnectionPool, util::secrets::to_secret_map, +}; +use log::info; + +#[derive(Debug, Default)] +pub struct MysqlExtension {} + +impl MysqlExtension { + pub fn new() -> Self { + Self {} + } +} + +#[async_trait::async_trait] +impl Extension for MysqlExtension { + async fn register( + &self, + config: ExecutionConfig, + builder: &mut DftSessionStateBuilder, + ) -> datafusion::common::Result<()> { + let Some(database_config) = &config.database else { + return Ok(()); + }; + + let Some(mysql_configs) = &database_config.mysql else { + return Ok(()); + }; + + info!("MySQL configs exists"); + for mysql_config in mysql_configs { + let mysql_params = to_secret_map(HashMap::from([ + ( + "connection_string".to_string(), + format!("mysql://{}:{}@{}:{}/{}", mysql_config.user, mysql_config.password, mysql_config.host, mysql_config.port, mysql_config.database), + ), + ("sslmode".to_string(), mysql_config.sslmode.clone()), + ])); + let mysql_pool = Arc::new( + MySQLConnectionPool::new(mysql_params) + .await + .expect("unable to create MySQL connection pool"), + ); + let catalog = DatabaseCatalogProvider::try_new(mysql_pool).await.unwrap(); + builder.add_catalog_provider(&mysql_config.name, Arc::new(catalog)); + } + + Ok(()) + } +}