diff --git a/.gitignore b/.gitignore index 238409e..91ee8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ rpm/usr/local/bin/dispenser *.deb *.rpm **/**/.DS_Store +/data diff --git a/Cargo.lock b/Cargo.lock index d00e60a..745588e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -60,6 +61,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -118,6 +125,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "arc-swap" version = "1.8.0" @@ -127,12 +140,238 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.14.5", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", + "lz4_flex", +] + +[[package]] +name = "arrow-json" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap 2.13.0", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" +dependencies = [ + "serde", +] + +[[package]] +name = "arrow-select" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + [[package]] name = "asn1-rs" version = "0.7.1" @@ -157,7 +396,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", "synstructure", ] @@ -169,7 +408,25 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", +] + +[[package]] +name = "async-compression" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +dependencies = [ + "bzip2 0.5.2", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", ] [[package]] @@ -191,7 +448,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -202,7 +459,16 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -217,7 +483,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -228,6 +494,48 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96571e6996817bf3d58f6b569e4b9fd2e9d2fcf9f7424eed07b2ce9bb87535e5" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.0", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.15.2" @@ -251,135 +559,499 @@ dependencies = [ ] [[package]] -name = "backtrace" -version = "0.3.76" +name = "aws-runtime" +version = "1.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +checksum = "959dab27ce613e6c9658eb3621064d0e2027e5f2acb65bc526a43577facea557" dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", ] [[package]] -name = "base64" -version = "0.22.1" +name = "aws-sdk-dynamodb" +version = "1.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "f5f7e6a53cf5ee8b7041c73106d9a93480b47f8b955466262b043aab0b5bf489" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] [[package]] -name = "bitflags" -version = "1.3.2" +name = "aws-sdk-sso" +version = "1.92.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "b7d63bd2bdeeb49aa3f9b00c15e18583503b778b2e792fc06284d54e7d5b6566" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] [[package]] -name = "bitflags" -version = "2.10.0" +name = "aws-sdk-ssooidc" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "532d93574bf731f311bafb761366f9ece345a0416dbcc273d81d6d1a1205239b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] [[package]] -name = "blake2" -version = "0.10.6" +name = "aws-sdk-sts" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "357e9a029c7524db6a0099cd77fbd5da165540339e7296cca603531bc783b56c" dependencies = [ - "digest", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] -name = "block-buffer" -version = "0.10.4" +name = "aws-sigv4" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" dependencies = [ - "generic-array", + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2", + "time", + "tracing", ] [[package]] -name = "bollard" -version = "0.19.4" +name = "aws-smithy-async" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" +checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c" dependencies = [ - "base64", - "bollard-stubs", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", + "bytes-utils", "futures-core", "futures-util", - "hex", - "http", - "http-body-util", - "hyper", - "hyper-named-pipe", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.12", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", "hyper-util", - "hyperlocal", - "log", "pin-project-lite", - "serde", - "serde_derive", - "serde_json", - "serde_repr", - "serde_urlencoded", - "thiserror 2.0.17", + "rustls 0.21.12", + "rustls 0.23.35", + "rustls-native-certs", + "rustls-pki-types", "tokio", - "tokio-util", - "tower-service", - "url", - "winapi", + "tokio-rustls 0.26.4", + "tower", + "tracing", ] [[package]] -name = "bollard-stubs" -version = "1.49.1-rc.28.4.0" +name = "aws-smithy-json" +version = "0.61.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5731fe885755e92beff1950774068e0cae67ea6ec7587381536fca84f1779623" +checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" dependencies = [ - "serde", - "serde_json", - "serde_repr", - "serde_with", + "aws-smithy-types", ] [[package]] -name = "bon" -version = "3.8.1" +name = "aws-smithy-observability" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +checksum = "ef1fcbefc7ece1d70dcce29e490f269695dfca2d2bacdeaf9e5c3f799e4e6a42" dependencies = [ - "bon-macros", - "rustversion", + "aws-smithy-runtime-api", ] [[package]] -name = "bon-macros" -version = "3.8.1" +name = "aws-smithy-query" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +checksum = "ae5d689cf437eae90460e944a58b5668530d433b4ff85789e69d2f2a556e057d" dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.87", + "aws-smithy-types", + "urlencoding", ] [[package]] -name = "brotli" -version = "3.5.0" +name = "aws-smithy-runtime" +version = "1.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "bb5b6167fcdf47399024e81ac08e795180c576a20e4d4ce67949f9a88ae37dc1" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efce7aaaf59ad53c5412f14fc19b2d5c6ab2c3ec688d272fd31f76ec12f44fb0" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65f172bcb02424eb94425db8aed1b6d583b5104d4d5ddddf22402c661a320048" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "tokio", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bollard" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http 1.4.0", + "http-body-util", + "hyper 1.8.1", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.17", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.49.1-rc.28.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5731fe885755e92beff1950774068e0cae67ea6ec7587381536fca84f1779623" +dependencies = [ + "serde", + "serde_json", + "serde_repr", + "serde_with", +] + +[[package]] +name = "bon" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.114", +] + +[[package]] +name = "brotli" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 2.5.1", +] + +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 4.0.3", ] [[package]] @@ -392,12 +1064,28 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "brotli-decompressor" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -413,6 +1101,45 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" version = "1.2.51" @@ -473,16 +1200,39 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-targets 0.52.6", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", ] [[package]] @@ -546,7 +1296,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -590,117 +1340,757 @@ dependencies = [ ] [[package]] -name = "core-foundation" -version = "0.10.1" +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cron" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" +dependencies = [ + "chrono", + "once_cell", + "serde", + "winnow", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "daemonize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" +dependencies = [ + "libc", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.114", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + +[[package]] +name = "datafusion" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "async-compression", + "async-trait", + "bytes", + "bzip2 0.4.4", + "chrono", + "dashmap", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-sql", + "flate2", + "futures", + "glob", + "half", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "itertools 0.12.1", + "log", + "num_cpus", + "object_store", + "parking_lot", + "parquet", + "paste", + "pin-project-lite", + "rand 0.8.5", + "sqlparser 0.49.0", + "tempfile", + "tokio", + "tokio-util", + "url", + "uuid", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", +] + +[[package]] +name = "datafusion-common" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.5", + "instant", + "libc", + "num_cpus", + "object_store", + "parquet", + "sqlparser 0.49.0", +] + +[[package]] +name = "datafusion-common-runtime" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1" +dependencies = [ + "tokio", +] + +[[package]] +name = "datafusion-execution" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" +dependencies = [ + "arrow", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "hashbrown 0.14.5", + "log", + "object_store", + "parking_lot", + "rand 0.8.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "chrono", + "datafusion-common", + "paste", + "serde_json", + "sqlparser 0.49.0", + "strum", + "strum_macros", +] + +[[package]] +name = "datafusion-functions" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hashbrown 0.14.5", + "hex", + "itertools 0.12.1", + "log", + "md-5", + "rand 0.8.5", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb" +dependencies = [ + "ahash", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", + "paste", + "sqlparser 0.49.0", +] + +[[package]] +name = "datafusion-functions-nested" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "itertools 0.12.1", + "log", + "paste", + "rand 0.8.5", +] + +[[package]] +name = "datafusion-optimizer" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "itertools 0.12.1", + "log", + "paste", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", + "base64", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "hex", + "indexmap 2.13.0", + "itertools 0.12.1", + "log", + "paste", + "petgraph", + "regex", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "hashbrown 0.14.5", + "rand 0.8.5", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0" +dependencies = [ + "datafusion-common", + "datafusion-execution", + "datafusion-physical-expr", + "datafusion-physical-plan", +] + +[[package]] +name = "datafusion-physical-plan" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "itertools 0.12.1", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "datafusion-proto" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +checksum = "cf1d25864c18178d0e51438648f5e0fa08417dbbc39b642c1752cbbb1013abf0" dependencies = [ - "core-foundation-sys", - "libc", + "arrow", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-proto-common", + "object_store", + "prost", ] [[package]] -name = "core-foundation-sys" -version = "0.8.7" +name = "datafusion-proto-common" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +checksum = "96a683253732334526b1cc5314a73a0f786803831f7e189ed3fe387ac50d7222" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "object_store", + "prost", +] [[package]] -name = "crc32fast" -version = "1.5.0" +name = "datafusion-sql" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0" dependencies = [ - "cfg-if", + "arrow", + "arrow-array", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "log", + "regex", + "sqlparser 0.49.0", + "strum", ] [[package]] -name = "cron" -version = "0.15.0" +name = "delta_kernel" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740" +checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8" dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-cast", + "arrow-json", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", "chrono", - "once_cell", + "delta_kernel_derive", + "either", + "fix-hidden-lifetime-bug", + "indexmap 2.13.0", + "itertools 0.13.0", + "lazy_static", + "parquet", + "roaring", + "rustc_version", "serde", - "winnow", + "serde_json", + "strum", + "thiserror 1.0.69", + "tracing", + "url", + "uuid", + "visibility", + "z85", ] [[package]] -name = "crossbeam-queue" -version = "0.3.12" +name = "delta_kernel_derive" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff" dependencies = [ - "crossbeam-utils", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.7" +name = "deltalake" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "0cdaf5eed6cf6be7d94ce89ee9d7325324fc7c6c0b1ca8b911b0a5d95f6b1af5" dependencies = [ - "generic-array", - "typenum", + "deltalake-aws", + "deltalake-azure", + "deltalake-core", + "deltalake-gcp", ] [[package]] -name = "daemonize" -version = "0.5.0" +name = "deltalake-aws" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" +checksum = "4e9d1a7b1f51be9509d29a4bfd95686646c973710a00d210707ec3672cb00a9a" dependencies = [ - "libc", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-dynamodb", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "backon", + "bytes", + "chrono", + "deltalake-core", + "futures", + "lazy_static", + "maplit", + "object_store", + "regex", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", + "uuid", ] [[package]] -name = "darling" -version = "0.21.3" +name = "deltalake-azure" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "2149d82f72ea31082f0279cd661962d9b4fea57d9578a100d2e30c16eb27b98b" dependencies = [ - "darling_core", - "darling_macro", + "async-trait", + "bytes", + "deltalake-core", + "futures", + "lazy_static", + "object_store", + "regex", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", ] [[package]] -name = "darling_core" -version = "0.21.3" +name = "deltalake-core" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "0ebab21c8c8820f9accb3ee74cc6ab7d930adf44323f39b7a764fd34e34aa7f4" dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.11.1", - "syn 2.0.87", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "cfg-if", + "chrono", + "dashmap", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-proto", + "datafusion-sql", + "delta_kernel", + "either", + "errno", + "fix-hidden-lifetime-bug", + "futures", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "itertools 0.13.0", + "lazy_static", + "libc", + "maplit", + "num-bigint", + "num-traits", + "num_cpus", + "object_store", + "once_cell", + "parking_lot", + "parquet", + "percent-encoding", + "pin-project-lite", + "rand 0.8.5", + "regex", + "roaring", + "serde", + "serde_json", + "sqlparser 0.51.0", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", + "urlencoding", + "uuid", + "z85", ] [[package]] -name = "darling_macro" -version = "0.21.3" +name = "deltalake-gcp" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "0994a88f86d5db28fc42f9d5e190dbb45c59a42dadaa0ca28a7c75cd4074be7a" dependencies = [ - "darling_core", - "quote", - "syn 2.0.87", + "async-trait", + "bytes", + "deltalake-core", + "futures", + "lazy_static", + "object_store", + "regex", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", ] -[[package]] -name = "data-encoding" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" - [[package]] name = "der-parser" version = "10.0.0" @@ -751,18 +2141,20 @@ dependencies = [ name = "dispenser" version = "0.11.0" dependencies = [ + "arrow", "async-trait", "base64", "bollard", "chrono", "clap 4.5.18", "cron", + "deltalake", "env_logger", "futures", "futures-util", "google-cloud-gax", "google-cloud-secretmanager-v1", - "http", + "http 1.4.0", "instant-acme", "log", "minijinja", @@ -776,7 +2168,7 @@ dependencies = [ "pingora-proxy", "rcgen", "reqwest", - "rustls", + "rustls 0.23.35", "sd-notify", "serde", "serde_json", @@ -785,6 +2177,7 @@ dependencies = [ "tokio", "toml", "urlencoding", + "uuid", "x509-parser", ] @@ -796,9 +2189,15 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + [[package]] name = "dunce" version = "1.0.5" @@ -846,12 +2245,64 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.1", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +[[package]] +name = "fix-hidden-lifetime-bug" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7b4994e93dd63050356bdde7d417591d1b348523638dc1c1f539f16e338d55" +dependencies = [ + "fix-hidden-lifetime-bug-proc_macros", +] + +[[package]] +name = "fix-hidden-lifetime-bug-proc_macros" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f0de9daf465d763422866d0538f07be1596e05623e120b37b4f715f5585200" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.5" @@ -875,6 +2326,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -961,7 +2418,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -1037,6 +2494,12 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "google-cloud-auth" version = "1.3.0" @@ -1048,10 +2511,10 @@ dependencies = [ "bon", "bytes", "google-cloud-gax", - "http", + "http 1.4.0", "reqwest", "rustc_version", - "rustls", + "rustls 0.23.35", "rustls-pemfile", "serde", "serde_json", @@ -1071,7 +2534,7 @@ dependencies = [ "futures", "google-cloud-rpc", "google-cloud-wkt", - "http", + "http 1.4.0", "pin-project", "rand 0.9.2", "serde", @@ -1091,9 +2554,9 @@ dependencies = [ "google-cloud-auth", "google-cloud-gax", "google-cloud-rpc", - "http", + "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "opentelemetry-semantic-conventions", "percent-encoding", "reqwest", @@ -1207,6 +2670,25 @@ dependencies = [ "url", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.12" @@ -1218,14 +2700,26 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", - "indexmap 2.5.0", + "http 1.4.0", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1237,6 +2731,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "hashbrown" @@ -1246,7 +2744,18 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -1270,12 +2779,27 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hostname" version = "0.4.2" @@ -1287,6 +2811,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.0" @@ -1297,6 +2832,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1304,7 +2850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1315,8 +2861,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1338,6 +2884,30 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1348,9 +2918,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.12", + "http 1.4.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1368,7 +2938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -1376,21 +2946,36 @@ dependencies = [ "winapi", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", - "rustls", + "rustls 0.23.35", "rustls-native-certs", "rustls-pki-types", "rustls-platform-verifier", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", "webpki-roots", ] @@ -1406,14 +2991,14 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", - "hyper", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.1", "tokio", "tower-service", "tracing", @@ -1427,7 +3012,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -1579,13 +3164,26 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1597,16 +3195,16 @@ dependencies = [ "async-trait", "base64", "bytes", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "httpdate", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "rcgen", "ring", - "rustls", + "rustls 0.23.35", "rustls-pki-types", "serde", "serde_json", @@ -1614,6 +3212,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "ipnet" version = "2.11.0" @@ -1636,6 +3240,24 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1690,12 +3312,82 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "libz-ng-sys" version = "1.1.23" @@ -1712,6 +3404,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1760,6 +3458,42 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash 2.1.2", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1871,6 +3605,20 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1881,6 +3629,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -1896,6 +3653,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1903,6 +3682,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi 0.5.2", + "libc", ] [[package]] @@ -1914,6 +3704,37 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "futures", + "humantime", + "hyper 1.8.1", + "itertools 0.13.0", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.8.5", + "reqwest", + "ring", + "rustls-pemfile", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "oid-registry" version = "0.8.1" @@ -1952,7 +3773,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -1993,52 +3814,166 @@ dependencies = [ name = "opentelemetry-semantic-conventions" version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "52.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli 6.0.0", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.14.5", + "lz4_flex", + "num", + "num-bigint", + "object_store", + "paste", + "seq-macro", + "snap", + "thrift", + "tokio", + "twox-hash 1.6.3", + "zstd", + "zstd-sys", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "os_str_bytes" -version = "6.6.1" +name = "petgraph" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.13.0", +] [[package]] -name = "parking_lot" -version = "0.12.5" +name = "phf" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "lock_api", - "parking_lot_core", + "phf_shared", ] [[package]] -name = "parking_lot_core" -version = "0.9.12" +name = "phf_codegen" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", + "phf_generator", + "phf_shared", ] [[package]] -name = "pem" -version = "3.0.6" +name = "phf_generator" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "base64", - "serde_core", + "phf_shared", + "rand 0.8.5", ] [[package]] -name = "percent-encoding" -version = "2.3.2" +name = "phf_shared" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] [[package]] name = "pin-project" @@ -2057,7 +3992,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -2099,7 +4034,7 @@ dependencies = [ "cf-rustracing", "cf-rustracing-jaeger", "hex", - "http", + "http 1.4.0", "httparse", "httpdate", "indexmap 1.9.3", @@ -2130,7 +4065,7 @@ checksum = "76f63d3f67d99c95a1f85623fc43242fd644dd12ccbaa18c38a54e1580c6846a" dependencies = [ "ahash", "async-trait", - "brotli", + "brotli 3.5.0", "bytes", "chrono", "clap 3.2.25", @@ -2138,8 +4073,8 @@ dependencies = [ "derivative", "flate2", "futures", - "h2", - "http", + "h2 0.4.12", + "http 1.4.0", "httparse", "httpdate", "libc", @@ -2161,7 +4096,7 @@ dependencies = [ "serde", "serde_yaml", "sfv", - "socket2", + "socket2 0.6.1", "strum", "strum_macros", "tokio", @@ -2184,7 +4119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252a16def05c7adbbdda776e87b2be36e9481c8a77249207a2f3b563e8933b35" dependencies = [ "bytes", - "http", + "http 1.4.0", "httparse", "pingora-error", "pingora-http", @@ -2200,7 +4135,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3542fd0fd0a83212882c5066ae739ba51804f20d624ff7e12ec85113c5c89a" dependencies = [ "bytes", - "http", + "http 1.4.0", "pingora-error", ] @@ -2224,7 +4159,7 @@ dependencies = [ "derivative", "fnv", "futures", - "http", + "http 1.4.0", "log", "pingora-core", "pingora-error", @@ -2242,7 +4177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba89e4400cb978f0d7be1c14bd7ab4168c8e2c00d97ff19f964fc0048780237c" dependencies = [ "arrayvec", - "hashbrown 0.15.5", + "hashbrown 0.16.1", "parking_lot", "rand 0.8.5", ] @@ -2285,8 +4220,8 @@ dependencies = [ "bytes", "clap 3.2.25", "futures", - "h2", - "http", + "h2 0.4.12", + "http 1.4.0", "log", "once_cell", "pingora-cache", @@ -2360,7 +4295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -2411,12 +4346,45 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "protobuf" version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -2429,8 +4397,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", - "socket2", + "rustls 0.23.35", + "socket2 0.6.1", "thiserror 2.0.17", "tokio", "tracing", @@ -2449,7 +4417,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls", + "rustls 0.23.35", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -2467,16 +4435,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.1", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" dependencies = [ "proc-macro2", ] @@ -2586,14 +4554,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -2603,20 +4571,26 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" + [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" @@ -2627,31 +4601,36 @@ dependencies = [ "base64", "bytes", "futures-core", - "http", - "http-body", + "futures-util", + "h2 0.4.12", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.35", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots", ] @@ -2689,6 +4668,16 @@ dependencies = [ "serde", ] +[[package]] +name = "roaring" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "rust_decimal" version = "1.39.0" @@ -2729,6 +4718,31 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.1", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.35" @@ -2740,7 +4754,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.8", "subtle", "zeroize", ] @@ -2787,10 +4801,10 @@ dependencies = [ "jni", "log", "once_cell", - "rustls", + "rustls 0.23.35", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki", + "rustls-webpki 0.103.8", "security-framework", "security-framework-sys", "webpki-root-certs", @@ -2803,6 +4817,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.8" @@ -2875,6 +4899,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sd-notify" version = "0.4.2" @@ -2910,6 +4944,12 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -2937,7 +4977,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -2961,7 +5001,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -2995,7 +5035,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.13.0", "schemars 0.9.0", "schemars 1.1.0", "serde_core", @@ -3013,7 +5053,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3035,10 +5075,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fa1f336066b758b7c9df34ed049c0e693a426afe2b27ff7d5b14f410ab1a132" dependencies = [ "base64", - "indexmap 2.5.0", + "indexmap 2.13.0", "rust_decimal", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -3070,6 +5121,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.11" @@ -3082,6 +5139,44 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.1" @@ -3092,12 +5187,48 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "sqlparser" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +dependencies = [ + "log", +] + +[[package]] +name = "sqlparser_derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -3129,7 +5260,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3151,9 +5282,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -3177,7 +5308,20 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.1", ] [[package]] @@ -3221,7 +5365,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3232,7 +5376,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3244,6 +5388,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + [[package]] name = "thrift_codec" version = "0.3.2" @@ -3285,6 +5440,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -3322,7 +5486,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.1", "tokio-macros", "windows-sys 0.61.1", ] @@ -3335,7 +5499,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3349,13 +5513,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.35", "tokio", ] @@ -3423,7 +5597,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime", @@ -3454,8 +5628,8 @@ dependencies = [ "bitflags 2.10.0", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tower", @@ -3481,6 +5655,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3494,7 +5669,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3531,6 +5706,22 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.19.0" @@ -3545,9 +5736,21 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "untrusted" @@ -3585,6 +5788,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde_core", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -3597,6 +5812,23 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -3654,7 +5886,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", "wasm-bindgen-shared", ] @@ -3689,7 +5921,7 @@ checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3703,6 +5935,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.81" @@ -3793,7 +6038,7 @@ checksum = "edb307e42a74fb6de9bf3a02d9712678b22399c87e6fa869d6dfcd8c1b7754e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -3804,7 +6049,7 @@ checksum = "c0abd1ddbc6964ac14db11c7213d6532ef34bd9aa042c2e5935f59d7908b46a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -4101,6 +6346,21 @@ dependencies = [ "time", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -4138,10 +6398,16 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", "synstructure", ] +[[package]] +name = "z85" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3a41ce106832b4da1c065baa4c31cf640cf965fa1483816402b7f6b96f0a64" + [[package]] name = "zerocopy" version = "0.8.27" @@ -4159,7 +6425,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -4179,7 +6445,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", "synstructure", ] @@ -4219,7 +6485,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.114", ] [[package]] @@ -4233,18 +6499,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 1e74df6..f06d1ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,13 @@ license = "MIT" [dependencies] rustls = { version = "0.23", features = ["ring"] } +arrow = "52.2.0" base64 = "0.22.1" bollard = "0.19.4" -chrono = "0.4.42" +chrono = "=0.4.39" clap = { version = "4.5.18", features = ["derive"] } cron = { version = "0.15.0", features = ["serde"] } +deltalake = { version = "0.20.1", features = ["datafusion", "s3", "gcs", "azure"] } env_logger = "0.11.5" futures = "0.3.31" futures-util = "0.3.31" @@ -28,6 +30,7 @@ thiserror = "2.0.17" tokio = { version = "1.48.0", features = ["full"] } toml = "0.8.19" urlencoding = "2.1.3" +uuid = { version = "1.10.0", features = ["v7", "serde"] } async-trait = "0.1.89" diff --git a/README.md b/README.md index 6749d18..a5a4cd9 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,24 @@ # Dispenser -Dispenser is a simple, declarative, and deterministic container orchestrator designed for single virtual machines. It combines continuous deployment (CD), a built-in reverse proxy with automatic SSL, and cron scheduling into a single binary, eliminating the need for complex external tooling or manual bash scripts. +Dispenser is a simple, declarative, and deterministic container orchestrator designed for single virtual machines. It combines continuous deployment (CD), a built-in reverse proxy with automatic SSL, cron scheduling, and **native telemetry** into a single binary, eliminating the need for complex external tooling or manual bash scripts. This tool manages containerized applications by continuously monitoring your artifact registry for new versions of Docker images. When updates are detected, dispenser automatically redeploys your services, ensuring the running containers on the host machine match the latest versions in your registry. dispenser operates as a daemon that runs in the background on the host server that watches your artifact registry, detecting when new versions of your container images are published. +## Key Features + +- **Continuous Deployment**: Automatically watches your registry and updates containers when new images are published. +- **Built-in Reverse Proxy**: Powered by [Pingora](https://github.com/cloudflare/pingora), handling SSL/TLS termination and zero-downtime reloads. +- **Observability & Telemetry**: Native integration with **Delta Lake** to stream deployment events and container health metrics directly to S3, GCS, Azure, or local storage. +- **Cron Scheduling**: Run containers on a schedule using standard cron syntax, perfect for backups and batch jobs. +- **Declarative Configuration**: Define your entire infrastructure in simple TOML files. + ## Documentation - **[CLI Reference](CLI.md)** - Complete command-line options and usage - **[Service Configuration](SERVICE_CONFIG.md)** - Detailed `service.toml` reference +- **[Telemetry Configuration](TELEMETRY.md)** - Delta Lake integration - **[Reverse Proxy](PROXY.md)** - Built-in proxy and SSL management - **[Network Configuration](NETWORKS.md)** - Docker network setup guide - **[Cron Scheduling](CRON.md)** - Scheduled deployments diff --git a/TELEMETRY.md b/TELEMETRY.md new file mode 100644 index 0000000..b719a63 --- /dev/null +++ b/TELEMETRY.md @@ -0,0 +1,147 @@ +# Telemetry Configuration + +Dispenser includes a built-in, high-performance telemetry system powered by [Delta Lake](https://delta.io/). It allows you to automatically collect deployment events and container health status, writing them directly to data lakes (S3, GCS, Azure) or local filesystems in Parquet format. + +## Overview + +The telemetry system runs in a dedicated, isolated thread to ensure that heavy I/O operations never block the main orchestration loop. It provides: + +1. **Deployment Tracking**: Every time a container is created, updated, or restarted, a detailed event is logged. +2. **Health Monitoring**: Periodically samples the status of all managed containers (CPU, memory, uptime, health checks). +3. **Delta Lake Integration**: Writes data using the Delta Lake protocol, enabling ACID transactions, scalable metadata handling, and direct compatibility with tools like Spark, Trino, Athena, and Databricks. + +## Configuration + +Telemetry is disabled by default. To enable it, add a `[telemetry]` section to your global `dispenser.toml` configuration file. + +```toml +# dispenser.toml + +[telemetry] +enabled = true + +# URIs for the Delta tables. +# Supported schemes: file://, s3://, gs://, az://, adls:// +table_uri_deployments = "s3://my-data-lake/dispenser/deployments" +table_uri_status = "s3://my-data-lake/dispenser/status" + +# Optional: How often to sample container status (default: 60 seconds) +status_interval = 60 + +# Optional: Number of events to buffer in memory before flushing to storage (default: 1000) +# Lower values write more frequently (lower latency) but create more small files. +buffer_size = 1000 +``` + +### Supported Storage Backends + +Dispenser uses the `deltalake` Rust crate, which supports multiple storage backends natively. The backend is determined by the URI scheme. + +#### 1. Local Filesystem (`file://`) +Writes to a local directory. Useful for development or single-node setups where an external agent (like Fluentbit) ships the logs. + +```toml +table_uri_deployments = "file:///var/log/dispenser/deployments" +``` + +#### 2. AWS S3 (`s3://`) +Writes directly to Amazon S3. + +**Authentication:** +Dispenser automatically looks for credentials in the environment: +* Environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`. +* IAM Instance Profiles (if running on EC2). +* IAM Roles for Service Accounts (IRSA) (if running on EKS). + +#### 3. Google Cloud Storage (`gs://`) +Writes directly to Google Cloud Storage. + +**Authentication:** +* Environment variable: `GOOGLE_APPLICATION_CREDENTIALS` pointing to your Service Account JSON key file. +* Workload Identity (if running on GKE/GCE). + +#### 4. Azure Blob Storage / Data Lake Gen2 (`az://` or `adls://`) +Writes directly to Azure Storage. + +**Authentication:** +Dispenser supports several authentication methods via environment variables: +* Storage Account Key: `AZURE_STORAGE_ACCOUNT`, `AZURE_STORAGE_KEY`. +* Service Principal: `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`. +* Managed Identity (if running on Azure VMs/AKS). + +## Data Schemas + +Dispenser automatically manages two Delta tables. It will create them if they do not exist. + +### Deployments Table (`dispenser-deployments`) + +Records every lifecycle event that results in a container change (creation, recreation, update). + +| Column | Type | Description | +| :--- | :--- | :--- | +| `date` | `DATE` | Partition column. Derived from timestamp. | +| `timestamp` | `TIMESTAMP (UTC)` | Exact time of the event. | +| `service` | `STRING` | Service name defined in `service.toml`. | +| `image` | `STRING` | Image name and tag (e.g., `nginx:latest`). | +| `image_sha` | `STRING` | SHA256 digest of the image. | +| `image_size_mb` | `LONG` | Size of the image in MB. | +| `container_id` | `STRING` | Docker container ID. | +| `container_created_at` | `TIMESTAMP (UTC)` | Creation time reported by Docker. | +| `trigger_type` | `STRING` | Cause of deployment (`startup`, `cron`, `image_update`, `manual_reload`). | +| `dispenser_version` | `STRING` | Version of the Dispenser binary. | +| `restart_policy` | `STRING` | Configured restart policy. | +| `memory_limit` | `STRING` | Configured memory limit. | +| `cpu_limit` | `STRING` | Configured CPU limit. | +| `proxy_enabled` | `BOOLEAN` | Whether the service uses the reverse proxy. | +| `proxy_host` | `STRING` | Hostname configured for the proxy. | +| `port_mappings_count` | `INTEGER` | Number of exposed ports. | +| `volume_count` | `INTEGER` | Number of mounted volumes. | +| `network_count` | `INTEGER` | Number of connected networks. | + +### Container Status Table (`dispenser-container-status`) + +Records periodic snapshots of the runtime state of containers. + +| Column | Type | Description | +| :--- | :--- | :--- | +| `date` | `DATE` | Partition column. Derived from timestamp. | +| `timestamp` | `TIMESTAMP (UTC)` | Exact time of the sample. | +| `service` | `STRING` | Service name. | +| `container_id` | `STRING` | Short container ID. | +| `state` | `STRING` | Execution state (e.g., `running`, `exited`, `dead`). | +| `health_status` | `STRING` | Docker healthcheck status (`healthy`, `unhealthy`, `starting`, `none`). | +| `exit_code` | `INTEGER` | Exit code (only relevant if state is `exited`). | +| `restart_count` | `INTEGER` | Number of times Docker has restarted this container. | +| `uptime_seconds` | `LONG` | Seconds since the container started. | +| `failing_streak` | `INTEGER` | Consecutive healthcheck failures. | +| `last_health_output` | `STRING` | Output of the last failed healthcheck (truncated). | + +## Performance Tuning + +### Buffering & Latency + +The `buffer_size` setting controls the trade-off between latency and file fragmentation. + +* **Small Buffer (e.g., 1-10)**: Events appear in the data lake almost instantly. However, this generates many small Parquet files ("small file problem"), which can degrade query performance and increase storage costs. +* **Large Buffer (e.g., 1000-5000)**: Events are batched into larger files. This is much more efficient for query engines (Presto/Trino, Spark) but introduces a delay before data is visible. + +Dispenser also enforces a time-based flush every **5 minutes** to ensure data is not held in memory indefinitely during periods of low activity. + +### Resource Isolation + +The telemetry service runs on a dedicated Tokio runtime spawned in a separate OS thread. This design ensures that network latency when talking to S3/GCS or CPU-intensive compression of Parquet files does not impact the responsiveness of the main Dispenser loop or the reverse proxy. + +## Data Management + +### Partitioning & Optimization + +* **Partitioning**: Data is automatically partitioned by `date`. Query engines should always filter by date (e.g., `WHERE date = CURRENT_DATE`) for optimal performance. +* **Target File Size**: Dispenser aims for **32MB** Parquet files to balance ingestion latency with query efficiency. +* **Data Skipping**: Key columns like `service` and `container_id` are indexed in Delta Lake stats to allow engines to skip irrelevant files. + +### Retention Policies + +To prevent indefinite storage growth, Dispenser applies the following default retention policies during table creation: + +* **Log Retention**: 30 days (Deployments), 7 days (Status). Delta log history is kept for time-travel queries. +* **Deleted Files**: 7 days (Deployments), 1 day (Status). Vacuum operations can reclaim space after this period. \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index bb8df33..d697a6b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ mod proxy; mod secrets; mod service; mod signals; +mod telemetry; #[tokio::main] async fn main() -> ExitCode { @@ -79,13 +80,43 @@ async fn main() -> ExitCode { } }; - let manager = match ServicesManager::from_config(service_manager_config, None).await { - Ok(m) => Arc::new(m), - Err(e) => { - log::error!("Failed to create services manager: {e}"); - return ExitCode::FAILURE; - } - }; + let telemetry_client = + if let Some(telemetry_config) = &service_manager_config.entrypoint_file.telemetry { + if telemetry_config.enabled { + let (tx, rx) = tokio::sync::mpsc::channel(10000); + let config = telemetry_config.clone(); + + std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("Failed to build telemetry runtime"); + + rt.block_on(async { + let service = crate::telemetry::TelemetryService::new(config, rx); + service.run().await; + }); + }); + + Some(crate::telemetry::TelemetryClient::new(tx)) + } else { + None + } + } else { + None + }; + + let manager = + match ServicesManager::from_config(service_manager_config, None, telemetry_client.clone()) + .await + { + Ok(m) => Arc::new(m), + Err(e) => { + log::error!("Failed to create services manager: {e}"); + return ExitCode::FAILURE; + } + }; if let Err(e) = manager.validate_containers_not_present().await { log::error!("{e}"); @@ -162,7 +193,10 @@ async fn main() -> ExitCode { t.abort(); } - if let Err(e) = signals::reload_manager(manager_holder.clone(), service_filter).await { + if let Err(e) = + signals::reload_manager(manager_holder.clone(), service_filter, telemetry_client.clone()) + .await + { log::error!("Reload failed: {e}"); } diff --git a/src/service/file.rs b/src/service/file.rs index cd0acfb..8a8c55b 100644 --- a/src/service/file.rs +++ b/src/service/file.rs @@ -20,6 +20,7 @@ pub struct EntrypointFile { #[serde(default = "default_delay")] pub delay: u64, pub certbot: Option, + pub telemetry: Option, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] @@ -60,6 +61,20 @@ pub struct CertbotSettings { pub email: String, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct TelemetryConfig { + pub enabled: bool, + pub table_uri_deployments: String, + pub table_uri_status: String, + pub buffer_size: Option, + #[serde(default = "default_status_interval")] + pub status_interval: u64, +} + +fn default_status_interval() -> u64 { + 60 +} + fn default_delay() -> u64 { 60 } diff --git a/src/service/instance.rs b/src/service/instance.rs index a9aefff..870de0c 100644 --- a/src/service/instance.rs +++ b/src/service/instance.rs @@ -25,6 +25,7 @@ use crate::service::{ manifest::{ImageWatcher, ImageWatcherStatus}, network::DEFAULT_NETWORK_NAME, }; +use crate::telemetry::types::{ContainerState, HealthStatus as TelemetryHealthStatus, TriggerType}; #[derive(Debug, PartialEq, Eq)] pub struct ServiceInstanceConfig { @@ -47,6 +48,7 @@ pub struct ServiceInstance { pub config: Arc, pub cron_watcher: Option, pub image_watcher: Option, + pub telemetry: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -99,7 +101,7 @@ async fn get_container_status(container_name: &str) -> Result Result<(), ServiceConfigError> { + pub async fn run_container(&self, trigger_type: TriggerType) -> Result<(), ServiceConfigError> { let mut depends_on_conditions = Vec::with_capacity(self.config.depends_on.len()); loop { for (container, condition) in &self.config.depends_on { @@ -140,7 +142,7 @@ impl ServiceInstance { if self.config.dispenser.pull == PullOptions::Always || self.container_does_not_exist().await { - self.recreate_container().await?; + self.recreate_container(trigger_type).await?; } let docker = get_docker(); @@ -290,7 +292,10 @@ impl ServiceInstance { } } - pub async fn create_container(&self) -> Result<(), ServiceConfigError> { + pub async fn create_container( + &self, + trigger_type: TriggerType, + ) -> Result<(), ServiceConfigError> { log::info!("Creating container: {}", self.config.service.name); let docker = get_docker(); @@ -423,7 +428,42 @@ impl ServiceInstance { .name(&self.config.service.name) .build(); - docker.create_container(Some(options), config).await?; + let container = docker.create_container(Some(options), config).await?; + + if let Some(telemetry) = &self.telemetry { + let container_id = container.id.clone(); + let options = InspectContainerOptionsBuilder::new().size(false).build(); + if let Ok(info) = docker.inspect_container(&container_id, Some(options)).await { + let mut image_sha = info.image.clone().unwrap_or_default(); + let mut image_size = 0; + + if let Ok(img_info) = docker.inspect_image(&image_sha).await { + image_size = img_info.size.unwrap_or(0); + if let Some(digests) = img_info.repo_digests { + if let Some(d) = digests.first() { + image_sha = d.clone(); + } + } + } + + let created_at = info + .created + .as_ref() + .and_then(|c| chrono::DateTime::parse_from_rfc3339(c).ok()) + .map(|dt| dt.timestamp_micros()) + .unwrap_or_else(|| chrono::Utc::now().timestamp_micros()); + + telemetry.track_deployment( + self, + container_id, + image_sha, + image_size / 1_000_000, + trigger_type, + env!("CARGO_PKG_VERSION").to_string(), + created_at, + ); + } + } // Connect to user-defined networks (default dispenser network is already connected) for network in &self.config.network { @@ -452,11 +492,14 @@ impl ServiceInstance { Ok(()) } - pub async fn recreate_container(&self) -> Result<(), ServiceConfigError> { + pub async fn recreate_container( + &self, + trigger_type: TriggerType, + ) -> Result<(), ServiceConfigError> { self.pull_image().await?; let _ = self.stop_container().await; let _ = self.remove_container().await; - self.create_container().await?; + self.create_container(trigger_type).await?; Ok(()) } @@ -503,7 +546,7 @@ impl ServiceInstance { pub async fn recreate_if_required(&self, other: &Self) { if self.requires_recreate(other).await { - if let Err(e) = self.recreate_container().await { + if let Err(e) = self.recreate_container(TriggerType::ManualReload).await { log::error!( "Failed to recreate container {}: {}", self.config.service.name, @@ -513,10 +556,90 @@ impl ServiceInstance { } } - pub async fn poll(&self, poll_images: bool, init: bool) { + async fn check_and_report_status(&self) { + if let Some(telemetry) = &self.telemetry { + let docker = get_docker(); + let options = InspectContainerOptionsBuilder::new().size(false).build(); + + match docker + .inspect_container(&self.config.service.name, Some(options)) + .await + { + Ok(info) => { + let state = info.state.as_ref(); + let container_state = state + .and_then(|s| s.status) + .map(ContainerState::from) + .unwrap_or(ContainerState::Unknown); + let health_status = state + .and_then(|s| s.health.as_ref()) + .and_then(|h| h.status) + .map(TelemetryHealthStatus::from) + .unwrap_or(TelemetryHealthStatus::None); + let exit_code = state.and_then(|s| s.exit_code).map(|c| c as i32); + let restart_count = info.restart_count.unwrap_or(0) as i32; + let uptime_seconds = if let Some(started_at) = + state.and_then(|s| s.started_at.as_ref()) + { + match chrono::DateTime::parse_from_rfc3339(started_at) { + Ok(dt) => chrono::Utc::now().signed_duration_since(dt).num_seconds(), + Err(_) => 0, + } + } else { + 0 + }; + let failing_streak = state + .and_then(|s| s.health.as_ref()) + .and_then(|h| h.failing_streak) + .unwrap_or(0) as i32; + let last_health_output = state + .and_then(|s| s.health.as_ref()) + .and_then(|h| h.log.as_ref()) + .and_then(|logs| logs.last()) + .and_then(|l| l.output.clone()); + + telemetry.track_status( + self.config.service.name.clone(), + info.id.unwrap_or_default(), + container_state, + health_status, + exit_code, + restart_count, + uptime_seconds, + failing_streak, + last_health_output, + ); + } + Err(bollard::errors::Error::DockerResponseServerError { + status_code: 404, .. + }) => { + telemetry.track_status( + self.config.service.name.clone(), + String::new(), + ContainerState::NotFound, + TelemetryHealthStatus::None, + None, + 0, + 0, + 0, + None, + ); + } + Err(e) => { + log::warn!( + "Failed to inspect container {} for telemetry: {}", + self.config.service.name, + e + ); + } + } + } + } + + pub async fn poll(&self, poll_images: bool, poll_status: bool, init: bool) { if init && self.config.dispenser.initialize == Initialize::Immediately { log::info!("Starting {} immediately", self.config.service.name); - if let Err(e) = self.run_container().await { + if let Err(e) = self.run_container(TriggerType::Startup).await { log::error!( "Failed to run container {}: {}", self.config.service.name, @@ -530,7 +653,7 @@ impl ServiceInstance { if let Some(cron_watcher) = &self.cron_watcher { if cron_watcher.is_ready() { // If the cron matches we can short circuit the function - if let Err(e) = self.run_container().await { + if let Err(e) = self.run_container(TriggerType::Cron).await { log::error!( "Failed to run container {} from cron: {}", self.config.service.name, @@ -553,14 +676,14 @@ impl ServiceInstance { "Image updated for service {}, recreating container...", self.config.service.name ); - if let Err(e) = self.recreate_container().await { + if let Err(e) = self.recreate_container(TriggerType::ImageUpdate).await { log::error!( "Failed to recreate container {}: {}", self.config.service.name, e ); } - if let Err(e) = self.run_container().await { + if let Err(e) = self.run_container(TriggerType::ImageUpdate).await { log::error!( "Failed to run container {}: {}", self.config.service.name, @@ -575,6 +698,10 @@ impl ServiceInstance { } } } + + if poll_status { + self.check_and_report_status().await; + } } } diff --git a/src/service/manager.rs b/src/service/manager.rs index 86c2c85..1808a25 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -18,8 +18,8 @@ use crate::service::{ }; pub struct ServiceMangerConfig { - entrypoint_file: EntrypointFile, - services: Vec<(PathBuf, ServiceFile)>, + pub entrypoint_file: EntrypointFile, + pub services: Vec<(PathBuf, ServiceFile)>, } impl ServiceMangerConfig { @@ -77,6 +77,7 @@ struct ServiceManagerInner { delay: Duration, certbot: Option, proxy: GlobalProxyConfig, + telemetry_status_interval: Duration, } pub struct ServicesManager { @@ -235,9 +236,17 @@ impl ServicesManager { pub async fn from_config( mut config: ServiceMangerConfig, existing_ips: Option>, + telemetry: Option, ) -> Result { // Get the delay from config (in seconds) let delay = Duration::from_secs(config.entrypoint_file.delay); + let telemetry_status_interval = config + .entrypoint_file + .telemetry + .as_ref() + .map(|t| Duration::from_secs(t.status_interval)) + .unwrap_or(Duration::from_secs(60)); + let mut instances = Vec::new(); let mut networks = Vec::new(); let mut service_names = Vec::new(); @@ -298,6 +307,7 @@ impl ServicesManager { .get(&service_file.service.name) .copied() .expect("IP should have been allocated for all services"); + let telemetry = telemetry.clone(); join_set.spawn(async move { log::debug!("Initializing config for {entry_path:?}"); @@ -331,6 +341,7 @@ impl ServicesManager { config: Arc::new(config), cron_watcher, image_watcher, + telemetry, }; (service_name, Arc::new(instance)) @@ -378,6 +389,7 @@ impl ServicesManager { router, proxy, certbot: config.entrypoint_file.certbot, + telemetry_status_interval, }; Ok(ServicesManager { @@ -394,6 +406,7 @@ impl ServicesManager { pub async fn start_polling(&self) { log::info!("Starting polling task"); let delay = self.inner.delay; + let status_delay = self.inner.telemetry_status_interval; let polls = self .inner @@ -403,15 +416,22 @@ impl ServicesManager { let instance = instance.clone(); async move { let mut last_image_poll = std::time::Instant::now(); + let mut last_status_poll = std::time::Instant::now(); let mut init = true; loop { let poll_images = last_image_poll.elapsed() >= delay; if poll_images { last_image_poll = std::time::Instant::now(); } + + let poll_status = last_status_poll.elapsed() >= status_delay; + if poll_status { + last_status_poll = std::time::Instant::now(); + } + // Scope to release the lock let poll_start = std::time::Instant::now(); - instance.poll(poll_images, init).await; + instance.poll(poll_images, poll_status, init).await; let poll_duration = poll_start.elapsed(); log::debug!( "Polling for {} took {:?}", @@ -784,6 +804,7 @@ mod tests { config: Arc::new(config), cron_watcher: None, image_watcher: None, + telemetry: None, })); } @@ -796,6 +817,7 @@ mod tests { router, proxy: GlobalProxyConfig::default(), certbot: None, + telemetry_status_interval: Duration::from_secs(60), }, cancel_tx: tokio::sync::mpsc::channel(1).0, cancel_rx: Mutex::new(tokio::sync::mpsc::channel(1).1), diff --git a/src/signals.rs b/src/signals.rs index 1a69738..89fbce8 100644 --- a/src/signals.rs +++ b/src/signals.rs @@ -64,6 +64,7 @@ pub fn handle_reload(reload_signal: Arc) { pub async fn reload_manager( manager_holder: Arc>>, service_filter: Option<&[String]>, + telemetry: Option, ) -> Result<(), String> { let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Reloading]); @@ -86,7 +87,9 @@ pub async fn reload_manager( // Create a new manager with the new configuration, passing existing IPs let new_manager = - match ServicesManager::from_config(service_manager_config, Some(existing_ips)).await { + match ServicesManager::from_config(service_manager_config, Some(existing_ips), telemetry) + .await + { Ok(manager) => Arc::new(manager), Err(e) => { log::error!("Failed to create new services manager: {e}"); diff --git a/src/telemetry/buffer.rs b/src/telemetry/buffer.rs new file mode 100644 index 0000000..cab57eb --- /dev/null +++ b/src/telemetry/buffer.rs @@ -0,0 +1,300 @@ +use super::events::{ContainerStatusEvent, DeploymentEvent}; +use super::schema::{deployments_schema, status_schema}; +use arrow::array::{ + BooleanBuilder, Date32Builder, Int32Builder, Int64Builder, StringBuilder, + StringDictionaryBuilder, TimestampMicrosecondBuilder, +}; +use arrow::datatypes::Int8Type; +use arrow::record_batch::RecordBatch; +use std::sync::Arc; + +pub struct DeploymentsBuffer { + date: Date32Builder, + timestamp: TimestampMicrosecondBuilder, + service: StringBuilder, + image: StringBuilder, + image_sha: StringBuilder, + image_size_mb: Int64Builder, + container_id: StringBuilder, + container_created_at: TimestampMicrosecondBuilder, + trigger_type: StringDictionaryBuilder, + dispenser_version: StringBuilder, + restart_policy: StringDictionaryBuilder, + memory_limit: StringBuilder, + cpu_limit: StringBuilder, + proxy_enabled: BooleanBuilder, + proxy_host: StringBuilder, + port_mappings_count: Int32Builder, + volume_count: Int32Builder, + network_count: Int32Builder, + + count: usize, +} + +impl DeploymentsBuffer { + pub fn new(capacity: usize) -> Self { + Self { + date: Date32Builder::with_capacity(capacity), + timestamp: TimestampMicrosecondBuilder::with_capacity(capacity), + service: StringBuilder::with_capacity(capacity, capacity * 20), + image: StringBuilder::with_capacity(capacity, capacity * 50), + image_sha: StringBuilder::with_capacity(capacity, capacity * 64), + image_size_mb: Int64Builder::with_capacity(capacity), + container_id: StringBuilder::with_capacity(capacity, capacity * 64), + container_created_at: TimestampMicrosecondBuilder::with_capacity(capacity), + trigger_type: StringDictionaryBuilder::new(), + dispenser_version: StringBuilder::with_capacity(capacity, capacity * 10), + restart_policy: StringDictionaryBuilder::new(), + memory_limit: StringBuilder::with_capacity(capacity, capacity * 5), + cpu_limit: StringBuilder::with_capacity(capacity, capacity * 5), + proxy_enabled: BooleanBuilder::with_capacity(capacity), + proxy_host: StringBuilder::with_capacity(capacity, capacity * 30), + port_mappings_count: Int32Builder::with_capacity(capacity), + volume_count: Int32Builder::with_capacity(capacity), + network_count: Int32Builder::with_capacity(capacity), + count: 0, + } + } + + pub fn push(&mut self, event: &DeploymentEvent) { + let date_days = (event.timestamp / (86400 * 1_000_000)) as i32; + + self.date.append_value(date_days); + self.timestamp.append_value(event.timestamp); + self.service.append_value(&event.service); + self.image.append_value(&event.image); + self.image_sha.append_value(&event.image_sha); + self.image_size_mb.append_value(event.image_size_mb); + self.container_id.append_value(&event.container_id); + self.container_created_at + .append_value(event.container_created_at); + self.trigger_type.append_value(event.trigger_type.as_ref()); + self.dispenser_version + .append_value(&event.dispenser_version); + + let restart_policy_str = match event.restart_policy { + crate::service::file::Restart::Always => "always", + crate::service::file::Restart::No => "no", + crate::service::file::Restart::OnFailure => "on-failure", + crate::service::file::Restart::UnlessStopped => "unless-stopped", + }; + self.restart_policy.append_value(restart_policy_str); + + if let Some(val) = &event.memory_limit { + self.memory_limit.append_value(val); + } else { + self.memory_limit.append_null(); + } + + if let Some(val) = &event.cpu_limit { + self.cpu_limit.append_value(val); + } else { + self.cpu_limit.append_null(); + } + + self.proxy_enabled.append_value(event.proxy_enabled); + + if let Some(val) = &event.proxy_host { + self.proxy_host.append_value(val); + } else { + self.proxy_host.append_null(); + } + + self.port_mappings_count + .append_value(event.port_mappings_count); + self.volume_count.append_value(event.volume_count); + self.network_count.append_value(event.network_count); + + self.count += 1; + } + + pub fn len(&self) -> usize { + self.count + } + + pub fn is_empty(&self) -> bool { + self.count == 0 + } + + pub fn into_record_batch(mut self) -> arrow::error::Result { + let schema = deployments_schema(); + + RecordBatch::try_new( + schema, + vec![ + Arc::new(self.date.finish()), + Arc::new(self.timestamp.finish().with_timezone("UTC")), + Arc::new(self.service.finish()), + Arc::new(self.image.finish()), + Arc::new(self.image_sha.finish()), + Arc::new(self.image_size_mb.finish()), + Arc::new(self.container_id.finish()), + Arc::new(self.container_created_at.finish().with_timezone("UTC")), + Arc::new(self.trigger_type.finish()), + Arc::new(self.dispenser_version.finish()), + Arc::new(self.restart_policy.finish()), + Arc::new(self.memory_limit.finish()), + Arc::new(self.cpu_limit.finish()), + Arc::new(self.proxy_enabled.finish()), + Arc::new(self.proxy_host.finish()), + Arc::new(self.port_mappings_count.finish()), + Arc::new(self.volume_count.finish()), + Arc::new(self.network_count.finish()), + ], + ) + } +} + +pub struct StatusBuffer { + date: Date32Builder, + timestamp: TimestampMicrosecondBuilder, + service: StringBuilder, + container_id: StringBuilder, + state: StringDictionaryBuilder, + health_status: StringDictionaryBuilder, + exit_code: Int32Builder, + restart_count: Int32Builder, + uptime_seconds: Int64Builder, + failing_streak: Int32Builder, + last_health_output: StringBuilder, + + count: usize, +} + +impl StatusBuffer { + pub fn new(capacity: usize) -> Self { + Self { + date: Date32Builder::with_capacity(capacity), + timestamp: TimestampMicrosecondBuilder::with_capacity(capacity), + service: StringBuilder::with_capacity(capacity, capacity * 20), + container_id: StringBuilder::with_capacity(capacity, capacity * 64), + state: StringDictionaryBuilder::new(), + health_status: StringDictionaryBuilder::new(), + exit_code: Int32Builder::with_capacity(capacity), + restart_count: Int32Builder::with_capacity(capacity), + uptime_seconds: Int64Builder::with_capacity(capacity), + failing_streak: Int32Builder::with_capacity(capacity), + last_health_output: StringBuilder::with_capacity(capacity, capacity * 50), + count: 0, + } + } + + pub fn push(&mut self, event: &ContainerStatusEvent) { + let date_days = (event.timestamp / (86400 * 1_000_000)) as i32; + + self.date.append_value(date_days); + self.timestamp.append_value(event.timestamp); + self.service.append_value(&event.service); + self.container_id.append_value(&event.container_id); + self.state.append_value(event.state.as_ref()); + self.health_status + .append_value(event.health_status.as_ref()); + + if let Some(val) = event.exit_code { + self.exit_code.append_value(val); + } else { + self.exit_code.append_null(); + } + + self.restart_count.append_value(event.restart_count); + self.uptime_seconds.append_value(event.uptime_seconds); + self.failing_streak.append_value(event.failing_streak); + + if let Some(val) = &event.last_health_output { + self.last_health_output.append_value(val); + } else { + self.last_health_output.append_null(); + } + + self.count += 1; + } + + pub fn len(&self) -> usize { + self.count + } + + pub fn is_empty(&self) -> bool { + self.count == 0 + } + + pub fn into_record_batch(mut self) -> arrow::error::Result { + let schema = status_schema(); + + RecordBatch::try_new( + schema, + vec![ + Arc::new(self.date.finish()), + Arc::new(self.timestamp.finish().with_timezone("UTC")), + Arc::new(self.service.finish()), + Arc::new(self.container_id.finish()), + Arc::new(self.state.finish()), + Arc::new(self.health_status.finish()), + Arc::new(self.exit_code.finish()), + Arc::new(self.restart_count.finish()), + Arc::new(self.uptime_seconds.finish()), + Arc::new(self.failing_streak.finish()), + Arc::new(self.last_health_output.finish()), + ], + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::service::file::Restart; + use crate::telemetry::events::DeploymentEvent; + use crate::telemetry::types::TriggerType; + use arrow::datatypes::{DataType, TimeUnit}; + use uuid::Uuid; + + #[test] + fn test_deployments_buffer_schema_compliance() { + let mut buffer = DeploymentsBuffer::new(10); + + let event = DeploymentEvent { + event_id: Uuid::now_v7(), + timestamp: 1700000000000000, // UTC mic + service: "test-svc".to_string(), + image: "nginx".to_string(), + image_sha: "sha256:1234".to_string(), + image_size_mb: 100, + container_id: "cid-1".to_string(), + container_created_at: 1700000000000000, + trigger_type: TriggerType::ManualReload, + dispenser_version: "1.0".to_string(), + restart_policy: Restart::Always, + memory_limit: Some("512m".to_string()), + cpu_limit: None, + proxy_enabled: true, + proxy_host: Some("test.com".to_string()), + port_mappings_count: 2, + volume_count: 1, + network_count: 1, + }; + + buffer.push(&event); + assert!(!buffer.is_empty()); + assert_eq!(buffer.len(), 1); + + let batch = buffer + .into_record_batch() + .expect("Failed to create record batch"); + + // Verify Schema correctness + let schema = batch.schema(); + + // Check Timestamp is UTC (Crucial fix verification) + let ts_field = schema.field_with_name("timestamp").unwrap(); + match ts_field.data_type() { + DataType::Timestamp(TimeUnit::Microsecond, Some(tz)) => { + assert_eq!(tz.as_ref(), "UTC", "Timestamp timezone mismatch"); + } + dt => panic!("Expected Timestamp(Microsecond, UTC), found {:?}", dt), + } + + // Check Nullable fields + let cpu_field = schema.field_with_name("cpu_limit").unwrap(); + assert!(cpu_field.is_nullable()); + } +} diff --git a/src/telemetry/client.rs b/src/telemetry/client.rs new file mode 100644 index 0000000..452d234 --- /dev/null +++ b/src/telemetry/client.rs @@ -0,0 +1,100 @@ +use super::events::{ContainerStatusEvent, DeploymentEvent, DispenserEvent}; +use super::types::{ContainerState, HealthStatus, TriggerType}; +use crate::service::instance::ServiceInstance; +use log::error; +use tokio::sync::mpsc::Sender; +use uuid::Uuid; + +#[derive(Clone, Debug)] +pub struct TelemetryClient { + tx: Sender, +} + +impl TelemetryClient { + pub fn new(tx: Sender) -> Self { + Self { tx } + } + + pub fn track_deployment( + &self, + service: &ServiceInstance, + container_id: String, + image_sha: String, + image_size_mb: i64, + trigger_type: TriggerType, + dispenser_version: String, + container_created_at: i64, + ) { + let now = chrono::Utc::now(); + let timestamp = now.timestamp_micros(); + + let config = &service.config; + let svc_entry = &config.service; + + let event = DeploymentEvent { + event_id: Uuid::now_v7(), + timestamp, + service: svc_entry.name.clone(), + image: svc_entry.image.clone(), + image_sha, + image_size_mb, + container_id, + container_created_at, + trigger_type, + dispenser_version, + restart_policy: svc_entry.restart.clone(), + memory_limit: svc_entry.memory.clone(), + cpu_limit: svc_entry.cpus.clone(), + proxy_enabled: config.proxy.is_some(), + proxy_host: config.proxy.as_ref().map(|p| p.host.clone()), + port_mappings_count: config.ports.len() as i32, + volume_count: config.volume.len() as i32, + network_count: config.network.len() as i32, + }; + + self.send(DispenserEvent::Deployment(Box::new(event))); + } + + pub fn track_status( + &self, + service_name: String, + container_id: String, + state: ContainerState, + health_status: HealthStatus, + exit_code: Option, + restart_count: i32, + uptime_seconds: i64, + failing_streak: i32, + last_health_output: Option, + ) { + let now = chrono::Utc::now(); + let timestamp = now.timestamp_micros(); + + let event = ContainerStatusEvent { + event_id: Uuid::now_v7(), + timestamp, + service: service_name, + container_id, + state, + health_status, + exit_code, + restart_count, + uptime_seconds, + failing_streak, + last_health_output, + }; + + self.send(DispenserEvent::ContainerStatus(Box::new(event))); + } + + fn send(&self, event: DispenserEvent) { + // Use try_send to avoid blocking the main loop. + // If the channel is full, we drop the event and log an error. + if let Err(e) = self.tx.try_send(event) { + error!( + "Failed to send telemetry event (channel full or closed): {:?}", + e + ); + } + } +} diff --git a/src/telemetry/events.rs b/src/telemetry/events.rs new file mode 100644 index 0000000..10176e0 --- /dev/null +++ b/src/telemetry/events.rs @@ -0,0 +1,50 @@ +use super::types::{ContainerState, HealthStatus, TriggerType}; +use crate::service::file::Restart; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeploymentEvent { + pub event_id: Uuid, + /// Timestamp in microseconds (UTC) + pub timestamp: i64, + pub service: String, + pub image: String, + pub image_sha: String, + pub image_size_mb: i64, + pub container_id: String, + /// Timestamp in microseconds (UTC) + pub container_created_at: i64, + pub trigger_type: TriggerType, + pub dispenser_version: String, + pub restart_policy: Restart, + pub memory_limit: Option, + pub cpu_limit: Option, + pub proxy_enabled: bool, + pub proxy_host: Option, + pub port_mappings_count: i32, + pub volume_count: i32, + pub network_count: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContainerStatusEvent { + pub event_id: Uuid, + /// Timestamp in microseconds (UTC) + pub timestamp: i64, + pub service: String, + pub container_id: String, + pub state: ContainerState, + pub health_status: HealthStatus, + pub exit_code: Option, + pub restart_count: i32, + pub uptime_seconds: i64, + pub failing_streak: i32, + pub last_health_output: Option, +} + +#[derive(Debug)] +pub enum DispenserEvent { + Deployment(Box), + ContainerStatus(Box), +} diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs new file mode 100644 index 0000000..135e208 --- /dev/null +++ b/src/telemetry/mod.rs @@ -0,0 +1,9 @@ +pub mod buffer; +pub mod client; +pub mod events; +pub mod schema; +pub mod service; +pub mod types; + +pub use client::TelemetryClient; +pub use service::TelemetryService; diff --git a/src/telemetry/schema.rs b/src/telemetry/schema.rs new file mode 100644 index 0000000..752bc55 --- /dev/null +++ b/src/telemetry/schema.rs @@ -0,0 +1,250 @@ +use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use deltalake::kernel::{DataType as DeltaDataType, PrimitiveType, StructField}; +use deltalake::operations::create::CreateBuilder; +use deltalake::protocol::SaveMode; +use deltalake::{DeltaTable, DeltaTableError, TableProperty}; +use std::sync::Arc; + +pub async fn create_deployments_table(table_uri: &str) -> Result { + let columns = vec![ + StructField::new("date", DeltaDataType::Primitive(PrimitiveType::Date), false), + StructField::new( + "timestamp", + DeltaDataType::Primitive(PrimitiveType::Timestamp), + false, + ), + StructField::new( + "service", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "image", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "image_sha", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "image_size_mb", + DeltaDataType::Primitive(PrimitiveType::Long), + false, + ), + StructField::new( + "container_id", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "container_created_at", + DeltaDataType::Primitive(PrimitiveType::Timestamp), + false, + ), + StructField::new( + "trigger_type", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "dispenser_version", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "restart_policy", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "memory_limit", + DeltaDataType::Primitive(PrimitiveType::String), + true, + ), + StructField::new( + "cpu_limit", + DeltaDataType::Primitive(PrimitiveType::String), + true, + ), + StructField::new( + "proxy_enabled", + DeltaDataType::Primitive(PrimitiveType::Boolean), + false, + ), + StructField::new( + "proxy_host", + DeltaDataType::Primitive(PrimitiveType::String), + true, + ), + StructField::new( + "port_mappings_count", + DeltaDataType::Primitive(PrimitiveType::Integer), + false, + ), + StructField::new( + "volume_count", + DeltaDataType::Primitive(PrimitiveType::Integer), + false, + ), + StructField::new( + "network_count", + DeltaDataType::Primitive(PrimitiveType::Integer), + false, + ), + ]; + + CreateBuilder::new() + .with_location(table_uri) + .with_columns(columns) + .with_partition_columns(vec!["date"]) + .with_save_mode(SaveMode::Ignore) + .with_configuration_property( + TableProperty::LogRetentionDuration, + Some("interval 30 days"), + ) + .with_configuration_property( + TableProperty::DeletedFileRetentionDuration, + Some("interval 7 days"), + ) + // 32MB target file size (good for streaming ingestion) + .with_configuration_property(TableProperty::TargetFileSize, Some("33554432")) + .await +} + +pub async fn create_status_table(table_uri: &str) -> Result { + let columns = vec![ + StructField::new("date", DeltaDataType::Primitive(PrimitiveType::Date), false), + StructField::new( + "timestamp", + DeltaDataType::Primitive(PrimitiveType::Timestamp), + false, + ), + StructField::new( + "service", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "container_id", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "state", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "health_status", + DeltaDataType::Primitive(PrimitiveType::String), + false, + ), + StructField::new( + "exit_code", + DeltaDataType::Primitive(PrimitiveType::Integer), + true, + ), + StructField::new( + "restart_count", + DeltaDataType::Primitive(PrimitiveType::Integer), + false, + ), + StructField::new( + "uptime_seconds", + DeltaDataType::Primitive(PrimitiveType::Long), + false, + ), + StructField::new( + "failing_streak", + DeltaDataType::Primitive(PrimitiveType::Integer), + false, + ), + StructField::new( + "last_health_output", + DeltaDataType::Primitive(PrimitiveType::String), + true, + ), + ]; + + CreateBuilder::new() + .with_location(table_uri) + .with_columns(columns) + .with_partition_columns(vec!["date"]) + .with_save_mode(SaveMode::Ignore) + .with_configuration_property(TableProperty::LogRetentionDuration, Some("interval 7 days")) + .with_configuration_property( + TableProperty::DeletedFileRetentionDuration, + Some("interval 1 days"), + ) + .with_configuration_property(TableProperty::TargetFileSize, Some("33554432")) + .await +} + +pub fn deployments_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("date", DataType::Date32, false), + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Field::new("service", DataType::Utf8, false), + Field::new("image", DataType::Utf8, false), + Field::new("image_sha", DataType::Utf8, false), + Field::new("image_size_mb", DataType::Int64, false), + Field::new("container_id", DataType::Utf8, false), + Field::new( + "container_created_at", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Field::new( + "trigger_type", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + false, + ), + Field::new("dispenser_version", DataType::Utf8, false), + Field::new( + "restart_policy", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + false, + ), + Field::new("memory_limit", DataType::Utf8, true), + Field::new("cpu_limit", DataType::Utf8, true), + Field::new("proxy_enabled", DataType::Boolean, false), + Field::new("proxy_host", DataType::Utf8, true), + Field::new("port_mappings_count", DataType::Int32, false), + Field::new("volume_count", DataType::Int32, false), + Field::new("network_count", DataType::Int32, false), + ])) +} + +pub fn status_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("date", DataType::Date32, false), + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Field::new("service", DataType::Utf8, false), + Field::new("container_id", DataType::Utf8, false), + Field::new( + "state", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + false, + ), + Field::new( + "health_status", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + false, + ), + Field::new("exit_code", DataType::Int32, true), + Field::new("restart_count", DataType::Int32, false), + Field::new("uptime_seconds", DataType::Int64, false), + Field::new("failing_streak", DataType::Int32, false), + Field::new("last_health_output", DataType::Utf8, true), + ])) +} diff --git a/src/telemetry/service.rs b/src/telemetry/service.rs new file mode 100644 index 0000000..678937c --- /dev/null +++ b/src/telemetry/service.rs @@ -0,0 +1,160 @@ +use super::buffer::{DeploymentsBuffer, StatusBuffer}; +use super::events::DispenserEvent; +use super::schema::{create_deployments_table, create_status_table}; +use crate::service::file::TelemetryConfig; +use deltalake::{DeltaOps, DeltaTableError}; +use log::{error, info, warn}; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc::Receiver; + +const DEFAULT_BUFFER_SIZE: usize = 1000; +const FLUSH_INTERVAL: Duration = Duration::from_secs(300); // 5 minutes + +pub struct TelemetryService { + config: TelemetryConfig, + rx: Receiver, + deployments_buffer: DeploymentsBuffer, + status_buffer: StatusBuffer, + buffer_limit: usize, +} + +impl TelemetryService { + pub fn new(config: TelemetryConfig, rx: Receiver) -> Self { + let buffer_limit = config.buffer_size.unwrap_or(DEFAULT_BUFFER_SIZE); + Self { + config, + rx, + deployments_buffer: DeploymentsBuffer::new(buffer_limit), + status_buffer: StatusBuffer::new(buffer_limit), + buffer_limit, + } + } + + pub async fn run(mut self) { + info!("Telemetry service started"); + // Start with a tick so we don't wait 5 mins for the first check if needed, + // but actually we only want to flush if time passes. + // interval.tick() completes immediately the first time. + let mut flush_interval = tokio::time::interval(FLUSH_INTERVAL); + // Consume the first immediate tick + flush_interval.tick().await; + + loop { + tokio::select! { + maybe_event = self.rx.recv() => { + match maybe_event { + Some(event) => { + self.handle_event(event); + if self.should_flush() { + self.flush().await; + } + } + None => { + info!("Telemetry channel closed, flushing remaining events"); + self.flush().await; + break; + } + } + } + _ = flush_interval.tick() => { + if !self.deployments_buffer.is_empty() || !self.status_buffer.is_empty() { + self.flush().await; + } + } + } + } + info!("Telemetry service stopped"); + } + + fn handle_event(&mut self, event: DispenserEvent) { + match event { + DispenserEvent::Deployment(e) => self.deployments_buffer.push(&e), + DispenserEvent::ContainerStatus(e) => self.status_buffer.push(&e), + } + } + + fn should_flush(&self) -> bool { + self.deployments_buffer.len() >= self.buffer_limit + || self.status_buffer.len() >= self.buffer_limit + } + + async fn flush(&mut self) { + let start = Instant::now(); + + // Flush Deployments + if !self.deployments_buffer.is_empty() { + let count = self.deployments_buffer.len(); + let old_buffer = std::mem::replace( + &mut self.deployments_buffer, + DeploymentsBuffer::new(self.buffer_limit), + ); + + match old_buffer.into_record_batch() { + Ok(batch) => { + if let Err(e) = self + .write_to_delta(&self.config.table_uri_deployments, batch, true) + .await + { + error!("Failed to write deployment events to Delta Lake: {:?}", e); + } else { + info!("Flushed {} deployment events to Delta Lake", count); + } + } + Err(e) => error!("Failed to create record batch for deployments: {:?}", e), + } + } + + // Flush Status + if !self.status_buffer.is_empty() { + let count = self.status_buffer.len(); + let old_buffer = std::mem::replace( + &mut self.status_buffer, + StatusBuffer::new(self.buffer_limit), + ); + + match old_buffer.into_record_batch() { + Ok(batch) => { + if let Err(e) = self + .write_to_delta(&self.config.table_uri_status, batch, false) + .await + { + error!("Failed to write status events to Delta Lake: {:?}", e); + } else { + info!("Flushed {} status events to Delta Lake", count); + } + } + Err(e) => error!("Failed to create record batch for status: {:?}", e), + } + } + + let duration = start.elapsed(); + if duration.as_secs() > 1 { + warn!("Telemetry flush took {:?}", duration); + } + } + + async fn write_to_delta( + &self, + table_uri: &str, + batch: arrow::record_batch::RecordBatch, + is_deployments: bool, + ) -> Result<(), DeltaTableError> { + let table = match deltalake::open_table(table_uri).await { + Ok(table) => table, + Err(DeltaTableError::NotATable(_)) => { + if is_deployments { + create_deployments_table(table_uri).await? + } else { + create_status_table(table_uri).await? + } + } + Err(e) => return Err(e), + }; + + let ops = DeltaOps(table); + ops.write(vec![batch]) + .with_save_mode(deltalake::protocol::SaveMode::Append) + .await?; + Ok(()) + } +} diff --git a/src/telemetry/types.rs b/src/telemetry/types.rs new file mode 100644 index 0000000..0bfff86 --- /dev/null +++ b/src/telemetry/types.rs @@ -0,0 +1,98 @@ +use bollard::models::{ContainerStateStatusEnum, HealthStatusEnum}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum TriggerType { + Startup, + Cron, + ImageUpdate, + ManualReload, +} + +impl AsRef for TriggerType { + fn as_ref(&self) -> &str { + match self { + Self::Startup => "startup", + Self::Cron => "cron", + Self::ImageUpdate => "image_update", + Self::ManualReload => "manual_reload", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ContainerState { + Empty, + Created, + Running, + Paused, + Restarting, + Removing, + Exited, + Dead, + // Custom states + NotFound, + Unknown, +} + +impl AsRef for ContainerState { + fn as_ref(&self) -> &str { + match self { + Self::Empty => "empty", + Self::Created => "created", + Self::Running => "running", + Self::Paused => "paused", + Self::Restarting => "restarting", + Self::Removing => "removing", + Self::Exited => "exited", + Self::Dead => "dead", + Self::NotFound => "not_found", + Self::Unknown => "unknown", + } + } +} + +impl From for ContainerState { + fn from(status: ContainerStateStatusEnum) -> Self { + match status { + ContainerStateStatusEnum::EMPTY => Self::Empty, + ContainerStateStatusEnum::CREATED => Self::Created, + ContainerStateStatusEnum::RUNNING => Self::Running, + ContainerStateStatusEnum::PAUSED => Self::Paused, + ContainerStateStatusEnum::RESTARTING => Self::Restarting, + ContainerStateStatusEnum::REMOVING => Self::Removing, + ContainerStateStatusEnum::EXITED => Self::Exited, + ContainerStateStatusEnum::DEAD => Self::Dead, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum HealthStatus { + Healthy, + Unhealthy, + Starting, + None, +} + +impl AsRef for HealthStatus { + fn as_ref(&self) -> &str { + match self { + Self::Healthy => "healthy", + Self::Unhealthy => "unhealthy", + Self::Starting => "starting", + Self::None => "none", + } + } +} + +impl From for HealthStatus { + fn from(status: HealthStatusEnum) -> Self { + match status { + HealthStatusEnum::HEALTHY => Self::Healthy, + HealthStatusEnum::UNHEALTHY => Self::Unhealthy, + HealthStatusEnum::STARTING => Self::Starting, + HealthStatusEnum::EMPTY | HealthStatusEnum::NONE => Self::None, + } + } +}