diff --git a/.secrets.baseline b/.secrets.baseline index eb560c0280..cb5e20ce55 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline|package-lock.json|Cargo.lock|scripts/sign_image.sh|scripts/zap|sonar-project.properties|uv.lock|go.sum|mcpgateway/sri_hashes.json|^.secrets.baseline$", "lines": null }, - "generated_at": "2026-04-16T07:17:13Z", + "generated_at": "2026-04-16T14:05:33Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -416,7 +416,7 @@ "hashed_secret": "93ac8946882128457cd9e283b30ca851945e6690", "is_secret": false, "is_verified": false, - "line_number": 7877, + "line_number": 7883, "type": "Secret Keyword", "verified_result": null } diff --git a/Cargo.lock b/Cargo.lock index 98578b1d13..8c44c76013 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,6 +13,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -57,6 +63,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -137,6 +149,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -147,6 +165,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -166,7 +196,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -177,7 +207,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -302,6 +332,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "btoi" version = "0.5.0" @@ -329,12 +368,27 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.60" @@ -458,7 +512,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -505,6 +559,37 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "compression-codecs" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "const-oid" version = "0.9.6" @@ -540,6 +625,43 @@ dependencies = [ "wiremock", ] +[[package]] +name = "contextforge_benchmark_console" +version = "1.0.0-BETA-3" +dependencies = [ + "crossterm", + "ratatui", + "shlex", + "toml 0.8.23", +] + +[[package]] +name = "contextforge_benchmark_runner" +version = "1.0.0-BETA-3" +dependencies = [ + "anyhow", + "chrono", + "clap", + "csv", + "serde", + "serde_json", + "serde_yaml", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "contextforge_goose" +version = "1.0.0-BETA-3" +dependencies = [ + "goose", + "goose-eggs", + "rand 0.8.5", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "contextforge_mcp_runtime" version = "1.0.0-BETA-3" @@ -577,6 +699,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -611,6 +762,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -681,6 +841,31 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -698,6 +883,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -707,6 +913,52 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys 0.61.2", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -721,6 +973,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + [[package]] name = "deadpool" version = "0.12.3" @@ -776,7 +1034,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -799,6 +1057,18 @@ dependencies = [ "subtle", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -807,7 +1077,16 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", ] [[package]] @@ -816,6 +1095,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dunce" version = "1.0.5" @@ -871,6 +1156,28 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] + [[package]] name = "flume" version = "0.12.0" @@ -895,6 +1202,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -966,7 +1288,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1014,7 +1336,7 @@ version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width", + "unicode-width 0.2.0", ] [[package]] @@ -1068,6 +1390,74 @@ dependencies = [ "polyval", ] +[[package]] +name = "goose" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5e77ce04d5086ca6659396f81b5d63c9c0c4453c05b7fa38fcc83ea1b2b8e7" +dependencies = [ + "async-trait", + "chrono", + "ctrlc", + "downcast-rs", + "flume 0.11.1", + "futures", + "gumdrop", + "http", + "itertools 0.14.0", + "lazy_static", + "log", + "num-format", + "rand 0.9.4", + "regex", + "reqwest 0.12.28", + "serde", + "serde_json", + "simplelog", + "strum 0.27.2", + "strum_macros 0.27.2", + "tokio", + "tokio-tungstenite", + "tungstenite", + "url", +] + +[[package]] +name = "goose-eggs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd230c2b6a53026456ddf7ff626811fa6e334cfd9caf19b768baa9084fa66185" +dependencies = [ + "goose", + "html-escape", + "http", + "log", + "rand 0.9.4", + "regex", + "reqwest 0.12.28", + "tokio", +] + +[[package]] +name = "gumdrop" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc700f989d2f6f0248546222d9b4258f5b02a171a431f8285a81c08142629e3" +dependencies = [ + "gumdrop_derive", +] + +[[package]] +name = "gumdrop_derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729f9bd3449d77e7831a18abfb7ba2f99ee813dfd15b8c2167c9a54ba20aa99d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "h2" version = "0.4.13" @@ -1110,6 +1500,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -1140,6 +1532,15 @@ dependencies = [ "digest", ] +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "http" version = "1.4.0" @@ -1236,6 +1637,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -1370,6 +1787,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1403,6 +1826,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "inout" version = "0.1.4" @@ -1412,6 +1844,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instability" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6778b0196eefee7df739db78758e5cf9b37412268bfa5650bfeed028aed20d9c" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "inventory" version = "0.3.24" @@ -1446,7 +1891,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1484,6 +1929,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -1540,7 +1994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1623,6 +2077,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1635,6 +2095,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -1650,6 +2116,15 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1697,7 +2172,7 @@ dependencies = [ "bytes", "clap", "dotenvy", - "flume", + "flume 0.12.0", "futures", "http", "jsonrpc-core", @@ -1757,6 +2232,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.0" @@ -1764,6 +2249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", + "log", "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -1793,6 +2279,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.17.2" @@ -1864,6 +2376,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1892,6 +2414,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "numpy" version = "0.28.0" @@ -1908,6 +2439,15 @@ dependencies = [ "rustc-hash 2.1.2", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -1917,6 +2457,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + [[package]] name = "objc2-system-configuration" version = "0.3.2" @@ -1950,12 +2496,50 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.31.0" @@ -2063,6 +2647,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2143,7 +2733,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2152,6 +2742,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plotters" version = "0.3.7" @@ -2269,7 +2865,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -2315,7 +2911,23 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", ] [[package]] @@ -2360,7 +2972,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2373,7 +2985,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2413,7 +3025,7 @@ dependencies = [ "proc-macro2", "quote", "rustpython-parser", - "syn", + "syn 2.0.117", ] [[package]] @@ -2569,6 +3181,27 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum 0.26.3", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -2675,6 +3308,8 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", + "cookie", + "cookie_store", "futures-core", "futures-util", "h2", @@ -2683,9 +3318,11 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -2697,6 +3334,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -2798,6 +3436,19 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -2807,7 +3458,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -3037,7 +3688,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3064,6 +3715,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -3085,6 +3745,30 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -3117,6 +3801,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -3127,12 +3832,29 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + [[package]] name = "siphasher" version = "1.0.2" @@ -3222,12 +3944,63 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -3256,7 +4029,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3274,10 +4047,19 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -3304,7 +4086,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3315,7 +4097,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3335,7 +4117,9 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde_core", "time-core", @@ -3420,7 +4204,7 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3448,7 +4232,17 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", ] [[package]] @@ -3524,6 +4318,18 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -3537,6 +4343,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -3545,7 +4363,7 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ "indexmap", "serde_core", - "serde_spanned", + "serde_spanned 1.1.1", "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", @@ -3560,13 +4378,22 @@ checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", - "serde_spanned", + "serde_spanned 1.1.1", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", "winnow 1.0.1", ] +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -3585,6 +4412,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -3594,6 +4435,12 @@ dependencies = [ "winnow 1.0.1", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.1+spec-1.1.0" @@ -3662,13 +4509,18 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -3718,7 +4570,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3794,7 +4646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3803,6 +4655,23 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "typenum" version = "1.19.0" @@ -3888,11 +4757,34 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unicode-xid" @@ -3932,6 +4824,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -3950,6 +4848,18 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3979,6 +4889,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -4088,7 +5004,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4203,6 +5119,22 @@ dependencies = [ "web-sys", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -4212,6 +5144,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.62.2" @@ -4265,7 +5203,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4276,7 +5214,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4549,6 +5487,9 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] [[package]] name = "winnow" @@ -4615,7 +5556,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4631,7 +5572,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -4710,7 +5651,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -4731,7 +5672,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4751,7 +5692,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -4772,7 +5713,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4805,7 +5746,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/MANIFEST.in b/MANIFEST.in index d5ea71fe9d..e3df41158f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -117,6 +117,7 @@ recursive-include crates *.rs recursive-include crates *.md recursive-include crates *.py recursive-include crates *.pyi +recursive-include crates *.json recursive-include crates *.sh recursive-include crates Makefile recursive-include crates pyproject.toml diff --git a/Makefile b/Makefile index 477710c099..eb8e199280 100644 --- a/Makefile +++ b/Makefile @@ -7696,6 +7696,12 @@ profile-compare: --current $(PROFILE_DIR)/mcp_calls_profile.prof \ --output $(REPORTS_DIR)/profile-comparison.json +# help: benchmark - Open the interactive benchmark launcher +.PHONY: benchmark +benchmark: + @echo "Starting benchmark console (first run may compile; wait for TUI)..." + cargo run --manifest-path crates/contextforge_benchmark_console/Cargo.toml -- + .PHONY: async-validate async-validate: @echo "✅ Validating async code patterns..." diff --git a/crates/a2a_runtime/src/event_store.rs b/crates/a2a_runtime/src/event_store.rs index 0abd2ba11c..7c20d0c40d 100644 --- a/crates/a2a_runtime/src/event_store.rs +++ b/crates/a2a_runtime/src/event_store.rs @@ -320,7 +320,7 @@ impl EventStore { }; let mut result = Vec::with_capacity(entries.len()); - for ((event_id, score), payload) in entries.into_iter().zip(payloads.into_iter()) { + for ((event_id, score), payload) in entries.into_iter().zip(payloads) { result.push(StoredEvent { event_id, sequence: score as i64, diff --git a/crates/a2a_runtime/src/server.rs b/crates/a2a_runtime/src/server.rs index 03190e1d2f..5cc9d17c09 100644 --- a/crates/a2a_runtime/src/server.rs +++ b/crates/a2a_runtime/src/server.rs @@ -267,18 +267,14 @@ fn normalize_task_proxy_params(action: &str, body: &Value, resolved_agent_id: &s if let Value::Object(ref mut map) = params { match action { - "get" | "cancel" => { - if !map.contains_key("task_id") { - if let Some(task_id) = map.get("id").cloned() { - map.insert("task_id".to_string(), task_id); - } + "get" | "cancel" if !map.contains_key("task_id") => { + if let Some(task_id) = map.get("id").cloned() { + map.insert("task_id".to_string(), task_id); } } - "list" => { - if !map.contains_key("state") { - if let Some(state) = map.get("status").cloned() { - map.insert("state".to_string(), state); - } + "list" if !map.contains_key("state") => { + if let Some(state) = map.get("status").cloned() { + map.insert("state".to_string(), state); } } _ => {} diff --git a/crates/contextforge_benchmark_console/Cargo.lock b/crates/contextforge_benchmark_console/Cargo.lock new file mode 100644 index 0000000000..5dd1444ca2 --- /dev/null +++ b/crates/contextforge_benchmark_console/Cargo.lock @@ -0,0 +1,712 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "contextforge_benchmark_console" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", + "shlex", + "toml", +] + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] diff --git a/crates/contextforge_benchmark_console/Cargo.toml b/crates/contextforge_benchmark_console/Cargo.toml new file mode 100644 index 0000000000..b44f4d0dd0 --- /dev/null +++ b/crates/contextforge_benchmark_console/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "contextforge_benchmark_console" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +crossterm = "0.28.1" +ratatui = "0.29.0" +shlex = "1.3.0" +toml = "0.8.23" + +[lints] +workspace = true diff --git a/crates/contextforge_benchmark_console/src/main.rs b/crates/contextforge_benchmark_console/src/main.rs new file mode 100644 index 0000000000..b573346b5e --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main.rs @@ -0,0 +1,281 @@ +// Allow duplicate transitive deps in this benchmark-only binary target. +#![allow(clippy::multiple_crate_versions)] + +use std::env; +use std::fs; +use std::io::{self, BufRead, BufReader, Stdout}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{self, Receiver}; +use std::thread; +use std::time::Duration; + +use crossterm::cursor::{Hide, Show}; +use crossterm::event::{self, Event, KeyCode, KeyEvent}; +use crossterm::execute; +use crossterm::terminal::{ + EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode, +}; +use ratatui::Terminal; +use ratatui::backend::CrosstermBackend; +use ratatui::layout::{Constraint, Direction, Layout, Rect}; +use ratatui::style::{Color, Modifier, Style}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{Block, Borders, List, ListItem, ListState, Paragraph, Tabs, Wrap}; +use toml::Value as TomlValue; + +type AppResult = Result>; +const MAX_LOG_LINES: usize = 500; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LogSource { + Stdout, + Stderr, + System, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct LogLine { + source: LogSource, + text: String, +} + +#[derive(Debug)] +struct RunningCommand { + child: Child, + receiver: Receiver, + command_label: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct SuiteSummary { + file_stem: String, + suite_name: String, + description: String, +} + +impl SuiteSummary { + fn label(&self) -> &str { + &self.file_stem + } + + fn suite_name(&self) -> &str { + if self.suite_name.is_empty() { + &self.file_stem + } else { + &self.suite_name + } + } + + fn description(&self) -> &str { + if self.description.is_empty() { + "No suite description is defined in this scenario TOML yet." + } else { + &self.description + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct PreviewSections { + run_plan: Vec, + execution: Vec, + checks: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct SelectionSummary { + action_label: String, + suite_label: String, + clean_label: String, + run_mode_label: String, + run_path_label: String, + extra_args_label: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AppView { + Launcher, + SuiteInspector, + RunMonitor, + Generator, +} + +impl AppView { + const ALL: [AppView; 4] = [ + AppView::Launcher, + AppView::SuiteInspector, + AppView::RunMonitor, + AppView::Generator, + ]; + + fn label(self) -> &'static str { + match self { + AppView::Launcher => "Launcher", + AppView::SuiteInspector => "Inspector", + AppView::RunMonitor => "Run Monitor", + AppView::Generator => "Generator", + } + } + + fn supports_suite_navigation(self) -> bool { + matches!(self, AppView::Launcher | AppView::SuiteInspector) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct ScenarioCardSummary { + name: String, + description: String, + scenario_type: String, + settings: Vec<(String, String)>, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct SuiteInspectorSummary { + suite_name: String, + suite_description: String, + scenario_count_label: String, + comparison_question: String, + scenario_cards: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct RunScenarioSummary { + name: String, + status: String, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct GeneratorFocusSummary { + section_filter: String, + field_label: String, + config_key: String, + value: String, + kind: String, + schema: String, + format_hint: String, + visibility: String, + purpose: String, + effect: String, + example: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Action { + Run, + Validate, + Smoke, + CheckRuntime, + List, + Report, + Compare, + Generate, +} + +impl Action { + const ALL: [Action; 8] = [ + Action::Run, + Action::Validate, + Action::Smoke, + Action::CheckRuntime, + Action::List, + Action::Report, + Action::Compare, + Action::Generate, + ]; + + fn label(self) -> &'static str { + match self { + Action::Run => "Run", + Action::Validate => "Validate", + Action::Smoke => "Smoke", + Action::CheckRuntime => "Check", + Action::List => "List", + Action::Report => "Report", + Action::Compare => "Compare", + Action::Generate => "Generate", + } + } + + fn help(self) -> &'static str { + match self { + Action::Run => "Execute the selected scenario end to end.", + Action::Validate => "Resolve configs and generate reports without load.", + Action::Smoke => "Run the selected scenario in smoke mode.", + Action::CheckRuntime => "Check container runtime prerequisites only.", + Action::List => "List committed scenarios and exit.", + Action::Report => "Re-render a saved run summary.", + Action::Compare => "Re-render comparison output for a saved run.", + Action::Generate => { + "Generate a fully Rust-native TOML scenario template with all supported sections." + } + } + } + + fn supports_scenario(self) -> bool { + !matches!( + self, + Action::List | Action::Report | Action::Compare | Action::Generate + ) + } + + fn supports_all(self) -> bool { + matches!( + self, + Action::Run | Action::Validate | Action::Smoke | Action::CheckRuntime + ) + } + + fn supports_clean(self) -> bool { + matches!(self, Action::Run | Action::Smoke) + } + + fn needs_run_path(self) -> bool { + matches!(self, Action::Report | Action::Compare) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum InputMode { + Normal, + EditRunPath, + EditExtraArgs, + EditGeneratorField, +} + +impl InputMode { + fn label(self) -> &'static str { + match self { + InputMode::Normal => "Normal", + InputMode::EditRunPath => "Editing Run Path", + InputMode::EditExtraArgs => "Editing Extra Args", + InputMode::EditGeneratorField => "Editing Generator Field", + } + } +} + +mod main_parts; + +use main_parts::{App, discover_scenarios, restore_terminal, run_app, setup_terminal}; + +fn main() -> AppResult<()> { + let root = env::current_dir()?; + let scenarios = discover_scenarios(&root)?; + + if env::args().nth(1).as_deref() == Some("--list-scenarios") { + for scenario in scenarios { + println!("{}", scenario.label()); + } + return Ok(()); + } + + let mut terminal = setup_terminal()?; + let result = run_app(&mut terminal, App::new(scenarios), &root); + restore_terminal(&mut terminal)?; + result +} + +#[cfg(test)] +#[path = "main_parts/tests.rs"] +mod tests; diff --git a/crates/contextforge_benchmark_console/src/main_parts/app_state.rs b/crates/contextforge_benchmark_console/src/main_parts/app_state.rs new file mode 100644 index 0000000000..8673f01b3f --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/app_state.rs @@ -0,0 +1,192 @@ +pub(crate) struct App { + pub(crate) active_view: AppView, + pub(crate) action_index: usize, + pub(crate) last_standard_action_index: usize, + pub(crate) scenario_index: usize, + pub(crate) scenarios: Vec, + pub(crate) run_path: String, + pub(crate) extra_args: String, + pub(crate) all: bool, + pub(crate) clean: bool, + pub(crate) mode: InputMode, + pub(crate) status: String, + pub(crate) should_quit: bool, + pub(crate) generator: GeneratorState, + pub(crate) log_lines: Vec, + pub(crate) dropped_log_lines: usize, + pub(crate) log_scroll: usize, + pub(crate) running_command: Option, + pub(crate) current_run_scenario: Option, + pub(crate) run_scenarios: Vec, + pub(crate) last_command_label: Option, + pub(crate) last_run_dir: Option, + pub(crate) last_run_outcome: Option, +} + +impl App { + pub(crate) fn new(scenarios: Vec) -> Self { + Self { + active_view: AppView::Launcher, + action_index: 0, + last_standard_action_index: 0, + scenario_index: 0, + scenarios, + run_path: String::new(), + extra_args: String::new(), + all: false, + clean: true, + mode: InputMode::Normal, + status: "Use 1-8 or left/right for action, Enter to run, g=save template when Generate is selected.".to_string(), + should_quit: false, + generator: GeneratorState::new(), + log_lines: Vec::new(), + dropped_log_lines: 0, + log_scroll: 0, + running_command: None, + current_run_scenario: None, + run_scenarios: Vec::new(), + last_command_label: None, + last_run_dir: None, + last_run_outcome: None, + } + } + + pub(crate) fn action(&self) -> Action { + Action::ALL[self.action_index] + } + + pub(crate) fn scenario(&self) -> &str { + self.scenarios + .get(self.scenario_index) + .map(SuiteSummary::label) + .unwrap_or("rust-mcp-runtime-300") + } + + pub(crate) fn selected_suite(&self) -> Option<&SuiteSummary> { + self.scenarios.get(self.scenario_index) + } + + pub(crate) fn set_action_index(&mut self, index: usize) { + self.action_index = index % Action::ALL.len(); + if self.action() != Action::Generate { + self.last_standard_action_index = self.action_index; + } + if !self.action().supports_all() { + self.all = false; + } + if !self.action().supports_clean() { + self.clean = false; + } + if self.action() == Action::Generate { + self.active_view = AppView::Generator; + } else if self.active_view == AppView::Generator { + self.active_view = AppView::Launcher; + } + self.status = self.action().help().to_string(); + } + + pub(crate) fn move_action(&mut self, delta: isize) { + let len = Action::ALL.len() as isize; + let next = (self.action_index as isize + delta).rem_euclid(len) as usize; + self.set_action_index(next); + } + + pub(crate) fn move_scenario(&mut self, delta: isize) { + if self.scenarios.is_empty() { + return; + } + let len = self.scenarios.len() as isize; + self.scenario_index = (self.scenario_index as isize + delta).rem_euclid(len) as usize; + self.status = format!("Selected scenario: {}", self.scenario()); + } + + pub(crate) fn set_view(&mut self, view: AppView) { + self.active_view = view; + match view { + AppView::Generator => { + if self.action() != Action::Generate { + self.last_standard_action_index = self.action_index; + self.action_index = Action::ALL + .iter() + .position(|action| *action == Action::Generate) + .unwrap_or(self.action_index); + } + } + _ => { + if self.action() == Action::Generate { + self.action_index = self.last_standard_action_index; + } + } + } + self.status = match self.active_view { + AppView::Launcher => "Launcher view: choose a suite and action.".to_string(), + AppView::SuiteInspector => { + "Suite Inspector: compare scenario cards for the selected suite.".to_string() + } + AppView::RunMonitor => { + "Run Monitor: follow live logs and per-scenario progress.".to_string() + } + AppView::Generator => "Generator: edit and save a benchmark template.".to_string(), + }; + } + + pub(crate) fn cycle_view(&mut self, delta: isize) { + let views = [ + AppView::Launcher, + AppView::SuiteInspector, + AppView::RunMonitor, + AppView::Generator, + ]; + let current = views + .iter() + .position(|view| *view == self.active_view) + .unwrap_or(0) as isize; + let next = (current + delta).rem_euclid(views.len() as isize) as usize; + self.set_view(views[next]); + } + + pub(crate) fn push_log_line(&mut self, source: LogSource, text: String) { + if text.trim().is_empty() { + return; + } + self.apply_progress_line(&text); + self.status = text.clone(); + self.log_lines.push(LogLine { source, text }); + self.log_scroll = 0; + if self.log_lines.len() > MAX_LOG_LINES { + let drop_count = self.log_lines.len() - MAX_LOG_LINES; + self.log_lines.drain(0..drop_count); + self.dropped_log_lines += drop_count; + } + } + + pub(crate) fn apply_progress_line(&mut self, text: &str) { + if let Some(name) = parse_scenario_start(text) { + self.current_run_scenario = Some(name.clone()); + self.upsert_run_scenario(&name, "running"); + } + if let Some((name, status)) = parse_scenario_completion(text) { + self.current_run_scenario = None; + self.upsert_run_scenario(&name, &status); + } + if let Some(run_dir) = parse_run_dir(text) { + self.last_run_dir = Some(run_dir); + } + if let Some(outcome) = parse_run_outcome(text) { + self.last_run_outcome = Some(outcome); + } + } + + pub(crate) fn upsert_run_scenario(&mut self, name: &str, status: &str) { + if let Some(item) = self.run_scenarios.iter_mut().find(|item| item.name == name) { + item.status = status.to_string(); + return; + } + self.run_scenarios.push(RunScenarioSummary { + name: name.to_string(), + status: status.to_string(), + }); + } +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_copy.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_copy.rs new file mode 100644 index 0000000000..38ebcea23c --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_copy.rs @@ -0,0 +1,435 @@ +pub(crate) fn generator_explanation(key: &str) -> &'static str { + match key { + "file_stem" => "Sets the filename stem used when the generator saves a scenario TOML.", + "template_kind" => "Chooses which starter benchmark shape the generator should prefill.", + "suite_name" => "Sets the suite title stored in `[suite].name`.", + "suite_description" => { + "Sets the operator-facing explanation stored in `[suite].description`." + } + "output_root" => "Sets the root directory where this suite writes reports and artifacts.", + "continue_on_failure" => { + "Controls whether the suite continues running after one scenario fails." + } + "save_intermediate_artifacts" => { + "Controls whether intermediate raw outputs are kept between stages." + } + "flamegraph_enabled" => { + "Marks the suite as able to produce flamegraph-style profiling artifacts." + } + "baseline_run" => { + "Points the suite at a previously saved run summary that should act as the comparison baseline." + } + "baseline_rps_drop_pct" => { + "Sets the allowed throughput drop threshold when comparing the current run against the baseline run." + } + "baseline_p95_regression_pct" => { + "Sets the allowed p95 latency regression threshold when comparing the current run against the baseline run." + } + "baseline_failure_increase" => { + "Sets the allowed increase in failure rate when comparing the current run against the baseline run." + } + "scenario_name" => "Sets the scenario name stored in the first `[[scenario]]` entry.", + "scenario_description" => { + "Sets the scenario description stored in the first `[[scenario]]` entry." + } + "scenario_type" => "Sets the scenario classification label used in reports.", + "target_kind" => "Defines whether the benchmark is aimed at gateway or agent behavior.", + "auth_mode" => "Defines which authentication mode the generated scenario expects.", + "plugins_enabled" => "Defines whether plugin-aware setup is enabled by default.", + "expected_mcp_runtime" => "Defines the MCP runtime expectation stored in scenario setup.", + "expected_mcp_runtime_mode" => { + "Defines the expected MCP runtime mode when MCP runtime assertions are active." + } + "expected_a2a_runtime" => "Defines the A2A runtime expectation stored in scenario setup.", + "rust_plugins" => { + "Defines whether the benchmark image should include Rust plugin artifacts." + } + "profiling_image" => { + "Defines whether the benchmark image should contain profiling tooling." + } + "container_file" => "Defines which Containerfile the benchmark image build should use.", + "image_name" => "Defines the container image repository name for the benchmark image.", + "image_tag" => "Defines the container image tag used for this generated suite.", + "rebuild_policy" => "Defines when the runner is allowed to rebuild the benchmark image.", + "build_args" => "Defines additional build arguments passed into the benchmark image build.", + "http_server" => { + "Defines which application server implementation the benchmark image should run." + } + "runtime_host" => { + "Defines which host address the app binds to inside the benchmark container." + } + "transport_type" => { + "Defines which gateway transport path the benchmark traffic should exercise." + } + "gunicorn_workers" => "Defines how many Gunicorn worker processes should be launched.", + "gunicorn_timeout" => "Defines Gunicorn's timeout for slow requests and worker startup.", + "gunicorn_graceful_timeout" => "Defines Gunicorn's graceful shutdown timeout.", + "gunicorn_keep_alive" => "Defines how long Gunicorn keeps idle connections open.", + "gunicorn_max_requests" => "Defines Gunicorn's worker recycling request limit.", + "gunicorn_max_requests_jitter" => { + "Defines the jitter applied to Gunicorn worker recycling." + } + "gunicorn_backlog" => "Defines the Gunicorn listen backlog size.", + "gunicorn_preload_app" => { + "Defines whether Gunicorn preloads the application before forking." + } + "gunicorn_dev_mode" => "Defines whether Gunicorn should use development-friendly behavior.", + "granian_workers" => "Defines how many Granian worker processes should be launched.", + "granian_runtime_mode" => "Defines Granian's runtime execution mode.", + "granian_runtime_threads" => "Defines how many runtime threads each Granian worker uses.", + "granian_blocking_threads" => "Defines how many blocking helper threads Granian can use.", + "granian_http" => "Defines which Granian HTTP stack should be used.", + "granian_loop" => "Defines which event loop implementation Granian should use.", + "granian_task_impl" => "Defines which async task runtime Granian should use internally.", + "granian_http1_pipeline_flush" => { + "Defines whether Granian flushes pipelined HTTP/1 responses aggressively." + } + "granian_http1_buffer_size" => "Defines the Granian HTTP/1 input buffer size.", + "granian_backlog" => "Defines the Granian listen backlog size.", + "granian_backpressure" => "Defines Granian's in-flight backpressure threshold.", + "granian_respawn_failed" => { + "Defines whether failed Granian workers are restarted automatically." + } + "granian_workers_lifetime" => "Defines a maximum lifetime for Granian workers.", + "granian_workers_max_rss" => "Defines an RSS threshold for Granian worker recycling.", + "granian_dev_mode" => "Defines whether Granian should use development-friendly behavior.", + "granian_log_level" => "Defines Granian's server log level.", + "uvicorn_workers" => "Defines how many Uvicorn worker processes should be launched.", + "uvicorn_loop" => "Defines which event loop implementation Uvicorn should use.", + "uvicorn_http" => "Defines which HTTP parser Uvicorn should use.", + "uvicorn_backlog" => "Defines the Uvicorn listen backlog size.", + "uvicorn_timeout_keep_alive" => "Defines Uvicorn's keep-alive timeout.", + "uvicorn_limit_max_requests" => "Defines Uvicorn's worker recycling request limit.", + "uvicorn_log_level" => "Defines Uvicorn's server log level.", + "uvicorn_dev_mode" => "Defines whether Uvicorn should use development-friendly behavior.", + "trust_proxy_auth" => { + "Defines whether proxy-provided auth headers are trusted by the gateway." + } + "disable_access_log" => { + "Defines whether request access logging is disabled during the run." + } + "templates_auto_reload" => { + "Defines whether templates auto-reload during the benchmark run." + } + "structured_logging_database_enabled" => { + "Defines whether structured database logging is enabled." + } + "sqlalchemy_echo" => "Defines whether SQLAlchemy emits SQL statements to logs.", + "gateway_log_level" => "Defines the gateway application's log verbosity.", + "gateway_environment" => { + "Defines extra environment variables injected into the gateway container." + } + "target_service" => "Defines which compose service receives benchmark traffic.", + "driver" => "Defines which Rust benchmark driver binary is invoked for load generation.", + "headless" => "Defines whether the load driver should minimize interactive output.", + "only_summary" => { + "Defines whether the load driver should prefer summary output over verbose logs." + } + "html_report" => "Defines whether the run should emit an HTML report artifact.", + "users" => "Defines the target number of concurrent simulated users.", + "spawn_rate" => "Defines how quickly those simulated users are started.", + "run_time" => "Defines the total wall-clock duration of the benchmark run.", + "request_count" => "Defines an explicit request-count stop condition for the run.", + "load_host" => "Defines an explicit host URL override for the load driver.", + "seed" => "Defines the random seed used when workload selection is randomized.", + "tags" => "Defines which tagged request-catalog entries should stay eligible.", + "exclude_tags" => "Defines which tagged request-catalog entries should be excluded.", + "load_extra_args" => "Defines raw CLI flags appended to the load driver invocation.", + "load_env" => "Defines extra environment variables passed to the load driver.", + "workload_selection" => "Defines how workload endpoints are selected or mixed.", + "fallback_endpoint" => "Defines the endpoint used when no other workload target is chosen.", + "workload_endpoints" => { + "Defines explicit endpoint weights and enablement for the workload mix." + } + "warmup_seconds" => "Defines how long the suite warms up before measuring.", + "measure_seconds" => "Defines how long the primary metrics window lasts.", + "profile_seconds" => "Defines how long the dedicated profiling window lasts.", + "cooldown_seconds" => "Defines how long the suite cools down after measurement ends.", + "enabled_groups" => "Defines which request-catalog groups are explicitly kept.", + "disabled_groups" => "Defines which request-catalog groups are explicitly removed.", + "enabled_endpoints" => "Defines which request-catalog endpoints are explicitly kept.", + "disabled_endpoints" => "Defines which request-catalog endpoints are explicitly removed.", + "enabled_tags" => "Defines which request-catalog tags are explicitly kept.", + "disabled_tags" => "Defines which request-catalog tags are explicitly removed.", + "include_admin_endpoints" => { + "Defines whether admin endpoints remain eligible in request selection." + } + "include_mcp_endpoints" => { + "Defines whether MCP endpoints remain eligible in request selection." + } + "include_resource_endpoints" => { + "Defines whether resource endpoints remain eligible in request selection." + } + "include_prompt_endpoints" => { + "Defines whether prompt endpoints remain eligible in request selection." + } + "include_tool_endpoints" => { + "Defines whether tool endpoints remain eligible in request selection." + } + "profiling_enabled" => { + "Defines whether profiling behavior is active for the generated suite." + } + "profiling_tools" => "Defines which profiling tools the suite should request.", + "profiling_duration_seconds" => "Defines how long profiling should run when enabled.", + "profiling_required" => { + "Defines whether missing profiling artifacts should fail the scenario." + } + "retry_enabled" => "Defines whether failed scenarios should be retried automatically.", + "max_attempts" => "Defines the maximum number of attempts allowed for a scenario.", + "capture_logs" => "Defines whether logs are captured into benchmark artifacts.", + "save_raw_results" => "Defines whether raw result files are preserved after a run.", + "reuse_stack" => "Defines whether scenarios are allowed to reuse the same running stack.", + "defaults_plugins_snippet" => { + "Defines a raw plugin configuration snippet under the defaults block." + } + "scenario_setup_snippet" => "Defines a raw setup override for the generated scenario.", + "scenario_build_snippet" => "Defines a raw build override for the generated scenario.", + "scenario_runtime_snippet" => "Defines a raw runtime override for the generated scenario.", + "scenario_gateway_snippet" => "Defines a raw gateway override for the generated scenario.", + "scenario_load_snippet" => "Defines a raw load override for the generated scenario.", + "scenario_measurement_snippet" => { + "Defines a raw measurement override for the generated scenario." + } + "scenario_requests_snippet" => { + "Defines a raw request-selection override for the generated scenario." + } + "scenario_profiling_snippet" => { + "Defines a raw profiling override for the generated scenario." + } + "scenario_execution_snippet" => { + "Defines a raw execution override for the generated scenario." + } + "scenario_plugins_snippet" => "Defines a raw plugin override for the generated scenario.", + _ => "Defines a generator field.", + } +} + +pub(crate) fn generator_change_reason(key: &str) -> &'static str { + match key { + "file_stem" => "Changing it changes which scenario file is created or overwritten.", + "template_kind" => { + "Changing it swaps in a different preset workload shape and default values." + } + "suite_name" => "Changing it changes the suite title shown in metadata and reports.", + "suite_description" => { + "Changing it changes the explanation operators read in the console and TOML." + } + "output_root" => "Changing it moves where reports and artifacts are written.", + "continue_on_failure" => { + "Changing it changes whether later scenarios still run after an earlier failure." + } + "save_intermediate_artifacts" => { + "Changing it changes whether transient artifacts are preserved." + } + "flamegraph_enabled" => { + "Changing it changes whether flamegraph-oriented suite behavior is expected." + } + "baseline_run" => { + "Changing it points the suite at a different saved run to treat as the benchmark baseline." + } + "baseline_rps_drop_pct" => { + "Changing it makes the suite more or less strict about tolerated throughput loss." + } + "baseline_p95_regression_pct" => { + "Changing it makes the suite more or less strict about tolerated p95 latency regression." + } + "baseline_failure_increase" => { + "Changing it makes the suite more or less strict about tolerated failure-rate increase." + } + "scenario_name" => "Changing it renames the generated scenario in reports and logs.", + "scenario_description" => "Changing it changes how the scenario is explained to operators.", + "scenario_type" => "Changing it changes how the scenario is categorized downstream.", + "target_kind" => { + "Changing it changes whether the generated scenario targets gateway or agent behavior." + } + "auth_mode" => "Changing it changes which auth path the scenario is configured to use.", + "plugins_enabled" => { + "Changing it changes whether plugin-specific fields and setup matter in the template." + } + "expected_mcp_runtime" => { + "Changing it changes the MCP runtime assertion recorded in the scenario." + } + "expected_mcp_runtime_mode" => "Changing it changes the asserted MCP runtime mode.", + "expected_a2a_runtime" => { + "Changing it changes the A2A runtime assertion recorded in the scenario." + } + "rust_plugins" => { + "Changing it changes whether Rust plugin artifacts are built into the benchmark image." + } + "profiling_image" => { + "Changing it changes whether profiling tooling is installed into the benchmark image." + } + "container_file" => "Changing it changes which image definition file the build uses.", + "image_name" => "Changing it changes the image repository name used during build and run.", + "image_tag" => "Changing it changes which benchmark image tag is built or reused.", + "rebuild_policy" => "Changing it changes when the runner rebuilds the benchmark image.", + "build_args" => { + "Changing it changes the build-time feature flags or values passed into the image build." + } + "http_server" => "Changing it switches the server implementation under the same workload.", + "runtime_host" => { + "Changing it changes which interface the app binds to inside the container." + } + "transport_type" => { + "Changing it changes which gateway transport path the load test exercises." + } + "gunicorn_workers" => "Changing it changes Gunicorn process concurrency.", + "gunicorn_timeout" => { + "Changing it changes how long Gunicorn waits before timing out slow work." + } + "gunicorn_graceful_timeout" => "Changing it changes Gunicorn shutdown grace periods.", + "gunicorn_keep_alive" => "Changing it changes Gunicorn keep-alive behavior.", + "gunicorn_max_requests" => "Changing it changes Gunicorn worker recycling frequency.", + "gunicorn_max_requests_jitter" => { + "Changing it changes how staggered Gunicorn worker recycling is." + } + "gunicorn_backlog" => { + "Changing it changes how many pending connections Gunicorn can queue." + } + "gunicorn_preload_app" => { + "Changing it changes whether the app is loaded once before workers fork." + } + "gunicorn_dev_mode" => { + "Changing it changes whether Gunicorn behaves more like development mode." + } + "granian_workers" => "Changing it changes Granian process concurrency.", + "granian_runtime_mode" => "Changing it changes how Granian executes async work.", + "granian_runtime_threads" => { + "Changing it changes how many runtime threads each Granian worker gets." + } + "granian_blocking_threads" => { + "Changing it changes how much blocking work Granian can offload." + } + "granian_http" => "Changing it changes the Granian HTTP protocol stack.", + "granian_loop" => "Changing it changes the event loop implementation Granian uses.", + "granian_task_impl" => "Changing it changes the async runtime Granian uses internally.", + "granian_http1_pipeline_flush" => "Changing it changes Granian's HTTP/1 flush behavior.", + "granian_http1_buffer_size" => "Changing it changes Granian's HTTP/1 buffering behavior.", + "granian_backlog" => "Changing it changes how many pending connections Granian can queue.", + "granian_backpressure" => "Changing it changes when Granian starts applying backpressure.", + "granian_respawn_failed" => { + "Changing it changes whether failed Granian workers come back automatically." + } + "granian_workers_lifetime" => { + "Changing it changes how often Granian workers recycle by age." + } + "granian_workers_max_rss" => { + "Changing it changes when Granian workers recycle for memory growth." + } + "granian_dev_mode" => { + "Changing it changes whether Granian behaves more like development mode." + } + "granian_log_level" => "Changing it changes Granian's server log verbosity.", + "uvicorn_workers" => "Changing it changes Uvicorn process concurrency.", + "uvicorn_loop" => "Changing it changes the event loop implementation Uvicorn uses.", + "uvicorn_http" => "Changing it changes the HTTP parser Uvicorn uses.", + "uvicorn_backlog" => "Changing it changes how many pending connections Uvicorn can queue.", + "uvicorn_timeout_keep_alive" => "Changing it changes Uvicorn keep-alive behavior.", + "uvicorn_limit_max_requests" => "Changing it changes Uvicorn worker recycling frequency.", + "uvicorn_log_level" => "Changing it changes Uvicorn's server log verbosity.", + "uvicorn_dev_mode" => { + "Changing it changes whether Uvicorn behaves more like development mode." + } + "trust_proxy_auth" => { + "Changing it changes whether proxy auth headers are accepted as authoritative." + } + "disable_access_log" => { + "Changing it changes whether access logs are emitted during the run." + } + "templates_auto_reload" => "Changing it changes whether template files auto-reload.", + "structured_logging_database_enabled" => { + "Changing it changes whether structured logs are written to the database." + } + "sqlalchemy_echo" => "Changing it changes whether SQL statements appear in logs.", + "gateway_log_level" => "Changing it changes gateway log volume.", + "gateway_environment" => { + "Changing it changes which environment variables are injected into the gateway container." + } + "target_service" => "Changing it changes which service the load generator targets.", + "driver" => "Changing it changes which benchmark driver binary is executed.", + "headless" => "Changing it changes how interactive the load output is.", + "only_summary" => "Changing it changes how much output the load run prints.", + "html_report" => "Changing it changes whether an HTML report is requested.", + "users" => "Changing it changes target concurrency.", + "spawn_rate" => "Changing it changes ramp-up speed.", + "run_time" => "Changing it changes total benchmark duration.", + "request_count" => "Changing it changes whether the run stops after a fixed request total.", + "load_host" => "Changing it changes which host URL the load driver calls.", + "seed" => "Changing it changes the workload randomization sequence.", + "tags" => "Changing it changes which tagged requests remain active.", + "exclude_tags" => "Changing it changes which tagged requests are filtered out.", + "load_extra_args" => "Changing it changes the raw flags appended to the driver command.", + "load_env" => "Changing it changes the environment passed to the load driver.", + "workload_selection" => "Changing it changes how request targets are selected or weighted.", + "fallback_endpoint" => "Changing it changes the default endpoint used by the workload.", + "workload_endpoints" => "Changing it changes explicit endpoint weighting and enablement.", + "warmup_seconds" => { + "Changing it changes how long the run warms up before measurement starts." + } + "measure_seconds" => "Changing it changes how long the primary metrics window lasts.", + "profile_seconds" => "Changing it changes how long profiling stays active.", + "cooldown_seconds" => "Changing it changes how long the run cools down before teardown.", + "enabled_groups" => { + "Changing it changes which request groups are allowed into the workload." + } + "disabled_groups" => { + "Changing it changes which request groups are removed from the workload." + } + "enabled_endpoints" => "Changing it changes which endpoints are kept in the workload.", + "disabled_endpoints" => { + "Changing it changes which endpoints are removed from the workload." + } + "enabled_tags" => "Changing it changes which tagged endpoints remain active.", + "disabled_tags" => "Changing it changes which tagged endpoints are removed.", + "include_admin_endpoints" => "Changing it changes whether admin endpoints remain eligible.", + "include_mcp_endpoints" => "Changing it changes whether MCP endpoints remain eligible.", + "include_resource_endpoints" => { + "Changing it changes whether resource endpoints remain eligible." + } + "include_prompt_endpoints" => { + "Changing it changes whether prompt endpoints remain eligible." + } + "include_tool_endpoints" => "Changing it changes whether tool endpoints remain eligible.", + "profiling_enabled" => "Changing it turns profiling behavior on or off.", + "profiling_tools" => "Changing it changes which profiling tools the suite requests.", + "profiling_duration_seconds" => "Changing it changes requested profiling duration.", + "profiling_required" => "Changing it changes whether missing profiles fail the scenario.", + "retry_enabled" => "Changing it changes whether failed scenarios are retried.", + "max_attempts" => "Changing it changes how many attempts the runner may make.", + "capture_logs" => "Changing it changes whether logs are captured into artifacts.", + "save_raw_results" => "Changing it changes whether raw result files are preserved.", + "reuse_stack" => "Changing it changes whether scenarios can reuse the same running stack.", + "defaults_plugins_snippet" => { + "Changing it changes the raw plugin configuration written into the defaults block." + } + "scenario_setup_snippet" => { + "Changing it changes only the scenario-specific setup override." + } + "scenario_build_snippet" => { + "Changing it changes only the scenario-specific build override." + } + "scenario_runtime_snippet" => { + "Changing it changes only the scenario-specific runtime override." + } + "scenario_gateway_snippet" => { + "Changing it changes only the scenario-specific gateway override." + } + "scenario_load_snippet" => "Changing it changes only the scenario-specific load override.", + "scenario_measurement_snippet" => { + "Changing it changes only the scenario-specific measurement override." + } + "scenario_requests_snippet" => { + "Changing it changes only the scenario-specific request-selection override." + } + "scenario_profiling_snippet" => { + "Changing it changes only the scenario-specific profiling override." + } + "scenario_execution_snippet" => { + "Changing it changes only the scenario-specific execution override." + } + "scenario_plugins_snippet" => { + "Changing it changes only the scenario-specific plugin override." + } + _ => "Changing it changes the generated benchmark template.", + } +} diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_examples.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_examples.rs new file mode 100644 index 0000000000..ada9f2dd28 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_examples.rs @@ -0,0 +1,168 @@ +pub(crate) fn generator_visibility_note(key: &str) -> &'static str { + match key { + "expected_mcp_runtime_mode" => { + "Visible only after expected_mcp_runtime is set, because runtime mode only matters when you are asserting an MCP runtime." + } + "gunicorn_workers" + | "gunicorn_timeout" + | "gunicorn_graceful_timeout" + | "gunicorn_keep_alive" + | "gunicorn_max_requests" + | "gunicorn_max_requests_jitter" + | "gunicorn_backlog" + | "gunicorn_preload_app" + | "gunicorn_dev_mode" => "Visible only when http_server is gunicorn.", + "granian_workers" + | "granian_runtime_mode" + | "granian_runtime_threads" + | "granian_blocking_threads" + | "granian_http" + | "granian_loop" + | "granian_task_impl" + | "granian_http1_pipeline_flush" + | "granian_http1_buffer_size" + | "granian_backlog" + | "granian_backpressure" + | "granian_respawn_failed" + | "granian_workers_lifetime" + | "granian_workers_max_rss" + | "granian_dev_mode" + | "granian_log_level" => "Visible only when http_server is granian.", + "uvicorn_workers" + | "uvicorn_loop" + | "uvicorn_http" + | "uvicorn_backlog" + | "uvicorn_timeout_keep_alive" + | "uvicorn_limit_max_requests" + | "uvicorn_log_level" + | "uvicorn_dev_mode" => "Visible only when http_server is uvicorn.", + "profiling_tools" | "profiling_duration_seconds" | "profiling_required" => { + "Visible only when profiling_enabled is true." + } + "defaults_plugins_snippet" | "scenario_plugins_snippet" => { + "Visible only when plugins_enabled is true." + } + "workload_endpoints" => { + "Visible once the workload area is in use. Keep it empty if you just want the preset selection and fallback endpoint." + } + _ => "Always visible for this generator.", + } +} + +pub(crate) fn generator_example(key: &str) -> &'static str { + match key { + "file_stem" => "a2a-invoke-300", + "template_kind" => "a2a", + "suite_name" => "contextforge-a2a-compare", + "suite_description" => "Compare Rust A2A invoke throughput", + "output_root" => "reports/benchmarks", + "continue_on_failure" => "false", + "save_intermediate_artifacts" => "true", + "flamegraph_enabled" => "false", + "baseline_run" => "reports/benchmarks/prior-run/run_summary.json", + "baseline_rps_drop_pct" => "5", + "baseline_p95_regression_pct" => "10", + "baseline_failure_increase" => "0", + "scenario_name" => "gunicorn-a2a-invoke-rust", + "scenario_description" => "A2A invoke benchmark against Rust mode", + "scenario_type" => "comparison", + "target_kind" => "gateway", + "auth_mode" => "jwt", + "plugins_enabled" => "false", + "expected_mcp_runtime" => "rust", + "expected_mcp_runtime_mode" => "rust-managed", + "expected_a2a_runtime" => "rust", + "rust_plugins" => "true", + "profiling_image" => "false", + "container_file" => "crates/contextforge_benchmark_runner/assets/Containerfile", + "image_name" => "mcpgateway/mcpgateway", + "image_tag" => "benchmark-suite-modular-design", + "rebuild_policy" => "missing", + "build_args" => "ENABLE_RUST_MCP_RMCP = \"true\" | ENABLE_A2A = \"true\"", + "http_server" => "granian", + "runtime_host" => "127.0.0.1", + "transport_type" => "streamablehttp", + "gunicorn_workers" | "granian_workers" | "uvicorn_workers" => "12", + "gunicorn_timeout" => "30", + "gunicorn_graceful_timeout" => "30", + "gunicorn_keep_alive" => "10", + "gunicorn_max_requests" | "uvicorn_limit_max_requests" => "0", + "gunicorn_max_requests_jitter" => "0", + "gunicorn_backlog" | "granian_backlog" | "uvicorn_backlog" => "2048", + "gunicorn_preload_app" | "granian_respawn_failed" => "true", + "gunicorn_dev_mode" | "granian_dev_mode" | "uvicorn_dev_mode" => "false", + "granian_runtime_mode" => "mt", + "granian_runtime_threads" => "1", + "granian_blocking_threads" => "512", + "granian_http" => "1", + "granian_loop" | "uvicorn_loop" => "auto", + "granian_task_impl" => "async-std", + "granian_http1_pipeline_flush" => "false", + "granian_http1_buffer_size" => "8192", + "granian_backpressure" => "1024", + "granian_workers_lifetime" | "granian_workers_max_rss" => "0", + "granian_log_level" | "uvicorn_log_level" | "gateway_log_level" => "warning", + "uvicorn_http" => "auto", + "uvicorn_timeout_keep_alive" => "5", + "trust_proxy_auth" + | "sqlalchemy_echo" + | "templates_auto_reload" + | "structured_logging_database_enabled" => "false", + "disable_access_log" => "true", + "gateway_environment" => "RUST_MCP_MODE = \"edge\" | MCPGATEWAY_UI_ENABLED = \"false\"", + "target_service" => "nginx", + "driver" => "contextforge_goose", + "headless" | "only_summary" | "retry_enabled" | "capture_logs" | "save_raw_results" + | "reuse_stack" => "true", + "html_report" + | "include_admin_endpoints" + | "include_mcp_endpoints" + | "include_resource_endpoints" + | "include_prompt_endpoints" + | "include_tool_endpoints" + | "profiling_enabled" + | "profiling_required" => "false", + "users" => "300", + "spawn_rate" => "60", + "run_time" => "180s", + "request_count" => "10000", + "load_host" => "http://gateway:4444", + "seed" => "1234", + "tags" => "a2a,hot-path", + "exclude_tags" => "admin", + "load_extra_args" => "--report-file,custom-goose-report.html", + "load_env" => "BENCH_MCP_SESSION_MODE = \"reuse\" | BENCHMARK_TARGET = \"a2a\"", + "workload_selection" => "weighted-random", + "fallback_endpoint" => "/health", + "workload_endpoints" => { + "[defaults.load.workload.endpoints.\"/a2a/a2a-echo-agent/invoke\"] | enabled = true | weight = 1" + } + "warmup_seconds" => "30", + "measure_seconds" => "120", + "profile_seconds" => "0", + "cooldown_seconds" => "30", + "enabled_groups" => "tools,resources", + "disabled_groups" => "admin", + "enabled_endpoints" => "/servers,/health", + "disabled_endpoints" => "/admin/plugins", + "enabled_tags" => "mcp,a2a", + "disabled_tags" => "slow", + "profiling_tools" => "perf,flamegraph", + "profiling_duration_seconds" => "30", + "max_attempts" => "2", + "defaults_plugins_snippet" => "mode = \"rust\" | timeout_ms = 250", + "scenario_setup_snippet" => "plugins_enabled = true", + "scenario_build_snippet" => "image_tag = \"benchmark-override\"", + "scenario_runtime_snippet" => "http_server = \"granian\"", + "scenario_gateway_snippet" => "log_level = \"WARNING\"", + "scenario_load_snippet" => "users = 100", + "scenario_measurement_snippet" => "warmup_seconds = 10", + "scenario_requests_snippet" => "enabled_groups = [\"resources\"]", + "scenario_profiling_snippet" => { + "enabled = true | tools = [\"perf\", \"flamegraph\"] | duration_seconds = 30 | required = true" + } + "scenario_execution_snippet" => "max_attempts = 1", + "scenario_plugins_snippet" => "mode = \"rust\" | timeout_ms = 500", + _ => "Set this to the value you want written into the generated scenario.", + } +} diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_fields_execution.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_execution.rs new file mode 100644 index 0000000000..4ca9c663ce --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_execution.rs @@ -0,0 +1,126 @@ +use crate::main_parts::*; + +pub(crate) fn generator_fields_execution() -> Vec { + vec![ + bool_field( + "Profiling On", + "profiling_enabled", + false, + "defaults.profiling.enabled", + ), + text_field( + "Rust Profilers", + "profiling_tools", + "perf,flamegraph", + "Comma-separated Rust-native profilers such as perf and flamegraph.", + ), + text_field( + "Profile Dur", + "profiling_duration_seconds", + "0", + "defaults.profiling.duration_seconds", + ), + bool_field( + "Profile Required", + "profiling_required", + false, + "defaults.profiling.required", + ), + bool_field( + "Retry Enabled", + "retry_enabled", + true, + "defaults.execution.retry_enabled", + ), + text_field( + "Max Attempts", + "max_attempts", + "2", + "defaults.execution.max_attempts", + ), + bool_field( + "Capture Logs", + "capture_logs", + true, + "defaults.execution.capture_logs", + ), + bool_field( + "Save Raw", + "save_raw_results", + true, + "defaults.execution.save_raw_results", + ), + bool_field( + "Reuse Stack", + "reuse_stack", + true, + "defaults.execution.reuse_stack", + ), + text_field( + "Defaults Plugins", + "defaults_plugins_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [defaults.plugins.].", + ), + text_field( + "Scenario Setup", + "scenario_setup_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.setup].", + ), + text_field( + "Scenario Build", + "scenario_build_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.build].", + ), + text_field( + "Scenario Runtime", + "scenario_runtime_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.runtime].", + ), + text_field( + "Scenario Gateway", + "scenario_gateway_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.gateway].", + ), + text_field( + "Scenario Load", + "scenario_load_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.load].", + ), + text_field( + "Scenario Measure", + "scenario_measurement_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.measurement].", + ), + text_field( + "Scenario Requests", + "scenario_requests_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.requests].", + ), + text_field( + "Scenario Profiling", + "scenario_profiling_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.profiling], using Rust-native profiling settings.", + ), + text_field( + "Scenario Execution", + "scenario_execution_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.execution].", + ), + text_field( + "Scenario Plugins", + "scenario_plugins_snippet", + "", + "Optional raw TOML lines with ' | ' separators for [scenario.plugins.].", + ), + ] +} diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_fields_runtime.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_runtime.rs new file mode 100644 index 0000000000..9eb804ac5c --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_runtime.rs @@ -0,0 +1,255 @@ +pub(crate) fn generator_fields_runtime() -> Vec { + vec![ + choice_field( + "HTTP Server", + "http_server", + &["gunicorn", "granian", "uvicorn"], + "gunicorn", + "defaults.runtime.http_server", + ), + text_field( + "Runtime Host", + "runtime_host", + "127.0.0.1", + "defaults.runtime.host", + ), + choice_field( + "Transport", + "transport_type", + &["streamablehttp", "sse", "websocket"], + "streamablehttp", + "defaults.runtime.transport_type", + ), + text_field( + "Gunicorn Workers", + "gunicorn_workers", + "12", + "defaults.runtime.gunicorn.workers", + ), + text_field( + "Gunicorn Timeout", + "gunicorn_timeout", + "30", + "defaults.runtime.gunicorn.timeout", + ), + text_field( + "Gunicorn Grace", + "gunicorn_graceful_timeout", + "30", + "defaults.runtime.gunicorn.graceful_timeout", + ), + text_field( + "Gunicorn KeepAlive", + "gunicorn_keep_alive", + "10", + "defaults.runtime.gunicorn.keep_alive", + ), + text_field( + "Gunicorn MaxReq", + "gunicorn_max_requests", + "0", + "defaults.runtime.gunicorn.max_requests", + ), + text_field( + "Gunicorn Jitter", + "gunicorn_max_requests_jitter", + "0", + "defaults.runtime.gunicorn.max_requests_jitter", + ), + text_field( + "Gunicorn Backlog", + "gunicorn_backlog", + "16384", + "defaults.runtime.gunicorn.backlog", + ), + bool_field( + "Gunicorn Preload", + "gunicorn_preload_app", + true, + "defaults.runtime.gunicorn.preload_app", + ), + bool_field( + "Gunicorn Dev", + "gunicorn_dev_mode", + false, + "defaults.runtime.gunicorn.dev_mode", + ), + text_field( + "Granian Workers", + "granian_workers", + "", + "Worker process count when using Granian.", + ), + text_field( + "Granian Mode", + "granian_runtime_mode", + "", + "Granian runtime_mode, for example st or mt.", + ), + text_field( + "Granian Threads", + "granian_runtime_threads", + "", + "Async runtime threads per worker.", + ), + text_field( + "Granian Blocking", + "granian_blocking_threads", + "", + "Blocking thread pool size.", + ), + text_field( + "Granian HTTP", + "granian_http", + "", + "HTTP protocol mode used by Granian.", + ), + text_field( + "Granian Loop", + "granian_loop", + "", + "Granian event loop selection.", + ), + text_field( + "Granian Task Impl", + "granian_task_impl", + "", + "Task implementation backend for Granian.", + ), + bool_field( + "Granian Flush", + "granian_http1_pipeline_flush", + false, + "Flush HTTP/1 pipelined responses immediately.", + ), + text_field( + "Granian Buf Size", + "granian_http1_buffer_size", + "", + "HTTP/1 buffer size in bytes.", + ), + text_field( + "Granian Backlog", + "granian_backlog", + "", + "Listen backlog for pending connections.", + ), + text_field( + "Granian Pressure", + "granian_backpressure", + "", + "Backpressure queue limit.", + ), + bool_field( + "Granian Respawn", + "granian_respawn_failed", + true, + "Respawn failed workers automatically.", + ), + text_field( + "Granian Lifetime", + "granian_workers_lifetime", + "", + "Maximum worker lifetime.", + ), + text_field( + "Granian Max RSS", + "granian_workers_max_rss", + "", + "Restart workers over this RSS threshold.", + ), + bool_field( + "Granian Dev", + "granian_dev_mode", + false, + "Enable Granian dev mode.", + ), + text_field("Granian Log", "granian_log_level", "", "Granian log level."), + text_field( + "Uvicorn Workers", + "uvicorn_workers", + "", + "Worker process count when using Uvicorn.", + ), + text_field( + "Uvicorn Loop", + "uvicorn_loop", + "", + "Event loop implementation, for example auto or uvloop.", + ), + text_field( + "Uvicorn HTTP", + "uvicorn_http", + "", + "HTTP protocol implementation.", + ), + text_field( + "Uvicorn Backlog", + "uvicorn_backlog", + "", + "Listen backlog for pending connections.", + ), + text_field( + "Uvicorn KeepAlive", + "uvicorn_timeout_keep_alive", + "", + "Keep-alive timeout in seconds.", + ), + text_field( + "Uvicorn MaxReq", + "uvicorn_limit_max_requests", + "", + "Restart worker after this many requests.", + ), + text_field("Uvicorn Log", "uvicorn_log_level", "", "Uvicorn log level."), + bool_field( + "Uvicorn Dev", + "uvicorn_dev_mode", + false, + "Enable Uvicorn dev mode.", + ), + bool_field( + "Trust Proxy", + "trust_proxy_auth", + false, + "defaults.gateway.trust_proxy_auth", + ), + bool_field( + "Disable Access Log", + "disable_access_log", + true, + "defaults.gateway.disable_access_log", + ), + bool_field( + "Templates Reload", + "templates_auto_reload", + false, + "defaults.gateway.templates_auto_reload", + ), + bool_field( + "Structured DB Log", + "structured_logging_database_enabled", + false, + "defaults.gateway.structured_logging_database_enabled", + ), + bool_field( + "SQL Echo", + "sqlalchemy_echo", + false, + "defaults.gateway.sqlalchemy_echo", + ), + text_field( + "Gateway Log", + "gateway_log_level", + "ERROR", + "defaults.gateway.log_level", + ), + text_field( + "Gateway Env", + "gateway_environment", + "", + "Optional lines with ' | ' separators, e.g. RUST_MCP_MODE = \"edge\"", + ), + ] +} +use crate::main_parts::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_fields_suite.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_suite.rs new file mode 100644 index 0000000000..e7827c53c9 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_suite.rs @@ -0,0 +1,223 @@ +pub(crate) fn text_field( + label: &'static str, + key: &'static str, + value: &'static str, + help: &'static str, +) -> GeneratorField { + GeneratorField { + label, + key, + kind: GeneratorFieldKind::Text, + value: value.to_string(), + help, + } +} + +pub(crate) fn bool_field( + label: &'static str, + key: &'static str, + value: bool, + help: &'static str, +) -> GeneratorField { + GeneratorField { + label, + key, + kind: GeneratorFieldKind::Bool, + value: if value { "true" } else { "false" }.to_string(), + help, + } +} + +pub(crate) fn choice_field( + label: &'static str, + key: &'static str, + options: &'static [&'static str], + value: &'static str, + help: &'static str, +) -> GeneratorField { + GeneratorField { + label, + key, + kind: GeneratorFieldKind::Choice(options), + value: value.to_string(), + help, + } +} + +pub(crate) fn generator_fields_suite() -> Vec { + vec![ + text_field( + "File Stem", + "file_stem", + "new-scenario", + "Output file name under crates/contextforge_benchmark_runner/assets/scenarios/.", + ), + choice_field( + "Template Kind", + "template_kind", + &["blank", "mcp", "a2a"], + "blank", + "Choose a starter workload shape.", + ), + text_field( + "Suite Name", + "suite_name", + "benchmark-generated-suite", + "The [suite].name value.", + ), + text_field( + "Suite Desc", + "suite_description", + "Generated benchmark scenario template", + "The [suite].description value.", + ), + text_field( + "Output Root", + "output_root", + "reports/benchmarks", + "Benchmark output directory.", + ), + bool_field( + "Continue Fail", + "continue_on_failure", + false, + "suite.continue_on_failure", + ), + bool_field( + "Save Artifacts", + "save_intermediate_artifacts", + true, + "suite.save_intermediate_artifacts", + ), + bool_field( + "Flamegraphs", + "flamegraph_enabled", + false, + "suite.flamegraph_enabled", + ), + text_field( + "Baseline Run", + "baseline_run", + "", + "Optional prior run_summary.json path.", + ), + text_field( + "Baseline RPS%", + "baseline_rps_drop_pct", + "", + "Optional allowed RPS drop percentage.", + ), + text_field( + "Baseline P95%", + "baseline_p95_regression_pct", + "", + "Optional allowed p95 regression percentage.", + ), + text_field( + "Baseline Fail+", + "baseline_failure_increase", + "", + "Optional allowed failure increase.", + ), + text_field( + "Scenario Name", + "scenario_name", + "generated-scenario", + "Name for the first [[scenario]] entry.", + ), + text_field( + "Scenario Desc", + "scenario_description", + "Generated benchmark scenario", + "Description for the first [[scenario]] entry.", + ), + text_field( + "Scenario Type", + "scenario_type", + "custom", + "Freeform scenario_type label.", + ), + choice_field( + "Target Kind", + "target_kind", + &["gateway", "agent"], + "gateway", + "defaults.setup.target_kind", + ), + choice_field( + "Auth Mode", + "auth_mode", + &["jwt", "basic", "none"], + "jwt", + "defaults.setup.auth_mode", + ), + bool_field( + "Plugins", + "plugins_enabled", + false, + "defaults.setup.plugins_enabled", + ), + text_field( + "Expect MCP", + "expected_mcp_runtime", + "", + "Optional defaults.setup.expected_mcp_runtime", + ), + text_field( + "Expect MCP Mode", + "expected_mcp_runtime_mode", + "", + "Optional defaults.setup.expected_mcp_runtime_mode", + ), + text_field( + "Expect A2A", + "expected_a2a_runtime", + "", + "Optional defaults.setup.expected_a2a_runtime", + ), + bool_field( + "Rust Plugins", + "rust_plugins", + false, + "defaults.build.rust_plugins", + ), + bool_field( + "Profiling Img", + "profiling_image", + false, + "defaults.build.profiling_image", + ), + text_field( + "Container File", + "container_file", + "crates/contextforge_benchmark_runner/assets/Containerfile", + "defaults.build.container_file", + ), + text_field( + "Image Name", + "image_name", + "mcpgateway/mcpgateway", + "defaults.build.image_name", + ), + text_field( + "Image Tag", + "image_tag", + "benchmark-suite-generated", + "defaults.build.image_tag", + ), + choice_field( + "Rebuild", + "rebuild_policy", + &["never", "missing", "always"], + "missing", + "defaults.build.rebuild_policy", + ), + text_field( + "Build Args", + "build_args", + "", + "Optional build args. Use 'KEY = \"value\" | OTHER = \"x\"'.", + ), + ] +} +use crate::main_parts::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_fields_workload.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_workload.rs new file mode 100644 index 0000000000..4b8738f4d8 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_fields_workload.rs @@ -0,0 +1,170 @@ +use crate::main_parts::*; + +pub(crate) fn generator_fields_workload() -> Vec { + vec![ + choice_field( + "Target Service", + "target_service", + &["nginx", "gateway"], + "nginx", + "defaults.load.target_service", + ), + text_field( + "Load Driver", + "driver", + "contextforge_goose", + "defaults.load.driver", + ), + bool_field("Headless", "headless", true, "defaults.load.headless"), + bool_field( + "Only Summary", + "only_summary", + true, + "defaults.load.only_summary", + ), + bool_field( + "HTML Report", + "html_report", + false, + "defaults.load.html_report", + ), + text_field("Users", "users", "300", "defaults.load.users"), + text_field("Spawn Rate", "spawn_rate", "60", "defaults.load.spawn_rate"), + text_field("Run Time", "run_time", "180s", "defaults.load.run_time"), + text_field( + "Request Count", + "request_count", + "", + "Optional defaults.load.request_count", + ), + text_field("Load Host", "load_host", "", "Optional defaults.load.host"), + text_field("Seed", "seed", "", "Optional defaults.load.seed"), + text_field("Tags", "tags", "", "Comma-separated defaults.load.tags"), + text_field( + "Exclude Tags", + "exclude_tags", + "", + "Comma-separated defaults.load.exclude_tags", + ), + text_field( + "Extra Args CSV", + "load_extra_args", + "", + "Comma-separated defaults.load.extra_args", + ), + text_field( + "Load Env", + "load_env", + "", + "Optional lines with ' | ' separators, e.g. BENCH_MCP_SESSION_MODE = \"reuse\"", + ), + text_field( + "Selection", + "workload_selection", + "", + "Optional defaults.load.workload.selection", + ), + text_field( + "Fallback", + "fallback_endpoint", + "/health", + "defaults.load.workload.fallback_endpoint", + ), + text_field( + "Workload Endpoints", + "workload_endpoints", + "", + "Optional raw TOML lines with ' | ' separators for workload endpoint tables.", + ), + text_field( + "Warmup", + "warmup_seconds", + "30", + "defaults.measurement.warmup_seconds", + ), + text_field( + "Measure", + "measure_seconds", + "120", + "defaults.measurement.measure_seconds", + ), + text_field( + "Profile", + "profile_seconds", + "0", + "defaults.measurement.profile_seconds", + ), + text_field( + "Cooldown", + "cooldown_seconds", + "30", + "defaults.measurement.cooldown_seconds", + ), + text_field( + "Req Enabled Groups", + "enabled_groups", + "", + "Comma-separated defaults.requests.enabled_groups", + ), + text_field( + "Req Disabled Groups", + "disabled_groups", + "", + "Comma-separated defaults.requests.disabled_groups", + ), + text_field( + "Req Enabled Endp", + "enabled_endpoints", + "", + "Comma-separated defaults.requests.enabled_endpoints", + ), + text_field( + "Req Disabled Endp", + "disabled_endpoints", + "", + "Comma-separated defaults.requests.disabled_endpoints", + ), + text_field( + "Req Enabled Tags", + "enabled_tags", + "", + "Comma-separated defaults.requests.enabled_tags", + ), + text_field( + "Req Disabled Tags", + "disabled_tags", + "", + "Comma-separated defaults.requests.disabled_tags", + ), + bool_field( + "Incl Admin", + "include_admin_endpoints", + false, + "defaults.requests.include_admin_endpoints", + ), + bool_field( + "Incl MCP", + "include_mcp_endpoints", + false, + "defaults.requests.include_mcp_endpoints", + ), + bool_field( + "Incl Resource", + "include_resource_endpoints", + false, + "defaults.requests.include_resource_endpoints", + ), + bool_field( + "Incl Prompt", + "include_prompt_endpoints", + false, + "defaults.requests.include_prompt_endpoints", + ), + bool_field( + "Incl Tool", + "include_tool_endpoints", + false, + "defaults.requests.include_tool_endpoints", + ), + ] +} diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_metadata.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_metadata.rs new file mode 100644 index 0000000000..00d5329932 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_metadata.rs @@ -0,0 +1,352 @@ +pub(crate) fn generator_section(key: &str) -> &'static str { + match key { + "file_stem" | "template_kind" => "Generator", + "suite_name" + | "suite_description" + | "output_root" + | "continue_on_failure" + | "save_intermediate_artifacts" + | "flamegraph_enabled" + | "baseline_run" + | "baseline_rps_drop_pct" + | "baseline_p95_regression_pct" + | "baseline_failure_increase" => "Suite", + "scenario_name" | "scenario_description" | "scenario_type" => "Scenario", + "target_kind" + | "auth_mode" + | "plugins_enabled" + | "expected_mcp_runtime" + | "expected_mcp_runtime_mode" + | "expected_a2a_runtime" + | "scenario_setup_snippet" => "Setup", + "rust_plugins" + | "profiling_image" + | "container_file" + | "image_name" + | "image_tag" + | "rebuild_policy" + | "build_args" + | "scenario_build_snippet" => "Build", + "http_server" + | "runtime_host" + | "transport_type" + | "gunicorn_workers" + | "gunicorn_timeout" + | "gunicorn_graceful_timeout" + | "gunicorn_keep_alive" + | "gunicorn_max_requests" + | "gunicorn_max_requests_jitter" + | "gunicorn_backlog" + | "gunicorn_preload_app" + | "gunicorn_dev_mode" + | "granian_workers" + | "granian_runtime_mode" + | "granian_runtime_threads" + | "granian_blocking_threads" + | "granian_http" + | "granian_loop" + | "granian_task_impl" + | "granian_http1_pipeline_flush" + | "granian_http1_buffer_size" + | "granian_backlog" + | "granian_backpressure" + | "granian_respawn_failed" + | "granian_workers_lifetime" + | "granian_workers_max_rss" + | "granian_dev_mode" + | "granian_log_level" + | "uvicorn_workers" + | "uvicorn_loop" + | "uvicorn_http" + | "uvicorn_backlog" + | "uvicorn_timeout_keep_alive" + | "uvicorn_limit_max_requests" + | "uvicorn_log_level" + | "uvicorn_dev_mode" + | "scenario_runtime_snippet" => "Runtime", + "trust_proxy_auth" + | "disable_access_log" + | "templates_auto_reload" + | "structured_logging_database_enabled" + | "sqlalchemy_echo" + | "gateway_log_level" + | "gateway_environment" + | "scenario_gateway_snippet" => "Gateway", + "target_service" + | "driver" + | "headless" + | "only_summary" + | "html_report" + | "users" + | "spawn_rate" + | "run_time" + | "request_count" + | "load_host" + | "seed" + | "tags" + | "exclude_tags" + | "load_extra_args" + | "load_env" + | "workload_selection" + | "fallback_endpoint" + | "workload_endpoints" + | "scenario_load_snippet" => "Load", + "warmup_seconds" + | "measure_seconds" + | "profile_seconds" + | "cooldown_seconds" + | "scenario_measurement_snippet" => "Measurement", + "enabled_groups" + | "disabled_groups" + | "enabled_endpoints" + | "disabled_endpoints" + | "enabled_tags" + | "disabled_tags" + | "include_admin_endpoints" + | "include_mcp_endpoints" + | "include_resource_endpoints" + | "include_prompt_endpoints" + | "include_tool_endpoints" + | "scenario_requests_snippet" => "Requests", + "profiling_enabled" + | "profiling_tools" + | "profiling_duration_seconds" + | "profiling_required" + | "scenario_profiling_snippet" => "Profiling", + "retry_enabled" + | "max_attempts" + | "capture_logs" + | "save_raw_results" + | "reuse_stack" + | "scenario_execution_snippet" => "Execution", + "defaults_plugins_snippet" | "scenario_plugins_snippet" => "Plugins", + _ => "Other", + } +} + +pub(crate) fn generator_config_path(key: &str) -> &'static str { + match key { + "file_stem" => "output file name", + "template_kind" => "starter preset", + "suite_name" => "suite.name", + "suite_description" => "suite.description", + "output_root" => "suite.output_root", + "continue_on_failure" => "suite.continue_on_failure", + "save_intermediate_artifacts" => "suite.save_intermediate_artifacts", + "flamegraph_enabled" => "suite.flamegraph_enabled", + "baseline_run" => "suite.baseline_run", + "baseline_rps_drop_pct" => "suite.baseline_rps_drop_pct", + "baseline_p95_regression_pct" => "suite.baseline_p95_regression_pct", + "baseline_failure_increase" => "suite.baseline_failure_increase", + "scenario_name" => "scenario.name", + "scenario_description" => "scenario.description", + "scenario_type" => "scenario.scenario_type", + "target_kind" => "defaults.setup.target_kind", + "auth_mode" => "defaults.setup.auth_mode", + "plugins_enabled" => "defaults.setup.plugins_enabled", + "expected_mcp_runtime" => "defaults.setup.expected_mcp_runtime", + "expected_mcp_runtime_mode" => "defaults.setup.expected_mcp_runtime_mode", + "expected_a2a_runtime" => "defaults.setup.expected_a2a_runtime", + "rust_plugins" => "defaults.build.rust_plugins", + "profiling_image" => "defaults.build.profiling_image", + "container_file" => "defaults.build.container_file", + "image_name" => "defaults.build.image_name", + "image_tag" => "defaults.build.image_tag", + "rebuild_policy" => "defaults.build.rebuild_policy", + "build_args" => "defaults.build.args", + "http_server" => "defaults.runtime.http_server", + "runtime_host" => "defaults.runtime.host", + "transport_type" => "defaults.runtime.transport_type", + "gunicorn_workers" => "defaults.runtime.gunicorn.workers", + "gunicorn_timeout" => "defaults.runtime.gunicorn.timeout", + "gunicorn_graceful_timeout" => "defaults.runtime.gunicorn.graceful_timeout", + "gunicorn_keep_alive" => "defaults.runtime.gunicorn.keep_alive", + "gunicorn_max_requests" => "defaults.runtime.gunicorn.max_requests", + "gunicorn_max_requests_jitter" => "defaults.runtime.gunicorn.max_requests_jitter", + "gunicorn_backlog" => "defaults.runtime.gunicorn.backlog", + "gunicorn_preload_app" => "defaults.runtime.gunicorn.preload_app", + "gunicorn_dev_mode" => "defaults.runtime.gunicorn.dev_mode", + "granian_workers" => "defaults.runtime.granian.workers", + "granian_runtime_mode" => "defaults.runtime.granian.runtime_mode", + "granian_runtime_threads" => "defaults.runtime.granian.runtime_threads", + "granian_blocking_threads" => "defaults.runtime.granian.blocking_threads", + "granian_http" => "defaults.runtime.granian.http", + "granian_loop" => "defaults.runtime.granian.loop", + "granian_task_impl" => "defaults.runtime.granian.task_impl", + "granian_http1_pipeline_flush" => "defaults.runtime.granian.http1_pipeline_flush", + "granian_http1_buffer_size" => "defaults.runtime.granian.http1_buffer_size", + "granian_backlog" => "defaults.runtime.granian.backlog", + "granian_backpressure" => "defaults.runtime.granian.backpressure", + "granian_respawn_failed" => "defaults.runtime.granian.respawn_failed", + "granian_workers_lifetime" => "defaults.runtime.granian.workers_lifetime", + "granian_workers_max_rss" => "defaults.runtime.granian.workers_max_rss", + "granian_dev_mode" => "defaults.runtime.granian.dev_mode", + "granian_log_level" => "defaults.runtime.granian.log_level", + "uvicorn_workers" => "defaults.runtime.uvicorn.workers", + "uvicorn_loop" => "defaults.runtime.uvicorn.loop", + "uvicorn_http" => "defaults.runtime.uvicorn.http", + "uvicorn_backlog" => "defaults.runtime.uvicorn.backlog", + "uvicorn_timeout_keep_alive" => "defaults.runtime.uvicorn.timeout_keep_alive", + "uvicorn_limit_max_requests" => "defaults.runtime.uvicorn.limit_max_requests", + "uvicorn_log_level" => "defaults.runtime.uvicorn.log_level", + "uvicorn_dev_mode" => "defaults.runtime.uvicorn.dev_mode", + "trust_proxy_auth" => "defaults.gateway.trust_proxy_auth", + "disable_access_log" => "defaults.gateway.disable_access_log", + "templates_auto_reload" => "defaults.gateway.templates_auto_reload", + "structured_logging_database_enabled" => { + "defaults.gateway.structured_logging_database_enabled" + } + "sqlalchemy_echo" => "defaults.gateway.sqlalchemy_echo", + "gateway_log_level" => "defaults.gateway.log_level", + "gateway_environment" => "defaults.gateway.environment", + "target_service" => "defaults.load.target_service", + "driver" => "defaults.load.driver", + "headless" => "defaults.load.headless", + "only_summary" => "defaults.load.only_summary", + "html_report" => "defaults.load.html_report", + "users" => "defaults.load.users", + "spawn_rate" => "defaults.load.spawn_rate", + "run_time" => "defaults.load.run_time", + "request_count" => "defaults.load.request_count", + "load_host" => "defaults.load.host", + "seed" => "defaults.load.seed", + "tags" => "defaults.load.tags", + "exclude_tags" => "defaults.load.exclude_tags", + "load_extra_args" => "defaults.load.extra_args", + "load_env" => "defaults.load.env", + "workload_selection" => "defaults.load.workload.selection", + "fallback_endpoint" => "defaults.load.workload.fallback_endpoint", + "workload_endpoints" => "defaults.load.workload.endpoints", + "warmup_seconds" => "defaults.measurement.warmup_seconds", + "measure_seconds" => "defaults.measurement.measure_seconds", + "profile_seconds" => "defaults.measurement.profile_seconds", + "cooldown_seconds" => "defaults.measurement.cooldown_seconds", + "enabled_groups" => "defaults.requests.enabled_groups", + "disabled_groups" => "defaults.requests.disabled_groups", + "enabled_endpoints" => "defaults.requests.enabled_endpoints", + "disabled_endpoints" => "defaults.requests.disabled_endpoints", + "enabled_tags" => "defaults.requests.enabled_tags", + "disabled_tags" => "defaults.requests.disabled_tags", + "include_admin_endpoints" => "defaults.requests.include_admin_endpoints", + "include_mcp_endpoints" => "defaults.requests.include_mcp_endpoints", + "include_resource_endpoints" => "defaults.requests.include_resource_endpoints", + "include_prompt_endpoints" => "defaults.requests.include_prompt_endpoints", + "include_tool_endpoints" => "defaults.requests.include_tool_endpoints", + "profiling_enabled" => "defaults.profiling.enabled", + "profiling_tools" => "defaults.profiling.tools", + "profiling_duration_seconds" => "defaults.profiling.duration_seconds", + "profiling_required" => "defaults.profiling.required", + "retry_enabled" => "defaults.execution.retry_enabled", + "max_attempts" => "defaults.execution.max_attempts", + "capture_logs" => "defaults.execution.capture_logs", + "save_raw_results" => "defaults.execution.save_raw_results", + "reuse_stack" => "defaults.execution.reuse_stack", + "defaults_plugins_snippet" => "defaults.plugins.", + "scenario_setup_snippet" => "scenario.setup", + "scenario_build_snippet" => "scenario.build", + "scenario_runtime_snippet" => "scenario.runtime", + "scenario_gateway_snippet" => "scenario.gateway", + "scenario_load_snippet" => "scenario.load", + "scenario_measurement_snippet" => "scenario.measurement", + "scenario_requests_snippet" => "scenario.requests", + "scenario_profiling_snippet" => "scenario.profiling", + "scenario_execution_snippet" => "scenario.execution", + "scenario_plugins_snippet" => "scenario.plugins.", + _ => "custom", + } +} + +pub(crate) fn generator_format_hint(key: &str) -> &'static str { + match key { + "template_kind" => "blank, mcp, or a2a", + "target_kind" => "gateway or agent", + "auth_mode" => "jwt, basic, or none", + "rebuild_policy" => "never, missing, or always", + "http_server" => "gunicorn, granian, or uvicorn", + "transport_type" => "streamablehttp, sse, or websocket", + "target_service" => "nginx or gateway", + "continue_on_failure" + | "save_intermediate_artifacts" + | "flamegraph_enabled" + | "plugins_enabled" + | "rust_plugins" + | "profiling_image" + | "gunicorn_preload_app" + | "gunicorn_dev_mode" + | "granian_http1_pipeline_flush" + | "granian_respawn_failed" + | "granian_dev_mode" + | "trust_proxy_auth" + | "disable_access_log" + | "templates_auto_reload" + | "structured_logging_database_enabled" + | "sqlalchemy_echo" + | "headless" + | "only_summary" + | "html_report" + | "include_admin_endpoints" + | "include_mcp_endpoints" + | "include_resource_endpoints" + | "include_prompt_endpoints" + | "include_tool_endpoints" + | "profiling_enabled" + | "profiling_required" + | "retry_enabled" + | "capture_logs" + | "save_raw_results" + | "reuse_stack" + | "uvicorn_dev_mode" => "true or false", + "tags" | "exclude_tags" | "enabled_groups" | "disabled_groups" | "enabled_endpoints" + | "disabled_endpoints" | "enabled_tags" | "disabled_tags" | "profiling_tools" + | "load_extra_args" => "comma-separated list", + "build_args" + | "gateway_environment" + | "load_env" + | "workload_endpoints" + | "defaults_plugins_snippet" + | "scenario_setup_snippet" + | "scenario_build_snippet" + | "scenario_runtime_snippet" + | "scenario_gateway_snippet" + | "scenario_load_snippet" + | "scenario_measurement_snippet" + | "scenario_requests_snippet" + | "scenario_profiling_snippet" + | "scenario_execution_snippet" + | "scenario_plugins_snippet" => "raw TOML lines separated by ' | '", + "users" + | "spawn_rate" + | "warmup_seconds" + | "measure_seconds" + | "profile_seconds" + | "cooldown_seconds" + | "max_attempts" + | "gunicorn_workers" + | "gunicorn_timeout" + | "gunicorn_graceful_timeout" + | "gunicorn_keep_alive" + | "gunicorn_max_requests" + | "gunicorn_max_requests_jitter" + | "gunicorn_backlog" + | "granian_workers" + | "granian_runtime_threads" + | "granian_blocking_threads" + | "granian_http1_buffer_size" + | "granian_backlog" + | "granian_backpressure" + | "granian_workers_lifetime" + | "granian_workers_max_rss" + | "uvicorn_workers" + | "uvicorn_backlog" + | "uvicorn_timeout_keep_alive" + | "uvicorn_limit_max_requests" + | "request_count" + | "profiling_duration_seconds" => "integer number", + "baseline_rps_drop_pct" | "baseline_p95_regression_pct" | "baseline_failure_increase" => { + "numeric threshold" + } + "run_time" => "duration like 180s or 5m", + "file_stem" => "filename stem without .toml", + _ => "plain text", + } +} diff --git a/crates/contextforge_benchmark_console/src/main_parts/generator_state.rs b/crates/contextforge_benchmark_console/src/main_parts/generator_state.rs new file mode 100644 index 0000000000..f051a46df4 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/generator_state.rs @@ -0,0 +1,201 @@ +use crate::main_parts::{ + generator_fields_execution, generator_fields_runtime, generator_fields_suite, + generator_fields_workload, generator_section, +}; + +#[derive(Clone, Copy)] +pub(crate) enum GeneratorFieldKind { + Text, + Bool, + Choice(&'static [&'static str]), +} + +pub(crate) struct GeneratorField { + pub(crate) label: &'static str, + pub(crate) key: &'static str, + pub(crate) kind: GeneratorFieldKind, + pub(crate) value: String, + pub(crate) help: &'static str, +} + +pub(crate) struct GeneratorState { + pub(crate) fields: Vec, + pub(crate) selected: usize, + pub(crate) selected_section: usize, +} + +impl GeneratorState { + pub(crate) fn new() -> Self { + let mut fields = Vec::new(); + fields.extend(generator_fields_suite()); + fields.extend(generator_fields_runtime()); + fields.extend(generator_fields_workload()); + fields.extend(generator_fields_execution()); + Self { + fields, + selected: 0, + selected_section: 0, + } + } + + pub(crate) fn sections() -> &'static [&'static str] { + &[ + "All", + "Generator", + "Suite", + "Scenario", + "Setup", + "Build", + "Runtime", + "Gateway", + "Load", + "Measurement", + "Requests", + "Profiling", + "Execution", + "Plugins", + ] + } + + pub(crate) fn selected_section_name(&self) -> &'static str { + Self::sections()[self.selected_section] + } + + pub(crate) fn visible_indices(&self) -> Vec { + self.fields + .iter() + .enumerate() + .filter_map(|(index, field)| { + let in_section = self.selected_section_name() == "All" + || generator_section(field.key) == self.selected_section_name(); + (in_section && self.is_visible(field.key)).then_some(index) + }) + .collect() + } + + pub(crate) fn ensure_visible_selection(&mut self) { + let visible = self.visible_indices(); + if visible.is_empty() { + self.selected = 0; + return; + } + if visible.contains(&self.selected) { + return; + } + self.selected = *visible + .iter() + .find(|index| **index > self.selected) + .unwrap_or(&visible[0]); + } + + pub(crate) fn selected_field(&self) -> &GeneratorField { + &self.fields[self.selected] + } + + pub(crate) fn selected_field_mut(&mut self) -> &mut GeneratorField { + &mut self.fields[self.selected] + } + + pub(crate) fn move_selected(&mut self, delta: isize) { + let visible = self.visible_indices(); + if visible.is_empty() { + return; + } + let current_pos = visible + .iter() + .position(|index| *index == self.selected) + .unwrap_or(0) as isize; + let len = visible.len() as isize; + let next_pos = (current_pos + delta).rem_euclid(len) as usize; + self.selected = visible[next_pos]; + } + + pub(crate) fn move_section(&mut self, delta: isize) { + let len = Self::sections().len() as isize; + self.selected_section = (self.selected_section as isize + delta).rem_euclid(len) as usize; + self.ensure_visible_selection(); + } + + pub(crate) fn get(&self, key: &str) -> &str { + self.fields + .iter() + .find(|field| field.key == key) + .map(|field| field.value.as_str()) + .unwrap_or("") + } + + pub(crate) fn toggle_or_cycle(&mut self) { + let field = self.selected_field_mut(); + match field.kind { + GeneratorFieldKind::Bool => { + field.value = if field.value == "true" { + "false" + } else { + "true" + } + .to_string(); + } + GeneratorFieldKind::Choice(options) => { + let current = options + .iter() + .position(|value| *value == field.value) + .unwrap_or(0); + field.value = options[(current + 1) % options.len()].to_string(); + } + GeneratorFieldKind::Text => {} + } + self.ensure_visible_selection(); + } + + pub(crate) fn is_visible(&self, key: &str) -> bool { + let http_server = self.get("http_server"); + let profiling_enabled = self.get("profiling_enabled") == "true"; + let plugins_enabled = self.get("plugins_enabled") == "true"; + let workload_selection_present = !self.get("workload_selection").trim().is_empty() + || self.get("template_kind") != "blank"; + + match key { + "expected_mcp_runtime_mode" => !self.get("expected_mcp_runtime").trim().is_empty(), + "gunicorn_workers" + | "gunicorn_timeout" + | "gunicorn_graceful_timeout" + | "gunicorn_keep_alive" + | "gunicorn_max_requests" + | "gunicorn_max_requests_jitter" + | "gunicorn_backlog" + | "gunicorn_preload_app" + | "gunicorn_dev_mode" => http_server == "gunicorn", + "granian_workers" + | "granian_runtime_mode" + | "granian_runtime_threads" + | "granian_blocking_threads" + | "granian_http" + | "granian_loop" + | "granian_task_impl" + | "granian_http1_pipeline_flush" + | "granian_http1_buffer_size" + | "granian_backlog" + | "granian_backpressure" + | "granian_respawn_failed" + | "granian_workers_lifetime" + | "granian_workers_max_rss" + | "granian_dev_mode" + | "granian_log_level" => http_server == "granian", + "uvicorn_workers" + | "uvicorn_loop" + | "uvicorn_http" + | "uvicorn_backlog" + | "uvicorn_timeout_keep_alive" + | "uvicorn_limit_max_requests" + | "uvicorn_log_level" + | "uvicorn_dev_mode" => http_server == "uvicorn", + "profiling_tools" | "profiling_duration_seconds" | "profiling_required" => { + profiling_enabled + } + "defaults_plugins_snippet" | "scenario_plugins_snippet" => plugins_enabled, + "workload_selection" | "fallback_endpoint" => true, + "workload_endpoints" => workload_selection_present, + _ => true, + } + } +} diff --git a/crates/contextforge_benchmark_console/src/main_parts/interaction.rs b/crates/contextforge_benchmark_console/src/main_parts/interaction.rs new file mode 100644 index 0000000000..a04a9ae22c --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/interaction.rs @@ -0,0 +1,200 @@ +pub(crate) fn setup_terminal() -> AppResult>> { + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen, Hide)?; + Ok(Terminal::new(CrosstermBackend::new(stdout))?) +} + +pub(crate) fn restore_terminal(terminal: &mut Terminal>) -> AppResult<()> { + disable_raw_mode()?; + execute!(terminal.backend_mut(), Show, LeaveAlternateScreen)?; + terminal.show_cursor()?; + Ok(()) +} + +pub(crate) fn run_app( + terminal: &mut Terminal>, + mut app: App, + root: &Path, +) -> AppResult<()> { + while !app.should_quit { + drain_running_command(&mut app)?; + terminal.draw(|frame| draw(frame, &app))?; + if event::poll(Duration::from_millis(100))? { + if let Event::Key(key) = event::read()? { + handle_key_event(&mut app, key, root, terminal)?; + } + } + } + Ok(()) +} + +pub(crate) fn handle_key_event( + app: &mut App, + key: KeyEvent, + root: &Path, + _terminal: &mut Terminal>, +) -> AppResult<()> { + match app.mode { + InputMode::Normal => handle_normal_mode(app, key, root), + InputMode::EditRunPath => handle_text_input(app, key, InputMode::EditRunPath), + InputMode::EditExtraArgs => handle_text_input(app, key, InputMode::EditExtraArgs), + InputMode::EditGeneratorField => handle_text_input(app, key, InputMode::EditGeneratorField), + } +} + +pub(crate) fn handle_normal_mode(app: &mut App, key: KeyEvent, root: &Path) -> AppResult<()> { + if app.active_view == AppView::Generator { + return handle_generate_mode(app, key, root); + } + + match key.code { + KeyCode::Char('q') | KeyCode::Esc => app.should_quit = true, + KeyCode::Tab => app.cycle_view(1), + KeyCode::BackTab => app.cycle_view(-1), + KeyCode::Left => app.move_action(-1), + KeyCode::Right => app.move_action(1), + KeyCode::Up | KeyCode::Char('k') if app.active_view.supports_suite_navigation() => { + app.move_scenario(-1) + } + KeyCode::Down | KeyCode::Char('j') if app.active_view.supports_suite_navigation() => { + app.move_scenario(1) + } + KeyCode::Char('1') => app.set_action_index(0), + KeyCode::Char('2') => app.set_action_index(1), + KeyCode::Char('3') => app.set_action_index(2), + KeyCode::Char('4') => app.set_action_index(3), + KeyCode::Char('5') => app.set_action_index(4), + KeyCode::Char('6') => app.set_action_index(5), + KeyCode::Char('7') => app.set_action_index(6), + KeyCode::Char('8') => app.set_action_index(7), + KeyCode::Char('i') => app.set_view(AppView::SuiteInspector), + KeyCode::Char('l') => app.set_view(AppView::Launcher), + KeyCode::Char('m') => app.set_view(AppView::RunMonitor), + KeyCode::PageUp | KeyCode::Char('[') if app.active_view == AppView::RunMonitor => { + app.log_scroll = app + .log_scroll + .saturating_add(10) + .min(app.log_lines.len().saturating_sub(1)); + app.status = format!("Log scroll offset: {}", app.log_scroll); + } + KeyCode::PageDown | KeyCode::Char(']') if app.active_view == AppView::RunMonitor => { + app.log_scroll = app.log_scroll.saturating_sub(10); + app.status = format!("Log scroll offset: {}", app.log_scroll); + } + KeyCode::Char('a') => { + if app.action().supports_all() { + app.all = !app.all; + app.status = format!("Run all scenarios: {}", yes_no(app.all)); + } else { + app.status = "This action does not support all-scenario mode.".to_string(); + } + } + KeyCode::Char('c') => { + if app.action().supports_clean() { + app.clean = !app.clean; + app.status = format!("Clean before launch: {}", yes_no(app.clean)); + } else { + app.status = "This action does not use cleanup.".to_string(); + } + } + KeyCode::Char('p') => { + if app.action().needs_run_path() { + app.mode = InputMode::EditRunPath; + app.status = + "Editing run path. Type, Backspace to delete, Enter to finish.".to_string(); + } else { + app.status = "Run path is only used for Report and Compare.".to_string(); + } + } + KeyCode::Char('e') => { + app.mode = InputMode::EditExtraArgs; + app.status = + "Editing extra args. Type, Backspace to delete, Enter to finish.".to_string(); + } + KeyCode::Enter | KeyCode::Char('r') => launch_action(app, root)?, + _ => {} + } + Ok(()) +} + +pub(crate) fn handle_generate_mode(app: &mut App, key: KeyEvent, root: &Path) -> AppResult<()> { + match key.code { + KeyCode::Char('q') | KeyCode::Esc => app.should_quit = true, + KeyCode::Tab => app.cycle_view(1), + KeyCode::BackTab => app.cycle_view(-1), + KeyCode::Left => app.move_action(-1), + KeyCode::Right => app.move_action(1), + KeyCode::Char('[') | KeyCode::PageUp => { + app.generator.move_section(-1); + app.status = format!("Section: {}", app.generator.selected_section_name()); + } + KeyCode::Char(']') | KeyCode::PageDown => { + app.generator.move_section(1); + app.status = format!("Section: {}", app.generator.selected_section_name()); + } + KeyCode::Up | KeyCode::Char('k') => app.generator.move_selected(-1), + KeyCode::Down | KeyCode::Char('j') => app.generator.move_selected(1), + KeyCode::Char('1') => app.set_action_index(0), + KeyCode::Char('2') => app.set_action_index(1), + KeyCode::Char('3') => app.set_action_index(2), + KeyCode::Char('4') => app.set_action_index(3), + KeyCode::Char('5') => app.set_action_index(4), + KeyCode::Char('6') => app.set_action_index(5), + KeyCode::Char('7') => app.set_action_index(6), + KeyCode::Char('8') => app.set_action_index(7), + KeyCode::Char('t') => { + app.generator.toggle_or_cycle(); + app.status = format!("Updated {}", app.generator.selected_field().label); + } + KeyCode::Enter | KeyCode::Char('e') => match app.generator.selected_field().kind { + GeneratorFieldKind::Text => { + app.mode = InputMode::EditGeneratorField; + app.status = format!("Editing {}", app.generator.selected_field().label); + } + GeneratorFieldKind::Bool | GeneratorFieldKind::Choice(_) => { + app.generator.toggle_or_cycle(); + app.status = format!("Updated {}", app.generator.selected_field().label); + } + }, + KeyCode::Char('g') | KeyCode::Char('s') => { + let path = save_generated_template(root, &mut app.scenarios, &app.generator)?; + app.status = format!("Saved scenario template to {}", path.display()); + } + _ => {} + } + Ok(()) +} + +pub(crate) fn handle_text_input(app: &mut App, key: KeyEvent, mode: InputMode) -> AppResult<()> { + let buffer: &mut String = match mode { + InputMode::EditRunPath => &mut app.run_path, + InputMode::EditExtraArgs => &mut app.extra_args, + InputMode::EditGeneratorField => &mut app.generator.selected_field_mut().value, + InputMode::Normal => return Ok(()), + }; + + match key.code { + KeyCode::Esc => { + app.mode = InputMode::Normal; + app.status = "Cancelled edit.".to_string(); + } + KeyCode::Enter => { + app.mode = InputMode::Normal; + if mode == InputMode::EditGeneratorField { + app.generator.ensure_visible_selection(); + } + app.status = "Saved input.".to_string(); + } + KeyCode::Backspace => { + buffer.pop(); + } + KeyCode::Char(c) => { + buffer.push(c); + } + _ => {} + } + Ok(()) +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/mod.rs b/crates/contextforge_benchmark_console/src/main_parts/mod.rs new file mode 100644 index 0000000000..0406e179d3 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/mod.rs @@ -0,0 +1,58 @@ +pub(crate) mod app_state; +pub(crate) mod generator_copy; +pub(crate) mod generator_examples; +pub(crate) mod generator_fields_execution; +pub(crate) mod generator_fields_runtime; +pub(crate) mod generator_fields_suite; +pub(crate) mod generator_fields_workload; +pub(crate) mod generator_metadata; +pub(crate) mod generator_state; +pub(crate) mod interaction; +pub(crate) mod run_actions; +pub(crate) mod run_cleanup; +pub(crate) mod runtime_helpers; +pub(crate) mod template_endpoints; +pub(crate) mod template_helpers; +pub(crate) mod template_writer; +pub(crate) mod ui_layout; +pub(crate) mod ui_panels; +pub(crate) mod view_models; + +pub(crate) use app_state::App; +pub(crate) use generator_copy::{generator_change_reason, generator_explanation}; +pub(crate) use generator_examples::{generator_example, generator_visibility_note}; +pub(crate) use generator_fields_execution::generator_fields_execution; +pub(crate) use generator_fields_runtime::generator_fields_runtime; +pub(crate) use generator_fields_suite::{ + bool_field, choice_field, generator_fields_suite, text_field, +}; +pub(crate) use generator_fields_workload::generator_fields_workload; +pub(crate) use generator_metadata::{ + generator_config_path, generator_format_hint, generator_section, +}; +pub(crate) use generator_state::{GeneratorField, GeneratorFieldKind, GeneratorState}; +#[cfg(test)] +pub(crate) use interaction::handle_normal_mode; +pub(crate) use interaction::{restore_terminal, run_app, setup_terminal}; +pub(crate) use run_actions::{CommandSpec, build_command, launch_action}; +pub(crate) use run_cleanup::run_cleanup; +pub(crate) use runtime_helpers::{ + drain_running_command, escape_toml, format_command, parse_run_dir, parse_run_outcome, + parse_scenario_completion, parse_scenario_start, start_command_capture, yes_no, +}; +pub(crate) use template_endpoints::template_endpoints; +pub(crate) use template_helpers::{ + append_optional_block, append_runtime_block_from_fields, parse_pipe_lines, push_bool_line, + push_optional_array_line, push_optional_scalar_line, push_optional_string_line, + push_scalar_line, push_string_line, quoted_csv, save_generated_template, +}; +pub(crate) use template_writer::generate_template_toml; +pub(crate) use ui_layout::draw; +pub(crate) use ui_panels::{ + draw_generator_fields, draw_generator_reference, draw_generator_selection, draw_help, + draw_live_logs, draw_preview, line_pair, +}; +pub(crate) use view_models::{ + build_generator_focus_summary, build_preview_sections, build_selection_summary, + build_suite_inspector_summary, discover_scenarios, +}; diff --git a/crates/contextforge_benchmark_console/src/main_parts/run_actions.rs b/crates/contextforge_benchmark_console/src/main_parts/run_actions.rs new file mode 100644 index 0000000000..0ca0006be7 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/run_actions.rs @@ -0,0 +1,98 @@ +use std::env; +use std::path::Path; + +use crate::{Action, AppResult, LogSource}; + +pub(crate) fn launch_action(app: &mut App, root: &Path) -> AppResult<()> { + let command_spec = build_command(app, root)?; + if app.running_command.is_some() { + app.status = "A benchmark command is already running.".to_string(); + return Ok(()); + } + if app.clean && app.action().supports_clean() { + app.push_log_line( + LogSource::System, + "Cleanup: removing prior benchmark containers and staging artifacts.".to_string(), + ); + let cleanup_status = run_cleanup()?; + app.push_log_line( + LogSource::System, + format!("Cleanup finished with status: {cleanup_status}"), + ); + } + start_command_capture(app, command_spec, root)?; + Ok(()) +} + +pub(crate) struct CommandSpec { + pub(crate) command: String, + pub(crate) args: Vec, + pub(crate) env: Vec<(String, String)>, +} + +pub(crate) fn build_command(app: &App, _root: &Path) -> AppResult { + let action = app.action(); + let mut args = vec![ + "cargo".to_string(), + "run".to_string(), + "--manifest-path".to_string(), + "crates/contextforge_benchmark_runner/Cargo.toml".to_string(), + "--quiet".to_string(), + "--".to_string(), + ]; + + match action { + Action::List => args.push("list".to_string()), + Action::Run | Action::Validate | Action::Smoke | Action::CheckRuntime => { + let uses_run_all = app.all && matches!(action, Action::Run | Action::Smoke); + args.push(match action { + Action::Run | Action::Smoke if uses_run_all => "run-all".to_string(), + Action::Run | Action::Smoke => "run".to_string(), + Action::Validate => "validate".to_string(), + Action::CheckRuntime => "check-runtime".to_string(), + _ => unreachable!(), + }); + if !uses_run_all { + args.push("--scenario".to_string()); + args.push(app.scenario().to_string()); + } + if action == Action::Smoke { + args.push("--smoke".to_string()); + } + } + Action::Report => { + if app.run_path.trim().is_empty() { + return Err("Report needs a run path. Press 'p' to edit it.".into()); + } + args.push("regenerate-report".to_string()); + args.push("--run-dir".to_string()); + args.push(app.run_path.trim().to_string()); + } + Action::Compare => { + if app.run_path.trim().is_empty() { + return Err("Compare needs a run path. Press 'p' to edit it.".into()); + } + args.push("compare-run".to_string()); + args.push("--run-dir".to_string()); + args.push(app.run_path.trim().to_string()); + } + Action::Generate => { + return Err("Generate uses 'g' to save a scenario file, not Enter to run.".into()); + } + } + + if !app.extra_args.trim().is_empty() { + args.extend(shlex::split(&app.extra_args).ok_or("Could not parse extra args.")?); + } + + Ok(CommandSpec { + command: args.remove(0), + args, + env: vec![( + "CONTAINER_RUNTIME".to_string(), + env::var("CONTAINER_RUNTIME").unwrap_or_else(|_| "podman".to_string()), + )], + }) +} + +use crate::main_parts::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/run_cleanup.rs b/crates/contextforge_benchmark_console/src/main_parts/run_cleanup.rs new file mode 100644 index 0000000000..adcf4f42cb --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/run_cleanup.rs @@ -0,0 +1,72 @@ +pub(crate) fn run_cleanup() -> AppResult { + let engine = env::var("CONTAINER_RUNTIME").unwrap_or_else(|_| "podman".to_string()); + let chosen_engine = if Command::new(&engine) + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false) + { + engine + } else { + "docker".to_string() + }; + + if chosen_engine == "podman" { + if let Ok(output) = Command::new("podman") + .args(["pod", "ps", "-a", "--format", "{{.Name}}"]) + .output() + { + for pod in String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::trim) + .filter(|name| name.starts_with("bench-")) + { + let _ = Command::new("podman") + .args(["pod", "rm", "-f", pod]) + .status(); + } + } + } + + if let Ok(output) = Command::new(&chosen_engine) + .args(["ps", "-a", "--format", "{{.Names}}"]) + .output() + { + for container in String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::trim) + .filter(|name| name.starts_with("bench-")) + { + let _ = Command::new(&chosen_engine) + .args(["rm", "-f", container]) + .status(); + } + } + + let reports_dir = Path::new("reports/benchmarks"); + if reports_dir.exists() { + for entry in fs::read_dir(reports_dir)? { + let path = entry?.path(); + let name = path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or(""); + if name.starts_with("all-scenarios_") + || name.starts_with("rust-mcp-runtime-300_") + || name.starts_with("a2a-invoke-300_") + || name == "_runtime_staging" + { + if path.is_dir() { + let _ = fs::remove_dir_all(&path); + } else { + let _ = fs::remove_file(&path); + } + } + } + } + + Command::new("true").status().map_err(Into::into) +} +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/runtime_helpers.rs b/crates/contextforge_benchmark_console/src/main_parts/runtime_helpers.rs new file mode 100644 index 0000000000..ac01a164f8 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/runtime_helpers.rs @@ -0,0 +1,162 @@ +pub(crate) fn escape_toml(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +pub(crate) fn format_command(command: &str, args: &[String]) -> String { + std::iter::once(command.to_string()) + .chain(args.iter().cloned()) + .collect::>() + .join(" ") +} + +pub(crate) fn start_command_capture( + app: &mut App, + command_spec: CommandSpec, + root: &Path, +) -> AppResult<()> { + let command_label = format_command(&command_spec.command, &command_spec.args); + let mut child = Command::new(&command_spec.command) + .args(&command_spec.args) + .envs(command_spec.env.clone()) + .current_dir(root) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let stdout = child + .stdout + .take() + .ok_or("Could not capture child stdout")?; + let stderr = child + .stderr + .take() + .ok_or("Could not capture child stderr")?; + let (sender, receiver) = mpsc::channel::(); + spawn_log_reader(stdout, LogSource::Stdout, sender.clone()); + spawn_log_reader(stderr, LogSource::Stderr, sender); + app.run_scenarios.clear(); + app.current_run_scenario = None; + app.last_run_dir = None; + app.last_run_outcome = None; + app.log_lines.clear(); + app.dropped_log_lines = 0; + app.log_scroll = 0; + app.last_command_label = Some(command_label.clone()); + app.push_log_line( + LogSource::System, + format!("Started command inside console: {command_label}"), + ); + app.running_command = Some(RunningCommand { + child, + receiver, + command_label, + }); + app.active_view = AppView::RunMonitor; + Ok(()) +} + +fn spawn_log_reader(reader: R, source: LogSource, sender: mpsc::Sender) +where + R: std::io::Read + Send + 'static, +{ + thread::spawn(move || { + let reader = BufReader::new(reader); + for line in reader.lines() { + match line { + Ok(text) => { + let _ = sender.send(LogLine { source, text }); + } + Err(error) => { + let _ = sender.send(LogLine { + source: LogSource::System, + text: format!("Log capture error: {error}"), + }); + break; + } + } + } + }); +} + +pub(crate) fn drain_running_command(app: &mut App) -> AppResult<()> { + let Some(mut running) = app.running_command.take() else { + return Ok(()); + }; + + while let Ok(line) = running.receiver.try_recv() { + app.push_log_line(line.source, line.text); + } + + match running.child.try_wait()? { + Some(status) => { + while let Ok(line) = running.receiver.try_recv() { + app.push_log_line(line.source, line.text); + } + let outcome = if status.success() { + "finished" + } else { + "failed" + }; + app.push_log_line( + LogSource::System, + format!( + "Command {outcome} with status {status}: {}", + running.command_label + ), + ); + } + None => { + app.running_command = Some(running); + } + } + + Ok(()) +} + +pub(crate) fn parse_scenario_start(text: &str) -> Option { + text.split("starting: ") + .nth(1) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) +} + +pub(crate) fn parse_scenario_completion(text: &str) -> Option<(String, String)> { + let prefix = "Scenario '"; + let rest = text.strip_prefix("[benchmark] ").unwrap_or(text); + let rest = rest.strip_prefix(prefix)?; + if let Some((name, suffix)) = rest.split_once("' completed with status ") { + return Some((name.to_string(), suffix.trim().to_string())); + } + if let Some((name, _)) = rest.split_once("' failed:") { + return Some((name.to_string(), "failed".to_string())); + } + None +} + +pub(crate) fn parse_run_dir(text: &str) -> Option { + if text.contains("reports/benchmarks/") { + return text + .split_whitespace() + .find(|part| part.contains("reports/benchmarks/")) + .map(|value| value.trim().to_string()); + } + None +} + +pub(crate) fn parse_run_outcome(text: &str) -> Option { + let rest = text.strip_prefix("[benchmark] ").unwrap_or(text); + if rest.starts_with("Benchmark run completed successfully") { + return Some("ok".to_string()); + } + if rest.starts_with("Benchmark run completed with failed scenarios") { + return Some("failed".to_string()); + } + None +} + +pub(crate) fn yes_no(value: bool) -> &'static str { + if value { "yes" } else { "no" } +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/template_endpoints.rs b/crates/contextforge_benchmark_console/src/main_parts/template_endpoints.rs new file mode 100644 index 0000000000..3168a0456f --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/template_endpoints.rs @@ -0,0 +1,82 @@ +pub(crate) fn template_endpoints(generator: &GeneratorState) -> String { + let custom = parse_pipe_lines(generator.get("workload_endpoints")); + if !custom.is_empty() { + return format!( + "[defaults.load.workload]\nselection = \"{}\"\nfallback_endpoint = \"{}\"\n\n{}", + escape_toml(generator.get("workload_selection")), + escape_toml(generator.get("fallback_endpoint")), + custom.join("\n") + ); + } + + match generator.get("template_kind") { + "a2a" => format!( + r#"[defaults.load.workload] +selection = "{}" +fallback_endpoint = "{}" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = false + +[defaults.load.workload.endpoints."/a2a"] +enabled = false + +[defaults.load.workload.endpoints."/a2a/a2a-echo-agent/invoke"] +enabled = true +weight = 1 +"#, + generator.get("workload_selection"), + generator.get("fallback_endpoint") + ), + "mcp" => format!( + r#"[defaults.load.workload] +selection = "{}" +fallback_endpoint = "{}" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/ready"] +enabled = false + +[defaults.load.workload.endpoints."/admin/plugins"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = true +weight = 2 + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 6 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 14 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-convert-time"] +enabled = true +weight = 12 +"#, + generator.get("workload_selection"), + generator.get("fallback_endpoint") + ), + _ => format!( + r#"[defaults.load.workload] +# selection = "{}" +fallback_endpoint = "{}" + +# Add endpoint tables as needed: +# [defaults.load.workload.endpoints."/health"] +# enabled = true +# weight = 1 +"#, + generator.get("workload_selection"), + generator.get("fallback_endpoint") + ), + } +} +use crate::main_parts::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/template_helpers.rs b/crates/contextforge_benchmark_console/src/main_parts/template_helpers.rs new file mode 100644 index 0000000000..72a9939127 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/template_helpers.rs @@ -0,0 +1,129 @@ +pub(crate) fn save_generated_template( + root: &Path, + scenarios: &mut Vec, + generator: &GeneratorState, +) -> AppResult { + let file_stem = sanitize_file_stem(generator.get("file_stem")); + let target = root + .join("crates/contextforge_benchmark_runner/assets/scenarios") + .join(format!("{file_stem}.toml")); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&target, generate_template_toml(generator))?; + *scenarios = discover_scenarios(root)?; + Ok(target) +} + +fn sanitize_file_stem(value: &str) -> String { + let mut stem = value + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' { + c + } else { + '-' + } + }) + .collect::() + .trim_matches('-') + .to_string(); + if stem.is_empty() { + stem = "generated-scenario".to_string(); + } + stem +} + +pub(crate) fn parse_pipe_lines(value: &str) -> Vec { + value + .split('|') + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToString::to_string) + .collect() +} + +fn parse_csv_items(value: &str) -> Vec { + value + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(ToString::to_string) + .collect() +} + +pub(crate) fn quoted_csv(value: &str) -> String { + parse_csv_items(value) + .into_iter() + .map(|item| format!("\"{}\"", escape_toml(&item))) + .collect::>() + .join(", ") +} + +pub(crate) fn push_string_line(lines: &mut Vec, key: &str, value: &str) { + lines.push(format!("{key} = \"{}\"", escape_toml(value))); +} + +pub(crate) fn push_bool_line(lines: &mut Vec, key: &str, value: &str) { + lines.push(format!( + "{key} = {}", + if value == "true" { "true" } else { "false" } + )); +} + +pub(crate) fn push_scalar_line(lines: &mut Vec, key: &str, value: &str) { + lines.push(format!("{key} = {value}")); +} + +pub(crate) fn push_optional_string_line(lines: &mut Vec, key: &str, value: &str) { + if !value.trim().is_empty() { + push_string_line(lines, key, value.trim()); + } +} + +pub(crate) fn push_optional_scalar_line(lines: &mut Vec, key: &str, value: &str) { + if !value.trim().is_empty() { + push_scalar_line(lines, key, value.trim()); + } +} + +pub(crate) fn push_optional_array_line(lines: &mut Vec, key: &str, value: &str) { + let items = quoted_csv(value); + if !items.is_empty() { + lines.push(format!("{key} = [{items}]")); + } +} + +pub(crate) fn append_optional_block(lines: &mut Vec, title: &str, raw: &str) { + let entries = parse_pipe_lines(raw); + if !entries.is_empty() { + lines.push(String::new()); + lines.push(title.to_string()); + lines.extend(entries); + } +} + +pub(crate) fn append_runtime_block_from_fields( + lines: &mut Vec, + title: &str, + fields: &[(&str, &str, &str)], +) { + let mut block = Vec::new(); + for (key, value, kind) in fields { + if value.trim().is_empty() { + continue; + } + match *kind { + "bool" => push_bool_line(&mut block, key, value), + "string" => push_string_line(&mut block, key, value.trim()), + _ => push_scalar_line(&mut block, key, value.trim()), + } + } + if !block.is_empty() { + lines.push(String::new()); + lines.push(title.to_string()); + lines.extend(block); + } +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/template_writer.rs b/crates/contextforge_benchmark_console/src/main_parts/template_writer.rs new file mode 100644 index 0000000000..7648dfa19b --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/template_writer.rs @@ -0,0 +1,440 @@ +pub(crate) fn generate_template_toml(generator: &GeneratorState) -> String { + let mut lines = Vec::new(); + + lines.push("[suite]".to_string()); + push_string_line(&mut lines, "name", generator.get("suite_name")); + push_string_line( + &mut lines, + "description", + generator.get("suite_description"), + ); + push_string_line(&mut lines, "output_root", generator.get("output_root")); + push_bool_line( + &mut lines, + "continue_on_failure", + generator.get("continue_on_failure"), + ); + push_bool_line( + &mut lines, + "save_intermediate_artifacts", + generator.get("save_intermediate_artifacts"), + ); + push_bool_line( + &mut lines, + "flamegraph_enabled", + generator.get("flamegraph_enabled"), + ); + push_optional_string_line(&mut lines, "baseline_run", generator.get("baseline_run")); + push_optional_scalar_line( + &mut lines, + "baseline_rps_drop_pct", + generator.get("baseline_rps_drop_pct"), + ); + push_optional_scalar_line( + &mut lines, + "baseline_p95_regression_pct", + generator.get("baseline_p95_regression_pct"), + ); + push_optional_scalar_line( + &mut lines, + "baseline_failure_increase", + generator.get("baseline_failure_increase"), + ); + + lines.push(String::new()); + lines.push("[defaults.setup]".to_string()); + push_string_line(&mut lines, "target_kind", generator.get("target_kind")); + push_string_line(&mut lines, "auth_mode", generator.get("auth_mode")); + push_bool_line( + &mut lines, + "plugins_enabled", + generator.get("plugins_enabled"), + ); + push_optional_string_line( + &mut lines, + "expected_mcp_runtime", + generator.get("expected_mcp_runtime"), + ); + push_optional_string_line( + &mut lines, + "expected_mcp_runtime_mode", + generator.get("expected_mcp_runtime_mode"), + ); + push_optional_string_line( + &mut lines, + "expected_a2a_runtime", + generator.get("expected_a2a_runtime"), + ); + + lines.push(String::new()); + lines.push("[defaults.build]".to_string()); + push_bool_line(&mut lines, "rust_plugins", generator.get("rust_plugins")); + push_bool_line( + &mut lines, + "profiling_image", + generator.get("profiling_image"), + ); + push_string_line( + &mut lines, + "container_file", + generator.get("container_file"), + ); + push_string_line(&mut lines, "image_name", generator.get("image_name")); + push_string_line(&mut lines, "image_tag", generator.get("image_tag")); + push_string_line( + &mut lines, + "rebuild_policy", + generator.get("rebuild_policy"), + ); + append_optional_block( + &mut lines, + "[defaults.build.args]", + generator.get("build_args"), + ); + + lines.push(String::new()); + lines.push("[defaults.runtime]".to_string()); + push_string_line(&mut lines, "http_server", generator.get("http_server")); + push_string_line(&mut lines, "host", generator.get("runtime_host")); + push_string_line( + &mut lines, + "transport_type", + generator.get("transport_type"), + ); + + lines.push(String::new()); + lines.push("[defaults.runtime.gunicorn]".to_string()); + push_scalar_line(&mut lines, "workers", generator.get("gunicorn_workers")); + push_scalar_line(&mut lines, "timeout", generator.get("gunicorn_timeout")); + push_scalar_line( + &mut lines, + "graceful_timeout", + generator.get("gunicorn_graceful_timeout"), + ); + push_scalar_line( + &mut lines, + "keep_alive", + generator.get("gunicorn_keep_alive"), + ); + push_scalar_line( + &mut lines, + "max_requests", + generator.get("gunicorn_max_requests"), + ); + push_scalar_line( + &mut lines, + "max_requests_jitter", + generator.get("gunicorn_max_requests_jitter"), + ); + push_scalar_line(&mut lines, "backlog", generator.get("gunicorn_backlog")); + push_bool_line( + &mut lines, + "preload_app", + generator.get("gunicorn_preload_app"), + ); + push_bool_line(&mut lines, "dev_mode", generator.get("gunicorn_dev_mode")); + append_runtime_block_from_fields( + &mut lines, + "[defaults.runtime.granian]", + &[ + ("workers", generator.get("granian_workers"), "number"), + ( + "runtime_mode", + generator.get("granian_runtime_mode"), + "string", + ), + ( + "runtime_threads", + generator.get("granian_runtime_threads"), + "number", + ), + ( + "blocking_threads", + generator.get("granian_blocking_threads"), + "number", + ), + ("http", generator.get("granian_http"), "number"), + ("loop", generator.get("granian_loop"), "string"), + ("task_impl", generator.get("granian_task_impl"), "string"), + ( + "http1_pipeline_flush", + generator.get("granian_http1_pipeline_flush"), + "bool", + ), + ( + "http1_buffer_size", + generator.get("granian_http1_buffer_size"), + "number", + ), + ("backlog", generator.get("granian_backlog"), "number"), + ( + "backpressure", + generator.get("granian_backpressure"), + "number", + ), + ( + "respawn_failed", + generator.get("granian_respawn_failed"), + "bool", + ), + ( + "workers_lifetime", + generator.get("granian_workers_lifetime"), + "number", + ), + ( + "workers_max_rss", + generator.get("granian_workers_max_rss"), + "number", + ), + ("dev_mode", generator.get("granian_dev_mode"), "bool"), + ("log_level", generator.get("granian_log_level"), "string"), + ], + ); + append_runtime_block_from_fields( + &mut lines, + "[defaults.runtime.uvicorn]", + &[ + ("workers", generator.get("uvicorn_workers"), "number"), + ("loop", generator.get("uvicorn_loop"), "string"), + ("http", generator.get("uvicorn_http"), "string"), + ("backlog", generator.get("uvicorn_backlog"), "number"), + ( + "timeout_keep_alive", + generator.get("uvicorn_timeout_keep_alive"), + "number", + ), + ( + "limit_max_requests", + generator.get("uvicorn_limit_max_requests"), + "number", + ), + ("log_level", generator.get("uvicorn_log_level"), "string"), + ("dev_mode", generator.get("uvicorn_dev_mode"), "bool"), + ], + ); + + lines.push(String::new()); + lines.push("[defaults.gateway]".to_string()); + push_bool_line( + &mut lines, + "trust_proxy_auth", + generator.get("trust_proxy_auth"), + ); + push_bool_line( + &mut lines, + "disable_access_log", + generator.get("disable_access_log"), + ); + push_bool_line( + &mut lines, + "templates_auto_reload", + generator.get("templates_auto_reload"), + ); + push_bool_line( + &mut lines, + "structured_logging_database_enabled", + generator.get("structured_logging_database_enabled"), + ); + push_bool_line( + &mut lines, + "sqlalchemy_echo", + generator.get("sqlalchemy_echo"), + ); + push_string_line(&mut lines, "log_level", generator.get("gateway_log_level")); + append_optional_block( + &mut lines, + "[defaults.gateway.environment]", + generator.get("gateway_environment"), + ); + + lines.push(String::new()); + lines.push("[defaults.load]".to_string()); + push_string_line(&mut lines, "driver", generator.get("driver")); + push_bool_line(&mut lines, "headless", generator.get("headless")); + push_bool_line(&mut lines, "only_summary", generator.get("only_summary")); + push_bool_line(&mut lines, "html_report", generator.get("html_report")); + push_scalar_line(&mut lines, "users", generator.get("users")); + push_scalar_line(&mut lines, "spawn_rate", generator.get("spawn_rate")); + push_string_line(&mut lines, "run_time", generator.get("run_time")); + push_optional_scalar_line(&mut lines, "request_count", generator.get("request_count")); + push_optional_string_line(&mut lines, "host", generator.get("load_host")); + push_optional_string_line(&mut lines, "seed", generator.get("seed")); + push_optional_array_line(&mut lines, "tags", generator.get("tags")); + push_optional_array_line(&mut lines, "exclude_tags", generator.get("exclude_tags")); + push_optional_array_line(&mut lines, "extra_args", generator.get("load_extra_args")); + push_string_line( + &mut lines, + "target_service", + generator.get("target_service"), + ); + append_optional_block(&mut lines, "[defaults.load.env]", generator.get("load_env")); + + lines.push(String::new()); + lines.push(template_endpoints(generator)); + + lines.push(String::new()); + lines.push("[defaults.measurement]".to_string()); + push_scalar_line( + &mut lines, + "warmup_seconds", + generator.get("warmup_seconds"), + ); + push_scalar_line( + &mut lines, + "measure_seconds", + generator.get("measure_seconds"), + ); + push_scalar_line( + &mut lines, + "profile_seconds", + generator.get("profile_seconds"), + ); + push_scalar_line( + &mut lines, + "cooldown_seconds", + generator.get("cooldown_seconds"), + ); + + lines.push(String::new()); + lines.push("[defaults.requests]".to_string()); + push_optional_array_line( + &mut lines, + "enabled_groups", + generator.get("enabled_groups"), + ); + push_optional_array_line( + &mut lines, + "disabled_groups", + generator.get("disabled_groups"), + ); + push_optional_array_line( + &mut lines, + "enabled_endpoints", + generator.get("enabled_endpoints"), + ); + push_optional_array_line( + &mut lines, + "disabled_endpoints", + generator.get("disabled_endpoints"), + ); + push_optional_array_line(&mut lines, "enabled_tags", generator.get("enabled_tags")); + push_optional_array_line(&mut lines, "disabled_tags", generator.get("disabled_tags")); + push_bool_line( + &mut lines, + "include_admin_endpoints", + generator.get("include_admin_endpoints"), + ); + push_bool_line( + &mut lines, + "include_mcp_endpoints", + generator.get("include_mcp_endpoints"), + ); + push_bool_line( + &mut lines, + "include_resource_endpoints", + generator.get("include_resource_endpoints"), + ); + push_bool_line( + &mut lines, + "include_prompt_endpoints", + generator.get("include_prompt_endpoints"), + ); + push_bool_line( + &mut lines, + "include_tool_endpoints", + generator.get("include_tool_endpoints"), + ); + + lines.push(String::new()); + lines.push("[defaults.profiling]".to_string()); + push_bool_line(&mut lines, "enabled", generator.get("profiling_enabled")); + let profiling_tools = quoted_csv(generator.get("profiling_tools")); + lines.push(format!("tools = [{}]", profiling_tools)); + push_scalar_line( + &mut lines, + "duration_seconds", + generator.get("profiling_duration_seconds"), + ); + push_bool_line(&mut lines, "required", generator.get("profiling_required")); + + lines.push(String::new()); + lines.push("[defaults.execution]".to_string()); + push_bool_line(&mut lines, "retry_enabled", generator.get("retry_enabled")); + push_scalar_line(&mut lines, "max_attempts", generator.get("max_attempts")); + push_bool_line(&mut lines, "capture_logs", generator.get("capture_logs")); + push_bool_line( + &mut lines, + "save_raw_results", + generator.get("save_raw_results"), + ); + push_bool_line(&mut lines, "reuse_stack", generator.get("reuse_stack")); + append_optional_block( + &mut lines, + "[defaults.plugins.example-plugin]", + generator.get("defaults_plugins_snippet"), + ); + + lines.push(String::new()); + lines.push("[[scenario]]".to_string()); + push_string_line(&mut lines, "name", generator.get("scenario_name")); + push_string_line( + &mut lines, + "description", + generator.get("scenario_description"), + ); + push_string_line(&mut lines, "scenario_type", generator.get("scenario_type")); + append_optional_block( + &mut lines, + "[scenario.setup]", + generator.get("scenario_setup_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.build]", + generator.get("scenario_build_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.runtime]", + generator.get("scenario_runtime_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.gateway]", + generator.get("scenario_gateway_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.load]", + generator.get("scenario_load_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.measurement]", + generator.get("scenario_measurement_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.requests]", + generator.get("scenario_requests_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.profiling]", + generator.get("scenario_profiling_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.execution]", + generator.get("scenario_execution_snippet"), + ); + append_optional_block( + &mut lines, + "[scenario.plugins.example-plugin]", + generator.get("scenario_plugins_snippet"), + ); + + lines.join("\n") + "\n" +} +use crate::main_parts::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/tests.rs b/crates/contextforge_benchmark_console/src/main_parts/tests.rs new file mode 100644 index 0000000000..98fc28d8fa --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/tests.rs @@ -0,0 +1,483 @@ +#[test] +fn discovers_suite_metadata_with_description() { + let tempdir = std::env::temp_dir().join("benchmark-console-suite-metadata"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(tempdir.join("crates/contextforge_benchmark_runner/assets/scenarios")) + .unwrap(); + std::fs::write( + tempdir.join("crates/contextforge_benchmark_runner/assets/scenarios/example-suite.toml"), + r#" +[suite] +name = "benchmark-example-suite" +description = "Explains what this benchmark covers and what comparison it is meant to answer." +"#, + ) + .unwrap(); + + let suites = discover_scenarios(&tempdir).unwrap(); + assert_eq!(suites.len(), 1); + assert_eq!(suites[0].file_stem, "example-suite"); + assert_eq!(suites[0].suite_name, "benchmark-example-suite"); + assert!(suites[0].description.contains("what this benchmark covers")); + + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn preview_summary_includes_run_plan_execution_and_checks() { + let tempdir = std::env::temp_dir().join("benchmark-console-preview-summary"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(tempdir.join("crates/contextforge_benchmark_runner/assets/scenarios")) + .unwrap(); + std::fs::write( + tempdir.join("crates/contextforge_benchmark_runner/assets/scenarios/example-suite.toml"), + r#" +[suite] +name = "benchmark-example-suite" +description = "Exercises a representative benchmark flow." + +[[scenario]] +name = "baseline-scenario" + +[[scenario]] +name = "variant-scenario" +"#, + ) + .unwrap(); + + let mut app = App::new(discover_scenarios(&tempdir).unwrap()); + app.action_index = 0; + app.clean = true; + app.extra_args = "--smoke-note enabled".to_string(); + + let preview = build_preview_sections(&app, &tempdir).unwrap(); + + assert!( + preview + .run_plan + .iter() + .any(|line| line.contains("benchmark-example-suite")) + ); + assert!( + preview + .run_plan + .iter() + .any(|line| line.contains("2 scenario(s)")) + ); + assert!( + preview + .run_plan + .iter() + .any(|line| line.contains("baseline-scenario vs variant-scenario")) + ); + assert!( + preview + .execution + .iter() + .any(|line| line.contains("cargo run --manifest-path")) + ); + assert!( + preview + .checks + .iter() + .any(|line| line.contains("Clean-first is enabled")) + ); + assert!( + preview + .checks + .iter() + .any(|line| line.contains("Extra args will be appended")) + ); + + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn selection_summary_tracks_toggle_state_and_inputs() { + let mut app = App::new(vec![SuiteSummary { + file_stem: "rest-discovery-300".to_string(), + suite_name: "benchmark-rest-discovery".to_string(), + description: "Exercises discovery endpoints.".to_string(), + }]); + app.all = true; + app.clean = true; + app.extra_args = "--profile brief".to_string(); + + let summary = build_selection_summary(&app); + + assert_eq!(summary.action_label, "Run"); + assert_eq!(summary.suite_label, "rest-discovery-300"); + assert_eq!(summary.run_mode_label, "all-scenarios"); + assert_eq!(summary.clean_label, "yes"); + assert_eq!(summary.extra_args_label, "--profile brief"); +} + +#[test] +fn build_command_uses_run_all_only_for_run_and_smoke() { + let scenarios = vec![SuiteSummary { + file_stem: "rest-discovery-300".to_string(), + suite_name: "benchmark-rest-discovery".to_string(), + description: "Exercises discovery endpoints.".to_string(), + }]; + + let mut run_app = App::new(scenarios.clone()); + run_app.all = true; + let run_command = build_command(&run_app, Path::new(".")).unwrap(); + assert!(run_command.args.iter().any(|arg| arg == "run-all")); + assert_eq!( + run_command + .args + .iter() + .filter(|arg: &&String| arg.as_str() == "--scenario") + .count(), + 0 + ); + + let mut validate_app = App::new(scenarios.clone()); + validate_app.all = true; + validate_app.set_action_index( + Action::ALL + .iter() + .position(|action| *action == Action::Validate) + .unwrap(), + ); + let validate_command = build_command(&validate_app, Path::new(".")).unwrap(); + assert!(validate_command.args.iter().any(|arg| arg == "validate")); + assert_eq!( + validate_command + .args + .iter() + .filter(|arg: &&String| arg.as_str() == "--scenario") + .count(), + 1 + ); + + let mut runtime_app = App::new(scenarios); + runtime_app.all = true; + runtime_app.set_action_index( + Action::ALL + .iter() + .position(|action| *action == Action::CheckRuntime) + .unwrap(), + ); + let runtime_command = build_command(&runtime_app, Path::new(".")).unwrap(); + assert!( + runtime_command + .args + .iter() + .any(|arg| arg == "check-runtime") + ); + assert_eq!( + runtime_command + .args + .iter() + .filter(|arg: &&String| arg.as_str() == "--scenario") + .count(), + 1 + ); +} + +#[test] +fn app_view_switching_tracks_generator_and_monitor_modes() { + let mut app = App::new(Vec::new()); + assert_eq!(app.active_view, AppView::Launcher); + + app.set_view(AppView::SuiteInspector); + assert_eq!(app.active_view, AppView::SuiteInspector); + + app.set_view(AppView::Generator); + assert_eq!(app.active_view, AppView::Generator); + assert_eq!(app.action(), Action::Generate); + + app.set_view(AppView::Launcher); + assert_eq!(app.active_view, AppView::Launcher); + assert_ne!(app.action(), Action::Generate); +} + +#[test] +fn suite_navigation_is_blocked_outside_suite_focused_views() { + let mut app = App::new(vec![ + SuiteSummary { + file_stem: "suite-one".to_string(), + suite_name: "suite-one".to_string(), + description: "First suite".to_string(), + }, + SuiteSummary { + file_stem: "suite-two".to_string(), + suite_name: "suite-two".to_string(), + description: "Second suite".to_string(), + }, + ]); + let root = std::env::temp_dir(); + app.set_view(AppView::RunMonitor); + handle_normal_mode(&mut app, KeyEvent::from(KeyCode::Char('j')), &root).unwrap(); + assert_eq!(app.scenario_index, 0); + + app.set_view(AppView::SuiteInspector); + handle_normal_mode(&mut app, KeyEvent::from(KeyCode::Char('j')), &root).unwrap(); + assert_eq!(app.scenario_index, 1); +} + +#[test] +fn suite_inspector_summary_builds_scenario_cards_with_settings() { + let tempdir = std::env::temp_dir().join("benchmark-console-suite-inspector"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(tempdir.join("crates/contextforge_benchmark_runner/assets/scenarios")) + .unwrap(); + std::fs::write( + tempdir.join("crates/contextforge_benchmark_runner/assets/scenarios/example-suite.toml"), + r#" +[suite] +name = "benchmark-example-suite" +description = "Compare Python and Rust runtime paths." + +[defaults.setup] +plugins_enabled = false + +[defaults.build] +rust_plugins = false + +[[scenario]] +name = "baseline-scenario" +description = "Python baseline" +scenario_type = "baseline" + +[[scenario]] +name = "rust-scenario" +description = "Rust comparison" +scenario_type = "compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +"#, + ) + .unwrap(); + + let app = App::new(discover_scenarios(&tempdir).unwrap()); + let summary = build_suite_inspector_summary(&app, &tempdir).unwrap(); + + assert_eq!(summary.scenario_cards.len(), 2); + assert_eq!(summary.scenario_cards[0].name, "baseline-scenario"); + assert!( + summary.scenario_cards[1] + .settings + .iter() + .any(|(key, value)| key == "expected_mcp_runtime" && value == "rust") + ); + assert!( + summary.scenario_cards[0] + .settings + .iter() + .any(|(key, value)| key == "rust_plugins" && value == "false") + ); + assert!( + summary.scenario_cards[1] + .settings + .iter() + .any(|(key, value)| key == "RUST_MCP_MODE" && value == "edge") + ); + + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn generator_focus_summary_exposes_field_guidance() { + let app = App::new(Vec::new()); + let summary = build_generator_focus_summary(&app); + + assert_eq!(summary.section_filter, "All"); + assert_eq!(summary.field_label, "File Stem"); + assert_eq!(summary.config_key, "output file name"); + assert!(summary.purpose.contains("filename")); + assert!(summary.effect.contains("scenario file")); + assert!(summary.example.contains("a2a-invoke-300")); +} + +#[test] +fn every_generator_field_has_specific_purpose_and_effect_copy() { + let generator = GeneratorState::new(); + for field in &generator.fields { + let purpose = generator_explanation(field.key); + let effect = generator_change_reason(field.key); + assert!( + !purpose.contains("maps directly to the benchmark scenario schema"), + "generic purpose for {}", + field.key + ); + assert_ne!( + purpose, "Defines a generator field.", + "fallback purpose for {}", + field.key + ); + assert!( + !effect.contains("default generated value does not match"), + "generic effect for {}", + field.key + ); + assert_ne!( + effect, "Changing it changes the generated benchmark template.", + "fallback effect for {}", + field.key + ); + } +} + +#[test] +fn generator_template_uses_rust_only_defaults() { + let generator = GeneratorState::new(); + let template = generate_template_toml(&generator); + + assert!(template.contains("driver = ")); + assert!(template.contains("tools = [\"perf\", \"flamegraph\"]")); + assert!(!template.contains("repo_url = ")); + assert!(!template.contains("git_ref = ")); + assert!(!template.contains("git_commit = ")); +} + +#[test] +fn generator_metadata_uses_rust_profiling_field_names() { + assert_eq!(generator_section("driver"), "Load"); + assert_eq!(generator_config_path("driver"), "defaults.load.driver"); + assert!(generator_example("driver").contains("contextforge_goose")); + assert!(generator_example("profiling_tools").contains("perf,flamegraph")); + assert!(generator_example("scenario_profiling_snippet").contains("perf")); +} + +#[test] +fn app_log_buffer_keeps_recent_entries_and_updates_status() { + let mut app = App::new(Vec::new()); + app.push_log_line(LogSource::Stdout, "first line".to_string()); + for index in 0..520 { + app.push_log_line(LogSource::Stdout, format!("line {index}")); + } + + assert_eq!(app.log_lines.len(), MAX_LOG_LINES); + assert!( + app.log_lines + .last() + .map(|line| line.text.as_str()) + .unwrap_or_default() + .contains("line 519") + ); + assert_eq!(app.dropped_log_lines, 21); + assert!(app.status.contains("line 519")); +} + +#[test] +fn progress_parsing_tracks_failed_scenarios_and_run_outcome() { + let mut app = App::new(Vec::new()); + app.push_log_line( + LogSource::System, + "[benchmark] Scenario 2/4 starting: gunicorn-rest-discovery-rust-runtime".to_string(), + ); + app.push_log_line( + LogSource::System, + "[benchmark] Scenario 'gunicorn-rest-discovery-rust-runtime' failed: compose command failed".to_string(), + ); + app.push_log_line( + LogSource::System, + "[benchmark] Benchmark run completed with failed scenarios [gunicorn-rest-discovery-rust-runtime]: reports/benchmarks/example".to_string(), + ); + + assert_eq!(app.current_run_scenario, None); + assert_eq!( + app.run_scenarios, + vec![RunScenarioSummary { + name: "gunicorn-rest-discovery-rust-runtime".to_string(), + status: "failed".to_string(), + }] + ); + assert_eq!(app.last_run_outcome.as_deref(), Some("failed")); + assert_eq!( + app.last_run_dir.as_deref(), + Some("reports/benchmarks/example") + ); +} + +#[test] +fn launcher_command_runs_inside_console_capture() { + let mut app = App::new(vec![SuiteSummary { + file_stem: "rest-discovery-300".to_string(), + suite_name: "benchmark-rest-discovery".to_string(), + description: "Exercises discovery endpoints.".to_string(), + }]); + let command = CommandSpec { + command: "sh".to_string(), + args: vec![ + "-c".to_string(), + "printf 'hello from stdout\\n'; printf 'hello from stderr\\n' >&2".to_string(), + ], + env: vec![], + }; + + start_command_capture(&mut app, command, Path::new(".")).unwrap(); + for _ in 0..40 { + drain_running_command(&mut app).unwrap(); + if app.running_command.is_none() { + break; + } + std::thread::sleep(Duration::from_millis(25)); + } + + assert!(app.running_command.is_none()); + assert_eq!(app.active_view, AppView::RunMonitor); + let combined = app + .log_lines + .iter() + .map(|line| line.text.as_str()) + .collect::>() + .join("\n"); + assert!(combined.contains("hello from stdout")); + assert!(combined.contains("hello from stderr")); + assert!(app.status.contains("finished")); +} + +#[test] +fn start_command_capture_resets_run_monitor_state() { + let mut app = App::new(vec![SuiteSummary { + file_stem: "rest-discovery-300".to_string(), + suite_name: "benchmark-rest-discovery".to_string(), + description: "Exercises discovery endpoints.".to_string(), + }]); + app.log_lines.push(LogLine { + source: LogSource::System, + text: "old log".to_string(), + }); + app.dropped_log_lines = 9; + app.last_run_outcome = Some("failed".to_string()); + app.last_run_dir = Some("reports/benchmarks/old".to_string()); + app.run_scenarios.push(RunScenarioSummary { + name: "old-scenario".to_string(), + status: "failed".to_string(), + }); + + let command = CommandSpec { + command: "sh".to_string(), + args: vec!["-c".to_string(), "printf 'hello\\n'".to_string()], + env: vec![], + }; + + start_command_capture(&mut app, command, Path::new(".")).unwrap(); + + assert_eq!(app.dropped_log_lines, 0); + assert_eq!(app.last_run_outcome, None); + assert_eq!(app.last_run_dir, None); + assert!(app.run_scenarios.is_empty()); + assert_eq!(app.log_scroll, 0); + assert_eq!(app.log_lines.len(), 1); + assert!( + app.log_lines[0] + .text + .contains("Started command inside console") + ); +} +use super::*; +use crate::main_parts::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/ui_layout.rs b/crates/contextforge_benchmark_console/src/main_parts/ui_layout.rs new file mode 100644 index 0000000000..91d49e5b8c --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/ui_layout.rs @@ -0,0 +1,451 @@ +pub(crate) fn draw(frame: &mut ratatui::Frame<'_>, app: &App) { + let chunks = if app.active_view == AppView::Generator { + Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(5), + Constraint::Min(16), + Constraint::Length(4), + ]) + .split(frame.area()) + } else { + Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(14), + Constraint::Length(5), + Constraint::Min(10), + Constraint::Length(4), + ]) + .split(frame.area()) + }; + + let header = Paragraph::new(vec![ + Line::from(Span::styled( + "ContextForge Benchmark Console", + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + )), + Line::from(format!("Mode: {}", app.mode.label())), + ]) + .block(Block::default().borders(Borders::ALL).title("Console")); + frame.render_widget(header, chunks[0]); + + let view_tabs = Tabs::new( + AppView::ALL + .iter() + .map(|view| Line::from(view.label().to_string())) + .collect::>(), + ) + .select( + AppView::ALL + .iter() + .position(|view| *view == app.active_view) + .unwrap_or(0), + ) + .block(Block::default().borders(Borders::ALL).title("Views")) + .highlight_style( + Style::default() + .fg(Color::Black) + .bg(Color::Green) + .add_modifier(Modifier::BOLD), + ); + frame.render_widget(view_tabs, chunks[1]); + + let tabs = Tabs::new( + Action::ALL + .iter() + .enumerate() + .map(|(index, action)| Line::from(format!("{} {}", index + 1, action.label()))) + .collect::>(), + ) + .select(app.action_index) + .block(Block::default().borders(Borders::ALL).title("Actions")) + .highlight_style( + Style::default() + .fg(Color::Black) + .bg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ); + frame.render_widget(tabs, chunks[2]); + + draw_status_banner(frame, chunks[3], app); + + if app.active_view == AppView::Generator { + draw_generator_sections(frame, chunks[2], app); + let body = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(56), Constraint::Percentage(44)]) + .split(chunks[4]); + let left = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Min(10), Constraint::Length(11)]) + .split(body[0]); + draw_generator_fields(frame, left[0], app); + draw_generator_selection(frame, left[1], app); + draw_generator_reference(frame, body[1], app); + } else { + match app.active_view { + AppView::Launcher => draw_launcher_view(frame, chunks[4], app), + AppView::SuiteInspector => draw_suite_inspector_view(frame, chunks[4], app), + AppView::RunMonitor => draw_run_monitor_view(frame, chunks[4], app), + AppView::Generator => {} + } + } + draw_help(frame, chunks[5], app); +} + +fn draw_status_banner(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let selected_suite = app + .selected_suite() + .map(SuiteSummary::label) + .unwrap_or("(none)"); + let status_lines = vec![ + Line::from(vec![ + Span::styled("Action ", Style::default().fg(Color::Gray)), + Span::styled( + app.action().label(), + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ), + Span::raw(" "), + Span::styled("View ", Style::default().fg(Color::Gray)), + Span::styled( + app.active_view.label(), + Style::default() + .fg(Color::Green) + .add_modifier(Modifier::BOLD), + ), + Span::raw(" "), + Span::styled("Suite ", Style::default().fg(Color::Gray)), + Span::styled( + selected_suite, + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ), + ]), + Line::from(vec![ + Span::styled("Status ", Style::default().fg(Color::Gray)), + Span::styled( + app.status.as_str(), + Style::default() + .fg(if app.running_command.is_some() { + Color::Yellow + } else { + Color::Green + }) + .add_modifier(Modifier::BOLD), + ), + ]), + Line::from(vec![ + Span::styled("Live Run ", Style::default().fg(Color::Gray)), + Span::styled( + if app.running_command.is_some() { + "active" + } else { + "idle" + }, + Style::default() + .fg(if app.running_command.is_some() { + Color::LightYellow + } else { + Color::DarkGray + }) + .add_modifier(Modifier::BOLD), + ), + ]), + ]; + let widget = Paragraph::new(status_lines) + .block( + Block::default() + .borders(Borders::ALL) + .title("Operator Status"), + ) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +fn draw_generator_sections(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let tabs = Tabs::new( + GeneratorState::sections() + .iter() + .map(|section| Line::from((*section).to_string())) + .collect::>(), + ) + .select(app.generator.selected_section) + .block( + Block::default() + .borders(Borders::ALL) + .title("Generator Sections"), + ) + .highlight_style( + Style::default() + .fg(Color::Black) + .bg(Color::Green) + .add_modifier(Modifier::BOLD), + ); + frame.render_widget(tabs, area); +} + +fn draw_scenarios(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let items = app + .scenarios + .iter() + .map(|scenario| { + ListItem::new(vec![ + Line::from(Span::styled( + scenario.label().to_string(), + Style::default().add_modifier(Modifier::BOLD), + )), + Line::from(Span::styled( + scenario.suite_name().to_string(), + Style::default().fg(Color::Gray), + )), + ]) + }) + .collect::>(); + let list = List::new(items) + .block( + Block::default() + .borders(Borders::ALL) + .title("Benchmark Suites"), + ) + .highlight_style( + Style::default() + .fg(Color::Black) + .bg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ) + .highlight_symbol(">> ") + .highlight_spacing(ratatui::widgets::HighlightSpacing::Always); + let mut state = ListState::default(); + state.select(Some(app.scenario_index)); + frame.render_stateful_widget(list, area, &mut state); +} + +fn draw_launcher_view(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let body = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(36), Constraint::Percentage(64)]) + .split(area); + draw_scenarios(frame, body[0], app); + let right = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(12), Constraint::Min(10)]) + .split(body[1]); + let top = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(42), Constraint::Percentage(58)]) + .split(right[0]); + draw_selection(frame, top[0], app); + draw_launcher_summary(frame, top[1], app); + draw_preview(frame, right[1], app); +} + +fn draw_suite_inspector_view(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let body = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(8), Constraint::Min(14)]) + .split(area); + draw_inspector_header(frame, body[0], app); + draw_scenario_cards(frame, body[1], app); +} + +fn draw_run_monitor_view(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let body = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(10), Constraint::Min(14)]) + .split(area); + draw_run_monitor_summary(frame, body[0], app); + draw_live_logs(frame, body[1], app); +} + +fn draw_selection(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let summary = build_selection_summary(app); + let lines = vec![ + line_pair("Action", &summary.action_label), + line_pair("Suite", &summary.suite_label), + line_pair("Run Mode", &summary.run_mode_label), + line_pair("Clean First", &summary.clean_label), + line_pair("Run Path", &summary.run_path_label), + line_pair("Extra Args", &summary.extra_args_label), + ]; + let widget = Paragraph::new(lines) + .block( + Block::default() + .borders(Borders::ALL) + .title("Selection State"), + ) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +fn draw_launcher_summary(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let summary = build_suite_inspector_summary(app, Path::new(".")).unwrap_or_default(); + let lines = vec![ + Line::from(Span::styled( + summary.suite_name, + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + )), + Line::from(""), + line_pair("Intent", &summary.suite_description), + line_pair("Comparison Set", &summary.scenario_count_label), + line_pair( + "Inspector", + "Press 'i' or Tab to open full scenario comparison cards", + ), + ]; + let widget = Paragraph::new(lines) + .block( + Block::default() + .borders(Borders::ALL) + .title("Suite Summary"), + ) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +fn draw_inspector_header(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let summary = build_suite_inspector_summary(app, Path::new(".")).unwrap_or_default(); + let lines = vec![ + Line::from(Span::styled( + summary.suite_name, + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + )), + Line::from(summary.suite_description), + Line::from(""), + line_pair("Comparison Set", &summary.scenario_count_label), + line_pair("Question", &summary.comparison_question), + ]; + let widget = Paragraph::new(lines) + .block( + Block::default() + .borders(Borders::ALL) + .title("Suite Inspector"), + ) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +fn draw_scenario_cards(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let summary = build_suite_inspector_summary(app, Path::new(".")).unwrap_or_default(); + if summary.scenario_cards.is_empty() { + let widget = Paragraph::new("No scenarios found for the selected suite.").block( + Block::default() + .borders(Borders::ALL) + .title("Scenario Comparison"), + ); + frame.render_widget(widget, area); + return; + } + + let constraints = vec![ + Constraint::Ratio(1, summary.scenario_cards.len() as u32); + summary.scenario_cards.len() + ]; + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints(constraints) + .split(area); + + for (index, card) in summary.scenario_cards.iter().enumerate() { + let is_active = app.current_run_scenario.as_deref() == Some(card.name.as_str()); + let mut lines = vec![ + Line::from(Span::styled( + card.name.clone(), + Style::default() + .fg(if is_active { + Color::Yellow + } else { + Color::White + }) + .add_modifier(Modifier::BOLD), + )), + Line::from(card.description.clone()), + Line::from(format!("Type: {}", card.scenario_type)), + ]; + if card.settings.is_empty() { + lines.push(Line::from("Settings: inherits suite defaults")); + } else { + lines.push(Line::from("Settings:")); + lines.extend( + card.settings + .iter() + .map(|(key, value)| Line::from(format!(" {} = {}", key, value))), + ); + } + let title = if is_active { + format!("Scenario {} (active)", index + 1) + } else { + format!("Scenario {}", index + 1) + }; + let widget = Paragraph::new(lines) + .block(Block::default().borders(Borders::ALL).title(title)) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, chunks[index]); + } +} + +fn draw_run_monitor_summary(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let selected_suite = app + .selected_suite() + .map(SuiteSummary::suite_name) + .unwrap_or("(none)"); + let current = app.current_run_scenario.as_deref().unwrap_or("(idle)"); + let buffered_logs = app.log_lines.len().to_string(); + let dropped_logs = app.dropped_log_lines.to_string(); + let statuses = if app.run_scenarios.is_empty() { + vec![Line::from("No run scenarios recorded yet.")] + } else { + app.run_scenarios + .iter() + .map(|item| Line::from(format!("{} -> {}", item.name, item.status))) + .collect::>() + }; + let mut lines = vec![ + line_pair("Suite", selected_suite), + line_pair( + "Command", + app.last_command_label + .as_deref() + .unwrap_or("(no command launched)"), + ), + line_pair("Current Scenario", current), + line_pair( + "Run Dir", + app.last_run_dir.as_deref().unwrap_or("(pending)"), + ), + line_pair( + "Outcome", + app.last_run_outcome + .as_deref() + .unwrap_or("(running or pending)"), + ), + line_pair("Buffered Logs", &buffered_logs), + line_pair("Dropped Logs", &dropped_logs), + Line::from(""), + Line::from(Span::styled( + "Scenario Status", + Style::default().add_modifier(Modifier::BOLD), + )), + ]; + lines.extend(statuses); + let widget = Paragraph::new(lines) + .block(Block::default().borders(Borders::ALL).title("Run Monitor")) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/ui_panels.rs b/crates/contextforge_benchmark_console/src/main_parts/ui_panels.rs new file mode 100644 index 0000000000..2ebb086671 --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/ui_panels.rs @@ -0,0 +1,243 @@ +pub(crate) fn draw_preview(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let preview = build_preview_sections(app, Path::new(".")).unwrap_or_else(|error| { + let mut fallback = PreviewSections::default(); + fallback.execution.push(format!( + "Command error: failed to build preview sections: {error}" + )); + fallback + }); + let mut lines = vec![Line::from(Span::styled( + app.action().help(), + Style::default().fg(Color::Cyan), + ))]; + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + "Run Plan", + Style::default().add_modifier(Modifier::BOLD), + ))); + lines.extend(preview.run_plan.iter().map(|line| Line::from(line.clone()))); + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + "Execution", + Style::default().add_modifier(Modifier::BOLD), + ))); + lines.extend(preview.execution.iter().map(|line| { + if line.starts_with("Command error:") { + Line::from(Span::styled(line.clone(), Style::default().fg(Color::Red))) + } else { + Line::from(line.clone()) + } + })); + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + "Checks", + Style::default().add_modifier(Modifier::BOLD), + ))); + lines.extend(preview.checks.iter().map(|line| Line::from(line.clone()))); + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + format!("Status: {}", app.status), + Style::default().fg(Color::Magenta), + ))); + let widget = Paragraph::new(lines) + .block( + Block::default() + .borders(Borders::ALL) + .title("Execution Dashboard"), + ) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +pub(crate) fn draw_live_logs(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let visible_height = area.height.saturating_sub(2) as usize; + let total = app.log_lines.len(); + let end = total.saturating_sub(app.log_scroll); + let start = end.saturating_sub(visible_height); + let lines = app.log_lines[start..end] + .iter() + .map(|line| { + let prefix = match line.source { + LogSource::Stdout => ("OUT", Color::Green), + LogSource::Stderr => ("ERR", Color::Red), + LogSource::System => ("SYS", Color::Cyan), + }; + Line::from(vec![ + Span::styled( + format!("[{}] ", prefix.0), + Style::default().fg(prefix.1).add_modifier(Modifier::BOLD), + ), + Span::raw(line.text.clone()), + ]) + }) + .collect::>(); + let empty = vec![Line::from(Span::styled( + "Run a benchmark action to see live logs here.", + Style::default().fg(Color::DarkGray), + ))]; + let widget = Paragraph::new(if lines.is_empty() { empty } else { lines }) + .block(Block::default().borders(Borders::ALL).title(format!( + "Live Logs ({}/{}, scroll {}, dropped {})", + end.saturating_sub(start), + total, + app.log_scroll, + app.dropped_log_lines + ))) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +pub(crate) fn draw_generator_fields(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let visible = app.generator.visible_indices(); + let items = visible + .iter() + .map(|index| { + let field = &app.generator.fields[*index]; + ListItem::new(vec![ + Line::from(vec![ + Span::styled( + format!("{}{}", generator_indent(field.key), field.label), + Style::default().add_modifier(Modifier::BOLD), + ), + Span::raw(" "), + Span::styled( + generator_section(field.key), + Style::default().fg(Color::Blue), + ), + ]), + Line::from(Span::styled( + field.value.clone(), + Style::default().fg(Color::Green), + )), + ]) + }) + .collect::>(); + let visible_pos = visible + .iter() + .position(|index| *index == app.generator.selected) + .unwrap_or(0); + let list = List::new(items) + .block(Block::default().borders(Borders::ALL).title(format!( + "{} Fields ({}/{} visible, {} total)", + app.generator.selected_section_name(), + visible_pos + 1, + visible.len(), + app.generator.fields.len() + ))) + .highlight_style( + Style::default() + .fg(Color::Black) + .bg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ) + .highlight_symbol(">> ") + .highlight_spacing(ratatui::widgets::HighlightSpacing::Always); + let mut state = ListState::default(); + state.select(Some(visible_pos)); + frame.render_stateful_widget(list, area, &mut state); +} + +pub(crate) fn draw_generator_selection(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let summary = build_generator_focus_summary(app); + let lines = vec![ + line_pair("Section Filter", &summary.section_filter), + line_pair("Field", &summary.field_label), + line_pair("Config Key", &summary.config_key), + line_pair("Value", &summary.value), + line_pair("Kind", &summary.kind), + line_pair("Schema", &summary.schema), + line_pair("Format", &summary.format_hint), + line_pair("Visibility", &summary.visibility), + line_pair("Edit", "Enter/e edits, t toggles bool or choice"), + line_pair("Save", "g or s writes the scenario file"), + ]; + let widget = Paragraph::new(lines) + .block( + Block::default() + .borders(Borders::ALL) + .title("Current Field"), + ) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +pub(crate) fn draw_generator_reference(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let summary = build_generator_focus_summary(app); + let detail = format!( + "What it is for:\n{}\n\nWhat it does:\n{}\n\nAccepted values:\n{}\n\nVisibility:\n{}\n\nExample:\n{}", + summary.purpose, summary.effect, summary.format_hint, summary.visibility, summary.example + ); + let widget = Paragraph::new(detail) + .block(Block::default().borders(Borders::ALL).title("Field Guide")) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} + +fn generator_indent(key: &str) -> &'static str { + match key { + "gunicorn_workers" + | "gunicorn_timeout" + | "gunicorn_graceful_timeout" + | "gunicorn_keep_alive" + | "gunicorn_max_requests" + | "gunicorn_max_requests_jitter" + | "gunicorn_backlog" + | "gunicorn_preload_app" + | "gunicorn_dev_mode" + | "granian_workers" + | "granian_runtime_mode" + | "granian_runtime_threads" + | "granian_blocking_threads" + | "granian_http" + | "granian_loop" + | "granian_task_impl" + | "granian_http1_pipeline_flush" + | "granian_http1_buffer_size" + | "granian_backlog" + | "granian_backpressure" + | "granian_respawn_failed" + | "granian_workers_lifetime" + | "granian_workers_max_rss" + | "granian_dev_mode" + | "granian_log_level" + | "uvicorn_workers" + | "uvicorn_loop" + | "uvicorn_http" + | "uvicorn_backlog" + | "uvicorn_timeout_keep_alive" + | "uvicorn_limit_max_requests" + | "uvicorn_log_level" + | "uvicorn_dev_mode" + | "profiling_tools" + | "profiling_duration_seconds" + | "profiling_required" => " ", + _ => "", + } +} + +pub(crate) fn line_pair<'a>(label: &'a str, value: &'a str) -> Line<'a> { + Line::from(vec![ + Span::styled(format!("{label}: "), Style::default().fg(Color::White)), + Span::styled(value.to_string(), Style::default().fg(Color::Green)), + ]) +} + +pub(crate) fn draw_help(frame: &mut ratatui::Frame<'_>, area: Rect, app: &App) { + let help = match app.mode { + InputMode::EditRunPath | InputMode::EditExtraArgs | InputMode::EditGeneratorField => { + "Type text, Backspace deletes, Enter saves, Esc cancels" + } + InputMode::Normal if app.active_view == AppView::Generator => { + "Tab/BackTab: switch view 1-8/left-right: action [ ] or PgUp/PgDn: section j/k: field e/Enter: edit t: toggle/cycle g or s: save template q: quit" + } + _ => { + "Tab/BackTab: switch view 1-8/left-right: action j/k: suite (launcher/inspector) i: inspector m: monitor l: launcher a: all c: clean p: run path e: extra args PgUp/PgDn or [ ]: scroll logs in monitor Enter/r: run q: quit" + } + }; + let widget = Paragraph::new(help) + .block(Block::default().borders(Borders::ALL).title("Keys")) + .wrap(Wrap { trim: false }); + frame.render_widget(widget, area); +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_console/src/main_parts/view_models.rs b/crates/contextforge_benchmark_console/src/main_parts/view_models.rs new file mode 100644 index 0000000000..fa38639d7d --- /dev/null +++ b/crates/contextforge_benchmark_console/src/main_parts/view_models.rs @@ -0,0 +1,418 @@ +pub(crate) fn discover_scenarios(root: &Path) -> AppResult> { + let mut scenarios = + fs::read_dir(root.join("crates/contextforge_benchmark_runner/assets/scenarios"))? + .filter_map(|entry| { + let path = entry.ok()?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("toml") { + return None; + } + let file_stem = path.file_stem()?.to_str()?.to_string(); + let raw = fs::read_to_string(&path).ok()?; + let parsed = toml::from_str::(&raw).ok()?; + let suite = parsed.get("suite")?.as_table()?; + Some(SuiteSummary { + file_stem, + suite_name: suite + .get("name") + .and_then(TomlValue::as_str) + .unwrap_or_default() + .to_string(), + description: suite + .get("description") + .and_then(TomlValue::as_str) + .unwrap_or_default() + .to_string(), + }) + }) + .collect::>(); + scenarios.sort_by(|left, right| left.file_stem.cmp(&right.file_stem)); + Ok(scenarios) +} + +fn load_selected_suite_doc(root: &Path, app: &App) -> AppResult> { + let Some(selected) = app.selected_suite() else { + return Ok(None); + }; + let path = root + .join("crates/contextforge_benchmark_runner/assets/scenarios") + .join(format!("{}.toml", selected.label())); + if !path.exists() { + return Ok(None); + } + let raw = fs::read_to_string(&path)?; + Ok(Some(toml::from_str::(&raw)?)) +} + +pub(crate) fn build_preview_sections(app: &App, root: &Path) -> AppResult { + let mut preview = PreviewSections::default(); + let selected_suite = app.selected_suite(); + + if let Some(suite) = selected_suite { + preview + .run_plan + .push(format!("Suite: {}", suite.suite_name())); + preview + .run_plan + .push(format!("Focus: {}", suite.description())); + if let Some(doc) = load_selected_suite_doc(root, app)? { + if let Some(scenarios) = doc.get("scenario").and_then(TomlValue::as_array) { + preview + .run_plan + .push(format!("Comparison set: {} scenario(s)", scenarios.len())); + let scenario_names = scenarios + .iter() + .filter_map(|scenario| { + scenario + .get("name") + .and_then(TomlValue::as_str) + .map(ToString::to_string) + }) + .collect::>(); + if !scenario_names.is_empty() { + preview + .run_plan + .push(format!("Variants: {}", scenario_names.join(" vs "))); + } + } + } + } else { + preview.run_plan.push("Suite: (none selected)".to_string()); + } + + match build_command(app, root) { + Ok(command) => { + preview.execution.push(format!( + "Command: {}", + format_command(&command.command, &command.args) + )); + preview.execution.push(format!( + "Runtime env: {}={}", + command.env[0].0, command.env[0].1 + )); + } + Err(error) => { + preview.execution.push(format!("Command error: {error}")); + } + } + + if app.action().supports_all() && app.all { + preview.checks.push( + "Run-all is enabled, so the selected suite entry will only seed the preview context." + .to_string(), + ); + } else { + preview + .checks + .push(format!("Selected suite will run as '{}'.", app.scenario())); + } + if app.action().supports_clean() && app.clean { + preview + .checks + .push("Clean-first is enabled, so benchmark containers and report staging will be cleared before launch.".to_string()); + } + if app.extra_args.trim().is_empty() { + preview.checks.push("No extra args are set.".to_string()); + } else { + preview.checks.push(format!( + "Extra args will be appended exactly as typed: {}", + app.extra_args.trim() + )); + } + if app.action().needs_run_path() { + if app.run_path.trim().is_empty() { + preview.checks.push( + "This action requires a run path. Press 'p' to set it before running.".to_string(), + ); + } else { + preview + .checks + .push(format!("Run path: {}", app.run_path.trim())); + } + } else { + preview + .checks + .push("Run path is ignored for this action.".to_string()); + } + if app.action() == Action::Smoke { + preview + .checks + .push("Smoke mode cuts benchmark duration down to a fast validation run.".to_string()); + } + + Ok(preview) +} + +pub(crate) fn build_selection_summary(app: &App) -> SelectionSummary { + let action_label = app.action().label().to_string(); + let suite_label = if app.action().supports_scenario() { + app.scenario().to_string() + } else { + "(not used)".to_string() + }; + let clean_label = if app.action().supports_clean() { + yes_no(app.clean).to_string() + } else { + "(not used)".to_string() + }; + let run_mode_label = if app.action().supports_all() { + if app.all { + "all-scenarios" + } else { + "selected-suite" + } + .to_string() + } else { + "single-action".to_string() + }; + let run_path_label = if app.action().needs_run_path() { + if app.run_path.trim().is_empty() { + "press 'p' to set".to_string() + } else { + app.run_path.trim().to_string() + } + } else { + "(not used)".to_string() + }; + let extra_args_label = if app.extra_args.trim().is_empty() { + "press 'e' to edit".to_string() + } else { + app.extra_args.trim().to_string() + }; + + SelectionSummary { + action_label, + suite_label, + clean_label, + run_mode_label, + run_path_label, + extra_args_label, + } +} + +pub(crate) fn build_suite_inspector_summary( + app: &App, + root: &Path, +) -> AppResult { + let Some(suite) = app.selected_suite() else { + return Ok(SuiteInspectorSummary { + suite_name: "(none selected)".to_string(), + suite_description: "Choose a suite to see its benchmark intent and comparison shape." + .to_string(), + scenario_count_label: "0 scenarios".to_string(), + comparison_question: "No active suite".to_string(), + scenario_cards: Vec::new(), + }); + }; + + let mut summary = SuiteInspectorSummary { + suite_name: suite.suite_name().to_string(), + suite_description: suite.description().to_string(), + scenario_count_label: "0 scenarios".to_string(), + comparison_question: suite.description().to_string(), + scenario_cards: Vec::new(), + }; + + if let Some(doc) = load_selected_suite_doc(root, app)? { + let defaults = doc.get("defaults"); + if let Some(scenarios) = doc.get("scenario").and_then(TomlValue::as_array) { + summary.scenario_count_label = format!("{} scenario(s)", scenarios.len()); + for scenario in scenarios { + summary + .scenario_cards + .push(build_scenario_card_summary(defaults, scenario)); + } + } + } + + Ok(summary) +} + +fn build_scenario_card_summary( + defaults: Option<&TomlValue>, + scenario: &TomlValue, +) -> ScenarioCardSummary { + let name = scenario + .get("name") + .and_then(TomlValue::as_str) + .unwrap_or("(unnamed scenario)") + .to_string(); + let description = scenario + .get("description") + .and_then(TomlValue::as_str) + .unwrap_or("No scenario description is defined yet.") + .to_string(); + let scenario_type = scenario + .get("scenario_type") + .and_then(TomlValue::as_str) + .unwrap_or("(no type)") + .to_string(); + let mut settings = Vec::new(); + + if let Some(value) = merged_bool(defaults, scenario, "setup", "plugins_enabled") { + settings.push(("plugins_enabled".to_string(), value.to_string())); + } + if let Some(value) = merged_bool(defaults, scenario, "build", "rust_plugins") { + settings.push(("rust_plugins".to_string(), value.to_string())); + } + if let Some(value) = merged_string(defaults, scenario, "setup", "auth_mode") { + settings.push(("auth_mode".to_string(), value)); + } + if let Some(value) = merged_string(defaults, scenario, "runtime", "http_server") { + settings.push(("http_server".to_string(), value)); + } + if let Some(value) = merged_string(defaults, scenario, "build", "image_tag") { + settings.push(("image_tag".to_string(), value)); + } + if let Some(value) = merged_string(defaults, scenario, "load", "target_service") { + settings.push(("target_service".to_string(), value)); + } + for key in ["users", "spawn_rate", "run_time"] { + if let Some(value) = merged_scalar_string(defaults, scenario, "load", key) { + settings.push((key.to_string(), value)); + } + } + if let Some(value) = merged_bool(defaults, scenario, "profiling", "enabled") { + settings.push(("profiling.enabled".to_string(), value.to_string())); + } + if let Some(value) = merged_bool(defaults, scenario, "execution", "retry_enabled") { + settings.push(("retry_enabled".to_string(), value.to_string())); + } + for key in [ + "expected_mcp_runtime", + "expected_mcp_runtime_mode", + "expected_a2a_runtime", + ] { + if let Some(value) = merged_string(defaults, scenario, "setup", key) { + settings.push((key.to_string(), value)); + } + } + for key in [ + "EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED", + "RUST_MCP_MODE", + "RUST_MCP_LOG", + ] { + if let Some(value) = merged_nested_string(defaults, scenario, "gateway", "environment", key) + { + settings.push((key.to_string(), value)); + } + } + + ScenarioCardSummary { + name, + description, + scenario_type, + settings, + } +} + +fn merged_bool( + defaults: Option<&TomlValue>, + scenario: &TomlValue, + section: &str, + key: &str, +) -> Option { + scenario + .get(section) + .and_then(|value| value.get(key)) + .and_then(TomlValue::as_bool) + .or_else(|| { + defaults + .and_then(|value| value.get(section)) + .and_then(|value| value.get(key)) + .and_then(TomlValue::as_bool) + }) +} + +fn merged_string( + defaults: Option<&TomlValue>, + scenario: &TomlValue, + section: &str, + key: &str, +) -> Option { + scenario + .get(section) + .and_then(|value| value.get(key)) + .and_then(TomlValue::as_str) + .map(ToString::to_string) + .or_else(|| { + defaults + .and_then(|value| value.get(section)) + .and_then(|value| value.get(key)) + .and_then(TomlValue::as_str) + .map(ToString::to_string) + }) +} + +fn merged_nested_string( + defaults: Option<&TomlValue>, + scenario: &TomlValue, + section: &str, + nested: &str, + key: &str, +) -> Option { + scenario + .get(section) + .and_then(|value| value.get(nested)) + .and_then(|value| value.get(key)) + .and_then(TomlValue::as_str) + .map(ToString::to_string) + .or_else(|| { + defaults + .and_then(|value| value.get(section)) + .and_then(|value| value.get(nested)) + .and_then(|value| value.get(key)) + .and_then(TomlValue::as_str) + .map(ToString::to_string) + }) +} + +fn merged_scalar_string( + defaults: Option<&TomlValue>, + scenario: &TomlValue, + section: &str, + key: &str, +) -> Option { + fn format_value(value: &TomlValue) -> Option { + value + .as_str() + .map(ToString::to_string) + .or_else(|| value.as_integer().map(|v| v.to_string())) + .or_else(|| value.as_float().map(|v| v.to_string())) + .or_else(|| value.as_bool().map(|v| v.to_string())) + } + + scenario + .get(section) + .and_then(|value| value.get(key)) + .and_then(format_value) + .or_else(|| { + defaults + .and_then(|value| value.get(section)) + .and_then(|value| value.get(key)) + .and_then(format_value) + }) +} + +pub(crate) fn build_generator_focus_summary(app: &App) -> GeneratorFocusSummary { + let field = app.generator.selected_field(); + let kind = match field.kind { + GeneratorFieldKind::Text => "text", + GeneratorFieldKind::Bool => "bool", + GeneratorFieldKind::Choice(_) => "choice", + }; + GeneratorFocusSummary { + section_filter: app.generator.selected_section_name().to_string(), + field_label: field.label.to_string(), + config_key: generator_config_path(field.key).to_string(), + value: field.value.clone(), + kind: kind.to_string(), + schema: field.help.to_string(), + format_hint: generator_format_hint(field.key).to_string(), + visibility: generator_visibility_note(field.key).to_string(), + purpose: generator_explanation(field.key).to_string(), + effect: generator_change_reason(field.key).to_string(), + example: generator_example(field.key).to_string(), + } +} +use crate::main_parts::*; +use crate::*; diff --git a/crates/contextforge_benchmark_runner/Cargo.lock b/crates/contextforge_benchmark_runner/Cargo.lock new file mode 100644 index 0000000000..d4cf88dceb --- /dev/null +++ b/crates/contextforge_benchmark_runner/Cargo.lock @@ -0,0 +1,622 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "contextforge_benchmark_runner" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "csv", + "serde", + "serde_json", + "serde_yaml", + "toml", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +dependencies = [ + "winnow 1.0.0", +] + +[[package]] +name = "toml_writer" +version = "1.1.0+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/crates/contextforge_benchmark_runner/Cargo.toml b/crates/contextforge_benchmark_runner/Cargo.toml new file mode 100644 index 0000000000..cf922485ec --- /dev/null +++ b/crates/contextforge_benchmark_runner/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "contextforge_benchmark_runner" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +chrono = { version = "0.4.42", features = ["serde"] } +clap.workspace = true +csv = "1.3.1" +serde.workspace = true +serde_json.workspace = true +serde_yaml = "0.9.34" +toml = "0.9.7" + +[lints] +workspace = true diff --git a/crates/contextforge_benchmark_runner/assets/Containerfile b/crates/contextforge_benchmark_runner/assets/Containerfile new file mode 100644 index 0000000000..5f02cb7322 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/Containerfile @@ -0,0 +1,135 @@ +# syntax=docker/dockerfile:1.7 + +ARG PYTHON_VERSION=3.12 +ARG ENABLE_RUST=false +ARG ENABLE_PROFILING=false + +FROM quay.io/pypa/manylinux2014:2026.03.06-3 AS rust-builder-base +ARG ENABLE_RUST +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV CARGO_HOME=/usr/local/cargo +ENV RUSTUP_HOME=/usr/local/rustup +ENV PATH="${CARGO_HOME}/bin:${PATH}" +WORKDIR /build + +RUN if [ "$ENABLE_RUST" != "true" ]; then \ + mkdir -p /build/rust-wheels; \ + exit 0; \ + fi + +RUN if [ "$ENABLE_RUST" = "true" ]; then \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable; \ + fi + +COPY plugins_rust/ /build/plugins_rust/ + +RUN if [ "$ENABLE_RUST" = "true" ]; then \ + if [ -f "${CARGO_HOME}/env" ]; then . "${CARGO_HOME}/env"; fi \ + && mkdir -p /build/rust-wheels \ + && /opt/python/cp312-cp312/bin/python -m pip install --upgrade pip maturin \ + && for plugin_dir in /build/plugins_rust/*/; do \ + if [ -f "$plugin_dir/Cargo.toml" ] && [ -f "$plugin_dir/pyproject.toml" ]; then \ + /opt/python/cp312-cp312/bin/maturin build --release --compatibility manylinux2014 --manifest-path "$plugin_dir/Cargo.toml" --out /build/rust-wheels || exit 1; \ + fi; \ + done; \ + fi + +FROM rust-builder-base AS rust-builder + +FROM registry.access.redhat.com/ubi10/ubi:10.1-1772441712 AS builder +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] + +ARG PYTHON_VERSION +ARG ENABLE_RUST=false +ARG ENABLE_PROFILING=false +ARG GRPC_PYTHON_BUILD_SYSTEM_OPENSSL='False' + +RUN dnf upgrade -y \ + && dnf install -y \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ + binutils \ + openssl-devel \ + gcc \ + gcc-c++ \ + postgresql-devel \ + curl \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && dnf clean all + +WORKDIR /app + +RUN if [ "$(uname -m)" = "s390x" ] || [ "$(uname -m)" = "ppc64le" ]; then \ + echo "export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL='True'" > /etc/profile.d/use-openssl.sh; \ + else \ + echo "export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL='False'" > /etc/profile.d/use-openssl.sh; \ + fi \ + && chmod 644 /etc/profile.d/use-openssl.sh + +COPY pyproject.toml /app/ +COPY --from=rust-builder /build/rust-wheels/ /tmp/rust-wheels/ + +RUN . /etc/profile.d/use-openssl.sh \ + && python3 -m venv /app/.venv \ + && /app/.venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel pdm uv \ + && /app/.venv/bin/uv pip install ".[redis,postgres,mysql,observability,granian]" \ + && if [ "$ENABLE_RUST" = "true" ] && ls /tmp/rust-wheels/*.whl >/dev/null 2>&1; then \ + /app/.venv/bin/python3 -m pip install /tmp/rust-wheels/*.whl; \ + fi \ + && if [ "$ENABLE_PROFILING" = "true" ]; then \ + /app/.venv/bin/pip install --no-cache-dir "memray>=1.17.0" "py-spy>=0.4.0"; \ + fi \ + && /app/.venv/bin/pip uninstall --yes uv pip setuptools wheel pdm \ + && rm -rf /tmp/rust-wheels /root/.cache /var/cache/dnf \ + && rm -rf /app/*.egg-info /app/build /app/dist /app/.eggs /app/.venv/share/python-wheels + +COPY run-gunicorn.sh run-granian.sh run.sh /app/ +COPY gunicorn.config.py /app/ +COPY mcpgateway/ /app/mcpgateway/ +COPY plugins/ /app/plugins/ +COPY crates/contextforge_benchmark_runner/assets/ /app/crates/contextforge_benchmark_runner/assets/ +COPY mcp-catalog.yml /app/ + +RUN cp /app/crates/contextforge_benchmark_runner/assets/docker-entrypoint.sh /app/docker-entrypoint.sh \ + && chmod +x /app/run-gunicorn.sh /app/run-granian.sh /app/docker-entrypoint.sh /app/run.sh /app/crates/contextforge_benchmark_runner/assets/run-uvicorn.sh \ + && python3 -OO -m compileall -q /app/.venv /app/mcpgateway /app/plugins /app/crates/contextforge_benchmark_runner/assets 2>/dev/null || true \ + && find /app -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true \ + && chown -R 1001:0 /app \ + && chmod -R g=u /app + +FROM registry.access.redhat.com/ubi10/ubi-minimal:10.1-1772441549 AS runtime + +ARG PYTHON_VERSION=3.12 +ARG ENABLE_PROFILING=false + +RUN microdnf upgrade -y --nodocs --setopt=install_weak_deps=0 \ + && microdnf install -y --nodocs --setopt=install_weak_deps=0 \ + python${PYTHON_VERSION} \ + ca-certificates \ + procps-ng \ + shadow-utils \ + && if [ "$ENABLE_PROFILING" = "true" ]; then \ + microdnf install -y --nodocs --setopt=install_weak_deps=0 gdb; \ + fi \ + && microdnf clean all \ + && rm -rf /var/cache/yum \ + && ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 \ + && useradd --uid 1001 --gid 0 --home-dir /app --shell /sbin/nologin --no-create-home --comment app app + +COPY --from=builder --chown=1001:0 /app /app + +ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app +EXPOSE 4444 +USER 1001 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD ["python3", "-c", "import httpx,sys;sys.exit(0 if httpx.get('http://localhost:4444/health',timeout=5).status_code==200 else 1)"] + +ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/crates/contextforge_benchmark_runner/assets/docker-entrypoint.sh b/crates/contextforge_benchmark_runner/assets/docker-entrypoint.sh new file mode 100755 index 0000000000..b06b76c048 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/docker-entrypoint.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +HTTP_SERVER="${HTTP_SERVER:-gunicorn}" + +case "${HTTP_SERVER}" in + granian) + echo "Starting ContextForge benchmark image with Granian..." + exec ./run-granian.sh "$@" + ;; + gunicorn) + echo "Starting ContextForge benchmark image with Gunicorn..." + exec ./run-gunicorn.sh "$@" + ;; + uvicorn) + echo "Starting ContextForge benchmark image with Uvicorn..." + exec ./crates/contextforge_benchmark_runner/assets/run-uvicorn.sh "$@" + ;; + *) + echo "ERROR: Unknown HTTP_SERVER value: ${HTTP_SERVER}" + echo "Valid options: granian, gunicorn, uvicorn" + exit 1 + ;; +esac diff --git a/crates/contextforge_benchmark_runner/assets/payloads/prompts/get_compare_timezones.json b/crates/contextforge_benchmark_runner/assets/payloads/prompts/get_compare_timezones.json new file mode 100644 index 0000000000..6743e5118d --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/prompts/get_compare_timezones.json @@ -0,0 +1,11 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "prompts/get", + "params": { + "name": "fast-time-compare-timezones", + "arguments": { + "timezones": "America/New_York,Europe/London,Asia/Tokyo" + } + } +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/prompts/get_schedule_meeting.json b/crates/contextforge_benchmark_runner/assets/payloads/prompts/get_schedule_meeting.json new file mode 100644 index 0000000000..59afc6c87f --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/prompts/get_schedule_meeting.json @@ -0,0 +1,14 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "prompts/get", + "params": { + "name": "fast-time-schedule-meeting", + "arguments": { + "participants": "New York,London,Tokyo", + "duration": "60", + "preferred_hours": "09:00-17:00", + "date_range": "next-week" + } + } +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/prompts/list_prompts.json b/crates/contextforge_benchmark_runner/assets/payloads/prompts/list_prompts.json new file mode 100644 index 0000000000..2417f2b93d --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/prompts/list_prompts.json @@ -0,0 +1,6 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "prompts/list", + "params": {} +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/resources/list_resources.json b/crates/contextforge_benchmark_runner/assets/payloads/resources/list_resources.json new file mode 100644 index 0000000000..ced96b057e --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/resources/list_resources.json @@ -0,0 +1,6 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "resources/list", + "params": {} +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/resources/read_timezone_info.json b/crates/contextforge_benchmark_runner/assets/payloads/resources/read_timezone_info.json new file mode 100644 index 0000000000..b9e07655ac --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/resources/read_timezone_info.json @@ -0,0 +1,8 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "resources/read", + "params": { + "uri": "timezone://info" + } +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/resources/read_world_times.json b/crates/contextforge_benchmark_runner/assets/payloads/resources/read_world_times.json new file mode 100644 index 0000000000..791c9801c5 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/resources/read_world_times.json @@ -0,0 +1,8 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "resources/read", + "params": { + "uri": "time://current/world" + } +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/tools/convert_time.json b/crates/contextforge_benchmark_runner/assets/payloads/tools/convert_time.json new file mode 100644 index 0000000000..16a861e79a --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/tools/convert_time.json @@ -0,0 +1,13 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "fast-time-convert-time", + "arguments": { + "time": "09:00", + "source_timezone": "Europe/London", + "target_timezone": "Asia/Tokyo" + } + } +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/tools/get_system_time.json b/crates/contextforge_benchmark_runner/assets/payloads/tools/get_system_time.json new file mode 100644 index 0000000000..1193e32d8e --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/tools/get_system_time.json @@ -0,0 +1,11 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "fast-time-get-system-time", + "arguments": { + "timezone": "America/New_York" + } + } +} diff --git a/crates/contextforge_benchmark_runner/assets/payloads/tools/list_tools.json b/crates/contextforge_benchmark_runner/assets/payloads/tools/list_tools.json new file mode 100644 index 0000000000..f621da5618 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/payloads/tools/list_tools.json @@ -0,0 +1,6 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/list", + "params": {} +} diff --git a/crates/contextforge_benchmark_runner/assets/run-uvicorn.sh b/crates/contextforge_benchmark_runner/assets/run-uvicorn.sh new file mode 100755 index 0000000000..dfa1d35fff --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/run-uvicorn.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +APP_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +cd "${APP_ROOT}" || exit 1 + +if [[ -z "${VIRTUAL_ENV:-}" && -f "${APP_ROOT}/.venv/bin/activate" ]]; then + # shellcheck disable=SC1090 + source "${APP_ROOT}/.venv/bin/activate" +fi + +PYTHON="${PYTHON:-}" +if [[ -z "${PYTHON}" ]]; then + if [[ -n "${VIRTUAL_ENV:-}" && -x "${VIRTUAL_ENV}/bin/python" ]]; then + PYTHON="${VIRTUAL_ENV}/bin/python" + elif command -v python3 >/dev/null 2>&1; then + PYTHON="$(command -v python3)" + else + PYTHON="$(command -v python)" + fi +fi + +HOST="${HOST:-0.0.0.0}" +PORT="${PORT:-4444}" +UVICORN_WORKERS="${UVICORN_WORKERS:-1}" +UVICORN_LOOP="${UVICORN_LOOP:-auto}" +UVICORN_HTTP="${UVICORN_HTTP:-auto}" +UVICORN_BACKLOG="${UVICORN_BACKLOG:-2048}" +UVICORN_TIMEOUT_KEEP_ALIVE="${UVICORN_TIMEOUT_KEEP_ALIVE:-5}" +UVICORN_LIMIT_MAX_REQUESTS="${UVICORN_LIMIT_MAX_REQUESTS:-0}" +LOG_LEVEL="${LOG_LEVEL:-error}" +DEVELOPER_MODE="${DEVELOPER_MODE:-false}" +DISABLE_ACCESS_LOG="${DISABLE_ACCESS_LOG:-true}" + +args=( + -m uvicorn + mcpgateway.main:app + --host "${HOST}" + --port "${PORT}" + --workers "${UVICORN_WORKERS}" + --loop "${UVICORN_LOOP}" + --http "${UVICORN_HTTP}" + --backlog "${UVICORN_BACKLOG}" + --timeout-keep-alive "${UVICORN_TIMEOUT_KEEP_ALIVE}" + --log-level "${LOG_LEVEL}" + --proxy-headers +) + +if [[ "${UVICORN_LIMIT_MAX_REQUESTS}" != "0" ]]; then + args+=(--limit-max-requests "${UVICORN_LIMIT_MAX_REQUESTS}") +fi + +if [[ "${DISABLE_ACCESS_LOG}" == "true" ]]; then + args+=(--no-access-log) +fi + +if [[ "${DEVELOPER_MODE}" == "true" ]]; then + args+=(--reload) +fi + +exec "${PYTHON}" "${args[@]}" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/a2a-invoke-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/a2a-invoke-300.toml new file mode 100644 index 0000000000..c041630a6e --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/a2a-invoke-300.toml @@ -0,0 +1,112 @@ +[suite] +name = "benchmark-gunicorn-a2a-invoke-compare" +description = "Exercises the A2A invoke hot path through nginx and the gateway, then compares two images built from the same checkout to show whether adding optional Rust plugin artifacts to the benchmark image changes A2A throughput or latency. This suite is intentionally not a Python-vs-Rust runtime comparison because A2A remains embedded in Python today." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-a2a" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.workload] +fallback_endpoint = "/a2a/a2a-echo-agent/invoke" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = false + +[defaults.load.workload.endpoints."/a2a"] +enabled = false + +[defaults.load.workload.endpoints."/a2a/a2a-echo-agent/invoke"] +enabled = true +weight = 1 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-a2a-invoke-rust-baseline" +description = "A2A invoke baseline on the benchmark image without optional Rust plugin artifacts" +scenario_type = "a2a_invoke_baseline" + +[scenario.setup] +expected_a2a_runtime = "rust" + +[scenario.build] +rust_plugins = false +image_tag = "benchmark-suite-a2a-rust-baseline" + +[[scenario]] +name = "gunicorn-a2a-invoke-rust-compare" +description = "A2A invoke comparison on the same checkout with optional Rust plugin artifacts installed in the benchmark image" +scenario_type = "a2a_invoke_compare" + +[scenario.setup] +expected_a2a_runtime = "rust" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-a2a-rust" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/admin-plugins-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/admin-plugins-300.toml new file mode 100644 index 0000000000..4edb534b82 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/admin-plugins-300.toml @@ -0,0 +1,112 @@ +[suite] +name = "benchmark-gunicorn-admin-plugins-compare" +description = "Exercises the admin plugin inventory endpoint with plugins enabled and matched load settings, then compares the default benchmark image against an image with Rust plugin artifacts installed to show whether plugin packaging changes admin control-plane performance. This suite is intentionally not a Python-vs-Rust runtime comparison because the admin plugin path does not move onto the Rust MCP runtime." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = true + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-admin-plugins" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.workload] +fallback_endpoint = "/admin/plugins" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/ready"] +enabled = false + +[defaults.load.workload.endpoints."/admin/plugins"] +enabled = true +weight = 1 + +[defaults.load.workload.endpoints."/servers"] +enabled = false + +[defaults.load.workload.endpoints."/resources"] +enabled = false + +[defaults.load.workload.endpoints."/prompts"] +enabled = false + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-admin-plugins-baseline" +description = "Admin plugin inventory benchmark on the default benchmark image without Rust plugin artifacts installed" +scenario_type = "admin_plugins_baseline" + +[scenario.build] +rust_plugins = false +image_tag = "benchmark-suite-admin-plugins-baseline" + +[[scenario]] +name = "gunicorn-admin-plugins-rust-plugins" +description = "Admin plugin inventory benchmark on the benchmark image with Rust plugin artifacts installed for control-plane packaging comparison" +scenario_type = "admin_plugins_rust_plugins_compare" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-admin-plugins-rust" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/agentgateway-mcp-server-time-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/agentgateway-mcp-server-time-300.toml new file mode 100644 index 0000000000..1e0f726097 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/agentgateway-mcp-server-time-300.toml @@ -0,0 +1,116 @@ +[suite] +name = "benchmark-agentgateway-mcp-server-time-compare" +description = "Covers the agentgateway MCP server time Locust family with a direct MCP-heavy time-service workload." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-agentgateway-time" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/list" + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 12 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-convert-time"] +enabled = true +weight = 9 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-agentgateway-mcp-server-time-baseline" +description = "MCP server time baseline on the default gateway stack." +scenario_type = "agentgateway_mcp_server_time_baseline" + +[scenario.build] +image_tag = "benchmark-suite-agentgateway-time-baseline" + +[[scenario]] +name = "gunicorn-agentgateway-mcp-server-time-rust-runtime" +description = "MCP server time workload with the Rust MCP runtime enabled." +scenario_type = "agentgateway_mcp_server_time_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-agentgateway-time-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/baseline-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/baseline-300.toml new file mode 100644 index 0000000000..6cce95aba5 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/baseline-300.toml @@ -0,0 +1,133 @@ +[suite] +name = "benchmark-gunicorn-baseline-compare" +description = "Covers the baseline Locust family with a mixed fast-time and gateway benchmark workload across health, REST discovery, and core MCP calls." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-baseline" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/health" + +[defaults.load.workload.endpoints."/health"] +enabled = true +weight = 6 + +[defaults.load.workload.endpoints."/servers"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/resources"] +enabled = true +weight = 3 + +[defaults.load.workload.endpoints."/prompts"] +enabled = true +weight = 3 + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = true +weight = 3 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-baseline-workload" +description = "Baseline benchmark workload on the default gateway path." +scenario_type = "baseline_workload" + +[scenario.build] +rust_plugins = false +image_tag = "benchmark-suite-baseline-default" + +[[scenario]] +name = "gunicorn-baseline-rust-runtime" +description = "Baseline workload with the Rust MCP runtime enabled behind the shared gateway." +scenario_type = "baseline_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-baseline-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/echo-delay-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/echo-delay-300.toml new file mode 100644 index 0000000000..3b25208aee --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/echo-delay-300.toml @@ -0,0 +1,116 @@ +[suite] +name = "benchmark-echo-delay-compare" +description = "Covers the echo-delay Locust family with a small-payload round-trip MCP workload tuned for latency-sensitive comparisons." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-echo-delay" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/call fast-time-get-system-time" + +[defaults.load.workload.endpoints."/health"] +enabled = true +weight = 2 + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 14 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-echo-delay-baseline" +description = "Latency-sensitive round-trip baseline on the default gateway path." +scenario_type = "echo_delay_baseline" + +[scenario.build] +image_tag = "benchmark-suite-echo-delay-baseline" + +[[scenario]] +name = "gunicorn-echo-delay-rust-runtime" +description = "Latency-sensitive round-trip benchmark with the Rust MCP runtime enabled." +scenario_type = "echo_delay_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-echo-delay-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/highthroughput-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/highthroughput-300.toml new file mode 100644 index 0000000000..047b50b3b3 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/highthroughput-300.toml @@ -0,0 +1,121 @@ +[suite] +name = "benchmark-highthroughput-compare" +description = "Covers the high-throughput Locust family with a read-heavy control-plane workload focused on cheap gateway endpoints." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-highthroughput" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.workload] +fallback_endpoint = "/health" + +[defaults.load.workload.endpoints."/health"] +enabled = true +weight = 20 + +[defaults.load.workload.endpoints."/servers"] +enabled = true +weight = 12 + +[defaults.load.workload.endpoints."/resources"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/prompts"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/ready"] +enabled = true +weight = 8 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-highthroughput-baseline" +description = "Read-heavy high-throughput control-plane baseline." +scenario_type = "highthroughput_baseline" + +[scenario.build] +image_tag = "benchmark-suite-highthroughput-baseline" + +[[scenario]] +name = "gunicorn-highthroughput-rust-runtime" +description = "Read-heavy high-throughput control-plane workload with Rust MCP runtime enabled in the same gateway." +scenario_type = "highthroughput_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-highthroughput-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/mcp-isolation-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-isolation-300.toml new file mode 100644 index 0000000000..d01c9512b9 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-isolation-300.toml @@ -0,0 +1,112 @@ +[suite] +name = "benchmark-mcp-isolation-compare" +description = "Covers the MCP isolation Locust family with an ownership-sensitive MCP workload focused on repeated tools/list and tools/call requests." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-mcp-isolation" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/list" + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 12 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 10 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-mcp-isolation-baseline" +description = "Session reuse and MCP call baseline aligned with the isolation workload family." +scenario_type = "mcp_isolation_baseline" + +[scenario.build] +image_tag = "benchmark-suite-mcp-isolation-baseline" + +[[scenario]] +name = "gunicorn-mcp-isolation-rust-runtime" +description = "Session reuse and MCP call workload with the Rust MCP runtime enabled." +scenario_type = "mcp_isolation_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-mcp-isolation-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/mcp-prompts-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-prompts-300.toml new file mode 100644 index 0000000000..60597f2118 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-prompts-300.toml @@ -0,0 +1,141 @@ +[suite] +name = "benchmark-gunicorn-mcp-prompts-compare" +description = "Exercises MCP prompt discovery and prompt fetch requests over the shared benchmark stack, then compares the default Python MCP path against a Rust-managed MCP runtime variant to show whether prompt-heavy MCP traffic regresses or improves when Rust owns /mcp." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-mcp-prompts" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp prompts/list" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/ready"] +enabled = false + +[defaults.load.workload.endpoints."/admin/plugins"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = false + +[defaults.load.workload.endpoints."/resources"] +enabled = false + +[defaults.load.workload.endpoints."/prompts"] +enabled = false + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = false + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = false + +[defaults.load.workload.endpoints."/mcp prompts/list"] +enabled = true +weight = 7 + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-schedule-meeting"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-compare-timezones"] +enabled = true +weight = 8 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-mcp-prompts-baseline" +description = "MCP prompts benchmark on the default Python-managed /mcp path" +scenario_type = "mcp_prompts_baseline" + +[scenario.build] +rust_plugins = false +image_tag = "benchmark-suite-mcp-prompts-baseline" + +[[scenario]] +name = "gunicorn-mcp-prompts-rust-runtime" +description = "MCP prompts benchmark with the Rust-managed /mcp runtime enabled for the same prompt-heavy workload" +scenario_type = "mcp_prompts_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-mcp-prompts-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/mcp-protocol-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-protocol-300.toml new file mode 100644 index 0000000000..3c32252a5a --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-protocol-300.toml @@ -0,0 +1,120 @@ +[suite] +name = "benchmark-mcp-protocol-compare" +description = "Covers the MCP protocol Locust family with MCP discovery, tools/call, prompts, and resources over the streamable HTTP path." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-mcp-protocol" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/list" + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 15 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/mcp prompts/list"] +enabled = true +weight = 5 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-mcp-protocol-baseline" +description = "Pure MCP protocol baseline on the default gateway path." +scenario_type = "mcp_protocol_baseline" + +[scenario.build] +image_tag = "benchmark-suite-mcp-protocol-baseline" + +[[scenario]] +name = "gunicorn-mcp-protocol-rust-runtime" +description = "Pure MCP protocol benchmark with the Rust MCP runtime enabled." +scenario_type = "mcp_protocol_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-mcp-protocol-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/mcp-resources-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-resources-300.toml new file mode 100644 index 0000000000..7fc8e55a18 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/mcp-resources-300.toml @@ -0,0 +1,141 @@ +[suite] +name = "benchmark-gunicorn-mcp-resources-compare" +description = "Exercises MCP resource listing and resource read traffic against the time-service fixtures, then compares the default Python MCP path against a Rust-managed MCP runtime variant to show whether resource-heavy MCP traffic improves or regresses when the Rust runtime owns /mcp." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-mcp-resources" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp resources/list" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/ready"] +enabled = false + +[defaults.load.workload.endpoints."/admin/plugins"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = false + +[defaults.load.workload.endpoints."/resources"] +enabled = false + +[defaults.load.workload.endpoints."/prompts"] +enabled = false + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = false + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = true +weight = 7 + +[defaults.load.workload.endpoints."/mcp resources/read timezone://info"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/mcp resources/read time://current/world"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/mcp prompts/list"] +enabled = false + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-mcp-resources-baseline" +description = "MCP resources benchmark on the default Python-managed /mcp path" +scenario_type = "mcp_resources_baseline" + +[scenario.build] +rust_plugins = false +image_tag = "benchmark-suite-mcp-resources-baseline" + +[[scenario]] +name = "gunicorn-mcp-resources-rust-runtime" +description = "MCP resources benchmark with the Rust-managed /mcp runtime enabled for the same resource-heavy workload" +scenario_type = "mcp_resources_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-mcp-resources-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-300.toml new file mode 100644 index 0000000000..7d83346ba0 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-300.toml @@ -0,0 +1,103 @@ +[suite] +name = "benchmark-rate-limiter-compare" +description = "Covers the rate limiter correctness Locust family with a plugins-enabled MCP tools/call workload that stresses shared per-user limits." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = true + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-rate-limiter" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 1 +spawn_rate = 1 +run_time = "120s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/call fast-time-get-system-time" + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 2 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 14 + +[defaults.measurement] +warmup_seconds = 15 +measure_seconds = 90 +profile_seconds = 0 +cooldown_seconds = 15 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-rate-limiter-baseline" +description = "Rate limiter correctness baseline with plugins enabled." +scenario_type = "rate_limiter_baseline" + +[scenario.build] +image_tag = "benchmark-suite-rate-limiter-baseline" + +[[scenario]] +name = "gunicorn-rate-limiter-rust-plugins" +description = "Rate limiter correctness workload on the benchmark image with Rust plugin artifacts installed." +scenario_type = "rate_limiter_rust_plugins_compare" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-rate-limiter-rust-plugins" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-redis-capacity-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-redis-capacity-300.toml new file mode 100644 index 0000000000..f6ce340462 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-redis-capacity-300.toml @@ -0,0 +1,103 @@ +[suite] +name = "benchmark-rate-limiter-redis-capacity-compare" +description = "Covers the rate limiter Redis capacity Locust family with a prompt-heavy plugins-enabled benchmark workload." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = true + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-rate-limiter-redis-capacity" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 100 +spawn_rate = 10 +run_time = "120s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp prompts/get fast-time-schedule-meeting" + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-schedule-meeting"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-compare-timezones"] +enabled = true +weight = 6 + +[defaults.measurement] +warmup_seconds = 15 +measure_seconds = 90 +profile_seconds = 0 +cooldown_seconds = 15 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-rate-limiter-redis-capacity-baseline" +description = "Prompt-heavy Redis-backed rate limiter baseline." +scenario_type = "rate_limiter_redis_capacity_baseline" + +[scenario.build] +image_tag = "benchmark-suite-rate-limiter-redis-capacity-baseline" + +[[scenario]] +name = "gunicorn-rate-limiter-redis-capacity-rust-plugins" +description = "Prompt-heavy Redis-backed rate limiter workload with Rust plugin artifacts installed." +scenario_type = "rate_limiter_redis_capacity_rust_plugins_compare" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-rate-limiter-redis-capacity-rust-plugins" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-scale-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-scale-300.toml new file mode 100644 index 0000000000..bdfb075582 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/rate-limiter-scale-300.toml @@ -0,0 +1,103 @@ +[suite] +name = "benchmark-rate-limiter-scale-compare" +description = "Covers the rate limiter scale Locust family with a high-user plugins-enabled MCP tools/call workload." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = true + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-rate-limiter-scale" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 500 +spawn_rate = 20 +run_time = "300s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/call fast-time-get-system-time" + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 1 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 15 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 240 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-rate-limiter-scale-baseline" +description = "High-cardinality rate limiter workload without Rust plugin artifacts." +scenario_type = "rate_limiter_scale_baseline" + +[scenario.build] +image_tag = "benchmark-suite-rate-limiter-scale-baseline" + +[[scenario]] +name = "gunicorn-rate-limiter-scale-rust-plugins" +description = "High-cardinality rate limiter workload with Rust plugin artifacts installed." +scenario_type = "rate_limiter_scale_rust_plugins_compare" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-rate-limiter-scale-rust-plugins" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/rest-discovery-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/rest-discovery-300.toml new file mode 100644 index 0000000000..7ed6d74e4b --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/rest-discovery-300.toml @@ -0,0 +1,132 @@ +[suite] +name = "benchmark-gunicorn-rest-discovery-compare" +description = "Exercises the REST discovery surfaces for servers, resources, and prompts with matched load settings, then compares the default gateway path against the same gateway with the Rust MCP runtime enabled in the background to show whether turning on the Rust runtime changes catalog-style discovery behavior outside /mcp." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-rest-discovery" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.workload] +fallback_endpoint = "/servers" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/ready"] +enabled = false + +[defaults.load.workload.endpoints."/admin/plugins"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = true +weight = 6 + +[defaults.load.workload.endpoints."/resources"] +enabled = true +weight = 5 + +[defaults.load.workload.endpoints."/prompts"] +enabled = true +weight = 5 + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = false + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = false + +[defaults.load.workload.endpoints."/mcp prompts/list"] +enabled = false + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-rest-discovery-baseline" +description = "REST discovery benchmark on the default gateway path with the Rust MCP runtime disabled" +scenario_type = "rest_discovery_baseline" + +[scenario.build] +rust_plugins = false +image_tag = "benchmark-suite-rest-discovery-baseline" + +[[scenario]] +name = "gunicorn-rest-discovery-rust-runtime" +description = "REST discovery benchmark on the same gateway workload with the Rust MCP runtime enabled behind the REST discovery paths" +scenario_type = "rest_discovery_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-rest-discovery-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/rust-mcp-runtime-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/rust-mcp-runtime-300.toml new file mode 100644 index 0000000000..762d5ab134 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/rust-mcp-runtime-300.toml @@ -0,0 +1,173 @@ +[suite] +name = "benchmark-gunicorn-rust-mcp-runtime-compare" +description = "Exercises the main MCP-heavy workload across tools, resources, prompts, and discovery calls, then compares the current branch baseline against the Rust MCP runtime variant to show whether the Rust-managed MCP runtime changes core gateway behavior under representative load." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-rust-mcp-runtime" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +# Optional per-scenario overrides: +# [scenario.build] +# +# [scenario.build.args] +# ENABLE_RUST_MCP_RMCP = "true" +# +# [scenario.gateway.environment] +# RUST_MCP_MODE = "edge" +# +# [scenario.setup] +# expected_mcp_runtime = "rust" +# expected_mcp_runtime_mode = "rust-managed" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp tools/list" + +[defaults.load.workload.endpoints."/health"] +enabled = false + +[defaults.load.workload.endpoints."/ready"] +enabled = false + +[defaults.load.workload.endpoints."/admin/plugins"] +enabled = false + +[defaults.load.workload.endpoints."/servers"] +enabled = true +weight = 2 + +[defaults.load.workload.endpoints."/resources"] +enabled = false + +[defaults.load.workload.endpoints."/prompts"] +enabled = false + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 6 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 14 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-convert-time"] +enabled = true +weight = 12 + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/mcp resources/read timezone://info"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/mcp resources/read time://current/world"] +enabled = true +weight = 6 + +[defaults.load.workload.endpoints."/mcp prompts/list"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-schedule-meeting"] +enabled = true +weight = 7 + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-compare-timezones"] +enabled = true +weight = 6 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-rust-mcp-runtime-baseline" +description = "Current branch baseline on Gunicorn with the shared Rust MCP runtime benchmark workload" +scenario_type = "mcp_runtime_baseline" + +[[scenario]] +name = "gunicorn-rust-mcp-runtime-variant" +description = "Rust MCP runtime comparison on Gunicorn using the same MCP-heavy workload and edge-mode public /mcp routing" +scenario_type = "mcp_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-rust-mcp-runtime-variant" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" + +[scenario.execution] +retry_enabled = false +max_attempts = 1 diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/secret-detection-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/secret-detection-300.toml new file mode 100644 index 0000000000..7bdfc75661 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/secret-detection-300.toml @@ -0,0 +1,103 @@ +[suite] +name = "benchmark-secret-detection-compare" +description = "Covers the secret detection Locust family with a prompt-heavy plugins-enabled workload that exercises prompt retrieval under secret scanning." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = true + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-secret-detection" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/mcp prompts/get fast-time-schedule-meeting" + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-schedule-meeting"] +enabled = true +weight = 12 + +[defaults.load.workload.endpoints."/mcp prompts/get fast-time-compare-timezones"] +enabled = true +weight = 8 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-secret-detection-baseline" +description = "Secret detection benchmark on the default benchmark image." +scenario_type = "secret_detection_baseline" + +[scenario.build] +image_tag = "benchmark-suite-secret-detection-baseline" + +[[scenario]] +name = "gunicorn-secret-detection-rust-plugins" +description = "Secret detection benchmark with Rust plugin artifacts installed." +scenario_type = "secret_detection_rust_plugins_compare" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-secret-detection-rust-plugins" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/slow-time-server-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/slow-time-server-300.toml new file mode 100644 index 0000000000..4c57b0bebe --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/slow-time-server-300.toml @@ -0,0 +1,116 @@ +[suite] +name = "benchmark-slow-time-server-compare" +description = "Covers the slow time server Locust family with a lower-intensity time-service workload emphasizing health and time/resource calls." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-slow-time-server" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 150 +spawn_rate = 30 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/health" + +[defaults.load.workload.endpoints."/health"] +enabled = true +weight = 10 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/mcp resources/read timezone://info"] +enabled = true +weight = 6 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-slow-time-server-baseline" +description = "Lower-intensity time-service baseline aligned with the slow-time workload family." +scenario_type = "slow_time_server_baseline" + +[scenario.build] +image_tag = "benchmark-suite-slow-time-server-baseline" + +[[scenario]] +name = "gunicorn-slow-time-server-rust-runtime" +description = "Lower-intensity time-service benchmark with Rust MCP runtime enabled." +scenario_type = "slow_time_server_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-slow-time-server-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/assets/scenarios/spin-detector-300.toml b/crates/contextforge_benchmark_runner/assets/scenarios/spin-detector-300.toml new file mode 100644 index 0000000000..157cb6da37 --- /dev/null +++ b/crates/contextforge_benchmark_runner/assets/scenarios/spin-detector-300.toml @@ -0,0 +1,140 @@ +[suite] +name = "benchmark-spin-detector-compare" +description = "Covers the spin detector Locust family with a broad mixed gateway workload spanning health, REST discovery, and MCP traffic." +output_root = "reports/benchmarks" +continue_on_failure = false +save_intermediate_artifacts = true +flamegraph_enabled = false + +[defaults.setup] +target_kind = "gateway" +auth_mode = "jwt" +plugins_enabled = false + +[defaults.build] +rust_plugins = false +profiling_image = false +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" +image_name = "mcpgateway/mcpgateway" +image_tag = "benchmark-suite-spin-detector" +rebuild_policy = "missing" + +[defaults.runtime] +http_server = "gunicorn" +host = "127.0.0.1" +transport_type = "streamablehttp" + +[defaults.runtime.gunicorn] +workers = 12 +timeout = 30 +graceful_timeout = 30 +keep_alive = 10 +max_requests = 0 +max_requests_jitter = 0 +backlog = 16384 +preload_app = true +dev_mode = false + +[defaults.gateway] +disable_access_log = true +templates_auto_reload = false +structured_logging_database_enabled = false +sqlalchemy_echo = false +log_level = "ERROR" + +[defaults.load] +driver = "contextforge_goose" +headless = true +only_summary = true +html_report = false +users = 300 +spawn_rate = 60 +run_time = "180s" +target_service = "nginx" + +[defaults.load.env] +BENCH_MCP_SESSION_MODE = "reuse" + +[defaults.load.workload] +fallback_endpoint = "/health" + +[defaults.load.workload.endpoints."/health"] +enabled = true +weight = 8 + +[defaults.load.workload.endpoints."/ready"] +enabled = true +weight = 5 + +[defaults.load.workload.endpoints."/servers"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/resources"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/prompts"] +enabled = true +weight = 4 + +[defaults.load.workload.endpoints."/mcp tools/list"] +enabled = true +weight = 5 + +[defaults.load.workload.endpoints."/mcp tools/call fast-time-get-system-time"] +enabled = true +weight = 6 + +[defaults.load.workload.endpoints."/mcp resources/list"] +enabled = true +weight = 3 + +[defaults.load.workload.endpoints."/mcp prompts/list"] +enabled = true +weight = 3 + +[defaults.measurement] +warmup_seconds = 30 +measure_seconds = 120 +profile_seconds = 0 +cooldown_seconds = 30 + +[defaults.profiling] +enabled = false +tools = ["perf", "flamegraph"] +duration_seconds = 0 +required = false + +[defaults.execution] +retry_enabled = true +max_attempts = 2 +capture_logs = true +save_raw_results = true +reuse_stack = true + +[[scenario]] +name = "gunicorn-spin-detector-baseline" +description = "Mixed gateway workload baseline aligned with the spin-detector surface." +scenario_type = "spin_detector_baseline" + +[scenario.build] +image_tag = "benchmark-suite-spin-detector-baseline" + +[[scenario]] +name = "gunicorn-spin-detector-rust-runtime" +description = "Mixed gateway workload with the Rust MCP runtime enabled behind the same traffic mix." +scenario_type = "spin_detector_rust_runtime_compare" + +[scenario.setup] +expected_mcp_runtime = "rust" +expected_mcp_runtime_mode = "rust-managed" + +[scenario.build] +rust_plugins = true +image_tag = "benchmark-suite-spin-detector-rust-runtime" + +[scenario.gateway.environment] +EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED = "true" +RUST_MCP_MODE = "edge" +RUST_MCP_LOG = "warn" diff --git a/crates/contextforge_benchmark_runner/src/lib.rs b/crates/contextforge_benchmark_runner/src/lib.rs new file mode 100644 index 0000000000..1c42cb29f5 --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib.rs @@ -0,0 +1,408 @@ +// Allow duplicate transitive deps in this benchmark-only crate. +#![allow(clippy::multiple_crate_versions)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Write; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const DEFAULT_SCENARIO_DIR: &str = "crates/contextforge_benchmark_runner/assets/scenarios"; +pub const DEFAULT_OUTPUT_ROOT: &str = "reports/benchmarks"; +pub const DEFAULT_GOSE_BIN: &str = "contextforge_goose"; + +fn log_progress(message: impl AsRef) { + println!("[benchmark] {}", message.as_ref()); + let _ = std::io::stdout().flush(); +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SuiteDocument { + pub suite: SuiteMeta, + #[serde(default)] + pub defaults: ScenarioTemplate, + #[serde(default)] + pub scenario: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SuiteMeta { + #[serde(default)] + pub name: String, + #[serde(default)] + pub description: String, + #[serde(default = "default_output_root")] + pub output_root: String, + #[serde(default)] + pub continue_on_failure: bool, + #[serde(default)] + pub save_intermediate_artifacts: bool, + #[serde(default)] + pub flamegraph_enabled: bool, + #[serde(default)] + pub baseline_run: Option, + #[serde(default)] + pub baseline_rps_drop_pct: Option, + #[serde(default)] + pub baseline_p95_regression_pct: Option, + #[serde(default)] + pub baseline_failure_increase: Option, +} + +fn default_output_root() -> String { + DEFAULT_OUTPUT_ROOT.to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ScenarioTemplate { + #[serde(default)] + pub setup: SetupConfig, + #[serde(default)] + pub build: BuildConfig, + #[serde(default)] + pub runtime: RuntimeConfig, + #[serde(default)] + pub gateway: GatewayConfig, + #[serde(default)] + pub load: LoadConfig, + #[serde(default)] + pub measurement: MeasurementConfig, + #[serde(default)] + pub profiling: ProfilingConfig, + #[serde(default)] + pub execution: ExecutionConfig, + #[serde(default)] + pub requests: RequestsConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ScenarioEntry { + #[serde(default)] + pub name: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub scenario_type: String, + #[serde(default)] + pub setup: SetupConfig, + #[serde(default)] + pub build: BuildConfig, + #[serde(default)] + pub runtime: RuntimeConfig, + #[serde(default)] + pub gateway: GatewayConfig, + #[serde(default)] + pub load: LoadConfig, + #[serde(default)] + pub measurement: MeasurementConfig, + #[serde(default)] + pub profiling: ProfilingConfig, + #[serde(default)] + pub execution: ExecutionConfig, + #[serde(default)] + pub requests: RequestsConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SetupConfig { + #[serde(default)] + pub target_kind: String, + #[serde(default)] + pub auth_mode: String, + #[serde(default)] + pub plugins_enabled: bool, + #[serde(default)] + pub expected_mcp_runtime: Option, + #[serde(default)] + pub expected_mcp_runtime_mode: Option, + #[serde(default)] + pub expected_a2a_runtime: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct BuildConfig { + #[serde(default)] + pub rust_plugins: bool, + #[serde(default)] + pub profiling_image: bool, + #[serde(default)] + pub container_file: String, + #[serde(default)] + pub image_name: String, + #[serde(default)] + pub image_tag: String, + #[serde(default)] + pub rebuild_policy: String, + #[serde(default)] + pub args: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RuntimeConfig { + #[serde(default)] + pub http_server: String, + #[serde(default)] + pub host: String, + #[serde(default)] + pub transport_type: String, + #[serde(default)] + pub gunicorn: GunicornConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct GunicornConfig { + #[serde(default)] + pub workers: Option, + #[serde(default)] + pub timeout: Option, + #[serde(default)] + pub graceful_timeout: Option, + #[serde(default)] + pub keep_alive: Option, + #[serde(default)] + pub max_requests: Option, + #[serde(default)] + pub max_requests_jitter: Option, + #[serde(default)] + pub backlog: Option, + #[serde(default)] + pub preload_app: Option, + #[serde(default)] + pub dev_mode: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct GatewayConfig { + #[serde(default)] + pub disable_access_log: bool, + #[serde(default)] + pub templates_auto_reload: bool, + #[serde(default)] + pub structured_logging_database_enabled: bool, + #[serde(default)] + pub sqlalchemy_echo: bool, + #[serde(default)] + pub log_level: String, + #[serde(default)] + pub trust_proxy_auth: bool, // pragma: allowlist secret + #[serde(default)] + pub environment: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct LoadConfig { + #[serde(default = "default_driver")] + pub driver: String, + #[serde(default)] + pub headless: bool, + #[serde(default)] + pub only_summary: bool, + #[serde(default)] + pub html_report: bool, + #[serde(default)] + pub users: Option, + #[serde(default)] + pub spawn_rate: Option, + #[serde(default)] + pub run_time: Option, + #[serde(default)] + pub request_count: Option, + #[serde(default)] + pub seed: Option, + #[serde(default)] + pub target_service: String, + #[serde(default)] + pub host: Option, + #[serde(default)] + pub extra_args: Vec, + #[serde(default)] + pub env: BTreeMap, + #[serde(default)] + pub workload: WorkloadConfig, +} + +fn default_driver() -> String { + DEFAULT_GOSE_BIN.to_string() +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct WorkloadConfig { + #[serde(default)] + pub fallback_endpoint: Option, + #[serde(default)] + pub endpoints: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct EndpointOverride { + #[serde(default)] + pub enabled: Option, + #[serde(default)] + pub weight: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct MeasurementConfig { + #[serde(default)] + pub warmup_seconds: u32, + #[serde(default)] + pub measure_seconds: u32, + #[serde(default)] + pub profile_seconds: u32, + #[serde(default)] + pub cooldown_seconds: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ProfilingConfig { + #[serde(default)] + pub enabled: bool, + #[serde(default)] + pub tools: Vec, + #[serde(default)] + pub duration_seconds: u32, + #[serde(default)] + pub required: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ExecutionConfig { + #[serde(default)] + pub retry_enabled: bool, + #[serde(default)] + pub max_attempts: u32, + #[serde(default)] + pub capture_logs: bool, + #[serde(default)] + pub save_raw_results: bool, + #[serde(default)] + pub reuse_stack: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RequestsConfig { + #[serde(default)] + pub enabled_groups: Vec, + #[serde(default)] + pub disabled_groups: Vec, + #[serde(default)] + pub enabled_endpoints: Vec, + #[serde(default)] + pub disabled_endpoints: Vec, + #[serde(default)] + pub enabled_tags: Vec, + #[serde(default)] + pub disabled_tags: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResolvedSuite { + pub suite: SuiteMeta, + pub scenarios: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResolvedScenario { + pub name: String, + pub description: String, + pub scenario_type: String, + pub setup: SetupConfig, + pub build: BuildConfig, + pub runtime: RuntimeConfig, + pub gateway: GatewayConfig, + pub load: LoadConfig, + pub measurement: MeasurementConfig, + pub profiling: ProfilingConfig, + pub execution: ExecutionConfig, + pub requests: RequestsConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestDefinition { + pub name: String, + pub group: String, + pub tags: BTreeSet, + pub weight: u32, + pub request: RequestSpec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestSpec { + pub kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub payload: Option, + #[serde(default)] + pub auth: bool, // pragma: allowlist secret + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expect_result_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expect_result_min_items: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expect_list_min_items: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expect_list_item_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expect_content_text: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CommandSpec { + pub command: String, + pub args: Vec, + pub env: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RuntimeChoice { + pub engine: String, + pub compose_cmd: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ScenarioSummary { + pub scenario: String, + pub status: String, + pub setup: SetupConfig, + pub runtime: RuntimeConfig, + pub load: LoadConfig, + pub measurement: MeasurementConfig, + pub profiling: ProfilingConfig, + pub goose: Value, + pub endpoint_metrics: Value, + pub flamegraph_run: Value, + pub log_paths: Vec, + pub artifacts: BTreeMap, +} + +mod lib_parts; + +pub use lib_parts::catalog::{ + benchmark_catalog, benchmark_request_names, resolve_requests_from_workload, +}; +pub use lib_parts::reporting::{ + build_comparison_report, build_run_summary, collect_endpoint_metrics, regenerate_reports, + write_goose_stats_csv, +}; +pub use lib_parts::runtime::{build_goose_command, detect_runtime, run_benchmark, scenario_env}; +pub use lib_parts::scenario_loading::{ + discover_scenarios, load_suite, repo_root, resolve_profile_path, scenario_root, + validate_scenario, +}; + +#[cfg(test)] +use lib_parts::compose::run_command_streaming; +#[cfg(test)] +use lib_parts::runtime::{determine_scenario_success, has_endpoint_failures}; +#[cfg(test)] +use lib_parts::runtime_orchestration::{ + benchmark_token_command, write_compose_override, yaml_strings, +}; +#[cfg(test)] +#[path = "lib_parts/tests.rs"] +mod tests; diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/catalog.rs b/crates/contextforge_benchmark_runner/src/lib_parts/catalog.rs new file mode 100644 index 0000000000..6a94d0c8f3 --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/catalog.rs @@ -0,0 +1,391 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde_json::{Value, json}; + +use crate::{RequestDefinition, RequestSpec, WorkloadConfig}; + +fn payload_root(root: &Path) -> PathBuf { + root.join("crates/contextforge_benchmark_runner/assets/payloads") +} + +fn load_payload(root: &Path, group: &str, name: &str) -> Result { + let path = payload_root(root).join(group).join(name); + let raw = + fs::read_to_string(&path).with_context(|| format!("failed to read {}", path.display()))?; + serde_json::from_str(&raw).with_context(|| format!("failed to parse {}", path.display())) +} + +pub fn benchmark_catalog(root: &Path) -> Result> { + let default_server = "9779b6698cbd4b4995ee04a4fab38737".to_string(); // pragma: allowlist secret + Ok(vec![ + RequestDefinition { + name: "/health".into(), + group: "health".into(), + tags: set(&["health"]), + weight: 10, + request: RequestSpec { + kind: "get".into(), + path: Some("/health".into()), + payload: None, + auth: false, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/ready".into(), + group: "health".into(), + tags: set(&["health"]), + weight: 4, + request: RequestSpec { + kind: "get".into(), + path: Some("/ready".into()), + payload: None, + auth: false, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/admin/plugins".into(), + group: "admin".into(), + tags: set(&["admin", "plugins"]), + weight: 2, + request: RequestSpec { + kind: "get".into(), + path: Some("/admin/plugins".into()), + payload: None, + auth: true, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/servers".into(), + group: "servers".into(), + tags: set(&["servers", "rest", "discovery"]), + weight: 5, + request: RequestSpec { + kind: "get".into(), + path: Some("/servers".into()), + payload: None, + auth: true, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: Some(1), + expect_list_item_name: Some("Fast Time Server".into()), + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/a2a".into(), + group: "a2a".into(), + tags: set(&["a2a", "rest", "discovery"]), + weight: 3, + request: RequestSpec { + kind: "get".into(), + path: Some("/a2a".into()), + payload: None, + auth: true, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: Some(1), + expect_list_item_name: Some("a2a-echo-agent".into()), + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/a2a/a2a-echo-agent/invoke".into(), + group: "a2a".into(), + tags: set(&["a2a", "invoke", "echo"]), + weight: 8, + request: RequestSpec { + kind: "post".into(), + path: Some("/a2a/a2a-echo-agent/invoke".into()), + payload: Some( + json!({"parameters":{"message":{"kind":"message","role":"user","messageId":"benchmark-a2a-invoke","parts":[{"kind":"text","text":"benchmark ping"}]}},"interaction_type":"query"}), + ), + auth: true, + server_id: None, + expect_result_key: Some("result".into()), + expect_result_min_items: None, + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp tools/list".into(), + group: "mcp".into(), + tags: set(&["mcp", "tools", "discovery"]), + weight: 3, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "tools", "list_tools.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("tools".into()), + expect_result_min_items: Some(2), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp tools/call fast-time-get-system-time".into(), + group: "tools".into(), + tags: set(&["tools", "mcp", "plugin-heavy"]), + weight: 8, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "tools", "get_system_time.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(true), + name: None, + }, + }, + RequestDefinition { + name: "/mcp tools/call fast-time-convert-time".into(), + group: "tools".into(), + tags: set(&["tools", "mcp", "plugin-heavy"]), + weight: 6, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "tools", "convert_time.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(true), + name: None, + }, + }, + RequestDefinition { + name: "/resources".into(), + group: "resources".into(), + tags: set(&["resources", "rest"]), + weight: 3, + request: RequestSpec { + kind: "get".into(), + path: Some("/resources".into()), + payload: None, + auth: true, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: Some(1), + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp resources/list".into(), + group: "mcp".into(), + tags: set(&["mcp", "resources"]), + weight: 3, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "resources", "list_resources.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("resources".into()), + expect_result_min_items: Some(1), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp resources/read timezone://info".into(), + group: "resources".into(), + tags: set(&["resources", "mcp", "plugin-heavy"]), + weight: 5, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "resources", "read_timezone_info.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("contents".into()), + expect_result_min_items: Some(1), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp resources/read time://current/world".into(), + group: "resources".into(), + tags: set(&["resources", "mcp", "plugin-heavy"]), + weight: 4, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "resources", "read_world_times.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("contents".into()), + expect_result_min_items: Some(1), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/prompts".into(), + group: "prompts".into(), + tags: set(&["prompts", "rest"]), + weight: 3, + request: RequestSpec { + kind: "get".into(), + path: Some("/prompts".into()), + payload: None, + auth: true, + server_id: None, + expect_result_key: None, + expect_result_min_items: None, + expect_list_min_items: Some(1), + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp prompts/list".into(), + group: "mcp".into(), + tags: set(&["mcp", "prompts"]), + weight: 3, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "prompts", "list_prompts.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("prompts".into()), + expect_result_min_items: Some(1), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp prompts/get fast-time-schedule-meeting".into(), + group: "prompts".into(), + tags: set(&["prompts", "mcp", "plugin-heavy"]), + weight: 5, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "prompts", "get_schedule_meeting.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("messages".into()), + expect_result_min_items: Some(1), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + RequestDefinition { + name: "/mcp prompts/get fast-time-compare-timezones".into(), + group: "prompts".into(), + tags: set(&["prompts", "mcp", "plugin-heavy"]), + weight: 4, + request: RequestSpec { + kind: "mcp".into(), + path: None, + payload: Some(load_payload(root, "prompts", "get_compare_timezones.json")?), + auth: true, + server_id: Some(default_server.clone()), + expect_result_key: Some("messages".into()), + expect_result_min_items: Some(1), + expect_list_min_items: None, + expect_list_item_name: None, + expect_content_text: Some(false), + name: None, + }, + }, + ]) +} + +fn set(items: &[&str]) -> BTreeSet { + items.iter().map(|item| (*item).to_string()).collect() +} + +pub fn benchmark_request_names(root: &Path) -> Result> { + Ok(benchmark_catalog(root)? + .into_iter() + .map(|request| request.name) + .collect()) +} + +pub fn resolve_requests_from_workload( + root: &Path, + workload: &WorkloadConfig, +) -> Result> { + let catalog = benchmark_catalog(root)?; + if workload.endpoints.is_empty() { + return Ok(catalog); + } + let mut requests = Vec::new(); + for request in catalog.iter() { + if let Some(override_config) = workload.endpoints.get(&request.name) { + let enabled = override_config.enabled.unwrap_or(true); + let weight = override_config.weight.unwrap_or(request.weight); + if enabled && weight > 0 { + let mut resolved = request.clone(); + resolved.weight = weight; + requests.push(resolved); + } + } + } + if !requests.is_empty() { + return Ok(requests); + } + let fallback = workload.fallback_endpoint.as_deref().unwrap_or("/health"); + Ok(catalog + .into_iter() + .filter(|request| request.name == fallback) + .collect()) +} diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/compose.rs b/crates/contextforge_benchmark_runner/src/lib_parts/compose.rs new file mode 100644 index 0000000000..831c1d584e --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/compose.rs @@ -0,0 +1,243 @@ +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::thread; + +use anyhow::{Result, anyhow, bail}; +use chrono::Utc; +use serde_json::{Value, json}; + +use crate::lib_parts::{ + build_goose_command, ensure_benchmark_image, run_compose, slug, wait_for_gateway_health, + wait_for_service, write_compose_override, +}; +use crate::{CommandSpec, ResolvedScenario, RuntimeChoice, log_progress}; + +#[derive(Debug)] +pub(crate) struct RunOutput { + pub(crate) success: bool, + pub(crate) stdout: String, + pub(crate) stderr: String, +} + +pub(crate) fn run_command_spec( + root: &Path, + spec: &CommandSpec, + token: Option<&str>, +) -> Result { + let mut command = Command::new(&spec.command); + command + .args(&spec.args) + .current_dir(root) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (key, value) in &spec.env { + command.env(key, value); + } + if let Some(token) = token { + command.env("MCPGATEWAY_BEARER_TOKEN", token); + } + run_command_streaming(&mut command, |stream, line| { + log_progress(format!("{stream}: {line}")); + }) +} + +pub(crate) fn run_command_streaming(command: &mut Command, mut on_line: F) -> Result +where + F: FnMut(&str, &str), +{ + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = command.spawn()?; + let stdout = child + .stdout + .take() + .ok_or_else(|| anyhow!("missing child stdout"))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| anyhow!("missing child stderr"))?; + let (sender, receiver) = mpsc::channel::<(&'static str, String)>(); + + let stdout_sender = sender.clone(); + thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + match line { + Ok(value) => { + let _ = stdout_sender.send(("stdout", value)); + } + Err(error) => { + let _ = stdout_sender.send(("stderr", format!("stdout read error: {error}"))); + break; + } + } + } + }); + + let stderr_sender = sender.clone(); + thread::spawn(move || { + for line in BufReader::new(stderr).lines() { + match line { + Ok(value) => { + let _ = stderr_sender.send(("stderr", value)); + } + Err(error) => { + let _ = stderr_sender.send(("stderr", format!("stderr read error: {error}"))); + break; + } + } + } + }); + drop(sender); + + let mut stdout_log = String::new(); + let mut stderr_log = String::new(); + for (stream, line) in receiver { + on_line(stream, &line); + match stream { + "stdout" => { + stdout_log.push_str(&line); + stdout_log.push('\n'); + } + "stderr" => { + stderr_log.push_str(&line); + stderr_log.push('\n'); + } + _ => {} + } + } + + let status = child.wait()?; + Ok(RunOutput { + success: status.success(), + stdout: stdout_log, + stderr: stderr_log, + }) +} + +pub(crate) fn run_flamegraph( + root: &Path, + scenario: &ResolvedScenario, + scenario_dir: &Path, + token: Option<&str>, +) -> Result { + let svg = scenario_dir.join("goose_flamegraph_flamegraph.svg"); + let spec = build_goose_command(root, scenario, scenario_dir, "goose_flamegraph", true); + let output = run_command_spec(root, &spec, token)?; + Ok(json!({ + "status": if output.success { "ok" } else { "failed" }, + "stdout": output.stdout, + "stderr": output.stderr, + "svg": svg.display().to_string(), + "html_report": scenario_dir.join("goose_flamegraph_report.html").display().to_string(), + })) +} + +fn compose_args(runtime: &RuntimeChoice, project_name: &str, override_path: &Path) -> Vec { + let mut args = runtime.compose_cmd.clone(); + args.push("-p".to_string()); + args.push(project_name.to_string()); + args.push("-f".to_string()); + args.push(override_path.display().to_string()); + args +} + +pub(crate) fn start_stack( + root: &Path, + runtime: &RuntimeChoice, + scenario: &ResolvedScenario, + scenario_dir: &Path, +) -> Result<(Vec, String)> { + let image_name = ensure_benchmark_image(root, runtime, scenario)?; + let project = format!("bench-{}-{}", slug(&scenario.name), Utc::now().timestamp()); + let override_path = write_compose_override(root, scenario, scenario_dir, &image_name)?; + let compose = compose_args(runtime, &project, &override_path); + let mut services = vec!["postgres", "redis", "pgbouncer", "gateway"]; + if uses_fast_time_fixture(scenario) { + services.push("fast_time_server"); + services.push("register_fast_time"); + } + if uses_a2a_fixture(scenario) { + services.push("a2a_echo_agent"); + services.push("register_a2a_echo"); + } + if scenario.load.target_service != "gateway" { + services.push("nginx"); + } + for service in ["postgres", "redis", "pgbouncer", "gateway"] { + log_progress(format!("Compose up: {service}")); + run_compose(root, &compose, &["up", "-d", "--no-build", service])?; + log_progress(format!("Waiting for service health: {service}")); + wait_for_service(runtime, &compose, service, 120)?; + } + if !wait_for_gateway_health(&compose, 120)? { + bail!( + "gateway health check failed for scenario '{}'", + scenario.name + ); + } + if uses_fast_time_fixture(scenario) { + log_progress("Compose up: fast_time_server"); + run_compose( + root, + &compose, + &["up", "-d", "--no-build", "fast_time_server"], + )?; + log_progress("Waiting for service health: fast_time_server"); + wait_for_service(runtime, &compose, "fast_time_server", 60)?; + log_progress("Compose up: register_fast_time"); + run_compose( + root, + &compose, + &["up", "-d", "--no-build", "register_fast_time"], + )?; + } + if uses_a2a_fixture(scenario) { + log_progress("Compose up: a2a_echo_agent"); + run_compose( + root, + &compose, + &["up", "-d", "--no-build", "a2a_echo_agent"], + )?; + log_progress("Waiting for service health: a2a_echo_agent"); + wait_for_service(runtime, &compose, "a2a_echo_agent", 60)?; + log_progress("Compose up: register_a2a_echo"); + run_compose( + root, + &compose, + &["up", "-d", "--no-build", "register_a2a_echo"], + )?; + } + if scenario.load.target_service != "gateway" { + log_progress("Compose up: nginx"); + run_compose(root, &compose, &["up", "-d", "--no-build", "nginx"])?; + log_progress("Waiting for service health: nginx"); + wait_for_service(runtime, &compose, "nginx", 60)?; + } + Ok((compose, project)) +} + +pub(crate) fn stop_stack(compose_args: &[String]) { + let _ = Command::new(&compose_args[0]) + .args(&compose_args[1..]) + .args(["down", "--remove-orphans"]) + .status(); +} + +pub(crate) fn uses_fast_time_fixture(scenario: &ResolvedScenario) -> bool { + scenario + .load + .workload + .endpoints + .keys() + .any(|name| name.contains("fast-time") || name.contains("/mcp ")) +} + +pub(crate) fn uses_a2a_fixture(scenario: &ResolvedScenario) -> bool { + scenario + .load + .workload + .endpoints + .keys() + .any(|name| name.contains("/a2a")) +} diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/mod.rs b/crates/contextforge_benchmark_runner/src/lib_parts/mod.rs new file mode 100644 index 0000000000..95b74f5b80 --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/mod.rs @@ -0,0 +1,22 @@ +pub(crate) mod catalog; +pub(crate) mod compose; +pub(crate) mod reporting; +pub(crate) mod runtime; +pub(crate) mod runtime_orchestration; +pub(crate) mod scenario_loading; + +pub(crate) use catalog::{benchmark_request_names, resolve_requests_from_workload}; +pub(crate) use compose::{ + run_command_spec, run_flamegraph, start_stack, stop_stack, uses_a2a_fixture, + uses_fast_time_fixture, +}; +pub(crate) use reporting::{ + build_comparison_report, build_run_summary, collect_endpoint_metrics, render_comparison_html, + render_comparison_markdown, render_run_summary_markdown, slug, write_goose_stats_csv, + write_json, write_text, +}; +pub(crate) use runtime::build_goose_command; +pub(crate) use runtime_orchestration::{ + benchmark_token, ensure_benchmark_image, run_compose, wait_for_gateway_health, + wait_for_service, write_compose_override, +}; diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/reporting.rs b/crates/contextforge_benchmark_runner/src/lib_parts/reporting.rs new file mode 100644 index 0000000000..5cd6fe6072 --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/reporting.rs @@ -0,0 +1,482 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use csv::{ReaderBuilder, WriterBuilder}; +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::{MeasurementConfig, ScenarioSummary, SuiteMeta}; + +pub fn write_goose_stats_csv(request_log_path: &Path, csv_prefix: &Path) -> Result<()> { + if !request_log_path.exists() { + return Ok(()); + } + let mut rows = Vec::new(); + let mut reader = ReaderBuilder::new().from_path(request_log_path)?; + for row in reader.deserialize::>() { + rows.push(row?); + } + let mut groups: BTreeMap>> = BTreeMap::new(); + for row in rows.iter() { + let name = row + .get("name") + .cloned() + .unwrap_or_else(|| "unknown".to_string()); + groups.entry(name).or_default().push(row.clone()); + } + let stats_path = PathBuf::from(format!("{}_stats.csv", csv_prefix.display())); + let mut writer = WriterBuilder::new().from_path(stats_path)?; + writer.write_record([ + "Name", + "Request Count", + "Failure Count", + "Average Response Time", + "Min Response Time", + "Max Response Time", + "50%", + "95%", + "99%", + ])?; + let aggregate = aggregate_rows("Aggregated", &rows); + writer.serialize(&aggregate)?; + for (name, group) in groups { + writer.serialize(aggregate_rows(&name, &group))?; + } + writer.flush()?; + + let mut by_second: BTreeMap>> = BTreeMap::new(); + for row in rows.iter() { + let second = row + .get("elapsed") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0.0) as i64; + by_second.entry(second).or_default().push(row.clone()); + } + let history_path = PathBuf::from(format!("{}_stats_history.csv", csv_prefix.display())); + let mut history = WriterBuilder::new().from_path(history_path)?; + history.write_record([ + "Timestamp", + "Requests/s", + "95%", + "99%", + "Total Request Count", + "Total Failure Count", + "Total Median Response Time", + "Total Average Response Time", + ])?; + let mut cumulative = Vec::new(); + let mut cumulative_failures = 0u64; + for (second, batch) in by_second { + cumulative.extend(batch.clone()); + cumulative_failures += batch + .iter() + .filter(|row| { + row.get("success") + .map(|value| value != "true") + .unwrap_or(false) + }) + .count() as u64; + let response_times = batch.iter().map(response_time).collect::>(); + let cumulative_times = cumulative.iter().map(response_time).collect::>(); + history.write_record(&[ + second.to_string(), + batch.len().to_string(), + percentile(&response_times, 0.95).to_string(), + percentile(&response_times, 0.99).to_string(), + cumulative.len().to_string(), + cumulative_failures.to_string(), + percentile(&cumulative_times, 0.50).to_string(), + average(&cumulative_times).to_string(), + ])?; + } + history.flush()?; + Ok(()) +} + +#[derive(Serialize)] +struct GooseStatsRow { + #[serde(rename = "Name")] + name: String, + #[serde(rename = "Request Count")] + request_count: String, + #[serde(rename = "Failure Count")] + failure_count: String, + #[serde(rename = "Average Response Time")] + average_response_time: String, + #[serde(rename = "Min Response Time")] + min_response_time: String, + #[serde(rename = "Max Response Time")] + max_response_time: String, + #[serde(rename = "50%")] + p50: String, + #[serde(rename = "95%")] + p95: String, + #[serde(rename = "99%")] + p99: String, +} + +fn aggregate_rows(name: &str, rows: &[BTreeMap]) -> GooseStatsRow { + let response_times = rows.iter().map(response_time).collect::>(); + let failures = rows + .iter() + .filter(|row| { + row.get("success") + .map(|value| value != "true") + .unwrap_or(false) + }) + .count(); + GooseStatsRow { + name: name.to_string(), + request_count: rows.len().to_string(), + failure_count: failures.to_string(), + average_response_time: average(&response_times).to_string(), + min_response_time: response_times + .iter() + .cloned() + .fold(0.0_f64, f64::min) + .to_string(), + max_response_time: response_times + .iter() + .cloned() + .fold(0.0_f64, f64::max) + .to_string(), + p50: percentile(&response_times, 0.50).to_string(), + p95: percentile(&response_times, 0.95).to_string(), + p99: percentile(&response_times, 0.99).to_string(), + } +} + +fn response_time(row: &BTreeMap) -> f64 { + row.get("response_time") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0.0) +} + +fn average(values: &[f64]) -> f64 { + if values.is_empty() { + 0.0 + } else { + values.iter().sum::() / values.len() as f64 + } +} + +fn percentile(values: &[f64], pct: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + let mut sorted = values.to_vec(); + sorted.sort_by(|left, right| left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal)); + let index = ((sorted.len().saturating_sub(1)) as f64 * pct).round() as usize; + sorted[index.min(sorted.len().saturating_sub(1))] +} + +pub fn collect_endpoint_metrics( + csv_prefix: &Path, + measurement: &MeasurementConfig, +) -> Result { + let path = PathBuf::from(format!("{}_stats.csv", csv_prefix.display())); + if !path.exists() { + return Ok(json!({"status":"unavailable","reason":"Goose stats CSV not found"})); + } + let mut reader = ReaderBuilder::new().from_path(path)?; + let rows = reader + .deserialize::>() + .collect::, _>>()?; + let aggregate = rows + .iter() + .find(|row| { + row.get("Name") + .map(|value| value == "Aggregated") + .unwrap_or(false) + }) + .cloned() + .unwrap_or_default(); + let endpoints = rows + .into_iter() + .filter(|row| { + row.get("Name") + .map(|value| value != "Aggregated") + .unwrap_or(false) + }) + .collect::>(); + let window = measurement_window_summary(csv_prefix, measurement)?; + Ok(json!({ + "status":"ok", + "aggregated": aggregate, + "measurement_window": window, + "endpoints": endpoints, + })) +} + +fn measurement_window_summary(csv_prefix: &Path, measurement: &MeasurementConfig) -> Result { + let path = PathBuf::from(format!("{}_stats_history.csv", csv_prefix.display())); + if !path.exists() { + return Ok(json!({"status":"unavailable","reason":"Goose stats history CSV not found"})); + } + let mut reader = ReaderBuilder::new().from_path(path)?; + let rows = reader + .deserialize::>() + .collect::, _>>()?; + if rows.is_empty() { + return Ok(json!({"status":"unavailable","reason":"Goose stats history CSV was empty"})); + } + let warmup = measurement.warmup_seconds as i64; + let cooldown = measurement.cooldown_seconds as i64; + let max_timestamp = rows + .iter() + .filter_map(|row| { + row.get("Timestamp") + .and_then(|value| value.parse::().ok()) + }) + .max() + .unwrap_or(0); + let window = rows + .iter() + .filter(|row| { + let ts = row + .get("Timestamp") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + ts >= warmup && ts <= (max_timestamp - cooldown) + }) + .cloned() + .collect::>(); + if window.is_empty() { + return Ok( + json!({"status":"unavailable","reason":"Measurement window did not overlap with Goose stats history"}), + ); + } + Ok(json!({ + "status":"ok", + "source":"goose_stats_history_window", + "warmup_seconds": measurement.warmup_seconds, + "measure_seconds": measurement.measure_seconds, + "cooldown_seconds": measurement.cooldown_seconds, + "samples": window.len(), + "aggregated": { + "Request Count": window.last().and_then(|row| row.get("Total Request Count")).cloned().unwrap_or_else(|| "0".to_string()), + "Failure Count": window.last().and_then(|row| row.get("Total Failure Count")).cloned().unwrap_or_else(|| "0".to_string()), + "Requests/s": average(&window.iter().filter_map(|row| row.get("Requests/s").and_then(|value| value.parse::().ok())).collect::>()), + "95%": window.iter().filter_map(|row| row.get("95%").and_then(|value| value.parse::().ok())).fold(0.0_f64, f64::max), + "99%": window.iter().filter_map(|row| row.get("99%").and_then(|value| value.parse::().ok())).fold(0.0_f64, f64::max), + } + })) +} + +pub fn build_run_summary(suite: &SuiteMeta, summaries: &[ScenarioSummary]) -> Value { + json!({ + "suite_name": suite.name, + "scenario_count": summaries.len(), + "scenarios": summaries.iter().map(|summary| json!({ + "scenario": summary.scenario, + "status": summary.status, + "runtime": summary.runtime.http_server, + "auth_mode": summary.setup.auth_mode, + })).collect::>() + }) +} + +pub fn build_comparison_report(summaries: &[ScenarioSummary]) -> Value { + let mut comparisons = Vec::new(); + for pair in summaries.windows(2) { + let left = &pair[0]; + let right = &pair[1]; + let left_rps = metric_value(&left.endpoint_metrics, "Requests/s"); + let right_rps = metric_value(&right.endpoint_metrics, "Requests/s"); + let left_p95 = metric_value(&left.endpoint_metrics, "95%"); + let right_p95 = metric_value(&right.endpoint_metrics, "95%"); + comparisons.push(json!({ + "left": left.scenario, + "right": right.scenario, + "rps_delta": right_rps - left_rps, + "p95_delta": right_p95 - left_p95, + "changed_dimensions": changed_dimensions(left, right), + })); + } + json!({ "comparisons": comparisons }) +} + +fn metric_value(metrics: &Value, key: &str) -> f64 { + metrics + .get("measurement_window") + .and_then(|window| window.get("aggregated")) + .and_then(|aggregated| aggregated.get(key)) + .and_then(|value| { + value + .as_f64() + .or_else(|| value.as_str().and_then(|inner| inner.parse::().ok())) + }) + .unwrap_or(0.0) +} + +fn changed_dimensions(left: &ScenarioSummary, right: &ScenarioSummary) -> Vec { + let mut dimensions = Vec::new(); + if left.runtime.http_server != right.runtime.http_server { + dimensions.push("runtime.http_server".to_string()); + } + if left.setup.auth_mode != right.setup.auth_mode { + dimensions.push("setup.auth_mode".to_string()); + } + if left.load.driver != right.load.driver { + dimensions.push("load.driver".to_string()); + } + dimensions +} + +pub fn regenerate_reports(run_dir: &Path) -> Result { + let mut summaries = Vec::new(); + let scenarios_dir = run_dir.join("scenarios"); + for entry in fs::read_dir(&scenarios_dir)? { + let path = entry?.path().join("summary.json"); + if path.exists() { + let raw = fs::read_to_string(&path)?; + summaries.push(serde_json::from_str::(&raw)?); + } + } + let suite = SuiteMeta { + name: run_dir + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("benchmark-run") + .to_string(), + ..SuiteMeta::default() + }; + let run_summary = build_run_summary(&suite, &summaries); + write_json(&run_dir.join("run_summary.json"), &run_summary)?; + write_text( + &run_dir.join("run_summary.md"), + &render_run_summary_markdown(&run_summary), + )?; + let comparison = build_comparison_report(&summaries); + write_json( + &run_dir.join("scenario_comparison_report.json"), + &comparison, + )?; + write_text( + &run_dir.join("scenario_comparison_report.md"), + &render_comparison_markdown(&comparison), + )?; + write_text( + &run_dir.join("scenario_comparison_report.html"), + &render_comparison_html(&comparison), + )?; + Ok(run_dir.to_path_buf()) +} + +pub(crate) fn render_run_summary_markdown(summary: &Value) -> String { + let mut lines = vec![ + "# Benchmark Run Summary".to_string(), + String::new(), + format!( + "- Suite: `{}`", + summary + .get("suite_name") + .and_then(Value::as_str) + .unwrap_or("unknown") + ), + format!( + "- Scenario count: `{}`", + summary + .get("scenario_count") + .and_then(Value::as_u64) + .unwrap_or(0) + ), + String::new(), + ]; + if let Some(items) = summary.get("scenarios").and_then(Value::as_array) { + for item in items { + lines.push(format!( + "- `{}`: status=`{}` runtime=`{}` auth=`{}`", // pragma: allowlist secret + item.get("scenario") + .and_then(Value::as_str) + .unwrap_or("unknown"), + item.get("status") + .and_then(Value::as_str) + .unwrap_or("unknown"), + item.get("runtime") + .and_then(Value::as_str) + .unwrap_or("unknown"), + item.get("auth_mode") + .and_then(Value::as_str) + .unwrap_or("unknown") + )); + } + } + lines.join("\n") +} + +pub(crate) fn render_comparison_markdown(report: &Value) -> String { + let mut lines = vec!["# Scenario Comparison Report".to_string(), String::new()]; + if let Some(items) = report.get("comparisons").and_then(Value::as_array) { + for item in items { + lines.push(format!( + "- `{}` vs `{}`: rps_delta=`{:.2}` p95_delta=`{:.2}`", + item.get("left").and_then(Value::as_str).unwrap_or("left"), + item.get("right").and_then(Value::as_str).unwrap_or("right"), + item.get("rps_delta").and_then(Value::as_f64).unwrap_or(0.0), + item.get("p95_delta").and_then(Value::as_f64).unwrap_or(0.0) + )); + } + } + lines.join("\n") +} + +pub(crate) fn render_comparison_html(report: &Value) -> String { + let mut rows = String::new(); + if let Some(items) = report.get("comparisons").and_then(Value::as_array) { + for item in items { + rows.push_str(&format!( + "{}{}{:.2}{:.2}", + html_escape(item.get("left").and_then(Value::as_str).unwrap_or("left")), + html_escape(item.get("right").and_then(Value::as_str).unwrap_or("right")), + item.get("rps_delta").and_then(Value::as_f64).unwrap_or(0.0), + item.get("p95_delta").and_then(Value::as_f64).unwrap_or(0.0), + )); + } + } + format!( + "Scenario Comparison Report

Scenario Comparison Report

{rows}
LeftRightRPS DeltaP95 Delta
" + ) +} + +fn html_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +pub(crate) fn write_json(path: &Path, payload: &T) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, serde_json::to_vec_pretty(payload)?)?; + Ok(()) +} + +pub(crate) fn write_text(path: &Path, payload: &str) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, payload)?; + Ok(()) +} + +pub(crate) fn slug(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '-' + } + }) + .collect::() + .split('-') + .filter(|part| !part.is_empty()) + .collect::>() + .join("-") +} diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/runtime.rs b/crates/contextforge_benchmark_runner/src/lib_parts/runtime.rs new file mode 100644 index 0000000000..b851bb2e5b --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/runtime.rs @@ -0,0 +1,497 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anyhow::{Result, bail}; +use chrono::Utc; +use serde_json::{Value, json}; + +use crate::lib_parts::scenario_loading::{discover_scenarios, load_suite}; +use crate::lib_parts::{ + benchmark_token, build_comparison_report, build_run_summary, collect_endpoint_metrics, + render_comparison_html, render_comparison_markdown, render_run_summary_markdown, + resolve_requests_from_workload, run_command_spec, run_flamegraph, start_stack, stop_stack, + wait_for_gateway_health, write_goose_stats_csv, write_json, write_text, +}; +use crate::{ + CommandSpec, DEFAULT_GOSE_BIN, LoadConfig, ResolvedScenario, ResolvedSuite, RuntimeChoice, + ScenarioSummary, SuiteMeta, log_progress, +}; + +pub fn detect_runtime() -> Result { + let preferred = std::env::var("CONTAINER_RUNTIME").unwrap_or_else(|_| "docker".to_string()); + let candidates = if preferred == "podman" { + vec![ + ("podman", vec!["podman".to_string(), "compose".to_string()]), + ("docker", vec!["docker".to_string(), "compose".to_string()]), + ] + } else { + vec![ + ("docker", vec!["docker".to_string(), "compose".to_string()]), + ("podman", vec!["podman".to_string(), "compose".to_string()]), + ] + }; + for (engine, compose_cmd) in candidates { + if command_ok(engine, &["--version"]) + && command_ok( + &compose_cmd[0], + &compose_cmd[1..] + .iter() + .map(String::as_str) + .chain(["version"]) + .collect::>(), + ) + { + return Ok(RuntimeChoice { + engine: engine.to_string(), + compose_cmd, + }); + } + } + bail!("could not detect a working docker/podman runtime") +} + +fn command_ok(command: &str, args: &[&str]) -> bool { + Command::new(command) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +pub fn build_goose_command( + root: &Path, + scenario: &ResolvedScenario, + scenario_dir: &Path, + artifact_prefix: &str, + profiling_mode: bool, +) -> CommandSpec { + let manifest = root.join("crates/contextforge_goose/Cargo.toml"); + let request_log = scenario_dir.join(format!("{artifact_prefix}_requests.csv")); + let transaction_log = scenario_dir.join(format!("{artifact_prefix}_transactions.csv")); + let mut env = scenario_env(root, scenario).unwrap_or_default(); + let (command, args) = if profiling_mode { + let mut args = vec![ + "flamegraph".to_string(), + "--manifest-path".to_string(), + manifest.display().to_string(), + "--bin".to_string(), + DEFAULT_GOSE_BIN.to_string(), + "--output".to_string(), + scenario_dir + .join(format!("{artifact_prefix}_flamegraph.svg")) + .display() + .to_string(), + "--root".to_string(), + "--".to_string(), + "--host".to_string(), + target_host(&scenario.load), + "--users".to_string(), + scenario.load.users.unwrap_or(1).to_string(), + "--hatch-rate".to_string(), + scenario.load.spawn_rate.unwrap_or(1).to_string(), + "--request-log".to_string(), + request_log.display().to_string(), + "--request-format".to_string(), + "csv".to_string(), + "--transaction-log".to_string(), + transaction_log.display().to_string(), + "--transaction-format".to_string(), + "csv".to_string(), + ]; + if let Some(run_time) = &scenario.load.run_time { + args.push("--run-time".to_string()); + args.push(run_time.clone()); + } + if scenario.load.html_report { + args.push("--report-file".to_string()); + args.push( + scenario_dir + .join(format!("{artifact_prefix}_report.html")) + .display() + .to_string(), + ); + } + args.extend(scenario.load.extra_args.clone()); + ("cargo".to_string(), args) + } else { + let mut args = vec![ + "run".to_string(), + "--manifest-path".to_string(), + manifest.display().to_string(), + "--bin".to_string(), + DEFAULT_GOSE_BIN.to_string(), + "--release".to_string(), + "--".to_string(), + "--host".to_string(), + target_host(&scenario.load), + "--users".to_string(), + scenario.load.users.unwrap_or(1).to_string(), + "--hatch-rate".to_string(), + scenario.load.spawn_rate.unwrap_or(1).to_string(), + "--request-log".to_string(), + request_log.display().to_string(), + "--request-format".to_string(), + "csv".to_string(), + "--transaction-log".to_string(), + transaction_log.display().to_string(), + "--transaction-format".to_string(), + "csv".to_string(), + ]; + if let Some(run_time) = &scenario.load.run_time { + args.push("--run-time".to_string()); + args.push(run_time.clone()); + } + if scenario.load.html_report { + args.push("--report-file".to_string()); + args.push( + scenario_dir + .join(format!("{artifact_prefix}_report.html")) + .display() + .to_string(), + ); + } + args.extend(scenario.load.extra_args.clone()); + ("cargo".to_string(), args) + }; + if let Some(seed) = scenario.load.seed { + env.insert("BENCH_SEED".to_string(), seed.to_string()); + } + CommandSpec { command, args, env } +} + +fn target_host(load: &LoadConfig) -> String { + if let Some(host) = &load.host { + return host.clone(); + } + if load.target_service == "gateway" { + "http://127.0.0.1:14444".to_string() + } else { + "http://127.0.0.1:18080".to_string() + } +} + +pub fn scenario_env(root: &Path, scenario: &ResolvedScenario) -> Result> { + let mut env = BTreeMap::new(); + env.insert("LOADTEST_HOST".to_string(), target_host(&scenario.load)); + env.insert( + "LOADTEST_USERS".to_string(), + scenario.load.users.unwrap_or(1).to_string(), + ); + env.insert( + "LOADTEST_SPAWN_RATE".to_string(), + scenario.load.spawn_rate.unwrap_or(1).to_string(), + ); + env.insert( + "LOADTEST_RUN_TIME".to_string(), + scenario + .load + .run_time + .clone() + .unwrap_or_else(|| "10s".to_string()), + ); + env.insert( + "LOADTEST_REQUEST_COUNT".to_string(), + scenario.load.request_count.unwrap_or(0).to_string(), + ); + env.insert( + "BENCH_REQUEST_COUNT".to_string(), + scenario.load.request_count.unwrap_or(0).to_string(), + ); + env.insert( + "BENCH_TARGET_SERVICE".to_string(), + scenario.load.target_service.clone(), + ); + env.insert( + "BENCH_REQUEST_PLAN".to_string(), + serde_json::to_string(&resolve_requests_from_workload( + root, + &scenario.load.workload, + )?)?, + ); + for (key, value) in &scenario.load.env { + env.insert(key.clone(), value.clone()); + } + Ok(env) +} + +pub fn run_benchmark( + root: &Path, + selection: &str, + run_all: bool, + validate_only: bool, + smoke: bool, + check_runtime_only: bool, +) -> Result { + let runtime = detect_runtime()?; + log_progress(format!( + "Runtime detected: {} via {}", + runtime.engine, + runtime.compose_cmd.join(" ") + )); + let scenarios = if run_all { + discover_scenarios(root)? + } else { + vec![selection.to_string()] + }; + let resolved = scenarios + .into_iter() + .map(|name| load_suite(root, &name, smoke)) + .collect::>>()?; + let suite = if resolved.len() == 1 { + resolved.into_iter().next().unwrap() + } else { + let mut all = Vec::new(); + let mut suite_meta = SuiteMeta::default(); + for item in resolved { + if suite_meta.name.is_empty() { + suite_meta = item.suite.clone(); + } + all.extend(item.scenarios); + } + ResolvedSuite { + suite: suite_meta, + scenarios: all, + } + }; + let run_dir = PathBuf::from(&suite.suite.output_root).join(format!( + "{}_{}", + if run_all { "all-scenarios" } else { selection }, + Utc::now().format("%Y%m%d_%H%M%S") + )); + fs::create_dir_all(&run_dir)?; + log_progress(format!( + "Writing benchmark artifacts to {}", + run_dir.display() + )); + + let mut summaries = Vec::new(); + let mut failed_scenarios = Vec::new(); + for (index, scenario) in suite.scenarios.iter().enumerate() { + log_progress(format!( + "Scenario {}/{} starting: {}", + index + 1, + suite.scenarios.len(), + scenario.name + )); + let scenario_dir = run_dir.join("scenarios").join(&scenario.name); + fs::create_dir_all(&scenario_dir)?; + let result = if check_runtime_only { + run_runtime_check(root, &runtime, scenario, &scenario_dir) + } else if validate_only { + Ok(build_validation_summary(scenario)) + } else { + execute_scenario(root, &runtime, scenario, &scenario_dir) + }; + + match result { + Ok(summary) => { + log_progress(format!( + "Scenario '{}' completed with status {}", + scenario.name, summary.status + )); + if summary.status != "ok" && summary.status != "validated" { + failed_scenarios.push(scenario.name.clone()); + } + write_json(&scenario_dir.join("summary.json"), &summary)?; + summaries.push(summary); + } + Err(error) => { + log_progress(format!("Scenario '{}' failed: {error}", scenario.name)); + failed_scenarios.push(scenario.name.clone()); + let summary = build_error_summary(scenario, &error); + write_json(&scenario_dir.join("summary.json"), &summary)?; + summaries.push(summary); + } + } + } + let run_summary = build_run_summary(&suite.suite, &summaries); + write_json(&run_dir.join("run_summary.json"), &run_summary)?; + write_text( + &run_dir.join("run_summary.md"), + &render_run_summary_markdown(&run_summary), + )?; + let comparison = build_comparison_report(&summaries); + write_json( + &run_dir.join("scenario_comparison_report.json"), + &comparison, + )?; + write_text( + &run_dir.join("scenario_comparison_report.md"), + &render_comparison_markdown(&comparison), + )?; + write_text( + &run_dir.join("scenario_comparison_report.html"), + &render_comparison_html(&comparison), + )?; + if failed_scenarios.is_empty() { + log_progress(format!( + "Benchmark run completed successfully: {}", + run_dir.display() + )); + } else { + log_progress(format!( + "Benchmark run completed with failed scenarios [{}]: {}", + failed_scenarios.join(", "), + run_dir.display() + )); + } + Ok(run_dir) +} + +fn build_validation_summary(scenario: &ResolvedScenario) -> ScenarioSummary { + ScenarioSummary { + scenario: scenario.name.clone(), + status: "validated".to_string(), + setup: scenario.setup.clone(), + runtime: scenario.runtime.clone(), + load: scenario.load.clone(), + measurement: scenario.measurement.clone(), + profiling: scenario.profiling.clone(), + goose: json!({"status":"omitted","reason":"Validation mode"}), + endpoint_metrics: json!({"status":"omitted","reason":"Validation mode"}), + flamegraph_run: json!({"status":"omitted","reason":"Validation mode"}), + log_paths: Vec::new(), + artifacts: BTreeMap::new(), + } +} + +fn build_error_summary(scenario: &ResolvedScenario, error: &anyhow::Error) -> ScenarioSummary { + ScenarioSummary { + scenario: scenario.name.clone(), + status: "failed".to_string(), + setup: scenario.setup.clone(), + runtime: scenario.runtime.clone(), + load: scenario.load.clone(), + measurement: scenario.measurement.clone(), + profiling: scenario.profiling.clone(), + goose: json!({ + "status":"failed", + "error": error.to_string(), + }), + endpoint_metrics: json!({"status":"unavailable","reason":"scenario failed before metrics collection"}), + flamegraph_run: json!({"status":"omitted","reason":"scenario failed"}), + log_paths: Vec::new(), + artifacts: BTreeMap::new(), + } +} + +fn run_runtime_check( + root: &Path, + runtime: &RuntimeChoice, + scenario: &ResolvedScenario, + scenario_dir: &Path, +) -> Result { + let (compose_args, _project) = start_stack(root, runtime, scenario, scenario_dir)?; + let health_ok = wait_for_gateway_health(&compose_args, 120)?; + stop_stack(&compose_args); + Ok(ScenarioSummary { + scenario: scenario.name.clone(), + status: if health_ok { + "ok".to_string() + } else { + "failed".to_string() + }, + setup: scenario.setup.clone(), + runtime: scenario.runtime.clone(), + load: scenario.load.clone(), + measurement: scenario.measurement.clone(), + profiling: scenario.profiling.clone(), + goose: json!({"status":"omitted","reason":"check-runtime"}), + endpoint_metrics: json!({"status":"omitted","reason":"check-runtime"}), + flamegraph_run: json!({"status":"omitted","reason":"check-runtime"}), + log_paths: Vec::new(), + artifacts: BTreeMap::new(), + }) +} + +fn execute_scenario( + root: &Path, + runtime: &RuntimeChoice, + scenario: &ResolvedScenario, + scenario_dir: &Path, +) -> Result { + log_progress(format!("Starting stack for scenario '{}'", scenario.name)); + let (compose_args, _project) = start_stack(root, runtime, scenario, scenario_dir)?; + let result = (|| -> Result { + let token = benchmark_token(&compose_args).ok(); + let artifact_prefix = "goose"; + let command = build_goose_command(root, scenario, scenario_dir, artifact_prefix, false); + log_progress(format!( + "Launching Goose for scenario '{}': {} {}", + scenario.name, + command.command, + command.args.join(" ") + )); + let goose_result = run_command_spec(root, &command, token.as_deref())?; + let request_log = scenario_dir.join("goose_requests.csv"); + let csv_prefix = scenario_dir.join("goose"); + write_goose_stats_csv(&request_log, &csv_prefix)?; + let endpoint_metrics = collect_endpoint_metrics(&csv_prefix, &scenario.measurement)?; + let scenario_success = determine_scenario_success(goose_result.success, &endpoint_metrics); + let mut flamegraph_run = json!({"status":"omitted","reason":"profiling disabled"}); + let mut artifacts = BTreeMap::new(); + if scenario.load.html_report { + artifacts.insert( + "goose_html".to_string(), + scenario_dir.join("goose_report.html").display().to_string(), + ); + } + if scenario.profiling.enabled { + log_progress(format!( + "Collecting flamegraph for scenario '{}'", + scenario.name + )); + flamegraph_run = run_flamegraph(root, scenario, scenario_dir, token.as_deref())?; + if let Some(path) = flamegraph_run.get("svg").and_then(Value::as_str) { + if Path::new(path).exists() { + artifacts.insert("goose_flamegraph".to_string(), path.to_string()); + } + } + } + Ok(ScenarioSummary { + scenario: scenario.name.clone(), + status: if scenario_success { + "ok".to_string() + } else { + "failed".to_string() + }, + setup: scenario.setup.clone(), + runtime: scenario.runtime.clone(), + load: scenario.load.clone(), + measurement: scenario.measurement.clone(), + profiling: scenario.profiling.clone(), + goose: json!({ + "status": if scenario_success { "ok" } else { "failed" }, + "stdout": goose_result.stdout, + "stderr": goose_result.stderr, + "html_report": scenario_dir.join("goose_report.html").display().to_string(), + "csv_prefix": csv_prefix.display().to_string(), + }), + endpoint_metrics, + flamegraph_run, + log_paths: Vec::new(), + artifacts, + }) + })(); + log_progress(format!("Stopping stack for scenario '{}'", scenario.name)); + stop_stack(&compose_args); + result +} + +pub(crate) fn determine_scenario_success(process_success: bool, endpoint_metrics: &Value) -> bool { + process_success && !has_endpoint_failures(endpoint_metrics) +} + +pub(crate) fn has_endpoint_failures(endpoint_metrics: &Value) -> bool { + endpoint_metrics + .get("aggregated") + .and_then(|value| value.get("Failure Count")) + .and_then(Value::as_str) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0) + > 0 +} diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/runtime_orchestration.rs b/crates/contextforge_benchmark_runner/src/lib_parts/runtime_orchestration.rs new file mode 100644 index 0000000000..3345757653 --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/runtime_orchestration.rs @@ -0,0 +1,455 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Result, anyhow, bail}; +use serde_json::{Value, json}; + +use crate::lib_parts::{slug, uses_a2a_fixture, uses_fast_time_fixture, write_text}; +use crate::{DEFAULT_OUTPUT_ROOT, ResolvedScenario, RuntimeChoice, log_progress}; + +pub(crate) fn ensure_benchmark_image( + root: &Path, + runtime: &RuntimeChoice, + scenario: &ResolvedScenario, +) -> Result { + let image_name = format!( + "{}:{}", + if scenario.build.image_name.is_empty() { + "mcpgateway/mcpgateway" + } else { + &scenario.build.image_name + }, + if scenario.build.image_tag.is_empty() { + "benchmark-suite-rust" + } else { + &scenario.build.image_tag + } + ); + if scenario.build.rebuild_policy == "missing" + && Command::new(&runtime.engine) + .args(["image", "inspect", &image_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) + { + log_progress(format!("Using existing benchmark image {image_name}")); + return Ok(image_name); + } + let container_file = if scenario.build.container_file.is_empty() { + "crates/contextforge_benchmark_runner/assets/Containerfile".to_string() + } else { + scenario.build.container_file.clone() + }; + let mut command = Command::new(&runtime.engine); + command + .current_dir(root) + .arg("build") + .arg("-f") + .arg(container_file) + .arg("-t") + .arg(&image_name) + .arg("--build-arg") + .arg(format!( + "ENABLE_RUST={}", + if scenario.build.rust_plugins { + "true" + } else { + "false" + } + )) + .arg("--build-arg") + .arg(format!( + "ENABLE_PROFILING={}", + if scenario.profiling.enabled || scenario.build.profiling_image { + "true" + } else { + "false" + } + )); + for (key, value) in &scenario.build.args { + command.arg("--build-arg").arg(format!("{key}={value}")); + } + command.arg("."); + log_progress(format!("Building benchmark image {image_name}")); + let status = command.status()?; + if !status.success() { + bail!( + "failed to build benchmark image for scenario '{}'", + scenario.name + ); + } + Ok(image_name) +} + +pub(crate) fn write_compose_override( + root: &Path, + scenario: &ResolvedScenario, + scenario_dir: &Path, + image_name: &str, +) -> Result { + let compose_path = root.join("docker-compose.yml"); + let base_raw = fs::read_to_string(&compose_path)?; + let mut base: serde_yaml::Value = serde_yaml::from_str(&base_raw)?; + let base_map = base + .as_mapping_mut() + .ok_or_else(|| anyhow!("docker-compose.yml must be a mapping"))?; + let base_services = base_map + .get(yaml_key("services")) + .and_then(serde_yaml::Value::as_mapping) + .cloned() + .ok_or_else(|| anyhow!("docker-compose.yml must define services"))?; + let networks = base_map + .get(yaml_key("networks")) + .cloned() + .unwrap_or_else(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + let volumes = base_map + .get(yaml_key("volumes")) + .cloned() + .unwrap_or_else(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + + let mut selected = vec!["postgres", "redis", "pgbouncer", "gateway"]; + if scenario.load.target_service != "gateway" { + selected.push("nginx"); + } + if uses_fast_time_fixture(scenario) { + selected.push("fast_time_server"); + selected.push("register_fast_time"); + } + if uses_a2a_fixture(scenario) { + selected.push("a2a_echo_agent"); + selected.push("register_a2a_echo"); + } + + let mut services = serde_yaml::Mapping::new(); + for name in selected { + let mut service = base_services + .get(yaml_key(name)) + .and_then(serde_yaml::Value::as_mapping) + .cloned() + .ok_or_else(|| anyhow!("missing compose service '{name}'"))?; + service.remove(yaml_key("profiles")); + service.remove(yaml_key("build")); + service.insert( + yaml_key("deploy"), + serde_yaml::to_value(json!({ + "resources": { + "limits": {"cpus": "1"}, + "reservations": {"cpus": "0.25"} + } + }))?, + ); + if !matches!(name, "gateway" | "nginx") { + service.remove(yaml_key("ports")); + } + if let Some(volumes) = service.get(yaml_key("volumes")).cloned() { + let normalized = yaml_strings(Some(&volumes)) + .into_iter() + .map(|entry| normalize_volume_entry(root, &entry)) + .collect::>(); + service.insert(yaml_key("volumes"), serde_yaml::to_value(normalized)?); + } + if name == "gateway" { + service.insert( + yaml_key("image"), + serde_yaml::Value::String(image_name.to_string()), + ); + if scenario.load.target_service == "gateway" { + service.insert(yaml_key("ports"), serde_yaml::to_value(vec!["14444:4444"])?); + } else { + service.remove(yaml_key("ports")); + } + service.insert( + yaml_key("cap_add"), + serde_yaml::to_value(vec!["SYS_PTRACE"])?, + ); + service.insert( + yaml_key("security_opt"), + serde_yaml::to_value(vec!["seccomp:unconfined"])?, + ); + let mut volumes_list = yaml_strings(service.get(yaml_key("volumes"))); + volumes_list.push(format!( + "{}:/mnt/bench", + scenario_dir + .canonicalize() + .unwrap_or_else(|_| scenario_dir.to_path_buf()) + .display() + )); + service.insert(yaml_key("volumes"), serde_yaml::to_value(volumes_list)?); + service.insert( + yaml_key("environment"), + serde_yaml::to_value(merge_environment( + service.get(yaml_key("environment")), + &gateway_environment(scenario), + ))?, + ); + } + if name == "nginx" { + service.insert(yaml_key("ports"), serde_yaml::to_value(vec!["18080:80"])?); + } + services.insert(yaml_key(name), serde_yaml::Value::Mapping(service)); + } + + let mut root_map = serde_yaml::Mapping::new(); + root_map.insert(yaml_key("services"), serde_yaml::Value::Mapping(services)); + root_map.insert(yaml_key("networks"), networks); + root_map.insert(yaml_key("volumes"), volumes); + let path = root.join(DEFAULT_OUTPUT_ROOT).join("_runtime_staging"); + fs::create_dir_all(&path)?; + let override_path = path.join(format!("{}_compose.yml", slug(&scenario.name))); + write_text(&override_path, &serde_yaml::to_string(&root_map)?)?; + Ok(override_path) +} + +pub(crate) fn yaml_strings(value: Option<&serde_yaml::Value>) -> Vec { + value + .and_then(serde_yaml::Value::as_sequence) + .map(|items| { + items + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(ToString::to_string) + .collect() + }) + .unwrap_or_default() +} + +fn merge_environment( + existing: Option<&serde_yaml::Value>, + overrides: &BTreeMap, +) -> Vec { + let mut merged = BTreeMap::new(); + for item in yaml_strings(existing) { + if let Some((key, value)) = item.split_once('=') { + merged.insert(key.trim().to_string(), value.trim().to_string()); + } + } + for (key, value) in overrides { + merged.insert(key.clone(), value.clone()); + } + merged + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect() +} + +fn normalize_volume_entry(root: &Path, entry: &str) -> String { + let mut parts = entry + .split(':') + .map(ToString::to_string) + .collect::>(); + if parts.is_empty() { + return entry.to_string(); + } + let source = parts[0].clone(); + if source.starts_with('/') || source.starts_with("${") || source.is_empty() { + return entry.to_string(); + } + if source == "." || source.starts_with("./") || source.contains('/') { + parts[0] = root.join(source).display().to_string(); + return parts.join(":"); + } + entry.to_string() +} + +fn gateway_environment(scenario: &ResolvedScenario) -> BTreeMap { + let mut env = BTreeMap::new(); + env.insert("IMAGE_LOCAL".to_string(), scenario.build.image_name.clone()); + env.insert( + "HTTP_SERVER".to_string(), + scenario.runtime.http_server.clone(), + ); + env.insert( + "TRANSPORT_TYPE".to_string(), + scenario.runtime.transport_type.clone(), + ); + env.insert( + "PLUGINS_ENABLED".to_string(), + if scenario.setup.plugins_enabled { + "true" + } else { + "false" + } + .to_string(), + ); + env.insert( + "AUTH_REQUIRED".to_string(), + if scenario.setup.auth_mode == "none" { + "false" + } else { + "true" + } + .to_string(), + ); + env.insert( + "MCP_REQUIRE_AUTH".to_string(), + if scenario.setup.auth_mode == "none" { + "false" + } else { + "true" + } + .to_string(), + ); + env.insert( + "DISABLE_ACCESS_LOG".to_string(), + scenario.gateway.disable_access_log.to_string(), + ); + env.insert( + "TEMPLATES_AUTO_RELOAD".to_string(), + scenario.gateway.templates_auto_reload.to_string(), + ); + env.insert( + "STRUCTURED_LOGGING_DATABASE_ENABLED".to_string(), + scenario + .gateway + .structured_logging_database_enabled + .to_string(), + ); + env.insert( + "SQLALCHEMY_ECHO".to_string(), + scenario.gateway.sqlalchemy_echo.to_string(), + ); + if !scenario.gateway.log_level.is_empty() { + env.insert("LOG_LEVEL".to_string(), scenario.gateway.log_level.clone()); + } + if let Some(workers) = scenario.runtime.gunicorn.workers { + env.insert("GUNICORN_WORKERS".to_string(), workers.to_string()); + } + if let Some(timeout) = scenario.runtime.gunicorn.timeout { + env.insert("GUNICORN_TIMEOUT".to_string(), timeout.to_string()); + } + if let Some(backlog) = scenario.runtime.gunicorn.backlog { + env.insert("GUNICORN_BACKLOG".to_string(), backlog.to_string()); + } + if let Some(preload_app) = scenario.runtime.gunicorn.preload_app { + env.insert("GUNICORN_PRELOAD_APP".to_string(), preload_app.to_string()); + } + for key in [ + "EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED", + "EXPERIMENTAL_RUST_MCP_SESSION_CORE_ENABLED", + "EXPERIMENTAL_RUST_MCP_EVENT_STORE_ENABLED", + "EXPERIMENTAL_RUST_MCP_RESUME_CORE_ENABLED", + "EXPERIMENTAL_RUST_MCP_LIVE_STREAM_CORE_ENABLED", + "EXPERIMENTAL_RUST_MCP_AFFINITY_CORE_ENABLED", + "EXPERIMENTAL_RUST_MCP_SESSION_AUTH_REUSE_ENABLED", + ] { + env.entry(key.to_string()) + .or_insert_with(|| "false".to_string()); + } + env.extend(scenario.gateway.environment.clone()); + env +} + +fn yaml_key(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) +} + +pub(crate) fn run_compose(root: &Path, compose_args: &[String], extra_args: &[&str]) -> Result<()> { + let status = Command::new(&compose_args[0]) + .current_dir(root) + .args(&compose_args[1..]) + .args(extra_args) + .stdin(Stdio::null()) + .status()?; + if !status.success() { + bail!( + "compose command failed: {} {:?}", + compose_args[0], + extra_args + ); + } + Ok(()) +} + +pub(crate) fn service_container_id(compose_args: &[String], service: &str) -> Result { + let output = Command::new(&compose_args[0]) + .args(&compose_args[1..]) + .args(["ps", "-q", service]) + .output()?; + if !output.status.success() { + bail!("could not resolve compose service '{service}'"); + } + let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if value.is_empty() { + bail!("compose service '{service}' has no container id") + } + Ok(value) +} + +pub(crate) fn wait_for_service( + runtime: &RuntimeChoice, + compose_args: &[String], + service: &str, + timeout_secs: u64, +) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(timeout_secs); + while Instant::now() < deadline { + if let Ok(container_id) = service_container_id(compose_args, service) { + let output = Command::new(&runtime.engine) + .args(["inspect", &container_id, "--format", "{{json .State}}"]) + .output()?; + if output.status.success() { + let payload: Value = + serde_json::from_slice(&output.stdout).unwrap_or_else(|_| json!({})); + if payload + .get("Health") + .and_then(|health| health.get("Status")) + .and_then(Value::as_str) + == Some("healthy") + || payload.get("Running").and_then(Value::as_bool) == Some(true) + { + return Ok(()); + } + } + } + thread::sleep(Duration::from_secs(1)); + } + bail!("timed out waiting for compose service '{service}'") +} + +pub(crate) fn wait_for_gateway_health(compose_args: &[String], timeout_secs: u64) -> Result { + let deadline = Instant::now() + Duration::from_secs(timeout_secs); + let script = "python3 - <<'PY'\nimport json, sys, urllib.request\ntry:\n resp=urllib.request.urlopen('http://127.0.0.1:4444/health',timeout=2)\n payload=json.loads(resp.read())\n sys.exit(0 if payload.get('status')=='healthy' else 1)\nexcept Exception:\n sys.exit(1)\nPY"; + while Instant::now() < deadline { + let output = Command::new(&compose_args[0]) + .args(&compose_args[1..]) + .args(["exec", "-T", "gateway", "sh", "-lc", script]) + .output()?; + if output.status.success() { + return Ok(true); + } + thread::sleep(Duration::from_secs(1)); + } + Ok(false) +} + +pub(crate) fn benchmark_token_command() -> String { + "python3 -m mcpgateway.utils.create_jwt_token --username admin@example.com --admin --full-name 'Benchmark Admin' --exp 10080 --secret \"${JWT_SECRET_KEY}\" --algo HS256".to_string() +} + +pub(crate) fn benchmark_token(compose_args: &[String]) -> Result { + let command = benchmark_token_command(); + let output = Command::new(&compose_args[0]) + .args(&compose_args[1..]) + .args(["exec", "-T", "gateway", "sh", "-lc", &command]) + .output()?; + if !output.status.success() { + bail!("failed to mint benchmark token"); + } + let combined = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + for token in combined.split_whitespace() { + if token.starts_with("eyJ") && token.matches('.').count() == 2 { + return Ok(token.to_string()); + } + } + bail!("gateway token generation did not emit a jwt") +} diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/scenario_loading.rs b/crates/contextforge_benchmark_runner/src/lib_parts/scenario_loading.rs new file mode 100644 index 0000000000..9b4dbc65ee --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/scenario_loading.rs @@ -0,0 +1,403 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use toml::Value as TomlValue; + +use crate::lib_parts::benchmark_request_names; +use crate::{ + BuildConfig, DEFAULT_GOSE_BIN, DEFAULT_SCENARIO_DIR, ExecutionConfig, GatewayConfig, + LoadConfig, MeasurementConfig, ProfilingConfig, RequestsConfig, ResolvedScenario, + ResolvedSuite, RuntimeConfig, ScenarioEntry, ScenarioTemplate, SetupConfig, SuiteDocument, +}; + +pub fn repo_root() -> Result { + let cwd = std::env::current_dir()?; + Ok(cwd) +} + +pub fn scenario_root(root: &Path) -> PathBuf { + root.join(DEFAULT_SCENARIO_DIR) +} + +pub fn discover_scenarios(root: &Path) -> Result> { + let mut scenarios = Vec::new(); + for entry in fs::read_dir(scenario_root(root))? { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) == Some("toml") { + if let Some(stem) = path.file_stem().and_then(|value| value.to_str()) { + scenarios.push(stem.to_string()); + } + } + } + scenarios.sort(); + Ok(scenarios) +} + +pub fn resolve_profile_path(root: &Path, selection: &str) -> Result { + let candidate = Path::new(selection); + if candidate.exists() { + return Ok(candidate.to_path_buf()); + } + let scenario_path = scenario_root(root).join(format!("{selection}.toml")); + if scenario_path.exists() { + return Ok(scenario_path); + } + bail!("Benchmark scenario not found: {selection}") +} + +pub fn load_suite(root: &Path, selection: &str, smoke: bool) -> Result { + let path = resolve_profile_path(root, selection)?; + let raw = + fs::read_to_string(&path).with_context(|| format!("failed to read {}", path.display()))?; + let raw_value: TomlValue = + toml::from_str(&raw).with_context(|| format!("failed to parse {}", path.display()))?; + validate_rust_only_load_contract(&raw_value, &path)?; + let document: SuiteDocument = + toml::from_str(&raw).with_context(|| format!("failed to parse {}", path.display()))?; + if document.scenario.is_empty() { + bail!( + "{} does not declare any [[scenario]] entries", + path.display() + ); + } + let mut scenarios = Vec::new(); + for scenario in document.scenario { + let merged = merge_scenario(&document.defaults, &scenario, smoke); + validate_scenario(root, &merged)?; + scenarios.push(merged); + } + Ok(ResolvedSuite { + suite: document.suite, + scenarios, + }) +} + +fn merge_scenario( + defaults: &ScenarioTemplate, + scenario: &ScenarioEntry, + smoke: bool, +) -> ResolvedScenario { + let mut load = defaults.load.clone(); + merge_load(&mut load, &scenario.load); + if smoke { + load.users = Some(1); + load.spawn_rate = Some(1); + load.run_time = Some("10s".to_string()); + load.request_count = Some(5); + } + let mut setup = defaults.setup.clone(); + merge_setup(&mut setup, &scenario.setup); + let mut build = defaults.build.clone(); + merge_build(&mut build, &scenario.build); + let mut runtime = defaults.runtime.clone(); + merge_runtime(&mut runtime, &scenario.runtime); + let mut gateway = defaults.gateway.clone(); + merge_gateway(&mut gateway, &scenario.gateway); + let mut measurement = defaults.measurement.clone(); + merge_measurement(&mut measurement, &scenario.measurement); + let mut profiling = defaults.profiling.clone(); + merge_profiling(&mut profiling, &scenario.profiling); + let mut execution = defaults.execution.clone(); + merge_execution(&mut execution, &scenario.execution); + let mut requests = defaults.requests.clone(); + merge_requests(&mut requests, &scenario.requests); + + ResolvedScenario { + name: scenario.name.clone(), + description: scenario.description.clone(), + scenario_type: scenario.scenario_type.clone(), + setup, + build, + runtime, + gateway, + load, + measurement, + profiling, + execution, + requests, + } +} + +fn merge_setup(base: &mut SetupConfig, overlay: &SetupConfig) { + if !overlay.target_kind.is_empty() { + base.target_kind = overlay.target_kind.clone(); + } + if !overlay.auth_mode.is_empty() { + base.auth_mode = overlay.auth_mode.clone(); + } + base.plugins_enabled = overlay.plugins_enabled || base.plugins_enabled; + if overlay.expected_mcp_runtime.is_some() { + base.expected_mcp_runtime = overlay.expected_mcp_runtime.clone(); + } + if overlay.expected_mcp_runtime_mode.is_some() { + base.expected_mcp_runtime_mode = overlay.expected_mcp_runtime_mode.clone(); + } + if overlay.expected_a2a_runtime.is_some() { + base.expected_a2a_runtime = overlay.expected_a2a_runtime.clone(); + } +} + +fn merge_build(base: &mut BuildConfig, overlay: &BuildConfig) { + base.rust_plugins = overlay.rust_plugins || base.rust_plugins; + base.profiling_image = overlay.profiling_image || base.profiling_image; + if !overlay.container_file.is_empty() { + base.container_file = overlay.container_file.clone(); + } + if !overlay.image_name.is_empty() { + base.image_name = overlay.image_name.clone(); + } + if !overlay.image_tag.is_empty() { + base.image_tag = overlay.image_tag.clone(); + } + if !overlay.rebuild_policy.is_empty() { + base.rebuild_policy = overlay.rebuild_policy.clone(); + } + base.args.extend(overlay.args.clone()); +} + +fn merge_runtime(base: &mut RuntimeConfig, overlay: &RuntimeConfig) { + if !overlay.http_server.is_empty() { + base.http_server = overlay.http_server.clone(); + } + if !overlay.host.is_empty() { + base.host = overlay.host.clone(); + } + if !overlay.transport_type.is_empty() { + base.transport_type = overlay.transport_type.clone(); + } + macro_rules! set_if_some { + ($field:ident) => { + if overlay.gunicorn.$field.is_some() { + base.gunicorn.$field = overlay.gunicorn.$field; + } + }; + } + set_if_some!(workers); + set_if_some!(timeout); + set_if_some!(graceful_timeout); + set_if_some!(keep_alive); + set_if_some!(max_requests); + set_if_some!(max_requests_jitter); + set_if_some!(backlog); + set_if_some!(preload_app); + set_if_some!(dev_mode); +} + +fn merge_gateway(base: &mut GatewayConfig, overlay: &GatewayConfig) { + base.disable_access_log = overlay.disable_access_log || base.disable_access_log; + base.templates_auto_reload = overlay.templates_auto_reload || base.templates_auto_reload; + base.structured_logging_database_enabled = + overlay.structured_logging_database_enabled || base.structured_logging_database_enabled; + base.sqlalchemy_echo = overlay.sqlalchemy_echo || base.sqlalchemy_echo; + base.trust_proxy_auth = overlay.trust_proxy_auth || base.trust_proxy_auth; // pragma: allowlist secret + if !overlay.log_level.is_empty() { + base.log_level = overlay.log_level.clone(); + } + base.environment.extend(overlay.environment.clone()); +} + +fn merge_load(base: &mut LoadConfig, overlay: &LoadConfig) { + if !overlay.driver.is_empty() { + base.driver = overlay.driver.clone(); + } + base.headless = overlay.headless || base.headless; + base.only_summary = overlay.only_summary || base.only_summary; + base.html_report = overlay.html_report || base.html_report; + if overlay.users.is_some() { + base.users = overlay.users; + } + if overlay.spawn_rate.is_some() { + base.spawn_rate = overlay.spawn_rate; + } + if overlay.run_time.is_some() { + base.run_time = overlay.run_time.clone(); + } + if overlay.request_count.is_some() { + base.request_count = overlay.request_count; + } + if overlay.seed.is_some() { + base.seed = overlay.seed; + } + if !overlay.target_service.is_empty() { + base.target_service = overlay.target_service.clone(); + } + if overlay.host.is_some() { + base.host = overlay.host.clone(); + } + if !overlay.extra_args.is_empty() { + base.extra_args = overlay.extra_args.clone(); + } + base.env.extend(overlay.env.clone()); + if overlay.workload.fallback_endpoint.is_some() { + base.workload.fallback_endpoint = overlay.workload.fallback_endpoint.clone(); + } + base.workload + .endpoints + .extend(overlay.workload.endpoints.clone()); +} + +fn merge_measurement(base: &mut MeasurementConfig, overlay: &MeasurementConfig) { + if overlay.warmup_seconds > 0 { + base.warmup_seconds = overlay.warmup_seconds; + } + if overlay.measure_seconds > 0 { + base.measure_seconds = overlay.measure_seconds; + } + if overlay.profile_seconds > 0 { + base.profile_seconds = overlay.profile_seconds; + } + if overlay.cooldown_seconds > 0 { + base.cooldown_seconds = overlay.cooldown_seconds; + } +} + +fn merge_profiling(base: &mut ProfilingConfig, overlay: &ProfilingConfig) { + base.enabled = overlay.enabled || base.enabled; + if !overlay.tools.is_empty() { + base.tools = overlay.tools.clone(); + } + if overlay.duration_seconds > 0 { + base.duration_seconds = overlay.duration_seconds; + } + base.required = overlay.required || base.required; +} + +fn merge_execution(base: &mut ExecutionConfig, overlay: &ExecutionConfig) { + base.retry_enabled = overlay.retry_enabled || base.retry_enabled; + if overlay.max_attempts > 0 { + base.max_attempts = overlay.max_attempts; + } + base.capture_logs = overlay.capture_logs || base.capture_logs; + base.save_raw_results = overlay.save_raw_results || base.save_raw_results; + base.reuse_stack = overlay.reuse_stack || base.reuse_stack; +} + +fn merge_requests(base: &mut RequestsConfig, overlay: &RequestsConfig) { + if !overlay.enabled_groups.is_empty() { + base.enabled_groups = overlay.enabled_groups.clone(); + } + if !overlay.disabled_groups.is_empty() { + base.disabled_groups = overlay.disabled_groups.clone(); + } + if !overlay.enabled_endpoints.is_empty() { + base.enabled_endpoints = overlay.enabled_endpoints.clone(); + } + if !overlay.disabled_endpoints.is_empty() { + base.disabled_endpoints = overlay.disabled_endpoints.clone(); + } + if !overlay.enabled_tags.is_empty() { + base.enabled_tags = overlay.enabled_tags.clone(); + } + if !overlay.disabled_tags.is_empty() { + base.disabled_tags = overlay.disabled_tags.clone(); + } +} + +const LEGACY_LOAD_FIELDS: &[&str] = &[ + "goosefile", + "locustfile", + "repo_url", + "git_ref", + "git_commit", +]; +const UNSUPPORTED_GOOSE_ARGS: &[&str] = &["--reset-stats"]; + +fn validate_rust_only_load_contract(raw: &TomlValue, path: &Path) -> Result<()> { + let Some(table) = raw.as_table() else { + return Ok(()); + }; + + if let Some(defaults) = table.get("defaults").and_then(TomlValue::as_table) { + if let Some(load) = defaults.get("load") { + reject_legacy_load_fields(load, "defaults.load", path)?; + } + } + + if let Some(scenarios) = table.get("scenario").and_then(TomlValue::as_array) { + for (index, scenario) in scenarios.iter().enumerate() { + let Some(scenario_table) = scenario.as_table() else { + continue; + }; + if let Some(load) = scenario_table.get("load") { + let scenario_name = scenario_table + .get("name") + .and_then(TomlValue::as_str) + .unwrap_or(""); + let context = format!("scenario[{index}] '{scenario_name}'.load"); + reject_legacy_load_fields(load, &context, path)?; + } + } + } + + Ok(()) +} + +fn reject_legacy_load_fields(load: &TomlValue, context: &str, path: &Path) -> Result<()> { + let Some(table) = load.as_table() else { + return Ok(()); + }; + + if let Some(field) = LEGACY_LOAD_FIELDS + .iter() + .find(|field| table.contains_key(**field)) + { + bail!( + "{} in {} uses legacy load.{}; the Rust-only benchmark contract supports only load.driver = \"{}\"", + context, + path.display(), + field, + DEFAULT_GOSE_BIN + ); + } + + Ok(()) +} + +pub fn validate_scenario(root: &Path, scenario: &ResolvedScenario) -> Result<()> { + if scenario.name.trim().is_empty() { + bail!("scenario name must not be empty"); + } + if scenario.load.driver.trim().is_empty() { + bail!("scenario '{}' must define load.driver", scenario.name); + } + if scenario.load.driver != DEFAULT_GOSE_BIN { + bail!( + "scenario '{}' uses unsupported driver '{}'; only '{}' is supported", + scenario.name, + scenario.load.driver, + DEFAULT_GOSE_BIN + ); + } + if let Some(arg) = scenario + .load + .extra_args + .iter() + .find(|arg| UNSUPPORTED_GOOSE_ARGS.contains(&arg.as_str())) + { + bail!( + "scenario '{}' uses unsupported Goose extra arg '{}'; remove stale Locust-era flags from load.extra_args", + scenario.name, + arg + ); + } + if !scenario.build.container_file.is_empty() + && !root.join(&scenario.build.container_file).exists() + { + bail!( + "scenario '{}' container file does not exist: {}", + scenario.name, + root.join(&scenario.build.container_file).display() + ); + } + for endpoint in scenario.load.workload.endpoints.keys() { + if !benchmark_request_names(root)?.contains(endpoint) { + bail!( + "scenario '{}' workload references unknown endpoint: {}", + scenario.name, + endpoint + ); + } + } + Ok(()) +} diff --git a/crates/contextforge_benchmark_runner/src/lib_parts/tests.rs b/crates/contextforge_benchmark_runner/src/lib_parts/tests.rs new file mode 100644 index 0000000000..d136c3490e --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/lib_parts/tests.rs @@ -0,0 +1,474 @@ +use super::*; +use std::path::Path; +use std::process::Command; + +use serde_json::json; + +fn fixture_repo_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() +} + +#[test] +fn resolves_suite_with_driver_contract() { + let root = fixture_repo_root(); + let suite = load_suite(root, "rust-mcp-runtime-300", false).unwrap(); + assert_eq!(suite.scenarios.len(), 2); + assert_eq!(suite.scenarios[0].load.driver, DEFAULT_GOSE_BIN); +} + +#[test] +fn builds_goose_command_for_local_driver() { + let root = fixture_repo_root(); + let scenario = load_suite(root, "rust-mcp-runtime-300", true) + .unwrap() + .scenarios + .remove(0); + let temp = std::env::temp_dir().join("benchmark-runner-tests"); + let spec = build_goose_command(root, &scenario, &temp, "goose_metrics", false); + assert_eq!(spec.command, "cargo"); + assert!( + spec.args + .iter() + .any(|part| { part.ends_with("crates/contextforge_goose/Cargo.toml") }) + ); + assert!( + spec.args + .iter() + .any(|part| part.ends_with("goose_metrics_requests.csv")) + ); +} + +#[test] +fn rejects_legacy_goosefile_field() { + let tempdir = std::env::temp_dir().join("benchmark-runner-legacy-goosefile"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(&tempdir).unwrap(); + let path = tempdir.join("legacy-goosefile.toml"); + std::fs::write( + &path, + r#" +[suite] +name = "legacy" + +[defaults.load] +goosefile = "legacy/goosefile_benchmark.rs" + +[[scenario]] +name = "legacy-scenario" +"#, + ) + .unwrap(); + + let error = load_suite(&tempdir, path.to_str().unwrap(), false) + .unwrap_err() + .to_string(); + assert!(error.contains("legacy load.goosefile")); + assert!(error.contains("contextforge_goose")); + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn rejects_legacy_locust_fields() { + let tempdir = std::env::temp_dir().join("benchmark-runner-legacy-locust"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(&tempdir).unwrap(); + let path = tempdir.join("legacy-locust.toml"); + std::fs::write( + &path, + r#" +[suite] +name = "legacy" + +[defaults.load] +driver = "contextforge_goose" + +[[scenario]] +name = "legacy-scenario" + +[scenario.load] +driver = "contextforge_goose" +locustfile = "loadtests/old_locust.py" +repo_url = "https://example.invalid/repo.git" +git_ref = "main" +git_commit = "deadbeef" +"#, + ) + .unwrap(); + + let error = load_suite(&tempdir, path.to_str().unwrap(), false) + .unwrap_err() + .to_string(); + assert!(error.contains("legacy load.locustfile") || error.contains("legacy load.repo_url")); + assert!(error.contains("contextforge_goose")); + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn rejects_non_rust_driver() { + let tempdir = std::env::temp_dir().join("benchmark-runner-wrong-driver"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(&tempdir).unwrap(); + let path = tempdir.join("wrong-driver.toml"); + std::fs::write( + &path, + r#" +[suite] +name = "legacy" + +[defaults.load] +driver = "goosefile" + +[[scenario]] +name = "legacy-scenario" +"#, + ) + .unwrap(); + + let error = load_suite(&tempdir, path.to_str().unwrap(), false) + .unwrap_err() + .to_string(); + assert!(error.contains("unsupported driver")); + assert!(error.contains("contextforge_goose")); + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn rejects_locust_only_goose_extra_args() { + let tempdir = std::env::temp_dir().join("benchmark-runner-legacy-extra-args"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(tempdir.join("crates/contextforge_benchmark_runner/assets")).unwrap(); + let path = tempdir.join("suite.toml"); + std::fs::write( + tempdir.join("crates/contextforge_benchmark_runner/assets/Containerfile"), + "FROM scratch\n", + ) + .unwrap(); + std::fs::write( + &path, + r#" +[suite] +name = "legacy-extra-args" + +[defaults.build] +container_file = "crates/contextforge_benchmark_runner/assets/Containerfile" + +[defaults.load] +driver = "contextforge_goose" +extra_args = ["--reset-stats"] + +[[scenario]] +name = "legacy-extra-args-scenario" +"#, + ) + .unwrap(); + + let error = load_suite(&tempdir, path.to_str().unwrap(), false) + .unwrap_err() + .to_string(); + assert!(error.contains("--reset-stats")); + assert!(error.contains("Goose")); + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn writes_goose_stats_csv_without_map_serialization_errors() { + let tempdir = std::env::temp_dir().join("benchmark-runner-goose-csv"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(&tempdir).unwrap(); + let request_log = tempdir.join("requests.csv"); + std::fs::write( + &request_log, + "name,elapsed,response_time,success\n/mcp tools/list,1,12.0,true\n/mcp tools/list,2,18.0,true\n", + ) + .unwrap(); + + let csv_prefix = tempdir.join("goose"); + write_goose_stats_csv(&request_log, &csv_prefix).unwrap(); + + let stats = std::fs::read_to_string(tempdir.join("goose_stats.csv")).unwrap(); + assert!(stats.contains("Aggregated")); + assert!(stats.contains("/mcp tools/list")); + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn resolves_new_uncharted_surface_suites() { + let root = fixture_repo_root(); + for suite in [ + "admin-plugins-300", + "rest-discovery-300", + "mcp-resources-300", + "mcp-prompts-300", + ] { + let resolved = load_suite(root, suite, false).unwrap(); + assert_eq!(resolved.scenarios.len(), 2, "{suite}"); + } +} + +#[test] +fn benchmark_scenarios_cover_locust_workload_families() { + let root = fixture_repo_root(); + let scenario_names = discover_scenarios(root).unwrap(); + let stems: std::collections::BTreeSet = scenario_names + .iter() + .map(|name| name.trim_end_matches("-300").to_string()) + .collect(); + + for required in [ + "agentgateway-mcp-server-time", + "baseline", + "echo-delay", + "highthroughput", + "mcp-isolation", + "mcp-protocol", + "rate-limiter", + "rate-limiter-scale", + "rate-limiter-redis-capacity", + "secret-detection", + "slow-time-server", + "spin-detector", + ] { + assert!( + stems.contains(required), + "missing benchmark scenario suite `{required}`" + ); + } + + for suite in scenario_names { + load_suite(root, &suite, false).unwrap_or_else(|error| { + panic!("failed to load suite `{suite}`: {error}"); + }); + } +} + +#[test] +fn mcp_focused_suites_compare_python_and_rust_runtime() { + let root = fixture_repo_root(); + for suite in [ + "rust-mcp-runtime-300", + "rest-discovery-300", + "mcp-resources-300", + "mcp-prompts-300", + ] { + let resolved = load_suite(root, suite, false).unwrap(); + let baseline = &resolved.scenarios[0]; + let variant = &resolved.scenarios[1]; + + assert_eq!( + baseline.setup.expected_mcp_runtime.as_deref(), + None, + "{suite}" + ); + assert_eq!( + variant.setup.expected_mcp_runtime.as_deref(), + Some("rust"), + "{suite}" + ); + assert_eq!( + variant.setup.expected_mcp_runtime_mode.as_deref(), + Some("rust-managed"), + "{suite}" + ); + assert_eq!( + variant + .gateway + .environment + .get("EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED") + .map(String::as_str), + Some("true"), + "{suite}" + ); + assert_eq!( + variant + .gateway + .environment + .get("RUST_MCP_MODE") + .map(String::as_str), + Some("edge"), + "{suite}" + ); + } +} + +#[test] +fn streaming_command_reports_live_stdout_and_stderr_lines() { + let mut command = Command::new("sh"); + command.args([ + "-c", + "printf 'alpha\\n'; sleep 0.1; printf 'beta\\n' >&2; sleep 0.1; printf 'gamma\\n'", + ]); + let mut events = Vec::new(); + + let result = run_command_streaming(&mut command, |stream, line| { + events.push(format!("{stream}:{line}")); + }) + .unwrap(); + + assert!(result.success); + assert_eq!( + events, + vec![ + "stdout:alpha".to_string(), + "stderr:beta".to_string(), + "stdout:gamma".to_string() + ] + ); + assert!(result.stdout.contains("alpha")); + assert!(result.stdout.contains("gamma")); + assert!(result.stderr.contains("beta")); +} + +#[test] +fn scenario_status_fails_when_endpoint_metrics_report_failures() { + let metrics = json!({ + "aggregated": { + "Failure Count": "5" + } + }); + + assert!(has_endpoint_failures(&metrics)); + assert!(!determine_scenario_success(true, &metrics)); +} + +#[test] +fn benchmark_token_command_uses_gateway_jwt_secret_env() { + let command = benchmark_token_command(); + assert!(command.contains("JWT_SECRET_KEY")); + assert!(!command.contains("my-test-key")); +} + +#[test] +fn nginx_targeted_override_does_not_bind_gateway_host_port() { + let tempdir = std::env::temp_dir().join("benchmark-runner-compose-ports"); + let _ = std::fs::remove_dir_all(&tempdir); + std::fs::create_dir_all(tempdir.join("reports/benchmarks/test-scenario")).unwrap(); + std::fs::write( + tempdir.join("docker-compose.yml"), + r#" +services: + postgres: + image: postgres:16 + ports: ["5432:5432"] + redis: + image: redis:7 + ports: ["6379:6379"] + pgbouncer: + image: edoburu/pgbouncer + ports: ["6432:6432"] + gateway: + image: mcpgateway/test:latest + environment: + - JWT_SECRET_KEY=my-test-key-but-now-longer-than-32-bytes + ports: ["4444:4444"] + nginx: + image: nginx:latest + ports: ["8080:80"] +networks: {} +volumes: {} +"#, + ) + .unwrap(); + + let scenario_dir = tempdir.join("reports/benchmarks/test-scenario"); + let scenario = ResolvedScenario { + name: "nginx-target".to_string(), + description: String::new(), + scenario_type: String::new(), + setup: SetupConfig::default(), + build: BuildConfig::default(), + runtime: RuntimeConfig::default(), + gateway: GatewayConfig::default(), + load: LoadConfig { + target_service: "nginx".to_string(), + ..LoadConfig::default() + }, + measurement: MeasurementConfig::default(), + profiling: ProfilingConfig::default(), + execution: ExecutionConfig::default(), + requests: RequestsConfig::default(), + }; + + let override_path = + write_compose_override(&tempdir, &scenario, &scenario_dir, "mcpgateway/test:latest") + .unwrap(); + let raw = std::fs::read_to_string(override_path).unwrap(); + let parsed: serde_yaml::Value = serde_yaml::from_str(&raw).unwrap(); + let services = parsed + .get("services") + .and_then(serde_yaml::Value::as_mapping) + .unwrap(); + let gateway = services + .get(serde_yaml::Value::String("gateway".to_string())) + .and_then(serde_yaml::Value::as_mapping) + .unwrap(); + let nginx = services + .get(serde_yaml::Value::String("nginx".to_string())) + .and_then(serde_yaml::Value::as_mapping) + .unwrap(); + + assert!(gateway.get("ports").is_none()); + assert_eq!( + yaml_strings(nginx.get("ports")), + vec!["18080:80".to_string()] + ); + + let _ = std::fs::remove_dir_all(&tempdir); +} + +#[test] +fn comparison_report_tracks_changed_dimensions() { + let left = ScenarioSummary { + scenario: "left".to_string(), + status: "ok".to_string(), + runtime: RuntimeConfig { + http_server: "gunicorn".to_string(), + ..RuntimeConfig::default() + }, + setup: SetupConfig { + auth_mode: "jwt".to_string(), + ..SetupConfig::default() + }, + load: LoadConfig { + driver: DEFAULT_GOSE_BIN.to_string(), + ..LoadConfig::default() + }, + endpoint_metrics: json!({"measurement_window":{"aggregated":{"Requests/s":5.0,"95%":10.0}}}), + ..ScenarioSummary::default() + }; + let right = ScenarioSummary { + scenario: "right".to_string(), + status: "ok".to_string(), + runtime: RuntimeConfig { + http_server: "granian".to_string(), + ..RuntimeConfig::default() + }, + setup: SetupConfig { + auth_mode: "jwt".to_string(), + ..SetupConfig::default() + }, + load: LoadConfig { + driver: DEFAULT_GOSE_BIN.to_string(), + ..LoadConfig::default() + }, + endpoint_metrics: json!({"measurement_window":{"aggregated":{"Requests/s":8.0,"95%":7.0}}}), + ..ScenarioSummary::default() + }; + let report = build_comparison_report(&[left, right]); + let first = report + .get("comparisons") + .and_then(Value::as_array) + .and_then(|items| items.first()) + .unwrap(); + assert_eq!(first.get("rps_delta").and_then(Value::as_f64).unwrap(), 3.0); + assert!( + first + .get("changed_dimensions") + .unwrap() + .to_string() + .contains("runtime.http_server") + ); +} diff --git a/crates/contextforge_benchmark_runner/src/main.rs b/crates/contextforge_benchmark_runner/src/main.rs new file mode 100644 index 0000000000..d33db3edc3 --- /dev/null +++ b/crates/contextforge_benchmark_runner/src/main.rs @@ -0,0 +1,87 @@ +// Allow duplicate transitive deps in this benchmark-only binary target. +#![allow(clippy::multiple_crate_versions)] + +use std::path::PathBuf; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use contextforge_benchmark_runner::{ + DEFAULT_SCENARIO_DIR, discover_scenarios, regenerate_reports, repo_root, run_benchmark, +}; + +#[derive(Parser, Debug)] +#[command(name = "contextforge-benchmark-runner")] +#[command(about = "Rust-native benchmark runner for crates/contextforge_benchmark_runner")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + List, + Validate { + #[arg(long)] + scenario: String, + #[arg(long, default_value_t = false)] + smoke: bool, + }, + Run { + #[arg(long)] + scenario: String, + #[arg(long, default_value_t = false)] + smoke: bool, + }, + RunAll { + #[arg(long, default_value_t = false)] + smoke: bool, + }, + CheckRuntime { + #[arg(long)] + scenario: String, + #[arg(long, default_value_t = false)] + smoke: bool, + }, + RegenerateReport { + #[arg(long)] + run_dir: PathBuf, + }, + CompareRun { + #[arg(long)] + run_dir: PathBuf, + }, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let root = repo_root()?; + match cli.command { + Commands::List => { + for scenario in discover_scenarios(&root)? { + println!("{scenario}"); + } + } + Commands::Validate { scenario, smoke } => { + let run_dir = run_benchmark(&root, &scenario, false, true, smoke, false)?; + println!("{}", run_dir.display()); + } + Commands::Run { scenario, smoke } => { + let run_dir = run_benchmark(&root, &scenario, false, false, smoke, false)?; + println!("{}", run_dir.display()); + } + Commands::RunAll { smoke } => { + let run_dir = run_benchmark(&root, "all-scenarios", true, false, smoke, false)?; + println!("{}", run_dir.display()); + } + Commands::CheckRuntime { scenario, smoke } => { + let run_dir = run_benchmark(&root, &scenario, false, false, smoke, true)?; + println!("{}", run_dir.display()); + } + Commands::RegenerateReport { run_dir } | Commands::CompareRun { run_dir } => { + let output = regenerate_reports(&run_dir)?; + println!("{}", output.display()); + } + } + let _ = DEFAULT_SCENARIO_DIR; + Ok(()) +} diff --git a/crates/contextforge_goose/Cargo.lock b/crates/contextforge_goose/Cargo.lock new file mode 100644 index 0000000000..aaa2fa06ff --- /dev/null +++ b/crates/contextforge_goose/Cargo.lock @@ -0,0 +1,2591 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "compression-codecs" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "contextforge_goose" +version = "0.1.0" +dependencies = [ + "goose", + "goose-eggs", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys 0.61.2", +] + +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "goose" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5e77ce04d5086ca6659396f81b5d63c9c0c4453c05b7fa38fcc83ea1b2b8e7" +dependencies = [ + "async-trait", + "chrono", + "ctrlc", + "downcast-rs", + "flume", + "futures", + "gumdrop", + "http", + "itertools", + "lazy_static", + "log", + "num-format", + "rand 0.9.2", + "regex", + "reqwest", + "serde", + "serde_json", + "simplelog", + "strum", + "strum_macros", + "tokio", + "tokio-tungstenite", + "tungstenite", + "url", +] + +[[package]] +name = "goose-eggs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd230c2b6a53026456ddf7ff626811fa6e334cfd9caf19b768baa9084fa66185" +dependencies = [ + "goose", + "html-escape", + "http", + "log", + "rand 0.9.2", + "regex", + "reqwest", + "tokio", +] + +[[package]] +name = "gumdrop" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc700f989d2f6f0248546222d9b4258f5b02a171a431f8285a81c08142629e3" +dependencies = [ + "gumdrop_derive", +] + +[[package]] +name = "gumdrop_derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729f9bd3449d77e7831a18abfb7ba2f99ee813dfd15b8c2167c9a54ba20aa99d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl" +version = "0.10.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "cookie", + "cookie_store", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "iri-string", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1", + "thiserror", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/crates/contextforge_goose/Cargo.toml b/crates/contextforge_goose/Cargo.toml new file mode 100644 index 0000000000..9da75a9f3f --- /dev/null +++ b/crates/contextforge_goose/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "contextforge_goose" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[[bin]] +name = "contextforge_goose" +path = "goosefile_benchmark.rs" + +[dependencies] +goose = "0.18.1" +goose-eggs = "0.6.0" +rand.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true + +[lints] +workspace = true diff --git a/crates/contextforge_goose/goosefile_benchmark.rs b/crates/contextforge_goose/goosefile_benchmark.rs new file mode 100644 index 0000000000..863dbecb0b --- /dev/null +++ b/crates/contextforge_goose/goosefile_benchmark.rs @@ -0,0 +1,312 @@ +// Allow duplicate transitive deps in this benchmark-only binary target. +#![allow(clippy::multiple_crate_versions)] + +use std::env; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; + +use goose::goose::{GooseMethod, GooseRequest}; +use goose::prelude::*; +use goose_eggs::{Validate, validate_page}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use reqwest::header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue}; +use serde::Deserialize; +use serde_json::Value; + +static REQUEST_PLAN: OnceLock> = OnceLock::new(); +static RNG: OnceLock> = OnceLock::new(); +static REQUEST_COUNT: AtomicUsize = AtomicUsize::new(0); + +#[derive(Clone, Debug, Deserialize)] +struct RequestPlanEntry { + name: String, + weight: usize, + request: RequestDefinition, +} + +#[derive(Clone, Debug, Deserialize)] +struct RequestDefinition { + kind: String, + path: Option, + payload: Option, + auth: Option, // pragma: allowlist secret + server_id: Option, + expect_json: Option, + expect_list_min_items: Option, + expect_list_item_name: Option, + expect_result_key: Option, + expect_result_min_items: Option, + expect_content_text: Option, +} + +fn request_plan() -> &'static Vec { + REQUEST_PLAN.get_or_init(|| { + serde_json::from_str(&env::var("BENCH_REQUEST_PLAN").unwrap_or_else(|_| "[]".to_string())) + .unwrap_or_default() + }) +} + +fn benchmark_rng() -> &'static Mutex { + RNG.get_or_init(|| { + let seed = env::var("BENCH_SEED") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(42); + Mutex::new(StdRng::seed_from_u64(seed)) + }) +} + +fn choose_entry() -> Option { + let plan = request_plan(); + if plan.is_empty() { + return None; + } + let total_weight: usize = plan.iter().map(|entry| entry.weight.max(1)).sum(); + let mut rng = benchmark_rng().lock().expect("benchmark rng poisoned"); + let needle = rng.gen_range(0..total_weight); + let mut offset = 0usize; + for entry in plan { + offset += entry.weight.max(1); + if needle < offset { + return Some(entry.clone()); + } + } + plan.last().cloned() +} + +fn request_limit_reached() -> bool { + let limit = env::var("BENCH_REQUEST_COUNT") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if limit == 0 { + return false; + } + REQUEST_COUNT.fetch_add(1, Ordering::Relaxed) >= limit +} + +fn auth_header() -> Option { + let token = env::var("MCPGATEWAY_BEARER_TOKEN").unwrap_or_default(); + if token.trim().is_empty() { + return None; + } + HeaderValue::from_str(&format!("Bearer {}", token.trim())).ok() +} + +fn add_common_headers(headers: &mut HeaderMap, definition: &RequestDefinition) { + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + if matches!(definition.kind.as_str(), "post" | "rpc" | "mcp") { + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + } + if definition.auth.unwrap_or(false) { + if let Some(value) = auth_header() { + headers.insert(AUTHORIZATION, value); + } + } +} + +fn mcp_path(definition: &RequestDefinition) -> String { + let server_id = definition + .server_id + .clone() + .unwrap_or_else(|| "9779b6698cbd4b4995ee04a4fab38737".to_string()); // pragma: allowlist secret + format!("/servers/{server_id}/mcp") +} + +async fn send_named_json_request( + user: &mut GooseUser, + method: GooseMethod, + path: &str, + name: &str, + payload: Option<&Value>, + definition: &RequestDefinition, +) -> Result> { + let mut headers = HeaderMap::new(); + add_common_headers(&mut headers, definition); + let mut request_builder = user.get_request_builder(&method, path)?; + request_builder = request_builder.headers(headers); + if let Some(body) = payload { + request_builder = request_builder.json(body); + } + let request = GooseRequest::builder() + .method(method) + .path(path) + .name(name) + .set_request_builder(request_builder) + .build(); + user.request(request).await +} + +async fn initialize_mcp_session( + user: &mut GooseUser, + definition: &RequestDefinition, +) -> TransactionResult { + let payload = serde_json::json!({ + "jsonrpc": "2.0", + "id": "benchmark-init", + "method": "initialize", + "params": { + "protocolVersion": env::var("BENCH_MCP_PROTOCOL_VERSION").unwrap_or_else(|_| "2024-11-05".to_string()), + "capabilities": {}, + "clientInfo": {"name": "benchmark-goose", "version": "1.0"} + } + }); + let goose = send_named_json_request( + user, + GooseMethod::Post, + &mcp_path(definition), + "/mcp initialize [setup]", + Some(&payload), + definition, + ) + .await?; + let validate = Validate::builder().status(200).build(); + let _ = validate_page(user, goose, &validate).await?; + Ok(()) +} + +fn validate_json_body(body: &str, entry: &RequestPlanEntry) -> Result<(), String> { + let payload: Value = + serde_json::from_str(body).map_err(|error| format!("invalid json: {error}"))?; + let definition = &entry.request; + let root = if matches!(definition.kind.as_str(), "rpc" | "mcp") { + if payload.get("error").is_some() { + return Err("json-rpc response included error".to_string()); + } + payload + .get("result") + .ok_or_else(|| "json-rpc response missing result".to_string())? + .clone() + } else { + payload + }; + + if let Some(min_items) = definition.expect_list_min_items { + let items = root + .as_array() + .ok_or_else(|| "expected list response".to_string())?; + if items.len() < min_items { + return Err(format!("expected at least {min_items} list items")); + } + } + if let Some(expected_name) = &definition.expect_list_item_name { + let items = root + .as_array() + .ok_or_else(|| "expected list response".to_string())?; + let found = items + .iter() + .any(|item| item.get("name").and_then(Value::as_str) == Some(expected_name.as_str())); + if !found { + return Err(format!("expected item named {expected_name}")); + } + } + if let Some(expected_key) = &definition.expect_result_key { + let result = root + .as_object() + .ok_or_else(|| "expected object response".to_string())?; + let value = result + .get(expected_key) + .ok_or_else(|| format!("expected result key '{expected_key}'"))?; + if let Some(min_items) = definition.expect_result_min_items { + let items = value + .as_array() + .ok_or_else(|| format!("expected array at result.{expected_key}"))?; + if items.len() < min_items { + return Err(format!( + "expected at least {min_items} items in result.{expected_key}" + )); + } + } + } + if definition.expect_content_text.unwrap_or(false) { + let content = root + .get("content") + .and_then(Value::as_array) + .ok_or_else(|| "expected content array".to_string())?; + let has_text = content + .iter() + .any(|item| item.get("text").and_then(Value::as_str).is_some()); + if !has_text { + return Err("expected text content".to_string()); + } + } + Ok(()) +} + +async fn execute_entry(user: &mut GooseUser) -> TransactionResult { + if request_limit_reached() { + tokio::time::sleep(Duration::from_millis(50)).await; + return Ok(()); + } + + let Some(entry) = choose_entry() else { + return Ok(()); + }; + let definition = &entry.request; + let goose = match definition.kind.as_str() { + "get" => { + send_named_json_request( + user, + GooseMethod::Get, + definition.path.as_deref().unwrap_or("/"), + &entry.name, + None, + definition, + ) + .await? + } + "post" | "rpc" => { + send_named_json_request( + user, + GooseMethod::Post, + definition.path.as_deref().unwrap_or("/rpc"), + &entry.name, + definition.payload.as_ref(), + definition, + ) + .await? + } + "mcp" => { + initialize_mcp_session(user, definition).await?; + send_named_json_request( + user, + GooseMethod::Post, + &mcp_path(definition), + &entry.name, + definition.payload.as_ref(), + definition, + ) + .await? + } + other => unreachable!("unsupported request kind: {other}"), + }; + + let validate = Validate::builder().status(200).build(); + let mut request_metric = goose.request.clone(); + let body = validate_page(user, goose, &validate).await?; + if definition.expect_json.unwrap_or(false) + || matches!(definition.kind.as_str(), "rpc" | "mcp") + || definition.expect_result_key.is_some() + || definition.expect_content_text.unwrap_or(false) + { + if let Err(error) = validate_json_body(&body, &entry) { + return user.set_failure(&error, &mut request_metric, None, Some(&body)); + } + } + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), GooseError> { + let scenario = scenario!("ContextForgeBenchmark") + .set_wait_time(Duration::from_millis(0), Duration::from_millis(0))? + .register_transaction(transaction!(execute_entry)); + + GooseAttack::initialize()? + .register_scenario(scenario) + .execute() + .await?; + Ok(()) +} diff --git a/crates/wrapper/src/json_rpc_id_fast.rs b/crates/wrapper/src/json_rpc_id_fast.rs index 098451ccbf..863e808dd0 100644 --- a/crates/wrapper/src/json_rpc_id_fast.rs +++ b/crates/wrapper/src/json_rpc_id_fast.rs @@ -29,15 +29,15 @@ pub fn parse_field_fast(json: &[u8], field_name: &str) -> Id { break; } } - JsonEvent::FieldName => { - if depth == 1 && parser.current_str().is_ok_and(|name| name == field_name) { - // Get the very next event (the value) - if let Ok(Some(val_event)) = parser.next_event() { - match parser.current_str() { - Ok(s) => return to_id(&val_event, s), - Err(e) => { - error!("Invalid string: {e}"); - } + JsonEvent::FieldName + if depth == 1 && parser.current_str().is_ok_and(|name| name == field_name) => + { + // Get the very next event (the value) + if let Ok(Some(val_event)) = parser.next_event() { + match parser.current_str() { + Ok(s) => return to_id(&val_event, s), + Err(e) => { + error!("Invalid string: {e}"); } } } diff --git a/deny.toml b/deny.toml index e2b8b99292..7281e94f7e 100644 --- a/deny.toml +++ b/deny.toml @@ -11,6 +11,8 @@ ignore = [ "RUSTSEC-2025-0090", "RUSTSEC-2025-0098", "RUSTSEC-2025-0100", + # ratatui currently pulls paste 1.x; tracked here until upstream moves off the archived crate. + "RUSTSEC-2024-0436", ] [bans] diff --git a/docs/docs/testing/benchmark-suite.md b/docs/docs/testing/benchmark-suite.md new file mode 100644 index 0000000000..f13300ddfb --- /dev/null +++ b/docs/docs/testing/benchmark-suite.md @@ -0,0 +1,206 @@ +# Benchmark Suite + +Use the benchmark suite when you want repeatable, scenario-driven benchmark runs +against the Docker/Compose stack from a Rust-native runner, console, and load +driver. + +The committed scenarios cover three main benchmark shapes: + +- smoke-oriented validation runs + Fast sanity validation only. Do not use their numbers for performance claims. +- end-to-end runtime comparisons through `nginx` + Authenticated runs that measure gateway behavior under representative load. +- MCP prompt/resource/tool-heavy comparisons + Scenarios that exercise MCP prompt, resource, and tool paths through + `/servers/{server_id}/mcp`. + +## What It Runs + +Each scenario runs against the real containerized testing stack: + +- PostgreSQL +- Redis +- PgBouncer +- gateway +- nginx +- `contextforge_goose` as the load driver +- optional `perf`, `flamegraph`, and other Rust-friendly profiling tools in a + separate profiling pass + +The runner is: + +```bash +cargo run --manifest-path crates/contextforge_benchmark_runner/Cargo.toml -- +``` + +Committed scenarios now live in: + +```bash +crates/contextforge_benchmark_runner/assets/scenarios/ +``` + +The benchmark launcher now lives in: + +```bash +crates/contextforge_benchmark_console/ +``` + +The Goose driver now lives in: + +```bash +crates/contextforge_goose/ +``` + +## Quick Start + +Open the interactive launcher: + +```bash +make benchmark +``` + +Inside the launcher you can choose: + +- `Run` +- `Validate` +- `Smoke` +- `Check Runtime` +- `List` +- `Report` +- `Compare` +- `Generate` + +The `Generate` action opens a template builder that saves a new scenario file +under `crates/contextforge_benchmark_runner/assets/scenarios/`. It fills in the important fields in +the UI and writes a full TOML template containing all supported sections and +keys, including commented optional settings for advanced tuning. + +Build the benchmark image expected by scenarios with `rebuild_policy = "never"`: + +```bash +make container-build CONTAINER_FILE=crates/contextforge_benchmark_runner/assets/Containerfile ENABLE_RUST_BUILD=1 ENABLE_PROFILING_BUILD=1 CONTAINER_RUNTIME=podman +``` + +Validate the suite: + +```bash +cargo run --manifest-path crates/contextforge_benchmark_runner/Cargo.toml -- validate --scenario rust-mcp-runtime-300 +``` + +Run the smoke suite: + +```bash +cargo run --manifest-path crates/contextforge_benchmark_runner/Cargo.toml -- run --scenario a2a-invoke-300 --smoke +``` + +Run the suite: + +```bash +cargo run --manifest-path crates/contextforge_benchmark_runner/Cargo.toml -- run --scenario rust-mcp-runtime-300 +``` + +The TUI discovers committed scenarios automatically from +`crates/contextforge_benchmark_runner/assets/scenarios/`. + +## Scenario Contract + +Supported sections are: + +- `[suite]` +- `[defaults.setup]` +- `[defaults.build]` +- `[defaults.runtime]` +- `[defaults.gateway]` +- `[defaults.load]` +- `[defaults.load.env]` +- `[defaults.measurement]` +- `[defaults.requests]` +- `[defaults.profiling]` +- `[defaults.execution]` +- `[[scenario]]` +- `[scenario.runtime]` +- `[scenario.load]` +- `[scenario.requests]` +- `[scenario.execution]` + +Unknown keys are currently ignored unless the runner checks them explicitly, so +scenario authors should treat validation as shape checking for supported fields +rather than strict unknown-key rejection. + +## Important Fields + +- `load.target_service = "nginx" | "gateway"` + Use `nginx` for realistic end-to-end benchmarking. Use `gateway` only for + direct app-path microbenchmarks. +- `execution.retry_enabled`, `execution.max_attempts` + Control per-scenario retries. +- `execution.capture_logs` + Persist service logs on failed runs. +- `measurement.*` + Warmup, measurement, and cooldown are applied to Goose history when building + the aggregated summary. +- `profiling.tools` + Use Rust-native profiling tools such as `perf`, `flamegraph`, and + `process_stats`. +- `suite.baseline_run` + Optional path to a prior `run_summary.json` used for threshold-based + comparison output. + +## Request Mixes + +The Rust Goose driver in `crates/contextforge_goose/` uses real request +families: + +- health checks +- admin plugin UI +- REST discovery (`/servers`, `/resources`, `/prompts`) +- MCP JSON-RPC discovery (`tools/list`, `resources/list`, `prompts/list`) +- MCP JSON-RPC prompt/resource/tool calls from payload fixtures in + `crates/contextforge_benchmark_runner/assets/payloads/` + +This means the committed scenario mixes now hit real prompt/resource/tool and +REST discovery code paths over the transports named in the scenario files, not +just health or admin endpoints. + +## Reporting + +Runs write to: + +```text +reports/benchmarks/_/ +``` + +Start here: + +- `scenario_comparison_report.html` +- `scenario_comparison_report.json` +- `scenario_comparison_report.md` +- `run_summary.json` +- `run_summary.md` +- `comparison_matrix.json` +- `scenarios//summary.json` + +Key reporting behaviors: + +- unified report combines scenario metrics, pairwise deltas, fairness checks, + recommendations, and artifact links when files exist +- validation mode marks metrics as omitted instead of emitting fake zero deltas +- comparison output shows `changed_dimensions` so intentional runtime changes do + not look like fairness failures +- plugin timing is merged from per-process artifacts +- run metadata captures git SHA, runtime, compose version, and host facts +- optional `baseline_comparison.json` is written when `suite.baseline_run` is set + +## Report Regeneration + +Re-render a saved run: + +```bash +cargo run --manifest-path crates/contextforge_benchmark_runner/Cargo.toml -- regenerate-report --run-dir reports/benchmarks/ +``` + +Rebuild comparisons for a saved run: + +```bash +cargo run --manifest-path crates/contextforge_benchmark_runner/Cargo.toml -- compare-run --run-dir reports/benchmarks/ +``` diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 1cf2179b8e..1a0cdded3a 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -45,6 +45,10 @@ criteria = "safe-to-deploy" version = "1.9.1" criteria = "safe-to-deploy" +[[exemptions.async-compression]] +version = "0.4.41" +criteria = "safe-to-deploy" + [[exemptions.async-stream]] version = "0.3.6" criteria = "safe-to-deploy" @@ -53,10 +57,6 @@ criteria = "safe-to-deploy" version = "0.3.6" criteria = "safe-to-deploy" -[[exemptions.axum]] -version = "0.8.8" -criteria = "safe-to-deploy" - [[exemptions.axum]] version = "0.8.9" criteria = "safe-to-deploy" @@ -69,20 +69,16 @@ criteria = "safe-to-deploy" version = "1.6.0" criteria = "safe-to-deploy" -[[exemptions.bitflags]] -version = "2.11.0" -criteria = "safe-to-deploy" - -[[exemptions.bitflags]] -version = "2.11.1" +[[exemptions.btoi]] +version = "0.5.0" criteria = "safe-to-deploy" -[[exemptions.block-buffer]] -version = "0.12.0" +[[exemptions.cassowary]] +version = "0.3.0" criteria = "safe-to-deploy" -[[exemptions.btoi]] -version = "0.5.0" +[[exemptions.castaway]] +version = "0.2.4" criteria = "safe-to-deploy" [[exemptions.cc]] @@ -93,10 +89,6 @@ criteria = "safe-to-deploy" version = "1.1.0" criteria = "safe-to-deploy" -[[exemptions.cmov]] -version = "0.5.3" -criteria = "safe-to-deploy" - [[exemptions.colored]] version = "3.1.1" criteria = "safe-to-run" @@ -105,10 +97,34 @@ criteria = "safe-to-run" version = "4.6.7" criteria = "safe-to-deploy" +[[exemptions.compact_str]] +version = "0.8.1" +criteria = "safe-to-deploy" + +[[exemptions.compression-codecs]] +version = "0.4.37" +criteria = "safe-to-deploy" + +[[exemptions.compression-core]] +version = "0.4.31" +criteria = "safe-to-deploy" + +[[exemptions.cookie]] +version = "0.18.1" +criteria = "safe-to-deploy" + +[[exemptions.cookie_store]] +version = "0.22.0" +criteria = "safe-to-deploy" + [[exemptions.core-foundation]] version = "0.10.1" criteria = "safe-to-deploy" +[[exemptions.crc32fast]] +version = "1.5.0" +criteria = "safe-to-deploy" + [[exemptions.criterion]] version = "0.5.1" criteria = "safe-to-run" @@ -129,22 +145,50 @@ criteria = "safe-to-run" version = "0.8.8" criteria = "safe-to-deploy" -[[exemptions.crypto-common]] -version = "0.2.1" +[[exemptions.crossterm]] +version = "0.28.1" +criteria = "safe-to-deploy" + +[[exemptions.crossterm_winapi]] +version = "0.9.1" +criteria = "safe-to-deploy" + +[[exemptions.csv]] +version = "1.4.0" +criteria = "safe-to-deploy" + +[[exemptions.csv-core]] +version = "0.1.13" criteria = "safe-to-deploy" [[exemptions.ctr]] version = "0.9.2" criteria = "safe-to-deploy" -[[exemptions.ctutils]] -version = "0.4.2" +[[exemptions.ctrlc]] +version = "3.5.2" +criteria = "safe-to-deploy" + +[[exemptions.darling]] +version = "0.20.11" +criteria = "safe-to-deploy" + +[[exemptions.darling_core]] +version = "0.20.11" +criteria = "safe-to-deploy" + +[[exemptions.darling_macro]] +version = "0.13.4" criteria = "safe-to-deploy" [[exemptions.dashmap]] version = "6.1.0" criteria = "safe-to-deploy" +[[exemptions.data-encoding]] +version = "2.10.0" +criteria = "safe-to-deploy" + [[exemptions.deadpool]] version = "0.12.3" criteria = "safe-to-deploy" @@ -157,14 +201,18 @@ criteria = "safe-to-deploy" version = "0.1.4" criteria = "safe-to-deploy" -[[exemptions.digest]] -version = "0.11.2" +[[exemptions.dispatch2]] +version = "0.3.1" criteria = "safe-to-deploy" [[exemptions.dotenvy]] version = "0.15.7" criteria = "safe-to-deploy" +[[exemptions.downcast-rs]] +version = "2.0.2" +criteria = "safe-to-deploy" + [[exemptions.fastrand]] version = "2.4.1" criteria = "safe-to-deploy" @@ -177,6 +225,14 @@ criteria = "safe-to-deploy" version = "0.4.7" criteria = "safe-to-deploy" +[[exemptions.flate2]] +version = "1.1.9" +criteria = "safe-to-deploy" + +[[exemptions.flume]] +version = "0.11.1" +criteria = "safe-to-deploy" + [[exemptions.flume]] version = "0.12.0" criteria = "safe-to-deploy" @@ -217,14 +273,26 @@ criteria = "safe-to-deploy" version = "0.2.24" criteria = "safe-to-deploy" -[[exemptions.getrandom]] -version = "0.4.2" -criteria = "safe-to-deploy" - [[exemptions.ghash]] version = "0.5.1" criteria = "safe-to-deploy" +[[exemptions.goose]] +version = "0.18.1" +criteria = "safe-to-deploy" + +[[exemptions.goose-eggs]] +version = "0.6.0" +criteria = "safe-to-deploy" + +[[exemptions.gumdrop]] +version = "0.8.1" +criteria = "safe-to-deploy" + +[[exemptions.gumdrop_derive]] +version = "0.8.1" +criteria = "safe-to-deploy" + [[exemptions.half]] version = "2.7.1" criteria = "safe-to-run" @@ -237,12 +305,8 @@ criteria = "safe-to-deploy" version = "0.3.3" criteria = "safe-to-deploy" -[[exemptions.hmac]] -version = "0.13.0" -criteria = "safe-to-deploy" - -[[exemptions.hybrid-array]] -version = "0.4.10" +[[exemptions.html-escape]] +version = "0.2.13" criteria = "safe-to-deploy" [[exemptions.hyper-rustls]] @@ -257,6 +321,14 @@ criteria = "safe-to-deploy" version = "0.1.65" criteria = "safe-to-deploy" +[[exemptions.indoc]] +version = "2.0.7" +criteria = "safe-to-deploy" + +[[exemptions.instability]] +version = "0.3.10" +criteria = "safe-to-deploy" + [[exemptions.inventory]] version = "0.3.24" criteria = "safe-to-deploy" @@ -285,6 +357,10 @@ criteria = "safe-to-deploy" version = "0.11.0" criteria = "safe-to-deploy" +[[exemptions.itertools]] +version = "0.13.0" +criteria = "safe-to-deploy" + [[exemptions.jni-sys]] version = "0.3.1" criteria = "safe-to-deploy" @@ -313,6 +389,10 @@ criteria = "safe-to-deploy" version = "0.1.16" criteria = "safe-to-deploy" +[[exemptions.lru]] +version = "0.12.5" +criteria = "safe-to-deploy" + [[exemptions.lru-slab]] version = "0.1.2" criteria = "safe-to-deploy" @@ -329,10 +409,6 @@ criteria = "safe-to-deploy" version = "0.3.10" criteria = "safe-to-deploy" -[[exemptions.md-5]] -version = "0.11.0" -criteria = "safe-to-deploy" - [[exemptions.mimalloc]] version = "0.1.48" criteria = "safe-to-deploy" @@ -345,6 +421,10 @@ criteria = "safe-to-deploy" version = "1.7.2" criteria = "safe-to-run" +[[exemptions.nanorand]] +version = "0.7.0" +criteria = "safe-to-deploy" + [[exemptions.ndarray]] version = "0.17.2" criteria = "safe-to-deploy" @@ -357,10 +437,22 @@ criteria = "safe-to-deploy" version = "0.4.6" criteria = "safe-to-deploy" +[[exemptions.num-format]] +version = "0.4.4" +criteria = "safe-to-deploy" + +[[exemptions.num_threads]] +version = "0.1.7" +criteria = "safe-to-deploy" + [[exemptions.numpy]] version = "0.28.0" criteria = "safe-to-deploy" +[[exemptions.objc2]] +version = "0.6.4" +criteria = "safe-to-deploy" + [[exemptions.objc2-system-configuration]] version = "0.3.2" criteria = "safe-to-deploy" @@ -381,6 +473,10 @@ criteria = "safe-to-deploy" version = "5.3.0" criteria = "safe-to-deploy" +[[exemptions.paste]] +version = "1.0.10" +criteria = "safe-to-deploy" + [[exemptions.phf_codegen]] version = "0.11.3" criteria = "safe-to-deploy" @@ -433,6 +529,14 @@ criteria = "safe-to-deploy" version = "0.2.21" criteria = "safe-to-deploy" +[[exemptions.psl-types]] +version = "2.0.11" +criteria = "safe-to-deploy" + +[[exemptions.publicsuffix]] +version = "2.3.0" +criteria = "safe-to-deploy" + [[exemptions.pyo3]] version = "0.28.3" criteria = "safe-to-deploy" @@ -469,20 +573,16 @@ criteria = "safe-to-deploy" version = "6.0.0" criteria = "safe-to-deploy" -[[exemptions.rand]] -version = "0.9.3" -criteria = "safe-to-deploy" - [[exemptions.rand]] version = "0.9.4" criteria = "safe-to-deploy" -[[exemptions.rand]] +[[exemptions.rand_core]] version = "0.10.1" criteria = "safe-to-deploy" -[[exemptions.rand_core]] -version = "0.10.1" +[[exemptions.ratatui]] +version = "0.29.0" criteria = "safe-to-deploy" [[exemptions.rawpointer]] @@ -529,14 +629,30 @@ criteria = "safe-to-deploy" version = "0.7.1" criteria = "safe-to-deploy" -[[exemptions.sha2]] -version = "0.11.0" +[[exemptions.serde_yaml]] +version = "0.9.34+deprecated" +criteria = "safe-to-deploy" + +[[exemptions.signal-hook]] +version = "0.3.18" +criteria = "safe-to-deploy" + +[[exemptions.signal-hook-mio]] +version = "0.2.5" criteria = "safe-to-deploy" [[exemptions.signal-hook-registry]] version = "1.4.8" criteria = "safe-to-deploy" +[[exemptions.simd-adler32]] +version = "0.3.9" +criteria = "safe-to-deploy" + +[[exemptions.simplelog]] +version = "0.12.2" +criteria = "safe-to-deploy" + [[exemptions.siphasher]] version = "1.0.2" criteria = "safe-to-deploy" @@ -565,6 +681,10 @@ criteria = "safe-to-deploy" version = "3.3.0" criteria = "safe-to-deploy" +[[exemptions.termcolor]] +version = "1.4.1" +criteria = "safe-to-deploy" + [[exemptions.time]] version = "0.1.44" criteria = "safe-to-deploy" @@ -593,10 +713,26 @@ criteria = "safe-to-deploy" version = "0.13.0" criteria = "safe-to-deploy" +[[exemptions.tokio-tungstenite]] +version = "0.27.0" +criteria = "safe-to-deploy" + +[[exemptions.toml_datetime]] +version = "0.6.11" +criteria = "safe-to-deploy" + [[exemptions.toml_datetime]] version = "1.1.1+spec-1.1.0" criteria = "safe-to-deploy" +[[exemptions.toml_edit]] +version = "0.22.27" +criteria = "safe-to-deploy" + +[[exemptions.toml_write]] +version = "0.1.2" +criteria = "safe-to-deploy" + [[exemptions.tracing-opentelemetry]] version = "0.32.1" criteria = "safe-to-deploy" @@ -609,6 +745,10 @@ criteria = "safe-to-run" version = "0.2.6" criteria = "safe-to-run" +[[exemptions.tungstenite]] +version = "0.27.0" +criteria = "safe-to-deploy" + [[exemptions.typenum]] version = "1.19.0" criteria = "safe-to-deploy" @@ -617,6 +757,14 @@ criteria = "safe-to-deploy" version = "0.9.0" criteria = "safe-to-deploy" +[[exemptions.unicode-segmentation]] +version = "1.13.2" +criteria = "safe-to-deploy" + +[[exemptions.unicode-truncate]] +version = "1.1.0" +criteria = "safe-to-deploy" + [[exemptions.unicode_names2]] version = "1.3.0" criteria = "safe-to-deploy" @@ -625,6 +773,14 @@ criteria = "safe-to-deploy" version = "1.3.0" criteria = "safe-to-deploy" +[[exemptions.unsafe-libyaml]] +version = "0.2.11" +criteria = "safe-to-deploy" + +[[exemptions.utf8-width]] +version = "0.1.8" +criteria = "safe-to-deploy" + [[exemptions.uuid]] version = "1.23.0" criteria = "safe-to-deploy" @@ -657,6 +813,18 @@ criteria = "safe-to-deploy" version = "2.1.1" criteria = "safe-to-deploy" +[[exemptions.winapi]] +version = "0.3.9" +criteria = "safe-to-deploy" + +[[exemptions.winapi-i686-pc-windows-gnu]] +version = "0.4.0" +criteria = "safe-to-deploy" + +[[exemptions.winapi-x86_64-pc-windows-gnu]] +version = "0.4.0" +criteria = "safe-to-deploy" + [[exemptions.wiremock]] version = "0.6.5" criteria = "safe-to-run" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 0289fd4abf..185ca9c290 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -78,6 +78,13 @@ user-id = 267 user-login = "tarcieri" user-name = "Tony Arcieri" +[[publisher.block-buffer]] +version = "0.10.4" +when = "2023-03-09" +user-id = 5059 +user-login = "newpavlov" +user-name = "Artyom Pavlov" + [[publisher.bumpalo]] version = "3.20.2" when = "2026-02-19" @@ -153,13 +160,6 @@ user-id = 11235 user-login = "baloo" user-name = "Arthur Gautier" -[[publisher.const-oid]] -version = "0.10.2" -when = "2026-01-07" -user-id = 267 -user-login = "tarcieri" -user-name = "Tony Arcieri" - [[publisher.cpufeatures]] version = "0.2.17" when = "2025-01-25" @@ -181,6 +181,13 @@ user-id = 11235 user-login = "baloo" user-name = "Arthur Gautier" +[[publisher.digest]] +version = "0.10.7" +when = "2023-05-19" +user-id = 267 +user-login = "tarcieri" +user-name = "Tony Arcieri" + [[publisher.fallible-iterator]] version = "0.2.0" when = "2019-03-10" @@ -249,13 +256,6 @@ user-id = 359 user-login = "seanmonstar" user-name = "Sean McArthur" -[[publisher.hyper-rustls]] -version = "0.27.8" -when = "2026-04-12" -user-id = 4556 -user-login = "djc" -user-name = "Dirkjan Ochtman" - [[publisher.hyper-tls]] version = "0.6.0" when = "2023-11-27" @@ -317,6 +317,13 @@ when = "2026-04-13" user-id = 55123 user-login = "rust-lang-owner" +[[publisher.linux-raw-sys]] +version = "0.4.15" +when = "2025-01-08" +user-id = 6825 +user-login = "sunfishcode" +user-name = "Dan Gohman" + [[publisher.linux-raw-sys]] version = "0.12.1" when = "2025-12-23" @@ -338,6 +345,13 @@ user-id = 2915 user-login = "Amanieu" user-name = "Amanieu d'Antras" +[[publisher.md-5]] +version = "0.10.6" +when = "2023-09-22" +user-id = 5059 +user-login = "newpavlov" +user-name = "Artyom Pavlov" + [[publisher.memchr]] version = "2.8.0" when = "2026-02-06" @@ -472,8 +486,8 @@ user-login = "JohnTitor" user-name = "Yuki Okushi" [[publisher.postgres-protocol]] -version = "0.6.11" -when = "2026-03-30" +version = "0.6.10" +when = "2026-01-14" user-id = 55015 user-login = "paolobarbolini" user-name = "Paolo Barbolini" @@ -604,6 +618,13 @@ user-id = 341684 user-login = "alexhancock" user-name = "Alex Hancock" +[[publisher.rustix]] +version = "0.38.44" +when = "2025-01-21" +user-id = 6825 +user-login = "sunfishcode" +user-name = "Dan Gohman" + [[publisher.rustix]] version = "1.1.4" when = "2026-02-22" @@ -723,6 +744,13 @@ user-id = 3618 user-login = "dtolnay" user-name = "David Tolnay" +[[publisher.serde_spanned]] +version = "0.6.9" +when = "2025-06-06" +user-id = 6743 +user-login = "epage" +user-name = "Ed Page" + [[publisher.serde_spanned]] version = "1.1.1" when = "2026-03-31" @@ -730,6 +758,13 @@ user-id = 6743 user-login = "epage" user-name = "Ed Page" +[[publisher.sha2]] +version = "0.10.9" +when = "2025-04-30" +user-id = 5059 +user-login = "newpavlov" +user-name = "Artyom Pavlov" + [[publisher.slab]] version = "0.4.12" when = "2026-01-31" @@ -758,6 +793,13 @@ user-id = 5 user-login = "sfackler" user-name = "Steven Fackler" +[[publisher.syn]] +version = "1.0.109" +when = "2023-02-24" +user-id = 3618 +user-login = "dtolnay" +user-name = "David Tolnay" + [[publisher.syn]] version = "2.0.117" when = "2026-02-20" @@ -800,13 +842,6 @@ user-id = 1139 user-login = "Manishearth" user-name = "Manish Goregaokar" -[[publisher.tokio]] -version = "1.51.1" -when = "2026-04-08" -user-id = 6741 -user-login = "Darksonn" -user-name = "Alice Ryhl" - [[publisher.tokio-macros]] version = "2.7.0" when = "2026-04-03" @@ -849,6 +884,13 @@ user-id = 6741 user-login = "Darksonn" user-name = "Alice Ryhl" +[[publisher.toml]] +version = "0.8.23" +when = "2025-06-06" +user-id = 6743 +user-login = "epage" +user-name = "Ed Page" + [[publisher.toml]] version = "0.9.12+spec-1.1.0" when = "2026-02-10" @@ -976,8 +1018,15 @@ user-login = "Manishearth" user-name = "Manish Goregaokar" [[publisher.unicode-width]] -version = "0.2.2" -when = "2025-10-06" +version = "0.1.14" +when = "2024-09-19" +user-id = 1139 +user-login = "Manishearth" +user-name = "Manish Goregaokar" + +[[publisher.unicode-width]] +version = "0.2.0" +when = "2024-09-19" user-id = 1139 user-login = "Manishearth" user-name = "Manish Goregaokar" @@ -1541,6 +1590,12 @@ who = "Nils Ponsard " criteria = "safe-to-deploy" delta = "0.3.31 -> 0.3.32" +[[audits.ariel-os.audits.litrs]] +who = "Antoine Lavandier " +criteria = "safe-to-deploy" +version = "1.0.0" +notes = "No new unsafe functions, most of the changes are formatting and the new functional code seems reasonable to me" + [[audits.ariel-os.audits.num-conv]] who = "Nils Ponsard " criteria = "safe-to-deploy" @@ -1659,6 +1714,21 @@ start = "2025-08-14" end = "2027-01-08" notes = "The Bytecode Alliance is the author of this crate" +[[audits.bytecode-alliance.audits.adler2]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "2.0.0" +notes = "Fork of the original `adler` crate, zero unsfae code, works in `no_std`, does what it says on th tin." + +[[audits.bytecode-alliance.audits.allocator-api2]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +delta = "0.2.18 -> 0.2.20" +notes = """ +The changes appear to be reasonable updates from Rust's stdlib imported into +`allocator-api2`'s copy of this code. +""" + [[audits.bytecode-alliance.audits.anes]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -1671,6 +1741,46 @@ criteria = "safe-to-deploy" version = "1.1.2" notes = "Contains `unsafe` code but it's well-documented and scoped to what it's intended to be doing. Otherwise a well-focused and straightforward crate." +[[audits.bytecode-alliance.audits.bitflags]] +who = "Jamey Sharp " +criteria = "safe-to-deploy" +delta = "2.1.0 -> 2.2.1" +notes = """ +This version adds unsafe impls of traits from the bytemuck crate when built +with that library enabled, but I believe the impls satisfy the documented +safety requirements for bytemuck. The other changes are minor. +""" + +[[audits.bytecode-alliance.audits.bitflags]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "2.3.2 -> 2.3.3" +notes = """ +Nothing outside the realm of what one would expect from a bitflags generator, +all as expected. +""" + +[[audits.bytecode-alliance.audits.bitflags]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "2.4.1 -> 2.6.0" +notes = """ +Changes in how macros are invoked and various bits and pieces of macro-fu. +Otherwise no major changes and nothing dealing with `unsafe`. +""" + +[[audits.bytecode-alliance.audits.bitflags]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "2.7.0 -> 2.9.4" +notes = "Tweaks to the macro, nothing out of order." + +[[audits.bytecode-alliance.audits.bitflags]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "2.10.0 -> 2.11.1" +notes = "Minor updates, nothing awry here." + [[audits.bytecode-alliance.audits.cipher]] who = "Andrew Brown " criteria = "safe-to-deploy" @@ -1765,6 +1875,12 @@ criteria = "safe-to-deploy" delta = "0.3.27 -> 0.3.31" notes = 'New waker_ref module contains "FIXME: panics on Arc::clone / refcount changes could wreak havoc..." comment, but this corner case feels low risk.' +[[audits.bytecode-alliance.audits.getrandom]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.4.1 -> 0.4.2" +notes = "Nothing awry in this update, standard updates for some platforms and other misc things." + [[audits.bytecode-alliance.audits.heck]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -1845,6 +1961,42 @@ criteria = "safe-to-deploy" delta = "0.1.0 -> 0.2.0" notes = "Some unsafe code, but not more than before. Nothing awry." +[[audits.bytecode-alliance.audits.miniz_oxide]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "0.7.1" +notes = """ +This crate is a Rust implementation of zlib compression/decompression and has +been used by default by the Rust standard library for quite some time. It's also +a default dependency of the popular `backtrace` crate for decompressing debug +information. This crate forbids unsafe code and does not otherwise access system +resources. It's originally a port of the `miniz.c` library as well, and given +its own longevity should be relatively hardened against some of the more common +compression-related issues. +""" + +[[audits.bytecode-alliance.audits.miniz_oxide]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.7.1 -> 0.8.0" +notes = "Minor updates, using new Rust features like `const`, no major changes." + +[[audits.bytecode-alliance.audits.miniz_oxide]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.8.0 -> 0.8.5" +notes = """ +Lots of small updates here and there, for example around modernizing Rust +idioms. No new `unsafe` code and everything looks like what you'd expect a +compression library to be doing. +""" + +[[audits.bytecode-alliance.audits.miniz_oxide]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.8.5 -> 0.8.9" +notes = "No new unsafe code, just refactorings." + [[audits.bytecode-alliance.audits.nu-ansi-term]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -1873,6 +2025,18 @@ a few `unsafe` blocks related to utf-8 validation which are locally verifiable as correct and otherwise this crate is good to go. """ +[[audits.bytecode-alliance.audits.rand]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.10.0 -> 0.10.1" +notes = "Minor logging-based updated fixing a recent advisory for the crate." + +[[audits.bytecode-alliance.audits.sha1]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +delta = "0.10.5 -> 0.10.6" +notes = "Only new code is some loongarch64 additions which include assembly code for that platform." + [[audits.bytecode-alliance.audits.sha1_smol]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -1945,6 +2109,12 @@ criteria = "safe-to-deploy" version = "0.2.4" notes = "Implements a concurrency primitive with atomics, and is not obviously incorrect" +[[audits.bytecode-alliance.audits.utf-8]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +version = "0.7.6" +notes = "Small library that uses `unsafe` only around `str::from_utf8_unchecked` after explicitly verifying UTF-8." + [[audits.bytecode-alliance.audits.vcpkg]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -2021,6 +2191,12 @@ criteria = "safe-to-deploy" version = "0.1.1" notes = "No unsafe usage or ambient capabilities" +[[audits.embark-studios.audits.ident_case]] +who = "Johan Andersson " +criteria = "safe-to-deploy" +version = "1.0.1" +notes = "No unsafe usage or ambient capabilities" + [[audits.embark-studios.audits.idna]] who = "Johan Andersson " criteria = "safe-to-deploy" @@ -2094,6 +2270,19 @@ criteria = "safe-to-deploy" version = "0.1.0" notes = "No unsafe usage or ambient capabilities, sane build script" +[[audits.google.audits.arrayvec]] +who = "Lukasz Anforowicz " +criteria = "safe-to-deploy" +version = "0.7.6" +notes = ''' +Grepped for `-i cipher`, `-i crypto`, `'\bfs\b'`, `'\bnet\b'` and there were +no hits, except for some `net` usage in tests. + +The crate has quite a few bits of `unsafe` Rust. The audit comments can be +found in https://chromium-review.googlesource.com/c/chromium/src/+/6187726/2 +''' +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.autocfg]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -2107,6 +2296,22 @@ criteria = "safe-to-deploy" version = "0.22.1" aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.bitflags]] +who = "Lukasz Anforowicz " +criteria = "safe-to-deploy" +version = "1.3.2" +notes = """ +Security review of earlier versions of the crate can be found at +(Google-internal, sorry): go/image-crate-chromium-security-review + +The crate exposes a function marked as `unsafe`, but doesn't use any +`unsafe` blocks (except for tests of the single `unsafe` function). I +think this justifies marking this crate as `ub-risk-1`. + +Additional review comments can be found at https://crrev.com/c/4723145/31 +""" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.byteorder]] who = "danakj " criteria = "safe-to-deploy" @@ -2348,6 +2553,12 @@ delta = "2.0.0-beta1 -> 2.0.0-beta2" notes = "from_utf8_unchecked unsafe remove, all other unsafe not meaningfully changed" aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.itertools]] +who = "ChromeOS" +criteria = "safe-to-run" +version = "0.10.5" +aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/refs/heads/main/cargo-vet/audits.toml?format=TEXT" + [[audits.google.audits.lazy_static]] who = "Lukasz Anforowicz " criteria = "safe-to-deploy" @@ -2525,6 +2736,13 @@ delta = "1.0.19 -> 1.0.20" notes = "Only minor updates to documentation and the mock today used for testing." aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.sha1]] +who = "David Koloski " +criteria = "safe-to-deploy" +version = "0.10.5" +notes = "Reviewed on https://fxrev.dev/712371." +aggregated-from = "https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/third_party/rust_crates/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.smallvec]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -2556,6 +2774,28 @@ Previously reviewed during security review and the audit is grandparented in. """ aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.strum]] +who = "danakj@chromium.org" +criteria = "safe-to-deploy" +version = "0.25.0" +notes = """ +Reviewed in https://crrev.com/c/5171063 + +Previously reviewed during security review and the audit is grandparented in. +""" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + +[[audits.google.audits.strum_macros]] +who = "danakj@chromium.org" +criteria = "safe-to-deploy" +version = "0.25.3" +notes = """ +Reviewed in https://crrev.com/c/5171063 + +Previously reviewed during security review and the audit is grandparented in. +""" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.tinytemplate]] who = "Ying Hsu " criteria = "safe-to-run" @@ -2597,6 +2837,21 @@ who = "David Cook " criteria = "safe-to-deploy" delta = "0.3.3 -> 0.3.4" +[[audits.isrg.audits.getrandom]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.3.4 -> 0.4.0" + +[[audits.isrg.audits.getrandom]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.4.0 -> 0.4.1" + +[[audits.isrg.audits.hmac]] +who = "David Cook " +criteria = "safe-to-deploy" +version = "0.12.1" + [[audits.isrg.audits.once_cell]] who = "Brandon Pitman " criteria = "safe-to-deploy" @@ -2634,6 +2889,21 @@ who = "David Cook " criteria = "safe-to-deploy" version = "0.3.0" +[[audits.isrg.audits.rand]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.8.5 -> 0.9.1" + +[[audits.isrg.audits.rand]] +who = "Tim Geoghegan " +criteria = "safe-to-deploy" +delta = "0.9.1 -> 0.9.2" + +[[audits.isrg.audits.rand]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.9.2 -> 0.10.0" + [[audits.isrg.audits.rand_chacha]] who = "David Cook " criteria = "safe-to-deploy" @@ -2649,11 +2919,6 @@ who = "J.C. Jones " criteria = "safe-to-deploy" delta = "0.9.3 -> 0.9.5" -[[audits.isrg.audits.rand_core]] -who = "David Cook " -criteria = "safe-to-deploy" -delta = "0.9.5 -> 0.10.0" - [[audits.isrg.audits.rayon-core]] who = "Ameer Ghani " criteria = "safe-to-deploy" @@ -2715,6 +2980,24 @@ end = "2024-06-16" notes = "Maintained by Henri Sivonen who works at Mozilla." aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.adler2]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "2.0.0 -> 2.0.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.allocator-api2]] +who = "Nicolas Silva " +criteria = "safe-to-deploy" +version = "0.2.18" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.allocator-api2]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.2.20 -> 0.2.21" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.android_system_properties]] who = "Nicolas Silva " criteria = "safe-to-deploy" @@ -2734,6 +3017,60 @@ criteria = "safe-to-deploy" delta = "0.1.4 -> 0.1.5" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.bitflags]] +who = "Alex Franchuk " +criteria = "safe-to-deploy" +delta = "1.3.2 -> 2.0.2" +notes = "Removal of some unsafe code/methods. No changes to externals, just some refactoring (mostly internal)." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.bitflags]] +who = "Nicolas Silva " +criteria = "safe-to-deploy" +delta = "2.0.2 -> 2.1.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.bitflags]] +who = "Teodor Tanasoaia " +criteria = "safe-to-deploy" +delta = "2.2.1 -> 2.3.2" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.bitflags]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "2.3.3 -> 2.4.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.bitflags]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "2.4.0 -> 2.4.1" +notes = "Only allowing new clippy lints" +aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" + +[[audits.mozilla.audits.bitflags]] +who = [ + "Teodor Tanasoaia ", + "Erich Gubler ", +] +criteria = "safe-to-deploy" +delta = "2.6.0 -> 2.7.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.bitflags]] +who = "Benjamin VanderSloot " +criteria = "safe-to-deploy" +delta = "2.9.4 -> 2.10.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.block2]] +who = "Andy Leiserson " +criteria = "safe-to-deploy" +version = "0.6.2" +notes = "Contains unsafe code to interoperate with the ObjC runtime." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.cfg_aliases]] who = "Alex Franchuk " criteria = "safe-to-deploy" @@ -2810,6 +3147,30 @@ criteria = "safe-to-deploy" version = "0.2.3" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.darling_macro]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.13.4 -> 0.14.2" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.darling_macro]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.14.2 -> 0.14.3" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.darling_macro]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.14.3 -> 0.20.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.darling_macro]] +who = "Ben Dean-Kawamura " +criteria = "safe-to-deploy" +delta = "0.20.1 -> 0.20.10" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.deranged]] who = "Alex Franchuk " criteria = "safe-to-deploy" @@ -2834,6 +3195,30 @@ delta = "0.4.0 -> 0.5.8" notes = "New unsafe code is properly guarded" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.document-features]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +version = "0.2.8" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.document-features]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.2.8 -> 0.2.9" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.document-features]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.2.9 -> 0.2.10" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.document-features]] +who = "Teodor Tanasoaia " +criteria = "safe-to-deploy" +delta = "0.2.10 -> 0.2.11" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.errno]] who = "Mike Hommey " criteria = "safe-to-deploy" @@ -3112,6 +3497,13 @@ https://github.com/madsmtm/objc2/blob/main/crates/objc2/src/topics/frameworks_so """ aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.objc2-encode]] +who = "Andy Leiserson " +criteria = "safe-to-deploy" +version = "4.1.0" +notes = "Support library for objc2 with no unsafe code" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.once_cell]] who = "Mike Hommey " criteria = "safe-to-deploy" @@ -3156,6 +3548,12 @@ version = "11.1.5" notes = "Small random number generator, explicitly not cryptographically secure, no use of unsafe code, no dependencies" aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" +[[audits.mozilla.audits.paste]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "1.0.10 -> 1.0.15" +aggregated-from = "https://raw.githubusercontent.com/mozilla/glean/main/supply-chain/audits.toml" + [[audits.mozilla.audits.percent-encoding]] who = "Valentin Gosu " criteria = "safe-to-deploy" @@ -3270,6 +3668,30 @@ criteria = "safe-to-deploy" delta = "0.10.0 -> 0.11.1" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.strum]] +who = "Teodor Tanasoaia " +criteria = "safe-to-deploy" +delta = "0.25.0 -> 0.26.3" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.strum]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.26.3 -> 0.27.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.strum_macros]] +who = "Teodor Tanasoaia " +criteria = "safe-to-deploy" +delta = "0.25.3 -> 0.26.4" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.strum_macros]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.26.4 -> 0.27.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.subtle]] who = "Simon Friedberger " criteria = "safe-to-deploy" @@ -3524,6 +3946,19 @@ was being selected by the target OS instead of the host OS. """ aggregated-from = "https://raw.githubusercontent.com/zcash/zcash/master/qa/supply-chain/audits.toml" +[[audits.zcash.audits.darling_macro]] +who = "Schell Carl Scivally " +criteria = "safe-to-deploy" +delta = "0.20.10 -> 0.20.11" +notes = "Only includes changes to cargo packaging, the library source itself is unchanged." +aggregated-from = "https://raw.githubusercontent.com/zcash/librustzcash/main/supply-chain/audits.toml" + +[[audits.zcash.audits.document-features]] +who = "Jack Grigg " +criteria = "safe-to-deploy" +delta = "0.2.11 -> 0.2.12" +aggregated-from = "https://raw.githubusercontent.com/zcash/wallet/main/supply-chain/audits.toml" + [[audits.zcash.audits.dunce]] who = "Jack Grigg " criteria = "safe-to-deploy" @@ -3615,6 +4050,18 @@ delta = "1.0.21 -> 1.0.22" notes = "Changes to generated code are to prepend a clippy annotation." aggregated-from = "https://raw.githubusercontent.com/zcash/wallet/main/supply-chain/audits.toml" +[[audits.zcash.audits.strum]] +who = "Jack Grigg " +criteria = "safe-to-deploy" +delta = "0.27.1 -> 0.27.2" +aggregated-from = "https://raw.githubusercontent.com/zcash/librustzcash/main/supply-chain/audits.toml" + +[[audits.zcash.audits.strum_macros]] +who = "Jack Grigg " +criteria = "safe-to-deploy" +delta = "0.27.1 -> 0.27.2" +aggregated-from = "https://raw.githubusercontent.com/zcash/librustzcash/main/supply-chain/audits.toml" + [[audits.zcash.audits.sync_wrapper]] who = "Jack Grigg " criteria = "safe-to-deploy" diff --git a/tests/unit/test_rust_workspace_layout.py b/tests/unit/test_rust_workspace_layout.py index 37b95ab3a9..d2c1250bb5 100644 --- a/tests/unit/test_rust_workspace_layout.py +++ b/tests/unit/test_rust_workspace_layout.py @@ -69,7 +69,15 @@ def test_dockerignore_and_containerfile_keep_rust_servers_out_of_workspace_image def test_crates_directory_is_flat() -> None: remaining = [path.name for path in _top_level_crate_dirs()] - assert remaining == ["a2a_runtime", "mcp_runtime", "request_logging_masking_native_extension", "wrapper"], f"Expected only direct crate folders under crates/: {remaining}" + assert remaining == [ + "a2a_runtime", + "contextforge_benchmark_console", + "contextforge_benchmark_runner", + "contextforge_goose", + "mcp_runtime", + "request_logging_masking_native_extension", + "wrapper", + ], f"Expected only direct crate folders under crates/: {remaining}" def test_gateway_rs_services_directory_is_empty() -> None: diff --git a/uv.lock b/uv.lock index 37430f07e8..51b11ab6d2 100644 --- a/uv.lock +++ b/uv.lock @@ -8,7 +8,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-05T10:56:06.379155Z" +exclude-newer = "2026-04-06T14:03:52.59293Z" exclude-newer-span = "P10D" [options.exclude-newer-package] @@ -2786,6 +2786,7 @@ source = { editable = "." } dependencies = [ { name = "alembic" }, { name = "argon2-cffi" }, + { name = "brotli" }, { name = "cryptography" }, { name = "fastapi" }, { name = "filelock" }, @@ -2978,6 +2979,7 @@ requires-dist = [ { name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "asyncpg", marker = "extra == 'asyncpg'", specifier = ">=0.31.0" }, { name = "atheris", marker = "extra == 'fuzz-atheris'", specifier = ">=3.0.0" }, + { name = "brotli", specifier = "==1.2.0" }, { name = "cookiecutter", marker = "extra == 'templating'", specifier = ">=2.7.1" }, { name = "cpex-encoded-exfil-detection", marker = "extra == 'plugins'", specifier = ">=0.2.0" }, { name = "cpex-pii-filter", marker = "extra == 'plugins'", specifier = ">=0.2.0" },