feat(test-suite): gw reliable benchmark (#923)

* chore(test-suite): add gw benchmarking cli

* docs(test-suite): update README

* fix(test-suite): error handling in bench

* chore(test-suite): use http instead of ws

Author: eudelins-zama

test-suite/gateway-stress/Cargo.lock

@@ -22,6 +22,7 @@ aws-sdk-kms = { version = "1.79.0", default-features = true }
 aws-sdk-s3 = { version = "1.98.0", default-features = true }
 clap = { version = "4.5.38", features = ["cargo", "derive"] }
 config = { version = "0.15.13", default-features = false, features = ["toml"] }
+csv = { version = "1.3.1", default-features = false }
 futures = { version = "0.3.31", default-features = false }
 humantime = "2.2.0"
 humantime-serde = "1.1.1"

test-suite/gateway-stress/Cargo.toml

@@ -1,8 +1,28 @@
 # Gateway Stress-Test Tool
 
+## Introduction
+
 A simple tool to send a configurable number of parallel decryption requests (public or user
 decrypts at the time of writing), at a given frequency and for a specified duration.
 
+## Table of Contents
+- [Introduction](#introduction)
+- [Build](#build)
+- [Configuration](#configuration)
+- [Run](#run)
+  - [Stress testing](#stress-testing)
+  - [Benchmarking](#benchmarking)
+- [Tracing](#tracing)
+- [Bonus: Generating handles via coprocessor stress-test-generator](#bonus-generating-handles-via-coprocessor-stress-test-generator)
+
+## Build
+
+You can build the tool by running the `cargo build --release` command in the
+`test-suite/gateway-stress` directory.
+
+Alternatively, you can run the manual `gateway-stress-tool-docker-build` workflow to trigger the
+build of the Docker images for the tool.
+
 ## Configuration
 
 To configure the tool, you can use either a configuration file (TOML format) or environment
@@ -13,16 +33,10 @@ along with its associated environment variable.
 
 Configuration fields defined via environment variables override those in the configuration file.
 
-## Build
-
-You can build the tool by running the `cargo build --release` command in the
-`test-suite/gateway-stress` directory.
-
-Alternatively, you can run the manual `gateway-stress-tool-docker-build` workflow to trigger the
-build of the Docker images for the tool.
-
 ## Run
 
+### Stress testing
+
 Once the `gateway-stress` binary has been built, you can run the following commands:
 
 ```bash
@@ -48,6 +62,28 @@ cargo run -- -c config/config.toml user
 
 Note that the `mixed` command of the CLI is not implemented yet.
 
+### Benchmarking
+
+The `benchmark` command take a CSV file in input (and the global config file as well).
+Each line of this CSV represent a burst of decryption to benchmark, which is composed of:
+- The number of parallel requests in the burst (1st column)
+- The number of time we must measure this burst (2nd column)
+- The type of decryption in the burst (`public` or `user`)
+
+See the [templates](./templates) folder for examples.
+
+It will then run the benchmark and stores the results (average and standard deviation of latency
+and throughput) for each burst in a CSV file.
+
+```bash
+# Run a benchmarking session using `templates/small_bench.csv` as input and store the global
+# results in `/tmp/bench.csv`
+./gateway-stress -c config/config.toml benchmark -i templates/small_bench.csv -o /tmp/bench.csv
+
+# Same, but also store each burst result in `tmp/full.csv`
+./gateway-stress -c config/config.toml benchmark -i templates/small_bench.csv -o /tmp/bench.csv -r /tmp/full.csv
+```
+
 ## Tracing
 
 This tool aims to output only essential information regarding the status of the test. The main
@@ -67,17 +103,23 @@ RUST_LOG="gateway_stress=debug,alloy=debug" ./gateway-stress -c config/config.to
 RUST_LOG="debug" ./gateway-stress -c config/config.toml public
 ```
 
-## Bonus: Generating handles via coprocessor-stress-test-generator
+## Bonus: Generating handles via coprocessor stress-test-generator
+
+To use this tool, you would need already existing handles to decrypt. You could use coprocessor's
+`stress-test-generator` tool to generate these handles.
 
-First clone the repo. Then, look at the `README.md` and gather all the environment variable values
-needed (default values work only for e2e setup).
+The tool is located at `coprocessor/fhevm-engine/stress-test-generator` in the `fhevm` repo.
+Then, look at the `README.md` and gather all the environment variable values needed (default
+values work only for e2e setup).
 
 ```bash
-git checkout simon/chore/fhevm-0.8.3
 export EVGEN_DB_URL="TODO"
 export ACL_CONTRACT_ADDRESS="TODO"
 # ...
 EVGEN_SCENARIO=data/minitest_003_generate_handles_for_decryption.csv make run
 ```
 
 This will generate the `data/handles_for_pub_decryption` and `handles_for_usr_decryption` files.
+
+Make sure that the 6th column of the `EVGEN_SCENARIO` file match the `allowed_contract` value of
+this tool's configuration, and that the 7th column match the wallet address used by the tool.

test-suite/gateway-stress/README.md

@@ -17,7 +17,7 @@ decryption_address = "0xc9bAE822fE6793e3B456144AdB776D5A318CB71e"
 # The time during which the tool will send requests burst to the Gateway
 # Format documentation: https://docs.rs/humantime/latest/humantime/
 # ENV: TESTS_DURATION
-tests_duration = "10s"
+tests_duration = "500ms"
 
 # If requests burst must be sent sequentially when the previous one has been completed
 # Defaults to false, if set to true, then `tests_interval` is ignored
@@ -37,14 +37,14 @@ parallel_requests = 5
 # ENV: USER_CT_HANDLES (format: comma separated list)
 # Example: USER_CT_HANDLES="0x408a1510fb9ebf45651a6b0d40ce34c52a4bfc6338ff00000000000030390500,0x408a1510fb9ebf45651a6b0d40ce34c52a4bfc6338ff00000000000030390500"
 user_ct_handles = [
-    "0x7a88b93ad2e636eaba00eab25e3db6a39bb3965a2fff00000000000030390500",
+    "0x7e314aadb1fe25d0eedc986529cf05604b71b717fdff00000000000030390500",
 ]
 
 # The handles used for public decryptions
 # ENV: PUBLIC_CT_HANDLES (format: comma separated list)
 # Example: PUBLIC_CT_HANDLES="0xbbb647e5a8dba46ed6f2eff8aed91d96ce9a80f537ff00000000000030390500,0xbbb647e5a8dba46ed6f2eff8aed91d96ce9a80f537ff00000000000030390500"
 public_ct_handles = [
-    "0x1c7538f08f6054130981aca8a7d85896ca2a197067ff00000000000030390500",
+    "0xcc0fe2694c954beeea00716b20dbff2f4aa3756afbff00000000000030390500",
 ]
 
 # The contract allowed for user decrytion on ct_handles
@@ -53,7 +53,7 @@ allowed_contract = "0xa5880e99d86F081E8D3868A8C4732C8f65dfdB08"
 
 # Wallet's private key as a hex string (optional if `aws_kms_config` or `mnemonic` is configured)
 # ENV: PRIVATE_KEY
-private_key = "3f45b129a7fd099146e9fe63851a71646231f7743c712695f3b2d2bf0e41c774"
+private_key = "24af7cb5f6cd0f29df22c6f3e2f18ee5b3949f5a489a14b0674bef8fd89bfe91"
 
 # Wallet's mnemonic (optional if `aws_kms_config` or `private_key` is configured)
 # ENV: MNEMONIC