Merge branch 'codex/fhevm-orchestration-parity-refactor' into claude/test-fhEVM-bun-cli-1Fsxq

Author: claude

.gitignore

@@ -34,6 +34,7 @@ logs/
 !.env.sample
 # Allow shared env templates in subprojects
 !test-suite/e2e/.env.devnet
+!test-suite/fhevm/env/staging/.env.versions.example
 # If you have .env files specific to subprojects, e.g. subproject/.env,
 # the .env rule above will catch them.
 

gateway-contracts/tasks/addHostChains.ts

@@ -40,6 +40,11 @@ task("task:addHostChainsToGatewayConfig")
     // Add host chains
     const gatewayConfig = await hre.ethers.getContractAt("GatewayConfig", proxyAddress, deployer);
     for (const hostChain of hostChains) {
+      const isAlreadyRegistered = await gatewayConfig.isHostChainRegistered(hostChain.chainId);
+      if (isAlreadyRegistered) {
+        console.log(`Host chain ${hostChain.chainId} already registered, skipping.`);
+        continue;
+      }
       await gatewayConfig.addHostChain(hostChain);
     }
 

gateway-contracts/tasks/addPausers.ts

@@ -7,7 +7,7 @@ import { getRequiredEnvVar, loadGatewayAddresses } from "./utils/loadVariables";
 // for local testing. By default, we use the PAUSER_SET_ADDRESS env var, as done in deployment
 task("task:addGatewayPausers")
   .addParam("useInternalPauserSetAddress", "If internal PauserSet address should be used", false, types.boolean)
-  .setAction(async function ({ useInternalGatewayConfigAddress }, hre) {
+  .setAction(async function ({ useInternalPauserSetAddress }, hre) {
     await hre.run("compile:specific", { contract: "contracts/immutable" });
     console.log("Adding pausers to PauserSet contract");
 
@@ -21,14 +21,19 @@ task("task:addGatewayPausers")
       pausers.push(getRequiredEnvVar(`PAUSER_ADDRESS_${idx}`));
     }
 
-    if (useInternalGatewayConfigAddress) {
+    if (useInternalPauserSetAddress) {
       loadGatewayAddresses();
     }
     const pauserSetAddress = getRequiredEnvVar("PAUSER_SET_ADDRESS");
 
     // Add pauser(s)
     const pauserSet = await hre.ethers.getContractAt("PauserSet", pauserSetAddress, deployer);
     for (const pauser of pausers) {
+      const isAlreadyPauser = await pauserSet.isPauser(pauser);
+      if (isAlreadyPauser) {
+        console.log(`Pauser ${pauser} already registered, skipping.`);
+        continue;
+      }
       await pauserSet.addPauser(pauser);
     }
 

test-suite/README.md

@@ -19,8 +19,16 @@ KMS can be configured to two modes:
   - [Quickstart](#quickstart)
   - [Forcing Local Builds](#wip---forcing-local-builds---build)
   - [Local Developer Optimizations](#local-developer-optimizations)
+  - [CLI Reference](#cli-reference)
+  - [Telemetry Checks](#telemetry-checks)
   - [Resuming a Deployment](#resuming-a-deployment)
   - [Deploying a Single Step](#deploying-a-single-step)
+  - [Orchestration Source of Truth](#orchestration-source-of-truth)
+  - [Troubleshooting Deploy Failures](#troubleshooting-deploy-failures)
+  - [Behavior Parity Tests](#behavior-parity-tests)
+  - [CLI Parity Diff Tests](#cli-parity-diff-tests)
+  - [Validation Protocol (PR / Manual)](#validation-protocol-pr--manual)
+  - [Docker Project Isolation](#docker-project-isolation)
 - [Security Policy](#security-policy)
   - [Handling Sensitive Data](#handling-sensitive-data)
     - [Environment Files](#environment-files)
@@ -44,6 +52,12 @@ cd test-suite/fhevm
 # Deploy with local BuildKit cache (disables provenance attestations)
 ./fhevm-cli deploy --local
 
+# Deploy and fail if telemetry services are not visible in Jaeger
+./fhevm-cli deploy --build --telemetry-smoke
+
+# Deploy with versions scraped from the public testnet matrix
+./fhevm-cli deploy --network testnet
+
 # Deploy with threshold 2 out of 2 coprocessors (local multicoprocessor mode)
 ./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2
 
@@ -72,8 +86,29 @@ cd test-suite/fhevm
 
 # Clean up
 ./fhevm-cli clean
+
+# Hard purge for reproducible A/B runs
+./fhevm-cli clean --purge
+```
+
+If you prefer shorter commands with Bun scripts, you can run the same CLI via:
+
+```sh
+cd test-suite/fhevm
+bun run deploy --network testnet
+bun run test input-proof
+bun run telemetry-smoke
+bun run clean --purge-build-cache
 ```
 
+All `clean` purge flags are fhevm-scoped:
+- `--purge-images` removes images referenced by fhevm compose services.
+- `--purge-build-cache` removes the local Buildx cache directory (`.buildx-cache` by default, or `FHEVM_BUILDX_CACHE_DIR` if set).
+- `--purge-local-cache` is a compatibility alias for `--purge-build-cache`.
+- `--purge-networks` removes Docker networks labeled with the active compose project only.
+
+For `deploy --coprocessors N` with `N > 1`, `cast` (Foundry) must be installed locally to derive per-coprocessor accounts from `MNEMONIC`.
+
 ### WIP - Forcing Local Builds (`--build`)
 
 ⚠️ **IMPORTANT: THIS FEATURE IS STILL A WORK IN PROGRESS!** ⚠️
@@ -110,12 +145,63 @@ For faster local iteration, use `--local` to enable a local BuildKit cache (stor
 ./fhevm-cli deploy --local
 ```
 
+For code-path validation, prefer `--build --local` so your local changes are rebuilt while keeping warm cache layers.
+
+To align local versions with currently deployed environments, you can ask deploy to scrape the public version dashboard:
+
+```sh
+./fhevm-cli deploy --network testnet
+./fhevm-cli deploy --network mainnet
+```
+
+Notes:
+- This is best-effort scraping from the public Grafana dashboard DOM.
+- It applies known service version env vars (coprocessor services, kms-connector services, `CORE_VERSION`) before deployment.
+- Contract/relayer versions continue to use local defaults unless explicitly overridden.
+- If your Chromium path is custom, set `FHEVM_GRAFANA_CHROMIUM_BIN=/path/to/chromium`.
+- For deterministic testing, set `FHEVM_GRAFANA_DASHBOARD_HTML_FILE=/path/to/dashboard.html`.
+
 When running tests and you know your Hardhat artifacts are already up to date, you can skip compilation:
 
 ```sh
 ./fhevm-cli test input-proof --no-hardhat-compile
 ```
 
+### CLI Reference
+
+For agent workflows, prefer explicit command+flag forms from this table.
+
+| Command | Flags | Notes |
+| --- | --- | --- |
+| `deploy` | `--build` | Build buildable services before `up -d`. |
+| `deploy` | `--local` / `--dev` | Enable local BuildKit cache (`.buildx-cache` by default). |
+| `deploy` | `--network testnet\|mainnet` | Apply version profile from public dashboard before deploy. |
+| `deploy` | `--coprocessors <n>` | Configure local coprocessor topology size (`n`, max `5`). |
+| `deploy` | `--coprocessor-threshold <t>` | Override topology threshold (`t <= n`). |
+| `deploy` | `--resume <step>` | Redeploy from a specific step onward. |
+| `deploy` | `--only <step>` | Redeploy only one step. |
+| `deploy` | `--telemetry-smoke` | Run Jaeger service smoke-check after deployment. |
+| `deploy` | `--strict-otel` | Fail if OTEL endpoint expects Jaeger and Jaeger is not running. |
+| `test` | `-n, --network <name>` | Test-runtime network selection (default: `staging`). |
+| `test` | `-g, --grep <pattern>` | Override test grep pattern. |
+| `test` | `-v, --verbose` | Verbose test output. |
+| `test` | `-r, --no-relayer` | Disable Rust relayer in tests. |
+| `test` | `--no-hardhat-compile` | Skip compile when artifacts are already up-to-date. |
+| `clean` | `--purge` | Shorthand for `--purge-images --purge-build-cache --purge-networks`. |
+| `clean` | `--purge-images` | Remove images referenced by fhevm compose services only. |
+| `clean` | `--purge-build-cache` | Remove local Buildx cache dir (`.buildx-cache` or `FHEVM_BUILDX_CACHE_DIR`). |
+| `clean` | `--purge-networks` | Remove Docker networks labeled with the active compose project only. |
+| `clean` | `--purge-local-cache` | Alias of `--purge-build-cache` (kept for compatibility). |
+| `pause` / `unpause` | `host` or `gateway` | Contract pause controls. |
+| `upgrade` | `<service>` | Restart selected service compose stack. |
+| `logs` | `<service>` | Stream container logs for one service. |
+| `telemetry-smoke` | _none_ | Validate required Jaeger services are present. |
+
+Notes:
+- `--network` on `deploy` selects a **version profile** (`testnet`/`mainnet`).
+- `--network` on `test` selects a **test runtime network** (default `staging`).
+- They are intentionally different and command-scoped.
+
 ### Resuming a deployment
 
 If a deploy fails mid-way, you can resume from a specific step without tearing down containers or regenerating `.env` files:
@@ -132,6 +218,9 @@ When resuming:
 - Services **before** the resume step are preserved (containers + volumes kept)
 - Services **from** the resume step onwards are torn down and redeployed
 
+Multicoprocessor safety note:
+- If you change multicoprocessor topology (`--coprocessors` and/or `--coprocessor-threshold`) while using `--resume` from `coprocessor` or later, the CLI intentionally forces resume from `minio` to reset key material coherently across all coprocessors.
+
 ### Deploying a single step
 
 To redeploy only a single service without touching others:
@@ -149,6 +238,203 @@ You can combine `--only` or `--resume` with other flags:
 ./fhevm-cli deploy --only gateway-sc --build --local
 ```
 
+### Telemetry Checks
+
+The coprocessor env now ensures `OTEL_EXPORTER_OTLP_ENDPOINT` is present.
+If it is missing, deploy defaults it to `http://jaeger:4317` in `.env.coprocessor.local`.
+
+Use strict endpoint validation (requires Jaeger to be up first):
+
+```sh
+./fhevm-cli deploy --strict-otel
+```
+
+Run smoke validation on demand:
+
+```sh
+./fhevm-cli telemetry-smoke
+```
+
+`telemetry-smoke` retries for a short warm-up window before failing, to reduce false negatives while traces are still starting up.
+
+Or include it in deploy:
+
+```sh
+./fhevm-cli deploy --telemetry-smoke
+```
+
+### Orchestration source of truth
+
+The orchestration is Bun-first with shell entrypoint wrappers:
+
+- `test-suite/fhevm/fhevm-cli`
+- `test-suite/fhevm/scripts/deploy-fhevm-stack.sh`
+
+Canonical deploy/test metadata now lives in one TypeScript source:
+
+- `test-suite/fhevm/scripts/bun/manifest.ts`
+
+Runtime implementation:
+
+- `test-suite/fhevm/scripts/bun/cli.ts`
+- `test-suite/fhevm/scripts/bun/process.ts`
+
+Compatibility snapshots used for parity verification:
+
+- `test-suite/fhevm/fhevm-cli.legacy`
+- `test-suite/fhevm/scripts/deploy-fhevm-stack.legacy.sh`
+
+You can force legacy mode explicitly with:
+
+```sh
+FHEVM_CLI_IMPL=legacy ./fhevm-cli deploy
+```
+
+Version updates do not require editing many per-service vars manually.
+You can override them in one place:
+
+- `FHEVM_STACK_VERSION` (gateway/host/coprocessor/kms-connector/test-suite)
+- `FHEVM_CORE_VERSION`
+- `FHEVM_RELAYER_VERSION` (relayer + relayer-migrate)
+
+These can be set as environment variables, or in an optional file:
+
+- `test-suite/fhevm/env/staging/.env.versions`
+- (template: `test-suite/fhevm/env/staging/.env.versions.example`)
+
+Example:
+
+```sh
+FHEVM_STACK_VERSION=v0.12.0-rc.1 \
+FHEVM_CORE_VERSION=v0.14.0-rc.1 \
+FHEVM_RELAYER_VERSION=v0.10.0-rc.1 \
+./fhevm-cli deploy
+```
+
+### Troubleshooting deploy failures
+
+When deploy fails, the script now surfaces explicit hints for common operational failure modes.
+
+- OOM-killed critical service:
+  - Symptom: failure includes `looks OOM-killed`.
+  - Action: increase Docker memory and resume from the failed step, for example:
+    - `./fhevm-cli deploy --resume coprocessor`
+
+- Key bootstrap / CRS not ready:
+  - Symptom: failure includes `Detected key-bootstrap-not-ready state`.
+  - Action: wait for keygen/CRS generation to settle, then resume from gateway contracts:
+    - `./fhevm-cli deploy --resume gateway-sc`
+
+- Gateway helper image export conflict (`already exists`):
+  - Symptom: build fails while starting gateway contracts.
+  - Action: deploy now auto-retries once after removing conflicting `gateway-contracts` tags.
+  - Manual fallback for repeated collisions:
+    - `./fhevm-cli clean --purge-images --purge-build-cache`
+
+### Behavior parity tests
+
+A behavior-level shell test suite validates deploy orchestration outcomes (ordering, `--resume`, `--only`, build semantics, env patch timing, actionable failure hints, strict OTEL checks, purge flags, and telemetry smoke checks).
+
+Run it with:
+
+```sh
+./test-suite/fhevm/scripts/tests/deploy-fhevm-stack.behavior.sh
+```
+
+### CLI parity diff tests
+
+A dry-run parity harness executes legacy Bash and Bun CLI flows under the same mocked Docker environment, then diffs command traces and exit codes for sampled command cases.
+
+Run it with:
+
+```sh
+./test-suite/fhevm/scripts/tests/fhevm-cli-parity-diff.sh
+```
+
+### Validation protocol (PR / manual)
+
+Use this when you want high confidence that the CLI is safe and functionally correct.
+
+#### 1) Quick confidence protocol (10-20 min)
+
+```sh
+cd test-suite/fhevm
+
+# Optional but strongly recommended: isolate this run from any other local docker tests.
+export FHEVM_DOCKER_PROJECT=fhevm-pr
+
+# Fresh start
+./fhevm-cli clean --purge
+
+# Bring up stack with testnet profile + telemetry gate
+./fhevm-cli deploy --resume core --network testnet --telemetry-smoke
+
+# Fast deterministic suite
+./fhevm-cli test input-proof --no-hardhat-compile
+./fhevm-cli test user-decryption --no-hardhat-compile
+./fhevm-cli test erc20 --no-hardhat-compile
+
+# Operational commands
+./fhevm-cli pause host
+./fhevm-cli unpause host
+./fhevm-cli pause gateway
+./fhevm-cli unpause gateway
+./fhevm-cli upgrade coprocessor
+
+# Cleanup
+./fhevm-cli clean
+```
+
+#### 2) Full protocol (same matrix used for end-to-end QA)
+
+```sh
+cd /path/to/fhevm
+bash /private/tmp/fhevm_full_qa.sh
+```
+
+Expected outcome:
+- No `[QA][FAIL]` in the generated summary log
+- Final line is `[QA][DONE] artifact_dir=...`
+
+Inspect results:
+
+```sh
+ART_DIR=$(cat /tmp/fhevm-full-qa-last-artifacts.txt)
+cat "$ART_DIR/summary.log"
+```
+
+#### 3) Safety check for cleanup scope (no side effects)
+
+Use sentinels to confirm clean/purge commands do not affect unrelated docker resources.
+
+```sh
+docker run -d --name qa-clean-sentinel alpine:3.20 sleep 36000
+docker network create qa-clean-sentinel-net
+
+cd test-suite/fhevm
+./fhevm-cli clean --purge
+
+docker ps -a --filter name='^qa-clean-sentinel$' --format '{{.Names}}'
+docker network ls --filter name='^qa-clean-sentinel-net$' --format '{{.Name}}'
+
+docker rm -f qa-clean-sentinel
+docker network rm qa-clean-sentinel-net
+```
+
+Both sentinel checks must still return their names after clean/purge.
+
+### Docker project isolation
+
+To avoid collisions with other local Docker-based test workflows, set a dedicated compose project name:
+
+```sh
+export FHEVM_DOCKER_PROJECT=fhevm-dev
+cd test-suite/fhevm
+./fhevm-cli deploy
+```
+
+All deploy/clean/purge operations are then scoped to that project.
+
 ## Security policy
 
 ### Handling sensitive data

test-suite/fhevm/env/staging/.env.coprocessor

@@ -30,6 +30,7 @@ FHE_KEY_ID=421c8116661b2150a46badd3956564ad8d4981718d30fa66a36342bee1b13dbf
 # host_listener uses RPC_WS_URL (ws/wss), poller uses RPC_HTTP_URL (http/https)
 RPC_HTTP_URL=http://host-node:8545
 RPC_WS_URL=ws://host-node:8545
+OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4317
 CHAIN_ID=12345
 ACL_CONTRACT_ADDRESS=0x05fD9B5EFE0a996095f42Ed7e77c390810CF660c
 FHEVM_EXECUTOR_CONTRACT_ADDRESS=0xcCAe95fF1d11656358E782570dF0418F59fA40e1