test-suite: refine clean aliases and multicoprocessor resume UX

Eikix · Eikix · commit cdff7de21092 · 2026-02-16T21:06:44.000+01:00
diff --git a/test-suite/README.md b/test-suite/README.md
@@ -98,12 +98,13 @@ cd test-suite/fhevm
 bun run deploy --network testnet
 bun run test input-proof
 bun run telemetry-smoke
-bun run clean --purge-local-cache
+bun run clean --purge-build-cache
 ```
 
 All `clean` purge flags are fhevm-scoped:
 - `--purge-images` removes images referenced by fhevm compose services.
-- `--purge-build-cache` and `--purge-local-cache` remove local Buildx cache directory (`.buildx-cache` by default, or `FHEVM_BUILDX_CACHE_DIR` if set).
+- `--purge-build-cache` removes the local Buildx cache directory (`.buildx-cache` by default, or `FHEVM_BUILDX_CACHE_DIR` if set).
+- `--purge-local-cache` is a compatibility alias for `--purge-build-cache`.
 - `--purge-networks` removes Docker networks labeled with the active compose project only.
 
 For `deploy --coprocessors N` with `N > 1`, `cast` (Foundry) must be installed locally to derive per-coprocessor accounts from `MNEMONIC`.
@@ -186,11 +187,11 @@ For agent workflows, prefer explicit command+flag forms from this table.
 | `test` | `-v, --verbose` | Verbose test output. |
 | `test` | `-r, --no-relayer` | Disable Rust relayer in tests. |
 | `test` | `--no-hardhat-compile` | Skip compile when artifacts are already up-to-date. |
-| `clean` | `--purge` | Shorthand for all purge flags below. |
+| `clean` | `--purge` | Shorthand for `--purge-images --purge-build-cache --purge-networks`. |
 | `clean` | `--purge-images` | Remove images referenced by fhevm compose services only. |
 | `clean` | `--purge-build-cache` | Remove local Buildx cache dir (`.buildx-cache` or `FHEVM_BUILDX_CACHE_DIR`). |
 | `clean` | `--purge-networks` | Remove Docker networks labeled with the active compose project only. |
-| `clean` | `--purge-local-cache` | Remove local Buildx cache dir (`.buildx-cache` or `FHEVM_BUILDX_CACHE_DIR`). |
+| `clean` | `--purge-local-cache` | Alias of `--purge-build-cache` (kept for compatibility). |
 | `pause` / `unpause` | `host` or `gateway` | Contract pause controls. |
 | `upgrade` | `<service>` | Restart selected service compose stack. |
 | `logs` | `<service>` | Stream container logs for one service. |
@@ -217,6 +218,9 @@ When resuming:
 - Services **before** the resume step are preserved (containers + volumes kept)
 - Services **from** the resume step onwards are torn down and redeployed
 
+Multicoprocessor safety note:
+- If you change multicoprocessor topology (`--coprocessors` and/or `--coprocessor-threshold`) while using `--resume` from `coprocessor` or later, the CLI intentionally forces resume from `minio` to reset key material coherently across all coprocessors.
+
 ### Deploying a single step
 
 To redeploy only a single service without touching others:
diff --git a/test-suite/fhevm/scripts/bun/cli.ts b/test-suite/fhevm/scripts/bun/cli.ts
@@ -55,7 +55,6 @@ type CleanOptions = {
   purgeImages: boolean;
   purgeBuildCache: boolean;
   purgeNetworks: boolean;
-  purgeLocalCache: boolean;
 };
 
 type TestOptions = {
@@ -82,14 +81,16 @@ function printLogo(): void {
 }
 
 function usage(): void {
+  const testTypes = Object.keys(TEST_TYPE_CONFIG).join("|");
+
   printLogo();
   console.log(`${COLORS.bold}Usage:${COLORS.reset} ${COLORS.yellow}fhevm-cli${COLORS.reset} ${COLORS.cyan}COMMAND [OPTIONS]${COLORS.reset}`);
   console.log("");
   console.log(`${COLORS.bold}${COLORS.lightBlue}Commands:${COLORS.reset}`);
   console.log(`  ${COLORS.yellow}deploy${COLORS.reset} ${COLORS.cyan}[--build] [--local] [--network testnet|mainnet] [--coprocessors N] [--coprocessor-threshold T] [--resume STEP] [--only STEP] [--telemetry-smoke] [--strict-otel]${COLORS.reset}    Deploy the full fhevm stack`);
   console.log(`  ${COLORS.yellow}pause${COLORS.reset} ${COLORS.cyan}[CONTRACTS]${COLORS.reset}     Pause specific contracts (host|gateway)`);
   console.log(`  ${COLORS.yellow}unpause${COLORS.reset} ${COLORS.cyan}[CONTRACTS]${COLORS.reset}     Unpause specific contracts (host|gateway)`);
-  console.log(`  ${COLORS.yellow}test${COLORS.reset} ${COLORS.cyan}[TYPE]${COLORS.reset}         Run tests (input-proof|user-decryption|public-decryption|delegated-user-decryption|random|random-subset|operators|erc20|debug)`);
+  console.log(`  ${COLORS.yellow}test${COLORS.reset} ${COLORS.cyan}[TYPE]${COLORS.reset}         Run tests (${testTypes})`);
   console.log(`  ${COLORS.yellow}upgrade${COLORS.reset} ${COLORS.cyan}[SERVICE]${COLORS.reset}   Upgrade specific service`);
   console.log(`  ${COLORS.yellow}clean${COLORS.reset} ${COLORS.cyan}[--purge] [--purge-images] [--purge-build-cache] [--purge-networks] [--purge-local-cache]${COLORS.reset}  Clean stack resources`);
   console.log(`  ${COLORS.yellow}logs${COLORS.reset} ${COLORS.cyan}[SERVICE]${COLORS.reset}      View logs for a specific service`);
@@ -115,11 +116,11 @@ function usage(): void {
   console.log(`  ${COLORS.cyan}--no-hardhat-compile${COLORS.reset}        Skip Hardhat compilation step`);
   console.log("");
   console.log(`${COLORS.bold}${COLORS.lightBlue}Clean Options:${COLORS.reset}`);
-  console.log(`  ${COLORS.cyan}--purge${COLORS.reset}               Equivalent to --purge-images --purge-build-cache --purge-networks --purge-local-cache`);
+  console.log(`  ${COLORS.cyan}--purge${COLORS.reset}               Equivalent to --purge-images --purge-build-cache --purge-networks`);
   console.log(`  ${COLORS.cyan}--purge-images${COLORS.reset}        Remove images for fhevm compose services only`);
   console.log(`  ${COLORS.cyan}--purge-build-cache${COLORS.reset}   Remove local fhevm Buildx cache directory`);
-  console.log(`  ${COLORS.cyan}--purge-networks${COLORS.reset}      Remove fhevm-prefixed Docker networks`);
-  console.log(`  ${COLORS.cyan}--purge-local-cache${COLORS.reset}   Remove local Buildx cache dir (.buildx-cache or FHEVM_BUILDX_CACHE_DIR)`);
+  console.log(`  ${COLORS.cyan}--purge-networks${COLORS.reset}      Remove networks labeled for the active fhevm compose project only`);
+  console.log(`  ${COLORS.cyan}--purge-local-cache${COLORS.reset}   Alias of --purge-build-cache (kept for compatibility)`);
   console.log("");
   console.log(`${COLORS.bold}${COLORS.lightBlue}Examples:${COLORS.reset}`);
   console.log(`  ${COLORS.purple}./fhevm-cli deploy${COLORS.reset}`);
@@ -139,7 +140,7 @@ function usage(): void {
   console.log(`  ${COLORS.purple}./fhevm-cli upgrade coprocessor${COLORS.reset}`);
   console.log(`  ${COLORS.purple}./fhevm-cli telemetry-smoke${COLORS.reset}`);
   console.log(`  ${COLORS.purple}./fhevm-cli clean --purge${COLORS.reset}`);
-  console.log(`  ${COLORS.purple}./fhevm-cli clean --purge-local-cache${COLORS.reset}`);
+  console.log(`  ${COLORS.purple}./fhevm-cli clean --purge-build-cache${COLORS.reset}`);
   console.log(`  ${COLORS.purple}FHEVM_DOCKER_PROJECT=fhevm-dev ./fhevm-cli deploy${COLORS.reset}`);
   console.log(`${COLORS.blue}============================================================${COLORS.reset}`);
 }
@@ -2107,7 +2108,7 @@ function resolveEffectiveResumeStep(options: DeployOptions): string | undefined
     const adjustedStep = "minio";
     if (options.resumeStep !== adjustedStep) {
       logWarn(
-        `Requested resume step '${options.resumeStep}' is too late for multicoprocessor topology changes. Forcing resume from '${adjustedStep}' to reset key material and chain state coherently.`,
+        `Requested resume step '${options.resumeStep}' is too late for multicoprocessor topology changes. Forcing resume from '${adjustedStep}' for a full key-material reset (this intentionally redeploys the full stack to keep key digests coherent).`,
       );
     }
     return adjustedStep;
@@ -2569,15 +2570,13 @@ function parseCleanArgs(args: string[]): CleanOptions {
     purgeImages: false,
     purgeBuildCache: false,
     purgeNetworks: false,
-    purgeLocalCache: false,
   };
 
   for (const arg of args) {
     if (arg === "--purge") {
       options.purgeImages = true;
       options.purgeBuildCache = true;
       options.purgeNetworks = true;
-      options.purgeLocalCache = true;
       continue;
     }
     if (arg === "--purge-images") {
@@ -2593,7 +2592,7 @@ function parseCleanArgs(args: string[]): CleanOptions {
       continue;
     }
     if (arg === "--purge-local-cache") {
-      options.purgeLocalCache = true;
+      options.purgeBuildCache = true;
       continue;
     }
     usageError(`Unknown option for clean: ${arg}`);
@@ -2623,11 +2622,7 @@ function clean(args: string[]): void {
   }
 
   if (options.purgeBuildCache) {
-    logInfo("Removing local fhevm Buildx cache only.");
-    purgeLocalBuildxCache();
-  }
-
-  if (options.purgeLocalCache && !options.purgeBuildCache) {
+    logInfo("Removing local fhevm Buildx cache directory.");
     purgeLocalBuildxCache();
   }
 
diff --git a/test-suite/fhevm/scripts/tests/deploy-fhevm-stack.behavior.sh b/test-suite/fhevm/scripts/tests/deploy-fhevm-stack.behavior.sh
@@ -89,6 +89,36 @@ case "${subcommand}" in
         exit 1
       fi
     fi
+    exit 0
+    ;;
+  exec)
+    while [[ $# -gt 0 ]]; do
+      case "$1" in
+        -e)
+          shift 2
+          ;;
+        *)
+          break
+          ;;
+      esac
+    done
+
+    container="${1:-}"
+    shift || true
+
+    if [[ "${1:-}" == "curl" ]]; then
+      if [[ "$*" == *"http://fhevm-relayer:3000/v2/keyurl"* ]]; then
+        cat <<'JSON'
+{"response":{"fheKeyInfo":[{"fhePublicKey":{"urls":["http://mock-fhevm/key"]}}],"crs":{"2048":{"urls":["http://mock-fhevm/crs"]}}}}
+JSON
+        exit 0
+      fi
+
+      if [[ "$*" == *"http://mock-fhevm/key"* || "$*" == *"http://mock-fhevm/crs"* ]]; then
+        exit 0
+      fi
+    fi
+
     exit 0
     ;;
   ps)
@@ -130,6 +160,42 @@ case "${subcommand}" in
     fi
     exit 0
     ;;
+  volume)
+    action="${1:-}"
+    shift || true
+    if [[ "${action}" == "ls" ]]; then
+      filter_name=""
+      while [[ $# -gt 0 ]]; do
+        case "$1" in
+          --filter)
+            if [[ "${2:-}" == name=* ]]; then
+              filter_name="${2#name=}"
+            fi
+            shift 2
+            ;;
+          --format)
+            shift 2
+            ;;
+          *)
+            shift
+            ;;
+        esac
+      done
+
+      names_csv="${DOCKER_VOLUME_NAMES:-fhevm_minio_secrets}"
+      IFS=',' read -r -a names <<< "${names_csv}"
+      exact="${filter_name#^}"
+      exact="${exact%\$}"
+
+      for name in "${names[@]}"; do
+        if [[ -z "${exact}" || "${name}" == "${exact}" ]]; then
+          echo "${name}"
+        fi
+      done
+      exit 0
+    fi
+    exit 0
+    ;;
   inspect)
     if [[ "${1:-}" == "-f" ]]; then
       format="${2:-}"
@@ -298,6 +364,7 @@ run_deploy() {
     export TEST_COMMAND_LOG="${COMMAND_LOG}"
     export DOCKER_COMPLETE_SERVICES="${DOCKER_COMPLETE_SERVICES:-${DEFAULT_COMPLETE_SERVICES}}"
     export DOCKER_MINIO_IP="${DOCKER_MINIO_IP:-172.20.0.2}"
+    export DOCKER_RUNNING_NAMES="${DOCKER_RUNNING_NAMES:-fhevm-relayer,fhevm-test-suite-e2e-debug}"
 
     ./deploy-fhevm-stack.sh "$@"
   ) > "${output_file}" 2>&1
@@ -313,6 +380,7 @@ run_cli() {
     export TEST_COMMAND_LOG="${COMMAND_LOG}"
     export DOCKER_COMPLETE_SERVICES="${DOCKER_COMPLETE_SERVICES:-${DEFAULT_COMPLETE_SERVICES}}"
     export DOCKER_MINIO_IP="${DOCKER_MINIO_IP:-172.20.0.2}"
+    export DOCKER_RUNNING_NAMES="${DOCKER_RUNNING_NAMES:-fhevm-relayer,fhevm-test-suite-e2e-debug}"
 
     bun scripts/bun/cli.ts "$@"
   ) > "${output_file}" 2>&1
@@ -414,8 +482,8 @@ test_default_flow_and_env_patch() {
   assert_contains "${COMMAND_LOG}" "coprocessor-docker-compose.yml up -d"
   assert_contains "${COMMAND_LOG}" "kms-connector-docker-compose.yml up -d"
   assert_contains "${COMMAND_LOG}" "gateway-mocked-payment-docker-compose.yml up -d"
-  assert_contains "${COMMAND_LOG}" "gateway-sc-docker-compose.yml up -d"
-  assert_contains "${COMMAND_LOG}" "host-sc-docker-compose.yml up -d"
+  assert_contains "${COMMAND_LOG}" "gateway-sc-docker-compose.yml up --force-recreate -d gateway-sc-deploy"
+  assert_contains "${COMMAND_LOG}" "host-sc-docker-compose.yml up --force-recreate -d host-sc-deploy"
   assert_contains "${COMMAND_LOG}" "relayer-docker-compose.yml up -d"
   assert_contains "${COMMAND_LOG}" "test-suite-docker-compose.yml up -d"
   assert_contains "${COMMAND_LOG}" "setup-kms-signer"
@@ -453,26 +521,63 @@ test_resume_preserves_prior_steps_and_restarts_tail() {
 
   assert_contains "${COMMAND_LOG}" "kms-connector-docker-compose.yml up -d"
   assert_contains "${COMMAND_LOG}" "gateway-mocked-payment-docker-compose.yml up -d"
-  assert_contains "${COMMAND_LOG}" "gateway-sc-docker-compose.yml up -d"
-  assert_contains "${COMMAND_LOG}" "host-sc-docker-compose.yml up -d"
+  assert_contains "${COMMAND_LOG}" "gateway-sc-docker-compose.yml up --force-recreate -d gateway-sc-deploy"
+  assert_contains "${COMMAND_LOG}" "host-sc-docker-compose.yml up --force-recreate -d host-sc-deploy"
   assert_contains "${COMMAND_LOG}" "relayer-docker-compose.yml up -d"
   assert_contains "${COMMAND_LOG}" "test-suite-docker-compose.yml up -d"
 
-  assert_contains "${COMMAND_LOG}" "test-suite-docker-compose.yml down -v --remove-orphans"
-  assert_contains "${COMMAND_LOG}" "relayer-docker-compose.yml down -v --remove-orphans"
-  assert_contains "${COMMAND_LOG}" "host-sc-docker-compose.yml down -v --remove-orphans"
-  assert_contains "${COMMAND_LOG}" "gateway-sc-docker-compose.yml down -v --remove-orphans"
-  assert_contains "${COMMAND_LOG}" "gateway-mocked-payment-docker-compose.yml down -v --remove-orphans"
-  assert_contains "${COMMAND_LOG}" "kms-connector-docker-compose.yml down -v --remove-orphans"
-  assert_not_contains "${COMMAND_LOG}" "coprocessor-docker-compose.yml down -v --remove-orphans"
+  assert_contains "${COMMAND_LOG}" "test-suite-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "relayer-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "host-sc-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "gateway-sc-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "gateway-mocked-payment-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "kms-connector-docker-compose.yml down -v"
+  assert_not_contains "${COMMAND_LOG}" "coprocessor-docker-compose.yml down -v"
 
-  assert_order "${COMMAND_LOG}" "test-suite-docker-compose.yml down -v --remove-orphans" "kms-connector-docker-compose.yml down -v --remove-orphans"
+  assert_order "${COMMAND_LOG}" "test-suite-docker-compose.yml down -v" "kms-connector-docker-compose.yml down -v"
   assert_order "${COMMAND_LOG}" "docker inspect -f {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} fhevm-minio" "kms-connector-docker-compose.yml up -d"
 
   unset DOCKER_RUNNING_NAMES
   cleanup_fixture
 }
 
+test_multicoprocessor_resume_forces_minio_reset() {
+  setup_fixture
+
+  cat > "${FIXTURE_ROOT}/env/staging/.env.gateway-sc" <<'ENV'
+MNEMONIC=test test test test test test test test test test test junk
+COPROCESSOR_THRESHOLD=1
+NUM_COPROCESSORS=1
+COPROCESSOR_TX_SENDER_ADDRESS_0=0x1111111111111111111111111111111111111111
+COPROCESSOR_SIGNER_ADDRESS_0=0x1111111111111111111111111111111111111111
+COPROCESSOR_S3_BUCKET_URL_0=http://minio:9000/ct128
+ENV
+
+  cat > "${FIXTURE_ROOT}/env/staging/.env.host-sc" <<'ENV'
+COPROCESSOR_THRESHOLD=1
+NUM_COPROCESSORS=1
+COPROCESSOR_SIGNER_ADDRESS_0=0x1111111111111111111111111111111111111111
+ENV
+
+  cat > "${FIXTURE_ROOT}/env/staging/.env.coprocessor" <<'ENV'
+AWS_ENDPOINT_URL=http://minio:9000
+DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/coprocessor"
+TX_SENDER_PRIVATE_KEY=0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ENV
+
+  local output_file="${TEST_TMP_DIR}/multicoprocessor-resume-forced-minio.out"
+  if ! run_deploy "${output_file}" --resume coprocessor --coprocessors 2 --coprocessor-threshold 2; then
+    echo "Deploy with resume+multicoprocessor should succeed" >&2
+    cat "${output_file}" >&2
+    return 1
+  fi
+
+  assert_contains "${output_file}" "Forcing resume from 'minio'."
+  assert_contains "${COMMAND_LOG}" "minio-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "minio-docker-compose.yml up -d"
+  cleanup_fixture
+}
+
 test_only_runs_single_step() {
   setup_fixture
 
@@ -483,7 +588,7 @@ test_only_runs_single_step() {
     return 1
   fi
 
-  assert_contains "${COMMAND_LOG}" "coprocessor-docker-compose.yml down -v --remove-orphans"
+  assert_contains "${COMMAND_LOG}" "coprocessor-docker-compose.yml down -v"
   assert_contains "${COMMAND_LOG}" "coprocessor-docker-compose.yml up -d"
   assert_not_contains "${COMMAND_LOG}" "host-node-docker-compose.yml up -d"
   assert_not_contains "${COMMAND_LOG}" "gateway-node-docker-compose.yml up -d"
@@ -727,13 +832,13 @@ test_clean_purge_invokes_prunes() {
     return 1
   fi
 
-  assert_contains "${COMMAND_LOG}" "docker compose -p fhevm down -v --remove-orphans"
-  assert_contains "${COMMAND_LOG}" "docker network ls --format {{.Name}}"
+  assert_contains "${COMMAND_LOG}" "minio-docker-compose.yml down -v"
+  assert_contains "${COMMAND_LOG}" "docker network ls --filter label=com.docker.compose.project=fhevm --format {{.Name}}"
   assert_contains "${COMMAND_LOG}" "down -v --remove-orphans --rmi all"
   assert_not_contains "${COMMAND_LOG}" "docker image prune -af"
   assert_not_contains "${COMMAND_LOG}" "docker builder prune -af"
   assert_contains "${output_file}" "Removing images referenced by fhevm compose services only."
-  assert_contains "${output_file}" "Removing local fhevm Buildx cache only."
+  assert_contains "${output_file}" "Removing local fhevm Buildx cache directory."
   assert_contains "${output_file}" "Removed local Buildx cache directory"
   if [[ -d "${FIXTURE_ROOT}/.buildx-cache" ]]; then
     echo "Assertion failed: .buildx-cache should be removed by clean --purge" >&2
@@ -764,6 +869,9 @@ test_command_and_flag_matrix() {
   export FHEVM_GRAFANA_DASHBOARD_HTML_FILE="${dashboard_fixture}"
 
   run_cli_ok "help-command" help
+  assert_contains "${TEST_TMP_DIR}/help-command.out" "input-proof-compute-decrypt"
+  assert_contains "${TEST_TMP_DIR}/help-command.out" "paused-host-contracts"
+  assert_contains "${TEST_TMP_DIR}/help-command.out" "paused-gateway-contracts"
   run_cli_ok "help-short" -h
   run_cli_ok "help-long" --help
 
@@ -829,6 +937,7 @@ main() {
   test_quoted_otel_endpoint_is_accepted
   test_multicoprocessor_flags_are_validated
   test_multicoprocessor_env_and_extra_instances
+  test_multicoprocessor_resume_forces_minio_reset
   test_clean_purge_invokes_prunes
   test_telemetry_smoke_requires_jaeger
   test_command_and_flag_matrix
