feat(test-suite): local multi coprocessor deploy + e2e debug mode

test-suite/README.md

@@ -44,13 +44,16 @@ cd test-suite/fhevm
 # Deploy with local BuildKit cache (disables provenance attestations)
 ./fhevm-cli deploy --local
 
+# Deploy with threshold 2 out of 2 coprocessors (local multicoprocessor mode)
+./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2
+
 # Resume a failed deploy from a specific step (keeps existing containers/volumes)
 ./fhevm-cli deploy --resume kms-connector
 
 # Deploy only a single step (useful for redeploying one service)
 ./fhevm-cli deploy --only coprocessor
 
-# Run specific tests
+# Run specific tests (works for both 1/1 and n/t topologies)
 ./fhevm-cli test input-proof
 # Skip Hardhat compile when artifacts are already up to date
 ./fhevm-cli test input-proof --no-hardhat-compile

test-suite/fhevm/docker-compose/database-docker-compose.yml

@@ -3,6 +3,10 @@ services:
     container_name: coprocessor-and-kms-db
     image: postgres:15.7
     restart: always
+    command:
+      - postgres
+      - -c
+      - max_connections=500
     env_file:
       - ../env/staging/.env.database.local
     ports:
@@ -16,4 +20,4 @@ services:
       - db:/var/lib/postgresql/data
 
 volumes:
-  db:
+  db:

test-suite/fhevm/env/staging/.env.coprocessor

@@ -6,12 +6,6 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/coprocessor"
 
-# =============================================================================
-# API & AUTHENTICATION - SENSITIVE 
-# =============================================================================
-# IMPORTANT: For testing only - move to secure storage in production
-TENANT_API_KEY=a1503fb6-d79b-4e9e-826d-44cf262f3e05
-
 # =============================================================================
 # AWS/MINIO CONFIGURATION
 # =============================================================================

test-suite/fhevm/fhevm-cli

@@ -14,6 +14,25 @@ RESET='\033[0m'
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 PROJECT="fhevm"
 
+# Portable grep helpers: prefer rg when available, fallback to grep.
+function has_pattern() {
+  local pattern="$1"
+  if command -v rg >/dev/null 2>&1; then
+    rg -q "$pattern"
+  else
+    grep -qE "$pattern"
+  fi
+}
+
+function filter_pattern() {
+  local pattern="$1"
+  if command -v rg >/dev/null 2>&1; then
+    rg "$pattern"
+  else
+    grep -E "$pattern"
+  fi
+}
+
 # Default versions for the fhevm stack.
 
 # KMS connector services.
@@ -23,13 +42,13 @@ export CONNECTOR_KMS_WORKER_VERSION=${CONNECTOR_KMS_WORKER_VERSION:-"v0.11.0-1"}
 export CONNECTOR_TX_SENDER_VERSION=${CONNECTOR_TX_SENDER_VERSION:-"v0.11.0-1"}
 
 # Coprocessor services.
-export COPROCESSOR_DB_MIGRATION_VERSION=${COPROCESSOR_DB_MIGRATION_VERSION:-"v0.11.0-1"}
-export COPROCESSOR_GW_LISTENER_VERSION=${COPROCESSOR_GW_LISTENER_VERSION:-"v0.11.0-1"}
-export COPROCESSOR_HOST_LISTENER_VERSION=${COPROCESSOR_HOST_LISTENER_VERSION:-"v0.11.0-1"}
-export COPROCESSOR_TX_SENDER_VERSION=${COPROCESSOR_TX_SENDER_VERSION:-"v0.11.0-1"}
-export COPROCESSOR_TFHE_WORKER_VERSION=${COPROCESSOR_TFHE_WORKER_VERSION:-"v0.11.0-1"}
-export COPROCESSOR_SNS_WORKER_VERSION=${COPROCESSOR_SNS_WORKER_VERSION:-"v0.11.0-1"}
-export COPROCESSOR_ZKPROOF_WORKER_VERSION=${COPROCESSOR_ZKPROOF_WORKER_VERSION:-"v0.11.0-1"}
+export COPROCESSOR_DB_MIGRATION_VERSION=${COPROCESSOR_DB_MIGRATION_VERSION:-"a4a9aa4"}
+export COPROCESSOR_GW_LISTENER_VERSION=${COPROCESSOR_GW_LISTENER_VERSION:-"a4a9aa4"}
+export COPROCESSOR_HOST_LISTENER_VERSION=${COPROCESSOR_HOST_LISTENER_VERSION:-"a4a9aa4"}
+export COPROCESSOR_TX_SENDER_VERSION=${COPROCESSOR_TX_SENDER_VERSION:-"a4a9aa4"}
+export COPROCESSOR_TFHE_WORKER_VERSION=${COPROCESSOR_TFHE_WORKER_VERSION:-"a4a9aa4"}
+export COPROCESSOR_SNS_WORKER_VERSION=${COPROCESSOR_SNS_WORKER_VERSION:-"a4a9aa4"}
+export COPROCESSOR_ZKPROOF_WORKER_VERSION=${COPROCESSOR_ZKPROOF_WORKER_VERSION:-"a4a9aa4"}
 
 # Gateway and Host contracts.
 export GATEWAY_VERSION=${GATEWAY_VERSION:-"v0.11.0-1"}
@@ -58,10 +77,11 @@ function usage {
   echo -e "${BOLD}Usage:${RESET} ${YELLOW}fhevm-cli${RESET} ${CYAN}COMMAND [OPTIONS]${RESET}"
   echo
   echo -e "${BOLD}${LIGHT_BLUE}Commands:${RESET}"
-  echo -e "  ${YELLOW}deploy${RESET} ${CYAN}[--build] [--local]${RESET}    WIP: Deploy the full fhevm stack (optionally rebuild images)"
+  echo -e "  ${YELLOW}deploy${RESET} ${CYAN}[--build] [--local] [--coprocessors N] [--coprocessor-threshold T]${RESET}    Deploy fhevm stack"
   echo -e "  ${YELLOW}pause${RESET} ${CYAN}[CONTRACTS]${RESET}     Pause specific contracts (host|gateway)"
   echo -e "  ${YELLOW}unpause${RESET} ${CYAN}[CONTRACTS]${RESET}     Unpause specific contracts (host|gateway)"
   echo -e "  ${YELLOW}test${RESET} ${CYAN}[TYPE]${RESET}         Run tests (input-proof|user-decryption|public-decryption|delegated-user-decryption|random|random-subset|operators|erc20|debug)"
+  echo -e "  ${YELLOW}smoke${RESET} ${CYAN}[PROFILE]${RESET}      Run multicoproc smoke profile (multi-2-2|multi-3-5)"
   echo -e "  ${YELLOW}upgrade${RESET} ${CYAN}[SERVICE]${RESET}   Upgrade specific service (host|gateway|connector|coprocessor|relayer|test-suite)"
   echo -e "  ${YELLOW}clean${RESET}               Remove all containers and volumes"
   echo -e "  ${YELLOW}logs${RESET} ${CYAN}[SERVICE]${RESET}      View logs for a specific service"
@@ -78,6 +98,9 @@ function usage {
   echo -e "  ${PURPLE}./fhevm-cli deploy${RESET}"
   echo -e "  ${PURPLE}./fhevm-cli deploy --build${RESET}"
   echo -e "  ${PURPLE}./fhevm-cli deploy --local${RESET}"
+  echo -e "  ${PURPLE}./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2${RESET}"
+  echo -e "  ${PURPLE}./fhevm-cli smoke multi-2-2${RESET}"
+  echo -e "  ${PURPLE}./fhevm-cli smoke multi-3-5${RESET}"
   echo -e "  ${PURPLE}./fhevm-cli test input-proof${RESET}"
   echo -e "  ${PURPLE}./fhevm-cli test input-proof --no-hardhat-compile${RESET}"
   echo -e "  ${PURPLE}./fhevm-cli test user-decryption ${RESET}"
@@ -88,6 +111,77 @@ function usage {
   echo -e "${BLUE}============================================================${RESET}"
 }
 
+# Best-effort log dump across all coprocessor instances for quick local triage.
+function collect_coprocessor_topology_logs() {
+  echo -e "${LIGHT_BLUE}${BOLD}[LOGS] Collecting coprocessor topology logs (last 10m)...${RESET}"
+  local containers
+  containers=$(docker ps --format '{{.Names}}' | filter_pattern '^coprocessor[0-9-]*-(host-listener|gw-listener|tfhe-worker|sns-worker|transaction-sender)$' || true)
+  if [ -z "$containers" ]; then
+    echo -e "${YELLOW}[WARN]${RESET} No coprocessor containers found"
+    return
+  fi
+
+  while IFS= read -r container; do
+    [ -z "$container" ] && continue
+    echo -e "\n${CYAN}===== ${container} =====${RESET}"
+    docker logs --since=10m "$container" | tail -n 160 || true
+  done <<< "$containers"
+}
+
+function wait_for_coprocessor_key_bootstrap() {
+  local timeout_seconds=300
+  local poll_interval=5
+  local elapsed=0
+  echo -e "${LIGHT_BLUE}${BOLD}[WAIT] Waiting for coprocessor key bootstrap...${RESET}"
+  while [ "$elapsed" -lt "$timeout_seconds" ]; do
+    local ready_count=0
+    local total_count=0
+    local pending=""
+    local containers
+    containers=$(docker ps --format '{{.Names}}' | filter_pattern '^coprocessor([0-9]+)?-sns-worker$' || true)
+    if [ -z "$containers" ]; then
+      echo -e "${YELLOW}[WARN]${RESET} No sns-worker containers found yet"
+      sleep "$poll_interval"
+      elapsed=$((elapsed + poll_interval))
+      continue
+    fi
+
+    while IFS= read -r container; do
+      [ -z "$container" ] && continue
+      total_count=$((total_count + 1))
+      if docker logs --since=20m "$container" 2>&1 | has_pattern "Fetched keyset"; then
+        ready_count=$((ready_count + 1))
+      else
+        pending="${pending} ${container}"
+      fi
+    done <<< "$containers"
+
+    local configured_threshold=""
+    if [ -f "test-suite/fhevm/env/staging/.env.host-sc.local" ]; then
+      configured_threshold=$(filter_pattern '^COPROCESSOR_THRESHOLD=' test-suite/fhevm/env/staging/.env.host-sc.local | tail -n1 | cut -d'=' -f2 || true)
+    fi
+    if ! [[ "$configured_threshold" =~ ^[0-9]+$ ]] || [ "$configured_threshold" -lt 1 ]; then
+      configured_threshold="$total_count"
+    fi
+
+    if [ "$configured_threshold" -gt "$total_count" ]; then
+      configured_threshold="$total_count"
+    fi
+
+    if [ "$ready_count" -ge "$configured_threshold" ]; then
+      echo -e "${GREEN}[READY]${RESET} Key bootstrap confirmed for ${ready_count}/${total_count} sns-worker containers (threshold=${configured_threshold})"
+      return 0
+    fi
+
+    echo -e "${YELLOW}[WAIT]${RESET} Key bootstrap ready=${ready_count}/${total_count} threshold=${configured_threshold} pending:${pending}"
+    sleep "$poll_interval"
+    elapsed=$((elapsed + poll_interval))
+  done
+
+  echo -e "${RED}[ERROR]${RESET} Key bootstrap did not reach threshold within ${timeout_seconds}s"
+  return 1
+}
+
 COMMAND=$1
 shift
 
@@ -112,6 +206,26 @@ case $COMMAND in
           DEPLOY_ARGS+=("--local")
           shift
           ;;
+        --coprocessors)
+          if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+            DEPLOY_ARGS+=("--coprocessors" "$2")
+            shift 2
+          else
+            echo -e "${RED}[ERROR]${RESET} ${BOLD}Coprocessor count argument missing${RESET}"
+            usage
+            exit 1
+          fi
+          ;;
+        --coprocessor-threshold)
+          if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+            DEPLOY_ARGS+=("--coprocessor-threshold" "$2")
+            shift 2
+          else
+            echo -e "${RED}[ERROR]${RESET} ${BOLD}Coprocessor threshold argument missing${RESET}"
+            usage
+            exit 1
+          fi
+          ;;
         *)
           echo -e "${RED}[ERROR]${RESET} ${BOLD}Unknown argument for deploy: $1${RESET}"
           usage
@@ -167,6 +281,7 @@ case $COMMAND in
     GREP=""
     NO_RELAYER=""
     NO_COMPILE=""
+    WAIT_FOR_KEY_BOOTSTRAP=""
 
     while (( "$#" )); do
       case "$1" in
@@ -229,6 +344,7 @@ case $COMMAND in
         input-proof)
           log_message="${LIGHT_BLUE}${BOLD}[TEST] INPUT PROOF (uint64)${RESET}"
           docker_args+=("-g" "test user input uint64")
+          WAIT_FOR_KEY_BOOTSTRAP="1"
           ;;
         input-proof-compute-decrypt)
           log_message="${LIGHT_BLUE}${BOLD}[TEST] INPUT PROOF (uint64)${RESET}"
@@ -291,8 +407,49 @@ case $COMMAND in
     fi
     if [ "$TEST_TYPE" != "debug" ]; then
       echo -e "${log_message}"
-      docker exec fhevm-test-suite-e2e-debug "${docker_args[@]}"
+      if [ -n "$WAIT_FOR_KEY_BOOTSTRAP" ]; then
+        wait_for_coprocessor_key_bootstrap
+      fi
+      test_status=0
+      if docker exec fhevm-test-suite-e2e-debug "${docker_args[@]}"; then
+        test_status=0
+      else
+        test_status=$?
+      fi
+      if [ -n "$WAIT_FOR_KEY_BOOTSTRAP" ]; then
+        collect_coprocessor_topology_logs
+      fi
+      if [ "$test_status" -ne 0 ]; then
+        exit "$test_status"
+      fi
+    fi
+    ;;
+
+  smoke)
+    print_logo
+    PROFILE=$1
+    if [[ ! $PROFILE =~ ^(multi-2-2|multi-3-5)$ ]]; then
+      echo -e "${RED}[ERROR]${RESET} ${BOLD}Unknown smoke profile: ${PROFILE}${RESET}"
+      usage
+      exit 1
     fi
+
+    coprocessors=""
+    threshold=""
+    case "$PROFILE" in
+      multi-2-2)
+        coprocessors=2
+        threshold=2
+        ;;
+      multi-3-5)
+        coprocessors=5
+        threshold=3
+        ;;
+    esac
+
+    echo -e "${LIGHT_BLUE}${BOLD}[SMOKE] Running profile ${PROFILE} (n=${coprocessors}, t=${threshold})...${RESET}"
+    "${SCRIPT_DIR}/fhevm-cli" deploy --local --coprocessors "${coprocessors}" --coprocessor-threshold "${threshold}"
+    "${SCRIPT_DIR}/fhevm-cli" test input-proof
     ;;
 
   help|-h|--help)