Skip to content

Commit 70edec7

Browse files
mergify[bot]mergify[bot]
authored
Merge of #1959 
2 parents 9d781e3 + e4949a0 commit 70edec7

File tree

6 files changed

+414
-16
lines changed

6 files changed

+414
-16
lines changed

‎test-suite/README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,16 @@ cd test-suite/fhevm
4444
# Deploy with local BuildKit cache (disables provenance attestations)
4545
./fhevm-cli deploy --local
4646

47+
# Deploy with threshold 2 out of 2 coprocessors (local multicoprocessor mode)
48+
./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2
49+
4750
# Resume a failed deploy from a specific step (keeps existing containers/volumes)
4851
./fhevm-cli deploy --resume kms-connector
4952

5053
# Deploy only a single step (useful for redeploying one service)
5154
./fhevm-cli deploy --only coprocessor
5255

53-
# Run specific tests
56+
# Run specific tests (works for both 1/1 and n/t topologies)
5457
./fhevm-cli test input-proof
5558
# Skip Hardhat compile when artifacts are already up to date
5659
./fhevm-cli test input-proof --no-hardhat-compile

‎test-suite/fhevm/docker-compose/database-docker-compose.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ services:
33
container_name: coprocessor-and-kms-db
44
image: postgres:15.7
55
restart: always
6+
command:
7+
- postgres
8+
- -c
9+
- max_connections=500
610
env_file:
711
- ../env/staging/.env.database.local
812
ports:
@@ -16,4 +20,4 @@ services:
1620
- db:/var/lib/postgresql/data
1721

1822
volumes:
19-
db:
23+
db:

‎test-suite/fhevm/env/staging/.env.coprocessor‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,6 @@ POSTGRES_USER=postgres
66
POSTGRES_PASSWORD=postgres
77
DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/coprocessor"
88

9-
# =============================================================================
10-
# API & AUTHENTICATION - SENSITIVE
11-
# =============================================================================
12-
# IMPORTANT: For testing only - move to secure storage in production
13-
TENANT_API_KEY=a1503fb6-d79b-4e9e-826d-44cf262f3e05
14-
159
# =============================================================================
1610
# AWS/MINIO CONFIGURATION
1711
# =============================================================================

‎test-suite/fhevm/fhevm-cli‎

Lines changed: 159 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,25 @@ RESET='\033[0m'
1414
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
1515
PROJECT="fhevm"
1616

17+
# Portable grep helpers: prefer rg when available, fallback to grep.
18+
function has_pattern() {
19+
local pattern="$1"
20+
if command -v rg >/dev/null 2>&1; then
21+
rg -q "$pattern"
22+
else
23+
grep -qE "$pattern"
24+
fi
25+
}
26+
27+
function filter_pattern() {
28+
local pattern="$1"
29+
if command -v rg >/dev/null 2>&1; then
30+
rg "$pattern"
31+
else
32+
grep -E "$pattern"
33+
fi
34+
}
35+
1736
# Default versions for the fhevm stack.
1837

1938
# KMS connector services.
@@ -58,10 +77,11 @@ function usage {
5877
echo -e "${BOLD}Usage:${RESET} ${YELLOW}fhevm-cli${RESET} ${CYAN}COMMAND [OPTIONS]${RESET}"
5978
echo
6079
echo -e "${BOLD}${LIGHT_BLUE}Commands:${RESET}"
61-
echo -e " ${YELLOW}deploy${RESET} ${CYAN}[--build] [--local]${RESET} WIP: Deploy the full fhevm stack (optionally rebuild images)"
80+
echo -e " ${YELLOW}deploy${RESET} ${CYAN}[--build] [--local] [--coprocessors N] [--coprocessor-threshold T]${RESET} Deploy fhevm stack"
6281
echo -e " ${YELLOW}pause${RESET} ${CYAN}[CONTRACTS]${RESET} Pause specific contracts (host|gateway)"
6382
echo -e " ${YELLOW}unpause${RESET} ${CYAN}[CONTRACTS]${RESET} Unpause specific contracts (host|gateway)"
6483
echo -e " ${YELLOW}test${RESET} ${CYAN}[TYPE]${RESET} Run tests (input-proof|user-decryption|public-decryption|delegated-user-decryption|random|random-subset|operators|erc20|debug)"
84+
echo -e " ${YELLOW}smoke${RESET} ${CYAN}[PROFILE]${RESET} Run multicoproc smoke profile (multi-2-2|multi-3-5)"
6585
echo -e " ${YELLOW}upgrade${RESET} ${CYAN}[SERVICE]${RESET} Upgrade specific service (host|gateway|connector|coprocessor|relayer|test-suite)"
6686
echo -e " ${YELLOW}clean${RESET} Remove all containers and volumes"
6787
echo -e " ${YELLOW}logs${RESET} ${CYAN}[SERVICE]${RESET} View logs for a specific service"
@@ -78,6 +98,9 @@ function usage {
7898
echo -e " ${PURPLE}./fhevm-cli deploy${RESET}"
7999
echo -e " ${PURPLE}./fhevm-cli deploy --build${RESET}"
80100
echo -e " ${PURPLE}./fhevm-cli deploy --local${RESET}"
101+
echo -e " ${PURPLE}./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2${RESET}"
102+
echo -e " ${PURPLE}./fhevm-cli smoke multi-2-2${RESET}"
103+
echo -e " ${PURPLE}./fhevm-cli smoke multi-3-5${RESET}"
81104
echo -e " ${PURPLE}./fhevm-cli test input-proof${RESET}"
82105
echo -e " ${PURPLE}./fhevm-cli test input-proof --no-hardhat-compile${RESET}"
83106
echo -e " ${PURPLE}./fhevm-cli test user-decryption ${RESET}"
@@ -88,6 +111,77 @@ function usage {
88111
echo -e "${BLUE}============================================================${RESET}"
89112
}
90113

114+
# Best-effort log dump across all coprocessor instances for quick local triage.
115+
function collect_coprocessor_topology_logs() {
116+
echo -e "${LIGHT_BLUE}${BOLD}[LOGS] Collecting coprocessor topology logs (last 10m)...${RESET}"
117+
local containers
118+
containers=$(docker ps --format '{{.Names}}' | filter_pattern '^coprocessor[0-9-]*-(host-listener|gw-listener|tfhe-worker|sns-worker|transaction-sender)$' || true)
119+
if [ -z "$containers" ]; then
120+
echo -e "${YELLOW}[WARN]${RESET} No coprocessor containers found"
121+
return
122+
fi
123+
124+
while IFS= read -r container; do
125+
[ -z "$container" ] && continue
126+
echo -e "\n${CYAN}===== ${container} =====${RESET}"
127+
docker logs --since=10m "$container" | tail -n 160 || true
128+
done <<< "$containers"
129+
}
130+
131+
function wait_for_coprocessor_key_bootstrap() {
132+
local timeout_seconds=300
133+
local poll_interval=5
134+
local elapsed=0
135+
echo -e "${LIGHT_BLUE}${BOLD}[WAIT] Waiting for coprocessor key bootstrap...${RESET}"
136+
while [ "$elapsed" -lt "$timeout_seconds" ]; do
137+
local ready_count=0
138+
local total_count=0
139+
local pending=""
140+
local containers
141+
containers=$(docker ps --format '{{.Names}}' | filter_pattern '^coprocessor([0-9]+)?-sns-worker$' || true)
142+
if [ -z "$containers" ]; then
143+
echo -e "${YELLOW}[WARN]${RESET} No sns-worker containers found yet"
144+
sleep "$poll_interval"
145+
elapsed=$((elapsed + poll_interval))
146+
continue
147+
fi
148+
149+
while IFS= read -r container; do
150+
[ -z "$container" ] && continue
151+
total_count=$((total_count + 1))
152+
if docker logs --since=20m "$container" 2>&1 | has_pattern "Fetched keyset"; then
153+
ready_count=$((ready_count + 1))
154+
else
155+
pending="${pending} ${container}"
156+
fi
157+
done <<< "$containers"
158+
159+
local configured_threshold=""
160+
if [ -f "test-suite/fhevm/env/staging/.env.host-sc.local" ]; then
161+
configured_threshold=$(filter_pattern '^COPROCESSOR_THRESHOLD=' test-suite/fhevm/env/staging/.env.host-sc.local | tail -n1 | cut -d'=' -f2 || true)
162+
fi
163+
if ! [[ "$configured_threshold" =~ ^[0-9]+$ ]] || [ "$configured_threshold" -lt 1 ]; then
164+
configured_threshold="$total_count"
165+
fi
166+
167+
if [ "$configured_threshold" -gt "$total_count" ]; then
168+
configured_threshold="$total_count"
169+
fi
170+
171+
if [ "$ready_count" -ge "$configured_threshold" ]; then
172+
echo -e "${GREEN}[READY]${RESET} Key bootstrap confirmed for ${ready_count}/${total_count} sns-worker containers (threshold=${configured_threshold})"
173+
return 0
174+
fi
175+
176+
echo -e "${YELLOW}[WAIT]${RESET} Key bootstrap ready=${ready_count}/${total_count} threshold=${configured_threshold} pending:${pending}"
177+
sleep "$poll_interval"
178+
elapsed=$((elapsed + poll_interval))
179+
done
180+
181+
echo -e "${RED}[ERROR]${RESET} Key bootstrap did not reach threshold within ${timeout_seconds}s"
182+
return 1
183+
}
184+
91185
COMMAND=$1
92186
shift
93187

@@ -112,6 +206,26 @@ case $COMMAND in
112206
DEPLOY_ARGS+=("--local")
113207
shift
114208
;;
209+
--coprocessors)
210+
if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
211+
DEPLOY_ARGS+=("--coprocessors" "$2")
212+
shift 2
213+
else
214+
echo -e "${RED}[ERROR]${RESET} ${BOLD}Coprocessor count argument missing${RESET}"
215+
usage
216+
exit 1
217+
fi
218+
;;
219+
--coprocessor-threshold)
220+
if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
221+
DEPLOY_ARGS+=("--coprocessor-threshold" "$2")
222+
shift 2
223+
else
224+
echo -e "${RED}[ERROR]${RESET} ${BOLD}Coprocessor threshold argument missing${RESET}"
225+
usage
226+
exit 1
227+
fi
228+
;;
115229
*)
116230
echo -e "${RED}[ERROR]${RESET} ${BOLD}Unknown argument for deploy: $1${RESET}"
117231
usage
@@ -167,6 +281,7 @@ case $COMMAND in
167281
GREP=""
168282
NO_RELAYER=""
169283
NO_COMPILE=""
284+
WAIT_FOR_KEY_BOOTSTRAP=""
170285

171286
while (( "$#" )); do
172287
case "$1" in
@@ -229,6 +344,7 @@ case $COMMAND in
229344
input-proof)
230345
log_message="${LIGHT_BLUE}${BOLD}[TEST] INPUT PROOF (uint64)${RESET}"
231346
docker_args+=("-g" "test user input uint64")
347+
WAIT_FOR_KEY_BOOTSTRAP="1"
232348
;;
233349
input-proof-compute-decrypt)
234350
log_message="${LIGHT_BLUE}${BOLD}[TEST] INPUT PROOF (uint64)${RESET}"
@@ -291,8 +407,49 @@ case $COMMAND in
291407
fi
292408
if [ "$TEST_TYPE" != "debug" ]; then
293409
echo -e "${log_message}"
294-
docker exec fhevm-test-suite-e2e-debug "${docker_args[@]}"
410+
if [ -n "$WAIT_FOR_KEY_BOOTSTRAP" ]; then
411+
wait_for_coprocessor_key_bootstrap
412+
fi
413+
test_status=0
414+
if docker exec fhevm-test-suite-e2e-debug "${docker_args[@]}"; then
415+
test_status=0
416+
else
417+
test_status=$?
418+
fi
419+
if [ -n "$WAIT_FOR_KEY_BOOTSTRAP" ]; then
420+
collect_coprocessor_topology_logs
421+
fi
422+
if [ "$test_status" -ne 0 ]; then
423+
exit "$test_status"
424+
fi
425+
fi
426+
;;
427+
428+
smoke)
429+
print_logo
430+
PROFILE=$1
431+
if [[ ! $PROFILE =~ ^(multi-2-2|multi-3-5)$ ]]; then
432+
echo -e "${RED}[ERROR]${RESET} ${BOLD}Unknown smoke profile: ${PROFILE}${RESET}"
433+
usage
434+
exit 1
295435
fi
436+
437+
coprocessors=""
438+
threshold=""
439+
case "$PROFILE" in
440+
multi-2-2)
441+
coprocessors=2
442+
threshold=2
443+
;;
444+
multi-3-5)
445+
coprocessors=5
446+
threshold=3
447+
;;
448+
esac
449+
450+
echo -e "${LIGHT_BLUE}${BOLD}[SMOKE] Running profile ${PROFILE} (n=${coprocessors}, t=${threshold})...${RESET}"
451+
"${SCRIPT_DIR}/fhevm-cli" deploy --local --coprocessors "${coprocessors}" --coprocessor-threshold "${threshold}"
452+
"${SCRIPT_DIR}/fhevm-cli" test input-proof
296453
;;
297454

298455
help|-h|--help)

0 commit comments

Comments
 (0)