Skip to content

Commit 57efe28

Browse files
EikixEikix
committed
fix(fhevm-cli): auto-detect MinIO key prefix and treat unparseable versions conservatively
Three fixes for network target compatibility (devnet/testnet/mainnet): - Auto-detect MinIO key prefix during KMS signer discovery by probing both PUB/PUB (KMS <= v0.12) and PUB (KMS >= v0.13), then propagate the detected prefix through state for all MinIO URL construction. - Fix Docker volume mount conflicts where named volumes shadowed bind mounts to the same container path, preventing GatewayAddresses.sol from being accessible on the host. - Treat unparseable SHA versions conservatively in compat layer — apply legacy flags rather than skip them, since we cannot determine the actual version from a SHA tag.
1 parent df8f95c commit 57efe28

File tree

6 files changed

+115
-43
lines changed

6 files changed

+115
-43
lines changed

‎test-suite/fhevm/src/artifacts.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,8 +163,8 @@ describe("compose templates", () => {
163163
structuredClone(input),
164164
stubState({
165165
envOverrides: {
166-
RELAYER_VERSION: "sha-29b0750",
167-
RELAYER_MIGRATE_VERSION: "sha-29b0750",
166+
RELAYER_VERSION: "v0.10.0",
167+
RELAYER_MIGRATE_VERSION: "v0.10.0",
168168
},
169169
}),
170170
),

‎test-suite/fhevm/src/artifacts.ts‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,13 @@ const applyBuildPolicy = (service: Record<string, unknown>, isOverridden: boolea
9797

9898
const appendVolume = (service: Record<string, unknown>, value: string) => {
9999
const volumes = Array.isArray(service.volumes) ? [...service.volumes] : [];
100-
if (!volumes.includes(value)) {
101-
volumes.push(value);
100+
const target = value.split(":").slice(1).join(":");
101+
// Remove any existing mount to the same container path (e.g. named volumes)
102+
const filtered = target ? volumes.filter((v) => typeof v !== "string" || v.split(":").slice(1).join(":") !== target) : volumes;
103+
if (!filtered.includes(value)) {
104+
filtered.push(value);
102105
}
103-
service.volumes = volumes;
106+
service.volumes = filtered;
104107
};
105108

106109
const resolveComposePath = (value: string) =>
@@ -401,13 +404,15 @@ const writeRuntimeEnvFiles = async (state: State, deps: Pick<ArtifactDeps, "runn
401404
envs["coprocessor"].TENANT_API_KEY = DEFAULT_TENANT_API_KEY;
402405
envs["coprocessor"].COPROCESSOR_API_KEY = DEFAULT_TENANT_API_KEY;
403406
envs["coprocessor"].AWS_ENDPOINT_URL = state.discovery?.endpoints.minioExternal ?? "http://minio:9000";
407+
const kp = state.discovery?.minioKeyPrefix ?? "PUB";
408+
const minioInt = state.discovery?.endpoints.minioInternal ?? "http://minio:9000";
404409
envs["coprocessor"].FHE_KEY_ID = state.discovery?.actualFheKeyId ?? state.discovery?.fheKeyId ?? predictedKeyId();
405-
envs["coprocessor"].KMS_PUBLIC_KEY = `${state.discovery?.endpoints.minioInternal ?? "http://minio:9000"}/kms-public/PUB/PublicKey/${envs["coprocessor"].FHE_KEY_ID}`;
406-
envs["coprocessor"].KMS_SERVER_KEY = `${state.discovery?.endpoints.minioInternal ?? "http://minio:9000"}/kms-public/PUB/ServerKey/${envs["coprocessor"].FHE_KEY_ID}`;
407-
envs["coprocessor"].KMS_SNS_KEY = `${state.discovery?.endpoints.minioInternal ?? "http://minio:9000"}/kms-public/PUB/SnsKey/${envs["coprocessor"].FHE_KEY_ID}`;
408-
envs["coprocessor"].KMS_CRS_KEY = `${state.discovery?.endpoints.minioInternal ?? "http://minio:9000"}/kms-public/PUB/CRS/${state.discovery?.actualCrsKeyId ?? state.discovery?.crsKeyId ?? predictedCrsId()}`;
409-
envs["relayer"].APP_KEYURL__FHE_PUBLIC_KEY__URL = `${state.discovery?.endpoints.minioInternal ?? "http://minio:9000"}/kms-public/PUB/PublicKey/${state.discovery?.actualFheKeyId ?? state.discovery?.fheKeyId ?? predictedKeyId()}`;
410-
envs["relayer"].APP_KEYURL__CRS__URL = `${state.discovery?.endpoints.minioInternal ?? "http://minio:9000"}/kms-public/PUB/CRS/${state.discovery?.actualCrsKeyId ?? state.discovery?.crsKeyId ?? predictedCrsId()}`;
410+
envs["coprocessor"].KMS_PUBLIC_KEY = `${minioInt}/kms-public/${kp}/PublicKey/${envs["coprocessor"].FHE_KEY_ID}`;
411+
envs["coprocessor"].KMS_SERVER_KEY = `${minioInt}/kms-public/${kp}/ServerKey/${envs["coprocessor"].FHE_KEY_ID}`;
412+
envs["coprocessor"].KMS_SNS_KEY = `${minioInt}/kms-public/${kp}/SnsKey/${envs["coprocessor"].FHE_KEY_ID}`;
413+
envs["coprocessor"].KMS_CRS_KEY = `${minioInt}/kms-public/${kp}/CRS/${state.discovery?.actualCrsKeyId ?? state.discovery?.crsKeyId ?? predictedCrsId()}`;
414+
envs["relayer"].APP_KEYURL__FHE_PUBLIC_KEY__URL = `${minioInt}/kms-public/${kp}/PublicKey/${state.discovery?.actualFheKeyId ?? state.discovery?.fheKeyId ?? predictedKeyId()}`;
415+
envs["relayer"].APP_KEYURL__CRS__URL = `${minioInt}/kms-public/${kp}/CRS/${state.discovery?.actualCrsKeyId ?? state.discovery?.crsKeyId ?? predictedCrsId()}`;
411416
for (const [key, source] of Object.entries(compat.connectorEnv)) {
412417
if (envs["kms-connector"][source]) {
413418
envs["kms-connector"][key] = envs["kms-connector"][source];

‎test-suite/fhevm/src/cli.test.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -287,9 +287,17 @@ describe("runtime invariants", () => {
287287
expect(compatPolicyForState(makeState("v0.12.0")).coprocessorArgs["sns-worker"]).toBeUndefined();
288288
expect(compatPolicyForState(makeState("v0.12.0")).coprocessorArgs["transaction-sender"]).toBeUndefined();
289289

290-
// latest-main SHAs stay modern-only once resolution enforces the floor
291-
expect(compatPolicyForState(makeState("58aebb0")).coprocessorArgs["host-listener"]).toBeUndefined();
292-
expect(compatPolicyForState(makeState("58aebb0")).coprocessorArgs["transaction-sender"]).toBeUndefined();
290+
// unparseable SHAs are treated conservatively (apply all compat rules)
291+
expect(compatPolicyForState(makeState("58aebb0")).coprocessorArgs["host-listener"]).toEqual([
292+
["--coprocessor-api-key", { env: "COPROCESSOR_API_KEY" }],
293+
] as const);
294+
expect(compatPolicyForState(makeState("58aebb0")).coprocessorArgs["transaction-sender"]).toEqual([
295+
["--multichain-acl-address", { env: "MULTICHAIN_ACL_ADDRESS" }],
296+
["--delegation-fallback-polling", { value: "30" }],
297+
["--delegation-max-retry", { value: "100000" }],
298+
["--retry-immediately-on-nonce-error", { value: "2" }],
299+
["--host-chain-url", { env: "RPC_WS_URL" }],
300+
] as const);
293301
});
294302

295303
test("coprocessor depends_on rewrite only renames cloned services", () => {

‎test-suite/fhevm/src/compat.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,9 @@ const parseCompatVersion = (version: string) => {
8282
const versionLt = (version: string, target: CompatSemver) => {
8383
const parsed = parseCompatVersion(version);
8484
if (!parsed) {
85-
return false;
85+
// Non-semver (SHA tags): treat as old — safer to apply compat rules
86+
// than to skip them and have the service crash.
87+
return true;
8688
}
8789
for (let index = 0; index < parsed.length; index += 1) {
8890
if (parsed[index] !== target[index]) {

‎test-suite/fhevm/src/runtime.ts‎

Lines changed: 84 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ type UpOptions = {
6666
allowSchemaMismatch: boolean;
6767
resume: boolean;
6868
dryRun: boolean;
69+
reset: boolean;
6970
};
7071

7172
type CleanOptions = {
@@ -239,6 +240,7 @@ const parseCli = (argv: string[]) => {
239240
"instance-env": { type: "string", multiple: true, default: [] },
240241
"instance-arg": { type: "string", multiple: true, default: [] },
241242
"allow-schema-mismatch": { type: "boolean", default: false },
243+
reset: { type: "boolean", default: false },
242244
},
243245
});
244246
const target = parsed.values.target as string;
@@ -301,21 +303,45 @@ const bundleFromFile = async (target: UpOptions["target"], lockFile: string) =>
301303
return { ...bundle, target };
302304
};
303305

304-
const resolveBundle = async (options: Pick<UpOptions, "target" | "sha" | "lockFile">, deps: RuntimeDeps) => {
305-
const bundle = options.lockFile
306-
? await bundleFromFile(options.target, options.lockFile)
307-
: await resolveTarget(options.target, createGitHubClient(deps.runner), { sha: options.sha });
306+
const resolveCachePath = (target: string, sha?: string) =>
307+
path.join(LOCK_DIR, `.cache-${target}${sha ? `-${sha.toLowerCase().slice(0, 7)}` : ""}.json`);
308+
309+
const cachedResolve = async (
310+
options: Pick<UpOptions, "target" | "sha" | "lockFile" | "reset">,
311+
deps: RuntimeDeps,
312+
): Promise<VersionBundle> => {
313+
if (options.lockFile) {
314+
return bundleFromFile(options.target, options.lockFile);
315+
}
316+
const cachePath = resolveCachePath(options.target, options.sha);
317+
if (!options.reset) {
318+
try {
319+
const bundle = await readJson<VersionBundle>(cachePath);
320+
if (bundle.target === options.target) {
321+
log("[resolve] using cached bundle");
322+
return bundle;
323+
}
324+
} catch {
325+
// no cache or invalid — resolve fresh
326+
}
327+
}
328+
log("[resolve] fetching versions from GitHub...");
329+
const bundle = await resolveTarget(options.target, createGitHubClient(deps.runner), { sha: options.sha });
330+
await writeJson(cachePath, bundle);
331+
return bundle;
332+
};
333+
334+
const resolveBundle = async (options: Pick<UpOptions, "target" | "sha" | "lockFile" | "reset">, deps: RuntimeDeps) => {
335+
const bundle = await cachedResolve(options, deps);
308336
const resolved = applyVersionEnvOverrides(bundle, deps.env);
309337
const lockPath = await writeLock(resolved);
310338
return { bundle: resolved, lockPath };
311339
};
312340

313-
const previewBundle = (options: Pick<UpOptions, "target" | "sha" | "lockFile">, deps: RuntimeDeps) =>
314-
(options.lockFile
315-
? bundleFromFile(options.target, options.lockFile)
316-
: resolveTarget(options.target, createGitHubClient(deps.runner), { sha: options.sha })).then((bundle) =>
317-
applyVersionEnvOverrides(bundle, deps.env),
318-
);
341+
const previewBundle = async (options: Pick<UpOptions, "target" | "sha" | "lockFile" | "reset">, deps: RuntimeDeps) => {
342+
const bundle = await cachedResolve(options, deps);
343+
return applyVersionEnvOverrides(bundle, deps.env);
344+
};
319345

320346
const partialSchemaOverrides = (overrides: LocalOverride[]) =>
321347
overrides.filter(
@@ -468,29 +494,41 @@ const responseSnippet = async (response: Response) => {
468494
return text ? `: ${text.slice(0, 200)}` : "";
469495
};
470496

471-
const discoverSigner = async (deps: RuntimeDeps) => {
472-
for (let attempt = 0; attempt < 30; attempt += 1) {
497+
/** Try both `PUB/PUB/` (KMS ≤ v0.12) and `PUB/` (KMS ≥ v0.13) prefixes. */
498+
const MINIO_KEY_PREFIXES = ["PUB/PUB", "PUB"] as const;
499+
500+
const discoverSigner = async (deps: RuntimeDeps): Promise<{ address: string; minioKeyPrefix: string }> => {
501+
let lastHandle = "";
502+
for (let attempt = 0; attempt < 60; attempt += 1) {
473503
const logs = await deps.runner(["docker", "logs", "kms-core"], { allowFailure: true });
474504
const match = logs.stdout.match(/handle ([a-zA-Z0-9]+)/) ?? logs.stderr.match(/handle ([a-zA-Z0-9]+)/);
475505
if (match) {
476-
try {
477-
const response = await deps.fetch(`http://localhost:9000/kms-public/PUB/VerfAddress/${match[1]}`);
478-
if (!response.ok) {
479-
throw new Error(`Could not fetch KMS signer address (HTTP ${response.status})${await responseSnippet(response)}`);
480-
}
481-
return (await response.text()).trim();
482-
} catch (error) {
483-
if (shouldLogRetry(attempt)) {
484-
log(`[wait] kms signer fetch: ${toError(error).message}`);
506+
lastHandle = match[1];
507+
for (const prefix of MINIO_KEY_PREFIXES) {
508+
try {
509+
const response = await deps.fetch(`http://localhost:9000/kms-public/${prefix}/VerfAddress/${lastHandle}`);
510+
if (response.ok) {
511+
return { address: (await response.text()).trim(), minioKeyPrefix: prefix };
512+
}
513+
// Consume body to avoid leaking connections
514+
await response.text();
515+
} catch {
516+
// network error — retry
485517
}
486518
}
487-
}
488-
if (shouldLogRetry(attempt)) {
519+
if (shouldLogRetry(attempt)) {
520+
log(`[wait] kms signer fetch (handle: ${lastHandle.slice(0, 12)}…)`);
521+
}
522+
} else if (shouldLogRetry(attempt)) {
489523
log("[wait] kms signer handle");
490524
}
491525
await sleep(1000);
492526
}
493-
throw new Error("Could not extract KMS signer handle from kms-core logs after 30 attempts");
527+
throw new Error(
528+
lastHandle
529+
? `KMS signer address not available in MinIO after 60 attempts (handle: ${lastHandle})`
530+
: "Could not extract KMS signer handle from kms-core logs after 60 attempts",
531+
);
494532
};
495533

496534
const waitForKmsCore = async (deps: RuntimeDeps) => {
@@ -609,14 +647,15 @@ export const probeBootstrap = async (state: State, deps: RuntimeDeps, attempt =
609647
const actualFheKeyId = uint256ToId(actualKey);
610648
const actualCrsKeyId = uint256ToId(actualCrs);
611649
// Keys are on-chain — material MUST appear. Let ensureMaterialUrl throw if it doesn't.
650+
const kp = state.discovery!.minioKeyPrefix ?? "PUB";
612651
await Promise.all([
613652
ensureMaterialUrl(
614653
deps,
615-
hostReachableMaterialUrl(`${state.discovery!.endpoints.minioExternal}/kms-public/PUB/PublicKey/${actualFheKeyId}`),
654+
hostReachableMaterialUrl(`${state.discovery!.endpoints.minioExternal}/kms-public/${kp}/PublicKey/${actualFheKeyId}`),
616655
),
617656
ensureMaterialUrl(
618657
deps,
619-
hostReachableMaterialUrl(`${state.discovery!.endpoints.minioExternal}/kms-public/PUB/CRS/${actualCrsKeyId}`),
658+
hostReachableMaterialUrl(`${state.discovery!.endpoints.minioExternal}/kms-public/${kp}/CRS/${actualCrsKeyId}`),
620659
),
621660
]);
622661
state.discovery!.actualFheKeyId = actualFheKeyId;
@@ -892,7 +931,9 @@ const runStep = async (state: State, step: StepName, deps: RuntimeDeps) => {
892931
minioExternal: `http://${await minioIp(deps)}:9000`,
893932
},
894933
};
895-
state.discovery.kmsSigner = await discoverSigner(deps);
934+
const signer = await discoverSigner(deps);
935+
state.discovery.kmsSigner = signer.address;
936+
state.discovery.minioKeyPrefix = signer.minioKeyPrefix;
896937
await regen(state, deps);
897938
break;
898939
case "gateway-deploy":
@@ -908,6 +949,7 @@ const runStep = async (state: State, step: StepName, deps: RuntimeDeps) => {
908949
crsKeyId: state.discovery?.crsKeyId ?? predictedCrsId(),
909950
actualFheKeyId: state.discovery?.actualFheKeyId,
910951
actualCrsKeyId: state.discovery?.actualCrsKeyId,
952+
minioKeyPrefix: state.discovery?.minioKeyPrefix,
911953
endpoints: state.discovery?.endpoints ?? {
912954
gatewayHttp: "http://gateway-node:8546",
913955
gatewayWs: "ws://gateway-node:8546",
@@ -943,6 +985,7 @@ const runStep = async (state: State, step: StepName, deps: RuntimeDeps) => {
943985
crsKeyId: state.discovery?.crsKeyId ?? predictedCrsId(),
944986
actualFheKeyId: state.discovery?.actualFheKeyId,
945987
actualCrsKeyId: state.discovery?.actualCrsKeyId,
988+
minioKeyPrefix: state.discovery?.minioKeyPrefix,
946989
endpoints: state.discovery?.endpoints ?? {
947990
gatewayHttp: "http://gateway-node:8546",
948991
gatewayWs: "ws://gateway-node:8546",
@@ -1076,6 +1119,10 @@ const runUp = async (options: UpOptions, deps: RuntimeDeps) => {
10761119
if (options.resume && !state) {
10771120
throw new Error("No .fhevm/state.json to resume from");
10781121
}
1122+
if (!options.resume && (await loadState())) {
1123+
log("[up] cleaning previous run");
1124+
await runDown(deps);
1125+
}
10791126
if (!state) {
10801127
state = await bootstrapState(options, deps);
10811128
}
@@ -1454,6 +1501,7 @@ up options:
14541501
--from-step <${STEP_NAMES.join("|")}> requires --resume, except in --dry-run
14551502
--resume
14561503
--dry-run
1504+
--reset re-resolve versions from GitHub (ignore cache)
14571505
14581506
clean options:
14591507
--images remove CLI-owned local override images too
@@ -1462,9 +1510,10 @@ clean options:
14621510

14631511
export const main = async (argv = process.argv, deps: Partial<RuntimeDeps> = {}) => {
14641512
const runtime = { ...defaultDeps, ...deps };
1513+
let command: string | undefined;
14651514
try {
14661515
const parsed = parseCli(argv);
1467-
const command = parsed.command === "deploy" ? "up" : parsed.command;
1516+
command = parsed.command === "deploy" ? "up" : parsed.command;
14681517
const fromStep = ensureStep(parsed.parsed.values["from-step"] as string | undefined);
14691518
if (command === "up" && fromStep && !parsed.parsed.values.resume && !parsed.parsed.values["dry-run"]) {
14701519
throw new Error("--from-step requires --resume or --dry-run");
@@ -1481,6 +1530,7 @@ export const main = async (argv = process.argv, deps: Partial<RuntimeDeps> = {})
14811530
fromStep,
14821531
lockFile: parsed.parsed.values["lock-file"] as string | undefined,
14831532
allowSchemaMismatch: parsed.parsed.values["allow-schema-mismatch"],
1533+
reset: parsed.parsed.values.reset,
14841534
},
14851535
runtime,
14861536
);
@@ -1497,6 +1547,7 @@ export const main = async (argv = process.argv, deps: Partial<RuntimeDeps> = {})
14971547
allowSchemaMismatch: parsed.parsed.values["allow-schema-mismatch"],
14981548
resume: parsed.parsed.values.resume,
14991549
dryRun: parsed.parsed.values["dry-run"],
1550+
reset: parsed.parsed.values.reset,
15001551
},
15011552
runtime,
15021553
);
@@ -1534,6 +1585,8 @@ export const main = async (argv = process.argv, deps: Partial<RuntimeDeps> = {})
15341585
case "doctor":
15351586
throw new Error("`doctor` was removed; use `fhevm-cli up --dry-run ...`");
15361587
case "help":
1588+
case "--help":
1589+
case "-h":
15371590
case undefined:
15381591
usage();
15391592
return;
@@ -1542,6 +1595,9 @@ export const main = async (argv = process.argv, deps: Partial<RuntimeDeps> = {})
15421595
}
15431596
} catch (error) {
15441597
console.error(toError(error).message);
1598+
if (command === "up" && (await loadState())) {
1599+
console.error("Hint: run with --resume to continue, or without to start fresh.");
1600+
}
15451601
process.exitCode = 1;
15461602
}
15471603
};

‎test-suite/fhevm/src/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ export type Discovery = {
5353
crsKeyId: string;
5454
actualFheKeyId?: string;
5555
actualCrsKeyId?: string;
56+
minioKeyPrefix?: string;
5657
endpoints: {
5758
gatewayHttp: string;
5859
gatewayWs: string;

0 commit comments

Comments
 (0)