feat(fhevm-cli): cross-version compat check and post-boot health gate

Detect relayer v1/v2 API mismatch at validate time (fail fast instead
of runtime 500s) and catch containers that crash shortly after boot
by inspecting exit codes after a delay.

Author: Eikix

test-suite/fhevm/src/cli.test.ts

@@ -12,8 +12,8 @@ import {
 } from "./artifacts";
 import { REPO_ROOT, STATE_DIR, TEST_GREP, composePath, resolveServiceOverrides } from "./layout";
 import { STEP_NAMES } from "./types";
-import { main, overrideWarnings, probeBootstrap, resolveUpgradePlan } from "./runtime";
-import { compatPolicyForState, requiresMultichainAclAddress } from "./compat";
+import { main, overrideWarnings, postBootHealthGate, probeBootstrap, resolveUpgradePlan } from "./runtime";
+import { compatPolicyForState, requiresMultichainAclAddress, validateBundleCompatibility } from "./compat";
 import { predictedCrsId, predictedKeyId } from "./utils";
 import { applyVersionEnvOverrides, createGitHubClient, resolveTarget } from "./versions";
 import {
@@ -1353,3 +1353,84 @@ describe("version resolution edge cases", () => {
     expect(result).toBe(original);
   });
 });
+
+describe("validateBundleCompatibility", () => {
+  const stateWithVersions = (relayer: string, testSuite: string) =>
+    stubState({ envOverrides: { RELAYER_VERSION: relayer, TEST_SUITE_VERSION: testSuite } });
+
+  test("detects relayer v1 vs test-suite v2 mismatch", () => {
+    const issues = validateBundleCompatibility(stateWithVersions("v0.9.0", "v0.11.0"));
+    expect(issues).toHaveLength(1);
+    expect(issues[0].code).toBe("relayer-v1-vs-test-suite-v2");
+  });
+
+  test("modern relayer is OK", () => {
+    expect(validateBundleCompatibility(stateWithVersions("v0.10.0", "v0.11.0"))).toEqual([]);
+  });
+
+  test("legacy test-suite is OK", () => {
+    expect(validateBundleCompatibility(stateWithVersions("v0.9.0", "v0.10.0"))).toEqual([]);
+  });
+
+  test("both modern is OK", () => {
+    expect(validateBundleCompatibility(stateWithVersions("v0.10.0", "v0.12.0"))).toEqual([]);
+  });
+
+  test("SHA relayer treated as modern", () => {
+    expect(validateBundleCompatibility(stateWithVersions("abc1234", "v0.11.0"))).toEqual([]);
+  });
+
+  test("SHA test-suite treated as modern triggers mismatch", () => {
+    const issues = validateBundleCompatibility(stateWithVersions("v0.9.0", "abc1234"));
+    expect(issues).toHaveLength(1);
+    expect(issues[0].code).toBe("relayer-v1-vs-test-suite-v2");
+  });
+
+  test("empty versions treated as modern", () => {
+    expect(validateBundleCompatibility(stateWithVersions("", ""))).toEqual([]);
+  });
+
+  test("boundary v0.10.0 relayer is OK", () => {
+    expect(validateBundleCompatibility(stateWithVersions("v0.10.0", "v0.11.0"))).toEqual([]);
+  });
+});
+
+describe("postBootHealthGate", () => {
+  const inspectResult = (status: string, exitCode: number) =>
+    JSON.stringify([{ Name: "test", State: { Status: status, ExitCode: exitCode }, NetworkSettings: { Networks: {} } }]);
+
+  test("resolves when all containers are running", async () => {
+    const runner = fakeRunner({
+      "docker inspect container-a": inspectResult("running", 0),
+      "docker inspect container-b": inspectResult("running", 0),
+    });
+    await postBootHealthGate({ runner }, ["container-a", "container-b"], 0);
+  });
+
+  test("throws when a container crashed", async () => {
+    const runner = fakeRunner({
+      "docker inspect container-a": inspectResult("running", 0),
+      "docker inspect container-b": inspectResult("exited", 1),
+      "docker logs --tail 30 container-b": "Error: missing API key",
+    });
+    await expect(
+      postBootHealthGate({ runner }, ["container-a", "container-b"], 0),
+    ).rejects.toThrow(/container-b.*exit 1/);
+  });
+
+  test("throws when container not found", async () => {
+    const runner = fakeRunner({
+      "docker inspect container-a": { stdout: "", stderr: "", code: 1 },
+    });
+    await expect(
+      postBootHealthGate({ runner }, ["container-a"], 0),
+    ).rejects.toThrow(/container not found/);
+  });
+
+  test("ignores containers that exited with code 0 (migrations)", async () => {
+    const runner = fakeRunner({
+      "docker inspect container-a": inspectResult("exited", 0),
+    });
+    await postBootHealthGate({ runner }, ["container-a"], 0);
+  });
+});

test-suite/fhevm/src/compat.ts

@@ -108,6 +108,33 @@ export const requiresLegacyRelayerReadinessConfig = (state: Pick<State, "version
 export const requiresLegacyRelayerUrl = (state: Pick<State, "versions">) =>
   versionLt(state.versions.env.TEST_SUITE_VERSION ?? "", [0, 11, 0]);
 
+export type BundleIncompatibility = { severity: "error"; code: string; message: string };
+
+/**
+ * Detect cross-component version incompatibilities that would cause runtime failures.
+ * Returns an empty array when the bundle is consistent.
+ */
+export const validateBundleCompatibility = (
+  state: Pick<State, "versions">,
+): BundleIncompatibility[] => {
+  const issues: BundleIncompatibility[] = [];
+  const relayerVersion = state.versions.env.RELAYER_VERSION ?? "";
+  const testSuiteVersion = state.versions.env.TEST_SUITE_VERSION ?? "";
+  if (
+    versionLt(relayerVersion, [0, 10, 0]) &&
+    !versionLt(testSuiteVersion, [0, 11, 0])
+  ) {
+    issues.push({
+      severity: "error",
+      code: "relayer-v1-vs-test-suite-v2",
+      message:
+        `RELAYER_VERSION ${relayerVersion} only serves /v1 API, but TEST_SUITE_VERSION ${testSuiteVersion} expects /v2. ` +
+        `Upgrade the relayer to >= v0.10.0 or pin TEST_SUITE_VERSION below v0.11.0.`,
+    });
+  }
+  return issues;
+};
+
 export const compatPolicyForState = (state: State): CompatPolicy => {
   const policy: CompatPolicy = { coprocessorArgs: {}, connectorEnv: {} };
   for (const rule of COMPAT_RULES.coprocessor) {

test-suite/fhevm/src/runtime.ts

@@ -1,4 +1,4 @@
-import { requiresMultichainAclAddress } from "./compat";
+import { requiresMultichainAclAddress, validateBundleCompatibility } from "./compat";
 import path from "node:path";
 import { parseArgs } from "node:util";
 
@@ -117,6 +117,59 @@ const dockerInspect = async (runner: Runner, name: string) => {
   }>;
 };
 
+const POST_BOOT_HEALTH_GATE_DELAY_MS = 5_000;
+
+const KMS_CONNECTOR_HEALTH_CONTAINERS = [
+  "kms-connector-gw-listener",
+  "kms-connector-kms-worker",
+  "kms-connector-tx-sender",
+];
+
+const coprocessorHealthContainers = (state: Pick<State, "topology">): string[] => {
+  const suffixes = GROUP_SERVICE_SUFFIXES["coprocessor"].filter((s) => !s.includes("migration"));
+  const names: string[] = [];
+  for (let index = 0; index < state.topology.count; index += 1) {
+    for (const suffix of suffixes) {
+      names.push(toServiceName(suffix, index));
+    }
+  }
+  return names;
+};
+
+export const postBootHealthGate = async (
+  deps: Pick<RuntimeDeps, "runner">,
+  containerNames: string[],
+  delayMs = POST_BOOT_HEALTH_GATE_DELAY_MS,
+) => {
+  if (delayMs > 0) await sleep(delayMs);
+  const crashed: { name: string; exitCode: number; logs: string }[] = [];
+  for (const name of containerNames) {
+    const [inspect] = await dockerInspect(deps.runner, name);
+    if (!inspect) {
+      crashed.push({ name, exitCode: -1, logs: "(container not found)" });
+      continue;
+    }
+    if (inspect.State.Status === "exited" && inspect.State.ExitCode !== 0) {
+      const result = await deps.runner(["docker", "logs", "--tail", "30", name], {
+        allowFailure: true,
+      });
+      crashed.push({
+        name,
+        exitCode: inspect.State.ExitCode,
+        logs: (result.stdout + result.stderr).trim(),
+      });
+    }
+  }
+  if (crashed.length) {
+    const details = crashed
+      .map((c) => `  ${c.name} (exit ${c.exitCode}):\n    ${c.logs.split("\n").join("\n    ")}`)
+      .join("\n");
+    throw new Error(
+      `Post-boot health gate: ${crashed.length} container(s) crashed shortly after starting:\n${details}`,
+    );
+  }
+};
+
 const loadState = async () => (await exists(STATE_FILE) ? readJson<State>(STATE_FILE) : undefined);
 const saveState = async (state: State) => writeJson(STATE_FILE, state);
 
@@ -983,16 +1036,26 @@ const runStep = async (state: State, step: StepName, deps: RuntimeDeps) => {
     case "regenerate":
       await regen(state, deps);
       break;
-    case "validate":
+    case "validate": {
       validateDiscovery(state);
+      const incompatibilities = validateBundleCompatibility(state);
+      if (incompatibilities.length) {
+        throw new Error(
+          `Bundle version incompatibilities detected:\n` +
+            incompatibilities.map((i) => `  - ${i.message}`).join("\n"),
+        );
+      }
       break;
+    }
     case "coprocessor":
       await composeUp("coprocessor", state, deps, saveState, log, serviceNameList(state, "coprocessor"));
       await waitForCoprocessor(state, deps);
+      await postBootHealthGate(deps, coprocessorHealthContainers(state));
       break;
     case "kms-connector":
       await composeUp("kms-connector", state, deps, saveState, log);
       await waitForKmsConnector(deps);
+      await postBootHealthGate(deps, KMS_CONNECTOR_HEALTH_CONTAINERS);
       break;
     case "bootstrap":
       await composeUp("gateway-sc", state, deps, saveState, log, ["gateway-sc-add-network"], { noDeps: true });