feat(test-suite): add per-service override support to fhevm-cli (#2075)

* feat(test-suite): add per-service override support to fhevm-cli

* feat(test-suite): warn user when selecting services that share a database

Author: PanGan21

test-suite/README.md

@@ -57,9 +57,12 @@ cd test-suite/fhevm
 ./fhevm-cli test public-decrypt-http-ebool
 ./fhevm-cli test erc20
 
-# Boot with a local coprocessor override
+# Boot with a local coprocessor override (all services)
 ./fhevm-cli up --target latest-release --override coprocessor --profile local
 
+# Boot with only specific coprocessor services built locally
+./fhevm-cli up --target latest-release --override coprocessor:host-listener,tfhe-worker --profile local
+
 # View logs
 ./fhevm-cli logs relayer
 
@@ -74,11 +77,17 @@ For the local CLI entrypoint and architecture, see [test-suite/fhevm/README.md](
 To run one local component on top of an otherwise versioned stack, use `--override`:
 
 ```sh
+# Override an entire group (builds all services locally)
 ./fhevm-cli up --target latest-release --override coprocessor --profile local
+
+# Override specific services within a group (others pull from registry)
+./fhevm-cli up --target latest-release --override coprocessor:host-listener,tfhe-worker --profile local
 ```
 
 Supported override groups are `coprocessor`, `kms-connector`, `gateway-contracts`, `host-contracts`, and `test-suite`.
 
+When specifying individual services, use the short suffix after the group prefix (e.g., `host-listener` not `coprocessor-host-listener`). Services that share a Docker image are automatically co-selected (e.g., `host-listener` includes `host-listener-poller`).
+
 ### Resuming a deployment
 
 If a boot fails mid-way, you can resume from a specific step:

test-suite/fhevm/ARCHITECTURE.md

@@ -32,7 +32,7 @@ flowchart TD
   F8 --> F9["relayer"]
   F9 --> F10["test-suite"]
 
-  G["Local overrides"] --> E
+  G["Local overrides (group or per-service)"] --> E
   H["Multicopro topology + per-instance overrides"] --> E
   I["Compatibility policy"] --> E
   I --> F7

test-suite/fhevm/README.md

@@ -162,12 +162,68 @@ Supported groups:
 - `host-contracts`
 - `test-suite`
 
-Example:
+### Override an entire group
 
 ```sh
 ./fhevm-cli up --target latest-release --override coprocessor --profile local
 ```
 
+### Override specific services
+
+To build only specific services locally while pulling the rest from the registry:
+
+```sh
+./fhevm-cli up --target latest-release --override coprocessor:host-listener,tfhe-worker --profile local
+```
+
+Use the short service suffix after the colon (e.g., `host-listener` not `coprocessor-host-listener`).
+Multiple services are comma-separated. Services that share a Docker image are automatically
+co-selected (e.g., `host-listener` includes `host-listener-poller`).
+
+> **Note:** `coprocessor` and `kms-connector` services share a database. Per-service overrides
+> work when your local changes don't include DB migrations. If your changes add or alter
+> migrations, non-overridden services will fail against the mismatched schema — use
+> `--override coprocessor` (full group) instead.
+
+Available suffixes per group:
+
+| Group | Suffixes |
+|-------|----------|
+| `coprocessor` | `db-migration`, `host-listener`, `host-listener-poller`, `gw-listener`, `tfhe-worker`, `zkproof-worker`, `sns-worker`, `transaction-sender` |
+| `kms-connector` | `db-migration`, `gw-listener`, `kms-worker`, `tx-sender` |
+| `gateway-contracts` | `deploy-mocked-zama-oft`, `set-relayer-mocked-payment`, `sc-deploy`, `sc-add-network`, `sc-add-pausers`, `sc-trigger-keygen`, `sc-trigger-crsgen` |
+| `host-contracts` | `sc-deploy`, `sc-add-pausers` |
+| `test-suite` | `e2e-debug` |
+
+### Multiple overrides
+
+Repeat `--override` to override several groups at once:
+
+```sh
+# Two full groups
+./fhevm-cli up --target latest-release --override coprocessor --override gateway-contracts --profile local
+
+# Per-service across groups
+./fhevm-cli up --target latest-release --override coprocessor:host-listener --override gateway-contracts:sc-deploy --profile local
+
+# Mixed: per-service + full group
+./fhevm-cli up --target latest-release --override coprocessor:host-listener --override kms-connector --profile local
+```
+
+The `--profile` flag applies to every override that doesn't specify its own profile.
+
+### Combining with env var overrides
+
+You can mix per-service local builds with registry tag overrides:
+
+```sh
+COPROCESSOR_GW_LISTENER_VERSION=abc1234 \
+  ./fhevm-cli up --target latest-release --override coprocessor:host-listener --profile local
+```
+
+This builds `host-listener` (and `host-listener-poller`) locally, pulls `gw-listener` at tag
+`abc1234`, and pulls all other coprocessor services at the resolved target version.
+
 ## Multicopro
 
 Example:

test-suite/fhevm/src/artifacts.ts

@@ -13,6 +13,7 @@ import {
   ENV_DIR,
   GROUP_BUILD_COMPONENTS,
   GROUP_BUILD_SERVICES,
+  GROUP_SERVICE_SUFFIXES,
   TEMPLATE_COMPOSE_DIR,
   TEMPLATE_ENV_DIR,
   TEMPLATE_RELAYER_CONFIG,
@@ -70,9 +71,11 @@ const loadComposeDoc = async (component: string) =>
 
 const overriddenServicesForComponent = (state: State, component: string) =>
   new Set(
-    state.overrides.flatMap((o) =>
-      GROUP_BUILD_COMPONENTS[o.group].includes(component) ? GROUP_BUILD_SERVICES[o.group] : [],
-    ),
+    state.overrides.flatMap((o) => {
+      if (!GROUP_BUILD_COMPONENTS[o.group].includes(component)) return [];
+      if (o.services?.length) return o.services;
+      return GROUP_BUILD_SERVICES[o.group];
+    }),
   );
 
 const retagLocal = (image: unknown) =>
@@ -209,10 +212,10 @@ const buildCoprocessorOverride = async (state: State) => {
   const baseOverride = state.topology.instances["coprocessor-0"];
   const baseEnv = await readEnvFile(envPath("coprocessor"));
   const compat = compatPolicyForState(state);
-  // Skip compat args for overridden services — local builds use HEAD code, not the resolved version.
-  const compatArgs = overridden.size ? {} : compat.coprocessorArgs;
   for (const [name, service] of Object.entries(doc.services)) {
-    const adjusted = applyInstanceAdjustments(service, envPath("coprocessor"), baseEnv, baseOverride, compatArgs);
+    // Skip compat args for overridden services — local builds use HEAD code, not the resolved version.
+    const serviceCompatArgs = overridden.has(name) ? {} : compat.coprocessorArgs;
+    const adjusted = applyInstanceAdjustments(service, envPath("coprocessor"), baseEnv, baseOverride, serviceCompatArgs);
     applyBuildPolicy(adjusted, overridden.has(name));
     services[name] = adjusted;
   }
@@ -222,12 +225,13 @@ const buildCoprocessorOverride = async (state: State) => {
     const instanceEnv = await readEnvFile(envPath(`coprocessor.${index}`));
     for (const [name, service] of Object.entries(doc.services)) {
       const suffix = name.replace(/^coprocessor-/, "");
+      const instanceCompatArgs = overridden.has(name) ? {} : compat.coprocessorArgs;
       const cloned = applyInstanceAdjustments(
         service,
         envPath(`coprocessor.${index}`),
         instanceEnv,
         override,
-        compatArgs,
+        instanceCompatArgs,
       );
       cloned.container_name = prefix + suffix;
       applyBuildPolicy(cloned, overridden.has(name));
@@ -480,7 +484,10 @@ const maybeBuild = async (
     if (GROUP_BUILD_COMPONENTS[override.group].includes(component)) {
       const doc = YAML.parse(await fs.readFile(composePath(component), "utf8")) as ComposeDoc;
       const available = new Set(Object.keys(doc.services));
-      const services = GROUP_BUILD_SERVICES[override.group].filter((s) => available.has(s));
+      const candidates = override.services?.length
+        ? override.services
+        : GROUP_BUILD_SERVICES[override.group];
+      const services = candidates.filter((s) => available.has(s));
       if (!services.length) {
         continue;
       }
@@ -549,16 +556,7 @@ export const serviceNameList = (state: State, component: string) => {
   if (component !== "coprocessor") {
     return [];
   }
-  const suffixes = [
-    "db-migration",
-    "host-listener",
-    "host-listener-poller",
-    "gw-listener",
-    "tfhe-worker",
-    "zkproof-worker",
-    "sns-worker",
-    "transaction-sender",
-  ];
+  const suffixes = GROUP_SERVICE_SUFFIXES["coprocessor"];
   const names: string[] = [];
   for (let index = 0; index < state.topology.count; index += 1) {
     for (const suffix of suffixes) {

test-suite/fhevm/src/layout.ts

@@ -96,6 +96,61 @@ export const GROUP_BUILD_SERVICES: Record<OverrideGroup, string[]> = {
   "test-suite": ["test-suite-e2e-debug"],
 };
 
+const GROUP_PREFIX: Record<OverrideGroup, string> = {
+  "coprocessor": "coprocessor-",
+  "kms-connector": "kms-connector-",
+  "gateway-contracts": "gateway-",
+  "host-contracts": "host-",
+  "test-suite": "test-suite-",
+};
+
+/** Derived from GROUP_BUILD_SERVICES by stripping the group prefix. */
+export const GROUP_SERVICE_SUFFIXES: Record<OverrideGroup, string[]> = Object.fromEntries(
+  Object.entries(GROUP_BUILD_SERVICES).map(([group, services]) => [
+    group,
+    services.map((s) => s.slice(GROUP_PREFIX[group as OverrideGroup].length)),
+  ]),
+) as Record<OverrideGroup, string[]>;
+
+/** Canonical sibling groups — services sharing the same Docker image. */
+const SIBLING_GROUPS: string[][] = [
+  ["coprocessor-host-listener", "coprocessor-host-listener-poller"],
+];
+
+/** Bidirectional lookup derived from SIBLING_GROUPS. */
+export const IMAGE_SIBLINGS: Record<string, string[]> = {};
+for (const group of SIBLING_GROUPS) {
+  for (const member of group) {
+    IMAGE_SIBLINGS[member] = group.filter((s) => s !== member);
+  }
+}
+
+/** Groups where all services share a database — per-service override requires schema compatibility. */
+export const SCHEMA_COUPLED_GROUPS: OverrideGroup[] = ["coprocessor", "kms-connector"];
+
+export const suffixToServiceName = (group: OverrideGroup, suffix: string): string => {
+  const fullName = GROUP_PREFIX[group] + suffix;
+  if (!GROUP_BUILD_SERVICES[group].includes(fullName)) {
+    throw new Error(
+      `Unknown service "${suffix}" in group "${group}". Valid: ${GROUP_SERVICE_SUFFIXES[group].join(", ")}`,
+    );
+  }
+  return fullName;
+};
+
+/** Resolve suffixes to full service names, auto-including image siblings. */
+export const resolveServiceOverrides = (group: OverrideGroup, suffixes: string[]): string[] => {
+  const names = new Set<string>();
+  for (const suffix of suffixes) {
+    const name = suffixToServiceName(group, suffix);
+    names.add(name);
+    for (const sibling of IMAGE_SIBLINGS[name] ?? []) {
+      names.add(sibling);
+    }
+  }
+  return [...names];
+};
+
 export const TEST_GREP: Record<string, string> = {
   "paused-host-contracts": "test paused host user input|test paused host HTTP public decrypt|test paused host operators",
   "paused-gateway-contracts":

test-suite/fhevm/src/runtime.ts

@@ -16,6 +16,8 @@ import { composeDown, composeUp, inspectImageId, regen, resolvedComposeEnv, serv
 import {
   COMPONENT_BY_STEP,
   COMPONENTS,
+  GROUP_SERVICE_SUFFIXES,
+  SCHEMA_COUPLED_GROUPS,
   LOCK_DIR,
   LOG_TARGETS,
   PORTS,
@@ -28,6 +30,7 @@ import {
   envPath,
   gatewayAddressesPath,
   hostAddressesPath,
+  resolveServiceOverrides,
 } from "./layout";
 import type { Runner } from "./utils";
 import {
@@ -102,11 +105,34 @@ const createTopology = (count: number, threshold?: number, instances?: Record<st
 });
 
 const parseLocalOverride = (value: string): LocalOverride => {
-  const [group, profile] = value.split(":", 2);
+  const colonIdx = value.indexOf(":");
+  if (colonIdx < 0) {
+    if (!OVERRIDE_GROUPS.includes(value as OverrideGroup)) {
+      throw new Error(`Unsupported override ${value}`);
+    }
+    return { group: value as OverrideGroup };
+  }
+  const group = value.slice(0, colonIdx);
+  const rest = value.slice(colonIdx + 1);
   if (!OVERRIDE_GROUPS.includes(group as OverrideGroup)) {
-    throw new Error(`Unsupported override ${value}`);
+    throw new Error(`Unsupported override group "${group}"`);
+  }
+  const g = group as OverrideGroup;
+  const parts = rest.split(",").map((s) => s.trim()).filter(Boolean);
+  const isServiceList = parts.length > 1 || GROUP_SERVICE_SUFFIXES[g].includes(parts[0]);
+  if (isServiceList) {
+    const services = resolveServiceOverrides(g, parts);
+    return { group: g, services };
   }
-  return { group: group as OverrideGroup, profile };
+  // If the value contains a hyphen but isn't a known suffix, it's likely a typo.
+  if (parts.length === 1 && parts[0].includes("-")) {
+    throw new Error(
+      `Unknown service "${parts[0]}" in group "${g}". ` +
+      `Valid services: ${GROUP_SERVICE_SUFFIXES[g].join(", ")}. ` +
+      `If this is a cargo profile, use --profile instead.`,
+    );
+  }
+  return { group: g, profile: rest };
 };
 
 const parseKeyValue = (value: string) => {
@@ -580,12 +606,25 @@ const printBundle = (bundle: VersionBundle) => {
   log(describeBundle(bundle));
 };
 
+const describeOverride = (item: LocalOverride) => {
+  const profile = item.profile ? `:${item.profile}` : "";
+  const svc = item.services?.length ? `[${item.services.join(",")}]` : "";
+  return `${item.group}${profile}${svc}`;
+};
+
 const printPlan = (state: Pick<State, "target" | "overrides" | "topology">, fromStep?: StepName) => {
   log(`[plan] target=${state.target}`);
   if (state.overrides.length) {
-    log(
-      `[plan] overrides=${state.overrides.map((item) => `${item.group}${item.profile ? `:${item.profile}` : ""}`).join(", ")}`,
-    );
+    log(`[plan] overrides=${state.overrides.map(describeOverride).join(", ")}`);
+    for (const o of state.overrides) {
+      if (o.services?.length && SCHEMA_COUPLED_GROUPS.includes(o.group)) {
+        log(
+          `[warn] ${o.group}: per-service override with a shared database. ` +
+            `If your changes include DB migrations, non-overridden services may fail. ` +
+            `Use --override ${o.group} (full group) in that case.`,
+        );
+      }
+    }
   }
   log(`[plan] topology=n${state.topology.count}/t${state.topology.threshold}`);
   log(`[plan] steps=${STEP_NAMES.slice(stateStepIndex(fromStep ?? STEP_NAMES[0])).join(" -> ")}`);
@@ -924,7 +963,7 @@ const runStatus = async (deps: RuntimeDeps) => {
   if (state) {
     log(`[target] ${state.target}`);
     if (state.overrides.length) {
-      log(`[overrides] ${state.overrides.map((item) => `${item.group}${item.profile ? `:${item.profile}` : ""}`).join(", ")}`);
+      log(`[overrides] ${state.overrides.map(describeOverride).join(", ")}`);
     }
     log(`[topology] n=${state.topology.count} t=${state.topology.threshold}`);
     log(`[steps] ${state.completedSteps.join(", ") || "none"}`);
@@ -1090,7 +1129,7 @@ Commands:
 up options:
   --target latest-main|latest-release|devnet|testnet|mainnet
   --lock-file <path-to-bundle-json>
-  --override <group[:profile]>    repeated; groups: ${OVERRIDE_GROUPS.join(", ")}
+  --override <group[:profile|svc1,svc2]>  repeated; groups: ${OVERRIDE_GROUPS.join(", ")}
   --profile <cargo-profile>
   --coprocessors <n>
   --threshold <t>

test-suite/fhevm/src/types.ts

@@ -38,6 +38,7 @@ export type InstanceOverride = {
 export type LocalOverride = {
   group: OverrideGroup;
   profile?: string;
+  services?: string[];
 };
 
 export type Topology = {