feat(test-suite): assert on-chain digest divergence in ciphertext-drift test

After the gw-listener confirms the drift warning, query the gateway
chain for AddCiphertextMaterial events and verify the two coprocessor
submissions produced distinct ciphertext digests.

Reverts the earlier ConsensusWatchdog changes — the watchdog is a
safety net for normal tests, not the right place for intentional
drift assertions.

Author: Eikix

test-suite/e2e/test/consensusWatchdog.test.ts

@@ -123,39 +123,6 @@ describe('ConsensusWatchdog', function () {
     });
   });
 
-  describe('checkHealth — expected divergence', function () {
-    it('should not throw when divergence is expected', async function () {
-      const { watchdog, setBlock, setCiphertextEvents } = mockWatchdog();
-
-      setCiphertextEvents(
-        [
-          fakeEvent('0xhandle1', 1n, '0xdigestA', '0xsnsDigestA', '0xCoprocessor1'),
-          fakeEvent('0xhandle1', 1n, '0xdigestB', '0xsnsDigestA', '0xCoprocessor2'),
-        ],
-        [],
-      );
-
-      setBlock(1);
-      await watchdog.flush();
-
-      expect(() => watchdog.checkHealth(true)).to.not.throw();
-    });
-
-    it('should skip stall checks when divergence is expected', async function () {
-      const { watchdog, setBlock, setCiphertextEvents } = mockWatchdog();
-
-      setCiphertextEvents([fakeEvent('0xhandle1', 1n, '0xdigest', '0xsns', '0xCopro1')], []);
-
-      setBlock(1);
-      await watchdog.flush();
-
-      const pending = (watchdog as any).pendingHandles.get('0xhandle1');
-      pending.firstSeenAt = Date.now() - 4 * 60 * 1000;
-
-      expect(() => watchdog.checkHealth(true)).to.not.throw();
-    });
-  });
-
   describe('checkHealth — stall detection', function () {
     it('should throw when consensus is not reached within timeout', async function () {
       const { watchdog, setBlock, setCiphertextEvents } = mockWatchdog();

test-suite/e2e/test/consensusWatchdog.ts

@@ -320,23 +320,15 @@ export class ConsensusWatchdog {
    * Check for divergences (instant) and stalls (3-minute timeout).
    * Called in afterEach to fail the current test if consensus is broken.
    */
-  checkHealth(expectDivergence = false): void {
+  checkHealth(): void {
     // Force a sync check of divergences accumulated since last poll.
     if (this.divergences.length > 0) {
       const msg = this.divergences.join('\n\n');
       this.divergences = [];
       this.divergenceKeys.clear();
-      if (expectDivergence) {
-        console.log(`[consensus-watchdog] Expected divergence confirmed:\n\n${msg}`);
-        return;
-      }
       throw new Error(`Consensus divergence detected:\n\n${msg}`);
     }
 
-    // When divergence is expected, the corrupted handle will never reach consensus,
-    // so skip stall checks to avoid a false-positive timeout error.
-    if (expectDivergence) return;
-
     // Check for stalls: handles that received a first submission but no consensus within timeout.
     const now = Date.now();
 
@@ -421,7 +413,7 @@ export const mochaHooks = {
 
     // Force one last poll before checking health so we catch recent events.
     await watchdog.flush();
-    watchdog.checkHealth(process.env.EXPECT_CIPHERTEXT_DIVERGENCE === 'true');
+    watchdog.checkHealth();
   },
 
   async afterAll(this: Mocha.Context) {

test-suite/fhevm/src/commands/test.ts

@@ -5,6 +5,7 @@ import { compatPolicyForState } from "../compat/compat";
 import { DRIFT_CLEANUP_SQL, DRIFT_INSTALL_SQL, driftDatabaseName, parseDriftInstanceIndex, parsePositiveInteger } from "../drift";
 import { PreflightError } from "../errors";
 import { pause, shellEscape, unpause } from "../flow/up-flow";
+import { hostReachableRpcUrl } from "../utils/fs";
 import { run, runWithHeartbeat } from "../utils/process";
 import { loadState } from "../state/state";
 import { topologyForState } from "../stack-spec/stack-spec";
@@ -222,6 +223,44 @@ const waitForDriftWarning = async (
   throw new PreflightError(`drift warning was not observed after injecting handle ${handleHex}`);
 };
 
+/** Topic hash for AddCiphertextMaterial(bytes32 indexed ctHandle, uint256 keyId, bytes32 ciphertextDigest, bytes32 snsCiphertextDigest, address coprocessorTxSender). */
+const ADD_CIPHERTEXT_MATERIAL_TOPIC = "0x7249a80e5b91709d2170511b960e8a92e1d5849d200f320524dfffd8b50308f7";
+
+/** Queries on-chain AddCiphertextMaterial events and asserts the two submissions have divergent digests. */
+const assertOnChainDivergence = async (
+  gatewayRpcUrl: string,
+  contractAddress: string,
+  handleHex: string,
+) => {
+  const paddedHandle = `0x${handleHex.toLowerCase().padStart(64, "0")}`;
+  const response = await fetch(gatewayRpcUrl, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      jsonrpc: "2.0",
+      id: 1,
+      method: "eth_getLogs",
+      params: [{ fromBlock: "0x0", toBlock: "latest", address: contractAddress, topics: [ADD_CIPHERTEXT_MATERIAL_TOPIC, paddedHandle] }],
+    }),
+  });
+  if (!response.ok) {
+    throw new PreflightError(`eth_getLogs failed: ${response.status} ${response.statusText}`);
+  }
+  const payload = (await response.json()) as { result?: { data: string }[] };
+  const logs = payload.result ?? [];
+  if (logs.length < 2) {
+    throw new PreflightError(`expected 2+ AddCiphertextMaterial events for handle 0x${handleHex}, got ${logs.length}`);
+  }
+  // data layout: keyId (32B) | ciphertextDigest (32B) | snsCiphertextDigest (32B) | coprocessorTxSender (32B)
+  // ciphertextDigest starts at byte offset 32 (chars 66..130 in the 0x-prefixed hex)
+  const digests = logs.map((log) => log.data.slice(66, 130));
+  const unique = new Set(digests);
+  if (unique.size < 2) {
+    throw new PreflightError(`on-chain AddCiphertextMaterial events show identical digests — drift not visible on chain`);
+  }
+  console.log(`[drift] on-chain divergence confirmed: ${logs.length} submissions with ${unique.size} distinct digest(s)`);
+};
+
 /** Builds the `docker exec` argv used to run tests inside the test-suite container. */
 export const buildTestContainerArgs = (tail: string[], extraExecArgs: string[] = []) => [
   "docker",
@@ -542,7 +581,7 @@ export const test = async (testName: string | undefined, options: TestOptions) =
           postgresPassword: postgres.postgresPassword,
         });
         await runWithHeartbeat(
-          buildTestContainerArgs(["./run-tests.sh", "-n", options.network, "-g", grepPattern], ["-e", "EXPECT_CIPHERTEXT_DIVERGENCE=true"]),
+          buildTestContainerArgs(["./run-tests.sh", "-n", options.network, "-g", grepPattern], ["-e", "GATEWAY_RPC_URL="]),
           "test ciphertext-drift",
         );
         const injectedHandleHex = await injector;
@@ -552,6 +591,11 @@ export const test = async (testName: string | undefined, options: TestOptions) =
           pollIntervalSeconds: driftAlertPollIntervalSeconds,
         });
         console.log(`[drift] detected in ${warning.container} for injected handle 0x${injectedHandleHex}`);
+        const ciphertextCommitsAddress = state.discovery!.gateway.CIPHERTEXT_COMMITS_ADDRESS;
+        if (ciphertextCommitsAddress) {
+          const gatewayRpcUrl = hostReachableRpcUrl(state.discovery!.endpoints.gateway.http);
+          await assertOnChainDivergence(gatewayRpcUrl, ciphertextCommitsAddress, injectedHandleHex);
+        }
       });
     }
     if (name === "multi-chain-isolation") {