feat(test-suite): add multi-chain e2e testing infrastructure

.github/workflows/test-suite-e2e-tests.yml

@@ -67,6 +67,10 @@
         description: "KMS Core version"
         default: ""
         type: string
+      multi-chain:
+        description: "Deploy multi-chain setup and run multi-chain tests only"
+        default: false
+        type: boolean
       deploy-build:
         description: "Build local Docker images from the checked out repository before deploy"
         default: false
@@ -90,6 +94,7 @@
 
 jobs:
   fhevm-e2e-test:
+    if: ${{ !inputs.multi-chain }}
     permissions:
       contents: 'read' # Required to checkout repository code
       id-token: 'write' # Required for OIDC authentication
@@ -290,3 +295,107 @@
         if: always()
         run: |
           ./fhevm-cli clean
+
+  fhevm-e2e-multi-chain-test:
+    if: ${{ inputs.multi-chain }}
+    permissions:
+      contents: 'read' # Required to checkout repository code
+      id-token: 'write' # Required for OIDC authentication
+      packages: 'read' # Required to read GitHub packages/container registry
+    env:
+      COPROCESSOR_DB_MIGRATION_VERSION: ${{ inputs.coprocessor-db-migration-version }}
+      COPROCESSOR_HOST_LISTENER_VERSION: ${{ inputs.coprocessor-host-listener-version }}
+      COPROCESSOR_GW_LISTENER_VERSION: ${{ inputs.coprocessor-gw-listener-version }}
+      COPROCESSOR_TX_SENDER_VERSION: ${{ inputs.coprocessor-tx-sender-version }}
+      COPROCESSOR_TFHE_WORKER_VERSION: ${{ inputs.coprocessor-tfhe-worker-version }}
+      COPROCESSOR_SNS_WORKER_VERSION: ${{ inputs.coprocessor-sns-worker-version }}
+      COPROCESSOR_ZKPROOF_WORKER_VERSION: ${{ inputs.coprocessor-zkproof-worker-version }}
+      GATEWAY_VERSION: ${{ inputs.gateway-version }}
+      HOST_VERSION: ${{ inputs.host-version }}
+      CONNECTOR_DB_MIGRATION_VERSION: ${{ inputs.connector-db-migration-version }}
+      CONNECTOR_GW_LISTENER_VERSION: ${{ inputs.connector-gw-listener-version }}
+      CONNECTOR_KMS_WORKER_VERSION: ${{ inputs.connector-kms-worker-version }}
+      CONNECTOR_TX_SENDER_VERSION: ${{ inputs.connector-tx-sender-version }}
+      TEST_SUITE_VERSION: ${{ inputs.test-suite-version }}
+      RELAYER_VERSION: ${{ inputs.relayer-version }}
+      CORE_VERSION: ${{ inputs.kms-core-version }}
+    runs-on: large_ubuntu_32
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: 'false'
+
+      - name: Setup Docker
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1.4.0
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GHCR_READ_TOKEN }}
+
+      - name: Login to Chainguard Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: cgr.dev
+          username: ${{ secrets.CGR_USERNAME }}
+          password: ${{ secrets.CGR_PASSWORD }}
+
+      - name: Display component versions
+        env:
+          JSON_INPUT: ${{ toJSON(inputs) }}
+        run: |
+          echo "Component versions: $JSON_INPUT"
+
+      - name: Deploy fhevm Stack (multi-chain)
+        working-directory: test-suite/fhevm
+        env:
+          DEPLOY_BUILD: ${{ inputs.deploy-build }}
+        run: |
+          if [[ "$DEPLOY_BUILD" == 'true' ]]; then
+            ./fhevm-cli deploy --build --multi-chain
+          else
+            ./fhevm-cli deploy --multi-chain
+          fi
+
+      - name: Multi-chain isolation test
+        working-directory: test-suite/fhevm
+        run: |
+          ./fhevm-cli test multi-chain-isolation
+
+      - name: Show logs on test failure
+        working-directory: test-suite/fhevm
+        if: always()
+        run: |
+          echo "::group::Relayer Logs"
+          ./fhevm-cli logs fhevm-relayer
+          echo "::endgroup::"
+          echo "::group::SNS Worker Logs"
+          ./fhevm-cli logs coprocessor-sns-worker | grep -v "Selected 0 rows to process"
+          echo "::endgroup::"
+          echo "::group::Transaction Sender Logs (filtered)"
+          ./fhevm-cli logs coprocessor-transaction-sender | grep -v "Selected 0 rows to process"
+          echo "::endgroup::"
+          echo "::group::Host Listener"
+          ./fhevm-cli logs coprocessor-host-listener
+          echo "::endgroup::"
+          echo "::group::Gateway Listener"
+          ./fhevm-cli logs coprocessor-gw-listener
+          echo "::endgroup::"
+          echo "::group::ZKProof Worker"
+          ./fhevm-cli logs coprocessor-zkproof-worker
+          echo "::endgroup::"
+          echo "::group::TFHE Worker"
+          ./fhevm-cli logs coprocessor-tfhe-worker
+          echo "::endgroup::"
+
+      - name: Cleanup
+        working-directory: test-suite/fhevm
+        if: always()
+        run: |
+          ./fhevm-cli clean

test-suite/README.md

@@ -21,6 +21,7 @@ KMS can be configured to two modes:
   - [Local Developer Optimizations](#local-developer-optimizations)
   - [Resuming a Deployment](#resuming-a-deployment)
   - [Deploying a Single Step](#deploying-a-single-step)
+  - [Multi-chain Mode](#multi-chain-mode)
 - [Security Policy](#security-policy)
   - [Handling Sensitive Data](#handling-sensitive-data)
     - [Environment Files](#environment-files)
@@ -47,6 +48,12 @@ cd test-suite/fhevm
 # Deploy with threshold 2 out of 2 coprocessors (local multicoprocessor mode)
 ./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2
 
+# Deploy multi-chain setup (Chain A + Chain B with separate host nodes)
+./fhevm-cli deploy --multi-chain
+
+# Run multi-chain isolation tests
+./fhevm-cli test multi-chain-isolation
+
 # Resume a failed deploy from a specific step (keeps existing containers/volumes)
 ./fhevm-cli deploy --resume kms-connector
 
@@ -149,6 +156,23 @@ You can combine `--only` or `--resume` with other flags:
 ./fhevm-cli deploy --only gateway-sc --build --local
 ```
 
+### Multi-chain mode
+
+Deploys a second host chain (Chain B) alongside Chain A, sharing the same coprocessor, gateway, and KMS. Chain B env files are derived automatically from the Chain A base files.
+
+```sh
+# Deploy multi-chain
+./fhevm-cli deploy --multi-chain
+
+# Run isolation tests
+./fhevm-cli test multi-chain-isolation
+
+# Combine with other flags
+./fhevm-cli deploy --multi-chain --build --local
+```
+
+Additional services deployed: `host-node-b` (chain ID `67890`, port `8547`), `host-sc-b`, and `coprocessor-host-listener-b`.
+
 ## Security policy
 
 ### Handling sensitive data

test-suite/e2e/test/multiChain/multiChain.fixture.ts

@@ -0,0 +1,28 @@
+import { ethers } from 'ethers';
+
+import { deployContract } from './multiChainHelper';
+
+export interface ChainContracts {
+  erc20: ethers.Contract;
+  erc20Address: string;
+  userDecrypt: ethers.Contract;
+  userDecryptAddress: string;
+  rand: ethers.Contract;
+}
+
+export async function deployChainFixture(
+  deployer: ethers.Signer,
+): Promise<ChainContracts> {
+  const erc20 = await deployContract('EncryptedERC20', deployer, 'Token', 'TKN');
+  const erc20Address = await erc20.getAddress();
+
+  const mintTx = await erc20.connect(deployer).getFunction('mint')(1_000_000, { gasLimit: 10_000_000 });
+  await mintTx.wait();
+
+  const userDecrypt = await deployContract('UserDecrypt', deployer);
+  const userDecryptAddress = await userDecrypt.getAddress();
+
+  const rand = await deployContract('Rand', deployer);
+
+  return { erc20, erc20Address, userDecrypt, userDecryptAddress, rand };
+}

test-suite/e2e/test/multiChain/multiChainHelper.ts

@@ -0,0 +1,127 @@
+import { ethers as hardhatEthers } from 'hardhat';
+import { ethers } from 'ethers';
+import { createInstance as createFhevmInstance } from '@zama-fhe/relayer-sdk/node';
+import { vars } from 'hardhat/config';
+
+const defaultMnemonic =
+  'adapt mosquito move limb mobile illegal tree voyage juice mosquito burger raise father hope layer';
+const mnemonic: string = process.env.MNEMONIC ?? vars.get('MNEMONIC', defaultMnemonic);
+
+const aclAddress = process.env.ACL_CONTRACT_ADDRESS!;
+const kmsVerifierAddress = process.env.KMS_VERIFIER_CONTRACT_ADDRESS!;
+const inputVerifierAddress = process.env.INPUT_VERIFIER_CONTRACT_ADDRESS!;
+const decryptionAddress = process.env.DECRYPTION_ADDRESS!;
+const inputVerificationAddress = process.env.INPUT_VERIFICATION_ADDRESS!;
+const relayerUrl = process.env.RELAYER_URL!;
+const gatewayChainId = Number(process.env.CHAIN_ID_GATEWAY!);
+
+export interface ChainConfig {
+  rpcUrl: string;
+  chainId: number;
+}
+
+export const CHAIN_A: ChainConfig = {
+  rpcUrl: process.env.RPC_URL || 'http://localhost:8545',
+  chainId: Number(process.env.CHAIN_ID_HOST || 12345),
+};
+
+export const CHAIN_B: ChainConfig = {
+  rpcUrl: process.env.RPC_URL_CHAIN_B || 'http://localhost:8547',
+  chainId: Number(process.env.CHAIN_ID_HOST_B || 67890),
+};
+
+const providers = new Map<string, ethers.JsonRpcProvider>();
+
+export function getProvider(chain: ChainConfig): ethers.JsonRpcProvider {
+  if (!providers.has(chain.rpcUrl)) {
+    providers.set(chain.rpcUrl, new ethers.JsonRpcProvider(chain.rpcUrl));
+  }
+  return providers.get(chain.rpcUrl)!;
+}
+
+export type ManagedWallet = ethers.NonceManager & { address: string; reset: () => void };
+
+function wrapWithNonceManager(wallet: ethers.Wallet): ManagedWallet {
+  const nm = new ethers.NonceManager(wallet);
+  (nm as any).address = wallet.address;
+  return nm as ManagedWallet;
+}
+
+export interface NamedSigners {
+  alice: ManagedWallet;
+  bob: ManagedWallet;
+  carol: ManagedWallet;
+  dave: ManagedWallet;
+  eve: ManagedWallet;
+}
+
+const signersCache = new Map<string, NamedSigners>();
+
+export function getSigners(chain: ChainConfig): NamedSigners {
+  if (!signersCache.has(chain.rpcUrl)) {
+    const provider = getProvider(chain);
+    const hdNode = ethers.HDNodeWallet.fromMnemonic(
+      ethers.Mnemonic.fromPhrase(mnemonic),
+      "m/44'/60'/0'/0",
+    );
+    const mkWallet = (i: number) => wrapWithNonceManager(
+      new ethers.Wallet(hdNode.deriveChild(i).privateKey, provider),
+    );
+    signersCache.set(chain.rpcUrl, {
+      alice: mkWallet(0),
+      bob: mkWallet(1),
+      carol: mkWallet(2),
+      dave: mkWallet(3),
+      eve: mkWallet(4),
+    });
+  }
+  return signersCache.get(chain.rpcUrl)!;
+}
+
+export function getWallet(chain: ChainConfig, index: number): ManagedWallet {
+  const provider = getProvider(chain);
+  const hdNode = ethers.HDNodeWallet.fromMnemonic(
+    ethers.Mnemonic.fromPhrase(mnemonic),
+    "m/44'/60'/0'/0",
+  );
+  return wrapWithNonceManager(
+    new ethers.Wallet(hdNode.deriveChild(index).privateKey, provider),
+  );
+}
+
+export async function createInstance(chain: ChainConfig) {
+  return createFhevmInstance({
+    verifyingContractAddressDecryption: decryptionAddress,
+    verifyingContractAddressInputVerification: inputVerificationAddress,
+    kmsContractAddress: kmsVerifierAddress,
+    inputVerifierContractAddress: inputVerifierAddress,
+    aclContractAddress: aclAddress,
+    network: chain.rpcUrl,
+    relayerUrl,
+    gatewayChainId,
+    chainId: chain.chainId,
+  });
+}
+
+export async function deployContract(
+  contractName: string,
+  deployer: ethers.Signer,
+  ...constructorArgs: unknown[]
+): Promise<ethers.Contract> {
+  const artifact = await hardhatEthers.getContractFactory(contractName);
+  const factory = new ethers.ContractFactory(artifact.interface, artifact.bytecode, deployer);
+  const contract = await factory.deploy(...constructorArgs, { gasLimit: 10_000_000 });
+  await contract.waitForDeployment();
+  return contract as ethers.Contract;
+}
+
+export async function evmSnapshot(provider: { send: (method: string, params: unknown[]) => Promise<unknown> }): Promise<string> {
+  return provider.send('evm_snapshot', []) as Promise<string>;
+}
+
+export async function evmRevert(
+  provider: { send: (method: string, params: unknown[]) => Promise<unknown> },
+  snapshotId: string,
+): Promise<boolean> {
+  return provider.send('evm_revert', [snapshotId]) as Promise<boolean>;
+}