fix: fix interop integration tests (#4300)

## What ❔

CI for interop integration tests broke recently as a result of the
recent CI rework in:
#4255. This PR defines a
new (optional) secondary chain for integration tests, to be used for
testing interop behavior, which defaults to the chain the tests are
being run on.

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- The `Why` has to be clear to non-Matter Labs entities running their
own ZK Chain -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Is this a breaking change?
- [ ] Yes
- [ ] No

## Operational changes
<!-- Any config changes? Any new flags? Any changes to any scripts? -->
<!-- Please add anything that non-Matter Labs entities running their own
ZK Chain may need to know -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zkstack dev fmt` and `zkstack dev
lint`.

Author: zkzoomer

core/tests/highlevel-test-tools/src/run-integration-tests.ts

@@ -80,9 +80,17 @@ async function runTest(
     }
 }
 
-export async function runIntegrationTests(chainName: string, testPattern?: string): Promise<void> {
+export async function runIntegrationTests(
+    chainName: string,
+    secondChainName?: string | undefined,
+    testPattern?: string
+): Promise<void> {
     await initTestWallet(chainName);
-    await runTest('integration', chainName, testPattern, ['--verbose', '--ignore-prerequisites']);
+    await runTest('integration', chainName, testPattern, [
+        '--verbose',
+        '--ignore-prerequisites',
+        secondChainName ? `--second-chain=${secondChainName}` : ''
+    ]);
 }
 
 export async function feesTest(chainName: string): Promise<void> {
@@ -110,7 +118,12 @@ export async function genesisRecoveryTest(chainName: string): Promise<void> {
     await runTest('recovery', chainName, undefined, ['--no-kill', '--ignore-prerequisites', '--verbose']);
 }
 
-export async function enIntegrationTests(chainName: string): Promise<void> {
+export async function enIntegrationTests(chainName: string, secondChainName?: string | undefined): Promise<void> {
     await initTestWallet(chainName);
-    await runTest('integration', chainName, undefined, ['--verbose', '--ignore-prerequisites', '--external-node']);
+    await runTest('integration', chainName, undefined, [
+        '--verbose',
+        '--ignore-prerequisites',
+        '--external-node',
+        secondChainName ? `--second-chain=${secondChainName}` : ''
+    ]);
 }

core/tests/highlevel-test-tools/src/wait-for-batches.ts

@@ -40,6 +40,6 @@ export async function waitForAllBatchesToBeExecuted(chainName: string, timeoutMs
  */
 export async function generateRealisticLoad(chainName: string): Promise<void> {
     console.log(`🚀 Generating realistic load on chain: ${chainName}`);
-    await runIntegrationTests(chainName, 'ETH token checks');
+    await runIntegrationTests(chainName, undefined, 'ETH token checks');
     console.log(`✅ Realistic load generation completed for chain: ${chainName}`);
 }

core/tests/highlevel-test-tools/tests/en-integration-test.test.ts

@@ -5,6 +5,9 @@ import { enIntegrationTests } from '../src/run-integration-tests';
 describe('External Node Integration tests Test', () => {
     it(`for ${TESTED_CHAIN_TYPE} chain`, async () => {
         const testChain = await createChainAndStartServer(TESTED_CHAIN_TYPE, 'External Node Integration tests Test');
+        // Define some chain B used for interop tests
+        const secondChainType = TESTED_CHAIN_TYPE === 'era' ? 'validium' : 'era';
+        const testSecondChain = await createChainAndStartServer(secondChainType, 'External Node Integration tests');
 
         await testChain.generateRealisticLoad();
 
@@ -14,6 +17,6 @@ describe('External Node Integration tests Test', () => {
 
         await testChain.runExternalNode();
 
-        await enIntegrationTests(testChain.chainName);
+        await enIntegrationTests(testChain.chainName, testSecondChain.chainName);
     });
 });

core/tests/highlevel-test-tools/tests/main-node-integration-test.test.ts

@@ -4,7 +4,10 @@ import { TESTED_CHAIN_TYPE, createChainAndStartServer, runIntegrationTests } fro
 describe('Integration Test', () => {
     it(`for ${TESTED_CHAIN_TYPE} chain`, async () => {
         const testChain = await createChainAndStartServer(TESTED_CHAIN_TYPE, 'Main Node Integration Test');
+        // Define some chain B used for interop tests
+        const secondChainType = TESTED_CHAIN_TYPE === 'era' ? 'validium' : 'era';
+        const testSecondChain = await createChainAndStartServer(secondChainType, 'Main Node Integration Test');
 
-        await runIntegrationTests(testChain.chainName);
+        await runIntegrationTests(testChain.chainName, testSecondChain.chainName);
     });
 });

core/tests/ts-integration/src/constants.ts

@@ -3,6 +3,7 @@ import * as fs from 'fs';
 export const REQUIRED_L2_GAS_PRICE_PER_PUBDATA = 800;
 
 export const SYSTEM_UPGRADE_L2_TX_TYPE = 254;
+export const GATEWAY_CHAIN_ID = 505;
 export const ADDRESS_ONE = '0x0000000000000000000000000000000000000001';
 export const ETH_ADDRESS_IN_CONTRACTS = ADDRESS_ONE;
 export const L1_TO_L2_ALIAS_OFFSET = '0x1111000000000000000000000000000000001111';

core/tests/ts-integration/src/context-owner.ts

@@ -20,9 +20,11 @@ export const L1_DEFAULT_ETH_PER_ACCOUNT = ethers.parseEther('0.08');
 // Stress tests for L1->L2 transactions on localhost require a lot of upfront payment, but these are skipped during tests on normal environments
 export const L1_EXTENDED_TESTS_ETH_PER_ACCOUNT = ethers.parseEther('0.5');
 export const L2_DEFAULT_ETH_PER_ACCOUNT = ethers.parseEther('0.5');
+export const L2_SECOND_CHAIN_ETH_PER_ACCOUNT = ethers.parseEther('0.05');
 
 // Stress tests on local host may require a lot of additiomal funds, but these are skipped during tests on normal environments
 export const L2_EXTENDED_TESTS_ETH_PER_ACCOUNT = ethers.parseEther('50');
+export const L2_SECOND_CHAIN_EXTENDED_TESTS_ETH_PER_ACCOUNT = ethers.parseEther('5');
 export const ERC20_PER_ACCOUNT = ethers.parseEther('10000.0');
 
 interface VmPlaygroundHealth {
@@ -79,9 +81,11 @@ export class TestContextOwner {
 
     private mainEthersWallet: ethers.Wallet;
     private mainSyncWallet: zksync.Wallet;
+    private secondChainMainSyncWallet: zksync.Wallet | undefined;
 
     private l1Provider: ethers.JsonRpcProvider;
     private l2Provider: zksync.Provider;
+    private l2ProviderSecondChain: zksync.Provider | undefined;
 
     private reporter: Reporter = new Reporter();
 
@@ -109,6 +113,28 @@ export class TestContextOwner {
 
         this.mainEthersWallet = new ethers.Wallet(env.mainWalletPK, this.l1Provider);
         this.mainSyncWallet = new zksync.Wallet(env.mainWalletPK, this.l2Provider, this.l1Provider);
+
+        if (env.l2ChainIdSecondChain) {
+            this.reporter.message('Using second chain L2 provider: ' + env.l2NodeUrlSecondChain);
+            this.l2ProviderSecondChain = new RetryProvider(
+                {
+                    url: env.l2NodeUrlSecondChain!,
+                    timeout: 1200 * 1000
+                },
+                undefined,
+                this.reporter
+            );
+
+            if (isLocalHost(env.network)) {
+                this.l2ProviderSecondChain.pollingInterval = 100;
+            }
+
+            this.secondChainMainSyncWallet = new zksync.Wallet(
+                env.mainWalletPK,
+                this.l2ProviderSecondChain,
+                this.l1Provider
+            );
+        }
     }
 
     // Returns the required amount of L1 ETH
@@ -121,6 +147,13 @@ export class TestContextOwner {
         return isLocalHost(this.env.network) ? L2_EXTENDED_TESTS_ETH_PER_ACCOUNT : L2_DEFAULT_ETH_PER_ACCOUNT;
     }
 
+    // Returns the required amount of L2 ETH for the second chain
+    requiredL2ETHPerAccountSecondChain() {
+        return isLocalHost(this.env.network)
+            ? L2_SECOND_CHAIN_EXTENDED_TESTS_ETH_PER_ACCOUNT
+            : L2_SECOND_CHAIN_ETH_PER_ACCOUNT;
+    }
+
     /**
      * Performs the test context initialization.
      *
@@ -228,14 +261,20 @@ export class TestContextOwner {
         // `+ 1  for the main account (it has to send all these transactions).
         const accountsAmount = BigInt(suites.length) + 1n;
 
-        const l2ETHAmountToDeposit = await this.ensureBalances(accountsAmount);
+        const { l2ETHAmountToDeposit, l2ETHAmountToDepositSecondChain } = await this.ensureBalances(accountsAmount);
         const l2ERC20AmountToDeposit = ERC20_PER_ACCOUNT * accountsAmount;
         const wallets = this.createTestWallets(suites);
         const bridgehubContract = await this.mainSyncWallet.getBridgehubContract();
         const baseTokenAddress = await bridgehubContract.baseToken(this.env.l2ChainId);
         await this.distributeL1BaseToken(wallets, l2ERC20AmountToDeposit, baseTokenAddress);
         await this.cancelAllowances();
-        await this.distributeL1Tokens(wallets, l2ETHAmountToDeposit, l2ERC20AmountToDeposit, baseTokenAddress);
+        await this.distributeL1Tokens(
+            wallets,
+            l2ETHAmountToDeposit,
+            l2ETHAmountToDepositSecondChain,
+            l2ERC20AmountToDeposit,
+            baseTokenAddress
+        );
         await this.distributeL2Tokens(wallets);
 
         this.reporter.finishAction();
@@ -245,7 +284,9 @@ export class TestContextOwner {
     /**
      * Checks the operator account balances on L1 and L2 and deposits funds if required.
      */
-    private async ensureBalances(accountsAmount: bigint): Promise<bigint> {
+    private async ensureBalances(
+        accountsAmount: bigint
+    ): Promise<{ l2ETHAmountToDeposit: bigint; l2ETHAmountToDepositSecondChain: bigint }> {
         this.reporter.startAction(`Checking main account balance`);
 
         this.reporter.message(`Operator address is ${this.mainEthersWallet.address}`);
@@ -254,11 +295,26 @@ export class TestContextOwner {
         const actualL2ETHAmount = await this.mainSyncWallet.getBalance();
         this.reporter.message(`Operator balance on L2 is ${ethers.formatEther(actualL2ETHAmount)} ETH`);
 
+        let l2ETHAmountToDepositSecondChain = 0n;
+        if (this.env.l2ChainIdSecondChain) {
+            // We only need enough funds for a single test suite
+            const requiredL2SecondChainETHAmount = this.requiredL2ETHPerAccountSecondChain() * accountsAmount;
+            const actualL2SecondChainETHAmount = await this.secondChainMainSyncWallet!.getBalance();
+            this.reporter.message(
+                `Operator balance on second chain is ${ethers.formatEther(actualL2SecondChainETHAmount)} ETH`
+            );
+            if (requiredL2SecondChainETHAmount > actualL2SecondChainETHAmount) {
+                l2ETHAmountToDepositSecondChain = requiredL2SecondChainETHAmount - actualL2SecondChainETHAmount;
+            }
+        }
+
         // We may have enough funds in L2. If that's the case, no need to deposit more than required.
         const l2ETHAmountToDeposit =
             requiredL2ETHAmount > actualL2ETHAmount ? requiredL2ETHAmount - actualL2ETHAmount : 0n;
 
-        const requiredL1ETHAmount = this.requiredL1ETHPerAccount() * accountsAmount + l2ETHAmountToDeposit;
+        const requiredL1ETHAmount =
+            this.requiredL1ETHPerAccount() * accountsAmount + l2ETHAmountToDeposit + l2ETHAmountToDepositSecondChain;
+        // Both mainSyncWallet and secondChainMainSyncWallet share the same L1 wallet
         const actualL1ETHAmount = await this.mainSyncWallet.getBalanceL1();
         this.reporter.message(`Operator balance on L1 is ${ethers.formatEther(actualL1ETHAmount)} ETH`);
 
@@ -270,7 +326,7 @@ export class TestContextOwner {
         }
         this.reporter.finishAction();
 
-        return l2ETHAmountToDeposit;
+        return { l2ETHAmountToDeposit, l2ETHAmountToDepositSecondChain };
     }
 
     /**
@@ -386,6 +442,7 @@ export class TestContextOwner {
     private async distributeL1Tokens(
         wallets: TestWallets,
         l2ETHAmountToDeposit: bigint,
+        l2ETHAmountToDepositSecondChain: bigint,
         l2erc20DepositAmount: bigint,
         baseTokenAddress: zksync.types.Address
     ) {
@@ -430,6 +487,30 @@ export class TestContextOwner {
             );
             await depositHandle;
         }
+
+        // Deposit L2 tokens on the second chain (if needed).
+        if (l2ETHAmountToDepositSecondChain != 0n) {
+            // Given that we've already sent a number of transactions,
+            // we have to correctly send nonce.
+            const depositHandle = this.secondChainMainSyncWallet!.deposit({
+                token: zksync.utils.ETH_ADDRESS,
+                amount: l2ETHAmountToDepositSecondChain as BigNumberish,
+                overrides: {
+                    nonce,
+                    gasPrice
+                }
+            }).then((tx) => {
+                const amount = ethers.formatEther(l2ETHAmountToDeposit);
+                this.reporter.debug(
+                    `Sent ETH deposit on second chain. Nonce ${tx.nonce}, amount: ${amount}, hash: ${tx.hash}`
+                );
+                return tx.wait();
+            });
+            nonce = nonce + 1;
+            this.reporter.debug(`Nonce changed by 1 for ETH deposit on second chain, new nonce: ${nonce}`);
+            await depositHandle;
+        }
+
         // Define values for handling ERC20 transfers/deposits.
         const erc20Token = this.env.erc20Token.l1Address;
         const erc20MintAmount = l2erc20DepositAmount * 100n;
@@ -546,6 +627,21 @@ export class TestContextOwner {
         );
         l2startNonce += l2TxPromises.length;
 
+        let l2TxPromisesSecondChain: Promise<any>[] = [];
+        if (this.env.l2ChainIdSecondChain) {
+            const l2startNonceSecondChain = await this.secondChainMainSyncWallet!.getNonce();
+            // ETH transfers on second chain.
+            l2TxPromisesSecondChain = await sendTransfers(
+                zksync.utils.ETH_ADDRESS,
+                this.secondChainMainSyncWallet!,
+                wallets,
+                this.requiredL2ETHPerAccountSecondChain(),
+                l2startNonceSecondChain,
+                undefined,
+                this.reporter
+            );
+        }
+
         // ERC20 transfers.
         const l2TokenAddress = await this.mainSyncWallet.l2TokenAddress(this.env.erc20Token.l1Address);
         const erc20Promises = await sendTransfers(
@@ -557,7 +653,7 @@ export class TestContextOwner {
             undefined,
             this.reporter
         );
-        l2TxPromises.push(...erc20Promises);
+        l2TxPromises.push(...l2TxPromisesSecondChain, ...erc20Promises);
         await Promise.all(l2TxPromises);
 
         this.reporter.finishAction();
@@ -659,6 +755,7 @@ export class TestContextOwner {
             }
             try {
                 this.l2Provider.destroy();
+                this.l2ProviderSecondChain?.destroy();
             } catch (err: any) {
                 // Catch any request cancellation errors that propagate here after destroying L2 provider
                 console.log(`Caught error while destroying L2 provider: ${err}`);
@@ -677,6 +774,11 @@ export class TestContextOwner {
             await killPidWithAllChilds(this.env.l2NodePid, 9);
             this.reporter.finishAction();
         }
+        if (this.env.l2NodePidSecondChain !== undefined) {
+            this.reporter.startAction(`Terminating L2 node process`);
+            await killPidWithAllChilds(this.env.l2NodePidSecondChain, 9);
+            this.reporter.finishAction();
+        }
     }
 
     /**