chore: deflake e2e_reqresp tests

The issue was racing on network readiness/state sync and occasionally sending to the wrong proposer (attester ordering mismatch).

  Fixes:

  - Wait for full P2P mesh connectivity instead of a fixed sleep.
  - Wait for all validator nodes to catch up to the block where the funded account was deployed before preparing/sending txs.
  - Align proposer selection to the validator order used to create nodes.
  - Await sendTx and fail fast on errors to avoid silent drops

NOTE:
The plan is to delete the `no_handhsake` version, but in the meantime, there is significant code duplication between the 2 (they are essentially identical), and any change to one test would have to be copied over to the other.
I've decided to clean up this a bit to make our life easier.

Author: mralj

yarn-project/end-to-end/bootstrap.sh

@@ -45,6 +45,7 @@ function test_cmds {
   local tests=(
     # List all standalone and nested tests, except for the ones listed above.
     src/e2e_!(prover)/*.test.ts
+    src/e2e_p2p/reqresp/*.test.ts
     src/e2e_!(block_building).test.ts
   )
   for test in "${tests[@]}"; do

yarn-project/end-to-end/src/e2e_p2p/reqresp.test.ts

@@ -0,0 +1,35 @@
+import type { AztecNodeService } from '@aztec/aztec-node';
+
+import type { P2PNetworkTest } from '../p2p_network.js';
+import { cleanupReqrespTest, createReqrespDataDir, createReqrespTest, runReqrespTxTest } from './utils.js';
+
+const DATA_DIR = createReqrespDataDir();
+
+describe('e2e_p2p_reqresp_tx', () => {
+  let t: P2PNetworkTest;
+  let nodes: AztecNodeService[];
+
+  beforeEach(async () => {
+    t = await createReqrespTest();
+  });
+
+  afterEach(async () => {
+    await cleanupReqrespTest({ t, nodes, dataDir: DATA_DIR });
+  });
+
+  it('should produce an attestation by requesting tx data over the p2p network', async () => {
+    /**
+     * Birds eye overview of the test
+     * 1. We spin up x nodes
+     * 2. We turn off receiving a tx via gossip from two of the nodes
+     * 3. We send a transactions and gossip it to other nodes
+     * 4. The disabled nodes will receive an attestation that it does not have the data for
+     * 5. They will request this data over the p2p layer
+     * 6. We receive all of the attestations that we need and we produce the block
+     *
+     * Note: we do not attempt to let this node produce a block, as it will not have received any transactions
+     *       from the other pxes.
+     */
+    nodes = await runReqrespTxTest({ t, dataDir: DATA_DIR });
+  });
+});

yarn-project/end-to-end/src/e2e_p2p/reqresp/reqresp.test.ts

@@ -0,0 +1,39 @@
+import type { AztecNodeService } from '@aztec/aztec-node';
+
+import type { P2PNetworkTest } from '../p2p_network.js';
+import { cleanupReqrespTest, createReqrespDataDir, createReqrespTest, runReqrespTxTest } from './utils.js';
+
+// TODO: DELETE THIS FILE
+// This is a temporary copy of reqresp.test.ts with status handshake disabled
+// Delete this file once we have settled on the cause of the reqresp flakes.
+
+const DATA_DIR = createReqrespDataDir();
+
+describe('e2e_p2p_reqresp_tx_no_handshake', () => {
+  let t: P2PNetworkTest;
+  let nodes: AztecNodeService[];
+
+  beforeEach(async () => {
+    t = await createReqrespTest({ disableStatusHandshake: true });
+  });
+
+  afterEach(async () => {
+    await cleanupReqrespTest({ t, nodes, dataDir: DATA_DIR });
+  });
+
+  it('should produce an attestation by requesting tx data over the p2p network', async () => {
+    /**
+     * Birds eye overview of the test
+     * 1. We spin up x nodes
+     * 2. We turn off receiving a tx via gossip from two of the nodes
+     * 3. We send a transactions and gossip it to other nodes
+     * 4. The disabled nodes will receive an attestation that it does not have the data for
+     * 5. They will request this data over the p2p layer
+     * 6. We receive all of the attestations that we need and we produce the block
+     *
+     * Note: we do not attempt to let this node produce a block, as it will not have received any transactions
+     *       from the other pxes.
+     */
+    nodes = await runReqrespTxTest({ t, dataDir: DATA_DIR, disableStatusHandshake: true });
+  });
+});