chore: scenario network test fixes

- explicitly set timeout to be less than 6 hours so that workflow can complete network teardown
- when rolling pods, wait for every state to progress before continuing, wait for PVC to be completely wiped, and roll HA db first and initialize the DB, this will reduce k8 provisioning flakes
- reestablish prometheus port forwards on failure for prover-node test and wait for proven chain to update before killing prover broker to make the test more consistent
- add component label to HA postgres DB

Author: danielntmd

.github/workflows/ci3.yml

@@ -134,6 +134,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Run Network Scenarios
+        timeout-minutes: 350
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/test-network-scenarios.yml

@@ -32,12 +32,12 @@ concurrency:
 jobs:
   deploy-and-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 360
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Run Network Scenarios
+        timeout-minutes: 350
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

spartan/aztec-postgres/templates/_helpers.tpl

@@ -5,5 +5,6 @@
 {{- define "aztec-postgres.labels" -}}
 app.kubernetes.io/name: {{ .Chart.Name }}
 app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: {{ .Values.component | default .Chart.Name }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}

spartan/aztec-postgres/templates/statefulset.yaml

@@ -16,6 +16,7 @@ spec:
       labels:
         app.kubernetes.io/name: {{ .Chart.Name }}
         app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: {{ .Values.component | default .Chart.Name }}
     spec:
       containers:
         - name: postgres
@@ -62,6 +63,10 @@ spec:
   volumeClaimTemplates:
     - metadata:
         name: data
+        labels:
+          app.kubernetes.io/name: {{ .Chart.Name }}
+          app.kubernetes.io/instance: {{ .Release.Name }}
+          app.kubernetes.io/component: {{ .Values.component | default .Chart.Name }}
       spec:
         accessModes: ["ReadWriteOnce"]
         {{- if .Values.persistence.storageClass }}

spartan/aztec-postgres/values.yaml

@@ -3,6 +3,8 @@ image:
   tag: "18"
   pullPolicy: IfNotPresent
 
+component: "validator-ha-db"
+
 auth:
   database: validator_ha
   username: validator

spartan/bootstrap.sh

@@ -74,7 +74,6 @@ _emit_test() { echo "$_test_cmd_prefix $_test_cmd_run src/spartan/$1"; }
 
 function network_test_cmds_1 {
   _emit_test smoke.test.ts
-  _emit_test gating-passive.test.ts
   _emit_test reorg.test.ts
   _emit_test upgrade_rollup_version.test.ts
   _emit_test validator_ha.test.ts
@@ -86,6 +85,7 @@ function network_test_cmds_2 {
   _emit_test slash_inactivity.test.ts
   _emit_test proving.test.ts
   _emit_test prover-node.test.ts
+  _emit_test gating-passive.test.ts
   _emit_test invalidate_blocks.test.ts
   _emit_test mempool_limit.test.ts
   _emit_test upgrade_governance_proposer.test.ts

yarn-project/end-to-end/src/spartan/prover-node.test.ts

@@ -1,23 +1,26 @@
 import { createLogger } from '@aztec/foundation/log';
 import { retryUntil } from '@aztec/foundation/retry';
 
-import { AlertTriggeredError, GrafanaClient } from '../quality_of_service/grafana_client.js';
+import { AlertTriggeredError } from '../quality_of_service/grafana_client.js';
 import {
   ChainHealth,
   type ServiceEndpoint,
   applyProverBrokerKill,
   applyProverKill,
+  createResilientPrometheusConnection,
   deleteResourceByLabel,
   getGitProjectRoot,
+  getRPCEndpoint,
   setupEnvironment,
-  startPortForward,
+  waitForProvenToAdvance,
 } from './utils.js';
 
 const config = setupEnvironment(process.env);
 
 const logger = createLogger('e2e:spartan-test:prover-node');
 
 const epochDurationSeconds = config.AZTEC_EPOCH_DURATION * config.AZTEC_SLOT_DURATION;
+const slotDurationSeconds = config.AZTEC_SLOT_DURATION;
 
 /**
  * This test aims to check that a prover node is able to recover after a crash.
@@ -55,49 +58,20 @@ const enqueuedRootRollupJobs = {
 
 describe('prover node recovery', () => {
   const endpoints: ServiceEndpoint[] = [];
-  let alertChecker: GrafanaClient;
+  let runAlertCheck: ReturnType<typeof createResilientPrometheusConnection>['runAlertCheck'];
   let spartanDir: string;
+  let rpcEndpoint: ServiceEndpoint;
   const health = new ChainHealth(config.NAMESPACE, logger);
 
   beforeAll(async () => {
     await health.setup();
-    // Try Prometheus in a dedicated metrics namespace first; if not present, fall back to the network namespace
-    let promPort = 0;
-    let promUrl = '';
-    let promProc: Awaited<ReturnType<typeof startPortForward>>['process'];
-    {
-      const result = await startPortForward({
-        resource: `svc/metrics-prometheus-server`,
-        namespace: 'metrics',
-        containerPort: 80,
-      });
-      promProc = result.process;
-      promPort = result.port;
-      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
-      if (promPort === 0) {
-        result.process.kill();
-      }
-    }
-
-    if (promPort === 0) {
-      const result = await startPortForward({
-        resource: `svc/prometheus-server`,
-        namespace: config.NAMESPACE,
-        containerPort: 80,
-      });
-      promProc = result.process;
-      promPort = result.port;
-      promUrl = `http://127.0.0.1:${promPort}/api/v1`;
-    }
-
-    if (!promProc || promPort === 0) {
-      throw new Error('Unable to port-forward to Prometheus. Ensure the metrics stack is deployed.');
-    }
-
-    endpoints.push({ url: promUrl, process: promProc });
-    const grafanaEndpoint = promUrl;
-    const grafanaCredentials = '';
-    alertChecker = new GrafanaClient(logger, { grafanaEndpoint, grafanaCredentials });
+
+    rpcEndpoint = await getRPCEndpoint(config.NAMESPACE);
+    endpoints.push(rpcEndpoint);
+
+    const prometheus = createResilientPrometheusConnection(config.NAMESPACE, endpoints, logger);
+    await prometheus.connect();
+    runAlertCheck = prometheus.runAlertCheck;
 
     spartanDir = `${getGitProjectRoot()}/spartan`;
   });
@@ -120,7 +94,7 @@ describe('prover node recovery', () => {
     await retryUntil(
       async () => {
         try {
-          await alertChecker.runAlertCheck([enqueuedBlockRollupJobs]);
+          await runAlertCheck([enqueuedBlockRollupJobs]);
         } catch (err) {
           return err && err instanceof AlertTriggeredError;
         }
@@ -139,12 +113,11 @@ describe('prover node recovery', () => {
       values: { 'global.chaosResourceNamespace': config.NAMESPACE },
     });
 
-    // wait for the node to start proving again and
-    // validate it hits the cache
+    // Wait for the node to start proving again and validate it hits the cache
     const result = await retryUntil(
       async () => {
         try {
-          await alertChecker.runAlertCheck([cachedProvingJobs]);
+          await runAlertCheck([cachedProvingJobs]);
         } catch (err) {
           if (err && err instanceof AlertTriggeredError) {
             return true;
@@ -163,10 +136,11 @@ describe('prover node recovery', () => {
   it('should recover after a broker crash', async () => {
     logger.info(`Waiting for epoch proving job to start`);
 
+    // First, wait for proving to be active
     await retryUntil(
       async () => {
         try {
-          await alertChecker.runAlertCheck([enqueuedBlockRollupJobs]);
+          await runAlertCheck([enqueuedBlockRollupJobs]);
         } catch (err) {
           return err && err instanceof AlertTriggeredError;
         }
@@ -176,7 +150,11 @@ describe('prover node recovery', () => {
       5,
     );
 
-    logger.info(`Detected epoch proving job. Killing the broker`);
+    logger.info(`Detected epoch proving job. Waiting for proven block to advance...`);
+
+    await waitForProvenToAdvance(rpcEndpoint.url, logger, epochDurationSeconds * 3, slotDurationSeconds);
+
+    logger.info(`Proven block advanced. Killing the broker`);
 
     await applyProverBrokerKill({
       namespace: config.NAMESPACE,
@@ -185,16 +163,16 @@ describe('prover node recovery', () => {
       values: { 'global.chaosResourceNamespace': config.NAMESPACE },
     });
 
+    // Wait for the broker to recover and proving to resume
     const result = await retryUntil(
       async () => {
         try {
-          await alertChecker.runAlertCheck([enqueuedRootRollupJobs]);
+          await runAlertCheck([enqueuedRootRollupJobs]);
         } catch (err) {
           if (err && err instanceof AlertTriggeredError) {
             return true;
           }
         }
-
         return false;
       },
       'wait for root rollup',

yarn-project/end-to-end/src/spartan/upgrade_rollup_version.test.ts

@@ -26,7 +26,6 @@ import { mnemonicToAccount } from 'viem/accounts';
 
 import { MNEMONIC } from '../fixtures/fixtures.js';
 import {
-  ChainHealth,
   type ServiceEndpoint,
   getEthereumEndpoint,
   getRPCEndpoint,
@@ -47,16 +46,13 @@ describe('spartan_upgrade_rollup_version', () => {
   let ETHEREUM_HOSTS: string[];
   let originalL1ContractAddresses: L1ContractAddresses;
   const endpoints: ServiceEndpoint[] = [];
-  const health = new ChainHealth(config.NAMESPACE, debugLogger);
   jest.setTimeout(3 * 60 * 60 * 1000); // Governance flow can take a while
 
-  afterAll(async () => {
-    await health.teardown();
+  afterAll(() => {
     endpoints.forEach(e => e.process?.kill());
   });
 
   beforeAll(async () => {
-    await health.setup();
     const rpcEndpoint = await getRPCEndpoint(config.NAMESPACE);
     const ethEndpoint = await getEthereumEndpoint(config.NAMESPACE);
     endpoints.push(rpcEndpoint, ethEndpoint);

yarn-project/end-to-end/src/spartan/utils/index.ts

@@ -24,6 +24,7 @@ export {
   getServiceEndpoint,
   getRPCEndpoint,
   getEthereumEndpoint,
+  createResilientPrometheusConnection,
 } from './k8s.js';
 
 // Chaos Mesh
@@ -45,6 +46,7 @@ export { restartBot, installTransferBot, uninstallTransferBot } from './bot.js';
 // Node operations (sequencers, validators, pods)
 export {
   awaitCheckpointNumber,
+  waitForProvenToAdvance,
   getSequencers,
   updateSequencersConfig,
   getSequencersConfig,