feat: Improve proving networks setup

core/lib/dal/migrations/20251028130429_add-attempts-to-eth-proof-manager-txs.down.sql

@@ -0,0 +1 @@
+ALTER TABLE eth_proof_manager DROP COLUMN validation_tx_attempts;

core/lib/dal/migrations/20251028130429_add-attempts-to-eth-proof-manager-txs.up.sql

@@ -0,0 +1 @@
+ALTER TABLE eth_proof_manager ADD COLUMN validation_tx_attempts INT NOT NULL DEFAULT 0;

core/lib/dal/src/eth_proof_manager_dal.rs

@@ -206,6 +206,7 @@ impl EthProofManagerDal<'_, '_> {
 
     pub async fn get_batch_to_send_validation_result(
         &mut self,
+        max_attempts: u64,
     ) -> anyhow::Result<Option<(L1BatchNumber, bool)>> {
         let batch_stats = self
             .storage
@@ -227,14 +228,15 @@ impl EthProofManagerDal<'_, '_> {
 
         let result: Option<(L1BatchNumber, Option<bool>)> = sqlx::query!(
             r#"
-            SELECT e.l1_batch_number, e.proof_validation_result
+            SELECT e.l1_batch_number, e.proof_validation_result, e.validation_tx_attempts
             FROM eth_proof_manager e
             JOIN proof_generation_details p
                 ON e.l1_batch_number = p.l1_batch_number
             WHERE
                 (
                     (e.status = $1 AND e.l1_batch_number <= $2)
                     OR (e.status = $3 AND e.proof_validation_result = false)
+                    AND e.validation_tx_attempts < $4
                 )
                 AND p.proving_mode = 'proving_network'
             ORDER BY e.l1_batch_number ASC
@@ -243,6 +245,7 @@ impl EthProofManagerDal<'_, '_> {
             EthProofManagerStatus::Proven.as_str(),
             i64::from(latest_proven_batch.0),
             EthProofManagerStatus::Fallbacked.as_str(),
+            max_attempts as i32,
         )
         .instrument("get_batch_to_validate")
         .fetch_optional(self.storage)
@@ -262,6 +265,42 @@ impl EthProofManagerDal<'_, '_> {
         }
     }
 
+    pub async fn increment_validation_tx_attempts(
+        &mut self,
+        batch_number: L1BatchNumber,
+    ) -> DalResult<()> {
+        sqlx::query!(
+            r#"
+            UPDATE eth_proof_manager SET
+                validation_tx_attempts = validation_tx_attempts + 1, updated_at = NOW()
+            WHERE l1_batch_number = $1
+            "#,
+            i64::from(batch_number.0),
+        )
+        .instrument("increment_validation_tx_attempts")
+        .with_arg("batch_number", &batch_number)
+        .execute(self.storage)
+        .await?;
+
+        Ok(())
+    }
+
+    pub async fn get_reached_max_attempts_amount(&mut self, max_attempts: u64) -> DalResult<u64> {
+        let result = sqlx::query!(
+            r#"
+            SELECT COUNT(*) FROM eth_proof_manager WHERE validation_tx_attempts >= $1
+            "#,
+            max_attempts as i32,
+        )
+        .instrument("get_reached_max_attempts_amount")
+        .fetch_one(self.storage)
+        .await?
+        .count
+        .unwrap_or(0) as u64;
+
+        Ok(result)
+    }
+
     pub async fn fallback_batches(
         &mut self,
         acknowledgment_timeout: Duration,

core/node/eth_proof_manager/src/client.rs

@@ -9,7 +9,7 @@ use zksync_types::{
     api::Log,
     ethabi::{self},
     web3::{BlockId, BlockNumber, Filter, FilterBuilder},
-    SLChainId, H256, U256,
+    Address, SLChainId, H256, U256,
 };
 
 use crate::types::{ClientError, ProofRequestIdentifier, ProofRequestParams};
@@ -64,6 +64,12 @@ pub trait EthProofManagerClient: 'static + std::fmt::Debug + Send + Sync {
     ) -> Result<H256, ClientError>;
 
     fn chain_id(&self) -> SLChainId;
+
+    fn submitter_address(&self) -> Address;
+
+    fn contract_address(&self) -> Address;
+
+    async fn submitter_balance(&self) -> Result<f64, ClientError>;
 }
 
 impl ProofManagerClient {
@@ -422,4 +428,20 @@ impl EthProofManagerClient for ProofManagerClient {
     fn chain_id(&self) -> SLChainId {
         self.client.chain_id()
     }
+
+    fn submitter_address(&self) -> Address {
+        self.client.sender_account()
+    }
+
+    fn contract_address(&self) -> Address {
+        self.client.contract_addr()
+    }
+
+    async fn submitter_balance(&self) -> Result<f64, ClientError> {
+        self.client
+            .sender_eth_balance()
+            .await
+            .map(|balance| balance.as_u64() as f64)
+            .map_err(Into::into)
+    }
 }

core/node/eth_proof_manager/src/lib.rs

@@ -7,7 +7,6 @@ use zksync_object_store::ObjectStore;
 use zksync_types::L2ChainId;
 
 use crate::client::EthProofManagerClient;
-
 mod client;
 mod metrics;
 pub mod node;

core/node/eth_proof_manager/src/metrics.rs

@@ -1,4 +1,4 @@
-use vise::{Counter, EncodeLabelSet, EncodeLabelValue, Family, Metrics};
+use vise::{Counter, EncodeLabelSet, EncodeLabelValue, Family, Gauge, LabeledFamily, Metrics};
 
 use crate::types::ProvingNetwork;
 
@@ -19,11 +19,16 @@ pub(super) enum ValidationResult {
 #[derive(Debug, Metrics)]
 #[metrics(prefix = "server_eth_proof_manager")]
 pub(super) struct EthProofManagerMetrics {
-    pub failed_to_send_tx: Family<TxType, Counter>,
     pub validated_batches: Family<ValidationResult, Counter>,
-    pub proven_batches: Family<ProvingNetwork, Counter>,
-    pub acknowledged_batches: Family<ProvingNetwork, Counter>,
+    pub proven_batches: Family<ProvingNetwork, Gauge<u64>>,
+    pub acknowledged_batches: Family<ProvingNetwork, Gauge<u64>>,
     pub fallbacked_batches: Counter<u64>,
+    pub reached_max_attempts: Family<TxType, Gauge<u64>>,
+    #[metrics(labels = ["submitter_address"])]
+    pub submitter_address: LabeledFamily<&'static str, Gauge, 1>,
+    #[metrics(labels = ["contract_address"])]
+    pub contract_address: LabeledFamily<&'static str, Gauge, 1>,
+    pub submitter_balance: Gauge<f64>,
 }
 
 #[vise::register]

core/node/eth_proof_manager/src/sender/mod.rs

@@ -61,6 +61,7 @@ impl EthProofSender {
             self.client.clone_boxed(),
             self.connection_pool.clone(),
             self.l2_chain_id,
+            self.config.max_tx_sending_attempts,
         );
 
         tokio::select! {

core/node/eth_proof_manager/src/sender/submit_proof_request.rs

@@ -169,7 +169,7 @@ impl ProofRequestSubmitter {
                 );
             }
             Err(e) => {
-                METRICS.failed_to_send_tx[&TxType::ProofRequest].inc();
+                METRICS.reached_max_attempts[&TxType::ProofRequest].inc_by(1);
                 return Err(anyhow::anyhow!(
                     "Failed to submit proof request for batch {}, error: {}",
                     batch_id,

core/node/eth_proof_manager/src/sender/submit_proof_validation.rs

@@ -14,18 +14,21 @@ pub struct SubmitProofValidationSubmitter {
     client: Box<dyn EthProofManagerClient>,
     connection_pool: ConnectionPool<Core>,
     l2_chain_id: L2ChainId,
+    max_attempts: u64,
 }
 
 impl SubmitProofValidationSubmitter {
     pub fn new(
         client: Box<dyn EthProofManagerClient>,
         connection_pool: ConnectionPool<Core>,
         l2_chain_id: L2ChainId,
+        max_attempts: u64,
     ) -> Self {
         Self {
             client,
             connection_pool,
             l2_chain_id,
+            max_attempts,
         }
     }
 
@@ -48,12 +51,22 @@ impl SubmitProofValidationSubmitter {
     }
 
     pub async fn loop_iteration(&self) -> anyhow::Result<()> {
+        let reached_max_attempts_amount = self
+            .connection_pool
+            .connection()
+            .await?
+            .eth_proof_manager_dal()
+            .get_reached_max_attempts_amount(self.max_attempts)
+            .await?;
+
+        METRICS.reached_max_attempts[&TxType::ValidationResult].set(reached_max_attempts_amount);
+
         let next_batch_to_be_validated = self
             .connection_pool
             .connection()
             .await?
             .eth_proof_manager_dal()
-            .get_batch_to_send_validation_result()
+            .get_batch_to_send_validation_result(self.max_attempts)
             .await?;
 
         if let Some((batch_number, validation_result)) = next_batch_to_be_validated {
@@ -62,6 +75,13 @@ impl SubmitProofValidationSubmitter {
                 block_number: batch_number.0 as u64,
             };
 
+            self.connection_pool
+                .connection()
+                .await?
+                .eth_proof_manager_dal()
+                .increment_validation_tx_attempts(batch_number)
+                .await?;
+
             match self
                 .client
                 .submit_proof_validation_result(proof_request_identifier, validation_result)
@@ -82,7 +102,6 @@ impl SubmitProofValidationSubmitter {
                     return Ok(());
                 }
                 Err(e) => {
-                    METRICS.failed_to_send_tx[&TxType::ValidationResult].inc();
                     tracing::error!(
                         "Failed to submit proof validation for batch {}: {}",
                         batch_number,

core/node/eth_proof_manager/src/watcher/events/proof_request_acknowledged.rs

@@ -91,7 +91,7 @@ impl EventHandler for ProofRequestAcknowledgedHandler {
 
         tracing::info!("Received ProofRequestAcknowledgedEvent: {:?}", event);
 
-        METRICS.acknowledged_batches[&event.assigned_to].inc();
+        METRICS.acknowledged_batches[&event.assigned_to].set(event.block_number.as_u64());
 
         if accepted {
             self.connection_pool

core/node/eth_proof_manager/src/watcher/events/proof_request_proven.rs

@@ -145,7 +145,7 @@ impl EventHandler for ProofRequestProvenHandler {
                     .await?;
 
                 METRICS.validated_batches[&ValidationResult::Success].inc();
-                METRICS.proven_batches[&event.assigned_to].inc();
+                METRICS.proven_batches[&event.assigned_to].set(batch_number.0 as u64);
                 true
             }
             Err(e) => {
@@ -191,7 +191,7 @@ async fn verify_proof(
                 ZkSyncSnarkWrapperCircuitNoLookupCustomGate,
                 RollingKeccakTranscript<Fr>,
             >(&verification_key, &proof, None)
-            .map_err(|e| anyhow::anyhow!("Failed to verify fflonk proof: {}", e))?
+            .unwrap_or(false)
         }
         TypedL1BatchProofForL1::Plonk(_) => {
             return Err(anyhow::anyhow!(