feat(eth-proof-manager): Add proper metrics (#4439)

## What ❔

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- The `Why` has to be clear to non-Matter Labs entities running their
own ZK Chain -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Is this a breaking change?
- [ ] Yes
- [ ] No

## Operational changes
<!-- Any config changes? Any new flags? Any changes to any scripts? -->
<!-- Please add anything that non-Matter Labs entities running their own
ZK Chain may need to know -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zkstack dev fmt` and `zkstack dev
lint`.

Author: Artemka374

core/node/eth_proof_manager/src/lib.rs

@@ -9,6 +9,7 @@ use zksync_types::L2ChainId;
 use crate::client::EthProofManagerClient;
 
 mod client;
+mod metrics;
 pub mod node;
 mod sender;
 #[cfg(test)]

core/node/eth_proof_manager/src/metrics.rs

@@ -0,0 +1,30 @@
+use vise::{Counter, EncodeLabelSet, EncodeLabelValue, Family, Metrics};
+
+use crate::types::ProvingNetwork;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)]
+#[metrics(label = "tx_type", rename_all = "snake_case")]
+pub(super) enum TxType {
+    ProofRequest,
+    ValidationResult,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)]
+#[metrics(label = "validation_result", rename_all = "snake_case")]
+pub(super) enum ValidationResult {
+    Success,
+    Failed,
+}
+
+#[derive(Debug, Metrics)]
+#[metrics(prefix = "server_eth_proof_manager")]
+pub(super) struct EthProofManagerMetrics {
+    pub failed_to_send_tx: Family<TxType, Counter>,
+    pub validated_batches: Family<ValidationResult, Counter>,
+    pub proven_batches: Family<ProvingNetwork, Counter>,
+    pub acknowledged_batches: Family<ProvingNetwork, Counter>,
+    pub fallbacked_batches: Counter<u64>,
+}
+
+#[vise::register]
+pub(super) static METRICS: vise::Global<EthProofManagerMetrics> = vise::Global::new();

core/node/eth_proof_manager/src/sender/submit_proof_request.rs

@@ -12,6 +12,7 @@ use zksync_types::{L1BatchId, L1BatchNumber, L2ChainId};
 
 use crate::{
     client::EthProofManagerClient,
+    metrics::{TxType, METRICS},
     types::{ProofRequestIdentifier, ProofRequestParams},
 };
 
@@ -80,6 +81,7 @@ impl ProofRequestSubmitter {
                         batch_id,
                         e
                     );
+                    METRICS.fallbacked_batches.inc();
                     self.connection_pool
                         .connection()
                         .await?
@@ -146,31 +148,36 @@ impl ProofRequestSubmitter {
             max_reward: self.config.max_reward,
         };
 
-        let tx_hash = self
+        match self
             .client
             .submit_proof_request(proof_request_identifier, proof_request_parameters)
             .await
-            .map_err(|e| {
-                anyhow::anyhow!(
+        {
+            Ok(tx_hash) => {
+                self.connection_pool
+                    .connection()
+                    .await?
+                    .eth_proof_manager_dal()
+                    .mark_batch_as_sent(batch_id, tx_hash)
+                    .await?;
+
+                tracing::info!(
+                    "Submitted proof request for batch {}, chain_id: {}, with tx hash {}",
+                    proof_generation_data.l1_batch_number,
+                    proof_generation_data.chain_id,
+                    tx_hash
+                );
+            }
+            Err(e) => {
+                METRICS.failed_to_send_tx[&TxType::ProofRequest].inc();
+                return Err(anyhow::anyhow!(
                     "Failed to submit proof request for batch {}, error: {}",
                     batch_id,
                     e
-                )
-            })?;
-
-        self.connection_pool
-            .connection()
-            .await?
-            .eth_proof_manager_dal()
-            .mark_batch_as_sent(batch_id, tx_hash)
-            .await?;
+                ));
+            }
+        }
 
-        tracing::info!(
-            "Submitted proof request for batch {}, chain_id: {}, with tx hash {}",
-            proof_generation_data.l1_batch_number,
-            proof_generation_data.chain_id,
-            tx_hash
-        );
         Ok(())
     }
 }

core/node/eth_proof_manager/src/sender/submit_proof_validation.rs

@@ -4,7 +4,11 @@ use tokio::sync::watch;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_types::L2ChainId;
 
-use crate::{client::EthProofManagerClient, types::ProofRequestIdentifier};
+use crate::{
+    client::EthProofManagerClient,
+    metrics::{TxType, METRICS},
+    types::ProofRequestIdentifier,
+};
 
 pub struct SubmitProofValidationSubmitter {
     client: Box<dyn EthProofManagerClient>,
@@ -78,11 +82,17 @@ impl SubmitProofValidationSubmitter {
                     return Ok(());
                 }
                 Err(e) => {
+                    METRICS.failed_to_send_tx[&TxType::ValidationResult].inc();
                     tracing::error!(
                         "Failed to submit proof validation for batch {}: {}",
                         batch_number,
                         e
                     );
+                    return Err(anyhow::anyhow!(
+                        "Failed to submit proof validation for batch {}: {}",
+                        batch_number,
+                        e
+                    ));
                 }
             }
         }

core/node/eth_proof_manager/src/types.rs

@@ -1,10 +1,12 @@
 use bellman::bn256::Bn256;
 use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuitNoLookupCustomGate;
 use fflonk::FflonkVerificationKey;
+use vise::{EncodeLabelSet, EncodeLabelValue};
 use zksync_eth_client::{ContractCallError, EnrichedClientError, SigningError};
 use zksync_types::{ethabi, U256};
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)]
+#[metrics(label = "proving_network", rename_all = "snake_case")]
 pub enum ProvingNetwork {
     None = 0,
     Fermah = 1,

core/node/eth_proof_manager/src/watcher/events/proof_request_acknowledged.rs

@@ -3,7 +3,7 @@ use async_trait::async_trait;
 use zksync_dal::{eth_watcher_dal::EventType, ConnectionPool, Core, CoreDal};
 use zksync_types::{api::Log, ethabi, h256_to_u256, L1BatchNumber, H256, U256};
 
-use crate::{types::ProvingNetwork, watcher::events::EventHandler};
+use crate::{metrics::METRICS, types::ProvingNetwork, watcher::events::EventHandler};
 
 //event ProofRequestAcknowledged(
 //     uint256 indexed chainId,
@@ -91,6 +91,8 @@ impl EventHandler for ProofRequestAcknowledgedHandler {
 
         tracing::info!("Received ProofRequestAcknowledgedEvent: {:?}", event);
 
+        METRICS.acknowledged_batches[&event.assigned_to].inc();
+
         if accepted {
             self.connection_pool
                 .connection()
@@ -106,6 +108,7 @@ impl EventHandler for ProofRequestAcknowledgedHandler {
                 "Proof request for batch {} not accepted, moving to prover cluster",
                 event.block_number
             );
+            METRICS.fallbacked_batches.inc();
             self.connection_pool
                 .connection()
                 .await?

core/node/eth_proof_manager/src/watcher/events/proof_request_proven.rs

@@ -18,6 +18,7 @@ use zksync_types::{
 };
 
 use crate::{
+    metrics::{ValidationResult, METRICS},
     types::{FflonkFinalVerificationKey, ProvingNetwork},
     watcher::events::EventHandler,
 };
@@ -153,6 +154,11 @@ impl EventHandler for ProofRequestProvenHandler {
                 .proof_generation_dal()
                 .save_proof_artifacts_metadata(batch_number, &proof_blob_url)
                 .await?;
+
+            METRICS.validated_batches[&ValidationResult::Success].inc();
+            METRICS.proven_batches[&event.assigned_to].inc();
+        } else {
+            METRICS.validated_batches[&ValidationResult::Failed].inc();
         }
 
         tracing::info!(

core/node/proof_data_handler/src/metrics.rs

@@ -1,4 +1,4 @@
-use vise::{Histogram, Metrics};
+use vise::{Counter, Histogram, Metrics};
 use zksync_object_store::bincode;
 use zksync_prover_interface::inputs::WitnessInputData;
 
@@ -14,6 +14,7 @@ pub(super) struct ProofDataHandlerMetrics {
     pub eip_4844_blob_size_in_mb: Histogram<u64>,
     #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))]
     pub total_blob_size_in_mb: Histogram<u64>,
+    pub fallbacked_batches: Counter<u64>,
 }
 
 impl ProofDataHandlerMetrics {

core/node/proof_data_handler/src/proof_router.rs

@@ -3,6 +3,8 @@ use std::time::Duration;
 use tokio::sync::watch;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 
+use crate::metrics::METRICS;
+
 pub struct ProofRouter {
     connection_pool: ConnectionPool<Core>,
     acknowledgment_timeout: Duration,
@@ -48,6 +50,8 @@ impl ProofRouter {
 
             tracing::info!("Fallbacked {} batches with timeouts: acknowledgment timeout: {:?} and proving timeout: {:?}", amount, self.acknowledgment_timeout, self.proving_timeout);
 
+            METRICS.fallbacked_batches.inc_by(amount as u64);
+
             tokio::time::sleep(Duration::from_secs(10)).await;
         }
         Ok(())