Add metrics for RPC failures

Author: jmcph4

src/metrics.rs

@@ -6,6 +6,7 @@ use prometheus::{IntGauge, Opts, Registry};
 pub struct Metrics {
     pub rpc_requests: Arc<IntGauge>,
     pub blocks_added: Arc<IntGauge>,
+    pub failed_rpc_requests: Arc<IntGauge>,
     pub registry: Arc<Registry>,
 }
 
@@ -21,17 +22,26 @@ impl Metrics {
             "The number of blocks added to the index",
         ))
         .expect("Invalid blocks_added gauge definition");
+        let failed_rpc_requests = IntGauge::with_opts(Opts::new(
+            "failed_rpc_requests",
+            "The number of requests made to the RPC node that have received an error response",
+        ))
+        .expect("Invalid rpc_requests gauge definition");
         let registry = Registry::new();
         registry
             .register(Box::new(rpc_requests.clone()))
             .expect("Invalid metrics registry definition");
         registry
             .register(Box::new(blocks_added.clone()))
             .expect("Invalid metrics registry definition");
+        registry
+            .register(Box::new(failed_rpc_requests.clone()))
+            .expect("Invalid metrics registry definition");
 
         Self {
             rpc_requests: Arc::new(rpc_requests),
             blocks_added: Arc::new(blocks_added),
+            failed_rpc_requests: Arc::new(failed_rpc_requests),
             registry: Arc::new(registry),
         }
     }

src/services/blockchain.rs

@@ -49,7 +49,7 @@ impl BlockchainService {
                     client: AnyClient::new(rpc).await?,
                 };
                 while let Some(header) =
-                    this.client.block_headers().await?.next().await
+                    this.client.block_headers().await.inspect_err(|e| error!("Failed to acquire block header stream from RPC: {e:?}"))?.next().await
                 {
                     metrics.rpc_requests.inc();
                     let block = this
@@ -59,7 +59,7 @@ impl BlockchainService {
                             header.hash,
                             alloy::rpc::types::BlockTransactionsKind::Full,
                         )
-                        .await?
+                        .await.inspect_err(|e| {error!("Failed to retrieve block by hash from RPC: {e:?}"); metrics.failed_rpc_requests.inc();})?
                         .ok_or(eyre!("No such block"))?;
                     db.add_block(&block).inspect_err(|e| {
                         error!("Failed to write block to database: {e:?}")