feat: add counter for critical logs (#163)

Author: matthias-wright

Cargo.lock

@@ -269,6 +269,9 @@ impl<
                         self.context.sleep(std::time::Duration::from_secs(5)).await;
                     } else {
                         error!(target: "critical", "finalizer started with invalid forkchoice: {forkchoice:?}, height: {}, epoch: {}", self.canonical_state.get_latest_height(), self.canonical_state.get_epoch());
+                        #[cfg(feature = "prom")]
+                        counter!("critical_errors_total", "reason" => "invalid_forkchoice", "severity" => "critical")
+                            .increment(1);
                         panic!(
                             "finalizer started with invalid forkchoice: {forkchoice:?}, height: {}, epoch: {}",
                             self.canonical_state.get_latest_height(),
@@ -361,6 +364,8 @@ impl<
                             // orchestrator, the finalizer should never receive a GetEpochGenesisHash request for the wrong epoch.
                             if epoch != self.canonical_state.get_epoch() {
                                 error!(target: "critical", "Finalizer received epoch genesis hash request from a different epoch. This should not happen and is a bug. Our epoch: {}, requested epoch {}", self.canonical_state.get_epoch(), epoch);
+                                #[cfg(feature = "prom")]
+                                counter!("critical_errors_total", "reason" => "epoch_mismatch", "severity" => "critical").increment(1);
                             }
                             let _ = response.send(self.canonical_state.get_epoch_genesis_hash());
                         },
@@ -1381,6 +1386,8 @@ async fn execute_block<
             ?eth_block_hash,
             "block validation failed, not executing but keeping in chain"
         );
+        #[cfg(feature = "prom")]
+        counter!("critical_errors_total", "reason" => "block_validation_failed", "severity" => "critical").increment(1);
     }
 
     state.set_latest_height(new_height);
@@ -1484,6 +1491,8 @@ async fn parse_execution_requests<
                                                     // The deposit contract verifies that the withdrawal credentials
                                                     // follow the expected format, so this should never happen.
                                                     error!(target: "critical", reason = "failed to parse withdrawal credentials (this is not a Summit error)", ?deposit_request);
+                                                    #[cfg(feature = "prom")]
+                                                    counter!("critical_errors_total", "reason" => "invalid_withdrawal_credentials", "severity" => "critical").increment(1);
                                                     warn!(
                                                         "Failed to parse withdrawal credentials: {e}"
                                                     );
@@ -1516,6 +1525,8 @@ async fn parse_execution_requests<
                                             // The deposit contract verifies that the withdrawal credentials
                                             // follow the expected format, so this should never happen.
                                             error!(target: "critical", reason = "failed to parse withdrawal credentials (this is not a Summit error)", ?deposit_request);
+                                            #[cfg(feature = "prom")]
+                                            counter!("critical_errors_total", "reason" => "invalid_withdrawal_credentials", "severity" => "critical").increment(1);
                                             warn!("Failed to parse withdrawal credentials: {e}");
                                             continue;
                                         }

finalizer/src/actor.rs

@@ -50,6 +50,8 @@ impl<E: Clock + Storage + Metrics, V: Variant> FinalizerState<E, V> {
     /// Log a database error and initiate graceful shutdown.
     fn handle_db_error(&self, e: impl std::fmt::Display, op: &str) {
         error!(target: "critical", %e, op, "fatal database error, initiating shutdown");
+        #[cfg(feature = "prom")]
+        metrics::counter!("critical_errors_total", "reason" => "fatal_db_error", "severity" => "critical").increment(1);
         self.cancellation_token.cancel();
     }
 

finalizer/src/db.rs

@@ -118,6 +118,7 @@ tikv-jemalloc-ctl = { version = "0.6", optional = true, features = ["stats"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.17.0", optional = true }
+libc = { version = "0.2", optional = true }
 
 [dev-dependencies]
 commonware-macros.workspace = true
@@ -146,6 +147,7 @@ prom = [
   "http",
   "eyre",
   "procfs",
+  "libc",
   "summit-types/prom",
 ]
 tokio-console = ["console-subscriber"]

node/Cargo.toml

@@ -48,6 +48,7 @@ impl Default for HooksBuilder {
                 Box::new(|| Collector::default().collect()),
                 Box::new(collect_memory_stats),
                 Box::new(collect_io_stats),
+                Box::new(collect_disk_stats),
             ],
         }
     }
@@ -175,3 +176,56 @@ fn collect_io_stats() {
 
 #[cfg(not(target_os = "linux"))]
 const fn collect_io_stats() {}
+
+#[cfg(target_os = "linux")]
+fn collect_disk_stats() {
+    use metrics::gauge;
+    use std::ffi::CString;
+    use std::fs;
+    use std::mem::MaybeUninit;
+    use tracing::error;
+
+    let Ok(contents) = fs::read_to_string("/proc/mounts")
+        .map_err(|error| error!(%error, "Failed to read /proc/mounts"))
+    else {
+        return;
+    };
+
+    for line in contents.lines() {
+        let fields: Vec<&str> = line.split_whitespace().collect();
+        if fields.len() < 3 {
+            continue;
+        }
+        let device = fields[0];
+        if !device.starts_with('/') {
+            continue;
+        }
+        let mount_point = fields[1];
+
+        let Some(c_path) = CString::new(mount_point).ok() else {
+            continue;
+        };
+
+        let mut buf = MaybeUninit::<libc::statvfs>::uninit();
+        let ret = unsafe { libc::statvfs(c_path.as_ptr(), buf.as_mut_ptr()) };
+        if ret != 0 {
+            continue;
+        }
+        let stat = unsafe { buf.assume_init() };
+
+        let block_size = stat.f_frsize;
+        let total = stat.f_blocks * block_size;
+        let free = stat.f_bfree * block_size;
+        let available = stat.f_bavail * block_size;
+        let used = total.saturating_sub(free);
+
+        gauge!("disk.total_bytes", "mountpoint" => mount_point.to_string()).set(total as f64);
+        gauge!("disk.free_bytes", "mountpoint" => mount_point.to_string()).set(free as f64);
+        gauge!("disk.available_bytes", "mountpoint" => mount_point.to_string())
+            .set(available as f64);
+        gauge!("disk.used_bytes", "mountpoint" => mount_point.to_string()).set(used as f64);
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+const fn collect_disk_stats() {}

node/src/prom/hooks.rs

@@ -64,9 +64,11 @@ impl MetricServer {
         describe_db_metrics();
         describe_static_file_metrics();
         describe_ssz_metrics();
+        describe_critical_error_metrics();
         Collector::default().describe();
         describe_memory_stats();
         describe_io_stats();
+        describe_disk_stats();
 
         Ok(())
     }
@@ -204,6 +206,15 @@ fn describe_ssz_metrics() {
     );
 }
 
+fn describe_critical_error_metrics() {
+    use metrics::describe_counter;
+
+    describe_counter!(
+        "critical_errors_total",
+        "Critical errors requiring immediate attention (labelled by reason)"
+    );
+}
+
 #[cfg(all(feature = "jemalloc", unix))]
 fn describe_memory_stats() {
     describe_gauge!(
@@ -262,6 +273,33 @@ fn describe_io_stats() {
 #[cfg(not(target_os = "linux"))]
 const fn describe_io_stats() {}
 
+#[cfg(target_os = "linux")]
+fn describe_disk_stats() {
+    describe_gauge!(
+        "disk.total_bytes",
+        Unit::Bytes,
+        "Total size of the filesystem"
+    );
+    describe_gauge!(
+        "disk.free_bytes",
+        Unit::Bytes,
+        "Free bytes on the filesystem (including reserved)"
+    );
+    describe_gauge!(
+        "disk.available_bytes",
+        Unit::Bytes,
+        "Bytes available to non-root users"
+    );
+    describe_gauge!(
+        "disk.used_bytes",
+        Unit::Bytes,
+        "Used bytes on the filesystem"
+    );
+}
+
+#[cfg(not(target_os = "linux"))]
+const fn describe_disk_stats() {}
+
 #[cfg(test)]
 mod tests {
     use super::*;