feat(node): metrics for attestation freshness (#4236)

Author: barakeinav1

crates/node/src/indexer/tx_sender.rs

@@ -3,6 +3,9 @@ use super::IndexerState;
 use super::tx_signer::{TransactionSigner, TransactionSigners};
 use crate::config::RespondConfig;
 use crate::metrics;
+use crate::tee::attestation_freshness_metrics::{
+    record_attestation_landed, record_stored_attestation_expiry,
+};
 use crate::types::{
     LogTransaction, SignerContext, SubmittedTransaction, SubmittedTransactionStatus,
     SubmittedTxMetadata,
@@ -284,6 +287,8 @@ async fn observe_tx_result(
                 .get_participant_attestation(&indexer_state.mpc_contract_id, &args.tls_public_key)
                 .await?;
 
+            record_stored_attestation_expiry(stored_attestation.as_ref());
+
             let Some(stored_attestation) = stored_attestation else {
                 tracing::debug!(
                     "no attestation stored on chain for our key; submission not yet landed"
@@ -306,6 +311,7 @@ async fn observe_tx_result(
             );
 
             Ok(if attestation_landed {
+                record_attestation_landed(&Clock::real());
                 TransactionStatus::Executed
             } else {
                 TransactionStatus::NotExecuted

crates/node/src/metrics.rs

@@ -479,3 +479,31 @@ pub static MPC_TEE_ATTESTATION_ATTEMPTS_TOTAL: LazyLock<prometheus::IntCounterVe
 
 pub const MPC_TEE_ATTESTATION_OUTCOME_SUCCESS: &str = "success";
 pub const MPC_TEE_ATTESTATION_OUTCOME_FAILURE: &str = "failure";
+
+pub static MPC_ATTESTATION_EXPIRY_TIMESTAMP_SECONDS: LazyLock<prometheus::IntGauge> =
+    LazyLock::new(|| {
+        prometheus::register_int_gauge!(
+            "mpc_attestation_expiry_timestamp_seconds",
+            "NEAR block time at which the attestation stored on chain for this node's TLS key \
+             expires. -1 if the stored attestation carries no expiry; 0 if none is stored. \
+             Subtract mpc_indexer_latest_block_timestamp_seconds, not wall clock, for the \
+             remaining time"
+        )
+        .unwrap()
+    });
+
+pub static MPC_ATTESTATION_LAST_LANDED_TIMESTAMP_SECONDS: LazyLock<prometheus::IntGauge> =
+    LazyLock::new(|| {
+        prometheus::register_int_gauge!(
+            "mpc_attestation_last_landed_timestamp_seconds",
+            "Unix time, by this node's own clock, at which it last confirmed an attestation \
+             submission landed on chain"
+        )
+        .unwrap()
+    });
+
+/// An alert cannot fire on a series that does not exist yet.
+pub fn init_attestation_freshness_metrics() {
+    LazyLock::force(&MPC_ATTESTATION_EXPIRY_TIMESTAMP_SECONDS);
+    LazyLock::force(&MPC_ATTESTATION_LAST_LANDED_TIMESTAMP_SECONDS);
+}

crates/node/src/tee.rs

@@ -1,4 +1,5 @@
 pub mod allowed_image_hashes_watcher;
+pub mod attestation_freshness_metrics;
 pub mod image_expiry_metrics;
 pub mod remote_attestation;
 

crates/node/src/tee/attestation_freshness_metrics.rs

@@ -0,0 +1,84 @@
+//! Attestation-freshness gauges.
+//!
+//! Both are absolute timestamps rather than remaining durations: a node that stops re-attesting
+//! also stops updating them, and only the absolute form keeps decaying towards now, so a staleness
+//! alert still fires on a frozen gauge. At rest both advance hourly — the expiry is re-read on
+//! every submission observation, the landing timestamp only when one is confirmed.
+
+use near_mpc_contract_interface::types::VerifiedAttestation;
+use near_time::Clock;
+
+use crate::metrics::{
+    MPC_ATTESTATION_EXPIRY_TIMESTAMP_SECONDS, MPC_ATTESTATION_LAST_LANDED_TIMESTAMP_SECONDS,
+};
+
+const NO_ATTESTATION_STORED: i64 = 0;
+const NO_EXPIRY: i64 = -1;
+
+pub(crate) fn record_stored_attestation_expiry(stored: Option<&VerifiedAttestation>) {
+    MPC_ATTESTATION_EXPIRY_TIMESTAMP_SECONDS.set(expiry_gauge_value(stored));
+}
+
+pub(crate) fn record_attestation_landed(clock: &Clock) {
+    MPC_ATTESTATION_LAST_LANDED_TIMESTAMP_SECONDS.set(clock.now_utc().unix_timestamp());
+}
+
+fn expiry_gauge_value(stored: Option<&VerifiedAttestation>) -> i64 {
+    match stored.map(VerifiedAttestation::expiry_timestamp_seconds) {
+        None => NO_ATTESTATION_STORED,
+        Some(None) => NO_EXPIRY,
+        Some(Some(expiry)) => i64::try_from(expiry).unwrap_or(i64::MAX),
+    }
+}
+
+#[cfg(test)]
+#[expect(non_snake_case)]
+mod tests {
+    use super::*;
+    use near_mpc_contract_interface::types::MockAttestation;
+    use near_time::{FakeClock, Utc};
+    use rstest::rstest;
+
+    const EXPIRES_AT: u64 = 1_754_000_000;
+
+    fn mock_with_expiry(expiry_timestamp_seconds: Option<u64>) -> VerifiedAttestation {
+        VerifiedAttestation::Mock(MockAttestation::WithConstraints {
+            mpc_docker_image_hash: None,
+            launcher_docker_compose_hash: None,
+            expiry_timestamp_seconds,
+            expected_measurements: None,
+        })
+    }
+
+    #[test]
+    fn record_attestation_landed__should_set_the_gauge_to_the_landing_time() {
+        // Given
+        let landed_at = Utc::from_unix_timestamp(EXPIRES_AT as i64).unwrap();
+
+        // When
+        record_attestation_landed(&FakeClock::new(landed_at).clock());
+
+        // Then
+        assert_eq!(
+            MPC_ATTESTATION_LAST_LANDED_TIMESTAMP_SECONDS.get(),
+            EXPIRES_AT as i64
+        );
+    }
+
+    #[rstest]
+    #[case::nothing_stored(None, NO_ATTESTATION_STORED)]
+    #[case::stored_with_expiry(Some(mock_with_expiry(Some(EXPIRES_AT))), EXPIRES_AT as i64)]
+    #[case::stored_without_expiry(Some(mock_with_expiry(None)), NO_EXPIRY)]
+    #[case::unstamped_mock(Some(VerifiedAttestation::Mock(MockAttestation::Valid)), NO_EXPIRY)]
+    #[case::expiry_beyond_i64(Some(mock_with_expiry(Some(u64::MAX))), i64::MAX)]
+    fn expiry_gauge_value__should_report_stored_expiry_or_a_sentinel(
+        #[case] stored: Option<VerifiedAttestation>,
+        #[case] expected: i64,
+    ) {
+        // When
+        let value = expiry_gauge_value(stored.as_ref());
+
+        // Then
+        assert_eq!(value, expected);
+    }
+}

crates/node/src/web.rs

@@ -49,6 +49,7 @@ impl IntoResponse for AnyhowErrorWrapper {
 pub(crate) async fn metrics() -> String {
     // Ensure build info metric is always set before gathering metrics
     crate::metrics::init_build_info_metric();
+    crate::metrics::init_attestation_freshness_metrics();
 
     let metric_families = default_registry().gather();
     let mut buffer = vec![];

docs/design/node-operator-metrics.md

@@ -30,6 +30,15 @@ serves keyshares over the migration service to the backup service registered for
 | [`mpc_last_backup_served_timestamp_seconds`](../../crates/node/src/metrics.rs) | Unix time of the last keyshare set served to the backup service | should be recent. A large gap since the last resharing means backups are not being taken. Confirms the node served the keyshares, not that the backup service persisted them. |
 | [`mpc_current_epoch_id`](../../crates/node/src/metrics.rs) | epoch id of the keyset the contract currently holds | the reference point for `mpc_last_backup_served_epoch`. Increments on every resharing; unset until the first keyset exists. During a resharing it stays at the old epoch, which is the one still available to back up. |
 
+Attestation freshness, in [`metrics.rs`](../../crates/node/src/metrics.rs). Set from the
+on-chain confirmation of an attestation submission, in
+[`attestation_freshness_metrics.rs`](../../crates/node/src/tee/attestation_freshness_metrics.rs):
+
+| Metric | Measures | How to interpret |
+| --- | --- | --- |
+| [`mpc_attestation_last_landed_timestamp_seconds`](../../crates/node/src/metrics.rs) | Unix time of the last attestation submission this node confirmed on chain | should be under an `ATTESTATION_RESUBMISSION_INTERVAL` (1h) old. A sustained gap ends in this node being dropped from the participant set. |
+| [`mpc_attestation_expiry_timestamp_seconds`](../../crates/node/src/metrics.rs) | NEAR block time at which the attestation the contract stores for this node's TLS key expires | subtract `mpc_indexer_latest_block_timestamp_seconds` — the clock the contract expires entries against, not wall clock — for the runway before this node is dropped from the participant set. `0` = nothing stored (evicted, or never landed one), `-1` = stored without an expiry. |
+
 ## Recommended alerts
 
 ```promql
@@ -55,4 +64,26 @@ time() - mpc_last_backup_served_timestamp_seconds > 86400  for 1h
 # been served to the backup service since. Also fires if the node has never served
 # a backup, since the gauge starts at 0.
 mpc_last_backup_served_epoch < mpc_current_epoch_id  for 1h
+
+# Re-attestation stuck (warn): nothing landed in three re-attestation intervals.
+# The primary signal — it fires within hours, while the expiry alert below only
+# does so days later.
+time() - mpc_attestation_last_landed_timestamp_seconds > 3 * 3600  for 15m
+
+# Attestation runway low (page): under 3 days before the contract drops this node
+# from the participant set. Backstop for the alert above, and reached only about
+# four days after submissions stop landing. The threshold is a plain duration
+# rather than a fraction of the expiry window (7 days today): keep it below the
+# window, and revisit if the window changes. `> 0` drops the sentinels so they are
+# not read as timestamps.
+mpc_attestation_expiry_timestamp_seconds > 0
+  and mpc_attestation_expiry_timestamp_seconds - mpc_indexer_latest_block_timestamp_seconds
+      < 3 * 86400  for 15m
+
+# No attestation stored (page): the contract holds nothing for our TLS key, so this
+# node is out of the attested set. Also covers a node that never landed one.
+mpc_attestation_expiry_timestamp_seconds == 0  for 15m
 ```
+
+A `-1` expiry satisfies neither expiry alert, so a node holding an attestation stored
+without one is covered by the staleness alert alone.