feat(node): repeat the foreign chain probe every hour

A verdict taken once at boot goes stale: a provider can start serving
another network, or go down, while the node is up. The probe now runs on a
ticker and the gauges carry the latest round.

Tick and its test double move out of remote_attestation into a shared tick
module, so both periodic loops use one abstraction.

Author: haiyuechen-nearone

crates/node/src/foreign_chain_probe.rs

@@ -1,7 +1,8 @@
-//! Startup probe of every configured foreign-chain RPC provider, via
+//! Periodic probe of every configured foreign-chain RPC provider, via
 //! [`foreign_chain_health_check::probe`].
 
 use std::collections::BTreeSet;
+use std::future::Future;
 
 use foreign_chain_health_check::probe::{
     ProbeReport, ProviderHealth, ProviderStatus, probe_all_providers,
@@ -11,19 +12,31 @@ use near_mpc_contract_interface::types as dtos;
 use tracing::{info, warn};
 
 use crate::metrics;
+use crate::tick::Tick;
 
-/// Asks every configured RPC provider which network it serves and reports the verdicts as logs and
-/// metrics. Diagnostic only: nothing gates on the result.
-pub async fn run_startup_probe(foreign_chains: ForeignChainsConfig) {
+/// Asks every configured RPC provider which network it serves, once per tick of `ticker`, and
+/// reports the verdicts as logs and metrics. Diagnostic only: nothing gates on the result.
+pub async fn run_periodic_probe(foreign_chains: ForeignChainsConfig, ticker: impl Tick) {
     if foreign_chains.is_empty() {
         warn!("no foreign chain is configured: this node cannot verify foreign-chain transactions");
         return;
     }
 
-    info!("probing foreign-chain RPC providers");
-    let report = probe_all_providers(&foreign_chains).await;
-    publish_metrics(&report);
-    log_report(&report);
+    probe_periodically(|| probe_all_providers(&foreign_chains), ticker).await;
+}
+
+async fn probe_periodically<Probe: Future<Output = ProbeReport>>(
+    probe: impl Fn() -> Probe,
+    mut ticker: impl Tick,
+) {
+    loop {
+        ticker.tick().await;
+
+        info!("probing foreign-chain RPC providers");
+        let report = probe().await;
+        publish_metrics(&report);
+        log_report(&report);
+    }
 }
 
 fn is_probed(row: &ProviderHealth) -> bool {
@@ -71,7 +84,7 @@ fn log_report(report: &ProbeReport) {
         let chains: BTreeSet<&str> = rows.iter().map(|row| row.chain.label()).collect();
         warn!(
             ?chains,
-            "no RPC provider was checked at startup: none of the configured foreign chains has a probe"
+            "no RPC provider was checked: none of the configured foreign chains has a probe"
         );
         return;
     }
@@ -103,8 +116,12 @@ fn publish_metrics(report: &ProbeReport) {
 #[expect(non_snake_case)]
 mod tests {
     use super::*;
+    use crate::async_testing::{MaybeReady, run_future_once};
+    use crate::tick::MockTicker;
     use foreign_chain_health_check::probe::ProviderCounts;
     use prometheus::core::Collector as _;
+    use std::cell::{Cell, RefCell};
+    use std::collections::VecDeque;
 
     fn labelled_chains(gauge: &prometheus::IntGaugeVec) -> BTreeSet<String> {
         gauge
@@ -243,4 +260,83 @@ mod tests {
         assert!(chains.contains("bnb"));
         assert!(!chains.contains("solana"));
     }
+
+    #[test]
+    fn probe_periodically__should_probe_once_per_tick() {
+        // Given
+        let probe_count = Cell::new(0);
+        let probe = || {
+            probe_count.set(probe_count.get() + 1);
+            std::future::ready(ProbeReport::from(vec![]))
+        };
+
+        // When
+        run_future_once(probe_periodically(probe, MockTicker::new(3)));
+
+        // Then
+        assert_eq!(probe_count.get(), 3);
+    }
+
+    #[test]
+    fn probe_periodically__should_replace_the_gauges_of_the_previous_round() {
+        // Given
+        let rounds = RefCell::new(VecDeque::from([
+            ProbeReport::from(vec![row(
+                dtos::ForeignChain::Starknet,
+                "only",
+                ProviderStatus::Healthy,
+            )]),
+            ProbeReport::from(vec![row(
+                dtos::ForeignChain::Starknet,
+                "only",
+                ProviderStatus::Unreachable,
+            )]),
+        ]));
+        let probe_dispatch =
+            || std::future::ready(rounds.borrow_mut().pop_front().expect("a report per tick"));
+        let ticker = MockTicker::new(1);
+
+        // When
+        let MaybeReady::Future(parked_probe_loop) =
+            run_future_once(probe_periodically(probe_dispatch, ticker.clone()))
+        else {
+            panic!("the loop should park once its ticker runs out");
+        };
+        let metrics_after_the_first_round = gauges("starknet");
+        ticker.schedule(1);
+        run_future_once(parked_probe_loop);
+
+        // Then
+        assert_eq!(
+            metrics_after_the_first_round,
+            ProviderCounts {
+                configured: 1,
+                healthy: 1
+            }
+        );
+        assert_eq!(
+            gauges("starknet"),
+            ProviderCounts {
+                configured: 1,
+                healthy: 0
+            }
+        );
+    }
+
+    #[test]
+    fn run_periodic_probe__should_stop_when_no_foreign_chain_is_configured() {
+        // Given
+        let foreign_chains = ForeignChainsConfig::default();
+        let ticker = MockTicker::new(1);
+
+        // When
+        let outcome = run_future_once(run_periodic_probe(foreign_chains, ticker.clone()));
+
+        // Then
+        assert_eq!(ticker.unspent(), 1, "no round should have run");
+        assert!(
+            matches!(outcome, MaybeReady::Ready(())),
+            "the probe should return rather than park on its ticker"
+        );
+    }
 }

crates/node/src/lib.rs

@@ -58,4 +58,5 @@ mod storage;
 mod tee;
 #[cfg(test)]
 mod tests;
+mod tick;
 mod tracking;

crates/node/src/metrics.rs

@@ -494,7 +494,7 @@ pub static FOREIGN_CHAIN_RPC_PROVIDERS_HEALTHY: LazyLock<prometheus::IntGaugeVec
     LazyLock::new(|| {
         prometheus::register_int_gauge_vec!(
             "mpc_foreign_chain_rpc_providers_healthy",
-            "RPC providers that served the expected network at startup",
+            "RPC providers that served the expected network at the latest probe",
             &["chain"],
         )
         .unwrap()

crates/node/src/run.rs

@@ -50,6 +50,7 @@ use crate::tee::{
 };
 
 pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
+pub const FOREIGN_CHAIN_PROBE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
 
 pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> {
     init_logging(&config.log);
@@ -207,8 +208,9 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> {
     let _web_server_join_handle = root_runtime.spawn(web_server);
 
     // Detached: the report is diagnostic, nothing downstream waits on it.
-    root_runtime.spawn(crate::foreign_chain_probe::run_startup_probe(
+    root_runtime.spawn(crate::foreign_chain_probe::run_periodic_probe(
         node_config.foreign_chains.clone(),
+        tokio::time::interval(FOREIGN_CHAIN_PROBE_INTERVAL),
     ));
 
     // Create Indexer and wait for indexer to be synced.

crates/node/src/tee/remote_attestation.rs

@@ -5,6 +5,7 @@ use crate::{
         tx_sender::{TransactionSender, TransactionStatus},
         types::ChainSendTransactionRequest,
     },
+    tick::Tick,
     trait_extensions::convert_to_contract_dto::IntoContractInterfaceType,
 };
 use anyhow::Context;
@@ -362,21 +363,11 @@ pub async fn monitor_attestation_removal<T: TransactionSender + Clone>(
     Ok(())
 }
 
-/// Allows repeatedly awaiting for something, like a [`tokio::time::Interval`].
-pub trait Tick {
-    async fn tick(&mut self);
-}
-
-impl Tick for tokio::time::Interval {
-    async fn tick(&mut self) {
-        self.tick().await;
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::indexer::tx_sender::{TransactionProcessorError, TransactionStatus};
+    use crate::tick::MockTicker;
     use ed25519_dalek::SigningKey;
     use rand::SeedableRng;
     use std::sync::{Arc, Mutex};
@@ -386,26 +377,6 @@ mod tests {
     const TEST_EXPECTED_ATTESTATION_RESUBMISSION_TIMEOUT: Duration = Duration::from_millis(100);
     const TEST_VERIFY_NO_ATTESTATION_RESUBMISSION_TIMEOUT: Duration = Duration::from_millis(100);
 
-    struct MockTicker {
-        count: usize,
-    }
-
-    impl MockTicker {
-        fn new(count: usize) -> Self {
-            Self { count }
-        }
-    }
-
-    impl Tick for MockTicker {
-        async fn tick(&mut self) {
-            if self.count > 0 {
-                self.count -= 1;
-            } else {
-                std::future::pending::<()>().await;
-            }
-        }
-    }
-
     struct StubAttestationExpiryReader;
 
     impl crate::indexer::ReadAttestationExpiry for StubAttestationExpiryReader {

crates/node/src/tick.rs

@@ -0,0 +1,56 @@
+//! Waiting for the next round of a periodic task.
+
+#[cfg(test)]
+use std::sync::Arc;
+#[cfg(test)]
+use tokio::sync::Semaphore;
+
+/// Allows repeatedly awaiting for something, like a [`tokio::time::Interval`].
+pub trait Tick {
+    async fn tick(&mut self);
+}
+
+impl Tick for tokio::time::Interval {
+    async fn tick(&mut self) {
+        self.tick().await;
+    }
+}
+
+/// Advances the loop under test by a counted number of rounds rather than by elapsed time.
+#[cfg(test)]
+#[derive(Clone)]
+pub struct MockTicker {
+    scheduled: Arc<Semaphore>,
+}
+
+#[cfg(test)]
+impl MockTicker {
+    pub fn new(count: usize) -> Self {
+        Self {
+            scheduled: Arc::new(Semaphore::new(count)),
+        }
+    }
+
+    /// Lets the loop run `count` more rounds, from its next poll onwards.
+    pub fn schedule(&self, count: usize) {
+        self.scheduled.add_permits(count);
+    }
+
+    /// Rounds scheduled but not yet taken by a loop.
+    pub fn unspent(&self) -> usize {
+        self.scheduled.available_permits()
+    }
+}
+
+#[cfg(test)]
+impl Tick for MockTicker {
+    async fn tick(&mut self) {
+        let round = self
+            .scheduled
+            .acquire()
+            .await
+            .expect("the Semaphore is never closed");
+        // Spend the round.
+        round.forget();
+    }
+}

docs/foreign-chain-transactions.md

@@ -409,7 +409,7 @@ The per-participant registration model above leaves the network with no shared n
 | What the operator picks                | Full URL, auth scheme, token reference                                                      | `provider_id` (label) + token reference                                                                                                                                                        |
 | Adding a new provider                  | Every operator updates their yaml; the network effectively supports a chain once enough do  | Threshold of participants vote in `(chain, ProviderEntry)`; operators reference it by `provider_id` only                                                                                       |
 | Removing a compromised provider        | Every operator manually edits their yaml; coordination problem                              | Threshold of participants vote remove; nodes pick up the change via the indexer and drop the provider on next reconfigure                                                                       |
-| Testnet vs mainnet separation          | Implicit — operator decides what URL goes under which chain                                 | Per-`ForeignChain` map slot, plus a startup *network fingerprint probe* that calls the chain's self-identifying RPC and compares the response against the chain's `expected_network_fingerprint` from operator config — catches both lookup-level (wrong bucket) and content-level (wrong URL voted into the right bucket) confusion. |
+| Testnet vs mainnet separation          | Implicit — operator decides what URL goes under which chain                                 | Per-`ForeignChain` map slot, plus a recurring *network fingerprint probe* that calls the chain's self-identifying RPC and compares the response against the chain's `expected_network_fingerprint` from operator config — catches both lookup-level (wrong bucket) and content-level (wrong URL voted into the right bucket) confusion. |
 
 ### Whitelist storage shape
 
@@ -565,9 +565,9 @@ Voting uses the protocol's existing signing threshold (`self.threshold()?.value(
 
 The per-chain map key prevents *lookup* confusion: when the node resolves the operator's `ethereum:` section, only `entries[Ethereum]` is consulted, never `entries[Sepolia]`. What it doesn't prevent is a `ChainVote { chain: Ethereum, providers: [ProviderEntry { provider_id: "ankr", chain_routing: PathSegment { segment: "eth_sepolia" }, … }, …], threshold: _ }` getting voted in — the contract just stores what threshold consensus produces; it can't tell whether `"eth_sepolia"` actually corresponds to Ethereum mainnet. Threshold voter review is the first line of defense; the fan-out across a chain's providers is the structural one. The network fingerprint probe is a per-node diagnostic on top of both.
 
-At startup, every provider of a chain the node can identify gets its self-identifying RPC called and the response is compared against that chain's `expected_network_fingerprint` from the operator's config. The probe is report-only: a provider serving the wrong network is logged, but is not dropped, because a boot-time network blip should not take a chain out of signing. It runs detached, so it never delays startup. A node with no foreign chain configured at all is warned instead, because it cannot verify foreign-chain transactions.
+At startup and every hour after that, every provider of a chain the node can identify gets its self-identifying RPC called and the response is compared against that chain's `expected_network_fingerprint` from the operator's config. The probe is report-only: a provider serving the wrong network is logged, but is not dropped, because a network blip should not take a chain out of signing. It runs detached, so it never delays startup. Repeating it turns a single snapshot taken at boot into a signal that also catches a provider that starts serving another network, or goes down, while the node is up. A node with no foreign chain configured at all is warned once at startup instead, because it cannot verify foreign-chain transactions.
 
-The result is a line per provider, an `x/y providers healthy` summary counting only the providers a probe covers, and two gauges labelled by chain: `mpc_foreign_chain_rpc_providers_configured` and `mpc_foreign_chain_rpc_providers_healthy`. The gauges are per chain rather than per provider, because a provider name is operator chosen and would put an unbounded label on a time series. A chain whose providers cannot be identified, such as Solana, is left out of both the summary and the gauges: reporting `0` healthy against its configured count would read as every provider failing.
+The result of each round is a line per provider, an `x/y providers healthy` summary counting only the providers a probe covers, and two gauges labelled by chain, which carry the verdicts of the round that ran last: `mpc_foreign_chain_rpc_providers_configured` and `mpc_foreign_chain_rpc_providers_healthy`. The gauges are per chain rather than per provider, because a provider name is operator chosen and would put an unbounded label on a time series. A chain whose providers cannot be identified, such as Solana, is left out of both the summary and the gauges: reporting `0` healthy against its configured count would read as every provider failing.
 
 Taking the expected value from operator config rather than a constant in the attested binary is a deliberate trade. It makes mixed-network and local deployments checkable at all, since a config may pair one chain's mainnet with another's testnet and no binary can ship a value for a devnet. The cost is that the check no longer binds an operator: they can set the wrong value, or omit the field and get no check at all, and either way they fool only their own node's diagnostics. The network-level defenses against a wrong URL are unchanged: threshold voter review of the whitelist, and the provider fan-out, which fails the individual request when a provider disagrees with its siblings.