fix(guest): make the gateway checker loop self-recovering

Two changes, one dropping state and one closing the last failure mode
that had no automatic recovery.

Drop reported_degraded. Mirroring gateway state to the host meant the
loop had to remember what the host had been told, and that memory bought
nothing the loop needs: recovery does not depend on it. Boot still
reports the one signal that matters -- this CVM came up without a route.
The cost is that the reported error stays on the VMM after the checker
recovers, until the VM restarts, which is the intended trade.

Add a systemd watchdog. Every recovery path here only runs while the
loop runs, and nothing below the loop could guarantee that: a refresh
that never returns leaves a healthy-looking process that Restart= would
never fire on, and the CVM sits without a route indefinitely. That is
not hypothetical -- a refresh spends most of its time in blocking cmd!
shell-outs, and wg-quick resolves peer endpoints, so a hung DNS lookup
is enough.

tokio::time::timeout cannot fix this: those calls block a runtime worker
with no await point to cancel at. Neither can a watchdog task on another
worker, which would keep pinging while this loop is wedged. The ping has
to come from the loop itself, before the work rather than after, so that
a refresh which never returns stops the pings. systemd then kills and
restarts the service, covering a hang whatever its cause.

Follows the Type=notify + WatchdogSec + sd_notify pattern already used
by dstack-guest-agent. Readiness is reported before the checker decides
whether it has anything to do, so the exit-0 (gateway disabled) and
exit-3 (misconfigured) paths remain a started service that then stopped.
WatchdogSec=600 clears the worst legitimate refresh, which can span KMS
certificate requests plus every configured gateway URL at 60s each.

Verified end to end against a stub notify socket: the process emits
READY=1 at startup and WATCHDOG=1 on each iteration, and stays inert
when the unit does not arm the watchdog, so manual runs still work.

Author: kvinwang

dstack/Cargo.lock

@@ -25,6 +25,7 @@ serde.workspace = true
 serde-human-bytes.workspace = true
 semver.workspace = true
 serde_json.workspace = true
+sd-notify.workspace = true
 sha2.workspace = true
 tokio = { workspace = true, features = ["full"] }
 tracing.workspace = true

dstack/dstack-util/Cargo.toml

@@ -33,11 +33,10 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{Context, Result};
 use cmd_lib::run_fun as cmd;
+use sd_notify::NotifyState;
 use tracing::{error, info, warn};
 
-use crate::system_setup::{
-    gateway_unavailable_message, GatewayRefresher, WG_CONFIG_PATH, WG_INTERFACE,
-};
+use crate::system_setup::{GatewayRefresher, WG_CONFIG_PATH, WG_INTERFACE};
 
 /// How often the loop samples the world.
 const POLL_INTERVAL: Duration = Duration::from_secs(10);
@@ -279,7 +278,45 @@ fn observe(now: i64) -> Observation {
     }
 }
 
+/// systemd liveness reporting, inert when the unit has no `WatchdogSec`.
+///
+/// The recovery paths in this loop only work while the loop runs, and nothing
+/// below it can guarantee that: a wedged refresh leaves a healthy-looking
+/// process that systemd will never restart. Handing liveness to systemd covers
+/// a hang wherever it comes from, including causes not anticipated here.
+struct Watchdog {
+    enabled: bool,
+}
+
+impl Watchdog {
+    /// Report readiness and arm the watchdog. Readiness is sent before the
+    /// checker decides whether it has anything to do, so the paths that exit
+    /// straight away are still a started service that then stopped, not a
+    /// service that failed to start.
+    fn arm() -> Self {
+        let mut usec = 0;
+        let enabled = sd_notify::watchdog_enabled(false, &mut usec);
+        if let Err(error) = sd_notify::notify(false, &[NotifyState::Ready]) {
+            warn!("failed to report readiness to systemd: {error}");
+        }
+        if enabled {
+            info!("systemd watchdog armed, timeout={usec}us");
+        }
+        Self { enabled }
+    }
+
+    fn ping(&self) {
+        if !self.enabled {
+            return;
+        }
+        if let Err(error) = sd_notify::notify(false, &[NotifyState::Watchdog]) {
+            warn!("failed to ping the systemd watchdog: {error}");
+        }
+    }
+}
+
 pub async fn cmd_gateway_checker(args: GatewayCheckerArgs) -> Result<()> {
+    let watchdog = Watchdog::arm();
     let refresher =
         GatewayRefresher::load(&args.work_dir).context("failed to load gateway configuration")?;
 
@@ -306,41 +343,33 @@ pub async fn cmd_gateway_checker(args: GatewayCheckerArgs) -> Result<()> {
     }
 
     info!("watching dstack-gateway registration");
-    let vmm = refresher.host_api();
-    // Seed from the observable world rather than assuming health: no WireGuard
-    // config here means boot-time registration failed and already reported it,
-    // so the first success owes the host a retraction. Assuming health instead
-    // would leave that boot error on screen forever after we recover.
-    //
-    // Reporting is deliberately fire-and-forget. Tracking delivery would mean
-    // carrying a third state ("degraded, and the host may or may not know")
-    // through the loop to cover a host-API blip that the next refresh cycle
-    // already re-reports on the way in or out of the degraded state.
-    let config_present = wg_config_present();
-    let mut reported_degraded = !config_present;
-    let mut checker = Checker::starting(now_secs(), config_present);
+    // The checker does not report gateway state to the host. Boot already
+    // reports the one signal that matters -- this CVM came up without a route
+    // -- and mirroring every later transition would mean tracking what the host
+    // has been told, which is state this loop should not have to carry. The
+    // consequence is that a boot error stays on the VMM after the checker
+    // recovers, until the VM restarts.
+    let mut checker = Checker::starting(now_secs(), wg_config_present());
     loop {
+        // Ping before the work, not after, so a refresh that never returns
+        // stops the pings. Nothing else can do this for us: a refresh spends
+        // most of its time in blocking `cmd!` shell-outs (`wg-quick up` alone
+        // resolves peer endpoints), which occupy a runtime worker with no await
+        // point. tokio::time::timeout cannot cancel that, and a watchdog task
+        // on another worker would happily keep pinging while this loop is
+        // wedged. Only the loop itself can prove the loop is alive.
+        watchdog.ping();
+
         let now = now_secs();
         if let Some(refresh) = checker.decide(observe(now)) {
             info!("refreshing dstack-gateway: {}", refresh.reason);
             let succeeded = match refresher.refresh(refresh.force).await {
                 Ok(()) => {
                     info!("dstack-gateway refresh succeeded");
-                    if reported_degraded {
-                        info!("dstack-gateway route restored; clearing the reported error");
-                        // Empty body resets the host's boot_error field.
-                        vmm.notify_q("boot.error", "").await;
-                        reported_degraded = false;
-                    }
                     true
                 }
                 Err(error) => {
                     warn!("dstack-gateway refresh failed: {error:#}");
-                    if !reported_degraded {
-                        vmm.notify_q("boot.error", &gateway_unavailable_message(&error))
-                            .await;
-                        reported_degraded = true;
-                    }
                     false
                 }
             };

dstack/dstack-util/src/gateway_checker.rs

@@ -1936,14 +1936,6 @@ impl Stage0<'_> {
     }
 }
 
-/// The message reported to the host while this CVM has no gateway route.
-///
-/// Boot and the gateway checker share it so the operator sees one consistent
-/// string no matter which of the two noticed the outage.
-pub fn gateway_unavailable_message(error: &anyhow::Error) -> String {
-    format!("dstack-gateway registration failed, the app has no ingress route: {error:#}")
-}
-
 /// Owns the inputs needed to (re)register this CVM with dstack-gateway.
 ///
 /// Loading is separated from refreshing so a long-running caller (the gateway
@@ -1974,14 +1966,6 @@ impl GatewayRefresher {
         self.shared.app_compose.gateway_enabled()
     }
 
-    /// Client for reporting guest state back to the host.
-    pub fn host_api(&self) -> HostApi {
-        HostApi::new(
-            self.shared.sys_config.host_api_url.clone(),
-            self.shared.sys_config.collateral_urls().pccs,
-        )
-    }
-
     /// Validate the parts of the gateway config that can never become valid by
     /// waiting. These are deployment mistakes, not outages, so callers that
     /// retry should give up instead of looping forever.
@@ -2923,7 +2907,12 @@ impl Stage1<'_> {
             // visible from the VMM. The gateway checker clears this once it
             // manages to register.
             self.vmm
-                .notify_q("boot.error", &gateway_unavailable_message(&error))
+                .notify_q(
+                    "boot.error",
+                    &format!(
+                        "dstack-gateway registration failed, the app has no ingress route: {error:#}"
+                    ),
+                )
                 .await;
         }
         self.vmm

dstack/dstack-util/src/system_setup.rs

@@ -4,8 +4,16 @@ After=network-online.target dstack-prepare.service
 Wants=network-online.target
 
 [Service]
-Type=simple
+Type=notify
 ExecStart=/bin/dstack-util gateway-checker --work-dir /dstack
+# Every recovery path in the checker only runs while its loop runs, and a loop
+# that wedges leaves a healthy-looking process systemd would never restart. The
+# loop pings the watchdog itself, so a hang anywhere -- including the blocking
+# wg-quick/iptables shell-outs a refresh performs, which no in-process timeout
+# can cancel -- gets the service killed and restarted. The timeout is generous
+# because one refresh may legitimately spend minutes across KMS certificate
+# requests and every configured gateway URL.
+WatchdogSec=600
 # The checker exits 0 when the app never enabled dstack-gateway, because there
 # is then nothing to supervise. Restart=always would respawn that exit forever.
 Restart=on-failure

os/common/rootfs/dstack-gateway-checker.service

@@ -86,6 +86,12 @@ exit_code=$(sed -n 's/^const EXIT_MISCONFIGURED: i32 = \([0-9]\+\);$/\1/p' "$che
 [[ -n $exit_code ]] || { echo "cannot read EXIT_MISCONFIGURED from $checker_src"; exit 1; }
 grep -q "^RestartPreventExitStatus=${exit_code}\$" "$gw_unit" || {
   echo "dstack-gateway-checker.service must set RestartPreventExitStatus=$exit_code"; exit 1; }
+# The loop's recovery paths only run while the loop runs, so systemd has to be
+# the thing that notices a wedge. WatchdogSec is useless without Type=notify.
+grep -q '^Type=notify$' "$gw_unit" || {
+  echo 'dstack-gateway-checker.service must use Type=notify to arm the watchdog'; exit 1; }
+grep -q '^WatchdogSec=' "$gw_unit" || {
+  echo 'dstack-gateway-checker.service must set WatchdogSec'; exit 1; }
 test ! -e "$D/../common/rootfs/wg-checker.sh"
 test ! -e "$D/../common/rootfs/wg-checker.service"
 # systemd enables any unit that matches no preset rule, so the enable list is