Quicken transiently skipped heartbeats

Author: jcorbin

crates/core/src/heartbeat/events.rs

@@ -13,6 +13,8 @@ pub enum HeartbeatStatus {
     Ok,
     /// Heartbeat was skipped (outside active hours, empty file, etc.)
     Skipped,
+    /// Heartbeat was skipped transiently (a soon retry may be useful)
+    SkippedMayTry,
     /// Heartbeat failed with an error
     Failed,
 }

crates/core/src/heartbeat/runner.rs

@@ -132,6 +132,11 @@ impl HeartbeatRunner {
             first_at,
         );
 
+        // Exponential backoff for SkippedMayTry retries
+        let mut skips_since_last = 0;
+        let skip_retry_base = Duration::from_millis(1000);
+        let skip_retry_max = self.interval / 2;
+
         loop {
             interval.tick().await; // Sleep until next interval
 
@@ -170,6 +175,16 @@ impl HeartbeatRunner {
                         warn!(name: "Heartbeat", "response not OK: {}", response);
                     }
 
+                    if status == HeartbeatStatus::SkippedMayTry {
+                        skips_since_last += 1;
+                        let retry_after =
+                            (skip_retry_base * 2_u32.pow(skips_since_last)).min(skip_retry_max);
+                        interval.reset_after(retry_after);
+                        info!(name: "Heartbeat", "transient skip, retry quickly after: {:?}", retry_after);
+                    } else {
+                        skips_since_last = 0;
+                    }
+
                     HeartbeatEvent {
                         ts: now_ms(),
                         status: status.clone(),
@@ -248,15 +263,21 @@ impl HeartbeatRunner {
             && gate.is_busy()
         {
             info!(name: "Heartbeat", "skipping: agent turn in flight (TurnGate busy)");
-            return Ok((HEARTBEAT_OK_TOKEN.to_string(), HeartbeatStatus::Skipped));
+            return Ok((
+                HEARTBEAT_OK_TOKEN.to_string(),
+                HeartbeatStatus::SkippedMayTry,
+            ));
         }
 
         // Try to acquire the cross-process workspace lock (non-blocking)
         let _ws_guard = match self.workspace_lock.try_acquire()? {
             Some(guard) => guard,
             None => {
                 info!(name: "Heartbeat", "skipping: workspace locked by another process");
-                return Ok((HEARTBEAT_OK_TOKEN.to_string(), HeartbeatStatus::Skipped));
+                return Ok((
+                    HEARTBEAT_OK_TOKEN.to_string(),
+                    HeartbeatStatus::SkippedMayTry,
+                ));
             }
         };
 
@@ -267,7 +288,10 @@ impl HeartbeatRunner {
                 Some(permit) => Some(permit),
                 None => {
                     info!(name: "Heartbeat", "skipping: agent turn started between check and acquire");
-                    return Ok((HEARTBEAT_OK_TOKEN.to_string(), HeartbeatStatus::Skipped));
+                    return Ok((
+                        HEARTBEAT_OK_TOKEN.to_string(),
+                        HeartbeatStatus::SkippedMayTry,
+                    ));
                 }
             }
         } else {

crates/server/src/http.rs

@@ -1064,6 +1064,7 @@ async fn heartbeat_status(State(state): State<Arc<AppState>>) -> Json<HeartbeatS
             HeartbeatStatus::Sent => "sent",
             HeartbeatStatus::Ok => "ok",
             HeartbeatStatus::Skipped => "skipped",
+            HeartbeatStatus::SkippedMayTry => "skipped",
             HeartbeatStatus::Failed => "failed",
         };