Skip to content

Commit 02c2c4b

Browse files
committed
fix: report dead-task respawns as restarts
1 parent 33d159b commit 02c2c4b

4 files changed

Lines changed: 77 additions & 9 deletions

File tree

src/eval_run.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -824,7 +824,10 @@ fn run_eval_inner(spec: &Spec, eval: &Eval, spec_dir: &Path, catalog: &Path, hos
824824
),
825825
};
826826
if !report.launched.is_empty() {
827-
eval_log!("== supervise: respawned {:?} from spec ==", report.launched);
827+
eval_log!("== supervise: launched {:?} from spec ==", report.launched);
828+
}
829+
if !report.restarted.is_empty() {
830+
eval_log!("== supervise: restarted {:?} from spec ==", report.restarted);
828831
}
829832
}
830833
};

src/main.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1845,6 +1845,7 @@ fn up(
18451845

18461846
fn print_report(report: &UpReport) {
18471847
report_line("launched", &report.launched);
1848+
report_line("restarted", &report.restarted);
18481849
report_line("torn down", &report.torn_down);
18491850
report_line("gc", &report.gc);
18501851
report_line("flapping", &report.flapping);

src/run.rs

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -527,11 +527,14 @@ pub struct UpReport {
527527
/// The pass could not obtain an authoritative session snapshot, so it deliberately performed no
528528
/// reconciliation. Long-running supervisors retry; a one-shot caller must exit unsuccessfully.
529529
pub skipped: bool,
530-
/// pty ids spawned this pass.
530+
/// Task ids first spawned this pass (no prior runtime record was reaped).
531531
pub launched: Vec<String>,
532+
/// Task ids successfully restarted this pass: a dead active record was reaped, then its
533+
/// replacement was spawned. Kept distinct from both first launch and final garbage collection.
534+
pub restarted: Vec<String>,
532535
/// pty ids torn down (retired agents) this pass.
533536
pub torn_down: Vec<String>,
534-
/// pty ids garbage-collected (dead, non-`keep`) this pass.
537+
/// Task ids finally garbage-collected this pass, with no replacement spawned.
535538
pub gc: Vec<String>,
536539
/// pty ids whose GC/relaunch was DEFERRED this pass by the liveness debounce — a task that read
537540
/// not-alive but was alive within the grace window, i.e. a transient `pty list` flicker under load,
@@ -559,6 +562,7 @@ impl UpReport {
559562
pub fn is_noteworthy(&self) -> bool {
560563
self.skipped
561564
|| !self.launched.is_empty()
565+
|| !self.restarted.is_empty()
562566
|| !self.torn_down.is_empty()
563567
|| !self.gc.is_empty()
564568
|| !self.flapping.is_empty()
@@ -621,9 +625,10 @@ pub fn execute(
621625
}
622626
// Reap the corpse first (a dead session blocks respawn), preserving any backend-owned
623627
// bounded diagnostics, then respawn.
624-
if gc_set.contains(target.pty_id.as_str()) {
628+
let restarting = gc_set.contains(target.pty_id.as_str());
629+
if restarting {
625630
match runner.reap_for_restart(&target.pty_id) {
626-
Ok(()) => report.gc.push(target.pty_id.clone()),
631+
Ok(()) => {}
627632
Err(e) => {
628633
report
629634
.errors
@@ -635,7 +640,11 @@ pub fn execute(
635640
match runner.spawn(target, spec_dir) {
636641
Ok(()) => {
637642
cap.record(&target.pty_id, now);
638-
report.launched.push(target.pty_id.clone());
643+
if restarting {
644+
report.restarted.push(target.pty_id.clone());
645+
} else {
646+
report.launched.push(target.pty_id.clone());
647+
}
639648
}
640649
Err(e) => report.errors.push(format!("spawn {}: {e}", target.pty_id)),
641650
}

tests/run.rs

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ struct FakeRunner {
2121
killed: RefCell<Vec<String>>,
2222
reaped: RefCell<Vec<String>>,
2323
removed: RefCell<Vec<String>>,
24+
ops: RefCell<Vec<String>>,
2425
}
2526

2627
impl Runner for FakeRunner {
@@ -34,6 +35,9 @@ impl Runner for FakeRunner {
3435
if self.fail_spawn.as_deref() == Some(target.pty_id.as_str()) {
3536
anyhow::bail!("simulated spawn failure");
3637
}
38+
self.ops
39+
.borrow_mut()
40+
.push(format!("spawn:{}", target.pty_id));
3741
self.spawned.borrow_mut().push(target.pty_id.clone());
3842
self.spawn_dirs
3943
.borrow_mut()
@@ -45,6 +49,7 @@ impl Runner for FakeRunner {
4549
Ok(())
4650
}
4751
fn reap_for_restart(&self, pty_id: &str) -> anyhow::Result<()> {
52+
self.ops.borrow_mut().push(format!("reap:{pty_id}"));
4853
self.reaped.borrow_mut().push(pty_id.to_string());
4954
if self.fail_reap.as_deref() == Some(pty_id) {
5055
anyhow::bail!("reap broke");
@@ -93,6 +98,8 @@ fn up_once_launches_all_tasks_of_a_fresh_agent() {
9398
let mut launched = report.launched.clone();
9499
launched.sort();
95100
assert_eq!(launched, vec!["hetz.demo-claude", "hetz.demo.ding"]);
101+
assert!(report.restarted.is_empty());
102+
assert!(report.gc.is_empty());
96103
assert!(report.errors.is_empty());
97104
let dirs = runner.spawn_dirs.borrow();
98105
assert!(dirs.iter().all(|(_, d)| d.ends_with("agents/hetz/demo")));
@@ -168,7 +175,7 @@ fn up_once_collects_spawn_errors_without_aborting() {
168175
}
169176

170177
#[test]
171-
fn up_once_reaps_dead_nonkeep_then_respawns() {
178+
fn up_once_reports_dead_active_reap_spawn_as_restart_not_gc_or_launch() {
172179
let tmp = tempfile::tempdir().unwrap();
173180
write(tmp.path(), "agents/hetz/demo/agent.toml", AGENT);
174181
let runner = FakeRunner {
@@ -183,7 +190,24 @@ fn up_once_reaps_dead_nonkeep_then_respawns() {
183190
runner.removed.borrow().is_empty(),
184191
"a crash restart is not final retirement cleanup"
185192
);
186-
assert_eq!(report.launched.len(), 2);
193+
assert!(report.launched.is_empty());
194+
let mut restarted = report.restarted.clone();
195+
restarted.sort();
196+
assert_eq!(restarted, vec!["hetz.demo-claude", "hetz.demo.ding"]);
197+
assert!(
198+
report.gc.is_empty(),
199+
"a successful restart must not be reported as final GC"
200+
);
201+
assert_eq!(
202+
runner.ops.borrow().as_slice(),
203+
[
204+
"reap:hetz.demo-claude",
205+
"spawn:hetz.demo-claude",
206+
"reap:hetz.demo.ding",
207+
"spawn:hetz.demo.ding",
208+
],
209+
"report taxonomy must not change reap-before-spawn execution ordering"
210+
);
187211
}
188212

189213
#[test]
@@ -198,7 +222,9 @@ fn up_once_does_not_restart_a_task_when_diagnostic_reap_fails() {
198222

199223
let report = up_once(tmp.path(), "hetz", &runner).unwrap();
200224

201-
assert_eq!(report.launched, vec!["hetz.demo.ding"]);
225+
assert!(report.launched.is_empty());
226+
assert_eq!(report.restarted, vec!["hetz.demo.ding"]);
227+
assert!(report.gc.is_empty());
202228
assert_eq!(runner.spawned.borrow().as_slice(), ["hetz.demo.ding"]);
203229
assert!(
204230
report
@@ -208,6 +234,34 @@ fn up_once_does_not_restart_a_task_when_diagnostic_reap_fails() {
208234
);
209235
}
210236

237+
#[test]
238+
fn up_once_does_not_report_a_failed_replacement_as_restarted() {
239+
let tmp = tempfile::tempdir().unwrap();
240+
write(tmp.path(), "agents/hetz/demo/agent.toml", AGENT);
241+
let runner = FakeRunner {
242+
sessions: vec![dead("hetz.demo-claude"), live("hetz.demo.ding")],
243+
fail_spawn: Some("hetz.demo-claude".into()),
244+
..Default::default()
245+
};
246+
247+
let report = up_once(tmp.path(), "hetz", &runner).unwrap();
248+
249+
assert_eq!(
250+
runner.reaped.borrow().as_slice(),
251+
["hetz.demo-claude"],
252+
"the stale record is still reaped before the replacement attempt"
253+
);
254+
assert!(report.launched.is_empty());
255+
assert!(report.restarted.is_empty());
256+
assert!(report.gc.is_empty());
257+
assert!(
258+
report
259+
.errors
260+
.iter()
261+
.any(|error| error == "spawn hetz.demo-claude: simulated spawn failure")
262+
);
263+
}
264+
211265
#[test]
212266
fn up_once_finally_removes_dead_retired_tasks_without_restarting_them() {
213267
let tmp = tempfile::tempdir().unwrap();
@@ -229,6 +283,7 @@ fn up_once_finally_removes_dead_retired_tasks_without_restarting_them() {
229283
assert_eq!(removed, vec!["hetz.demo-claude", "hetz.demo.ding"]);
230284
assert!(runner.reaped.borrow().is_empty());
231285
assert!(report.launched.is_empty());
286+
assert!(report.restarted.is_empty());
232287
assert_eq!(report.gc.len(), 2);
233288
}
234289

0 commit comments

Comments
 (0)