-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathharness_state.rs
More file actions
2198 lines (2020 loc) · 97 KB
/
Copy pathharness_state.rs
File metadata and controls
2198 lines (2020 loc) · 97 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! Observed harness state: the driver-owned record of what a harness is seen doing.
//!
//! A `harness-state` file (sibling of `status` in the agent's dir) carries the latest observation a
//! session wrapper made of its provider: whether the harness is working, blocked on a human, or
//! ended, plus what its input buffer holds. This is the *observed* axis; `status` remains the
//! *declared* one, and neither speaks for the other. The record is written only by the owning
//! session's driver processes — the wrapper, its channel, or its hooks; one logical owner per
//! record, and nothing outside the driver writes it. Writes happen on state transitions plus a
//! slow heartbeat and follow the presence record's transport discipline: an embedded origin
//! timestamp (never file mtime), atomic tmp+rename writes serialized by a cross-process lock,
//! byte-distinct content on every write that lands, and a derived-only `unknown` — a writer that
//! loses sight of its harness stops heartbeating and lets the record age out rather than
//! refreshing a state it can no longer prove. Restating an unchanged state is free: it touches
//! the record only when the refresh cadence is due, so a chatty producer cannot flood the
//! transport.
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use anyhow::Context as _;
use serde::{Deserialize, Serialize};
/// A valid observation at least this old reads as `unknown`. Deliberately its own constant rather
/// than an alias of [`crate::status::STATUS_STALE`]: retuning presence must not silently retune
/// observed harness state.
pub const HARNESS_STATE_STALE: Duration = Duration::from_secs(15 * 60);
/// How often a live writer re-stamps a record it still has evidence for — the presence cadence, so
/// wrappers piggyback on the wakeup they already own.
pub const HARNESS_STATE_REFRESH: Duration = Duration::from_secs(5 * 60);
/// Maximum accepted positive difference between the writer's UTC clock and the reader's clock.
pub const HARNESS_STATE_FUTURE_SKEW: Duration = Duration::from_secs(60);
const SCHEMA: &str = "st2.harness-state.v1";
/// The claim-sequence floor sidecar, beside the record: claims stay monotonic even across a
/// record this version cannot parse.
const SEQ_FLOOR_NAME: &str = ".harness-state.seq";
const LOCK_NAME: &str = ".harness-state.lock";
/// What the harness is observed doing. `Child` is reserved: it is part of the contract so a v1
/// reader decodes it, but no producer emits it yet (the screen observer that would have was cut).
/// `Unknown` is DERIVED — staleness, malformation, or a dead session — and is never written; there
/// is no constructor path from missing evidence to `Idle`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Activity {
Idle,
Active,
Child,
/// The session ended or reached a terminal error; nothing further will be observed from this
/// incarnation without intervention. Unlike the live states, a fresh `Ended` survives the
/// session-liveness cross-check: a terminal record is *supposed* to outlive its writer.
Ended,
#[serde(other)]
Unknown,
}
impl Activity {
pub fn as_str(self) -> &'static str {
match self {
Activity::Idle => "idle",
Activity::Active => "active",
Activity::Child => "child",
Activity::Ended => "ended",
Activity::Unknown => "unknown",
}
}
}
/// Who the harness is waiting on. `Human` means the model is stopped and a person is the thing
/// that restarts it (a permission prompt, a review, a question) — neither working nor merely idle.
/// Unrecognized future values decode as `Unknown` (indeterminate), never as `None`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum BlockedOn {
None,
Human,
#[serde(other)]
Unknown,
}
impl BlockedOn {
pub fn as_str(self) -> &'static str {
match self {
BlockedOn::None => "none",
BlockedOn::Human => "human",
BlockedOn::Unknown => "unknown",
}
}
}
/// What the harness's composer holds. `Unknown` is writable on this axis: "I cannot see the
/// composer" is itself the observation most producers make.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum InputBuffer {
Empty,
Nonempty,
#[serde(other)]
Unknown,
}
impl InputBuffer {
pub fn as_str(self) -> &'static str {
match self {
InputBuffer::Empty => "empty",
InputBuffer::Nonempty => "nonempty",
InputBuffer::Unknown => "unknown",
}
}
}
/// What kind of human ask holds the harness, machine-readably — consumers filter on this axis
/// (`reason` stays diagnostic-only). Meaningful only while `blockedOn` is `human`; writers set
/// `none` otherwise. `Unknown` decodes future words and is never written.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Ask {
#[default]
None,
Permission,
Question,
Review,
#[serde(other)]
Unknown,
}
impl Ask {
pub fn as_str(self) -> &'static str {
match self {
Ask::None => "none",
Ask::Permission => "permission",
Ask::Question => "question",
Ask::Review => "review",
Ask::Unknown => "unknown",
}
}
}
/// The durable record. Additive-tolerant on read (no `deny_unknown_fields`): a reader pinned to an
/// older crate may be older than the writer.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Record {
schema: String,
agent: String,
harness: String,
state: Activity,
blocked_on: BlockedOn,
input_buffer: InputBuffer,
/// The machine-readable kind of human ask while `blockedOn` is `human`; `none` otherwise.
/// Absent in records from writers predating the axis, which defaults to `none`.
#[serde(default)]
ask: Ask,
/// Diagnostic only. No consumer branches on it.
#[serde(default, skip_serializing_if = "Option::is_none")]
reason: Option<String>,
/// `Ended` only: the exit outcome, e.g. `exit 0` or `signal 9`.
#[serde(default, skip_serializing_if = "Option::is_none")]
exit: Option<String>,
/// The pty session whose liveness vouches for the live states. Same-host readers cross-check
/// it; a record whose session is provably dead reads `unknown` even while fresh.
#[serde(default, skip_serializing_if = "Option::is_none")]
pty_session: Option<String>,
/// The writing session's incarnation token. Ownership is token equality, never a timestamp
/// comparison: same-millisecond takeovers and lingering predecessor writers are both real.
/// Empty in records from writers predating the field, which no session owns.
#[serde(default)]
incarnation: String,
/// The monotonic ownership sequence. Only a session claim (a wrapper or session-boundary
/// writer starting up) advances it, to the on-disk value plus one; every writer refuses to
/// touch a record whose sequence is beyond its own claim, which is what gives ownership a
/// DIRECTION — a lingering predecessor's late write cannot replace its successor's record,
/// while the successor's claim replaces the predecessor's.
#[serde(default)]
seq: u64,
/// When the current state was entered. Survives heartbeat re-stamps.
since_ms: u64,
/// The heartbeat: when the writer last held evidence for this state.
written_at_ms: u64,
/// Monotonic transition counter. Keeps every write byte-distinct and leaves room for a
/// compatible transition history later.
transitions: u64,
}
/// The record's file name inside an agent directory. Named rather than inlined because the
/// replication transport's include list carries it literally: see
/// [`crate::harness_context::REPLICATED_DRIVER_RECORDS`].
pub const RECORD_NAME: &str = "harness-state";
/// The observed-state file: `<agent_dir>/harness-state`.
pub fn harness_state_path(agent_dir: &Path) -> PathBuf {
agent_dir.join(RECORD_NAME)
}
/// One observation as a producer states it: everything except the derived pieces.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Observation {
pub state: Activity,
pub blocked_on: BlockedOn,
pub input_buffer: InputBuffer,
pub ask: Ask,
pub reason: Option<String>,
pub exit: Option<String>,
}
impl Observation {
pub fn new(state: Activity, blocked_on: BlockedOn, input_buffer: InputBuffer) -> Self {
Self {
state,
blocked_on,
input_buffer,
ask: Ask::None,
reason: None,
exit: None,
}
}
pub fn with_ask(mut self, ask: Ask) -> Self {
self.ask = ask;
self
}
pub fn with_reason(mut self, reason: impl Into<String>) -> Self {
self.reason = Some(reason.into());
self
}
pub fn with_exit(mut self, exit: impl Into<String>) -> Self {
self.exit = Some(exit.into());
self
}
}
/// The writer a driver process owns over one agent's record. Several driver processes may
/// legitimately hold writers over the same record — a wrapper heartbeat beside hook-process
/// transitions — so every operation takes the record's cross-process lock and treats the on-disk
/// record as the authoritative current state: rename alone is atomic but not isolated, and
/// without the re-read a stale process could resurrect the state it saw before a peer's write.
/// The caller's rule for indeterminacy: when evidence is lost (the observer no longer sees its
/// harness), call nothing — never heartbeat a state you cannot see, and never write `unknown`.
pub struct Writer {
path: PathBuf,
lock_path: PathBuf,
agent: String,
harness: &'static str,
pty_session: Option<String>,
interrupted: bool,
session: String,
/// The ownership sequence this writer acts under. `None` = a claiming writer: it resolves at
/// the first write — adopting the on-disk sequence when the record already carries this
/// session's token, else claiming on-disk + 1. `Some` = adopted ownership handed down by the
/// session's claimer (env-exported beside the token).
claimed_seq: Option<u64>,
}
impl Writer {
/// The transition counter continues from any readable record already on disk, so restarts and
/// sibling writers keep writes byte-distinct; an unreadable predecessor starts the counter
/// fresh.
pub fn new(
agent_dir: &Path,
agent: impl Into<String>,
harness: &'static str,
pty_session: Option<String>,
) -> Self {
Self {
path: harness_state_path(agent_dir),
lock_path: agent_dir.join(LOCK_NAME),
agent: agent.into(),
harness,
pty_session,
interrupted: false,
session: session_token(),
claimed_seq: None,
}
}
/// Adopt an explicit session incarnation token. Sibling writer processes of one session —
/// a wrapper beside its hook subprocesses, a channel beside its wrapper — must share one
/// token (typically minted by the wrapper and exported through the session environment), or
/// each writes as its own session: restatements open transitions and the wrapper can neither
/// re-stamp nor terminally fence its siblings' records.
pub fn with_session(mut self, token: impl Into<String>) -> Self {
self.session = token.into();
self
}
/// Adopt the full ownership a session's claimer exported — token and claimed sequence
/// together. A writer holding adopted ownership never claims: it writes only while the
/// on-disk record's sequence is at or below its claim, so a straggler from a superseded
/// session is refused in both the live and the terminal path.
pub fn with_ownership(mut self, token: impl Into<String>, seq: u64) -> Self {
self.session = token.into();
self.claimed_seq = Some(seq);
self
}
/// Mark this writer's observation stream discontinuous: its evidence was lost and has since
/// returned. The next observation opens a fresh transition even if it restates the
/// pre-interruption tuple — continuity (`sinceMs`, the counter) must never be claimed across
/// an interval the observer did not see.
pub fn interrupt(&mut self) {
self.interrupted = true;
}
/// Hold the record's exclusive cross-process lock for one read→decide→rename cycle. The lock
/// file is a permanent sibling; the guard releases on drop (close).
fn locked(&self) -> anyhow::Result<crate::flock::FileLock> {
lock_exclusive(&self.lock_path)
}
/// Record an observation. A genuine change writes a new transition with a fresh `since`. An
/// observation identical to the on-disk record is a no-op while that record is fresh —
/// producers may restate their state arbitrarily often (an SSE stream restates several times
/// per second, measured) and only the refresh cadence may reach the transport — and becomes a
/// heartbeat-equivalent re-stamp once the record is older than [`HARNESS_STATE_REFRESH`].
/// `Unknown` state is derived and cannot be written.
pub fn observe(&mut self, observation: Observation) -> anyhow::Result<()> {
self.observe_inner(observation, false).map(|_wrote| ())
}
/// [`Writer::observe`], except a live-state frame is dropped (returning `false`) when the
/// on-disk record is already terminal. Not a general rule — a harness may legally report
/// activity after a terminal error, and Codex does — but a producer whose live frames and
/// terminal record come from different processes opts in so a queued live frame can never
/// overwrite the incarnation's last word.
pub fn observe_unless_ended(&mut self, observation: Observation) -> anyhow::Result<bool> {
self.observe_inner(observation, true)
}
fn observe_inner(
&mut self,
observation: Observation,
skip_if_ended: bool,
) -> anyhow::Result<bool> {
anyhow::ensure!(
observation.state != Activity::Unknown,
"unknown is derived and cannot be written"
);
anyhow::ensure!(
observation.state == Activity::Ended || self.pty_session.is_some(),
"live observations require a pty session to vouch for them"
);
anyhow::ensure!(
observation.ask != Ask::Unknown,
"unknown is derived and cannot be written"
);
anyhow::ensure!(
observation.ask == Ask::None || observation.blocked_on == BlockedOn::Human,
"an ask kind is meaningful only while blocked on a human"
);
let _lock = self.locked()?;
let on_disk = match read_stored(&self.path) {
StoredRecord::Parsed(record) => Some(record),
StoredRecord::Absent => None,
// Bytes this version cannot parse are somebody's record, not a virgin seat: a
// non-claiming writer refuses rather than restarting the sequence and counter over
// foreign state. Only the explicit written claim supersedes.
StoredRecord::Unreadable => return Ok(false),
};
// Resolve this writer's ownership sequence, then enforce its direction. A claiming
// writer adopts the on-disk sequence when the record already carries its token (a
// sibling wrote first) and claims on-disk + 1 otherwise; an adopted-ownership writer
// holds whatever its session's claimer exported. Either way, a record whose sequence is
// beyond the claim belongs to a LATER session: this writer is the straggler, and its
// write — live or terminal — is refused rather than replacing its successor's record.
let seq = match self.claimed_seq {
Some(seq) => seq,
// A token-only writer NEVER claims: it adopts the on-disk sequence when the record
// already carries its token, starts a virgin record at one, and is refused outright
// against a foreign token — a mixed-version straggler minting claims would fence
// the true successor out permanently. New sequences are minted only by [`claim`],
// the written act, and adopted from it.
None => match on_disk.as_ref() {
// A virgin seat's first write mints sequence one — initial ownership,
// exactly what [`claim_locked`] establishes — so it persists the floor
// sidecar too: if this record later goes unreadable, a replacement claim
// must continue past it instead of colliding with this lingering writer.
None => {
persist_floor(&self.path, 1);
1
}
Some(current) if current.incarnation == self.session => current.seq,
Some(_) => return Ok(false),
},
};
if on_disk.as_ref().is_some_and(|current| current.seq > seq) {
return Ok(false);
}
// A foreign schema's `seq` decodes as serde-default zero, which every claim exceeds — a
// v1 straggler would otherwise replace a v2 record it cannot even read. Non-claiming
// writers refuse foreign schemas outright; only the explicit written [`claim`]
// supersedes an unsupported schema.
if on_disk
.as_ref()
.is_some_and(|current| current.schema != SCHEMA)
{
return Ok(false);
}
self.claimed_seq = Some(seq);
// Ownership is token equality: a record is this writer's only when it carries both this
// version's schema and this session's incarnation. Anything else — a foreign schema, a
// predecessor's or successor's token, the empty pre-token form — is never coalesced
// against and never treated as this session's terminal word; a genuine observation
// replaces it wholesale (one logical owner per record), continuing the counter for
// byte-distinctness. Timestamps deliberately play no part: a same-millisecond takeover
// and a lingering predecessor writer are both real and both ambiguous by clock.
let own_record = on_disk
.as_ref()
.filter(|current| current.schema == SCHEMA && current.incarnation == self.session);
if skip_if_ended
&& own_record.is_some_and(|current| {
// Only a REAL terminal record from this session suppresses queued live frames:
// one carrying an exit, which every wrapper's `ended` does. The claim record —
// this session's own `ended (superseded)` placeholder, deliberately exitless —
// must not suppress the session's first frames, and a predecessor incarnation's
// `ended` is history rather than this session's last word.
current.state == Activity::Ended && current.exit.is_some()
})
{
return Ok(false);
}
let now_ms = crate::message::now_ms();
let unchanged = !self.interrupted
&& own_record.is_some_and(|current| {
current.state == observation.state
&& current.blocked_on == observation.blocked_on
&& current.input_buffer == observation.input_buffer
&& current.ask == observation.ask
&& current.reason == observation.reason
&& current.exit == observation.exit
});
if unchanged
&& let Some(current) = own_record
// A restatement is a no-op only against a record this session already wrote (the
// token filter above) whose stamp a reader would trust: a stamp beyond the
// future-skew bound — a backward clock correction's leftover — would otherwise
// read "fresh" here forever while every reader derives future-skew unknown, so it
// falls through to the write below, whose next_stamp resets to the writer's clock.
&& current.written_at_ms <= now_ms.saturating_add(duration_ms(HARNESS_STATE_FUTURE_SKEW))
&& now_ms.saturating_sub(current.written_at_ms) < duration_ms(HARNESS_STATE_REFRESH)
{
return Ok(true);
}
// A landed write is byte-distinct even against a same-millisecond predecessor: the stamp
// is strictly monotonic per record, at the cost of a bounded forward skew of at most one
// millisecond per write (writes are transition-scale, so the skew never accumulates
// meaningfully against the staleness horizon). A stamp is only ever inherited from a
// record a reader would trust: one already past the future-skew bound is somebody's
// garbage (or an overflow probe), and inheriting it would poison every later write —
// the writer's own clock wins instead.
let written_at_ms = next_stamp(on_disk.as_ref(), now_ms);
let (since_ms, transitions) = match (own_record, unchanged) {
(Some(current), true) => (current.since_ms, current.transitions),
(Some(current), false) => (written_at_ms, current.transitions.saturating_add(1)),
(None, _) => (
written_at_ms,
on_disk
.as_ref()
.map_or(0, |current| current.transitions.saturating_add(1)),
),
};
let record = Record {
schema: SCHEMA.to_string(),
agent: self.agent.clone(),
harness: self.harness.to_string(),
state: observation.state,
blocked_on: observation.blocked_on,
input_buffer: observation.input_buffer,
ask: observation.ask,
reason: observation.reason,
exit: observation.exit,
pty_session: self.pty_session.clone(),
incarnation: self.session.clone(),
seq,
since_ms,
written_at_ms,
transitions,
};
write_record(&self.path, &record)?;
self.interrupted = false;
Ok(true)
}
/// Re-stamp the heartbeat for whatever live state is on disk. Nothing on disk means nothing
/// to keep fresh, and a terminal record is never re-stamped. The on-disk record is
/// authoritative: a wrapper heartbeat re-stamps the newest state, including one a hook or
/// channel process wrote after this writer's last observation. A predecessor session's record
/// — one written before this session started — is preserved for counter continuity but is
/// never heartbeat-eligible: re-stamping it would keep a dead session's state fresh forever.
/// It becomes eligible once any writer of this session observes something.
pub fn heartbeat(&mut self) -> anyhow::Result<()> {
let _lock = self.locked()?;
let Some(mut current) = read_record(&self.path) else {
return Ok(());
};
// A schema this writer does not own must not be round-tripped through this version's
// record type, and a record this *session* does not own must never be kept fresh: a
// lingering predecessor re-stamping its successor's record would keep a dead seat's
// state alive for cross-host readers, and a successor re-stamping a predecessor's would
// resurrect history. Token equality decides, in both directions.
if current.schema != SCHEMA
|| current.state == Activity::Ended
|| current.incarnation != self.session
{
return Ok(());
}
let now_ms = crate::message::now_ms();
current.written_at_ms = if current.written_at_ms
<= now_ms.saturating_add(duration_ms(HARNESS_STATE_FUTURE_SKEW))
{
now_ms.max(current.written_at_ms.saturating_add(1))
} else {
// Never inherit an untrusted future stamp — reset to this writer's clock.
now_ms
};
write_record(&self.path, ¤t)
}
/// Write the terminal record for this session. Idempotent-shaped: callers on racing teardown
/// paths may both call it.
pub fn ended(&mut self, exit: impl Into<String>) -> anyhow::Result<()> {
self.observe(
Observation::new(Activity::Ended, BlockedOn::None, InputBuffer::Unknown)
.with_exit(exit),
)
}
}
/// The derived view a consumer reads. `state` already folds in staleness, future skew,
/// malformation, and (when a probe is supplied) session liveness; `reason` names which derivation
/// produced an `unknown`, so no absence is silent.
#[derive(Debug, Clone, PartialEq)]
pub struct Observed {
pub state: Activity,
pub blocked_on: BlockedOn,
pub input_buffer: InputBuffer,
pub ask: Ask,
pub harness: Option<String>,
pub since_ms: Option<u64>,
pub exit: Option<String>,
pub reason: Option<String>,
}
impl Observed {
fn indeterminate(reason: &str, harness: Option<String>) -> Self {
// The single constructor for an indeterminate observation: every absence routes here, so
// no path can derive `idle` — or anything else — from missing evidence.
Self {
state: Activity::Unknown,
blocked_on: BlockedOn::Unknown,
input_buffer: InputBuffer::Unknown,
ask: Ask::Unknown,
harness,
since_ms: None,
exit: None,
reason: Some(reason.to_string()),
}
}
}
/// Result of a same-host session-liveness probe. `Indeterminate` (an unreadable registry, e.g. a
/// reader without the session dir the writer used) must not downgrade anything: unprovable
/// evidence is never reported as death.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SessionLiveness {
Alive,
Dead,
Indeterminate,
}
/// Read an agent's observed harness state. `None` means no record exists — no driver has ever
/// observed this agent, which is different from `unknown`. `probe` is the optional same-host
/// liveness cross-check for the record's pty session; pass `None` for cross-host reads.
pub fn read(path: &Path, probe: Option<&dyn Fn(&str) -> SessionLiveness>) -> Option<Observed> {
let raw = match fs::read(path) {
Ok(raw) => raw,
// Only proven absence is absence; a record that exists but cannot be read is
// indeterminate, never silently "no observation".
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None,
Err(_) => return Some(Observed::indeterminate("unreadable-record", None)),
};
Some(read_raw_at(&raw, probe, crate::message::now_ms()))
}
fn read_raw_at(
raw: &[u8],
probe: Option<&dyn Fn(&str) -> SessionLiveness>,
now_ms: u64,
) -> Observed {
let Ok(record) = serde_json::from_slice::<Record>(raw) else {
return Observed::indeterminate("malformed-record", None);
};
let harness = Some(record.harness.clone());
// The discriminator gates interpretation: a future schema's words may be spelled like this
// version's while meaning something else, so nothing definite may be derived from them.
if record.schema != SCHEMA {
return Observed::indeterminate("unsupported-schema", harness);
}
if record.written_at_ms > now_ms {
if record.written_at_ms - now_ms > duration_ms(HARNESS_STATE_FUTURE_SKEW) {
return Observed::indeterminate("future-skew", harness);
}
} else if now_ms - record.written_at_ms >= duration_ms(HARNESS_STATE_STALE) {
return Observed::indeterminate("stale", harness);
}
if record.state == Activity::Unknown {
// A literal `unknown` is never written by this crate; treat one like malformation.
return Observed::indeterminate("literal-unknown", harness);
}
if record.state == Activity::Ended
&& record.exit.is_none()
&& record.reason.as_deref() == Some("superseded")
{
// The claim placeholder is a fence, not an observation: the session wrote it at startup
// and has observed nothing yet. Reading it as definite `ended` would flip a live seat
// to dead for every consumer whose harness never publishes its first frame promptly —
// indeterminate, distinctly, until the first real observation or the ordinary horizon.
return Observed::indeterminate("claimed", harness);
}
if record.state != Activity::Ended
&& let Some(probe) = probe
{
// Same-host readers cross-check live states against the session registry. A live record
// that names no session offers nothing to check — without this rule it would stay
// definite through an external SIGKILL for the whole staleness horizon, which is exactly
// the window the cross-check exists to close. Writers therefore must fence live states
// (enforced in `observe`); a fenced record whose session is provably dead is downgraded,
// and an unreadable registry still downgrades nothing.
let Some(session) = record.pty_session.as_deref() else {
return Observed::indeterminate("unfenced-record", harness);
};
if probe(session) == SessionLiveness::Dead {
return Observed::indeterminate("session-dead", harness);
}
}
Observed {
state: record.state,
blocked_on: record.blocked_on,
input_buffer: record.input_buffer,
ask: record.ask,
harness,
since_ms: Some(record.since_ms),
exit: record.exit,
reason: record.reason,
}
}
fn duration_ms(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
/// What the record file holds, tri-state: absence, bytes this version cannot parse, or a parsed
/// record. Collapsing `Unreadable` into `Absent` would let a writer treat an undeserializable
/// v2 record as a virgin seat — restarting the sequence and counter over live foreign state.
enum StoredRecord {
Absent,
Unreadable,
Parsed(Record),
}
fn read_stored(path: &Path) -> StoredRecord {
match fs::read(path) {
// Only proven absence is absence: a file that exists but cannot be read (permissions,
// IO) is somebody's record — treating it as a virgin seat would let a token-only write
// or a wrapperless claim rename over live state it never saw.
Err(error) if error.kind() == std::io::ErrorKind::NotFound => StoredRecord::Absent,
Err(_) => StoredRecord::Unreadable,
Ok(bytes) => match serde_json::from_slice(&bytes) {
Ok(record) => StoredRecord::Parsed(record),
Err(_) => StoredRecord::Unreadable,
},
}
}
fn read_record(path: &Path) -> Option<Record> {
match read_stored(path) {
StoredRecord::Parsed(record) => Some(record),
StoredRecord::Absent | StoredRecord::Unreadable => None,
}
}
fn write_record(path: &Path, record: &Record) -> anyhow::Result<()> {
// This record stages beside itself, unchanged: the sibling driver record
// ([`crate::harness_context`]) stages outside the agent subtree because a replicated
// temporary name becomes a durable key, and moving this one's staging is a separate change.
let dir = path.parent().unwrap_or(Path::new(".")).to_path_buf();
write_json_atomic(path, record, &dir, ".harness-state")
}
/// Take one driver record's exclusive cross-process lock, held for a read→decide→rename cycle.
/// The lock file is a permanent sibling of the record and the guard releases on drop (close).
/// Shared with [`crate::harness_context`], which owns a sibling record with its own lock file:
/// the transport is common, the ownership protocol above it is not.
pub(crate) fn lock_exclusive(lock_path: &Path) -> anyhow::Result<crate::flock::FileLock> {
if let Some(dir) = lock_path.parent() {
fs::create_dir_all(dir)?;
}
let lock = crate::flock::open(lock_path, crate::flock::Open::Create)?;
crate::flock::FileLock::hold_blocking(lock, crate::flock::Mode::Exclusive)
.with_context(|| format!("locking {} failed", lock_path.display()))
}
/// Stage-and-rename one newline-terminated JSON record. Atomic when `staging_dir` is on the
/// record's filesystem, which every caller must ensure — `staging_dir` is explicit precisely
/// because the two driver records answer "where may a temporary name live" differently.
pub(crate) fn write_json_atomic<T: Serialize>(
path: &Path,
value: &T,
staging_dir: &Path,
tmp_prefix: &str,
) -> anyhow::Result<()> {
let mut bytes = serde_json::to_vec(value)?;
bytes.push(b'\n');
crate::fsatomic::replace(
path,
&bytes,
crate::fsatomic::Staging::new(tmp_prefix).in_dir(staging_dir),
crate::fsatomic::Durability::Rename,
)?;
Ok(())
}
/// The per-record monotonic stamp: strictly beyond the on-disk stamp when that stamp is inside
/// the future-skew trust bound (a stamp beyond it is somebody's garbage or an overflow probe,
/// and inheriting it would poison every later write), and the writer's own clock otherwise.
fn next_stamp(on_disk: Option<&Record>, now_ms: u64) -> u64 {
on_disk
.map(|current| current.written_at_ms)
.filter(|&previous| {
previous <= now_ms.saturating_add(duration_ms(HARNESS_STATE_FUTURE_SKEW))
})
.map_or(now_ms, |previous| now_ms.max(previous.saturating_add(1)))
}
/// Claim session ownership of an agent's record, as a WRITTEN act under the record's lock: the
/// takeover record supersedes whatever is on disk — `ended` with reason `superseded`, no exit,
/// the new session's token, and the next ownership sequence — and the claimed sequence is
/// returned for the wrapper to adopt and export beside its token. Writing the claim makes it
/// atomic: racing claimers serialize on the lock and mint DISTINCT sequences, and a
/// predecessor's fresh live record is superseded at relaunch even though the pty-name-based
/// probe cannot tell the sessions apart. Readers derive indeterminate (`claimed`) from the
/// fresh placeholder — a fence, not an observation — until the session's first real
/// observation replaces it.
pub fn claim(
agent_dir: &Path,
agent: impl Into<String>,
harness: &'static str,
token: &str,
) -> anyhow::Result<u64> {
let writer = Writer::new(agent_dir, agent, harness, None);
let _lock = writer.locked()?;
claim_locked(&writer, token)
}
/// The claim's body, under an already-held record lock.
fn claim_locked(writer: &Writer, token: &str) -> anyhow::Result<u64> {
// Unreadable bytes are superseded like anything else — that is exactly what the claim is
// for — but their CONTENT cannot be continued (the counter restarts). The SEQUENCE must
// survive them regardless: a claim restarting at one would sit below a lingering
// predecessor's claim, whose next write would replace the new claim and then permanently
// fence the new session out. The floor sidecar, written under this same lock on every
// claim, preserves monotonicity across records this version cannot parse; only both files
// being damaged loses the floor, and that residual is documented.
let on_disk = match read_stored(&writer.path) {
StoredRecord::Parsed(record) => Some(record),
StoredRecord::Absent | StoredRecord::Unreadable => None,
};
let floor_path = writer.path.with_file_name(SEQ_FLOOR_NAME);
let floor = fs::read_to_string(&floor_path)
.ok()
.and_then(|raw| raw.trim().parse::<u64>().ok());
let highest = on_disk.as_ref().map(|record| record.seq).max(floor);
// A saturated sequence would mint SHARED ownership forever after: every later claim would
// return the same MAX, and two sessions holding equal claims are exactly the ambiguity the
// sequence exists to remove. Fail loudly; producers degrade to token-only and stay alive.
anyhow::ensure!(
highest.is_none_or(|seq| seq < u64::MAX),
"ownership sequence exhausted; refusing a shared claim"
);
let seq = highest.map_or(1, |seq| seq.saturating_add(1));
let now_ms = crate::message::now_ms();
let written_at_ms = next_stamp(on_disk.as_ref(), now_ms);
let record = Record {
schema: SCHEMA.to_string(),
agent: writer.agent.clone(),
harness: writer.harness.to_string(),
state: Activity::Ended,
blocked_on: BlockedOn::None,
input_buffer: InputBuffer::Unknown,
ask: Ask::None,
reason: Some("superseded".to_string()),
exit: None,
pty_session: None,
incarnation: token.to_string(),
seq,
since_ms: written_at_ms,
written_at_ms,
transitions: on_disk
.as_ref()
.map_or(0, |record| record.transitions.saturating_add(1)),
};
write_record(&writer.path, &record)?;
// The floor accompanies every act that establishes ownership; its own failure
// modes must never be quiet ones.
persist_floor(&writer.path, seq);
// A session boundary empties the window, so the numeric sibling is removed with the same
// act that supersedes this record (HC-R15): the new incarnation reads "no context yet"
// rather than the previous one's 190k, which is what a crash-looping seat would otherwise
// show for the whole hour of that record's horizon. This runs while THIS record's lock is
// held and takes the sibling's lock inside it, so the order is state → context. That is
// the only place the two are ever held together and `harness_context` never takes this
// one, so the ordering is acyclic and no writer can deadlock against it. The claim stands
// whether or not the removal succeeds, but never silently.
if let Some(agent_dir) = writer.path.parent()
&& let Err(error) = crate::harness_context::remove(agent_dir)
{
tracing::warn!(
"st2 harness-state: clearing the harness-context record for {} failed: {error}",
agent_dir.display()
);
}
Ok(seq)
}
/// Persist the sequence-floor sidecar for `seq`. The floor is the safety net for the record
/// itself going unreadable, so its own failure modes must not be quiet ones: stage-and-rename
/// keeps a torn write from corrupting the current floor, and a failed write is logged — the
/// ownership still stands (losing the floor only matters if the record later becomes
/// unreadable), but never silently.
///
/// The staging name carries the writer's pid and a counter for the same reason every other
/// publication's does. It used to be the fixed literal `.harness-state.seq.tmp`, written with a
/// truncating, symlink-following `fs::write`: two writers persisting a floor for one agent shared
/// that one path, so each could truncate and rename the other's half-written bytes — a torn floor,
/// which defeats exactly the failure the floor exists for. One caller holds the record's lock; the
/// other is the token-only virgin-record path, whose lock coverage is not established here, so the
/// staging name must not depend on it.
fn persist_floor(record_path: &Path, seq: u64) {
let floor_path = record_path.with_file_name(SEQ_FLOOR_NAME);
if let Err(error) = crate::fsatomic::replace(
&floor_path,
format!("{seq}\n").as_bytes(),
crate::fsatomic::Staging::new(SEQ_FLOOR_NAME),
crate::fsatomic::Durability::Rename,
) {
tracing::warn!(
"st2 harness-state: writing the sequence floor {} failed: {error}",
floor_path.display()
);
}
}
/// The token prefix wrapperless Claude sessions derive from Claude's own session id.
pub const WRAPPERLESS_PREFIX: &str = "claude-session-";
/// A WRAPPERLESS session boundary's claim — eligibility and the written takeover as ONE act
/// under the record lock, because check-then-act across two acquisitions is a race: a
/// hooks-only SessionStart landing between a wrapper's startup reads could otherwise steal the
/// sequence the wrapper was about to export. A wrapper's claim is always legitimate — it owns
/// the seat's lifecycle — but a wrapperless claimer (a hook fired by any interactive session
/// that inherited the project-scoped registration) must not supersede live wrapper state. It
/// claims over nothing, over records no wrapper minted (fellow wrapperless tokens), over REAL
/// terminal records (exit-bearing), and over staleness — never over a live wrapper record, and
/// never over a wrapper's FRESH claim placeholder (`ended (superseded)`, exitless,
/// wrapper-shaped token): that placeholder is a session mid-startup, not an ended one, though
/// an abandoned placeholder past the staleness horizon is claimable like any orphan.
/// `Ok(None)` = ineligible; unreadable bytes are also ineligible for this cautious path.
pub fn claim_wrapperless(
agent_dir: &Path,
agent: impl Into<String>,
harness: &'static str,
token: &str,
) -> anyhow::Result<Option<u64>> {
let writer = Writer::new(agent_dir, agent, harness, None);
let _lock = writer.locked()?;
let eligible = match read_stored(&writer.path) {
StoredRecord::Absent => true,
StoredRecord::Unreadable => false,
StoredRecord::Parsed(record) => {
let now_ms = crate::message::now_ms();
let stale =
now_ms.saturating_sub(record.written_at_ms) >= duration_ms(HARNESS_STATE_STALE);
let wrapperless_owner =
record.incarnation.is_empty() || record.incarnation.starts_with(WRAPPERLESS_PREFIX);
let real_terminal = record.state == Activity::Ended && record.exit.is_some();
wrapperless_owner || real_terminal || stale
}
};
if !eligible {
return Ok(None);
}
claim_locked(&writer, token).map(Some)
}
/// A process-unique session incarnation token: pid, wall-clock, and a process-local counter.
/// Uniqueness across the writers that can actually race on one record (processes on one host)
/// is what matters; no cryptographic strength is implied or needed.
pub fn session_token() -> String {
format!(
"{}-{}-{}",
std::process::id(),
crate::message::now_ms(),
TMP_COUNTER.fetch_add(1, Ordering::Relaxed)
)
}
static TMP_COUNTER: AtomicU64 = AtomicU64::new(0);
#[cfg(test)]
mod tests {
use super::*;
/// [`write_json_atomic`]'s contract: one newline-terminated JSON record, replaced whole,
/// staged in the directory the CALLER named, and owner-only. The staging directory is not a
/// detail — the harness-context record stages in the catalog control plane precisely because a
/// staged name inside the replicated `agents` namespace becomes a durable replicated key
/// (INVARIANTS row 29, HC-R05) — so an unusable staging directory must fail the publication
/// instead of quietly staging beside the record. Proven with a staging path that is a regular
/// file, which no uid can turn into a directory.
///
/// The mode is the deliberate change of the fold onto `fsatomic`: this pair used to be
/// published at whatever an ordinary write produces (`0644` under the fleet's umask). These
/// are the two records a replication transport's include list names (HC-R05), and no such
/// transport runs on the fleet today (`DQ-C1`/`DQ-H2`), so nothing reads them as another uid;
/// the tightening is recorded against HC-T08 for whoever adopts one.
#[test]
fn a_record_is_one_json_line_staged_in_the_directory_the_caller_named() {
use std::os::unix::fs::PermissionsExt as _;
let tmp = tempfile::tempdir().unwrap();
let agent_dir = tmp.path().join("agents/hetz/worker");
let path = harness_state_path(&agent_dir);
let staging = tmp.path().join("staging");
let record = serde_json::json!({"schema": "test"});
write_json_atomic(&path, &record, &staging, ".harness-state").unwrap();
write_json_atomic(&path, &record, &staging, ".harness-state").unwrap();
assert_eq!(fs::read(&path).unwrap(), b"{\"schema\":\"test\"}\n");
assert_eq!(
fs::metadata(&path).unwrap().permissions().mode() & 0o777,
0o600,
"the driver record is published owner-only"
);
for dir in [&agent_dir, &staging] {
let residue = fs::read_dir(dir)
.unwrap()
.map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
.filter(|name| name.starts_with(".harness-state.tmp-"))
.collect::<Vec<_>>();
assert!(residue.is_empty(), "staging residue in {dir:?}: {residue:?}");
}
let blocked = tmp.path().join("blocked");
fs::write(&blocked, b"not a directory").unwrap();
assert!(
write_json_atomic(&path, &record, &blocked, ".harness-state").is_err(),
"an unusable staging directory must fail the publication, not fall back"
);
}
fn writer(dir: &Path) -> Writer {
Writer::new(dir, "hetz.worker", "codex", Some("worker".to_string()))
}
/// A new session arriving the way real wrappers do: a written claim, then adoption.
fn takeover(dir: &Path, harness: &'static str) -> Writer {
let token = session_token();
let seq = claim(dir, "hetz.worker", harness, &token).unwrap();
Writer::new(dir, "hetz.worker", harness, Some("worker".to_string()))
.with_ownership(token, seq)
}
fn active() -> Observation {
Observation::new(Activity::Active, BlockedOn::None, InputBuffer::Unknown)
}
#[test]
fn missing_record_reads_as_none_not_unknown() {
let tmp = tempfile::tempdir().unwrap();
assert_eq!(read(&harness_state_path(tmp.path()), None), None);
}
#[test]
fn observe_then_read_roundtrips_every_writable_state() {
let tmp = tempfile::tempdir().unwrap();
let mut writer = writer(tmp.path());
for (state, blocked, buffer) in [
(Activity::Idle, BlockedOn::None, InputBuffer::Empty),
(Activity::Active, BlockedOn::Human, InputBuffer::Unknown),
(Activity::Child, BlockedOn::None, InputBuffer::Nonempty),
(Activity::Ended, BlockedOn::None, InputBuffer::Unknown),
] {
writer
.observe(Observation::new(state, blocked, buffer))
.unwrap();
let observed = read(&harness_state_path(tmp.path()), None).unwrap();