1- import { writeFileSync } from "node:fs" ;
1+ import { existsSync , readFileSync , readdirSync , writeFileSync } from "node:fs" ;
22import { join } from "node:path" ;
33import { createChecks , evidenceDir , guardRealAuth , installCleanupHooks } from "./common.mjs" ;
44
@@ -20,6 +20,7 @@ function writeTtsrEvidence(slug, scenarioName, result, server) {
2020 writeFileSync ( join ( dir , `${ scenarioName } -stdout.txt` ) , `${ result . stdout } \n${ result . stderr } ` ) ;
2121 writeFileSync ( join ( dir , `${ scenarioName } -requests.json` ) , JSON . stringify ( server . requests , null , 2 ) ) ;
2222 process . stderr . write ( `evidence: ${ dir } \n` ) ;
23+ return dir ;
2324}
2425
2526const REPEATED_STATUS_TURNS = [
@@ -28,28 +29,91 @@ const REPEATED_STATUS_TURNS = [
2829 "I read this as continue supervising the portable matrix; it has started cleanly with 3 checks green and 6 gates pending." ,
2930] ;
3031
32+ function writeGoalMonitorFixture ( box ) {
33+ const eventLogPath = join ( box . dir , "goal-monitor-events.jsonl" ) ;
34+ const extensionPath = join ( box . dir , "goal-monitor-extension.mjs" ) ;
35+ const source = `
36+ import { appendFileSync } from "node:fs";
37+
38+ const eventLogPath = ${ JSON . stringify ( eventLogPath ) } ;
39+ const record = (event) => appendFileSync(eventLogPath, JSON.stringify(event) + "\\n");
40+
41+ export default function(pi) {
42+ pi.on("session_start", () => {
43+ pi.events?.emit("terminal_monitor_state", { activeCount: 1 });
44+ record({ type: "monitor_state", activeCount: 1 });
45+ });
46+ pi.on("agent_end", (event) => {
47+ record({ type: "agent_end", aborted: event.aborted, abortSource: event.abortSource });
48+ });
49+ pi.on("tool_result", (event) => {
50+ if (event.toolName === "create_goal") record({ type: "goal_created" });
51+ });
52+ pi.events?.on("goal_continuation_scheduled", (data) => {
53+ record({ type: "goal_continuation_scheduled", data });
54+ });
55+ }
56+ ` ;
57+ writeFileSync ( extensionPath , source ) ;
58+ return { extraArgs : [ "--extension" , extensionPath ] , eventLogPath } ;
59+ }
60+
61+ function readGoalState ( box ) {
62+ const goalDir = join ( box . sessionDir , "extensions" , "goal" ) ;
63+ if ( ! existsSync ( goalDir ) ) return undefined ;
64+ const goalFile = readdirSync ( goalDir )
65+ . filter ( ( name ) => name . endsWith ( ".json" ) )
66+ . map ( ( name ) => join ( goalDir , name ) )
67+ . at ( 0 ) ;
68+ return goalFile === undefined ? undefined : JSON . parse ( readFileSync ( goalFile , "utf8" ) ) ;
69+ }
70+
71+ function readFixtureEvents ( path ) {
72+ if ( ! existsSync ( path ) ) return [ ] ;
73+ return readFileSync ( path , "utf8" )
74+ . split ( / \r ? \n / )
75+ . filter ( Boolean )
76+ . map ( ( line ) => JSON . parse ( line ) ) ;
77+ }
78+
3179async function runRepetitiveTurnsScenario ( { apiName, driveTurn, evidenceSlug, checks, guard, finalMarker, scenarioName } ) {
32- const { box, server, result } = await driveTurn ( {
80+ const { box, server, result, prepared } = await driveTurn ( {
3381 apiName,
3482 turns : [
3583 { text : REPEATED_STATUS_TURNS [ 0 ] } ,
84+ { toolCalls : [ { name : "create_goal" , args : { objective : "Keep the live monitor wait active" } } ] } ,
3685 { text : REPEATED_STATUS_TURNS [ 1 ] } ,
37- { text : REPEATED_STATUS_TURNS [ 2 ] } ,
3886 { text : finalMarker } ,
3987 ] ,
4088 prompt : `Report status repeatedly and finish with ${ finalMarker } .` ,
4189 extraArgs : [ "--approve" ] ,
42- followUpPrompts : [ "continue" , "continue" ] ,
90+ followUpPrompts : [ "Create a Goal and continue monitoring" ] ,
91+ prepareSandbox : writeGoalMonitorFixture ,
4392 timeoutMs : 180000 ,
4493 } ) ;
4594
4695 try {
4796 const output = `${ result . stdout } \n${ result . stderr } ` ;
4897 const allBodies = JSON . stringify ( server . requests . map ( ( r ) => r . body ?? r . raw ?? "" ) ) ;
98+ const goalState = readGoalState ( box ) ;
99+ const fixtureEvents = readFixtureEvents ( prepared . eventLogPath ) ;
100+ const goalCreatedIndex = fixtureEvents . findIndex ( ( event ) => event . type === "goal_created" ) ;
101+ const systemAbortIndex = fixtureEvents . findIndex (
102+ ( event ) => event . type === "agent_end" && event . aborted === true && event . abortSource === "system" ,
103+ ) ;
104+ const recoveryIndex = fixtureEvents . findIndex (
105+ ( event , index ) => index > systemAbortIndex && event . type === "agent_end" && event . abortSource === undefined ,
106+ ) ;
107+ const monitorScheduleIndex = fixtureEvents . findIndex (
108+ ( event , index ) =>
109+ index > goalCreatedIndex &&
110+ event . type === "goal_continuation_scheduled" &&
111+ event . data ?. activeMonitorCount === 1 ,
112+ ) ;
49113 checks . ok ( `${ scenarioName } : CLI exits zero` , result . code === 0 && ! result . timedOut , `code=${ result . code } ` ) ;
50114 checks . ok (
51115 `${ scenarioName } : cross-turn repetition triggered an extra bounded turn` ,
52- server . requests . length > 2 ,
116+ server . requests . length === 4 ,
53117 `requests=${ server . requests . length } ` ,
54118 ) ;
55119 checks . ok (
@@ -58,8 +122,34 @@ async function runRepetitiveTurnsScenario({ apiName, driveTurn, evidenceSlug, ch
58122 `interruptPresent=${ allBodies . includes ( "repetitive-turns" ) } ` ,
59123 ) ;
60124 checks . ok ( `${ scenarioName } : recovery answer returned` , output . includes ( finalMarker ) , `marker=${ finalMarker } ` ) ;
125+ const hiddenRuntimeError = / A g e n t i s a l r e a d y p r o c e s s i n g | E x t e n s i o n e r r o r \( [ ^ ) ] * \) : T h i s e x t e n s i o n c t x i s s t a l e / . test ( output ) ;
126+ checks . ok (
127+ `${ scenarioName } : no hidden runtime or stale-context errors` ,
128+ ! hiddenRuntimeError ,
129+ `hiddenRuntimeError=${ hiddenRuntimeError } ` ,
130+ ) ;
131+ checks . ok (
132+ `${ scenarioName } : final persisted Goal remains active` ,
133+ goalState ?. goal ?. status === "active" ,
134+ `status=${ goalState ?. goal ?. status ?? "missing" } ` ,
135+ ) ;
136+ checks . ok (
137+ `${ scenarioName } : active Goal exists before the TTSR system abort` ,
138+ goalCreatedIndex >= 0 && systemAbortIndex > goalCreatedIndex ,
139+ `goalCreatedIndex=${ goalCreatedIndex } systemAbortIndex=${ systemAbortIndex } ` ,
140+ ) ;
141+ checks . ok (
142+ `${ scenarioName } : TTSR system abort is followed by recovery with monitor wait live` ,
143+ recoveryIndex > systemAbortIndex &&
144+ monitorScheduleIndex > goalCreatedIndex &&
145+ monitorScheduleIndex < recoveryIndex ,
146+ `systemAbortIndex=${ systemAbortIndex } recoveryIndex=${ recoveryIndex } monitorScheduleIndex=${ monitorScheduleIndex } ` ,
147+ ) ;
61148 guard . assertUnchanged ( ) ;
62- if ( evidenceSlug ) writeTtsrEvidence ( evidenceSlug , scenarioName , result , server ) ;
149+ if ( evidenceSlug ) {
150+ const dir = writeTtsrEvidence ( evidenceSlug , scenarioName , result , server ) ;
151+ writeFileSync ( join ( dir , `${ scenarioName } -state.json` ) , JSON . stringify ( { goalState, fixtureEvents } , null , 2 ) ) ;
152+ }
63153 } finally {
64154 await server . stop ( ) ;
65155 box . cleanup ( ) ;
0 commit comments