test(coding-agent): cover provider-error goal recovery

Exercise the real AgentSession path from terminal provider failure through direct-input reactivation and completion while preserving deliberate blocks.

Add an isolated RPC QA scenario that proves the same lifecycle through the source CLI without real credentials or provider traffic.

Author: realsigridjin

.agents/skills/senpi-qa/scripts/goal-provider-error-recovery.mjs

@@ -0,0 +1,247 @@
+#!/usr/bin/env node
+
+import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { createChecks, evidenceDir, guardRealAuth, installCleanupHooks, makeSandbox } from "./lib/common.mjs";
+import { startFakeModelServer } from "./lib/fake-model-server.mjs";
+import { API_PRESETS, hermeticEnv, writeMockModelsJson } from "./lib/mock-loop-support.mjs";
+import { RpcQaClient } from "./lib/rpc-qa-client.mjs";
+
+const PROVIDER_ERROR_REASON = "provider error ended the turn (retries exhausted)";
+const INTENTIONAL_REASON = "waiting on an explicit user decision";
+const TURNS = [
+	{ toolCalls: [{ id: "qa-create-goal", name: "create_goal", args: { objective: "Survive a provider outage" } }] },
+	{ error: { status: 400, message: "SENPI_QA_TERMINAL_PROVIDER_ERROR" } },
+	{ toolCalls: [{ id: "qa-get-reactivated-goal", name: "get_goal", args: {} }] },
+	{
+		toolCalls: [
+			{
+				id: "qa-intentional-block",
+				name: "update_goal",
+				args: { status: "blocked", reason: INTENTIONAL_REASON },
+			},
+		],
+	},
+	{ text: "SENPI-QA-INTENTIONAL-BLOCK-SET" },
+	{ toolCalls: [{ id: "qa-get-still-blocked-goal", name: "get_goal", args: {} }] },
+	{ text: "SENPI-QA-INTENTIONAL-BLOCK-PRESERVED" },
+];
+
+function flag(name) {
+	const index = process.argv.indexOf(name);
+	return index >= 0 ? process.argv[index + 1] : undefined;
+}
+
+function findJsonFiles(root) {
+	if (!existsSync(root)) return [];
+	return readdirSync(root, { withFileTypes: true }).flatMap((entry) => {
+		const path = join(root, entry.name);
+		if (entry.isDirectory()) return findJsonFiles(path);
+		return entry.isFile() && entry.name.endsWith(".json") ? [path] : [];
+	});
+}
+
+function readGoal(agentDir) {
+	const root = join(agentDir, "extensions", "goal");
+	const files = findJsonFiles(root);
+	if (files.length !== 1) throw new Error(`Expected one goal under ${root}, found ${files.length}`);
+	const parsed = JSON.parse(readFileSync(files[0], "utf8"));
+	const goal = parsed?.goal ?? parsed;
+	if (!goal || typeof goal.status !== "string") throw new Error(`Invalid goal record in ${files[0]}`);
+	return goal;
+}
+
+function toolGoal(events, toolCallId) {
+	const event = events.find(
+		(candidate) => candidate.type === "tool_execution_end" && candidate.toolCallId === toolCallId,
+	);
+	const text = event?.result?.content?.find((part) => part.type === "text")?.text;
+	if (typeof text !== "string") throw new Error(`Missing result for ${toolCallId}`);
+	const goal = JSON.parse(text)?.goal;
+	if (!goal || typeof goal.status !== "string") throw new Error(`Invalid goal result for ${toolCallId}`);
+	return goal;
+}
+
+function safeRequests(requests) {
+	return requests.map((request, index) => ({
+		index: index + 1,
+		method: request.method,
+		url: request.url,
+		model: request.model,
+		messageCount: Array.isArray(request.messages) ? request.messages.length : null,
+	}));
+}
+
+function writeJson(path, value) {
+	writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+async function runPrompt(client, message) {
+	const afterIndex = client.events.length;
+	const terminalPromise = client.waitForEvent(
+		(event) => event.type === "agent_end" || event.type === "agent_aborted",
+		afterIndex,
+		60_000,
+	);
+	const acknowledgement = await client.send({ type: "prompt", message }, 15_000);
+	return { acknowledgement, terminal: await terminalPromise };
+}
+
+async function main() {
+	if (!process.argv.includes("--self-test")) {
+		process.stderr.write("usage: goal-provider-error-recovery.mjs --self-test [--evidence SLUG]\n");
+		process.exitCode = 2;
+		return;
+	}
+
+	installCleanupHooks();
+	const checks = createChecks("goal-provider-error-recovery.mjs --self-test");
+	const evidence = evidenceDir(flag("--evidence") ?? "goal-provider-error-recovery");
+	const authGuard = guardRealAuth();
+	const box = makeSandbox("senpi-qa-goal-provider-error-recovery");
+	const preset = API_PRESETS["openai-completions"];
+	const observed = { terminals: [], goals: {}, requestCount: 0, localhostOnly: false };
+	let server;
+	let client;
+	let rpcExitCode = null;
+	let realAuthUnchanged = false;
+
+	try {
+		server = await startFakeModelServer({ turns: TURNS });
+		writeMockModelsJson(box.agentDir, server, "openai-completions", {}, {
+			retry: {
+				enabled: false,
+				maxRetries: 0,
+				baseDelayMs: 0,
+				provider: { maxRetries: 0, maxRetryDelayMs: 0 },
+				fallbackChains: {},
+			},
+		});
+		client = new RpcQaClient({
+			env: hermeticEnv(box.env),
+			cwd: box.cwd,
+			extraArgs: ["--provider", preset.provider, "--model", preset.modelId],
+		});
+
+		const state = await client.send({ type: "get_state" });
+		checks.ok("RPC booted in the isolated sandbox", state.success === true && state.command === "get_state");
+
+		const first = await runPrompt(client, "Create the scripted recovery goal and begin it.");
+		observed.terminals.push(first.terminal.type);
+		checks.ok(
+			"provider-error turn ended through agent_end",
+			first.acknowledgement.success === true && first.terminal.type === "agent_end",
+		);
+		observed.goals.providerError = readGoal(box.agentDir);
+		checks.ok("provider error stopped after exactly two localhost requests", server.requests.length === 2);
+		checks.ok(
+			"provider error persisted the mechanical blocked reason",
+			observed.goals.providerError.status === "blocked" &&
+				observed.goals.providerError.blockedReason === PROVIDER_ERROR_REASON,
+		);
+
+		const second = await runPrompt(
+			client,
+			"Retry the goal now, inspect its status, then apply the scripted intentional block.",
+		);
+		observed.terminals.push(second.terminal.type);
+		checks.ok(
+			"next direct prompt was accepted",
+			second.acknowledgement.success === true && second.terminal.type === "agent_end",
+		);
+		observed.goals.reactivated = toolGoal(client.events, "qa-get-reactivated-goal");
+		checks.ok(
+			"get_goal observed active before the next model action",
+			observed.goals.reactivated.status === "active" &&
+				observed.goals.reactivated.blockedReason === undefined,
+		);
+		observed.goals.intentional = readGoal(box.agentDir);
+		checks.ok(
+			"model-authored update_goal persisted an intentional block",
+			observed.goals.intentional.status === "blocked" &&
+				observed.goals.intentional.blockedReason === INTENTIONAL_REASON,
+		);
+
+		const third = await runPrompt(
+			client,
+			"Inspect the intentionally blocked goal without explicitly resuming it.",
+		);
+		observed.terminals.push(third.terminal.type);
+		checks.ok(
+			"later direct prompt was accepted",
+			third.acknowledgement.success === true && third.terminal.type === "agent_end",
+		);
+		observed.goals.toolPreserved = toolGoal(client.events, "qa-get-still-blocked-goal");
+		observed.goals.preserved = readGoal(box.agentDir);
+		checks.ok(
+			"get_goal observed the intentional block unchanged",
+			observed.goals.toolPreserved.status === "blocked" &&
+				observed.goals.toolPreserved.blockedReason === INTENTIONAL_REASON &&
+				observed.goals.preserved.id === observed.goals.intentional.id &&
+				observed.goals.preserved.status === "blocked" &&
+				observed.goals.preserved.blockedReason === INTENTIONAL_REASON &&
+				Number.isFinite(observed.goals.intentional.blockedAt) &&
+				observed.goals.preserved.blockedAt === observed.goals.intentional.blockedAt,
+		);
+
+		observed.requestCount = server.requests.length;
+		checks.ok("scripted run made exactly seven localhost provider requests", observed.requestCount === 7);
+		observed.localhostOnly =
+			server.origin.startsWith("http://127.0.0.1:") &&
+			server.requests.every(
+				(request) => request.method === "POST" && request.url?.endsWith("/chat/completions"),
+			);
+		checks.ok("zero real provider calls", observed.localhostOnly, server.origin);
+	} catch (error) {
+		checks.ok("scenario completed without an exception", false, error instanceof Error ? error.message : String(error));
+	} finally {
+		if (client) {
+			client.close();
+			try {
+				rpcExitCode = await client.waitForExit(5_000);
+			} catch {
+				client.kill();
+				rpcExitCode = await client.waitForExit(5_000).catch(() => null);
+			}
+			checks.ok("RPC process exited after stdin closed", rpcExitCode === 0, `exitCode=${rpcExitCode}`);
+		}
+		if (server) await server.stop();
+		try {
+			realAuthUnchanged = authGuard.assertUnchanged();
+		} catch {
+			realAuthUnchanged = false;
+		}
+		checks.ok("real auth unchanged", realAuthUnchanged, authGuard.path);
+		box.cleanup();
+		checks.ok("isolated sandbox removed", !existsSync(box.dir), box.dir);
+	}
+
+	const pass = checks.finish();
+	writeJson(join(evidence, "summary.json"), {
+		pass,
+		terminalEvents: observed.terminals,
+		providerRequestCount: observed.requestCount,
+		providerErrorStatus: observed.goals.providerError?.status ?? null,
+		reactivatedStatus: observed.goals.reactivated?.status ?? null,
+		intentionalStatus: observed.goals.preserved?.status ?? null,
+		blockedAtUnchanged:
+			observed.goals.intentional?.blockedAt === observed.goals.preserved?.blockedAt,
+		localhostOnly: observed.localhostOnly,
+		realAuthUnchanged,
+		rpcExitCode,
+		serverStopped: server !== undefined,
+		sandboxRemoved: !existsSync(box.dir),
+	});
+	writeFileSync(
+		join(evidence, "rpc-events.jsonl"),
+		`${(client?.events ?? []).map((event) => JSON.stringify(event)).join("\n")}\n`,
+	);
+	for (const [name, goal] of Object.entries(observed.goals)) {
+		writeJson(join(evidence, `goal-${name}.json`), goal);
+	}
+	writeJson(join(evidence, "mock-request-summary.json"), safeRequests(server?.requests ?? []));
+	process.stdout.write(`Evidence: ${evidence}\n`);
+	process.exitCode = pass ? 0 : 1;
+}
+
+await main();

packages/coding-agent/test/suite/goal-provider-error-recovery.test.ts

@@ -0,0 +1,135 @@
+import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
+import { afterEach, describe, expect, it } from "vitest";
+import goalExtension from "../../src/core/extensions/builtin/goal/index.ts";
+import { createGoal, readGoal, updateGoal } from "../../src/core/extensions/builtin/goal/store.ts";
+import { goalStoreRef } from "../../src/core/extensions/builtin/goal/store-ref.ts";
+import type { GoalStatus } from "../../src/core/extensions/builtin/goal/types.ts";
+import type { ExtensionAPI } from "../../src/core/extensions/types.ts";
+import { GOAL_CONTINUATION_MESSAGE_TYPE } from "../../src/core/messages.ts";
+import { createHarness, type Harness } from "./harness.ts";
+
+const PROVIDER_ERROR_BLOCKED_REASON = "provider error ended the turn (retries exhausted)";
+const INTENTIONAL_BLOCKED_REASON = "Waiting on a user decision";
+const harnesses: Harness[] = [];
+
+afterEach(() => {
+	for (const harness of harnesses.splice(0)) {
+		harness.cleanup();
+	}
+});
+
+function goalContinuationCount(harness: Harness): number {
+	return harness.sessionManager
+		.getEntries()
+		.filter((entry) => entry.type === "custom_message" && entry.customType === GOAL_CONTINUATION_MESSAGE_TYPE).length;
+}
+
+async function createProviderBlockedHarness(): Promise<{
+	harness: Harness;
+	ref: ReturnType<typeof goalStoreRef>;
+	statusesAtAgentStart: Array<GoalStatus | null>;
+}> {
+	const statusesAtAgentStart: Array<GoalStatus | null> = [];
+	let ref: ReturnType<typeof goalStoreRef> | undefined;
+	const observeGoalAtAgentStart = (pi: ExtensionAPI) => {
+		pi.on("agent_start", async () => {
+			statusesAtAgentStart.push(ref ? ((await readGoal(ref))?.status ?? null) : null);
+		});
+	};
+	const harness = await createHarness({
+		persistSession: true,
+		settings: { retry: { enabled: false, maxRetries: 0, baseDelayMs: 0 } },
+		extensionFactories: [goalExtension, observeGoalAtAgentStart],
+	});
+	harnesses.push(harness);
+	ref = goalStoreRef(harness.sessionManager, harness.tempDir);
+	harness.setResponses([
+		fauxAssistantMessage([fauxToolCall("create_goal", { objective: "Resume this goal after a provider outage" })], {
+			stopReason: "toolUse",
+		}),
+		fauxAssistantMessage("", {
+			stopReason: "error",
+			errorMessage: "SENPI_TEST_TERMINAL_PROVIDER_ERROR",
+		}),
+	]);
+
+	await harness.session.prompt("create the recovery goal");
+
+	expect(await readGoal(ref)).toMatchObject({
+		status: "blocked",
+		blockedReason: PROVIDER_ERROR_BLOCKED_REASON,
+	});
+	return { harness, ref, statusesAtAgentStart };
+}
+
+function completionResponses() {
+	return [
+		fauxAssistantMessage([fauxToolCall("update_goal", { status: "complete" })], {
+			stopReason: "toolUse",
+		}),
+		fauxAssistantMessage("goal recovered and completed"),
+	];
+}
+
+describe("provider-error goal recovery through the real AgentSession", () => {
+	it("reactivates a provider-error block before the next direct user run", async () => {
+		const { harness, ref, statusesAtAgentStart } = await createProviderBlockedHarness();
+		harness.setResponses(completionResponses());
+
+		await harness.session.prompt("retry the blocked goal");
+
+		expect(statusesAtAgentStart).toEqual([null, "active"]);
+		expect(await readGoal(ref)).toMatchObject({ status: "complete" });
+	});
+
+	it("does not auto-continue after an exhausted provider error", async () => {
+		const { harness, ref, statusesAtAgentStart } = await createProviderBlockedHarness();
+
+		expect(harness.eventsOfType("agent_end")).toHaveLength(1);
+		expect(harness.eventsOfType("agent_end")[0]?.willRetry).toBe(false);
+		expect(goalContinuationCount(harness)).toBe(0);
+		expect(harness.faux.state.callCount).toBe(2);
+
+		harness.setResponses(completionResponses());
+		await harness.session.prompt("resume exactly once");
+
+		expect(statusesAtAgentStart).toEqual([null, "active"]);
+		expect(harness.faux.state.callCount).toBe(4);
+		expect(goalContinuationCount(harness)).toBe(0);
+		expect(await readGoal(ref)).toMatchObject({ status: "complete" });
+	});
+
+	it("keeps a deliberate block blocked on direct user input", async () => {
+		const statusesAtAgentStart: Array<GoalStatus | null> = [];
+		let ref: ReturnType<typeof goalStoreRef> | undefined;
+		const observeGoalAtAgentStart = (pi: ExtensionAPI) => {
+			pi.on("agent_start", async () => {
+				statusesAtAgentStart.push(ref ? ((await readGoal(ref))?.status ?? null) : null);
+			});
+		};
+		const harness = await createHarness({
+			persistSession: true,
+			extensionFactories: [goalExtension, observeGoalAtAgentStart],
+		});
+		harnesses.push(harness);
+		await harness.session.bindExtensions({});
+		ref = goalStoreRef(harness.sessionManager, harness.tempDir);
+		await createGoal(ref, "Wait for a user decision");
+		const deliberatelyBlocked = await updateGoal(
+			ref,
+			{ status: "blocked", reason: INTENTIONAL_BLOCKED_REASON },
+			"model",
+		);
+		expect(deliberatelyBlocked.blockedAt).toEqual(expect.any(Number));
+		harness.setResponses([fauxAssistantMessage("the block remains intentional")]);
+
+		await harness.session.prompt("inspect without resuming");
+
+		expect(statusesAtAgentStart).toEqual(["blocked"]);
+		expect(await readGoal(ref)).toMatchObject({
+			status: "blocked",
+			blockedReason: INTENTIONAL_BLOCKED_REASON,
+			blockedAt: deliberatelyBlocked.blockedAt,
+		});
+	});
+});