fix(coding-agent): recover required compaction loops

packages/coding-agent/src/changes.md

@@ -1,3 +1,36 @@
+## Keep long-running compaction recovery progressing (2026-08-03)
+
+### What changed
+
+- The automatic-compaction soft cap now resets after each provider turn instead of lasting for the
+  whole multi-tool agent run. The completed turn's zero-yield recovery still observes its original cap
+  before the reset, while the absolute session cap remains authoritative for every route.
+- Required-compaction failures now resume queued work after an accepted recovery compaction without a
+  synthetic `continue`, while rejected recovery remains terminal.
+- Provenance-confirmed required recovery uses the persisted byte-derived estimate when no valid
+  provider usage sample exists.
+- Deterministic recovery measures the reconstructed suffix instead of stale cumulative assistant usage.
+  It keeps the prepared boundary when safe and otherwise advances to the latest complete persisted user
+  turn, including expanded skill text and its chronological suffix, with strict retained-message schemas.
+
+### Why
+
+- Long `ulw` runs could complete three valid compactions and then reject every later threshold
+  compaction as if the whole agent run were one provider turn.
+- When summarization then failed, a fitting skill-bearing suffix could be rejected because provider
+  usage still described the discarded pre-compaction prefix. Repeated continuations surfaced the same
+  threshold error instead of recovering.
+
+### Why this cannot be expressed externally
+
+- The fix depends on internal provider-turn lifecycle state, exact session entry boundaries, compaction
+  admission, and continuation ownership.
+
+### Expected merge conflict zones
+
+- `src/core/agent-session.ts` compaction retry/continuation ownership and upstream telemetry lifecycle.
+- `src/core/extensions/builtin/compaction/` admission, fallback, and provider-turn accounting.
+
 ## Compact completed apply_patch result details (2026-08-02)
 
 ### What changed

packages/coding-agent/src/core/agent-session.ts

@@ -610,6 +610,10 @@ export class AgentSession {
 	private readonly _sessionWorkBarrier = new SessionWorkBarrier();
 	private _overflowRecoveryAttempted = false;
 	private _requiredCompactionAdmissionError: RequiredCompactionError | undefined;
+	// Preserve provenance across agent-core's conversion of our admission error
+	// into an assistant error message. Matching provider text alone is not proof
+	// that AgentSession initiated required-compaction recovery.
+	private _requiredCompactionTurnError: RequiredCompactionError | undefined;
 	// A retry continuation immediately follows an accepted compaction. Its first
 	// response must not retrigger threshold compaction from stale provider usage.
 	private _skipNextPostRetryCompactionCheck = false;
@@ -997,8 +1001,11 @@ export class AgentSession {
 				try {
 					return await this._enforceCompactionBeforeProvider(turn.message, true, "threshold");
 				} catch (error) {
-					if (error instanceof RequiredCompactionError && this.agent.hasQueuedMessages()) {
-						this._requiredCompactionAdmissionError = error;
+					if (error instanceof RequiredCompactionError) {
+						this._requiredCompactionTurnError = error;
+						if (this.agent.hasQueuedMessages()) {
+							this._requiredCompactionAdmissionError = error;
+						}
 					}
 					throw error;
 				}
@@ -1451,16 +1458,17 @@ export class AgentSession {
 			const messages = filterContextExcludedMessages(this.agent.state.messages);
 			const estimate = estimateContextTokens(messages);
 			if (estimate.lastUsageIndex === null) {
-				return undefined;
-			}
-			const compactionEntry = getLatestCompactionEntry(this.sessionManager.getBranch());
-			const usageMessage = messages[estimate.lastUsageIndex];
-			if (
-				compactionEntry &&
-				usageMessage?.role === "assistant" &&
-				this._isAssistantFromBeforeLatestCompaction(usageMessage)
-			) {
-				return undefined;
+				if (!this._isRequiredCompactionError(message)) return undefined;
+			} else {
+				const compactionEntry = getLatestCompactionEntry(this.sessionManager.getBranch());
+				const usageMessage = messages[estimate.lastUsageIndex];
+				if (
+					compactionEntry &&
+					usageMessage?.role === "assistant" &&
+					this._isAssistantFromBeforeLatestCompaction(usageMessage)
+				) {
+					return undefined;
+				}
 			}
 			contextTokens = estimate.tokens;
 		}
@@ -1528,13 +1536,19 @@ export class AgentSession {
 	}
 
 	private _willRetryAfterAgentEnd(messages: AgentMessage[]): boolean {
-		const settings = this.settingsManager.getRetrySettings();
-		if (!settings.enabled) {
+		const lastAssistant = this._lastAssistantMessage ?? this._findLastAssistantInMessages(messages);
+		if (!lastAssistant) {
 			return false;
 		}
+		if (
+			this._isRequiredCompactionError(lastAssistant) &&
+			this._getRequiredAutoCompactionReason(lastAssistant) !== undefined
+		) {
+			return true;
+		}
 
-		const lastAssistant = this._lastAssistantMessage ?? this._findLastAssistantInMessages(messages);
-		if (!lastAssistant) {
+		const settings = this.settingsManager.getRetrySettings();
+		if (!settings.enabled) {
 			return false;
 		}
 
@@ -1567,7 +1581,18 @@ export class AgentSession {
 		return this._retryFallback.canTryFallback();
 	}
 
+	private _isRequiredCompactionError(message: AssistantMessage): boolean {
+		return (
+			this._requiredCompactionTurnError !== undefined &&
+			message.stopReason === "error" &&
+			message.errorMessage === this._requiredCompactionTurnError.message
+		);
+	}
+
 	private async _processAgentEvent(event: AgentEvent, signal: AbortSignal): Promise<void> {
+		if (event.type === "agent_start") {
+			this._requiredCompactionTurnError = undefined;
+		}
 		// When a user message starts, check if it's from either queue and remove it BEFORE emitting
 		// This ensures the UI sees the updated queue state
 		if (event.type === "message_start" && event.message.role === "user") {
@@ -1684,6 +1709,8 @@ export class AgentSession {
 			this._lastAssistantMessage = undefined;
 			this._skipNextPostRetryCompactionCheck = false;
 			const requiredAutoCompaction = this._getRequiredAutoCompactionReason(msg);
+			const retryAfterRequiredCompaction =
+				requiredAutoCompaction !== undefined && this._isRequiredCompactionError(msg);
 
 			// Retry transient failures normally and eligible hard errors only through a fallback.
 			const retryableError = this._isRetryableError(msg);
@@ -1718,7 +1745,7 @@ export class AgentSession {
 					this._scheduleContinuationAfterCurrentEvent();
 					launchedContinuation = true;
 				} else {
-					launchedContinuation = await this._checkCompaction(msg);
+					launchedContinuation = await this._checkCompaction(msg, true, undefined, retryAfterRequiredCompaction);
 					allowsPostCompactionUsageExemptContinuation = this._postCompactionUsageExemptAssistants.has(msg);
 					if (allowsPostCompactionUsageExemptContinuation) {
 						this._flushPostCompactionDeferredMessages();
@@ -4640,17 +4667,20 @@ export class AgentSession {
 			} else {
 				const messages = filterContextExcludedMessages(this.agent.state.messages);
 				const estimate = estimateContextTokens(messages);
-				if (estimate.lastUsageIndex === null) return false; // No usage data at all
-				// Verify the usage source is post-compaction. Kept pre-compaction messages
-				// have stale usage reflecting the old (larger) context and would falsely
-				// trigger compaction right after one just finished.
-				const usageMsg = messages[estimate.lastUsageIndex];
-				if (
-					compactionEntry &&
-					usageMsg.role === "assistant" &&
-					this._isAssistantFromBeforeLatestCompaction(usageMsg)
-				) {
-					return false;
+				if (estimate.lastUsageIndex === null) {
+					if (!this._isRequiredCompactionError(assistantMessage)) return false;
+				} else {
+					// Verify the usage source is post-compaction. Kept pre-compaction messages
+					// have stale usage reflecting the old (larger) context and would falsely
+					// trigger compaction right after one just finished.
+					const usageMsg = messages[estimate.lastUsageIndex];
+					if (
+						compactionEntry &&
+						usageMsg.role === "assistant" &&
+						this._isAssistantFromBeforeLatestCompaction(usageMsg)
+					) {
+						return false;
+					}
 				}
 				contextTokens = estimate.tokens;
 			}
@@ -4664,7 +4694,7 @@ export class AgentSession {
 					retryAfterCompaction,
 				);
 			} else {
-				const compacted = await this._runAutoCompaction("threshold", false);
+				const compacted = await this._runAutoCompaction("threshold", retryAfterCompaction);
 				if (
 					!compacted &&
 					this._compactionLifecycle.state.status === "failed" &&

packages/coding-agent/src/core/changes.md

@@ -21,6 +21,32 @@
 
 - LOW: `agent-session.ts` compaction start/end logging correlation and `test/session-log-routes.test.ts` lifecycle telemetry coverage.
 
+## Required-compaction continuation recovery (2026-08-03)
+
+### What changed
+
+- `AgentSession` marks only provenance-confirmed required-compaction admission errors as retrying.
+- Accepted post-turn threshold compaction resumes the exact interrupted continuation, including queued
+  steering input, without fabricating a user `continue`.
+- A locally proven required-compaction error can use the persisted byte estimate when every provider
+  usage sample is missing or zero.
+- Rejected recovery stays terminal, provider errors with the same text do not gain retry provenance,
+  and one recovery sequence persists one threshold error.
+
+### Why
+
+- Required admission previously surfaced as a terminal provider failure before the recovery compaction
+  finished, leaving active work idle even after a successful compaction.
+
+### Why this cannot be expressed externally
+
+- Only the session runtime owns the interrupted continuation, compaction lifecycle, provider-admission
+  ordering, and queued-input precedence.
+
+### Expected merge conflict zones
+
+- `agent-session.ts` required-compaction provenance, `_runAutoCompaction()`, and upstream request-ID telemetry.
+
 ## Prefer configured client fallback chains over server substitutions (2026-08-03)
 
 ### What changed

packages/coding-agent/src/core/extensions/builtin/compaction/changes.md

@@ -1,5 +1,38 @@
 # Builtin compaction extension changes
 
+## Reset the cap per provider turn and retain a safe deterministic suffix (2026-08-03)
+
+### What changed
+
+- `turn_end` now resets the soft compaction counters after the completed turn's degradation and
+  zero-yield recovery checks. `agent_end` keeps its existing final reset, and the absolute session cap
+  is checked before every manual, extension, or automatic route.
+- Deterministic required recovery projects the exact post-compaction context and ignores cumulative
+  assistant usage that refers to the discarded prefix.
+- The fallback prefers the prepared boundary, then tries the latest meaningful persisted user boundary
+  once and retains every following message in order.
+- Recovery remains fail-closed for oversized suffixes, images, provider-native blocks, opaque replay
+  signatures, branch summaries, malformed message envelopes or known block schemas, and empty or
+  default-ignorable user boundaries.
+
+### Why
+
+- The previous “per-turn” counter lasted for an entire multi-tool agent run, so the fourth valid
+  compaction was rejected even after three separate provider turns.
+- A loaded skill is ordinary user text, but stale assistant usage could make that small suffix appear
+  larger than the input cap, and the fallback had no later safe boundary to try.
+
+### Why this cannot be expressed externally
+
+- The behavior depends on builtin lifecycle state, canonical session reconstruction, and internal
+  replay-safety metadata.
+
+### Expected merge conflict zones
+
+- `index.ts` `turn_end`/`agent_end` lifecycle accounting and blocking-compaction admission.
+- `deterministic-fallback.ts` retained-suffix projection and metadata.
+- `retained-message-safety.ts` normalized replay-envelope and content validation.
+
 ## Idle warm-up retries transient failures while the session stays idle (2026-08-03)
 
 ### What changed

packages/coding-agent/src/core/extensions/builtin/compaction/deterministic-fallback.ts

@@ -1,7 +1,9 @@
-import { type CompactionPreparation, type CompactionResult, estimateContextTokens } from "../../../compaction/index.ts";
+import { type CompactionPreparation, type CompactionResult, estimateTokens } from "../../../compaction/index.ts";
 import { StreamDurationBudgetError, StreamIdleTimeoutError } from "../../../compaction/stream-watchdog.ts";
+import { filterContextExcludedMessages } from "../../../messages.ts";
 import { buildSessionContext, type CompactionEntry, type SessionEntry } from "../../../session-manager.ts";
 import { SummarizationOverflowExhaustedError } from "./overflow-retry.ts";
+import { hasUnsafeRetainedContent } from "./retained-message-safety.ts";
 import { SummaryRequestError } from "./speculative.ts";
 import { capUtf8Bytes } from "./task-intent.ts";
 
@@ -20,7 +22,36 @@ interface DeterministicFallbackDetails {
 	schema: "senpi.compaction.deterministic-fallback.v1";
 	origin: "required-compaction-recovery";
 	failureKind: RequiredCompactionFallbackFailure;
-	retainedSuffix?: "prepared";
+	taskIntent?: string;
+	retainedSuffix?: "prepared" | "latest-user-turn";
+}
+
+const NON_VISIBLE_USER_TEXT = /[\p{White_Space}\p{Default_Ignorable_Code_Point}]/gu;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function hasMeaningfulUserText(entry: SessionEntry): boolean {
+	if (entry.type !== "message" || !isRecord(entry.message) || entry.message.role !== "user") return false;
+	const content = entry.message.content;
+	const hasVisibleText = (text: unknown): boolean =>
+		typeof text === "string" && text.normalize("NFKC").replace(NON_VISIBLE_USER_TEXT, "").length > 0;
+	if (typeof content === "string") return hasVisibleText(content);
+	if (!Array.isArray(content)) return false;
+	return content.some((block) => isRecord(block) && block.type === "text" && hasVisibleText(block.text));
+}
+
+function estimateConservativeTokens(messages: ReturnType<typeof filterContextExcludedMessages>): number {
+	try {
+		return messages.reduce((tokens, message) => {
+			const serialized = JSON.stringify(message);
+			if (serialized === undefined) return Number.POSITIVE_INFINITY;
+			return tokens + Math.max(estimateTokens(message), Buffer.byteLength(serialized));
+		}, 0);
+	} catch {
+		return Number.POSITIVE_INFINITY;
+	}
 }
 
 export function classifyRequiredCompactionFallbackFailure(
@@ -45,7 +76,8 @@ export function createRequiredCompactionFallback(
 	metadata: RecoveryMetadata,
 	branchEntries: SessionEntry[] = [],
 ): CompactionResult<DeterministicFallbackDetails> | undefined {
-	if (!preparation.firstKeptEntryId || !branchEntries.some((entry) => entry.id === preparation.firstKeptEntryId)) {
+	const preparedBoundaryIndex = branchEntries.findIndex((entry) => entry.id === preparation.firstKeptEntryId);
+	if (!preparation.firstKeptEntryId || preparedBoundaryIndex === -1) {
 		return undefined;
 	}
 
@@ -75,30 +107,49 @@ export function createRequiredCompactionFallback(
 		failureKind,
 		...(taskIntent ? { taskIntent } : {}),
 	};
-	const result: CompactionResult<DeterministicFallbackDetails> = {
-		summary,
-		firstKeptEntryId: preparation.firstKeptEntryId,
-		tokensBefore: preparation.tokensBefore,
-		details: baseDetails,
-	};
-	const syntheticCompaction: CompactionEntry = {
-		type: "compaction",
-		id: "__senpi_deterministic_fallback_preview__",
-		parentId: branchEntries.at(-1)?.id ?? null,
-		timestamp: new Date(0).toISOString(),
-		summary: result.summary,
-		firstKeptEntryId: result.firstKeptEntryId,
-		tokensBefore: result.tokensBefore,
-		details: result.details,
-		fromHook: true,
-	};
-	const retainedTokens = estimateContextTokens(
-		buildSessionContext([...branchEntries, syntheticCompaction]).messages,
-	).tokens;
-	if (retainedTokens > contextWindow - preparation.settings.reserveTokens) return undefined;
-	return {
-		...result,
-		estimatedTokensAfter: retainedTokens,
-		details: { ...baseDetails, retainedSuffix: "prepared" },
+	const projectCandidate = (
+		firstKeptEntryId: string,
+		retainedSuffix: NonNullable<DeterministicFallbackDetails["retainedSuffix"]>,
+	): CompactionResult<DeterministicFallbackDetails> | undefined => {
+		const details = { ...baseDetails, retainedSuffix };
+		const result: CompactionResult<DeterministicFallbackDetails> = {
+			summary,
+			firstKeptEntryId,
+			tokensBefore: preparation.tokensBefore,
+			details,
+		};
+		const syntheticCompaction: CompactionEntry = {
+			type: "compaction",
+			id: "__senpi_deterministic_fallback_preview__",
+			parentId: branchEntries.at(-1)?.id ?? null,
+			timestamp: new Date(0).toISOString(),
+			summary: result.summary,
+			firstKeptEntryId: result.firstKeptEntryId,
+			tokensBefore: result.tokensBefore,
+			details: result.details,
+			fromHook: true,
+		};
+		let retainedMessages: ReturnType<typeof filterContextExcludedMessages>;
+		try {
+			retainedMessages = filterContextExcludedMessages(
+				buildSessionContext([...branchEntries, syntheticCompaction]).messages,
+			);
+		} catch {
+			return undefined;
+		}
+		if (hasUnsafeRetainedContent(retainedMessages)) return undefined;
+		const retainedTokens = estimateConservativeTokens(retainedMessages);
+		if (retainedTokens > contextWindow - preparation.settings.reserveTokens) return undefined;
+		return { ...result, estimatedTokensAfter: retainedTokens };
 	};
+
+	const prepared = projectCandidate(preparation.firstKeptEntryId, "prepared");
+	if (prepared) return prepared;
+
+	for (let index = branchEntries.length - 1; index > preparedBoundaryIndex; index--) {
+		const entry = branchEntries[index];
+		if (!hasMeaningfulUserText(entry)) continue;
+		return projectCandidate(entry.id, "latest-user-turn");
+	}
+	return undefined;
 }