From 76fc85130c2b274f20a632888f6b3d693e8d1947 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 9 Jun 2026 18:33:58 -0400 Subject: [PATCH 1/3] feat(ai): server-side compaction infrastructure (#17636) Provider-agnostic infrastructure for server-side compaction: - ai-core: `serverSideCompactionSupport` capability on model metadata (propagated to the frontend); `CompactionSettings` carried verbatim on the request; `resolveCompactionDefault` (global preference folded with the per-provider override) and the capability-gated `resolveServerSideCompaction`; the global `ai-features.serverSideCompaction` preference; opaque `CompactionResponsePart` / `CompactionMessage` marker types. - ai-chat: persisted `CompactionChatResponseContent` (+ deserializer), agent stream-to-content mapping, per-session `commonSettings.compaction` copied verbatim onto the request (the agent reads no preference). - ai-chat-ui: inline compaction marker renderer, token-usage tooltip (cumulative usage + "compacted Nx"), and the per-session tri-state control. - ai-ide / ai-copilot / ai-ollama: tolerate the compaction marker (ignore foreign-provider markers in Chat Completions / Ollama conversion). Signed-off-by: Christian W. Damus --- CHANGELOG.md | 4 + .../src/browser/ai-chat-ui-frontend-module.ts | 2 + .../src/browser/chat-input-widget.tsx | 2 +- .../compaction-part-renderer.tsx | 52 +++++++++ .../browser/chat-response-renderer/index.ts | 1 + .../chat-token-usage-indicator-util.spec.ts | 102 +++++++++++++++++- .../chat-token-usage-indicator-util.ts | 57 +++++++++- .../src/browser/session-settings-dialog.tsx | 87 ++++++++++++++- .../ai-chat-ui/src/browser/style/index.css | 19 ++++ .../ai-chat/src/common/chat-agents.spec.ts | 33 +++++- packages/ai-chat/src/common/chat-agents.ts | 10 ++ .../src/common/chat-content-deserializer.ts | 11 ++ .../ai-chat/src/common/chat-model.spec.ts | 42 ++++++++ packages/ai-chat/src/common/chat-model.ts | 74 +++++++++++++ .../src/node/copilot-language-model.ts | 2 +- .../ai-core/src/common/ai-core-preferences.ts | 11 ++ .../ai-core/src/common/language-model.spec.ts | 46 +++++++- packages/ai-core/src/common/language-model.ts | 88 ++++++++++++++- .../node/backend-language-model-registry.ts | 1 + .../src/node/ollama-language-model.ts | 3 + .../src/node/openai-response-api-utils.ts | 2 + 21 files changed, 633 insertions(+), 16 deletions(-) create mode 100644 packages/ai-chat-ui/src/browser/chat-response-renderer/compaction-part-renderer.tsx create mode 100644 packages/ai-chat/src/common/chat-model.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 52a726a553ee3..482dc990b43fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ - [Previous Changelogs](https://github.com/eclipse-theia/theia/tree/master/doc/changelogs/) +## 1.74.0 - tbd + +- [ai-anthropic, ai-chat, ai-chat-ui, ai-core, ai-openai] added provider-native server-side compaction for chat sessions, modeled as a model capability (`LanguageModelMetaData.serverSideCompactionSupport`) plus layered activation: a global preference (`ai-features.chat.serverSideCompaction`, default on), a per-provider override, and a per-session override (in the session settings dialog) that wins over both. Honored by the Anthropic Messages API (beta) and the OpenAI Responses API; the compaction marker is persisted in the session, replayed on subsequent requests, surfaced as an inline chat marker, and reflected in the token-usage tooltip (cumulative usage and a "compacted Nx" count). Providers/models without the capability ignore it [#17636](https://github.com/eclipse-theia/theia/issues/17636) + ## 1.73.0 - 6/25/2026 - [ai] supported deferred tool loading via provider tool search [#17449](https://github.com/eclipse-theia/theia/pull/17449) diff --git a/packages/ai-chat-ui/src/browser/ai-chat-ui-frontend-module.ts b/packages/ai-chat-ui/src/browser/ai-chat-ui-frontend-module.ts index 8727aebf83f03..6ac1b4cf27820 100644 --- a/packages/ai-chat-ui/src/browser/ai-chat-ui-frontend-module.ts +++ b/packages/ai-chat-ui/src/browser/ai-chat-ui-frontend-module.ts @@ -30,6 +30,7 @@ import { CodePartRenderer, CodePartRendererAction, CommandPartRenderer, + CompactionPartRenderer, CopyToClipboardButtonAction, ErrorPartRenderer, HorizontalLayoutPartRenderer, @@ -186,6 +187,7 @@ export default new ContainerModule((bind, _unbind, _isBound, rebind) => { bind(ChatResponsePartRenderer).to(NotAvailableToolCallRenderer).inSingletonScope(); bind(ChatResponsePartRenderer).to(ErrorPartRenderer).inSingletonScope(); bind(ChatResponsePartRenderer).to(ThinkingPartRenderer).inSingletonScope(); + bind(ChatResponsePartRenderer).to(CompactionPartRenderer).inSingletonScope(); bind(ChatResponsePartRenderer).to(QuestionPartRenderer).inSingletonScope(); bind(ChatResponsePartRenderer).to(ProgressPartRenderer).inSingletonScope(); bind(ChatResponsePartRenderer).to(TextPartRenderer).inSingletonScope(); diff --git a/packages/ai-chat-ui/src/browser/chat-input-widget.tsx b/packages/ai-chat-ui/src/browser/chat-input-widget.tsx index 69b1c2fa89725..dc12e18ab20b6 100644 --- a/packages/ai-chat-ui/src/browser/chat-input-widget.tsx +++ b/packages/ai-chat-ui/src/browser/chat-input-widget.tsx @@ -2235,7 +2235,7 @@ const ChatInput: React.FunctionComponent = (props: ChatInpu const tokenColorClass = showTokenUsage ? getUsageColorClass(totalTokens, props.tokenUsageWarningThreshold, props.contextWindowSize) : ''; const tokenIsWarningOrError = tokenColorClass === 'token-usage-yellow' || tokenColorClass === 'token-usage-red'; const tokenTooltip = showTokenUsage - ? buildBarTooltip(getLatestTokenUsage(props.chatModel), totalTokens, props.tokenUsageWarningThreshold, props.contextWindowSize) + ? buildBarTooltip(getLatestTokenUsage(props.chatModel), totalTokens, props.tokenUsageWarningThreshold, props.contextWindowSize, props.chatModel) : undefined; return ( diff --git a/packages/ai-chat-ui/src/browser/chat-response-renderer/compaction-part-renderer.tsx b/packages/ai-chat-ui/src/browser/chat-response-renderer/compaction-part-renderer.tsx new file mode 100644 index 0000000000000..50425305c8c0d --- /dev/null +++ b/packages/ai-chat-ui/src/browser/chat-response-renderer/compaction-part-renderer.tsx @@ -0,0 +1,52 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { ChatResponsePartRenderer } from '../chat-response-part-renderer'; +import { injectable } from '@theia/core/shared/inversify'; +import { ChatResponseContent, CompactionChatResponseContent } from '@theia/ai-chat/lib/common'; +import { ReactNode } from '@theia/core/shared/react'; +import { nls } from '@theia/core/lib/common/nls'; +import * as React from '@theia/core/shared/react'; + +@injectable() +export class CompactionPartRenderer implements ChatResponsePartRenderer { + + canHandle(response: ChatResponseContent): number { + if (CompactionChatResponseContent.is(response)) { + return 10; + } + return -1; + } + + render(response: CompactionChatResponseContent): ReactNode { + const label = nls.localize('theia/ai/chat-ui/compaction-part-renderer/compactionMarker', 'Conversation compacted'); + if (response.summary) { + return ( +
+
+ {label} +
{response.summary}
+
+
+ ); + } + return ( +
+ {label} +
+ ); + } +} diff --git a/packages/ai-chat-ui/src/browser/chat-response-renderer/index.ts b/packages/ai-chat-ui/src/browser/chat-response-renderer/index.ts index 4dee25ce5b984..e99cacc229d2d 100644 --- a/packages/ai-chat-ui/src/browser/chat-response-renderer/index.ts +++ b/packages/ai-chat-ui/src/browser/chat-response-renderer/index.ts @@ -26,6 +26,7 @@ export * from './toolcall-part-renderer'; export * from './server-toolcall-part-renderer'; export * from './not-available-toolcall-renderer'; export * from './thinking-part-renderer'; +export * from './compaction-part-renderer'; export * from './progress-part-renderer'; export * from './tool-confirmation'; export * from './delegation-tool-renderer'; diff --git a/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.spec.ts b/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.spec.ts index f6dc411fed737..4ec5d91eca31a 100644 --- a/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.spec.ts +++ b/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.spec.ts @@ -19,7 +19,9 @@ import { Emitter, Event } from '@theia/core'; import { ChatModel, ChatRequestModel, ResponseTokenUsage } from '@theia/ai-chat'; import { buildBarTooltip, + computeCumulativeTokenUsage, computeSessionTokenUsage, + countCompactions, decideTokenUsageWarning, formatTokenCount, getUsageColorClass, @@ -29,11 +31,16 @@ import { const THRESHOLD = 100; const CONTEXT_WINDOW = 200; -function createMockRequest(tokenUsage?: ResponseTokenUsage, isComplete = true): Partial { +function createMockRequest(tokenUsage?: ResponseTokenUsage, isComplete = true, contentKinds?: string[]): Partial { return { response: { tokenUsage, - isComplete + isComplete, + response: { + content: (contentKinds ?? []).map(kind => ({ kind })), + asString: () => '', + asDisplayString: () => '' + } } as ChatRequestModel['response'] }; } @@ -250,6 +257,97 @@ describe('ChatTokenUsageIndicator', () => { // The fallback (200k) should appear in the total line. expect(tooltip!.value).to.contain('200.0k'); }); + + it('includes cumulative usage line when model has token usage', () => { + const model = createMockChatModel([ + createMockRequest({ inputTokens: 50, outputTokens: 50 }), + createMockRequest({ inputTokens: 30, outputTokens: 20 }) + ]); + const usage = { inputTokens: 30, outputTokens: 20 }; + const tooltip = buildBarTooltip(usage, 50, THRESHOLD, CONTEXT_WINDOW, model); + expect(tooltip).to.not.equal(undefined); + expect(tooltip!.value).to.contain('Cumulative:'); + }); + + it('includes compaction line when model has a compaction content part', () => { + const model = createMockChatModel([ + createMockRequest({ inputTokens: 50, outputTokens: 50 }, true, ['compaction']) + ]); + const usage = { inputTokens: 50, outputTokens: 50 }; + const tooltip = buildBarTooltip(usage, 100, THRESHOLD, CONTEXT_WINDOW, model); + expect(tooltip).to.not.equal(undefined); + expect(tooltip!.value).to.contain('Compacted'); + }); + }); + + describe('computeCumulativeTokenUsage', () => { + it('returns 0 for an undefined model', () => { + expect(computeCumulativeTokenUsage(undefined)).to.equal(0); + }); + + it('returns 0 for a model with no requests', () => { + const model = createMockChatModel([]); + expect(computeCumulativeTokenUsage(model)).to.equal(0); + }); + + it('sums cumulative usage across all responses', () => { + const model = createMockChatModel([ + createMockRequest({ inputTokens: 100, outputTokens: 50 }), + createMockRequest({ inputTokens: 200, outputTokens: 60, cacheReadInputTokens: 10 }) + ]); + expect(computeCumulativeTokenUsage(model)).to.equal(420); + }); + + it('includes cache creation and read tokens in the cumulative total', () => { + const model = createMockChatModel([ + createMockRequest({ inputTokens: 100, outputTokens: 50, cacheCreationInputTokens: 20, cacheReadInputTokens: 30 }), + createMockRequest({ inputTokens: 200, outputTokens: 60 }) + ]); + expect(computeCumulativeTokenUsage(model)).to.equal(460); + }); + + it('skips requests with no token usage', () => { + const model = createMockChatModel([ + createMockRequest(undefined), + createMockRequest({ inputTokens: 100, outputTokens: 50 }) + ]); + expect(computeCumulativeTokenUsage(model)).to.equal(150); + }); + }); + + describe('countCompactions', () => { + it('returns 0 for an undefined model', () => { + expect(countCompactions(undefined)).to.equal(0); + }); + + it('returns 0 for a model with no requests', () => { + const model = createMockChatModel([]); + expect(countCompactions(model)).to.equal(0); + }); + + it('counts compaction markers across the session', () => { + const model = createMockChatModel([ + createMockRequest(undefined, true, ['text', 'compaction']), + createMockRequest(undefined, true, ['text']), + createMockRequest(undefined, true, ['compaction', 'text']) + ]); + expect(countCompactions(model)).to.equal(2); + }); + + it('returns 0 when no compaction markers are present', () => { + const model = createMockChatModel([ + createMockRequest(undefined, true, ['text']), + createMockRequest(undefined, true, ['toolCall']) + ]); + expect(countCompactions(model)).to.equal(0); + }); + + it('counts multiple compaction markers within a single response', () => { + const model = createMockChatModel([ + createMockRequest(undefined, true, ['compaction', 'compaction', 'text']) + ]); + expect(countCompactions(model)).to.equal(2); + }); }); describe('decideTokenUsageWarning', () => { diff --git a/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.ts b/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.ts index 38031cf4d0e73..713f21f15d522 100644 --- a/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.ts +++ b/packages/ai-chat-ui/src/browser/chat-token-usage-indicator-util.ts @@ -78,6 +78,13 @@ export function getUsageColorClass(totalTokens: number, threshold: number, conte return 'token-usage-red'; } +function sumUsageTokens(usage: ResponseTokenUsage): number { + return usage.inputTokens + + usage.outputTokens + + (usage.cacheCreationInputTokens ?? 0) + + (usage.cacheReadInputTokens ?? 0); +} + export function computeSessionTokenUsage(chatModel?: ChatModel): number { if (!chatModel) { return 0; @@ -86,15 +93,46 @@ export function computeSessionTokenUsage(chatModel?: ChatModel): number { for (let i = requests.length - 1; i >= 0; i--) { const usage: ResponseTokenUsage | undefined = requests[i].response.tokenUsage; if (usage) { - return usage.inputTokens - + usage.outputTokens - + (usage.cacheCreationInputTokens ?? 0) - + (usage.cacheReadInputTokens ?? 0); + return sumUsageTokens(usage); } } return 0; } +/** Sum of input + output (+ cache) tokens across every response in the session. Represents cumulative billed usage. */ +export function computeCumulativeTokenUsage(chatModel?: ChatModel): number { + if (!chatModel) { + return 0; + } + let total = 0; + for (const request of chatModel.getRequests()) { + const usage: ResponseTokenUsage | undefined = request.response.tokenUsage; + if (usage) { + total += sumUsageTokens(usage); + } + } + return total; +} + +/** + * Number of provider compaction events recorded across the session. + * Counts every `compaction` content part; a single response may contain more than one. + */ +export function countCompactions(chatModel?: ChatModel): number { + if (!chatModel) { + return 0; + } + let count = 0; + for (const request of chatModel.getRequests()) { + for (const content of request.response.response.content) { + if (content.kind === 'compaction') { + count++; + } + } + } + return count; +} + export function getLatestTokenUsage(chatModel?: ChatModel): ResponseTokenUsage | undefined { if (!chatModel) { return undefined; @@ -113,7 +151,8 @@ export function buildBarTooltip( usage: ResponseTokenUsage | undefined, totalTokens: number, threshold: number, - contextWindowSize: number = CHAT_CONTEXT_WINDOW_SIZE_FALLBACK + contextWindowSize: number = CHAT_CONTEXT_WINDOW_SIZE_FALLBACK, + chatModel?: ChatModel ): MarkdownString | undefined { if (!usage) { return undefined; @@ -148,5 +187,13 @@ export function buildBarTooltip( formatTokenCount(contextWindowSize), percentage )); + const cumulative = computeCumulativeTokenUsage(chatModel); + if (cumulative > 0) { + lines.push(nls.localize('theia/ai/chat-ui/tokenUsageTooltipCumulative', 'Cumulative: {0}', formatTokenCount(cumulative))); + } + const compactions = countCompactions(chatModel); + if (compactions > 0) { + lines.push(nls.localize('theia/ai/chat-ui/tokenUsageTooltipCompactions', 'Compacted {0}x', compactions)); + } return { value: lines.join(' \n') }; } diff --git a/packages/ai-chat-ui/src/browser/session-settings-dialog.tsx b/packages/ai-chat-ui/src/browser/session-settings-dialog.tsx index 5aca5d6c70f95..5a3ac7535d8ed 100644 --- a/packages/ai-chat-ui/src/browser/session-settings-dialog.tsx +++ b/packages/ai-chat-ui/src/browser/session-settings-dialog.tsx @@ -81,6 +81,66 @@ const ConfirmationTimeoutSection: React.FC = ({ ); +type CompactionOverride = boolean | undefined; + +interface ServerSideCompactionSectionProps { + compactionOverride: CompactionOverride; + onCompactionOverrideChange: (value: CompactionOverride) => void; +} + +const ServerSideCompactionSection: React.FC = ({ + compactionOverride, + onCompactionOverrideChange +}) => { + const selectValue = compactionOverride === true ? 'enabled' : compactionOverride === false ? 'disabled' : 'default'; + + const handleChange = (e: React.ChangeEvent): void => { + const v = e.target.value; + if (v === 'enabled') { + onCompactionOverrideChange(true); + } else if (v === 'disabled') { + onCompactionOverrideChange(false); + } else { + onCompactionOverrideChange(undefined); + } + }; + + return ( +
+
+ {nls.localize('theia/ai/session-settings-dialog/compactionHeader', 'Server-Side Compaction')} +
+
+ {nls.localize( + 'theia/ai/session-settings-dialog/compactionNote', + 'Overrides the global and per-provider compaction setting for this session. ' + + 'Applies only to providers that support server-side compaction.' + )} +
+
+ + +
+
+ ); +}; + interface AdvancedSettingsSectionProps { sectionHeader: string; } @@ -105,17 +165,21 @@ const ErrorMessage: React.FC = ({ message }) => ( interface DialogContentProps { confirmationTimeoutEnabled: boolean; confirmationTimeoutSeconds: number; + compactionOverride: CompactionOverride; errorMessage: string; onConfirmationTimeoutEnabledChange: (enabled: boolean) => void; onConfirmationTimeoutSecondsChange: (seconds: number) => void; + onCompactionOverrideChange: (value: CompactionOverride) => void; } const DialogContent: React.FC = ({ confirmationTimeoutEnabled, confirmationTimeoutSeconds, + compactionOverride, errorMessage, onConfirmationTimeoutEnabledChange, - onConfirmationTimeoutSecondsChange + onConfirmationTimeoutSecondsChange, + onCompactionOverrideChange }) => (
= ({ onEnabledChange={onConfirmationTimeoutEnabledChange} onTimeoutChange={onConfirmationTimeoutSecondsChange} /> + @@ -140,6 +208,8 @@ export class SessionSettingsDialog extends AbstractDialog { protected confirmationTimeoutEnabled: boolean; protected confirmationTimeoutSeconds: number; + protected compactionOverride: boolean | undefined; + protected preservedReasoning: CommonChatSessionSettings['reasoning']; protected contentRoot: Root; @@ -161,6 +231,9 @@ export class SessionSettingsDialog extends AbstractDialog { // Reasoning is edited in the chat input; preserve it across this dialog. this.preservedReasoning = this.settings.commonSettings?.reasoning; + // Read the per-session compaction override (undefined = use global/per-provider setting). + this.compactionOverride = this.settings.commonSettings?.compaction?.enabled; + // Extract confirmation timeout settings from commonSettings const savedTimeout = this.settings.commonSettings?.confirmationTimeout; this.confirmationTimeoutEnabled = savedTimeout !== undefined && savedTimeout > 0; @@ -214,9 +287,11 @@ export class SessionSettingsDialog extends AbstractDialog { ); } @@ -328,6 +403,13 @@ export class SessionSettingsDialog extends AbstractDialog { this.attachEditorContainer(); }; + protected handleCompactionOverrideChange = (value: boolean | undefined): void => { + this.compactionOverride = value; + this.updateSettingsFromCommonSettings(); + this.render(); + this.attachEditorContainer(); + }; + protected updateSettingsFromCommonSettings(): void { const commonSettings: CommonChatSessionSettings = {}; if (this.preservedReasoning) { @@ -336,6 +418,9 @@ export class SessionSettingsDialog extends AbstractDialog { if (this.confirmationTimeoutEnabled && !isNaN(this.confirmationTimeoutSeconds) && this.confirmationTimeoutSeconds > 0) { commonSettings.confirmationTimeout = this.confirmationTimeoutSeconds; } + if (this.compactionOverride !== undefined) { + commonSettings.compaction = { enabled: this.compactionOverride }; + } this.settings.commonSettings = Object.keys(commonSettings).length > 0 ? commonSettings : undefined; } diff --git a/packages/ai-chat-ui/src/browser/style/index.css b/packages/ai-chat-ui/src/browser/style/index.css index a851ba4a8d527..8ada161bc1117 100644 --- a/packages/ai-chat-ui/src/browser/style/index.css +++ b/packages/ai-chat-ui/src/browser/style/index.css @@ -2193,3 +2193,22 @@ details.theia-tool-confirmation-args>summary::marker { border-radius: 50%; background: var(--theia-editor-background); } + +.theia-ChatCompactionMarker { + width: 100%; + margin: 6px 0; + padding: 4px 8px; + border-top: 1px solid var(--theia-editorWidget-border); + border-bottom: 1px solid var(--theia-editorWidget-border); + color: var(--theia-descriptionForeground); + font-size: var(--theia-ui-font-size0); +} + +.theia-ChatCompactionMarker summary { + cursor: pointer; +} + +.theia-ChatCompactionMarker-summary { + margin-top: 4px; + white-space: pre-wrap; +} diff --git a/packages/ai-chat/src/common/chat-agents.spec.ts b/packages/ai-chat/src/common/chat-agents.spec.ts index 846ba804381d1..55fdfab4eed3b 100644 --- a/packages/ai-chat/src/common/chat-agents.spec.ts +++ b/packages/ai-chat/src/common/chat-agents.spec.ts @@ -17,9 +17,14 @@ import 'reflect-metadata'; import { expect } from 'chai'; -import { LanguageModel, LanguageModelMessage, LanguageModelRequirement, LanguageModelResponse, LanguageModelService, ServerToolDescriptor, UserRequest } from '@theia/ai-core'; -import { AbstractChatAgent, ChatAgentLocation } from './chat-agents'; import { + LanguageModel, LanguageModelMessage, LanguageModelRequirement, LanguageModelResponse, + LanguageModelService, LanguageModelStreamResponsePart, ServerToolDescriptor, UserRequest +} from '@theia/ai-core'; +import { AbstractChatAgent, AbstractStreamParsingChatAgent, ChatAgentLocation } from './chat-agents'; +import { + ChatResponseContent, + CompactionChatResponseContent, MutableChatModel, MutableChatRequestModel, ChatModel, @@ -182,3 +187,27 @@ describe('AbstractChatAgent.sendLlmRequest server tools', () => { expect(captured()!.serverTools).to.equal(undefined); }); }); + +class StreamParsingTestChatAgent extends AbstractStreamParsingChatAgent { + readonly id = 'stream-test-agent'; + readonly name = 'Stream Test Agent'; + readonly languageModelRequirements: LanguageModelRequirement[] = []; + protected readonly defaultLanguageModelPurpose = 'chat'; + + exposeParse(token: LanguageModelStreamResponsePart): ChatResponseContent | ChatResponseContent[] { + return this.parse(token, undefined as never); + } +} + +describe('AbstractChatAgent.parse compaction', () => { + it('creates compaction content from a compaction response part', () => { + const agent = new StreamParsingTestChatAgent(); + const content = agent.exposeParse({ compaction: { provider: 'anthropic', data: { b: 1 }, summary: 's' } }); + expect(ChatResponseContent.is(content)).to.equal(true); + expect(CompactionChatResponseContent.is(content)).to.equal(true); + const compaction = content as CompactionChatResponseContent; + expect(compaction.provider).to.equal('anthropic'); + expect(compaction.data).to.deep.equal({ b: 1 }); + expect(compaction.summary).to.equal('s'); + }); +}); diff --git a/packages/ai-chat/src/common/chat-agents.ts b/packages/ai-chat/src/common/chat-agents.ts index 0f23f2177e2fe..2c4fccd383e89 100644 --- a/packages/ai-chat/src/common/chat-agents.ts +++ b/packages/ai-chat/src/common/chat-agents.ts @@ -27,6 +27,7 @@ import { CapabilityAwareContext, GenericCapabilitySelections, getTextOfResponse, + isCompactionResponsePart, isLanguageModelStreamResponsePart, isServerToolCallResponsePart, isTextResponsePart, @@ -65,6 +66,7 @@ import { ChatRequestModel, ChatResponseContent, CommonChatSessionSettings, + CompactionChatResponseContentImpl, ErrorChatResponseContentImpl, MarkdownChatResponseContentImpl, MutableChatRequestModel, @@ -547,6 +549,7 @@ export abstract class AbstractChatAgent implements ChatAgent { serverTools: enabledServerTools.length > 0 ? enabledServerTools : undefined, settings, reasoning: commonSettings?.reasoning, + compaction: commonSettings?.compaction, agentId: this.id, sessionId: request.session.id, requestId: request.id, @@ -776,6 +779,13 @@ export abstract class AbstractStreamParsingChatAgent extends AbstractChatAgent { request.response.setTokenUsage(this.mapUsageResponsePart(token)); return []; } + if (isCompactionResponsePart(token)) { + return new CompactionChatResponseContentImpl( + token.compaction.provider, + token.compaction.data, + token.compaction.summary + ); + } return this.defaultContentFactory.create('', request); } diff --git a/packages/ai-chat/src/common/chat-content-deserializer.ts b/packages/ai-chat/src/common/chat-content-deserializer.ts index c1ca7e2793185..c3b74d0ce4cb6 100644 --- a/packages/ai-chat/src/common/chat-content-deserializer.ts +++ b/packages/ai-chat/src/common/chat-content-deserializer.ts @@ -19,6 +19,8 @@ import { ChatResponseContent, CodeChatResponseContentImpl, CommandChatResponseContentImpl, + CompactionChatResponseContentImpl, + CompactionContentData, ErrorChatResponseContentImpl, HorizontalLayoutChatResponseContentImpl, InformationalChatResponseContentImpl, @@ -250,6 +252,15 @@ export class DefaultChatContentDeserializerContribution implements ChatContentDe ) }); + registry.register({ + kind: 'compaction', + deserialize: (data: CompactionContentData) => new CompactionChatResponseContentImpl( + data.provider, + data.data, + data.summary + ) + }); + registry.register({ kind: 'markdownContent', deserialize: (data: MarkdownContentData) => new MarkdownChatResponseContentImpl(data.content) diff --git a/packages/ai-chat/src/common/chat-model.spec.ts b/packages/ai-chat/src/common/chat-model.spec.ts new file mode 100644 index 0000000000000..ff6207baf5668 --- /dev/null +++ b/packages/ai-chat/src/common/chat-model.spec.ts @@ -0,0 +1,42 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { expect } from 'chai'; +import { CompactionChatResponseContentImpl, CompactionChatResponseContent } from './chat-model'; + +describe('CompactionChatResponseContent', () => { + it('round-trips through serialization', () => { + const original = new CompactionChatResponseContentImpl('anthropic', { block: 'opaque' }, 'summary text'); + const serialized = original.toSerializable(); + expect(serialized.kind).to.equal('compaction'); + const restored = new CompactionChatResponseContentImpl( + serialized.data.provider, serialized.data.data, serialized.data.summary); + expect(restored.provider).to.equal('anthropic'); + expect(restored.summary).to.equal('summary text'); + expect(restored.data).to.deep.equal({ block: 'opaque' }); + }); + it('produces a compaction language model message', () => { + const content = new CompactionChatResponseContentImpl('openai-responses', { item: 'x' }); + const message = content.toLanguageModelMessage(); + expect(message).to.deep.equal({ actor: 'ai', type: 'compaction', provider: 'openai-responses', data: { item: 'x' }, summary: undefined }); + }); + it('is excluded from plain-text history but offers a display string', () => { + const content = new CompactionChatResponseContentImpl('anthropic', {}, 'sum'); + expect(content.asString()).to.equal(undefined); + expect(content.asDisplayString()).to.equal('sum'); + expect(CompactionChatResponseContent.is(content)).to.equal(true); + }); +}); diff --git a/packages/ai-chat/src/common/chat-model.ts b/packages/ai-chat/src/common/chat-model.ts index a61314b3ee9c4..7687650e2ee6d 100644 --- a/packages/ai-chat/src/common/chat-model.ts +++ b/packages/ai-chat/src/common/chat-model.ts @@ -21,6 +21,8 @@ import { AIVariableResolutionRequest, + CompactionMessage, + CompactionSettings, GenericCapabilitySelections, LanguageModelMessage, ReasoningSettings, @@ -226,6 +228,8 @@ export interface CommonChatSessionSettings { reasoning?: ReasoningSettings; /** Per-session tool confirmation timeout in seconds. Overrides the global preference when set. */ confirmationTimeout?: number; + /** Per-session server-side compaction settings; `compaction.enabled`, when set, wins over the per-provider and global settings. */ + compaction?: CompactionSettings; } export interface ChatSessionSettings { @@ -477,6 +481,12 @@ export interface ThinkingContentData { signature: string; } +export interface CompactionContentData { + provider: string; + data: unknown; + summary?: string; +} + export interface MarkdownContentData { content: string; } @@ -869,6 +879,18 @@ export namespace ThinkingChatResponseContent { } } +export interface CompactionChatResponseContent extends ChatResponseContent { + kind: 'compaction'; + provider: string; + data: unknown; + summary?: string; +} +export namespace CompactionChatResponseContent { + export function is(obj: unknown): obj is CompactionChatResponseContent { + return ChatResponseContent.is(obj) && obj.kind === 'compaction'; + } +} + export namespace ProgressChatResponseContent { export function is(obj: unknown): obj is ProgressChatResponseContent { return ( @@ -2242,6 +2264,58 @@ export class ThinkingChatResponseContentImpl implements ThinkingChatResponseCont } } +export class CompactionChatResponseContentImpl implements CompactionChatResponseContent { + readonly kind = 'compaction'; + protected _provider: string; + protected _data: unknown; + protected _summary?: string; + + constructor(provider: string, data: unknown, summary?: string) { + this._provider = provider; + this._data = data; + this._summary = summary; + } + + get provider(): string { + return this._provider; + } + get data(): unknown { + return this._data; + } + get summary(): string | undefined { + return this._summary; + } + + asString(): string | undefined { + return undefined; + } + + asDisplayString(): string | undefined { + return this._summary; + } + + toLanguageModelMessage(): CompactionMessage { + return { + actor: 'ai', + type: 'compaction', + provider: this._provider, + data: this._data, + summary: this._summary + }; + } + + toSerializable(): SerializableChatResponseContentData { + return { + kind: 'compaction', + data: { + provider: this._provider, + data: this._data, + summary: this._summary + } + }; + } +} + export class MarkdownChatResponseContentImpl implements MarkdownChatResponseContent { readonly kind = 'markdownContent'; protected _content: MarkdownStringImpl = new MarkdownStringImpl(); diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index 00f30433fcbff..e6fd14018b410 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -184,7 +184,7 @@ export class CopilotLanguageModel implements LanguageModel { } protected processMessages(messages: LanguageModelMessage[]): ChatCompletionMessageParam[] { - const converted = messages.filter(m => m.type !== 'thinking').map(m => this.toOpenAIMessage(m)); + const converted = messages.filter(m => m.type !== 'thinking' && m.type !== 'compaction').map(m => this.toOpenAIMessage(m)); return this.mergeConsecutiveAssistantMessages(converted); } diff --git a/packages/ai-core/src/common/ai-core-preferences.ts b/packages/ai-core/src/common/ai-core-preferences.ts index 16f37b5125742..667c848c18c54 100644 --- a/packages/ai-core/src/common/ai-core-preferences.ts +++ b/packages/ai-core/src/common/ai-core-preferences.ts @@ -34,11 +34,22 @@ export const PREFERENCE_NAME_REASONING = 'ai-features.reasoning.defaults'; export const PREFERENCE_NAME_MAX_RETRIES = 'ai-features.modelSettings.maxRetries'; export const PREFERENCE_NAME_DEFAULT_NOTIFICATION_TYPE = 'ai-features.notifications.default'; export const PREFERENCE_NAME_SKILL_DIRECTORIES = 'ai-features.skills.skillDirectories'; +export const PREFERENCE_NAME_SERVER_SIDE_COMPACTION = 'ai-features.chat.serverSideCompaction'; export const LANGUAGE_MODEL_ALIASES_PREFERENCE = 'ai-features.languageModelAliases'; export const aiCorePreferenceSchema: PreferenceSchema = { properties: { + [PREFERENCE_NAME_SERVER_SIDE_COMPACTION]: { + title: AI_CORE_PREFERENCES_TITLE, + type: 'boolean', + description: nls.localize('theia/ai/core/serverSideCompaction/description', + 'Enable provider-native server-side compaction by default for chat requests. When on, models that support it ' + + 'may automatically summarize older turns once the conversation grows past their own threshold, letting long ' + + 'sessions continue past the context limit. Each provider offers its own override, and individual chat sessions ' + + 'can override this in their session settings. Models without support ignore it.'), + default: true, + }, [PREFERENCE_NAME_PROMPT_TEMPLATES]: { title: AI_CORE_PREFERENCES_TITLE, description: nls.localize('theia/ai/core/promptTemplates/description', diff --git a/packages/ai-core/src/common/language-model.spec.ts b/packages/ai-core/src/common/language-model.spec.ts index 0977fc76e382d..ff6c558998436 100644 --- a/packages/ai-core/src/common/language-model.spec.ts +++ b/packages/ai-core/src/common/language-model.spec.ts @@ -15,7 +15,16 @@ // ***************************************************************************** import { expect } from 'chai'; -import { isModelMatching, isToolCallContent, LanguageModel, LanguageModelSelector } from './language-model'; +import { + isCompactionResponsePart, + isLanguageModelStreamResponsePart, + isModelMatching, + isToolCallContent, + LanguageModel, + LanguageModelSelector, + resolveCompactionDefault, + resolveServerSideCompaction +} from './language-model'; describe('isModelMatching', () => { it('returns false with one of two parameter mismatches', () => { @@ -163,3 +172,38 @@ describe('isToolCallContent', () => { }); }); + +describe('compaction contract', () => { + it('recognizes a compaction response part', () => { + const part = { compaction: { provider: 'anthropic', data: { foo: 1 } } }; + expect(isCompactionResponsePart(part)).to.equal(true); + expect(isLanguageModelStreamResponsePart(part)).to.equal(true); + }); + it('rejects a non-compaction part', () => { + expect(isCompactionResponsePart({ content: 'hi' })).to.equal(false); + // eslint-disable-next-line no-null/no-null + expect(isCompactionResponsePart(null)).to.equal(false); + expect(isCompactionResponsePart(undefined)).to.equal(false); + // eslint-disable-next-line no-null/no-null + expect(isCompactionResponsePart({ compaction: null })).to.equal(false); + expect(isCompactionResponsePart({ compaction: { provider: 42 } })).to.equal(false); + }); + it('resolves a model default from the global preference and the per-provider override', () => { + expect(resolveCompactionDefault(true, 'default')).to.equal(true); + expect(resolveCompactionDefault(false, 'default')).to.equal(false); + expect(resolveCompactionDefault(false, 'enabled')).to.equal(true); + expect(resolveCompactionDefault(true, 'disabled')).to.equal(false); + }); + it('resolves server-side compaction with capability gate and session-wins precedence', () => { + // capability gate + expect(resolveServerSideCompaction(false, true, { enabled: true })).to.equal(false); + expect(resolveServerSideCompaction(undefined, true, undefined)).to.equal(false); + // no per-session setting -> model default + expect(resolveServerSideCompaction(true, true, undefined)).to.equal(true); + expect(resolveServerSideCompaction(true, false, undefined)).to.equal(false); + expect(resolveServerSideCompaction(true, true, {})).to.equal(true); + // explicit per-session setting wins over the model default (which already folds in the per-provider override) + expect(resolveServerSideCompaction(true, false, { enabled: true })).to.equal(true); + expect(resolveServerSideCompaction(true, true, { enabled: false })).to.equal(false); + }); +}); diff --git a/packages/ai-core/src/common/language-model.ts b/packages/ai-core/src/common/language-model.ts index 29e49692962dd..ace289e0dda57 100644 --- a/packages/ai-core/src/common/language-model.ts +++ b/packages/ai-core/src/common/language-model.ts @@ -42,7 +42,8 @@ export interface ReasoningSupport { readonly defaultLevel?: ReasoningLevel; } -export type LanguageModelMessage = TextMessage | ThinkingMessage | ToolUseMessage | ToolResultMessage | ServerToolUseMessage | ImageMessage; +export type LanguageModelMessage = + TextMessage | ThinkingMessage | ToolUseMessage | ToolResultMessage | ServerToolUseMessage | ImageMessage | CompactionMessage; export namespace LanguageModelMessage { export function isTextMessage(obj: LanguageModelMessage): obj is TextMessage { @@ -63,6 +64,9 @@ export namespace LanguageModelMessage { export function isImageMessage(obj: LanguageModelMessage): obj is ImageMessage { return obj.type === 'image'; } + export function isCompactionMessage(obj: LanguageModelMessage): obj is CompactionMessage { + return obj.type === 'compaction'; + } } export interface TextMessage { actor: MessageActor; @@ -127,6 +131,17 @@ export interface ImageMessage { image: ImageContent; } +export interface CompactionMessage { + actor: 'ai'; + type: 'compaction'; + /** Originating provider tag; a backend replays the payload only when this matches its own provider. */ + provider: string; + /** Opaque provider payload to replay. */ + data: unknown; + /** Human-readable summary, when the provider exposes one. */ + summary?: string; +} + export const isLanguageModelRequestMessage = (obj: unknown): obj is LanguageModelMessage => !!(obj && typeof obj === 'object' && 'type' in obj && @@ -304,6 +319,50 @@ export namespace ToolRequest { (!('required' in obj) || (Array.isArray(obj.required) && obj.required.every(prop => typeof prop === 'string'))); } } +/** + * Per-session/per-request server-side compaction settings, carried verbatim from the chat + * session's common settings to the request. Kept as an object so further parameters can be + * added later. + */ +export interface CompactionSettings { + /** Explicit enablement for this session; when set it wins over the model's default. `undefined` means "no explicit choice". */ + enabled?: boolean; +} + +/** Per-provider override for server-side compaction; combined with the global preference by {@link resolveCompactionDefault}. */ +export type ServerSideCompactionSetting = 'default' | 'enabled' | 'disabled'; + +/** + * Resolves a model's default server-side compaction enablement from the global preference and + * the per-provider override. `'enabled'`/`'disabled'` force the result; `'default'` defers to + * the global preference. Intended to be called where the preferences are read (the provider's + * frontend contribution) and stored on the model. + */ +export function resolveCompactionDefault(globalEnabled: boolean, perProviderOverride: ServerSideCompactionSetting): boolean { + if (perProviderOverride === 'enabled') { + return true; + } + if (perProviderOverride === 'disabled') { + return false; + } + return globalEnabled; +} + +/** + * Resolves whether server-side compaction is effective for a request: the model must support it + * (capability), then an explicit per-session setting wins, otherwise the model's resolved default applies. + */ +export function resolveServerSideCompaction( + capability: boolean | undefined, + enabledByDefault: boolean, + compaction: CompactionSettings | undefined +): boolean { + if (!capability) { + return false; + } + return compaction?.enabled ?? enabledByDefault; +} + export interface LanguageModelRequest { messages: LanguageModelMessage[], tools?: ToolRequest[]; @@ -323,6 +382,8 @@ export interface LanguageModelRequest { clientSettings?: { keepToolCalls: boolean; keepThinking: boolean }; /** Provider-agnostic reasoning configuration; providers translate it to their native API. */ reasoning?: ReasoningSettings; + /** Provider-agnostic server-side compaction settings, copied verbatim from the chat session. Resolved against the model's capability and default in the backend. */ + compaction?: CompactionSettings; } export interface ResponseFormatJsonSchema { type: 'json_schema'; @@ -378,10 +439,12 @@ export interface LanguageModelTextResponse { export const isLanguageModelTextResponse = (obj: unknown): obj is LanguageModelTextResponse => !!(obj && typeof obj === 'object' && 'text' in obj && typeof (obj as { text: unknown }).text === 'string'); -export type LanguageModelStreamResponsePart = TextResponsePart | ToolCallResponsePart | ServerToolCallResponsePart | ThinkingResponsePart | UsageResponsePart; +export type LanguageModelStreamResponsePart = + TextResponsePart | ToolCallResponsePart | ServerToolCallResponsePart | ThinkingResponsePart | UsageResponsePart | CompactionResponsePart; export const isLanguageModelStreamResponsePart = (part: unknown): part is LanguageModelStreamResponsePart => - isUsageResponsePart(part) || isTextResponsePart(part) || isThinkingResponsePart(part) || isToolCallResponsePart(part) || isServerToolCallResponsePart(part); + isUsageResponsePart(part) || isTextResponsePart(part) || isThinkingResponsePart(part) || + isToolCallResponsePart(part) || isServerToolCallResponsePart(part) || isCompactionResponsePart(part); export interface UsageResponsePart { input_tokens: number; @@ -432,6 +495,23 @@ export interface ThinkingResponsePart { export const isThinkingResponsePart = (part: unknown): part is ThinkingResponsePart => !!(part && typeof part === 'object' && 'thought' in part && typeof part.thought === 'string'); +export interface CompactionResponsePart { + compaction: { + /** Originating provider tag, e.g. 'anthropic' or 'openai-responses'. */ + provider: string; + /** Opaque provider payload (Anthropic compaction block(s) / OpenAI compaction item). Never interpreted outside the originating backend. */ + data: unknown; + /** Human-readable summary, when the provider exposes one. */ + summary?: string; + }; +} +export const isCompactionResponsePart = (part: unknown): part is CompactionResponsePart => + !!(part && typeof part === 'object' && 'compaction' in part && + typeof (part as CompactionResponsePart).compaction === 'object' && + (part as CompactionResponsePart).compaction && + 'provider' in (part as CompactionResponsePart).compaction && + typeof (part as CompactionResponsePart).compaction.provider === 'string'); + export interface ToolCallTextResult { type: 'text', text: string; }; export interface ToolCallImageResult extends Base64ImageContent { type: 'image' }; export interface ToolCallAudioResult { type: 'audio', data: string; mimeType: string }; @@ -530,6 +610,8 @@ export interface LanguageModelMetaData { * **Note:** If you provide these, you must also provide `vendor` because server tools are vendor-specific. */ readonly serverTools?: ServerToolDescriptor[]; + /** Whether this model supports provider-native server-side compaction (capability, distinct from whether it is activated). */ + readonly serverSideCompactionSupport?: boolean; } export namespace LanguageModelMetaData { diff --git a/packages/ai-core/src/node/backend-language-model-registry.ts b/packages/ai-core/src/node/backend-language-model-registry.ts index ba1785480aebf..661695f89e438 100644 --- a/packages/ai-core/src/node/backend-language-model-registry.ts +++ b/packages/ai-core/src/node/backend-language-model-registry.ts @@ -64,6 +64,7 @@ export class BackendLanguageModelRegistryImpl extends DefaultLanguageModelRegist maxOutputTokens: model.maxOutputTokens, reasoningSupport: model.reasoningSupport, serverTools: model.serverTools, + serverSideCompactionSupport: model.serverSideCompactionSupport, }; } } diff --git a/packages/ai-ollama/src/node/ollama-language-model.ts b/packages/ai-ollama/src/node/ollama-language-model.ts index d8279400f6407..01d43db77211d 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.ts @@ -451,6 +451,9 @@ export class OllamaModel implements LanguageModel { result.thinking = message.thinking; } else if (LanguageModelMessage.isImageMessage(message) && ImageContent.isBase64(message.image)) { result.images = [message.image.base64data]; + } else if (LanguageModelMessage.isCompactionMessage(message)) { + // Ollama has no server-side compaction; the opaque marker carries no representable content and is dropped. + return undefined; } else { console.log(`Unknown message type encountered when converting message to Ollama format: ${JSON.stringify(message)}. Ignoring message.`); return undefined; diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index 8feafb99bd048..2906e14b61cc6 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -299,6 +299,8 @@ export class OpenAiResponseApiUtils { } else if (LanguageModelMessage.isServerToolUseMessage(message)) { // 'server_tool_use' replay messages can appear when switching providers within a // session; OpenAI has no equivalent, so they are skipped. + } else if (LanguageModelMessage.isCompactionMessage(message)) { + // The opaque compaction marker carries no representable input here; it is skipped. } else { unreachable(message); } From f9ee23b6a658720221128305ace15454d76d558b Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 9 Jun 2026 18:33:58 -0400 Subject: [PATCH 2/3] feat(ai-openai): server-side compaction via the Responses API Declare the capability (= useResponseApi), fold the global and per-provider preferences into the model's default enablement, enable `context_management` compaction on Responses requests when active, capture the streamed compaction item, and replay it via transcript prefix-drop. Chat Completions ignores it. Signed-off-by: Christian W. Damus --- ...penai-frontend-application-contribution.ts | 23 ++- .../common/openai-language-models-manager.ts | 2 + .../src/common/openai-preferences.ts | 19 ++- .../src/node/openai-language-model.spec.ts | 74 ++++++++- .../src/node/openai-language-model.ts | 30 +++- .../openai-language-models-manager-impl.ts | 6 +- .../node/openai-response-api-utils.spec.ts | 152 +++++++++++++++++- .../src/node/openai-response-api-utils.ts | 41 ++++- 8 files changed, 327 insertions(+), 20 deletions(-) diff --git a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts index db576b4ba688a..ad41798ea6055 100644 --- a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts +++ b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts @@ -16,10 +16,10 @@ import { FrontendApplicationContribution } from '@theia/core/lib/browser'; import { inject, injectable } from '@theia/core/shared/inversify'; -import { ReasoningSupport } from '@theia/ai-core'; +import { ReasoningSupport, resolveCompactionDefault, ServerSideCompactionSetting } from '@theia/ai-core'; import { OpenAiLanguageModelsManager, OpenAiModelDescription, OPENAI_PROVIDER_ID } from '../common'; -import { API_KEY_PREF, CUSTOM_ENDPOINTS_PREF, MODELS_PREF, USE_RESPONSE_API_PREF } from '../common/openai-preferences'; -import { AICorePreferences, PREFERENCE_NAME_MAX_RETRIES } from '@theia/ai-core/lib/common/ai-core-preferences'; +import { API_KEY_PREF, CUSTOM_ENDPOINTS_PREF, MODELS_PREF, SERVER_SIDE_COMPACTION_PREF, USE_RESPONSE_API_PREF } from '../common/openai-preferences'; +import { AICorePreferences, PREFERENCE_NAME_MAX_RETRIES, PREFERENCE_NAME_SERVER_SIDE_COMPACTION } from '@theia/ai-core/lib/common/ai-core-preferences'; import { PreferenceService } from '@theia/core'; @injectable() @@ -63,6 +63,10 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio this.handleCustomModelChanges(this.preferenceService.get[]>(CUSTOM_ENDPOINTS_PREF, [])); } else if (event.preferenceName === USE_RESPONSE_API_PREF) { this.updateAllModels(); + } else if (event.preferenceName === SERVER_SIDE_COMPACTION_PREF) { + this.updateAllModels(); + } else if (event.preferenceName === PREFERENCE_NAME_SERVER_SIDE_COMPACTION) { + this.updateAllModels(); } else if (event.preferenceName === 'http.proxy') { this.manager.setProxyUrl(this.preferenceService.get('http.proxy', undefined)); this.updateAllModels(); @@ -106,6 +110,7 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio model.supportsStructuredOutput === newModel.supportsStructuredOutput && model.enableStreaming === newModel.enableStreaming && model.useResponseApi === newModel.useResponseApi && + model.serverSideCompactionEnabledByDefault === newModel.serverSideCompactionEnabledByDefault && reasoningSupportEquals(model.reasoningSupport, newModel.reasoningSupport))); this.manager.removeLanguageModels(...modelsToRemove.map(model => model.id)); @@ -126,13 +131,17 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio const id = `${OPENAI_PROVIDER_ID}/${modelId}`; const maxRetries = this.aiCorePreferences.get(PREFERENCE_NAME_MAX_RETRIES) ?? 3; const useResponseApi = this.preferenceService.get(USE_RESPONSE_API_PREF, false); + const globalCompaction = this.preferenceService.get(PREFERENCE_NAME_SERVER_SIDE_COMPACTION, true); + const compactionOverride = this.preferenceService.get(SERVER_SIDE_COMPACTION_PREF, 'default'); + const serverSideCompactionEnabledByDefault = resolveCompactionDefault(globalCompaction, compactionOverride); return { id: id, model: modelId, apiKey: true, apiVersion: true, maxRetries: maxRetries, - useResponseApi: useResponseApi + useResponseApi: useResponseApi, + serverSideCompactionEnabledByDefault }; } @@ -140,6 +149,9 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio preferences: Partial[] ): OpenAiModelDescription[] { const maxRetries = this.aiCorePreferences.get(PREFERENCE_NAME_MAX_RETRIES) ?? 3; + const globalCompaction = this.preferenceService.get(PREFERENCE_NAME_SERVER_SIDE_COMPACTION, true); + const compactionOverride = this.preferenceService.get(SERVER_SIDE_COMPACTION_PREF, 'default'); + const serverSideCompactionEnabledByDefault = resolveCompactionDefault(globalCompaction, compactionOverride); return preferences.reduce((acc, pref) => { if (!pref.model || !pref.url || typeof pref.model !== 'string' || typeof pref.url !== 'string') { return acc; @@ -158,7 +170,8 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio enableStreaming: pref.enableStreaming, maxRetries: pref.maxRetries ?? maxRetries, useResponseApi: pref.useResponseApi ?? false, - reasoningSupport: isReasoningSupport(pref.reasoningSupport) ? pref.reasoningSupport : undefined + reasoningSupport: isReasoningSupport(pref.reasoningSupport) ? pref.reasoningSupport : undefined, + serverSideCompactionEnabledByDefault } ]; }, []); diff --git a/packages/ai-openai/src/common/openai-language-models-manager.ts b/packages/ai-openai/src/common/openai-language-models-manager.ts index 252e0f77fdf3e..2fbc35b787f79 100644 --- a/packages/ai-openai/src/common/openai-language-models-manager.ts +++ b/packages/ai-openai/src/common/openai-language-models-manager.ts @@ -50,6 +50,8 @@ export interface OpenAiModelDescription { supportsStructuredOutput?: boolean; /** When set, the UI exposes a reasoning selector. Defaults from the model id when unset. */ reasoningSupport?: ReasoningSupport; + /** Resolved default enablement of server-side compaction (global preference folded with the per-provider override). Defaults to disabled when omitted. */ + serverSideCompactionEnabledByDefault?: boolean; } export interface OpenAiLanguageModelsManager { apiKey: string | undefined; diff --git a/packages/ai-openai/src/common/openai-preferences.ts b/packages/ai-openai/src/common/openai-preferences.ts index 1e05c3dc05fba..37a78b33fe08f 100644 --- a/packages/ai-openai/src/common/openai-preferences.ts +++ b/packages/ai-openai/src/common/openai-preferences.ts @@ -14,12 +14,13 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { AI_CORE_PREFERENCES_TITLE } from '@theia/ai-core/lib/common/ai-core-preferences'; +import { AI_CORE_PREFERENCES_TITLE, PREFERENCE_NAME_SERVER_SIDE_COMPACTION } from '@theia/ai-core/lib/common/ai-core-preferences'; import { LINUX_ENV_HINT, nls, PreferenceSchema } from '@theia/core'; export const API_KEY_PREF = 'ai-features.openAiOfficial.openAiApiKey'; export const MODELS_PREF = 'ai-features.openAiOfficial.officialOpenAiModels'; export const USE_RESPONSE_API_PREF = 'ai-features.openAiOfficial.useResponseApi'; +export const SERVER_SIDE_COMPACTION_PREF = 'ai-features.openAiOfficial.serverSideCompaction'; export const CUSTOM_ENDPOINTS_PREF = 'ai-features.openAiCustom.customOpenAiModels'; export const OpenAiPreferencesSchema: PreferenceSchema = { @@ -59,6 +60,22 @@ This setting only applies to official OpenAI models - custom providers must conf Note that for the response API, tool call definitions must satisfy Open AI\'s [strict schema definition](https://platform.openai.com/docs/guides/function-calling#strict-mode).\ Best effort is made to convert non-conformant schemas, but errors are still possible.') }, + [SERVER_SIDE_COMPACTION_PREF]: { + type: 'string', + enum: ['default', 'enabled', 'disabled'], + enumDescriptions: [ + nls.localize('theia/ai/openai/compaction/default', 'Follow the global chat server-side compaction setting.'), + nls.localize('theia/ai/openai/compaction/enabled', 'Always request server-side compaction for official OpenAI models.'), + nls.localize('theia/ai/openai/compaction/disabled', 'Never request server-side compaction for official OpenAI models.') + ], + default: 'default', + markdownDescription: nls.localize('theia/ai/openai/compaction/description', + 'Override provider-native server-side compaction for official OpenAI models. This applies to the OpenAI Response API only; ' + + 'the Chat Completions API ignores it. "default" follows the global chat setting ({0}). When effectively ' + + 'enabled, the Response API is asked to summarize older turns once the conversation grows past the provider\'s threshold.', + `\`#${PREFERENCE_NAME_SERVER_SIDE_COMPACTION}#\``), + title: AI_CORE_PREFERENCES_TITLE, + }, [CUSTOM_ENDPOINTS_PREF]: { type: 'array', title: AI_CORE_PREFERENCES_TITLE, diff --git a/packages/ai-openai/src/node/openai-language-model.spec.ts b/packages/ai-openai/src/node/openai-language-model.spec.ts index 7a7c303afde74..e44c247d79927 100644 --- a/packages/ai-openai/src/node/openai-language-model.spec.ts +++ b/packages/ai-openai/src/node/openai-language-model.spec.ts @@ -15,7 +15,7 @@ // ***************************************************************************** import { expect } from 'chai'; -import { LanguageModelRequest, ReasoningSupport } from '@theia/ai-core'; +import { LanguageModelMessage, LanguageModelRequest, ReasoningSupport } from '@theia/ai-core'; import { OpenAiModel, OpenAiModelUtils } from './openai-language-model'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; @@ -33,6 +33,9 @@ class TestableOpenAiModel extends OpenAiModel { public callGetSettings(request: LanguageModelRequest, forResponseApi: boolean = false): Record { return this.getSettings(request, forResponseApi); } + public callApplyResponseApiCompaction(settings: Record, request: LanguageModelRequest): Record { + return this.applyResponseApiCompaction(settings, request); + } } function createModel(modelId: string, reasoningSupport?: ReasoningSupport): TestableOpenAiModel { @@ -45,6 +48,16 @@ function createModel(modelId: string, reasoningSupport?: ReasoningSupport): Test ); } +function createCompactionModel(serverSideCompactionEnabledByDefault: boolean, useResponseApi: boolean = true): TestableOpenAiModel { + return new TestableOpenAiModel( + 'test-id', 'gpt-5', { status: 'ready' }, true, + () => 'test-key', () => undefined, + false, undefined, undefined, + new OpenAiModelUtils(), new OpenAiResponseApiUtils(), + 'developer', 3, useResponseApi, undefined, undefined, undefined, serverSideCompactionEnabledByDefault + ); +} + describe('OpenAiModel reasoning translation', () => { describe('Responses API (GPT-5)', () => { @@ -102,3 +115,62 @@ describe('OpenAiModel reasoning translation', () => { }); }); }); + +describe('OpenAiModelUtils Chat Completions processMessages', () => { + it('drops a CompactionMessage without throwing when useResponseApi is off', () => { + const utils = new OpenAiModelUtils(); + const messages: LanguageModelMessage[] = [ + { actor: 'user', type: 'text', text: 'hello' }, + { actor: 'ai', type: 'compaction', provider: 'openai-responses', data: { id: 'c1', encrypted_content: 'enc' } }, + { actor: 'ai', type: 'text', text: 'world' } + ]; + + let result: ReturnType | undefined; + expect(() => { result = utils.processMessages(messages, 'developer'); }).to.not.throw(); + + // The compaction marker must not appear in the output + const hasCompaction = result?.some(m => 'content' in m && typeof m.content === 'string' && m.content.includes('enc')); + expect(hasCompaction).to.equal(false); + + // The surrounding real messages must still be present + const texts = result?.map(m => typeof m.content === 'string' ? m.content : '').join(' '); + expect(texts).to.contain('hello'); + expect(texts).to.contain('world'); + }); +}); + +describe('OpenAiModel server-side compaction (Response API)', () => { + + it('adds context_management when model default is on and request has no compaction setting', () => { + const model = createCompactionModel(true); + const result = model.callApplyResponseApiCompaction({ stream: true }, { messages: [] }); + expect(result.context_management).to.deep.equal([{ type: 'compaction' }]); + expect(result.stream).to.equal(true); + }); + + it('leaves settings unchanged when model default is off and request has no compaction setting', () => { + const model = createCompactionModel(false); + const result = model.callApplyResponseApiCompaction({ stream: true }, { messages: [] }); + expect(result.context_management).to.equal(undefined); + expect(result).to.deep.equal({ stream: true }); + }); + + it('session enabled=true activates compaction over a false model default', () => { + const model = createCompactionModel(false); + const result = model.callApplyResponseApiCompaction({}, { messages: [], compaction: { enabled: true } }); + expect(result.context_management).to.deep.equal([{ type: 'compaction' }]); + }); + + it('session enabled=false deactivates compaction even when model default is true', () => { + const model = createCompactionModel(true); + const result = model.callApplyResponseApiCompaction({}, { messages: [], compaction: { enabled: false } }); + expect(result.context_management).to.equal(undefined); + }); + + it('does not add context_management when capability is false (useResponseApi=false), even with session enabled=true', () => { + const model = createCompactionModel(true, false); + const result = model.callApplyResponseApiCompaction({ stream: true }, { messages: [], compaction: { enabled: true } }); + expect(result.context_management).to.equal(undefined); + expect(result).to.deep.equal({ stream: true }); + }); +}); diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index a611554a5a855..8b79b11bba677 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -24,7 +24,8 @@ import { UserRequest, ImageContent, LanguageModelStatus, - ReasoningSupport + ReasoningSupport, + resolveServerSideCompaction } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; import { injectable } from '@theia/core/shared/inversify'; @@ -89,6 +90,7 @@ export class OpenAiModel implements LanguageModel { * @param url the OpenAI API compatible endpoint where the model is hosted. If not provided the default OpenAI endpoint will be used. * @param maxRetries the maximum number of retry attempts when a request fails * @param useResponseApi whether to use the newer OpenAI Response API instead of the Chat Completion API + * @param serverSideCompactionEnabledByDefault resolved default enablement of server-side compaction (global preference folded with the per-provider override) */ constructor( public readonly id: string, @@ -107,9 +109,14 @@ export class OpenAiModel implements LanguageModel { public useResponseApi: boolean = false, public proxy?: string, public reasoningSupport?: ReasoningSupport, - public maxInputTokens?: number + public maxInputTokens?: number, + public serverSideCompactionEnabledByDefault: boolean = false ) { } + get serverSideCompactionSupport(): boolean { + return this.useResponseApi; + } + /** Reasoning-level translation lives in {@link openAiReasoningFor}. */ protected getSettings(request: LanguageModelRequest, forResponseApi: boolean = false): Record { return { @@ -248,8 +255,19 @@ export class OpenAiModel implements LanguageModel { } } + /** + * Augments the Response API settings with the server-side compaction directive when compaction is enabled for the + * given request. When disabled, the settings are returned unchanged so the default path is byte-for-byte identical. + */ + protected applyResponseApiCompaction(settings: Record, request: LanguageModelRequest): Record { + if (resolveServerSideCompaction(this.serverSideCompactionSupport, this.serverSideCompactionEnabledByDefault, request.compaction)) { + return { ...settings, context_management: [{ type: 'compaction' }] }; + } + return settings; + } + protected async handleResponseApiRequest(openai: OpenAI, request: UserRequest, cancellationToken?: CancellationToken): Promise { - const settings = this.getSettings(request, true); + const settings = this.applyResponseApiCompaction(this.getSettings(request, true), request); const isStreamingRequest = this.enableStreaming && !(typeof settings.stream === 'boolean' && !settings.stream); try { @@ -369,10 +387,10 @@ export class OpenAiModelUtils { model?: string ): ChatCompletionMessageParam[] { const processed = this.processSystemMessages(messages, developerMessageSettings); - // 'server_tool_use' replay messages can appear when switching providers within a session; - // OpenAI has no equivalent, so they are dropped (like 'thinking' messages). + // 'server_tool_use' and 'compaction' replay markers can appear when switching providers within a session; + // OpenAI Chat Completions has no equivalent, so they are dropped (like 'thinking' messages). const converted = processed - .filter(m => m.type !== 'thinking' && m.type !== 'server_tool_use') + .filter(m => m.type !== 'thinking' && m.type !== 'server_tool_use' && m.type !== 'compaction') .map(m => this.toOpenAIMessage(m, developerMessageSettings)); return this.mergeConsecutiveAssistantMessages(converted); } diff --git a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts index 5a8ade66e17b9..eb57d393e8f24 100644 --- a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts +++ b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts @@ -111,7 +111,8 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana useResponseApi: modelDescription.useResponseApi ?? false, proxy: proxyUrl, reasoningSupport: metadata.reasoningSupport, - maxInputTokens: metadata.maxInputTokens + maxInputTokens: metadata.maxInputTokens, + serverSideCompactionEnabledByDefault: modelDescription.serverSideCompactionEnabledByDefault ?? false }); } else { this.languageModelRegistry.addLanguageModels([ @@ -132,7 +133,8 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana modelDescription.useResponseApi ?? false, proxyUrl, metadata.reasoningSupport, - metadata.maxInputTokens + metadata.maxInputTokens, + modelDescription.serverSideCompactionEnabledByDefault ?? false ) ]); } diff --git a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts index 4327bac54c09c..f6041b5a00048 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts @@ -15,7 +15,7 @@ // ***************************************************************************** import { expect } from 'chai'; -import { isUsageResponsePart, LanguageModelStreamResponsePart, UserRequest } from '@theia/ai-core'; +import { CompactionMessage, isCompactionResponsePart, isUsageResponsePart, LanguageModelMessage, LanguageModelStreamResponsePart, UserRequest } from '@theia/ai-core'; import { OpenAiModelUtils } from './openai-language-model'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; @@ -106,4 +106,154 @@ describe('OpenAiResponseApiUtils', () => { { input_tokens: 200, output_tokens: 20 } ]); }); + + it('yields a compaction part when the stream contains a response.output_item.done compaction event', async () => { + const utils = new OpenAiResponseApiUtils(); + const streamEvents = [ + { + type: 'response.output_item.done', + item: { type: 'compaction', id: 'c1', encrypted_content: 'enc1' } + } as never, + { + type: 'response.completed', + response: { usage: { input_tokens: 10, output_tokens: 5 } } + } as never + ]; + const openai = { + responses: { + stream: () => toStream(streamEvents) + } + }; + const request: UserRequest = { + sessionId: 'session-1', + requestId: 'request-1', + messages: [{ actor: 'user', type: 'text', text: 'hello' }] + }; + + const response = await utils.handleRequest( + openai as never, + request, + {}, + 'gpt-5', + new OpenAiModelUtils(), + 'developer', + { maxChatCompletions: 3 }, + 'openai/gpt-5', + true + ); + const parts: LanguageModelStreamResponsePart[] = []; + if ('stream' in response) { + for await (const part of response.stream) { + parts.push(part); + } + } + + const compactionParts = parts.filter(isCompactionResponsePart); + expect(compactionParts).to.have.length(1); + expect(compactionParts[0]).to.deep.equal({ + compaction: { + provider: 'openai-responses', + data: { id: 'c1', encrypted_content: 'enc1' } + } + }); + }); + + describe('processMessages server-side compaction replay', () => { + + function userMessage(text: string): LanguageModelMessage { + return { actor: 'user', type: 'text', text }; + } + + function aiMessage(text: string): LanguageModelMessage { + return { actor: 'ai', type: 'text', text }; + } + + function compactionMessage(provider: string, encrypted: string, id?: string): CompactionMessage { + return { actor: 'ai', type: 'compaction', provider, data: { encrypted_content: encrypted, id } }; + } + + function textOf(input: ReturnType['input']): string { + return JSON.stringify(input); + } + + function findCompactionItem(input: ReturnType['input']): { type: string; encrypted_content: string } | undefined { + return input.find((item): item is { type: 'compaction'; encrypted_content: string } => + (item as { type?: string }).type === 'compaction') as { type: string; encrypted_content: string } | undefined; + } + + it('replays the openai-responses compaction marker and drops the prefix before it', () => { + const utils = new OpenAiResponseApiUtils(); + const messages: LanguageModelMessage[] = [ + userMessage('user A'), + aiMessage('ai B'), + compactionMessage('openai-responses', 'enc1'), + userMessage('user C') + ]; + + const { input } = utils.processMessages(messages, 'developer', 'gpt-5'); + + const compaction = findCompactionItem(input); + expect(compaction, 'compaction item present').to.not.equal(undefined); + expect(compaction!.encrypted_content).to.equal('enc1'); + expect(input[0]).to.deep.include({ type: 'compaction', encrypted_content: 'enc1' }); + + const serialized = textOf(input); + expect(serialized).to.contain('user C'); + expect(serialized).to.not.contain('user A'); + expect(serialized).to.not.contain('ai B'); + }); + + it('only replays the LAST openai-responses marker and drops everything before it', () => { + const utils = new OpenAiResponseApiUtils(); + const messages: LanguageModelMessage[] = [ + userMessage('user A'), + compactionMessage('openai-responses', 'enc1'), + userMessage('user B'), + compactionMessage('openai-responses', 'enc2'), + userMessage('user C') + ]; + + const { input } = utils.processMessages(messages, 'developer', 'gpt-5'); + + const compactionItems = input.filter(item => (item as { type?: string }).type === 'compaction'); + expect(compactionItems).to.have.length(1); + expect((compactionItems[0] as { encrypted_content: string }).encrypted_content).to.equal('enc2'); + + const serialized = textOf(input); + expect(serialized).to.contain('user C'); + expect(serialized).to.not.contain('user A'); + expect(serialized).to.not.contain('user B'); + expect(serialized).to.not.contain('enc1'); + }); + + it('skips a foreign-provider compaction marker without dropping the prefix', () => { + const utils = new OpenAiResponseApiUtils(); + const messages: LanguageModelMessage[] = [ + userMessage('user A'), + compactionMessage('anthropic', 'enc1'), + userMessage('user B') + ]; + + const { input } = utils.processMessages(messages, 'developer', 'gpt-5'); + + expect(findCompactionItem(input), 'no compaction item for foreign provider').to.equal(undefined); + const serialized = textOf(input); + expect(serialized).to.contain('user A'); + expect(serialized).to.contain('user B'); + expect(serialized).to.not.contain('enc1'); + }); + + it('converts all messages unchanged when there is no compaction marker', () => { + const utils = new OpenAiResponseApiUtils(); + const messages: LanguageModelMessage[] = [userMessage('user A'), aiMessage('ai B'), userMessage('user C')]; + + const { input } = utils.processMessages(messages, 'developer', 'gpt-5'); + + expect(findCompactionItem(input)).to.equal(undefined); + const serialized = textOf(input); + expect(serialized).to.contain('user A'); + expect(serialized).to.contain('ai B'); + expect(serialized).to.contain('user C'); + }); + }); }); diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index 2906e14b61cc6..2a7c7211be5ef 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -181,6 +181,13 @@ export class OpenAiResponseApiUtils { yield { content: event.delta }; + } else if (event.type === 'response.output_item.done' && event.item?.type === 'compaction') { + yield { + compaction: { + provider: 'openai-responses', + data: { id: event.item.id, encrypted_content: event.item.encrypted_content } + } + }; } else if (event.type === 'response.completed') { if (event.response?.usage) { usageYielded = true; @@ -242,8 +249,29 @@ export class OpenAiResponseApiUtils { const nonSystemMessages = processed.filter(m => m.actor !== 'system'); const input: ResponseInputItem[] = []; - for (const message of nonSystemMessages) { - if (LanguageModelMessage.isTextMessage(message)) { + // Server-side compaction replay: the latest openai-responses compaction marker carries the (encrypted) context for + // everything before it. Drop that prefix and replay the marker as a compaction input item; earlier markers are subsumed. + let sliceStart = 0; + for (let i = nonSystemMessages.length - 1; i >= 0; i--) { + const candidate = nonSystemMessages[i]; + if (LanguageModelMessage.isCompactionMessage(candidate) && candidate.provider === 'openai-responses') { + const data = candidate.data as { encrypted_content: string; id?: string }; + input.push({ + type: 'compaction', + encrypted_content: data.encrypted_content, + id: data.id + }); + sliceStart = i + 1; + break; + } + } + + for (const message of nonSystemMessages.slice(sliceStart)) { + if (LanguageModelMessage.isCompactionMessage(message)) { + // Skip any remaining compaction marker (foreign provider, or a non-final openai-responses one): + // it is not the prefix-drop item, so it carries no input for this provider. + continue; + } else if (LanguageModelMessage.isTextMessage(message)) { if (message.actor === 'ai') { // Assistant messages use ResponseOutputMessage format input.push({ @@ -299,8 +327,6 @@ export class OpenAiResponseApiUtils { } else if (LanguageModelMessage.isServerToolUseMessage(message)) { // 'server_tool_use' replay messages can appear when switching providers within a // session; OpenAI has no equivalent, so they are skipped. - } else if (LanguageModelMessage.isCompactionMessage(message)) { - // The opaque compaction marker carries no representable input here; it is skipped. } else { unreachable(message); } @@ -522,6 +548,13 @@ class ResponseApiToolCallIterator implements AsyncIterableIterator Date: Tue, 9 Jun 2026 18:33:58 -0400 Subject: [PATCH 3/3] feat(ai-anthropic): server-side compaction via the Beta Messages API Declare the capability (Opus/Sonnet 4.6+ heuristic), fold the global and per-provider preferences into the model's default enablement, route active requests through the Beta Messages API with the compact-2026-01-12 beta and the compact_20260112 edit, capture the streamed compaction block, and replay it while keeping surrounding history. Default path unchanged. Signed-off-by: Christian W. Damus --- ...ropic-frontend-application-contribution.ts | 20 +- .../anthropic-language-models-manager.ts | 3 + .../src/common/anthropic-preferences.ts | 19 +- .../src/node/anthropic-language-model.spec.ts | 208 +++++++++++++++++- .../src/node/anthropic-language-model.ts | 111 ++++++++-- ...ropic-language-models-manager-impl.spec.ts | 1 + .../anthropic-language-models-manager-impl.ts | 6 +- 7 files changed, 343 insertions(+), 25 deletions(-) diff --git a/packages/ai-anthropic/src/browser/anthropic-frontend-application-contribution.ts b/packages/ai-anthropic/src/browser/anthropic-frontend-application-contribution.ts index 27e1f2ba04329..b3bf8aaec703b 100644 --- a/packages/ai-anthropic/src/browser/anthropic-frontend-application-contribution.ts +++ b/packages/ai-anthropic/src/browser/anthropic-frontend-application-contribution.ts @@ -17,9 +17,10 @@ import { FrontendApplicationContribution } from '@theia/core/lib/browser'; import { inject, injectable } from '@theia/core/shared/inversify'; import { AnthropicLanguageModelsManager, AnthropicModelDescription } from '../common'; -import { API_KEY_PREF, CUSTOM_ENDPOINTS_PREF, MODELS_PREF } from '../common/anthropic-preferences'; -import { AICorePreferences, PREFERENCE_NAME_MAX_RETRIES } from '@theia/ai-core/lib/common/ai-core-preferences'; +import { API_KEY_PREF, CUSTOM_ENDPOINTS_PREF, MODELS_PREF, SERVER_SIDE_COMPACTION_PREF } from '../common/anthropic-preferences'; +import { AICorePreferences, PREFERENCE_NAME_MAX_RETRIES, PREFERENCE_NAME_SERVER_SIDE_COMPACTION } from '@theia/ai-core/lib/common/ai-core-preferences'; import { PreferenceService } from '@theia/core'; +import { resolveCompactionDefault, ServerSideCompactionSetting } from '@theia/ai-core'; const ANTHROPIC_PROVIDER_ID = 'anthropic'; @@ -63,6 +64,8 @@ export class AnthropicFrontendApplicationContribution implements FrontendApplica } else if (event.preferenceName === 'http.proxy') { this.manager.setProxyUrl(this.preferenceService.get('http.proxy', undefined)); this.updateAllModels(); + } else if (event.preferenceName === SERVER_SIDE_COMPACTION_PREF || event.preferenceName === PREFERENCE_NAME_SERVER_SIDE_COMPACTION) { + this.updateAllModels(); } else if (event.preferenceName === CUSTOM_ENDPOINTS_PREF) { this.handleCustomModelChanges(this.preferenceService.get[]>(CUSTOM_ENDPOINTS_PREF, [])); } @@ -101,6 +104,7 @@ export class AnthropicFrontendApplicationContribution implements FrontendApplica model.apiKey === newModel.apiKey && model.maxRetries === newModel.maxRetries && model.useCaching === newModel.useCaching && + model.serverSideCompactionEnabledByDefault === newModel.serverSideCompactionEnabledByDefault && model.enableStreaming === newModel.enableStreaming)); this.manager.removeLanguageModels(...modelsToRemove.map(model => model.id)); @@ -120,6 +124,9 @@ export class AnthropicFrontendApplicationContribution implements FrontendApplica protected createAnthropicModelDescription(modelId: string): AnthropicModelDescription { const id = `${ANTHROPIC_PROVIDER_ID}/${modelId}`; const maxRetries = this.aiCorePreferences.get(PREFERENCE_NAME_MAX_RETRIES) ?? 3; + const globalCompaction = this.preferenceService.get(PREFERENCE_NAME_SERVER_SIDE_COMPACTION, true); + const compactionOverride = this.preferenceService.get(SERVER_SIDE_COMPACTION_PREF, 'default'); + const serverSideCompactionEnabledByDefault = resolveCompactionDefault(globalCompaction, compactionOverride); return { id: id, @@ -127,12 +134,16 @@ export class AnthropicFrontendApplicationContribution implements FrontendApplica apiKey: true, enableStreaming: true, useCaching: true, - maxRetries: maxRetries + maxRetries: maxRetries, + serverSideCompactionEnabledByDefault }; } protected createCustomModelDescriptionsFromPreferences(preferences: Partial[]): AnthropicModelDescription[] { const maxRetries = this.aiCorePreferences.get(PREFERENCE_NAME_MAX_RETRIES) ?? 3; + const globalCompaction = this.preferenceService.get(PREFERENCE_NAME_SERVER_SIDE_COMPACTION, true); + const compactionOverride = this.preferenceService.get(SERVER_SIDE_COMPACTION_PREF, 'default'); + const serverSideCompactionEnabledByDefault = resolveCompactionDefault(globalCompaction, compactionOverride); return preferences.reduce((acc, pref) => { if (!pref.model || !pref.url || typeof pref.model !== 'string' || typeof pref.url !== 'string') { return acc; @@ -146,7 +157,8 @@ export class AnthropicFrontendApplicationContribution implements FrontendApplica apiKey: typeof pref.apiKey === 'string' || pref.apiKey === true ? pref.apiKey : undefined, enableStreaming: pref.enableStreaming ?? true, useCaching: pref.useCaching ?? true, - maxRetries: pref.maxRetries ?? maxRetries + maxRetries: pref.maxRetries ?? maxRetries, + serverSideCompactionEnabledByDefault } ]; }, []); diff --git a/packages/ai-anthropic/src/common/anthropic-language-models-manager.ts b/packages/ai-anthropic/src/common/anthropic-language-models-manager.ts index df968d25c9885..3549da5026a57 100644 --- a/packages/ai-anthropic/src/common/anthropic-language-models-manager.ts +++ b/packages/ai-anthropic/src/common/anthropic-language-models-manager.ts @@ -13,6 +13,7 @@ // // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** + export const ANTHROPIC_LANGUAGE_MODELS_MANAGER_PATH = '/services/anthropic/language-model-manager'; export const AnthropicLanguageModelsManager = Symbol('AnthropicLanguageModelsManager'); @@ -31,6 +32,8 @@ export interface AnthropicModelDescription { useCaching: boolean; /** Maximum number of retry attempts when a request fails. Default is 3. */ maxRetries: number; + /** Resolved default enablement of server-side compaction (global preference folded with the per-provider override). Defaults to disabled when omitted. */ + serverSideCompactionEnabledByDefault?: boolean; } export interface AnthropicLanguageModelsManager { apiKey: string | undefined; diff --git a/packages/ai-anthropic/src/common/anthropic-preferences.ts b/packages/ai-anthropic/src/common/anthropic-preferences.ts index ef286a2b0779b..0bc6d631b7528 100644 --- a/packages/ai-anthropic/src/common/anthropic-preferences.ts +++ b/packages/ai-anthropic/src/common/anthropic-preferences.ts @@ -14,12 +14,13 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { AI_CORE_PREFERENCES_TITLE } from '@theia/ai-core/lib/common/ai-core-preferences'; +import { AI_CORE_PREFERENCES_TITLE, PREFERENCE_NAME_SERVER_SIDE_COMPACTION } from '@theia/ai-core/lib/common/ai-core-preferences'; import { LINUX_ENV_HINT, nls, PreferenceSchema } from '@theia/core'; export const API_KEY_PREF = 'ai-features.anthropic.AnthropicApiKey'; export const MODELS_PREF = 'ai-features.anthropic.AnthropicModels'; export const CUSTOM_ENDPOINTS_PREF = 'ai-features.anthropicCustom.customAnthropicModels'; +export const SERVER_SIDE_COMPACTION_PREF = 'ai-features.anthropic.serverSideCompaction'; export const AnthropicPreferencesSchema: PreferenceSchema = { properties: { @@ -46,6 +47,22 @@ export const AnthropicPreferencesSchema: PreferenceSchema = { type: 'string' } }, + [SERVER_SIDE_COMPACTION_PREF]: { + type: 'string', + enum: ['default', 'enabled', 'disabled'], + enumDescriptions: [ + nls.localize('theia/ai/anthropic/compaction/default', 'Follow the global chat server-side compaction setting.'), + nls.localize('theia/ai/anthropic/compaction/enabled', 'Always request server-side compaction for Anthropic models.'), + nls.localize('theia/ai/anthropic/compaction/disabled', 'Never request server-side compaction for Anthropic models.') + ], + default: 'default', + markdownDescription: nls.localize('theia/ai/anthropic/compaction/description', + 'Override provider-native server-side compaction for Anthropic models. "default" follows the global chat setting ' + + '({0}). When effectively enabled, the Anthropic Beta Messages API is used so the provider can summarize ' + + 'older turns once the conversation grows past its threshold.', + `\`#${PREFERENCE_NAME_SERVER_SIDE_COMPACTION}#\``), + title: AI_CORE_PREFERENCES_TITLE, + }, [CUSTOM_ENDPOINTS_PREF]: { type: 'array', title: AI_CORE_PREFERENCES_TITLE, diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts b/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts index 9457dda2df3c7..f96f2186d3df5 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts @@ -15,9 +15,13 @@ // ***************************************************************************** import { expect } from 'chai'; -import { ANTHROPIC_RESULT_BLOCK_DATA_KEY, AnthropicModel, DEFAULT_MAX_TOKENS, addCacheControlToLastMessage, mergeConsecutiveSameRoleMessages } from './anthropic-language-model'; import { - isServerToolCallResponsePart, isUsageResponsePart, LanguageModelRequest, LanguageModelStreamResponsePart, ReasoningApi, ReasoningSupport, UserRequest + ANTHROPIC_RESULT_BLOCK_DATA_KEY, AnthropicModel, DEFAULT_MAX_TOKENS, addCacheControlToLastMessage, + mergeConsecutiveSameRoleMessages, transformToAnthropicParams +} from './anthropic-language-model'; +import { + CompactionMessage, isServerToolCallResponsePart, isUsageResponsePart, LanguageModelMessage, LanguageModelRequest, + LanguageModelStreamResponsePart, ReasoningApi, ReasoningSupport, UserRequest } from '@theia/ai-core'; import type { Anthropic } from '@anthropic-ai/sdk'; import type { MessageParam } from '@anthropic-ai/sdk/resources'; @@ -27,7 +31,7 @@ const REASONING_SUPPORT: ReasoningSupport = { defaultLevel: 'auto' }; -/** Test helper that exposes the otherwise protected getSettings()/createTools() methods. */ +/** Test helper that exposes the otherwise protected getSettings(), createTools(), applyCompactionParams(), and computeServerSideCompactionSupport() methods. */ class TestableAnthropicModel extends AnthropicModel { public callGetSettings(request: LanguageModelRequest): Readonly> { return this.getSettings(request); @@ -35,6 +39,12 @@ class TestableAnthropicModel extends AnthropicModel { public callCreateTools(request: LanguageModelRequest): Anthropic.Messages.ToolUnion[] | undefined { return this.createTools(request); } + public callApplyCompactionParams(params: T, request: LanguageModelRequest): T { + return this.applyCompactionParams(params, request); + } + public callComputeServerSideCompactionSupport(): boolean { + return this.computeServerSideCompactionSupport(); + } } function createReasoningModel( @@ -723,6 +733,198 @@ describe('AnthropicModel', () => { expect((resultBlock as { content: unknown }).content).to.deep.equal(rawBlock); }); }); + + describe('server-side compaction', () => { + + function textMessage(text: string, actor: 'user' | 'ai' = 'user'): LanguageModelMessage { + return { actor, type: 'text', text, query: text } as unknown as LanguageModelMessage; + } + + function compactionMessage(provider: string): CompactionMessage { + return { + actor: 'ai', + type: 'compaction', + provider, + data: { content: 'summary of earlier turns', encrypted_content: 'opaque-blob' }, + summary: 'summary of earlier turns' + }; + } + + function findCompactionBlock(messages: Anthropic.Messages.MessageParam[]): { type: string; content: unknown; encrypted_content: unknown } | undefined { + for (const message of messages) { + if (Array.isArray(message.content)) { + for (const block of message.content) { + if ((block as { type?: string }).type === 'compaction') { + return block as unknown as { type: string; content: unknown; encrypted_content: unknown }; + } + } + } + } + return undefined; + } + + function createCompactionModel(serverSideCompactionEnabledByDefault: boolean): TestableAnthropicModel { + return new TestableAnthropicModel( + 'test-id', 'claude-opus-4-6', { status: 'ready' }, true, false, () => 'test-key', undefined, + DEFAULT_MAX_TOKENS, 3, undefined, undefined, undefined, undefined, undefined, undefined, serverSideCompactionEnabledByDefault + ); + } + + describe('transformToAnthropicParams', () => { + it('emits a compaction block for an anthropic compaction message when compaction is enabled, retaining surrounding messages', () => { + const messages: LanguageModelMessage[] = [textMessage('first user turn'), compactionMessage('anthropic'), textMessage('second user turn')]; + + const { messages: result } = transformToAnthropicParams(messages, false, true); + + const compactionBlock = findCompactionBlock(result); + expect(compactionBlock, 'compaction block present').to.not.equal(undefined); + expect(compactionBlock!.content).to.equal('summary of earlier turns'); + expect(compactionBlock!.encrypted_content).to.equal('opaque-blob'); + + const flattened = JSON.stringify(result); + expect(flattened).to.contain('first user turn'); + expect(flattened).to.contain('second user turn'); + }); + + it('omits the compaction block when compaction is disabled, retaining surrounding messages', () => { + const messages: LanguageModelMessage[] = [textMessage('first user turn'), compactionMessage('anthropic'), textMessage('second user turn')]; + + const { messages: result } = transformToAnthropicParams(messages, false, false); + + expect(findCompactionBlock(result), 'no compaction block').to.equal(undefined); + const flattened = JSON.stringify(result); + expect(flattened).to.contain('first user turn'); + expect(flattened).to.contain('second user turn'); + }); + + it('skips a foreign-provider compaction message even when compaction is enabled', () => { + const messages: LanguageModelMessage[] = [textMessage('first user turn'), compactionMessage('openai-responses'), textMessage('second user turn')]; + + const { messages: result } = transformToAnthropicParams(messages, false, true); + + expect(findCompactionBlock(result), 'no compaction block for foreign provider').to.equal(undefined); + const flattened = JSON.stringify(result); + expect(flattened).to.contain('first user turn'); + expect(flattened).to.contain('second user turn'); + }); + }); + + describe('applyCompactionParams', () => { + function baseParams(): Anthropic.MessageCreateParams { + return { max_tokens: DEFAULT_MAX_TOKENS, model: 'claude-opus-4-6', messages: [] }; + } + + it('adds the beta flag and context_management when compaction is enabled by default (capable model)', () => { + // claude-opus-4-6 is capable; serverSideCompactionEnabledByDefault=true, no session override. + const model = createCompactionModel(true); + const params = model.callApplyCompactionParams(baseParams(), { + messages: [] + }); + const beta = params as Anthropic.MessageCreateParams & Anthropic.Beta.Messages.MessageCreateParams; + + expect(beta.betas).to.deep.equal(['compact-2026-01-12']); + expect(beta.context_management).to.deep.equal({ edits: [{ type: 'compact_20260112' }] }); + }); + + it('leaves params unchanged when compaction is disabled by default', () => { + const model = createCompactionModel(false); + const params = model.callApplyCompactionParams(baseParams(), { + messages: [] + }); + const beta = params as Anthropic.MessageCreateParams & Partial; + + expect(beta.betas).to.equal(undefined); + expect(beta.context_management).to.equal(undefined); + }); + + it('session enables compaction over a false default (capable model)', () => { + const model = createCompactionModel(false); + + const enabled = model.callApplyCompactionParams(baseParams(), { + messages: [], + compaction: { enabled: true } + }) as Anthropic.MessageCreateParams & Partial; + expect(enabled.betas).to.deep.equal(['compact-2026-01-12']); + }); + + it('session disables compaction over a true default', () => { + const model = createCompactionModel(true); + + // compaction.enabled=false wins even when serverSideCompactionEnabledByDefault is true + const forcedOff = model.callApplyCompactionParams(baseParams(), { + messages: [], + compaction: { enabled: false } + }) as Anthropic.MessageCreateParams & Partial; + expect(forcedOff.betas).to.equal(undefined); + }); + + it('leaves params unchanged when the model is incapable (e.g. claude-haiku-4-5), regardless of activation', () => { + const incapableModel = new TestableAnthropicModel( + 'test-id', 'claude-haiku-4-5', { status: 'ready' }, true, false, () => 'test-key', undefined, + DEFAULT_MAX_TOKENS, 3, undefined, undefined, undefined, undefined, undefined, undefined, true + ); + const params = incapableModel.callApplyCompactionParams(baseParams(), { + messages: [], + compaction: { enabled: true } + }); + const beta = params as Anthropic.MessageCreateParams & Partial; + + expect(beta.betas).to.equal(undefined); + expect(beta.context_management).to.equal(undefined); + }); + + it('leaves params unchanged when no compaction request settings are provided and default is false', () => { + const model = createCompactionModel(false); + const params = model.callApplyCompactionParams(baseParams(), { messages: [] }); + const beta = params as Anthropic.MessageCreateParams & Partial; + + expect(beta.betas).to.equal(undefined); + expect(beta.context_management).to.equal(undefined); + }); + }); + + describe('computeServerSideCompactionSupport', () => { + function capabilityFor(modelId: string): boolean { + return new TestableAnthropicModel( + 'test-id', modelId, { status: 'ready' }, true, false, () => 'test-key', undefined + ).callComputeServerSideCompactionSupport(); + } + + it('returns true for claude-opus-4-6', () => { + expect(capabilityFor('claude-opus-4-6')).to.equal(true); + }); + it('returns true for claude-sonnet-4-6', () => { + expect(capabilityFor('claude-sonnet-4-6')).to.equal(true); + }); + it('returns true for claude-opus-4-7 (newer minor version)', () => { + expect(capabilityFor('claude-opus-4-7')).to.equal(true); + }); + it('returns false for claude-haiku-4-5 (haiku variant)', () => { + expect(capabilityFor('claude-haiku-4-5')).to.equal(false); + }); + it('returns false for claude-opus-4-5 (older minor version)', () => { + expect(capabilityFor('claude-opus-4-5')).to.equal(false); + }); + it('returns false for claude-sonnet-4-5 (older minor version, sonnet variant)', () => { + expect(capabilityFor('claude-sonnet-4-5')).to.equal(false); + }); + it('returns true for claude-opus-5-0 (newer major version)', () => { + expect(capabilityFor('claude-opus-5-0')).to.equal(true); + }); + it('returns false for claude-3-opus-20240229 (old date-style id)', () => { + expect(capabilityFor('claude-3-opus-20240229')).to.equal(false); + }); + it('returns false for claude-sonnet-4-20250514 (4.0 with a date suffix, not minor 20250514)', () => { + expect(capabilityFor('claude-sonnet-4-20250514')).to.equal(false); + }); + it('returns true for claude-opus-4-6-20250101 (4.6 with a date suffix)', () => { + expect(capabilityFor('claude-opus-4-6-20250101')).to.equal(true); + }); + it('returns false for an unrecognized model id', () => { + expect(capabilityFor('gpt-4o')).to.equal(false); + }); + }); + }); }); /** Builds a mock Anthropic client whose messages.stream() yields the supplied raw events. */ diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.ts b/packages/ai-anthropic/src/node/anthropic-language-model.ts index 5c824ae8b7dcc..6ccec60d061e8 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.ts @@ -28,6 +28,7 @@ import { LanguageModelTextResponse, ReasoningApi, ReasoningSupport, + resolveServerSideCompaction, ServerToolCallResponsePart, ServerToolDescriptor, ToolCallContent, @@ -58,8 +59,16 @@ interface ToolCallback { args: string; } -const createMessageContent = (message: LanguageModelMessage): MessageParam['content'] => { - if (LanguageModelMessage.isTextMessage(message)) { +const createMessageContent = (message: LanguageModelMessage, compactionEnabled: boolean): MessageParam['content'] => { + if (LanguageModelMessage.isCompactionMessage(message)) { + // Only replay our own provider's compaction blocks, and only when the request will use the beta endpoint. + // Returning [] for a skipped/foreign/disabled compaction message lets the surrounding real history carry the context. + if (compactionEnabled && message.provider === 'anthropic') { + const data = message.data as { content: string | null; encrypted_content: string | null }; + return [{ type: 'compaction', content: data.content, encrypted_content: data.encrypted_content }] as unknown as MessageParam['content']; + } + return []; + } else if (LanguageModelMessage.isTextMessage(message)) { return [{ type: 'text', text: message.text }]; } else if (LanguageModelMessage.isThinkingMessage(message)) { return [{ signature: message.signature, thinking: message.thinking, type: 'thinking' }]; @@ -135,11 +144,14 @@ function isNonThinkingParam( /** * Transforms Theia language model messages to Anthropic API format * @param messages Array of LanguageModelRequestMessage to transform + * @param addCacheControl whether to add prompt-cache control to the system message + * @param compactionEnabled whether the request will use the beta endpoint, so compaction replay blocks may be emitted * @returns Object containing transformed messages and optional system message */ -function transformToAnthropicParams( +export function transformToAnthropicParams( messages: readonly LanguageModelMessage[], - addCacheControl: boolean = true + addCacheControl: boolean = true, + compactionEnabled: boolean = false ): { messages: MessageParam[]; systemMessage?: Anthropic.Messages.TextBlockParam[] } { // Extract the system message (if any), as it is a separate parameter in the Anthropic API. const systemMessageObj = messages.find(message => message.actor === 'system'); @@ -151,8 +163,10 @@ function transformToAnthropicParams( .filter(message => message.actor !== 'system') .map(message => ({ role: toAnthropicRole(message), - content: createMessageContent(message) - })); + content: createMessageContent(message, compactionEnabled) + })) + // Drop messages whose content converted to empty (e.g. a skipped compaction message), so no empty turns are sent. + .filter(message => !Array.isArray(message.content) || message.content.length > 0); return { messages: mergeConsecutiveSameRoleMessages(convertedMessages), @@ -311,9 +325,29 @@ export class AnthropicModel implements LanguageModel { public reasoningApi?: ReasoningApi, public supportsXHighEffort?: boolean, public maxInputTokens?: number, - public serverTools?: ServerToolDescriptor[] + public serverTools?: ServerToolDescriptor[], + public serverSideCompactionEnabledByDefault: boolean = false ) { } + get serverSideCompactionSupport(): boolean { + return this.computeServerSideCompactionSupport(); + } + + /** + * Server-side compaction (`compact_20260112`) is available on Claude Opus and Sonnet 4.6 and later. + * Heuristic over the model id; override for custom endpoints with different support. + */ + protected computeServerSideCompactionSupport(): boolean { + const match = /(opus|sonnet)-(\d+)-(\d+)/.exec(this.model); + if (!match) { + return false; + } + const major = Number(match[2]); + // A 4+ digit "minor" is a date suffix on a `.0` model id (e.g. claude-sonnet-4-20250514), not a minor version. + const minor = Number(match[3]) >= 1000 ? 0 : Number(match[3]); + return major > 4 || (major === 4 && minor >= 6); + } + protected getSettings(request: LanguageModelRequest): Readonly> { return { ...request.settings, @@ -321,6 +355,25 @@ export class AnthropicModel implements LanguageModel { }; } + /** Resolves whether this request will route through the Anthropic Beta Messages API for server-side compaction. */ + protected useServerSideCompaction(request: LanguageModelRequest): boolean { + return resolveServerSideCompaction(this.serverSideCompactionSupport, this.serverSideCompactionEnabledByDefault, request.compaction); + } + + /** + * Augments the base message-create params with the beta flag and context-management edit when server-side compaction + * is enabled for the given request. When disabled, the params are returned unchanged so the default (stable) path is byte-for-byte identical. + */ + protected applyCompactionParams(params: T, request: LanguageModelRequest): T { + if (!this.useServerSideCompaction(request)) { + return params; + } + const betaParams = params as T & Anthropic.Beta.Messages.MessageCreateParams; + betaParams.betas = ['compact-2026-01-12']; + betaParams.context_management = { edits: [{ type: 'compact_20260112' }] }; + return betaParams; + } + async request(request: UserRequest, cancellationToken?: CancellationToken): Promise { if (!request.messages?.length) { throw new Error('Request must contain at least one message'); @@ -346,7 +399,8 @@ export class AnthropicModel implements LanguageModel { toolMessages?: readonly Anthropic.Messages.MessageParam[] ): Promise { const settings = this.getSettings(request); - const { messages, systemMessage } = transformToAnthropicParams(request.messages, this.useCaching); + const useCompaction = this.useServerSideCompaction(request); + const { messages, systemMessage } = transformToAnthropicParams(request.messages, this.useCaching, useCompaction); let anthropicMessages = [...messages, ...(toolMessages ?? [])]; @@ -362,7 +416,7 @@ export class AnthropicModel implements LanguageModel { defer_loading: 'defer_loading' in tool ? tool.defer_loading : undefined }))); } - const params: Anthropic.MessageCreateParams = { + const params: Anthropic.MessageCreateParams = this.applyCompactionParams({ max_tokens: this.maxTokens, messages: anthropicMessages, tools, @@ -370,8 +424,14 @@ export class AnthropicModel implements LanguageModel { model: this.model, ...(systemMessage && { system: systemMessage }), ...settings - }; - const stream = anthropic.messages.stream(params, { maxRetries: this.maxRetries }); + }, request); + // The beta message params are a structural superset of the stable ones for the content we build, so the cast at the beta call boundary is safe. + // The beta stream is likewise structurally compatible with the stable MessageStream: we narrow compaction events via explicit casts in the + // loop below, and casting the stream here keeps the iterator and lifecycle handling identical on both paths. + const stream = useCompaction + ? anthropic.beta.messages.stream(params as unknown as Anthropic.Beta.Messages.MessageCreateParamsStreaming, + { maxRetries: this.maxRetries }) as unknown as ReturnType + : anthropic.messages.stream(params, { maxRetries: this.maxRetries }); cancellationToken?.onCancellationRequested(() => { stream.abort(); @@ -390,6 +450,8 @@ export class AnthropicModel implements LanguageModel { let currentMessage: Message | undefined = undefined; let currentOutputTokens: number = 0; let usageYielded = false; + // Accumulator for a streamed compaction block (beta path only): content + opaque encrypted_content across start + deltas. + let compaction: { index: number; content: string | null; encrypted_content: string | null } | undefined; try { for await (const event of stream) { @@ -402,6 +464,11 @@ export class AnthropicModel implements LanguageModel { if (contentBlock.type === 'text') { yield { content: contentBlock.text }; } + // Compaction blocks only occur on the beta path; guard via type narrowing. + if (contentBlock.type === 'compaction') { + const block = contentBlock as Anthropic.Beta.Messages.BetaCompactionBlock; + compaction = { index: event.index, content: block.content, encrypted_content: block.encrypted_content }; + } if (contentBlock.type === 'tool_use') { toolCall = { name: contentBlock.name!, args: '', id: contentBlock.id!, index: event.index }; yield { tool_calls: [{ finished: false, id: toolCall.id, function: { name: toolCall.name, arguments: toolCall.args } }] }; @@ -449,6 +516,11 @@ export class AnthropicModel implements LanguageModel { if (delta.type === 'text_delta') { yield { content: delta.text }; } + if (compaction && delta.type === 'compaction_delta') { + const compactionDelta = delta as Anthropic.Beta.Messages.BetaCompactionContentBlockDelta; + compaction.content = compactionDelta.content; + compaction.encrypted_content = compactionDelta.encrypted_content; + } if (toolCall && delta.type === 'input_json_delta') { toolCall.args += delta.partial_json; yield { tool_calls: [{ function: { arguments: delta.partial_json } }] }; @@ -466,6 +538,11 @@ export class AnthropicModel implements LanguageModel { serverToolCalls.push(serverToolCall); serverToolCall = undefined; } + if (compaction && compaction.index === event.index) { + const data = { content: compaction.content, encrypted_content: compaction.encrypted_content }; + yield { compaction: { provider: 'anthropic', data, summary: compaction.content ?? undefined } }; + compaction = undefined; + } } else if (event.type === 'message_delta') { currentOutputTokens = event.usage.output_tokens; if (event.delta.stop_reason === 'max_tokens') { @@ -588,18 +665,22 @@ export class AnthropicModel implements LanguageModel { request: UserRequest ): Promise { const settings = this.getSettings(request); - const { messages, systemMessage } = transformToAnthropicParams(request.messages); + const useCompaction = this.useServerSideCompaction(request); + const { messages, systemMessage } = transformToAnthropicParams(request.messages, true, useCompaction); - const params: Anthropic.MessageCreateParams = { + const params: Anthropic.MessageCreateParams = this.applyCompactionParams({ max_tokens: this.maxTokens, messages, model: this.model, ...(systemMessage && { system: systemMessage }), ...settings, - }; + }, request); try { - const response = await anthropic.messages.create(params); + // The beta message params are a structural superset of the stable ones, so the cast at the beta call boundary is safe. + const response = useCompaction + ? await anthropic.beta.messages.create(params as Anthropic.Beta.Messages.MessageCreateParamsNonStreaming) + : await anthropic.messages.create(params); const textContent = response.content[0]; const usage = response.usage ? { diff --git a/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.spec.ts b/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.spec.ts index a8365dc802e71..bdfbfaef43f00 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.spec.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.spec.ts @@ -64,6 +64,7 @@ function description(model: string, url?: string): AnthropicModelDescription { enableStreaming: true, useCaching: true, maxRetries: 3, + serverSideCompactionEnabledByDefault: false, url }; } diff --git a/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts b/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts index ef64aeff7129e..994dd9aeea443 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts @@ -91,7 +91,8 @@ export class AnthropicLanguageModelsManagerImpl implements AnthropicLanguageMode reasoningSupport: metadata.reasoningSupport, reasoningApi: metadata.reasoningApi, supportsXHighEffort: metadata.supportsXHighEffort, - maxInputTokens: metadata.maxInputTokens + maxInputTokens: metadata.maxInputTokens, + serverSideCompactionEnabledByDefault: modelDescription.serverSideCompactionEnabledByDefault ?? false }); } else { this.languageModelRegistry.addLanguageModels([ @@ -110,7 +111,8 @@ export class AnthropicLanguageModelsManagerImpl implements AnthropicLanguageMode metadata.reasoningApi, metadata.supportsXHighEffort, metadata.maxInputTokens, - ANTHROPIC_SERVER_TOOLS + ANTHROPIC_SERVER_TOOLS, + modelDescription.serverSideCompactionEnabledByDefault ?? false ) ]); }