feat(chat): add end-to-end context compression (mudler#11556)

* feat(config): add context compression policy

Define the opt-in model configuration contract before the chat middleware consumes it. Document each policy field so later request handling does not invent a second schema.\n\nRefs mudler#9534\n\nAssisted-by: Codex:gpt-5

* fix(config): register compression fields

The model editor metadata gate rejects new config fields without descriptions and suitable controls. Register the compression policy so operators can edit its six fields safely.

Assisted-by: Codex:gpt-5 [monitoring-prs]

* feat(chat): compress long contexts

Long conversations currently fail once they reach the model context window. The opt-in policy now summarizes complete older turns before primary inference and preserves the newest tool chains.

Both OpenAI and MCP chat routes share the same transformation. Usage metadata and metrics expose each compression event.

Refs mudler#9534

Assisted-by: Codex:gpt-5

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

Author: localai-org-maint-bot

core/config/meta/registry.go

@@ -185,6 +185,64 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Advanced: true,
 			Order:    22,
 		},
+		"compression.enabled": {
+			Section:     "llm",
+			Label:       "Context Compression",
+			Description: "Enable compression of chat history before it reaches the model context limit",
+			Component:   "checkbox",
+			Advanced:    true,
+			Order:       24,
+		},
+		"compression.trigger_at_ratio": {
+			Section:     "llm",
+			Label:       "Compression Trigger Ratio",
+			Description: "Fraction of the model context window that starts compression",
+			Component:   "slider",
+			Min:         f64(0),
+			Max:         f64(1),
+			Step:        f64(0.05),
+			Advanced:    true,
+			Order:       25,
+		},
+		"compression.keep_tail_tokens": {
+			Section:     "llm",
+			Label:       "Compression Tail Tokens",
+			Description: "Number of newest conversation tokens to keep outside the summary",
+			Component:   "number",
+			Min:         f64(0),
+			Advanced:    true,
+			Order:       26,
+		},
+		"compression.max_summary_tokens": {
+			Section:     "llm",
+			Label:       "Maximum Summary Tokens",
+			Description: "Maximum number of tokens produced by context compression",
+			Component:   "number",
+			Min:         f64(0),
+			Advanced:    true,
+			Order:       27,
+		},
+		"compression.compressor_model": {
+			Section:              "llm",
+			Label:                "Compressor Model",
+			Description:          "Chat model used to summarize context; empty uses this model",
+			Component:            "model-select",
+			AutocompleteProvider: ProviderModelsChat,
+			Advanced:             true,
+			Order:                28,
+		},
+		"compression.on_post_compression_overflow": {
+			Section:     "llm",
+			Label:       "Post-compression Overflow",
+			Description: "Action to take when compressed context still exceeds the context limit",
+			Component:   "select",
+			Options: []FieldOption{
+				{Value: "drop_oldest_summary", Label: "Drop Oldest Summary"},
+				{Value: "error", Label: "Return Error"},
+			},
+			Advanced: true,
+			Order:    29,
+		},
 		"cache_type_k": {
 			Section:     "llm",
 			Label:       "KV Cache Type (K)",

core/config/model_config.go

@@ -84,6 +84,7 @@ type ModelConfig struct {
 
 	FunctionsConfig functions.FunctionsConfig `yaml:"function,omitempty" json:"function,omitempty"`
 	ReasoningConfig reasoning.Config          `yaml:"reasoning,omitempty" json:"reasoning,omitempty"`
+	Compression     CompressionConfig         `yaml:"compression,omitempty" json:"compression,omitempty"`
 
 	// ReasoningEffort is the default reasoning effort (none|minimal|low|medium|high)
 	// for this model. A per-request reasoning_effort overrides it. It is forwarded
@@ -152,6 +153,18 @@ type ModelConfig struct {
 	Limits       LimitsConfig       `yaml:"limits,omitempty" json:"limits,omitempty"`
 }
 
+// CompressionConfig controls opt-in compression of chat history before inference.
+// The request middleware consumes this configuration; keeping it on ModelConfig
+// lets operators select a policy per context window and model workload.
+type CompressionConfig struct {
+	Enabled                   bool    `yaml:"enabled,omitempty" json:"enabled,omitempty"`
+	TriggerAtRatio            float64 `yaml:"trigger_at_ratio,omitempty" json:"trigger_at_ratio,omitempty"`
+	KeepTailTokens            int     `yaml:"keep_tail_tokens,omitempty" json:"keep_tail_tokens,omitempty"`
+	MaxSummaryTokens          int     `yaml:"max_summary_tokens,omitempty" json:"max_summary_tokens,omitempty"`
+	CompressorModel           string  `yaml:"compressor_model,omitempty" json:"compressor_model,omitempty"`
+	OnPostCompressionOverflow string  `yaml:"on_post_compression_overflow,omitempty" json:"on_post_compression_overflow,omitempty"`
+}
+
 // @Description Admission-control limits applied per request. The
 // admission middleware enforces these before invoking the handler;
 // requests that exceed a limit get 503 with a Retry-After hint so
@@ -1538,6 +1551,22 @@ func (cfg *ModelConfig) SetDefaults(opts ...ConfigLoaderOption) {
 }
 
 func (c *ModelConfig) Validate() (bool, error) {
+	if c.Compression.Enabled {
+		if c.IsCloudProxyBackendPassthrough() {
+			return false, fmt.Errorf("compression: cloud-proxy passthrough is unsupported; configure proxy mode translate")
+		}
+		if c.Compression.TriggerAtRatio < 0 || c.Compression.TriggerAtRatio > 1 {
+			return false, fmt.Errorf("compression: trigger_at_ratio must be between 0 and 1")
+		}
+		if c.Compression.KeepTailTokens < 0 || c.Compression.MaxSummaryTokens < 0 {
+			return false, fmt.Errorf("compression: token limits cannot be negative")
+		}
+		switch c.Compression.OnPostCompressionOverflow {
+		case "", "error", "drop_oldest_summary":
+		default:
+			return false, fmt.Errorf("compression: unknown on_post_compression_overflow %q", c.Compression.OnPostCompressionOverflow)
+		}
+	}
 	if c.IsAlias() && len(c.Artifacts) > 0 {
 		return false, fmt.Errorf("alias model %q cannot declare artifacts", c.Name)
 	}

core/config/model_config_test.go

@@ -52,6 +52,54 @@ parameters:
 		Expect(valid).To(BeTrue())
 	})
 
+	It("round-trips context compression settings", func() {
+		raw := []byte(`
+name: compressed-chat
+backend: llama-cpp
+parameters:
+  model: chat.gguf
+compression:
+  enabled: true
+  trigger_at_ratio: 0.75
+  keep_tail_tokens: 8000
+  max_summary_tokens: 2048
+  compressor_model: fast-summarizer
+  on_post_compression_overflow: error
+`)
+		var cfg ModelConfig
+		Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed())
+		Expect(cfg.Compression.Enabled).To(BeTrue())
+		Expect(cfg.Compression.TriggerAtRatio).To(Equal(0.75))
+		Expect(cfg.Compression.KeepTailTokens).To(Equal(8000))
+		Expect(cfg.Compression.MaxSummaryTokens).To(Equal(2048))
+		Expect(cfg.Compression.CompressorModel).To(Equal("fast-summarizer"))
+		Expect(cfg.Compression.OnPostCompressionOverflow).To(Equal("error"))
+	})
+
+	It("rejects invalid context compression policies", func() {
+		cfg := ModelConfig{Compression: CompressionConfig{Enabled: true, TriggerAtRatio: 1.1}}
+		valid, err := cfg.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("trigger_at_ratio")))
+
+		cfg.Compression.TriggerAtRatio = 0.75
+		cfg.Compression.OnPostCompressionOverflow = "truncate_anything"
+		valid, err = cfg.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("on_post_compression_overflow")))
+	})
+
+	It("rejects context compression for cloud proxy passthrough", func() {
+		cfg := ModelConfig{
+			Backend:     "cloud-proxy",
+			Proxy:       ProxyConfig{Mode: ProxyModePassthrough},
+			Compression: CompressionConfig{Enabled: true},
+		}
+		valid, err := cfg.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("cloud-proxy passthrough")))
+	})
+
 	It("derives a managed snapshot filename without replacing the logical model", func() {
 		const cacheKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
 		cfg := ModelConfig{

core/http/endpoints/localai/mcp.go

@@ -57,15 +57,15 @@ type MCPErrorEvent struct {
 // @Param request body schema.OpenAIRequest true "query params"
 // @Success 200 {object} schema.OpenAIResponse "Response"
 // @Router /v1/mcp/chat/completions [post]
-func MCPEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient) echo.HandlerFunc {
+func MCPEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient, compressor middleware.ChatCompressor) echo.HandlerFunc {
 	// The legacy /v1/mcp/chat/completions endpoint never opts into the
 	// in-process LocalAI Assistant tool surface — pass nil holder so the
 	// assistant branch in chat.go is unreachable from this code path.
 	// Stream-side PII filter is also nil: this legacy endpoint pre-dates
 	// the per-model PII config and is kept for backward compatibility.
 	// The request-side middleware on the main chat route handles
 	// filtering for the standard /v1/chat/completions path.
-	chatHandler := openai.ChatEndpoint(cl, ml, evaluator, appConfig, natsClient, nil)
+	chatHandler := openai.ChatEndpoint(cl, ml, evaluator, appConfig, natsClient, nil, compressor)
 
 	return func(c echo.Context) error {
 		input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)

core/http/endpoints/openai/chat.go

@@ -129,7 +129,7 @@ func applyAutoparserOverride(
 // @Param request body schema.OpenAIRequest true "query params"
 // @Success 200 {object} schema.OpenAIResponse "Response"
 // @Router /v1/chat/completions [post]
-func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, startupOptions *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient, assistantHolder *mcpTools.LocalAIAssistantHolder) echo.HandlerFunc {
+func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, startupOptions *config.ApplicationConfig, natsClient mcpTools.MCPNATSClient, assistantHolder *mcpTools.LocalAIAssistantHolder, compressor middleware.ChatCompressor) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		var textContentToReturn string
 		id := uuid.New().String()
@@ -155,6 +155,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 		// redaction already ran in the middleware; the response is
 		// forwarded unmodified.
 		if config.IsCloudProxyBackendPassthrough() {
+			if err := middleware.CompressChatRequest(c, compressor); err != nil {
+				return err
+			}
 			return forwardCloudProxyOpenAIViaBackend(c, config, input, ml, startupOptions)
 		}
 
@@ -201,7 +204,8 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 			// system message they're responsible for keeping the assistant
 			// safe, so we leave it alone.
 			if !hasSystemMessage(input.Messages) {
-				input.Messages = append([]schema.Message{{Role: "system", StringContent: assistantHolder.SystemPrompt()}}, input.Messages...)
+				prompt := assistantHolder.SystemPrompt()
+				input.Messages = append([]schema.Message{{Role: "system", Content: prompt, StringContent: prompt}}, input.Messages...)
 			}
 
 			xlog.Debug("LocalAI Assistant tools injected", "count", len(mcpFuncs))
@@ -244,6 +248,9 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 				xlog.Error("Failed to parse MCP config", "error", mcpErr)
 			}
 		}
+		if err := middleware.CompressChatRequest(c, compressor); err != nil {
+			return err
+		}
 
 		xlog.Debug("Tool call routing decision",
 			"shouldUseFn", shouldUseFn,
@@ -401,9 +408,17 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 
 			for mcpStreamIter := 0; mcpStreamIter <= mcpStreamMaxIterations; mcpStreamIter++ {
 				// Re-template on MCP iterations
-				if mcpStreamIter > 0 && !config.TemplateConfig.UseTokenizerTemplate {
-					predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
-					xlog.Debug("MCP stream re-templating", "iteration", mcpStreamIter)
+				if mcpStreamIter > 0 {
+					if err := middleware.CompressChatRequest(c, compressor); err != nil {
+						fmt.Fprintf(c.Response().Writer, "data: {\"error\":{\"message\":%q,\"type\":\"context_compression_error\"}}\n\n", err.Error())
+						fmt.Fprintf(c.Response().Writer, "data: [DONE]\n\n")
+						c.Response().Flush()
+						return nil
+					}
+					if !config.TemplateConfig.UseTokenizerTemplate {
+						predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
+						xlog.Debug("MCP stream re-templating", "iteration", mcpStreamIter)
+					}
 				}
 
 				responses := make(chan schema.OpenAIResponse)
@@ -658,6 +673,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 				// tools-path worker not surfacing this value at all.
 				if input.StreamOptions != nil && input.StreamOptions.IncludeUsage {
 					trailerUsage := streamUsageFromTokenUsage(finalUsage, extraUsage)
+					trailerUsage.CompressionMeta = middleware.CompressionMetadata(c)
 					trailer := streamUsageTrailerJSON(id, input.Model, created, trailerUsage)
 					_, _ = fmt.Fprintf(c.Response().Writer, "data: %s\n\n", trailer)
 				}
@@ -683,9 +699,14 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 
 			for mcpIteration := 0; mcpIteration <= mcpMaxIterations; mcpIteration++ {
 				// Re-template on each MCP iteration since messages may have changed
-				if mcpIteration > 0 && !config.TemplateConfig.UseTokenizerTemplate {
-					predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
-					xlog.Debug("MCP re-templating", "iteration", mcpIteration, "prompt_len", len(predInput))
+				if mcpIteration > 0 {
+					if err := middleware.CompressChatRequest(c, compressor); err != nil {
+						return err
+					}
+					if !config.TemplateConfig.UseTokenizerTemplate {
+						predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn)
+						xlog.Debug("MCP re-templating", "iteration", mcpIteration, "prompt_len", len(predInput))
+					}
 				}
 
 				// Detect if thinking token is already in prompt or template
@@ -1010,6 +1031,7 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 					usage.TimingTokenGeneration = tokenUsage.TimingTokenGeneration
 					usage.TimingPromptProcessing = tokenUsage.TimingPromptProcessing
 				}
+				usage.CompressionMeta = middleware.CompressionMetadata(c)
 
 				resp := &schema.OpenAIResponse{
 					ID:      id,

core/http/middleware/compression.go

@@ -0,0 +1,102 @@
+package middleware
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+	compressionservice "github.com/mudler/LocalAI/core/services/compression"
+	"github.com/mudler/LocalAI/pkg/tokens"
+)
+
+const contextKeyCompressionMetadata = "COMPRESSION_METADATA"
+
+type ChatCompressor interface {
+	Transform(context.Context, config.CompressionConfig, int, string, []schema.Message) ([]schema.Message, *compressionservice.Metadata, error)
+}
+
+func ContextCompression(compressor ChatCompressor) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			if err := CompressChatRequest(c, compressor); err != nil {
+				return err
+			}
+			return next(c)
+		}
+	}
+}
+
+func CompressChatRequest(c echo.Context, compressor ChatCompressor) error {
+	cfg, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
+	if !ok || cfg == nil || !cfg.Compression.Enabled {
+		return nil
+	}
+	if cfg.IsCloudProxyBackendPassthrough() {
+		return echo.NewHTTPError(http.StatusBadRequest, "context compression is not supported by cloud-proxy passthrough models; configure translate mode or a local compressor model")
+	}
+	input, ok := c.Get(CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
+	if !ok {
+		return echo.NewHTTPError(http.StatusBadRequest, "context compression requires a chat request")
+	}
+	contextSize := config.DefaultContextSize
+	if cfg.ContextSize != nil && *cfg.ContextSize > 0 {
+		contextSize = *cfg.ContextSize
+	}
+	extraPayload := make(map[string]any)
+	if len(input.Functions) > 0 {
+		extraPayload["functions"] = input.Functions
+	}
+	if len(input.Tools) > 0 {
+		extraPayload["tools"] = input.Tools
+	}
+	if input.FunctionCall != nil {
+		extraPayload["function_call"] = input.FunctionCall
+	}
+	if input.ToolsChoice != nil {
+		extraPayload["tool_choice"] = input.ToolsChoice
+	}
+	if input.ResponseFormat != nil {
+		extraPayload["response_format"] = input.ResponseFormat
+	}
+	requestOverhead, err := tokens.CountPayload(extraPayload)
+	if err != nil {
+		return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+	}
+	if cfg.Maxtokens != nil && *cfg.Maxtokens > 0 {
+		requestOverhead += *cfg.Maxtokens
+	}
+	messages, meta, err := compressor.Transform(c.Request().Context(), cfg.Compression, contextSize-requestOverhead, cfg.ModelID(), input.Messages)
+	if err != nil {
+		if compressionservice.IsOverflow(err) {
+			return echo.NewHTTPError(http.StatusRequestEntityTooLarge, err.Error())
+		}
+		return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
+	}
+	input.Messages = messages
+	if meta != nil {
+		meta.OriginalTokens += requestOverhead
+		meta.CompressedTokens += requestOverhead
+		if previous, ok := c.Get(contextKeyCompressionMetadata).(*compressionservice.Metadata); ok && previous != nil {
+			meta.OriginalTokens = previous.OriginalTokens
+			meta.DroppedTurns += previous.DroppedTurns
+			meta.SummaryTokens += previous.SummaryTokens
+			meta.OverflowRecoveries += previous.OverflowRecoveries
+		}
+		c.Set(contextKeyCompressionMetadata, meta)
+	}
+	return nil
+}
+
+func CompressionMetadata(c echo.Context) *schema.CompressionMetadata {
+	meta, ok := c.Get(contextKeyCompressionMetadata).(*compressionservice.Metadata)
+	if !ok || meta == nil {
+		return nil
+	}
+	return &schema.CompressionMetadata{
+		OriginalTokens: meta.OriginalTokens, CompressedTokens: meta.CompressedTokens,
+		DroppedTurns: meta.DroppedTurns, Compressor: meta.Compressor,
+		SummaryTokens: meta.SummaryTokens, OverflowRecoveries: meta.OverflowRecoveries,
+	}
+}