feat(prefix): auto-tune prefix match cap from model max_model_len

The approximate-prefix-cache producer caps prefix matching at a fixed
maxPrefixTokensToMatch (default 128k). On models with a larger context
window that under-matches long shared prefixes, and it's one more knob
to set per model.

When autoTune is on and maxPrefixTokensToMatch is left at its default,
derive the cap from each endpoint's max_model_len instead. The value is
read from the model server's /v1/models, fetched once per pod off the
request path and cached. Until it resolves (or if the endpoint can't be
reached) we fall back to the existing default, so behaviour is unchanged
in the common case. An explicitly configured cap is always respected.

The fetch reuses the model-server-metrics scheme/TLS flags so /v1/models
follows the same protocol as /metrics.

Signed-off-by: Mayur Das <mayur.das@neevcloud.com>

Author: mayur-tolexo

pkg/epp/framework/plugins/datalayer/attribute/models/data_types.go

@@ -37,6 +37,9 @@ type ModelDataCollection []ModelData
 type ModelData struct {
 	ID     string `json:"id"`
 	Parent string `json:"parent,omitempty"`
+	// MaxModelLen is the model's context window in tokens as reported by the
+	// model server; zero when the server does not report it.
+	MaxModelLen int `json:"max_model_len,omitempty"`
 }
 
 // String returns a string representation of the model info

pkg/epp/framework/plugins/datalayer/extractor/models/extractor_test.go

@@ -106,3 +106,32 @@ func TestExtractorExtract(t *testing.T) {
 		})
 	}
 }
+
+// TestExtractorCapturesMaxModelLen verifies the extractor surfaces max_model_len
+// from the /v1/models payload onto the endpoint attribute.
+func TestExtractorCapturesMaxModelLen(t *testing.T) {
+	ctx := context.Background()
+	extPlugin, err := ModelServerExtractorFactory("test-extractor", nil, nil)
+	if err != nil {
+		t.Fatalf("failed to create extractor: %v", err)
+	}
+	extractor := extPlugin.(*ModelExtractor)
+	ep := fwkdl.NewEndpoint(nil, nil)
+
+	resp := &ModelResponse{Data: []attrmodels.ModelData{{ID: "m", MaxModelLen: 131072}}}
+	if err := extractor.Extract(ctx, fwkdl.PollInput[*ModelResponse]{Payload: resp, Endpoint: ep}); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	val, ok := ep.GetAttributes().Get(attrmodels.ModelsAttributeKey.String())
+	if !ok {
+		t.Fatal("expected models attribute to be set")
+	}
+	models, ok := val.(attrmodels.ModelDataCollection)
+	if !ok {
+		t.Fatalf("unexpected attribute type %T", val)
+	}
+	if len(models) != 1 || models[0].MaxModelLen != 131072 {
+		t.Errorf("expected max_model_len 131072, got %+v", models)
+	}
+}

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/autotune_test.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package approximateprefix
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+
+	fwkdl "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/datalayer"
+	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin"
+	fwksched "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/scheduling"
+	attrmodels "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/models"
+	attrprefix "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/prefix"
+)
+
+// modelLenEndpoint builds an endpoint whose /v1/models attribute reports the
+// given max_model_len values, one per model entry.
+func modelLenEndpoint(name string, maxModelLens ...int) fwksched.Endpoint {
+	attrs := fwkdl.NewAttributes()
+	if maxModelLens != nil {
+		models := make(attrmodels.ModelDataCollection, len(maxModelLens))
+		for i, ml := range maxModelLens {
+			models[i] = attrmodels.ModelData{ID: "m", MaxModelLen: ml}
+		}
+		attrs.Put(attrmodels.ModelsAttributeKey.String(), models)
+	}
+	return fwksched.NewEndpoint(&fwkdl.EndpointMetadata{NamespacedName: k8stypes.NamespacedName{Name: name}}, fwkdl.NewMetrics(), attrs)
+}
+
+// TestProduceAutoTunesFromModelLen verifies the prefix cap is derived from the
+// endpoint's max_model_len attribute when auto-tuning, and falls back otherwise.
+func TestProduceAutoTunesFromModelLen(t *testing.T) {
+	disableMinBlockSizeClamp(t)
+
+	// A 20-block prompt at block size 1.
+	tokens := make([]uint32, 20)
+	for i := range tokens {
+		tokens[i] = uint32(i + 1)
+	}
+
+	produceBlocks := func(t *testing.T, p *dataProducer, ep fwksched.Endpoint) int {
+		t.Helper()
+		req := &fwksched.InferenceRequest{RequestID: uuid.NewString(), TargetModel: "m", Body: tokenizedBody(tokens)}
+		assert.NoError(t, p.Produce(context.Background(), req, []fwksched.Endpoint{ep}))
+		state, err := plugin.ReadPluginStateKey[*SchedulingContextState](
+			p.PluginState(), req.RequestID, plugin.StateKey(ApproxPrefixCachePluginType))
+		assert.NoError(t, err)
+		return len(state.PerPromptHashes[0])
+	}
+
+	autoTuneConfig := config{
+		AutoTune:               true,
+		BlockSizeTokens:        1,
+		MaxPrefixBlocksToMatch: defaultMaxPrefixBlocks,
+		MaxPrefixTokensToMatch: defaultMaxPrefixTokens, // left at default -> auto-tune on
+		LRUCapacityPerServer:   defaultLRUCapacityPerServer,
+	}
+
+	t.Run("caps at max_model_len when known", func(t *testing.T) {
+		p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, autoTuneConfig, testHandle())
+		assert.NoError(t, err)
+		assert.True(t, p.config.autoTuneMaxPrefixTokens)
+		// max_model_len = 10 -> cap = 10/blockSize(1) = 10 blocks.
+		assert.Equal(t, 10, produceBlocks(t, p, modelLenEndpoint("pod1", 10)))
+	})
+
+	t.Run("uses largest across models", func(t *testing.T) {
+		p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, autoTuneConfig, testHandle())
+		assert.NoError(t, err)
+		assert.Equal(t, 12, produceBlocks(t, p, modelLenEndpoint("pod1", 8, 12)))
+	})
+
+	t.Run("falls back to default when attribute absent", func(t *testing.T) {
+		p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, autoTuneConfig, testHandle())
+		assert.NoError(t, err)
+		// No models attribute -> default cap (131072) far exceeds 20 blocks.
+		assert.Equal(t, 20, produceBlocks(t, p, modelLenEndpoint("pod1")))
+	})
+
+	t.Run("falls back when max_model_len is zero", func(t *testing.T) {
+		p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, autoTuneConfig, testHandle())
+		assert.NoError(t, err)
+		// Attribute present but server did not report the field.
+		assert.Equal(t, 20, produceBlocks(t, p, modelLenEndpoint("pod1", 0)))
+	})
+
+	t.Run("explicit cap is not auto-tuned", func(t *testing.T) {
+		explicit := autoTuneConfig
+		explicit.MaxPrefixTokensToMatch = 5 // operator-pinned, non-default
+		p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, explicit, testHandle())
+		assert.NoError(t, err)
+		assert.False(t, p.config.autoTuneMaxPrefixTokens)
+		// Even with a known model length, the explicit cap (5) wins.
+		assert.Equal(t, 5, produceBlocks(t, p, modelLenEndpoint("pod1", 10)))
+	})
+
+	t.Run("empty pods uses default cap", func(t *testing.T) {
+		p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, autoTuneConfig, testHandle())
+		assert.NoError(t, err)
+
+		req := &fwksched.InferenceRequest{RequestID: uuid.NewString(), TargetModel: "m", Body: tokenizedBody(tokens)}
+		assert.NoError(t, p.Produce(context.Background(), req, []fwksched.Endpoint{}))
+		state, err := plugin.ReadPluginStateKey[*SchedulingContextState](
+			p.PluginState(), req.RequestID, plugin.StateKey(ApproxPrefixCachePluginType))
+		assert.NoError(t, err)
+		assert.Equal(t, 20, len(state.PerPromptHashes[0]))
+	})
+}
+
+// TestConsumesDeclaresModelsDependency verifies the producer declares the
+// /v1/models attribute as a required dependency so its source is run.
+func TestConsumesDeclaresModelsDependency(t *testing.T) {
+	p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, defaultConfig, testHandle())
+	assert.NoError(t, err)
+	_, ok := p.Consumes().Required[attrmodels.ModelsAttributeKey]
+	assert.True(t, ok, "models attribute must be a required dependency")
+}
+
+// TestMaxModelLenHelper covers the attribute reader's non-collection and
+// missing-attribute paths.
+func TestMaxModelLenHelper(t *testing.T) {
+	// Missing attribute -> 0.
+	assert.Equal(t, 0, maxModelLen(modelLenEndpoint("pod1")))
+
+	// Wrong type stored under the key -> 0.
+	attrs := fwkdl.NewAttributes()
+	attrs.Put(attrmodels.ModelsAttributeKey.String(), attrprefix.NewPrefixCacheMatchInfo(0, 0, 0))
+	ep := fwksched.NewEndpoint(&fwkdl.EndpointMetadata{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}}, fwkdl.NewMetrics(), attrs)
+	assert.Equal(t, 0, maxModelLen(ep))
+}

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/plugin.go

@@ -30,6 +30,7 @@ import (
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin"
 	"github.com/llm-d/llm-d-router/pkg/epp/framework/interface/requestcontrol"
 	fwksched "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/scheduling"
+	attrmodels "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/models"
 	attrprefix "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/prefix"
 	approxprefixconstants "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/constants"
 	tokenproducer "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer"
@@ -79,7 +80,11 @@ func (p *dataProducer) Produces() map[plugin.DataKey]any {
 // is configured.
 func (p *dataProducer) Consumes() plugin.DataDependencies {
 	return plugin.DataDependencies{
-		Required: map[plugin.DataKey]any{tokenproducer.TokenizedPromptDataKey: fwksched.TokenizedPrompt{}},
+		Required: map[plugin.DataKey]any{
+			tokenproducer.TokenizedPromptDataKey: fwksched.TokenizedPrompt{},
+			// Supplies each endpoint's max_model_len for auto-tuning the cap in Produce.
+			attrmodels.ModelsAttributeKey: attrmodels.ModelDataCollection{},
+		},
 	}
 }
 
@@ -118,6 +123,10 @@ func newDataProducer(ctx context.Context, name string, config config, handle plu
 
 	indexer := newIndexer(ctx, config.LRUCapacityPerServer, name, ApproxPrefixCachePluginType)
 
+	// Auto-tune the cap from max_model_len only when AutoTune is on and the
+	// operator left MaxPrefixTokensToMatch at its default (no explicit cap).
+	config.autoTuneMaxPrefixTokens = config.AutoTune && config.MaxPrefixTokensToMatch == defaultMaxPrefixTokens
+
 	p := &dataProducer{
 		typedName: plugin.TypedName{
 			Type: ApproxPrefixCachePluginType,
@@ -175,9 +184,16 @@ func (p *dataProducer) PluginState() *plugin.PluginState {
 // Produce is called by the director before scheduling requests.
 func (p *dataProducer) Produce(ctx context.Context, request *fwksched.InferenceRequest, pods []fwksched.Endpoint) error {
 	blockSize := p.GetBlockSize(pods)
+	maxTokens := p.config.MaxPrefixTokensToMatch
+	// Raise the cap to the model's full context window when max_model_len is known.
+	if p.config.autoTuneMaxPrefixTokens && len(pods) > 0 {
+		if modelLen := maxModelLen(pods[0]); modelLen > 0 {
+			maxTokens = modelLen
+		}
+	}
 	maxBlocks := p.config.MaxPrefixBlocksToMatch
-	if p.config.MaxPrefixTokensToMatch > 0 && blockSize > 0 {
-		maxBlocks = p.config.MaxPrefixTokensToMatch / blockSize
+	if maxTokens > 0 && blockSize > 0 {
+		maxBlocks = maxTokens / blockSize
 	}
 	perPromptHashes := getBlockHashes(ctx, request, blockSize, maxBlocks)
 
@@ -307,6 +323,26 @@ func (p *dataProducer) GetBlockSize(endpoints []fwksched.Endpoint) int {
 	return blockSize
 }
 
+// maxModelLen returns the largest max_model_len reported for ep via the
+// /v1/models attribute, or 0 when absent or non-positive.
+func maxModelLen(ep fwksched.Endpoint) int {
+	val, ok := ep.Get(attrmodels.ModelsAttributeKey.String())
+	if !ok {
+		return 0
+	}
+	models, ok := val.(attrmodels.ModelDataCollection)
+	if !ok {
+		return 0
+	}
+	maxLen := 0
+	for i := range models {
+		if models[i].MaxModelLen > maxLen {
+			maxLen = models[i].MaxModelLen
+		}
+	}
+	return maxLen
+}
+
 // ApproxPrefixCacheFactory is the factory function for the prefix cache data producer plugin.
 func ApproxPrefixCacheFactory(name string, rawParameters *json.Decoder, handle plugin.Handle) (plugin.Plugin, error) {
 	parameters := defaultConfig

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/types.go

@@ -140,6 +140,11 @@ type config struct {
 	MaxPrefixTokensToMatch int `json:"maxPrefixTokensToMatch"`
 	// Max capacity size of the LRU indexer in number of entries per server (pod).
 	LRUCapacityPerServer int `json:"lruCapacityPerServer"`
+
+	// autoTuneMaxPrefixTokens, set internally (never from JSON), derives the cap
+	// per endpoint from the model's max_model_len. Enabled when AutoTune is on and
+	// MaxPrefixTokensToMatch was left at its default (no operator-pinned cap).
+	autoTuneMaxPrefixTokens bool
 }
 
 // defaultConfig provides sensible defaults for the prefix cache plugins.