feat(prefix): auto-tune prefix match cap from model max_model_len

The approximate-prefix-cache producer caps prefix matching at a fixed
maxPrefixTokensToMatch (default 128k). On models with a larger context
window that under-matches long shared prefixes, and it's one more knob
to set per model.

When autoTune is on and maxPrefixTokensToMatch is left at its default,
derive the cap from each endpoint's max_model_len instead. The value is
read from the model server's /v1/models, fetched once per pod off the
request path and cached. Until it resolves (or if the endpoint can't be
reached) we fall back to the existing default, so behaviour is unchanged
in the common case. An explicitly configured cap is always respected.

The fetch reuses the model-server-metrics scheme/TLS flags so /v1/models
follows the same protocol as /metrics.

Signed-off-by: Mayur Das <mayur.das@neevcloud.com>

Author: mayur-tolexo

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/modellen.go

@@ -0,0 +1,221 @@
+/*
+Copyright 2026 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package approximateprefix
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/spf13/pflag"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	logutil "github.com/llm-d/llm-d-router/pkg/common/observability/logging"
+	fwksched "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/scheduling"
+)
+
+const (
+	// modelsPath reports each served model's max_model_len, on the same
+	// host:port as metrics scraping.
+	modelsPath = "/v1/models"
+
+	// modelLenFetchTimeout bounds a single /v1/models request, matching the
+	// metrics scraper's client timeout.
+	modelLenFetchTimeout = 10 * time.Second
+
+	// schemeFlag and insecureFlag are the metrics scraper's flags; /v1/models
+	// reuses them so it follows the same scheme and TLS policy as /metrics. The
+	// names are duplicated as strings to avoid an import cycle with the server
+	// package, and default to plain http when unset.
+	schemeFlag   = "model-server-metrics-scheme"
+	insecureFlag = "model-server-metrics-https-insecure-skip-verify"
+)
+
+// resolveModelServerScheme returns the scheme and TLS-skip policy for reaching
+// the model server, taken from the metrics scraper flags and defaulting to
+// plain http with verification on.
+func resolveModelServerScheme() (scheme string, insecureSkipVerify bool) {
+	scheme = "http"
+	if f := pflag.Lookup(schemeFlag); f != nil && f.Changed {
+		scheme = f.Value.String()
+	}
+	if f := pflag.Lookup(insecureFlag); f != nil && f.Changed {
+		insecureSkipVerify, _ = strconv.ParseBool(f.Value.String())
+	}
+	return scheme, insecureSkipVerify
+}
+
+// modelsResponse is the consumed subset of the /v1/models payload.
+type modelsResponse struct {
+	Data []struct {
+		MaxModelLen int `json:"max_model_len"`
+	} `json:"data"`
+}
+
+// modelLenCache resolves and caches each endpoint's max_model_len, which is
+// static per pod. Lookups never block the request path: a miss returns 0 and
+// schedules a background fetch, so the caller uses its default until resolved.
+type modelLenCache struct {
+	mu sync.RWMutex
+	// byServer holds resolved context windows; a missing key means unknown.
+	byServer map[ServerID]int
+	// inflight dedups concurrent fetches to one request per endpoint.
+	inflight map[ServerID]struct{}
+
+	// baseCtx scopes fetches to the plugin lifetime, not the triggering request.
+	baseCtx context.Context
+	// scheme is the protocol used to reach the model server (http or https).
+	scheme string
+	client *http.Client
+
+	// fetchFn is the resolver, swappable in tests to avoid a real HTTP server.
+	fetchFn func(ctx context.Context, address, port string) (int, error)
+}
+
+// newModelLenCache returns an empty cache whose fetches are scoped to baseCtx
+// and use the model server's configured scheme and TLS policy. The transport is
+// cloned from the default so connections are pooled across endpoint fetches.
+func newModelLenCache(baseCtx context.Context) *modelLenCache {
+	scheme, insecureSkipVerify := resolveModelServerScheme()
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	if scheme == "https" {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: insecureSkipVerify}
+	}
+	c := &modelLenCache{
+		byServer: make(map[ServerID]int),
+		inflight: make(map[ServerID]struct{}),
+		baseCtx:  baseCtx,
+		scheme:   scheme,
+		client:   &http.Client{Timeout: modelLenFetchTimeout, Transport: transport},
+	}
+	c.fetchFn = c.httpFetch
+	return c
+}
+
+// get returns ep's max_model_len, or 0 if not yet resolved (scheduling a
+// background fetch on the first miss).
+func (c *modelLenCache) get(ep fwksched.Endpoint) int {
+	meta := ep.GetMetadata()
+	if meta == nil {
+		return 0
+	}
+	id := ServerID(meta.NamespacedName)
+
+	c.mu.RLock()
+	val, known := c.byServer[id]
+	c.mu.RUnlock()
+	if known {
+		return val
+	}
+
+	c.scheduleFetch(id, meta.Address, meta.Port)
+	return 0
+}
+
+// scheduleFetch starts one background fetch for id, skipping if it is already
+// resolved or in flight.
+func (c *modelLenCache) scheduleFetch(id ServerID, address, port string) {
+	// Claim the in-flight slot, or bail if resolved/claimed meanwhile.
+	c.mu.Lock()
+	if _, ok := c.byServer[id]; ok {
+		c.mu.Unlock()
+		return
+	}
+	if _, ok := c.inflight[id]; ok {
+		c.mu.Unlock()
+		return
+	}
+	c.inflight[id] = struct{}{}
+	c.mu.Unlock()
+
+	go func() {
+		// Release the slot so a failed fetch can be retried later.
+		defer func() {
+			c.mu.Lock()
+			delete(c.inflight, id)
+			c.mu.Unlock()
+		}()
+
+		ctx, cancel := context.WithTimeout(c.baseCtx, modelLenFetchTimeout)
+		defer cancel()
+
+		val, err := c.fetchFn(ctx, address, port)
+		if err != nil {
+			// Stay unknown: caller keeps its default and a later request retries.
+			log.FromContext(ctx).V(logutil.DEFAULT).Info(
+				"failed to fetch max_model_len; using default prefix cap",
+				"endpoint", id.String(), "error", err.Error())
+			return
+		}
+		if val <= 0 {
+			return
+		}
+		c.mu.Lock()
+		c.byServer[id] = val
+		c.mu.Unlock()
+	}()
+}
+
+// remove drops id's cached value so a recreated pod re-resolves.
+func (c *modelLenCache) remove(id ServerID) {
+	c.mu.Lock()
+	delete(c.byServer, id)
+	delete(c.inflight, id)
+	c.mu.Unlock()
+}
+
+// httpFetch GETs /v1/models and returns the largest max_model_len reported, so
+// a multi-model pod's cap still covers its longest context.
+func (c *modelLenCache) httpFetch(ctx context.Context, address, port string) (int, error) {
+	url := fmt.Sprintf("%s://%s%s", c.scheme, net.JoinHostPort(address, port), modelsPath)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return 0, err
+	}
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("unexpected status %d from %s", resp.StatusCode, url)
+	}
+
+	var parsed modelsResponse
+	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
+		return 0, fmt.Errorf("decode /v1/models response: %w", err)
+	}
+
+	maxLen := 0
+	for _, m := range parsed.Data {
+		if m.MaxModelLen > maxLen {
+			maxLen = m.MaxModelLen
+		}
+	}
+	if maxLen <= 0 {
+		return 0, fmt.Errorf("no positive max_model_len in /v1/models response from %s", url)
+	}
+	return maxLen, nil
+}