feat(epp): match whole prompt when both prefix caps are zero (llm-d#2058)

Zeroing both maxPrefixTokensToMatch and maxPrefixBlocksToMatch hashed no
blocks at all, leaving the producer, its indexer, and its cleanup
goroutine running while contributing nothing to scoring. Nothing in the
config surface signalled that.

Treat the pair as "no cap" instead. Prompt length is already bounded by
the model server's context window, so an absent cap matches at most one
context window of tokens without operators having to name a token count
that tracks max_model_len by hand.

A token cap smaller than the block size still resolves to zero blocks;
that is a misconfiguration rather than a request for unlimited matching,
so it keeps hashing nothing.

Signed-off-by: Kaushik Mitra <kaushikmitra@google.com>

Author: kaushikmitr

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/README.md

@@ -12,8 +12,10 @@ For each request, the plugin consumes `request.Body.TokenizedPrompt` (token IDs)
 
 - `autoTune` (bool, optional, default: `true`): Infer `blockSizeTokens` and `lruCapacityPerServer` from endpoint metrics when available.
 - `blockSizeTokens` (int, optional, default: `16`): Prefix block size in tokens. Used when `autoTune` is false or endpoint metrics are unavailable; ignored when auto-tuned from metrics. Values below the minimum of `64` are clamped up at request time (see #1158), so the `16` default is effectively `64` absent a larger metric/configured value.
-- `maxPrefixBlocksToMatch` (int, optional, default: `2048`): Maximum number of prefix blocks hashed and matched per request. Not auto-tuned. `0` disables matching (zero blocks hashed).
+- `maxPrefixBlocksToMatch` (int, optional, default: `2048`): Maximum number of prefix blocks hashed and matched per request. Not auto-tuned. Applies only when `maxPrefixTokensToMatch` is `0`.
 - `maxPrefixTokensToMatch` (int, optional, default: `131072`): Cap expressed in tokens instead of blocks (`maxBlocks = maxPrefixTokensToMatch / blockSizeTokens`). Not auto-tuned. Takes precedence over `maxPrefixBlocksToMatch` when set (> 0); set to `0` to fall back to the block-based cap. The `131072` default (128K, the context window of large production models such as gpt-oss 120b) is a reasonable upper bound that covers the long-prompt use cases seen in production.
+
+Setting both caps to `0` matches the whole prompt. Prompt length is bounded by the model server's context window, so this caps matching at one context window without needing to name a token count. It raises EPP CPU on long prompts (see [operations](../../../../../../../docs/operations.md)); to turn prefix matching off, remove this producer from the configuration rather than zeroing the caps.
 - `lruCapacityPerServer` (int, optional, default: `31250`): Per-pod LRU index capacity. Used when `autoTune` is false or endpoint metrics are unavailable; ignored when auto-tuned from metrics.
 - `blockSize` (int, optional): Deprecated — character-based block size. Use `blockSizeTokens` instead.
 

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/plugin.go

@@ -227,11 +227,7 @@ func (p *dataProducer) PluginState() *plugin.PluginState {
 // Produce is called by the director before scheduling requests.
 func (p *dataProducer) Produce(ctx context.Context, request *fwksched.InferenceRequest, pods []fwksched.Endpoint) error {
 	blockSize := p.GetBlockSize(pods)
-	maxBlocks := p.config.MaxPrefixBlocksToMatch
-	if p.config.MaxPrefixTokensToMatch > 0 && blockSize > 0 {
-		maxBlocks = p.config.MaxPrefixTokensToMatch / blockSize
-	}
-	perPromptHashes := prefixhash.GetBlockHashes(ctx, request, blockSize, maxBlocks)
+	perPromptHashes := prefixhash.GetBlockHashes(ctx, request, blockSize, p.resolveMaxBlocks(blockSize))
 
 	prefixCacheServers := make(map[ServerID]int)
 	totalBlocks := 0
@@ -359,6 +355,24 @@ func (p *dataProducer) GetBlockSize(endpoints []fwksched.Endpoint) int {
 	return blockSize
 }
 
+// resolveMaxBlocks returns the per-request cap on hashed prefix blocks.
+//
+// MaxPrefixTokensToMatch wins when set, converting tokens to blocks at the
+// effective block size. Zeroing both caps means unlimited: prompt length is
+// already bounded by the model server's context window, so the whole prompt is
+// at most one context window of tokens. A token cap smaller than the block size
+// still resolves to 0 and hashes nothing; that is a misconfiguration, not a
+// request for unlimited matching.
+func (p *dataProducer) resolveMaxBlocks(blockSize int) int {
+	if p.config.MaxPrefixTokensToMatch > 0 && blockSize > 0 {
+		return p.config.MaxPrefixTokensToMatch / blockSize
+	}
+	if p.config.MaxPrefixBlocksToMatch == 0 {
+		return unlimitedPrefixBlocks
+	}
+	return p.config.MaxPrefixBlocksToMatch
+}
+
 // ApproxPrefixCacheFactory is the factory function for the prefix cache data producer plugin.
 func ApproxPrefixCacheFactory(name string, rawParameters *json.Decoder, handle plugin.Handle) (plugin.Plugin, error) {
 	parameters := defaultConfig

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/plugin_test.go

@@ -443,6 +443,40 @@ func TestMaxPrefixTokensToMatch(t *testing.T) {
 	assert.Equal(t, 3, len(state2.PerPromptHashes[0]), "should fall back to MaxPrefixBlocksToMatch when MaxPrefixTokensToMatch is 0")
 }
 
+// TestMaxPrefixBothCapsZeroMatchesEverything verifies that zeroing both caps
+// hashes the whole prompt rather than nothing. The prompt length is already
+// bounded by the model server's context window, so an absent cap is an implicit
+// max_model_len cap.
+func TestMaxPrefixBothCapsZeroMatchesEverything(t *testing.T) {
+	disableMinBlockSizeClamp(t)
+	cfg := config{
+		BlockSizeTokens:        1,
+		MaxPrefixTokensToMatch: 0,
+		MaxPrefixBlocksToMatch: 0,
+		LRUCapacityPerServer:   defaultLRUCapacityPerServer,
+	}
+	p, err := newDataProducer(context.Background(), ApproxPrefixCachePluginType, cfg, testHandle())
+	assert.NoError(t, err)
+
+	endpoint := fwksched.NewEndpoint(
+		&fwkdl.EndpointMetadata{NamespacedName: k8stypes.NamespacedName{Name: "pod1"}},
+		fwkdl.NewMetrics(), fwkdl.NewAttributes(),
+	)
+
+	req := &fwksched.InferenceRequest{
+		RequestID:   uuid.NewString(),
+		TargetModel: "test-model",
+		Body:        tokenizedBody([]uint32{1, 2, 3, 4}),
+	}
+
+	err = p.Produce(context.Background(), req, []fwksched.Endpoint{endpoint})
+	assert.NoError(t, err)
+
+	state, err := plugin.ReadPluginStateKey[*SchedulingContextState](p.PluginState(), req.RequestID, plugin.StateKey(ApproxPrefixCachePluginType))
+	assert.NoError(t, err)
+	assert.Equal(t, 4, len(state.PerPromptHashes[0]), "both caps at 0 should hash every block in the prompt")
+}
+
 // TestGetBlockSize_AutotuneClampsBelowMinimum verifies that when AutoTune is on and
 // the endpoint reports a small CacheBlockSize, GetBlockSize floors the result at
 // minBlockSizeTokens to bound EPP indexer memory. See issue #1158.

pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix/types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package approximateprefix
 
 import (
+	"math"
 	"time"
 
 	k8stypes "k8s.io/apimachinery/pkg/types"
@@ -99,7 +100,8 @@ const (
 	defaultBlockSizeTokens = 16
 
 	// defaultMaxPrefixBlocks is the fallback block cap, consulted only when
-	// MaxPrefixTokensToMatch is 0; the default token cap otherwise supersedes it.
+	// MaxPrefixTokensToMatch is 0 and MaxPrefixBlocksToMatch is non-zero; the
+	// default token cap otherwise supersedes it.
 	// Two long requests with the same prefix up to this limit will be indistinguishable.
 	// This parameter provides a trade-off between cache size, prefix matching speed and matching
 	// accuracy. Use a small value if most requests are short to reduce cache size and speed up the
@@ -112,6 +114,12 @@ const (
 	// over defaultMaxPrefixBlocks: maxBlocks = defaultMaxPrefixTokens / blockSizeTokens.
 	defaultMaxPrefixTokens = 131072
 
+	// unlimitedPrefixBlocks is the block cap used when both MaxPrefixTokensToMatch
+	// and MaxPrefixBlocksToMatch are 0. Prompt length is already bounded by the
+	// model server's context window, so an absent cap matches at most one context
+	// window of tokens.
+	unlimitedPrefixBlocks = math.MaxInt
+
 	// defaultLRUCapacityPerServer is the default capacity of the LRU indexer per server.
 	// The indexer is an approximation to the actual prefix LRU cache state on the model servers per server (pod).
 	// A small capacity ensures a high accuracy of cache hit on the model server, but it will
@@ -135,11 +143,12 @@ type config struct {
 	BlockSize int `json:"blockSize"`
 	// Deprecated: use MaxPrefixTokensToMatch, which caps prefix matching in tokens
 	// independent of BlockSizeTokens. MaxPrefixBlocksToMatch applies only when
-	// MaxPrefixTokensToMatch is 0.
+	// MaxPrefixTokensToMatch is 0. Setting both to 0 matches the whole prompt.
 	MaxPrefixBlocksToMatch int `json:"maxPrefixBlocksToMatch"`
 	// MaxPrefixTokensToMatch is the maximum number of prefix tokens to match.
 	// When set (> 0), it takes precedence over MaxPrefixBlocksToMatch by computing
-	// maxBlocks = MaxPrefixTokensToMatch / blockSizeTokens.
+	// maxBlocks = MaxPrefixTokensToMatch / blockSizeTokens. Setting both caps to 0
+	// matches the whole prompt.
 	MaxPrefixTokensToMatch int `json:"maxPrefixTokensToMatch"`
 	// Max capacity size of the LRU indexer in number of entries per server (pod).
 	LRUCapacityPerServer int `json:"lruCapacityPerServer"`