Merge branch 'main' into epd-sidecar

Author: roytman

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jellydator/ttlcache/v3 v3.4.0
-	github.com/llm-d/llm-d-kv-cache v0.5.1-0.20260218092800-cbfd0b78c70b
+	github.com/llm-d/llm-d-kv-cache v0.5.1-rc2
 	github.com/onsi/ginkgo/v2 v2.28.1
 	github.com/onsi/gomega v1.39.1
 	github.com/openai/openai-go v1.12.0

go.sum

@@ -211,6 +211,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/llm-d/llm-d-kv-cache v0.5.1-0.20260218092800-cbfd0b78c70b h1:246D1wR0JnYe6rz7E5RRmyHqM9L8a1S1qnXgnmo4GEg=
 github.com/llm-d/llm-d-kv-cache v0.5.1-0.20260218092800-cbfd0b78c70b/go.mod h1:/Llnuds5IN8xyc4y2ikyXeLR6XDdU5g2Ff2zwVB12Wc=
+github.com/llm-d/llm-d-kv-cache v0.5.1-rc2 h1:WQ0zd3nkgjQ5vxYHAExzTI2urdUim1I275Hn8dukcdA=
+github.com/llm-d/llm-d-kv-cache v0.5.1-rc2/go.mod h1:y78v3jkKXoSD5MKFnkuyNUZ0XliGfW6CwZjD4UClwOE=
 github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
 github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
 github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=

pkg/plugins/scorer/precise_prefix_cache_test.go

@@ -234,20 +234,13 @@ func TestPrefixCacheTracking_Score(t *testing.T) {
 				})
 				require.NoError(t, err)
 
-				// render the chat template
-				renderReq := &preprocessing.ApplyChatTemplateRequest{
+				// render the chat template and tokenize
+				renderReq := &preprocessing.RenderChatRequest{
 					Key:          tokenizerCacheKey,
-					Conversation: [][]preprocessing.Conversation{conversations},
+					Conversation: conversations,
 					ChatTemplate: req.ChatCompletions.ChatTemplate,
 				}
-				rendered, err := processor.ApplyChatTemplate(t.Context(), renderReq)
-				require.NoError(t, err)
-
-				// tokenize rendered prompt
-				testTokenizer, err := tokenization.NewCachedLocalTokenizer(t.Context(), model, localTokenizerConfig)
-				require.NoError(t, err)
-
-				tokens, _, err := testTokenizer.Encode(rendered, model, false)
+				tokens, _, err := processor.RenderChat(t.Context(), renderReq)
 				require.NoError(t, err)
 
 				tokenProcessor := kvblock.NewChunkedTokenDatabase(kvblock.DefaultTokenProcessorConfig())
@@ -539,10 +532,9 @@ func TestPrefixCacheTracking_Score(t *testing.T) {
 
 			kvcacheConfig, err := kvcache.NewDefaultConfig()
 			kvcacheConfig.TokenizersPoolConfig = &tokenization.Config{
-				ModelName:             "test-model",
-				WorkersCount:          1,
-				MinPrefixOverlapRatio: 0.8,
-				LocalTokenizerConfig:  &localTokenizerConfig,
+				ModelName:            "test-model",
+				WorkersCount:         1,
+				LocalTokenizerConfig: &localTokenizerConfig,
 			}
 			require.NoError(t, err)