@@ -22,12 +22,14 @@ import (
2222 "encoding/json"
2323 "errors"
2424 "strconv"
25+ "strings"
2526
2627 "github.com/cespare/xxhash/v2"
2728 "sigs.k8s.io/controller-runtime/pkg/log"
2829
2930 logutil "github.com/llm-d/llm-d-router/pkg/common/observability/logging"
3031 fwkrh "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/requesthandling"
32+ "github.com/llm-d/llm-d-router/pkg/epp/metadata"
3133)
3234
3335// bytesPerToken matches the scorer's averageCharactersPerToken, so a block of N
@@ -41,6 +43,69 @@ const blockTypeText = "text"
4143// so pairing this backend with the engine-correlated scorer yields misses, not bad routes.
4244type estimateBackend struct {
4345 img imageEstimator
46+ vid videoEstimator
47+ }
48+
49+ // parseMMMetadataHeaders reads the x-llm-d-* request headers into an mmMetadata.
50+ // Only video is populated today; image and audio parsing slot in here later.
51+ func parseMMMetadataHeaders (headers map [string ]string ) mmMetadata {
52+ return mmMetadata {video : parseVideoMetadataHeaders (headers )}
53+ }
54+
55+ // mmMetadataCtxKey keys the request-scoped mmMetadata on the context, carrying it
56+ // from Plugin.Produce (which holds request.Headers) to the estimate backend
57+ // without widening the shared tokenInputProducer.produce signature.
58+ type mmMetadataCtxKey struct {}
59+
60+ // withMMMetadata returns ctx carrying meta.
61+ func withMMMetadata (ctx context.Context , meta mmMetadata ) context.Context {
62+ return context .WithValue (ctx , mmMetadataCtxKey {}, meta )
63+ }
64+
65+ // mmMetadataFromContext returns the mmMetadata on ctx, or the zero value.
66+ func mmMetadataFromContext (ctx context.Context ) mmMetadata {
67+ meta , _ := ctx .Value (mmMetadataCtxKey {}).(mmMetadata )
68+ return meta
69+ }
70+
71+ // parseVideoMetadataHeaders reads the x-llm-d-video- request headers into a
72+ // videoMetadata, using metadata.GetLowerCaseHeaderValue so aliases resolve the
73+ // same way as the SLO headers. Missing or malformed values leave their field zero
74+ // so the estimator falls back per field to config and then defaults.
75+ func parseVideoMetadataHeaders (headers map [string ]string ) videoMetadata {
76+ var meta videoMetadata
77+ if s , ok := metadata .GetLowerCaseHeaderValue (headers , metadata .VideoFPSHeaderKey ); ok {
78+ if v , err := strconv .ParseFloat (s , 64 ); err == nil && v > 0 {
79+ meta .fps = v
80+ }
81+ }
82+ if s , ok := metadata .GetLowerCaseHeaderValue (headers , metadata .VideoDurationHeaderKey ); ok {
83+ if v , err := strconv .ParseFloat (s , 64 ); err == nil && v > 0 {
84+ meta .duration = v
85+ }
86+ }
87+ if s , ok := metadata .GetLowerCaseHeaderValue (headers , metadata .VideoResolutionHeaderKey ); ok {
88+ meta .width , meta .height = parseResolution (s )
89+ }
90+ return meta
91+ }
92+
93+ // parseResolution splits a "WIDTHxHEIGHT" value into pixel dimensions, returning
94+ // zeros when the value is empty or malformed.
95+ func parseResolution (s string ) (width , height int ) {
96+ i := strings .IndexAny (s , "xX" )
97+ if i <= 0 {
98+ return 0 , 0
99+ }
100+ w , err := strconv .Atoi (strings .TrimSpace (s [:i ]))
101+ if err != nil || w <= 0 {
102+ return 0 , 0
103+ }
104+ h , err := strconv .Atoi (strings .TrimSpace (s [i + 1 :]))
105+ if err != nil || h <= 0 {
106+ return 0 , 0
107+ }
108+ return w , h
44109}
45110
46111func (b estimateBackend ) produce (ctx context.Context , body * fwkrh.InferenceRequestBody ) (* fwkrh.TokenizedPrompt , error ) {
@@ -62,7 +127,7 @@ func (b estimateBackend) produce(ctx context.Context, body *fwkrh.InferenceReque
62127 // Chat and Anthropic messages fold multimodal placeholders into the stream
63128 // and report them as features.
64129 if body .ChatCompletions != nil {
65- raw , features := b .chatCompletionsBytes (body .ChatCompletions )
130+ raw , features := b .chatCompletionsBytes (body .ChatCompletions , mmMetadataFromContext ( ctx ) )
66131 return & fwkrh.TokenizedPrompt {PerPromptTokens : [][]uint32 {packBytes (raw )}, MultiModalFeatures : features }, nil
67132 }
68133 if body .Messages != nil {
@@ -128,7 +193,7 @@ func estimateBytes(body *fwkrh.InferenceRequestBody) ([]byte, error) {
128193// multimodal assets in on aligned boundaries. Each asset occupies N placeholder
129194// pseudo-tokens (its content hash repeated N times) so it carries weight in the
130195// stream, and is reported as a MultiModalFeature with its token offset and span.
131- func (b estimateBackend ) chatCompletionsBytes (chat * fwkrh.ChatCompletionsRequest ) ([]byte , []fwkrh.MultiModalFeature ) {
196+ func (b estimateBackend ) chatCompletionsBytes (chat * fwkrh.ChatCompletionsRequest , meta mmMetadata ) ([]byte , []fwkrh.MultiModalFeature ) {
132197 var out []byte
133198 var features []fwkrh.MultiModalFeature
134199 if len (chat .Tools ) > 0 {
@@ -137,14 +202,14 @@ func (b estimateBackend) chatCompletionsBytes(chat *fwkrh.ChatCompletionsRequest
137202 }
138203 }
139204 for _ , msg := range chat .Messages {
140- out , features = b .appendChatMessage (out , features , msg )
205+ out , features = b .appendChatMessage (out , features , msg , meta )
141206 }
142207 return out , features
143208}
144209
145210// appendChatMessage flattens a single chat-completions message into the byte
146211// stream, recording multimodal placeholders on aligned boundaries.
147- func (b estimateBackend ) appendChatMessage (out []byte , features []fwkrh.MultiModalFeature , msg fwkrh.Message ) ([]byte , []fwkrh.MultiModalFeature ) {
212+ func (b estimateBackend ) appendChatMessage (out []byte , features []fwkrh.MultiModalFeature , msg fwkrh.Message , meta mmMetadata ) ([]byte , []fwkrh.MultiModalFeature ) {
148213 if msg .Role != "" {
149214 out = append (out , []byte (msg .Role )... )
150215 }
@@ -159,7 +224,7 @@ func (b estimateBackend) appendChatMessage(out []byte, features []fwkrh.MultiMod
159224 case "image_url" :
160225 out , features = appendMMAsset (out , features , fwkrh .ModalityImage , block .ImageURL .URL , b .img .placeholderCount (block .ImageURL .URL ))
161226 case "video_url" :
162- out , features = appendMMAsset (out , features , fwkrh .ModalityVideo , block .VideoURL .URL , assetPlaceholderCount ( len ( block . VideoURL . URL ) ))
227+ out , features = appendMMAsset (out , features , fwkrh .ModalityVideo , block .VideoURL .URL , b . vid . placeholderCount ( meta . video ))
163228 case "input_audio" , "audio_url" :
164229 data := block .InputAudio .Data + block .InputAudio .Format
165230 out , features = appendMMAsset (out , features , fwkrh .ModalityAudio , data , assetPlaceholderCount (len (data )))
0 commit comments