diff --git a/config/charts/README.md b/config/charts/README.md index c30ef6441a..87955bfed0 100644 --- a/config/charts/README.md +++ b/config/charts/README.md @@ -316,7 +316,7 @@ router: Runs a tokenizer sidecar that EPP queries to tokenize incoming requests, enabling precise, token-count-aware routing policies (e.g., precise prefix-cache matching). -The sidecar runs vLLM's `vllm launch render ` and exposes `/v1/completions/render` and `/v1/chat/completions/render` over loopback HTTP. Wire EPP to it via `router.epp.pluginsCustomConfig` with `type: token-producer` and `vllm:`. +The sidecar runs vLLM's `vllm launch render ` and exposes `/v1/completions/render` and `/v1/chat/completions/render` over loopback HTTP. Wire EPP to it via `router.epp.pluginsCustomConfig` with `type: token-producer`, `backend: vllm`, and `vllm:`. #### Tokenizer Sidecar Parameters @@ -420,8 +420,8 @@ spec: **Step 2 — point EPP at the socket.** Configure the tokenizer plugin via `router.epp.pluginsCustomConfig` (the chart writes this into the EPP config -mounted at `/config`). Select the deprecated UDS backend with -`udsTokenizerConfig`: +mounted at `/config`). Select the deprecated UDS backend with `backend: uds` +and point it at the socket with `udsTokenizerConfig`: ```yaml router: @@ -431,6 +431,7 @@ router: plugins: - type: token-producer parameters: + backend: uds modelName: "Qwen/Qwen3-32B" udsTokenizerConfig: socketFile: /tmp/tokenizer/tokenizer-uds.socket diff --git a/deploy/config/epp-mm-embeddings-cache-config.yaml b/deploy/config/epp-mm-embeddings-cache-config.yaml index 6b157f6235..b537e1ab9e 100644 --- a/deploy/config/epp-mm-embeddings-cache-config.yaml +++ b/deploy/config/epp-mm-embeddings-cache-config.yaml @@ -4,6 +4,7 @@ kind: EndpointPickerConfig plugins: - type: token-producer parameters: + backend: vllm modelName: hf-repo/model-name # set to the model used in the vLLM deployment vllm: url: http://localhost:8000 diff --git a/deploy/config/epp-precise-prefix-cache-config.yaml b/deploy/config/epp-precise-prefix-cache-config.yaml index a2df8ee182..5f070043d1 100644 --- a/deploy/config/epp-precise-prefix-cache-config.yaml +++ b/deploy/config/epp-precise-prefix-cache-config.yaml @@ -5,6 +5,7 @@ kind: EndpointPickerConfig plugins: - type: token-producer parameters: + backend: vllm modelName: hf-repo/model-name # set to the model used in the vLLM deployment vllm: url: http://localhost:8000 diff --git a/deploy/config/sim-epp-kvcache-config.yaml b/deploy/config/sim-epp-kvcache-config.yaml index 546d47bc85..ba72a1413f 100644 --- a/deploy/config/sim-epp-kvcache-config.yaml +++ b/deploy/config/sim-epp-kvcache-config.yaml @@ -5,6 +5,7 @@ kind: EndpointPickerConfig plugins: - type: token-producer parameters: + backend: vllm modelName: TinyLlama/TinyLlama-1.1B-Chat-v1.0 # replace to use a different model vllm: url: http://localhost:8000 diff --git a/deploy/config/sim-epp-tokenizer-vllm-http-config.yaml b/deploy/config/sim-epp-tokenizer-vllm-http-config.yaml index ab46577d1a..5fce2309dc 100644 --- a/deploy/config/sim-epp-tokenizer-vllm-http-config.yaml +++ b/deploy/config/sim-epp-tokenizer-vllm-http-config.yaml @@ -7,6 +7,7 @@ kind: EndpointPickerConfig plugins: - type: token-producer parameters: + backend: vllm modelName: "${MODEL_NAME}" vllm: url: "http://localhost:8000" diff --git a/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/README.md b/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/README.md index 331c841e1b..85b143cd2b 100644 --- a/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/README.md +++ b/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/README.md @@ -21,6 +21,8 @@ upstream list shape, sorted by placeholder offset. Backend selection: +- Set **`backend: estimate|vllm|uds`** to select a backend explicitly. + When omitted, the plugin keeps the legacy inference rules below. - **`estimate`** (default): tokenizer-free byte-packing — no model, no service. Selected when no backend is set, and auto-created by the framework for any config whose plugins consume `TokenizedPrompt` (prefix cache, context-length, @@ -48,7 +50,8 @@ Backend selection: | Parameter | Default | Description | | ---------------- | ----------------------- | ----------------------------------------------------------------- | -| `modelName` | – (required for `vllm`) | Model whose tokenizer should be loaded / sent in render requests. | +| `backend` | inferred | Explicit backend selector: `estimate`, `vllm`, or `uds`. | +| `modelName` | – (required for `vllm`/`uds`) | Model whose tokenizer should be loaded / sent in render requests. | | `vllm.url` | `http://localhost:8000` | Base URL of the vLLM render endpoint (no trailing slash). | | `vllm.timeout` | `5s` | Per-request timeout for text-only requests. | | `vllm.mmTimeout` | `30s` | Per-request timeout for multimodal requests. | @@ -93,6 +96,7 @@ Plugin config — sidecar (loopback): ```yaml - type: token-producer parameters: + backend: vllm modelName: "${MODEL_NAME}" vllm: url: "http://localhost:8000" # optional; this is the default @@ -103,6 +107,7 @@ Plugin config — dedicated render Service: ```yaml - type: token-producer parameters: + backend: vllm modelName: "${MODEL_NAME}" vllm: url: "http://vllm-render.default.svc.cluster.local:8000" @@ -122,6 +127,7 @@ Before: ```yaml - type: token-producer parameters: + backend: uds modelName: "${MODEL_NAME}" udsTokenizerConfig: socketFile: /tmp/tokenizer/tokenizer-uds.socket @@ -132,6 +138,7 @@ After: ```yaml - type: token-producer parameters: + backend: vllm modelName: "${MODEL_NAME}" vllm: url: "http://localhost:8000" # or a shared render Service diff --git a/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer.go b/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer.go index 0a40564736..a55552c01f 100644 --- a/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer.go +++ b/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer.go @@ -58,13 +58,20 @@ const ( var TokenizedPromptDataKey = plugin.NewDataKey(tokenizedPromptKeyID, PluginType) +type tokenizerBackend string + +const ( + tokenizerBackendEstimate tokenizerBackend = "estimate" + tokenizerBackendVLLM tokenizerBackend = "vllm" + tokenizerBackendUDS tokenizerBackend = "uds" +) + // tokenizerPluginConfig holds the configuration for the tokenizer plugin. -// -// Backend selection: `vllm` or `modelName` selects the vLLM HTTP /render -// backend; `udsTokenizerConfig` selects the deprecated gRPC-over-UDS backend; -// `estimate` selects the tokenizer-free byte-packing backend, which is also the -// zero-config default when no backend is set. type tokenizerPluginConfig struct { + // Backend explicitly selects "estimate", "vllm", or "uds". When omitted, + // the plugin preserves legacy behavior by inferring the backend from + // existing config fields. + Backend tokenizerBackend `json:"backend,omitempty"` // TokenizerConfig configures the deprecated gRPC-over-UDS backend. // // Deprecated: the UDS tokenizer backend is deprecated and will be removed @@ -72,8 +79,9 @@ type tokenizerPluginConfig struct { TokenizerConfig tokenization.UdsTokenizerConfig `json:"udsTokenizerConfig,omitempty"` // VLLM configures the vLLM /render backend. VLLM *vllmConfig `json:"vllm,omitempty"` - // Estimate selects the tokenizer-free byte-packing backend; mutually - // exclusive with 'vllm'/'udsTokenizerConfig' and needs no 'modelName'. + // Estimate configures the tokenizer-free byte-packing backend. When + // backend is omitted, it selects the estimate backend and needs no + // 'modelName'. Estimate *estimateConfig `json:"estimate,omitempty"` // ModelName is the name of the model whose tokenizer should be loaded. ModelName string `json:"modelName"` @@ -128,29 +136,67 @@ func PluginFactory(name string, rawParameters *json.Decoder, handle plugin.Handl } } - estimate := config.Estimate != nil - uds := config.TokenizerConfig.IsEnabled() - vllm := config.VLLM != nil || config.ModelName != "" - if (estimate && (uds || vllm)) || (uds && vllm) { - return nil, fmt.Errorf("invalid configuration for '%s' plugin: only one of 'estimate', 'vllm', or 'udsTokenizerConfig' may be set", PluginType) + p, err := NewPlugin(handle.Context(), name, &config) + if err != nil { + return nil, err } - // modelName is required only by the real-tokenizer backends; the zero-config - // path selects the estimate backend, which needs none. - if (uds || vllm) && config.ModelName == "" { - return nil, fmt.Errorf("invalid configuration for '%s' plugin: 'modelName' must be specified", PluginType) + + return p, nil +} + +func (config *tokenizerPluginConfig) validateAndSelectBackend() (tokenizerBackend, error) { + selected, err := config.selectBackend() + if err != nil { + return "", err } - if config.Estimate != nil && config.Estimate.Image != nil { - if m := config.Estimate.Image.Mode; m != "" && m != imageModeDynamic && m != imageModeStatic { - return nil, fmt.Errorf("invalid configuration for '%s' plugin: estimate.image.mode must be %q or %q", PluginType, imageModeDynamic, imageModeStatic) + + switch selected { + case tokenizerBackendVLLM: + if config.ModelName == "" { + return "", fmt.Errorf("invalid configuration for '%s' plugin: 'modelName' must be specified", PluginType) + } + case tokenizerBackendUDS: + if config.ModelName == "" { + return "", fmt.Errorf("invalid configuration for '%s' plugin: 'modelName' must be specified", PluginType) + } + if !config.TokenizerConfig.IsEnabled() { + return "", fmt.Errorf("invalid configuration for '%s' plugin: 'udsTokenizerConfig' must be specified when backend is %q", PluginType, tokenizerBackendUDS) + } + case tokenizerBackendEstimate: + if config.Estimate != nil && config.Estimate.Image != nil { + if m := config.Estimate.Image.Mode; m != "" && m != imageModeDynamic && m != imageModeStatic { + return "", fmt.Errorf("invalid configuration for '%s' plugin: estimate.image.mode must be %q or %q", PluginType, imageModeDynamic, imageModeStatic) + } } } - p, err := NewPlugin(handle.Context(), name, &config) - if err != nil { - return nil, err + return selected, nil +} + +func (config *tokenizerPluginConfig) selectBackend() (tokenizerBackend, error) { + if config.Backend != "" { + switch config.Backend { + case tokenizerBackendEstimate, tokenizerBackendVLLM, tokenizerBackendUDS: + return config.Backend, nil + default: + return "", fmt.Errorf("invalid configuration for '%s' plugin: backend must be one of %q, %q, or %q", PluginType, tokenizerBackendEstimate, tokenizerBackendVLLM, tokenizerBackendUDS) + } } - return p, nil + estimate := config.Estimate != nil + uds := config.TokenizerConfig.IsEnabled() + vllm := config.VLLM != nil || config.ModelName != "" + if (estimate && (uds || vllm)) || (uds && vllm) { + return "", fmt.Errorf("invalid configuration for '%s' plugin: only one of 'estimate', 'vllm', or 'udsTokenizerConfig' may be set", PluginType) + } + switch { + case uds: + return tokenizerBackendUDS, nil + case vllm: + return tokenizerBackendVLLM, nil + default: + return tokenizerBackendEstimate, nil + } } // LegacyPluginFactory wraps PluginFactory for the deprecated `tokenizer` type @@ -166,13 +212,20 @@ func LegacyPluginFactory(name string, rawParameters *json.Decoder, handle plugin return PluginFactory(name, rawParameters, handle) } -// NewPlugin constructs the configured backend: udsTokenizerConfig (deprecated), -// vllm /render (selected by 'vllm' or 'modelName'), or estimate byte-packing -// (the default when no backend is set). +// NewPlugin constructs the configured backend. The explicit backend selector +// wins when set; otherwise legacy config inference is used. func NewPlugin(ctx context.Context, name string, config *tokenizerPluginConfig) (*Plugin, error) { + if config == nil { + config = &tokenizerPluginConfig{} + } + selected, err := config.validateAndSelectBackend() + if err != nil { + return nil, err + } + var backend tokenInputProducer - switch { - case config.TokenizerConfig.IsEnabled(): + switch selected { + case tokenizerBackendUDS: log.FromContext(ctx).Info( "DEPRECATION: the 'udsTokenizerConfig' parameter is deprecated and will be removed in a future release; set the 'vllm' parameter instead (see plugin README)", "pluginType", PluginType, @@ -182,7 +235,7 @@ func NewPlugin(ctx context.Context, name string, config *tokenizerPluginConfig) return nil, fmt.Errorf("failed to initialize UDS tokenizer for '%s' plugin - %w", PluginType, err) } backend = renderBackend{tk: uds} - case config.VLLM != nil || config.ModelName != "": + case tokenizerBackendVLLM: cfg := config.VLLM if cfg == nil { cfg = &vllmConfig{} @@ -192,7 +245,7 @@ func NewPlugin(ctx context.Context, name string, config *tokenizerPluginConfig) return nil, fmt.Errorf("failed to initialize vLLM HTTP renderer for '%s' plugin - %w", PluginType, err) } backend = renderBackend{tk: renderer} - default: + case tokenizerBackendEstimate: backend = estimateBackend{img: newImageEstimator(config.Estimate)} } diff --git a/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer_test.go b/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer_test.go index afb2f6f54c..558c822d5d 100644 --- a/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer_test.go +++ b/pkg/epp/framework/plugins/requestcontrol/dataproducer/tokenizer/tokenizer_test.go @@ -68,6 +68,11 @@ func TestProduceTimeout(t *testing.T) { require.NoError(t, err) assert.Equal(t, 45*time.Second, vp2.ProduceTimeout()) + // Explicit backend selection can use vLLM's default config block. + vp3, err := NewPlugin(ctx, "tok", &tokenizerPluginConfig{Backend: tokenizerBackendVLLM, ModelName: "m"}) + require.NoError(t, err) + assert.Equal(t, defaultHTTPRenderMMTimeout, vp3.ProduceTimeout()) + // Estimate backend declares none, so the director keeps its default. ep, err := NewPlugin(ctx, "tok", &tokenizerPluginConfig{Estimate: &estimateConfig{}}) require.NoError(t, err) @@ -108,6 +113,29 @@ func TestPluginFactory_Validation(t *testing.T) { params: `{"estimate":{"image":{"mode":"static","static":{"staticToken":8}}}}`, expectErr: false, }, + { + name: "explicit estimate backend overrides legacy vllm inference", + params: `{"backend":"estimate","modelName":"m","vllm":{}}`, + expectErr: false, + }, + { + name: "explicit vllm backend requires modelName", + params: `{"backend":"vllm","vllm":{}}`, + expectErr: true, + errContain: "'modelName' must be specified", + }, + { + name: "explicit uds backend requires tokenizer config", + params: `{"backend":"uds","modelName":"m"}`, + expectErr: true, + errContain: "'udsTokenizerConfig' must be specified", + }, + { + name: "invalid backend selector", + params: `{"backend":"unknown"}`, + expectErr: true, + errContain: "backend must be one of", + }, { name: "invalid estimate image mode", params: `{"estimate":{"image":{"mode":"bogus"}}}`, diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 6760a66242..31298f653b 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -185,6 +185,7 @@ kind: EndpointPickerConfig plugins: - type: token-producer parameters: + backend: vllm modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: url: http://localhost:8000 @@ -217,6 +218,7 @@ kind: EndpointPickerConfig plugins: - type: token-producer parameters: + backend: vllm modelName: Qwen/Qwen2.5-1.5B-Instruct vllm: url: http://localhost:8000