From f2b811ec81f082b9bcd84a03ac67d423d3143806 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Fri, 17 Jul 2026 20:54:39 -0700 Subject: [PATCH 1/7] docs/configs: replace deprecated maxPrefixBlocksToMatch and blockSize with maxPrefixTokensToMatch and blockSizeTokens Update sample deployment configurations, test fixtures, and documentation to use non-deprecated parameter names. Signed-off-by: Cong Liu --- deploy/config/pd-epp-config.yaml | 2 +- deploy/config/sim-epp-config.yaml | 2 +- deploy/config/sim-epp-no-hit-lru.yaml | 4 ++-- deploy/config/sim-pd-epp-config.yaml | 2 +- docs/architecture.md | 4 ++-- docs/disaggregation.md | 4 ++-- docs/operations.md | 2 +- test/e2e/configs_test.go | 10 +++++----- test/integration/epp/e2e_config_smoke_test.go | 10 +++++----- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/deploy/config/pd-epp-config.yaml b/deploy/config/pd-epp-config.yaml index cf17cdf68d..f173f68070 100644 --- a/deploy/config/pd-epp-config.yaml +++ b/deploy/config/pd-epp-config.yaml @@ -4,7 +4,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/deploy/config/sim-epp-config.yaml b/deploy/config/sim-epp-config.yaml index 0af90d0f0f..08641cdb28 100644 --- a/deploy/config/sim-epp-config.yaml +++ b/deploy/config/sim-epp-config.yaml @@ -5,7 +5,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: decode-filter diff --git a/deploy/config/sim-epp-no-hit-lru.yaml b/deploy/config/sim-epp-no-hit-lru.yaml index 7cfc178add..93adfb4578 100644 --- a/deploy/config/sim-epp-no-hit-lru.yaml +++ b/deploy/config/sim-epp-no-hit-lru.yaml @@ -7,9 +7,9 @@ plugins: parameters: indexerConfig: tokenProcessorConfig: - blockSize: 5 + blockSizeTokens: 5 kvBlockIndexConfig: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 - type: prefix-cache-scorer parameters: prefixMatchInfoProducerName: precise-prefix-cache-producer diff --git a/deploy/config/sim-pd-epp-config.yaml b/deploy/config/sim-pd-epp-config.yaml index 771974b2cc..052d0f471d 100644 --- a/deploy/config/sim-pd-epp-config.yaml +++ b/deploy/config/sim-pd-epp-config.yaml @@ -7,7 +7,7 @@ plugins: parameters: blockSizeTokens: 16 autoTune: false - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/docs/architecture.md b/docs/architecture.md index b4f86faa16..be86d9a30b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -152,9 +152,9 @@ plugins: parameters: indexerConfig: tokenProcessorConfig: - blockSize: 5 + blockSizeTokens: 5 kvBlockIndexConfig: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 - type: prefix-cache-scorer parameters: prefixMatchInfoProducerName: precise-prefix-cache-producer diff --git a/docs/disaggregation.md b/docs/disaggregation.md index 0a38201c72..af308d4874 100644 --- a/docs/disaggregation.md +++ b/docs/disaggregation.md @@ -279,7 +279,7 @@ plugins: parameters: autoTune: false blockSizeTokens: 5 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 31250 - type: max-score-picker - type: prefix-based-pd-decider @@ -342,7 +342,7 @@ plugins: parameters: autoTune: false blockSizeTokens: 5 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 31250 - type: max-score-picker - type: always-disagg-multimodal-decider diff --git a/docs/operations.md b/docs/operations.md index f6df65b080..a21f09cdd7 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -13,7 +13,7 @@ The EPP acts as the routing intelligence engine. Its resource usage scales prima #### CPU Allocation - **Rule of Thumb**: Allocate **0.5 to 1.0 CPU cores per request/second** of expected throughput for large agentic workloads (approximately 100k input / 1k output tokens). - **Scaling Behavior**: CPU utilization scales linearly with the request rate, and increases with both the input prompt size and output token length. -- **Prefix Matching Overhead**: Increasing the `maxPrefixBlocksToMatch` parameter increases EPP CPU utilization. At lower throughputs, a large prefix block limit (such as 6250 blocks) can increase EPP CPU utilization by over 100% compared to a small limit (256 blocks) due to the overhead of searching and matching blocks. +- **Prefix Matching Overhead**: Increasing the `maxPrefixBlocksToMatch` parameter (deprecated; use `maxPrefixTokensToMatch` instead) increases EPP CPU utilization. At lower throughputs, a large prefix block limit (such as 6250 blocks) can increase EPP CPU utilization by over 100% compared to a small limit (256 blocks) due to the overhead of searching and matching blocks. - **Idle CPU Scaling**: Idle CPU usage of the EPP container scales with the number of model-serving pods in the cluster due to continuous metric scraping. For example, in a cluster with 100 model-serving pods, the idle CPU usage of the EPP container grows to approximately **7.5 cores**. #### Memory Allocation diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 0afbb4abba..a3e5766cb0 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -8,7 +8,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -32,7 +32,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -92,7 +92,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -130,7 +130,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -164,7 +164,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter diff --git a/test/integration/epp/e2e_config_smoke_test.go b/test/integration/epp/e2e_config_smoke_test.go index 33aec8a68b..1a239fe74a 100644 --- a/test/integration/epp/e2e_config_smoke_test.go +++ b/test/integration/epp/e2e_config_smoke_test.go @@ -26,7 +26,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -47,7 +47,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -79,7 +79,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -115,7 +115,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -150,7 +150,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 131072 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter From e068babb6e59cde83464bb7139660f18787348a6 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Fri, 17 Jul 2026 21:08:58 -0700 Subject: [PATCH 2/7] docs/configs: calculate maxPrefixTokensToMatch from maxPrefixBlocksToMatch * blockSizeTokens Scale maxPrefixTokensToMatch accurately based on the block count and token block size. Signed-off-by: Cong Liu --- deploy/config/pd-epp-config.yaml | 2 +- deploy/config/sim-epp-config.yaml | 2 +- deploy/config/sim-epp-no-hit-lru.yaml | 2 +- deploy/config/sim-pd-epp-config.yaml | 2 +- docs/architecture.md | 2 +- docs/disaggregation.md | 4 ++-- test/e2e/configs_test.go | 10 +++++----- test/integration/epp/e2e_config_smoke_test.go | 10 +++++----- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/deploy/config/pd-epp-config.yaml b/deploy/config/pd-epp-config.yaml index f173f68070..fd1f818326 100644 --- a/deploy/config/pd-epp-config.yaml +++ b/deploy/config/pd-epp-config.yaml @@ -4,7 +4,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/deploy/config/sim-epp-config.yaml b/deploy/config/sim-epp-config.yaml index 08641cdb28..bdc73843e7 100644 --- a/deploy/config/sim-epp-config.yaml +++ b/deploy/config/sim-epp-config.yaml @@ -5,7 +5,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: decode-filter diff --git a/deploy/config/sim-epp-no-hit-lru.yaml b/deploy/config/sim-epp-no-hit-lru.yaml index 93adfb4578..7058757bed 100644 --- a/deploy/config/sim-epp-no-hit-lru.yaml +++ b/deploy/config/sim-epp-no-hit-lru.yaml @@ -9,7 +9,7 @@ plugins: tokenProcessorConfig: blockSizeTokens: 5 kvBlockIndexConfig: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 1280 - type: prefix-cache-scorer parameters: prefixMatchInfoProducerName: precise-prefix-cache-producer diff --git a/deploy/config/sim-pd-epp-config.yaml b/deploy/config/sim-pd-epp-config.yaml index 052d0f471d..52ecfe63ba 100644 --- a/deploy/config/sim-pd-epp-config.yaml +++ b/deploy/config/sim-pd-epp-config.yaml @@ -7,7 +7,7 @@ plugins: parameters: blockSizeTokens: 16 autoTune: false - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/docs/architecture.md b/docs/architecture.md index be86d9a30b..1c692b3947 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -154,7 +154,7 @@ plugins: tokenProcessorConfig: blockSizeTokens: 5 kvBlockIndexConfig: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 1280 - type: prefix-cache-scorer parameters: prefixMatchInfoProducerName: precise-prefix-cache-producer diff --git a/docs/disaggregation.md b/docs/disaggregation.md index af308d4874..1c3fe3eb0a 100644 --- a/docs/disaggregation.md +++ b/docs/disaggregation.md @@ -279,7 +279,7 @@ plugins: parameters: autoTune: false blockSizeTokens: 5 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 1280 lruCapacityPerServer: 31250 - type: max-score-picker - type: prefix-based-pd-decider @@ -342,7 +342,7 @@ plugins: parameters: autoTune: false blockSizeTokens: 5 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 1280 lruCapacityPerServer: 31250 - type: max-score-picker - type: always-disagg-multimodal-decider diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index a3e5766cb0..282c1d98a1 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -8,7 +8,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -32,7 +32,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -92,7 +92,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -130,7 +130,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -164,7 +164,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter diff --git a/test/integration/epp/e2e_config_smoke_test.go b/test/integration/epp/e2e_config_smoke_test.go index 1a239fe74a..6b63d290e5 100644 --- a/test/integration/epp/e2e_config_smoke_test.go +++ b/test/integration/epp/e2e_config_smoke_test.go @@ -26,7 +26,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -47,7 +47,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -79,7 +79,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -115,7 +115,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -150,7 +150,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 131072 + maxPrefixTokensToMatch: 4096 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter From fcd691afd161277610a1136d4f2cf8669cbb6832 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Fri, 17 Jul 2026 21:09:27 -0700 Subject: [PATCH 3/7] docs/operations: replace deprecated field name directly in sentence with maxPrefixTokensToMatch Signed-off-by: Cong Liu --- docs/operations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations.md b/docs/operations.md index a21f09cdd7..6982f5dcd0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -13,7 +13,7 @@ The EPP acts as the routing intelligence engine. Its resource usage scales prima #### CPU Allocation - **Rule of Thumb**: Allocate **0.5 to 1.0 CPU cores per request/second** of expected throughput for large agentic workloads (approximately 100k input / 1k output tokens). - **Scaling Behavior**: CPU utilization scales linearly with the request rate, and increases with both the input prompt size and output token length. -- **Prefix Matching Overhead**: Increasing the `maxPrefixBlocksToMatch` parameter (deprecated; use `maxPrefixTokensToMatch` instead) increases EPP CPU utilization. At lower throughputs, a large prefix block limit (such as 6250 blocks) can increase EPP CPU utilization by over 100% compared to a small limit (256 blocks) due to the overhead of searching and matching blocks. +- **Prefix Matching Overhead**: Increasing the `maxPrefixTokensToMatch` parameter increases EPP CPU utilization. At lower throughputs, a large prefix block limit (such as 6250 blocks) can increase EPP CPU utilization by over 100% compared to a small limit (256 blocks) due to the overhead of searching and matching blocks. - **Idle CPU Scaling**: Idle CPU usage of the EPP container scales with the number of model-serving pods in the cluster due to continuous metric scraping. For example, in a cluster with 100 model-serving pods, the idle CPU usage of the EPP container grows to approximately **7.5 cores**. #### Memory Allocation From e1452848c78a7da45dae637783a7f4e677032de6 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Fri, 17 Jul 2026 21:10:15 -0700 Subject: [PATCH 4/7] docs/operations: replace remaining sentence reference of maxPrefixBlocksToMatch with maxPrefixTokensToMatch Signed-off-by: Cong Liu --- docs/operations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations.md b/docs/operations.md index 6982f5dcd0..27a8aa7e70 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -58,7 +58,7 @@ Configuration used: [#1287](https://github.com/llm-d/llm-d-router/issues/1287#is These were run against 0.9.0 EPP container image. #### Output Length and Prefix Matching Complexity -This table shows EPP peak resource usage at a constant request rate of 50 requests/second with a 100k input token workload, varying the output token length and the `maxPrefixBlocksToMatch` configuration. +This table shows EPP peak resource usage at a constant request rate of 50 requests/second with a 100k input token workload, varying the output token length and the `maxPrefixTokensToMatch` configuration. | Input Tokens | Output Tokens | maxPrefixBlocksToMatch | Peak CPU (Cores) | Peak Memory (GiB) | | :--- | :--- | :--- | :--- | :--- | From 75975d8282b3eb2f3a3bb284b548fb13661c8534 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Fri, 17 Jul 2026 21:11:17 -0700 Subject: [PATCH 5/7] docs/operations: rephrase CPU overhead sentence to state token caps and blockSizeTokens Signed-off-by: Cong Liu --- docs/operations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations.md b/docs/operations.md index 27a8aa7e70..695bc9625a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -13,7 +13,7 @@ The EPP acts as the routing intelligence engine. Its resource usage scales prima #### CPU Allocation - **Rule of Thumb**: Allocate **0.5 to 1.0 CPU cores per request/second** of expected throughput for large agentic workloads (approximately 100k input / 1k output tokens). - **Scaling Behavior**: CPU utilization scales linearly with the request rate, and increases with both the input prompt size and output token length. -- **Prefix Matching Overhead**: Increasing the `maxPrefixTokensToMatch` parameter increases EPP CPU utilization. At lower throughputs, a large prefix block limit (such as 6250 blocks) can increase EPP CPU utilization by over 100% compared to a small limit (256 blocks) due to the overhead of searching and matching blocks. +- **Prefix Matching Overhead**: Increasing the `maxPrefixTokensToMatch` parameter increases EPP CPU utilization. At lower throughputs, a large prefix limit (such as 100,000 tokens / 6,250 blocks with `blockSizeTokens: 16`) can increase EPP CPU utilization by over 100% compared to a small limit (4,096 tokens / 256 blocks) due to the overhead of searching and matching prefix blocks. - **Idle CPU Scaling**: Idle CPU usage of the EPP container scales with the number of model-serving pods in the cluster due to continuous metric scraping. For example, in a cluster with 100 model-serving pods, the idle CPU usage of the EPP container grows to approximately **7.5 cores**. #### Memory Allocation From c693058cbd7bd51135884312df86b5811288260c Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Fri, 17 Jul 2026 21:19:00 -0700 Subject: [PATCH 6/7] docs/operations: update performance benchmark tables to use maxPrefixTokensToMatch Signed-off-by: Cong Liu --- docs/operations.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 695bc9625a..24c3a76d57 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -47,12 +47,12 @@ The following tables present empirical benchmark results for EPP running with ll #### Throughput and Prefix Block Sizing This table shows peak CPU and memory utilization for EPP under a 100k token workload (95k system prompt, 5k question prompt, and 1k output tokens) when using approximate prefix caching across 100 model-serving pods. -| Configuration | Request Rate (Req/s) | maxPrefixBlocksToMatch | Peak CPU (Cores) | Peak Memory (GiB) | Scheduler P50 Latency (s) | +| Configuration | Request Rate (Req/s) | maxPrefixTokensToMatch | Peak CPU (Cores) | Peak Memory (GiB) | Scheduler P50 Latency (s) | | :--- | :--- | :--- | :--- | :--- | :--- | -| Small Prefix Match | 5.0 | 256 | 1.19 | 0.26 | 0.00010 | -| Large Prefix Match | 5.0 | 6250 | 3.82 | 0.65 | 0.00010 | -| Small Prefix Match | 98.7 | 256 | 35.17 | 2.46 | 0.00014 | -| Large Prefix Match | 98.8 | 6250 | 46.50 | 3.41 | 0.00020 | +| Small Prefix Match | 5.0 | 4096 | 1.19 | 0.26 | 0.00010 | +| Large Prefix Match | 5.0 | 100000 | 3.82 | 0.65 | 0.00010 | +| Small Prefix Match | 98.7 | 4096 | 35.17 | 2.46 | 0.00014 | +| Large Prefix Match | 98.8 | 100000 | 46.50 | 3.41 | 0.00020 | Configuration used: [#1287](https://github.com/llm-d/llm-d-router/issues/1287#issuecomment-4666058475). These were run against 0.9.0 EPP container image. @@ -60,14 +60,14 @@ These were run against 0.9.0 EPP container image. #### Output Length and Prefix Matching Complexity This table shows EPP peak resource usage at a constant request rate of 50 requests/second with a 100k input token workload, varying the output token length and the `maxPrefixTokensToMatch` configuration. -| Input Tokens | Output Tokens | maxPrefixBlocksToMatch | Peak CPU (Cores) | Peak Memory (GiB) | +| Input Tokens | Output Tokens | maxPrefixTokensToMatch | Peak CPU (Cores) | Peak Memory (GiB) | | :--- | :--- | :--- | :--- | :--- | -| 100k | 500 | 256 | 15.13 | 2.27 | -| 100k | 500 | 2048 | 17.14 | 3.76 | -| 100k | 1000 | 256 | 17.51 | 3.66 | -| 100k | 1000 | 2048 | 20.28 | 5.23 | -| 100k | 5000 | 1024 | 30.95 | 12.54 | -| 100k | 10000 | 512 | 32.53 | 12.54 | +| 100k | 500 | 4096 | 15.13 | 2.27 | +| 100k | 500 | 32768 | 17.14 | 3.76 | +| 100k | 1000 | 4096 | 17.51 | 3.66 | +| 100k | 1000 | 32768 | 20.28 | 5.23 | +| 100k | 5000 | 16384 | 30.95 | 12.54 | +| 100k | 10000 | 8192 | 32.53 | 12.54 | Configuration used: [#1287](https://github.com/llm-d/llm-d-router/issues/1287#issuecomment-4619775397) These were run against 0.9.0 EPP container image. From 98684098c99ca2160afff718c426f7fc3e61adc1 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Mon, 20 Jul 2026 21:40:08 -0700 Subject: [PATCH 7/7] deploy/config: update maxPrefixTokensToMatch to 16384 to preserve 256-block cap with 64-token min block size Signed-off-by: Cong Liu --- deploy/config/pd-epp-config.yaml | 2 +- deploy/config/sim-epp-config.yaml | 2 +- deploy/config/sim-pd-epp-config.yaml | 2 +- test/e2e/configs_test.go | 10 +++++----- test/integration/epp/e2e_config_smoke_test.go | 10 +++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/deploy/config/pd-epp-config.yaml b/deploy/config/pd-epp-config.yaml index fd1f818326..a9e4c7a9be 100644 --- a/deploy/config/pd-epp-config.yaml +++ b/deploy/config/pd-epp-config.yaml @@ -4,7 +4,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/deploy/config/sim-epp-config.yaml b/deploy/config/sim-epp-config.yaml index bdc73843e7..68e718cfb2 100644 --- a/deploy/config/sim-epp-config.yaml +++ b/deploy/config/sim-epp-config.yaml @@ -5,7 +5,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: decode-filter diff --git a/deploy/config/sim-pd-epp-config.yaml b/deploy/config/sim-pd-epp-config.yaml index 52ecfe63ba..95262c303e 100644 --- a/deploy/config/sim-pd-epp-config.yaml +++ b/deploy/config/sim-pd-epp-config.yaml @@ -7,7 +7,7 @@ plugins: parameters: blockSizeTokens: 16 autoTune: false - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 282c1d98a1..4eaaa4312f 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -8,7 +8,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -32,7 +32,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -92,7 +92,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -130,7 +130,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -164,7 +164,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter diff --git a/test/integration/epp/e2e_config_smoke_test.go b/test/integration/epp/e2e_config_smoke_test.go index 6b63d290e5..509755af77 100644 --- a/test/integration/epp/e2e_config_smoke_test.go +++ b/test/integration/epp/e2e_config_smoke_test.go @@ -26,7 +26,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -47,7 +47,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -79,7 +79,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -115,7 +115,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -150,7 +150,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixTokensToMatch: 4096 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter