diff --git a/deploy/config/pd-epp-config.yaml b/deploy/config/pd-epp-config.yaml index cf17cdf68d..a9e4c7a9be 100644 --- a/deploy/config/pd-epp-config.yaml +++ b/deploy/config/pd-epp-config.yaml @@ -4,7 +4,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/deploy/config/sim-epp-config.yaml b/deploy/config/sim-epp-config.yaml index 0af90d0f0f..68e718cfb2 100644 --- a/deploy/config/sim-epp-config.yaml +++ b/deploy/config/sim-epp-config.yaml @@ -5,7 +5,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: decode-filter diff --git a/deploy/config/sim-epp-no-hit-lru.yaml b/deploy/config/sim-epp-no-hit-lru.yaml index 7cfc178add..7058757bed 100644 --- a/deploy/config/sim-epp-no-hit-lru.yaml +++ b/deploy/config/sim-epp-no-hit-lru.yaml @@ -7,9 +7,9 @@ plugins: parameters: indexerConfig: tokenProcessorConfig: - blockSize: 5 + blockSizeTokens: 5 kvBlockIndexConfig: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 1280 - type: prefix-cache-scorer parameters: prefixMatchInfoProducerName: precise-prefix-cache-producer diff --git a/deploy/config/sim-pd-epp-config.yaml b/deploy/config/sim-pd-epp-config.yaml index 771974b2cc..95262c303e 100644 --- a/deploy/config/sim-pd-epp-config.yaml +++ b/deploy/config/sim-pd-epp-config.yaml @@ -7,7 +7,7 @@ plugins: parameters: blockSizeTokens: 16 autoTune: false - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 31250 - type: prefix-cache-scorer - type: queue-scorer diff --git a/docs/architecture.md b/docs/architecture.md index b4f86faa16..1c692b3947 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -152,9 +152,9 @@ plugins: parameters: indexerConfig: tokenProcessorConfig: - blockSize: 5 + blockSizeTokens: 5 kvBlockIndexConfig: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 1280 - type: prefix-cache-scorer parameters: prefixMatchInfoProducerName: precise-prefix-cache-producer diff --git a/docs/disaggregation.md b/docs/disaggregation.md index 0a38201c72..1c3fe3eb0a 100644 --- a/docs/disaggregation.md +++ b/docs/disaggregation.md @@ -279,7 +279,7 @@ plugins: parameters: autoTune: false blockSizeTokens: 5 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 1280 lruCapacityPerServer: 31250 - type: max-score-picker - type: prefix-based-pd-decider @@ -342,7 +342,7 @@ plugins: parameters: autoTune: false blockSizeTokens: 5 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 1280 lruCapacityPerServer: 31250 - type: max-score-picker - type: always-disagg-multimodal-decider diff --git a/docs/operations.md b/docs/operations.md index f6df65b080..24c3a76d57 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -13,7 +13,7 @@ The EPP acts as the routing intelligence engine. Its resource usage scales prima #### CPU Allocation - **Rule of Thumb**: Allocate **0.5 to 1.0 CPU cores per request/second** of expected throughput for large agentic workloads (approximately 100k input / 1k output tokens). - **Scaling Behavior**: CPU utilization scales linearly with the request rate, and increases with both the input prompt size and output token length. -- **Prefix Matching Overhead**: Increasing the `maxPrefixBlocksToMatch` parameter increases EPP CPU utilization. At lower throughputs, a large prefix block limit (such as 6250 blocks) can increase EPP CPU utilization by over 100% compared to a small limit (256 blocks) due to the overhead of searching and matching blocks. +- **Prefix Matching Overhead**: Increasing the `maxPrefixTokensToMatch` parameter increases EPP CPU utilization. At lower throughputs, a large prefix limit (such as 100,000 tokens / 6,250 blocks with `blockSizeTokens: 16`) can increase EPP CPU utilization by over 100% compared to a small limit (4,096 tokens / 256 blocks) due to the overhead of searching and matching prefix blocks. - **Idle CPU Scaling**: Idle CPU usage of the EPP container scales with the number of model-serving pods in the cluster due to continuous metric scraping. For example, in a cluster with 100 model-serving pods, the idle CPU usage of the EPP container grows to approximately **7.5 cores**. #### Memory Allocation @@ -47,27 +47,27 @@ The following tables present empirical benchmark results for EPP running with ll #### Throughput and Prefix Block Sizing This table shows peak CPU and memory utilization for EPP under a 100k token workload (95k system prompt, 5k question prompt, and 1k output tokens) when using approximate prefix caching across 100 model-serving pods. -| Configuration | Request Rate (Req/s) | maxPrefixBlocksToMatch | Peak CPU (Cores) | Peak Memory (GiB) | Scheduler P50 Latency (s) | +| Configuration | Request Rate (Req/s) | maxPrefixTokensToMatch | Peak CPU (Cores) | Peak Memory (GiB) | Scheduler P50 Latency (s) | | :--- | :--- | :--- | :--- | :--- | :--- | -| Small Prefix Match | 5.0 | 256 | 1.19 | 0.26 | 0.00010 | -| Large Prefix Match | 5.0 | 6250 | 3.82 | 0.65 | 0.00010 | -| Small Prefix Match | 98.7 | 256 | 35.17 | 2.46 | 0.00014 | -| Large Prefix Match | 98.8 | 6250 | 46.50 | 3.41 | 0.00020 | +| Small Prefix Match | 5.0 | 4096 | 1.19 | 0.26 | 0.00010 | +| Large Prefix Match | 5.0 | 100000 | 3.82 | 0.65 | 0.00010 | +| Small Prefix Match | 98.7 | 4096 | 35.17 | 2.46 | 0.00014 | +| Large Prefix Match | 98.8 | 100000 | 46.50 | 3.41 | 0.00020 | Configuration used: [#1287](https://github.com/llm-d/llm-d-router/issues/1287#issuecomment-4666058475). These were run against 0.9.0 EPP container image. #### Output Length and Prefix Matching Complexity -This table shows EPP peak resource usage at a constant request rate of 50 requests/second with a 100k input token workload, varying the output token length and the `maxPrefixBlocksToMatch` configuration. +This table shows EPP peak resource usage at a constant request rate of 50 requests/second with a 100k input token workload, varying the output token length and the `maxPrefixTokensToMatch` configuration. -| Input Tokens | Output Tokens | maxPrefixBlocksToMatch | Peak CPU (Cores) | Peak Memory (GiB) | +| Input Tokens | Output Tokens | maxPrefixTokensToMatch | Peak CPU (Cores) | Peak Memory (GiB) | | :--- | :--- | :--- | :--- | :--- | -| 100k | 500 | 256 | 15.13 | 2.27 | -| 100k | 500 | 2048 | 17.14 | 3.76 | -| 100k | 1000 | 256 | 17.51 | 3.66 | -| 100k | 1000 | 2048 | 20.28 | 5.23 | -| 100k | 5000 | 1024 | 30.95 | 12.54 | -| 100k | 10000 | 512 | 32.53 | 12.54 | +| 100k | 500 | 4096 | 15.13 | 2.27 | +| 100k | 500 | 32768 | 17.14 | 3.76 | +| 100k | 1000 | 4096 | 17.51 | 3.66 | +| 100k | 1000 | 32768 | 20.28 | 5.23 | +| 100k | 5000 | 16384 | 30.95 | 12.54 | +| 100k | 10000 | 8192 | 32.53 | 12.54 | Configuration used: [#1287](https://github.com/llm-d/llm-d-router/issues/1287#issuecomment-4619775397) These were run against 0.9.0 EPP container image. diff --git a/test/e2e/configs_test.go b/test/e2e/configs_test.go index 0afbb4abba..4eaaa4312f 100644 --- a/test/e2e/configs_test.go +++ b/test/e2e/configs_test.go @@ -8,7 +8,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -32,7 +32,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -92,7 +92,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -130,7 +130,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -164,7 +164,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter diff --git a/test/integration/epp/e2e_config_smoke_test.go b/test/integration/epp/e2e_config_smoke_test.go index 33aec8a68b..509755af77 100644 --- a/test/integration/epp/e2e_config_smoke_test.go +++ b/test/integration/epp/e2e_config_smoke_test.go @@ -26,7 +26,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: decode-filter @@ -47,7 +47,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -79,7 +79,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: prefill-filter @@ -115,7 +115,7 @@ plugins: - type: approx-prefix-cache-producer parameters: blockSizeTokens: 16 - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: max-score-picker @@ -150,7 +150,7 @@ kind: EndpointPickerConfig plugins: - type: approx-prefix-cache-producer parameters: - maxPrefixBlocksToMatch: 256 + maxPrefixTokensToMatch: 16384 lruCapacityPerServer: 256 - type: prefix-cache-scorer - type: encode-filter