Skip to content

Commit ddc5e15

Browse files
committed
upd
Signed-off-by: Chi McIsaac <chixie.mcisaac@gmail.com>
1 parent b59f64b commit ddc5e15

1 file changed

Lines changed: 17 additions & 3 deletions

File tree

  • python/tokenspeed/runtime/layers/attention/kv_cache/recipes

python/tokenspeed/runtime/layers/attention/kv_cache/recipes/ordinary.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,12 +148,26 @@ def groups(self) -> tuple[CacheGroupDeclaration, ...]:
148148

149149
@override
150150
def num_lcm_blocks(self, layout: CacheLayout) -> int:
151-
if layout.lcm_block_bytes <= 0:
151+
parent_bytes = layout.lcm_block_bytes
152+
if not layout.group_packing:
153+
# Capacity-only probes may not carry a packed layout. Preserve
154+
# ordinary cache sizing from the declared per-token payload while
155+
# production plans use the exact (DCP-aware) packed parent bytes.
156+
bytes_per_token = self.attn_config.cache_cell_size() * _storage_layers(
157+
self.attn_config, self.num_target_layers
158+
)
159+
if self.draft_attn_config is not None:
160+
bytes_per_token += (
161+
self.draft_attn_config.cache_cell_size()
162+
* _storage_layers(self.draft_attn_config, self.num_draft_layers)
163+
)
164+
parent_bytes = bytes_per_token * self.prefix_granularity
165+
if parent_bytes <= 0:
152166
raise ValueError(
153-
f"KV cache LCM block size must be positive, got {layout.lcm_block_bytes}"
167+
f"KV cache LCM block size must be positive, got {parent_bytes}"
154168
)
155169
return self._capped_parents(
156-
self.cache_budget_bytes // layout.lcm_block_bytes - 1,
170+
self.cache_budget_bytes // parent_bytes - 1,
157171
parent_tokens=self.prefix_granularity,
158172
)
159173

0 commit comments

Comments
 (0)