We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 8387956 commit a9a4e41Copy full SHA for a9a4e41
1 file changed
src/cpp/src/continuous_batching/scheduler.hpp
@@ -191,7 +191,7 @@ class Scheduler {
191
if (tokens_in_last_block == 0) {
192
tokens_in_last_block = block_size;
193
}
194
- preempted_tokens = tokens_in_last_block + std::max<size_t>((int)logical_blocks_released - 1, 0) * block_size;
+ preempted_tokens = tokens_in_last_block + (logical_blocks_released == 0 ? 0 : logical_blocks_released - 1) * block_size;
195
196
// case when preemption requires preempt prompt tokens
197
if (!m_config.dynamic_split_fuse && processed_tokens - preempted_tokens < sequence_group->get_prompt_len()) {
0 commit comments