File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -116,7 +116,9 @@ def __init__(
116116 elif self .chunked_prefill_size is not None :
117117 self .chunked_prefill_size = chunked_prefill_size
118118 else :
119- self .chunked_prefill_size = max_sequence_length if max_sequence_length is not None else max_num_tokens_per_batch
119+ self .chunked_prefill_size = (
120+ max_sequence_length if max_sequence_length is not None else max_num_tokens_per_batch
121+ )
120122 if self .lora_paths is not None and len (self .lora_paths ) > 0 :
121123 self .check_lora_server_args ()
122124
Original file line number Diff line number Diff line change @@ -355,7 +355,9 @@ def form_batch(self) -> List[Request]:
355355 break
356356 cost = req .prompt_len
357357 if cost + inflight_tokens > self .max_num_tokens_per_batch :
358- logger .debug (f"prefill request { req .request_id } cost { cost } + inflight_tokens { inflight_tokens } > max_num_tokens_per_batch { self .max_num_tokens_per_batch } , breaking" )
358+ logger .debug (
359+ f"prefill request { req .request_id } cost { cost } + inflight_tokens { inflight_tokens } > max_num_tokens_per_batch { self .max_num_tokens_per_batch } , breaking"
360+ )
359361 continue
360362 batch .append (req )
361363 inflight_tokens += cost
You can’t perform that action at this time.
0 commit comments