Skip to content

Commit 652ae9f

Browse files
luoyc123MC952-arch
andauthored
[Serve] Fix vllm in online serving (flagos-ai#969)
### PR Category Serve ### PR Types Bug Fixes ### PR Description 1、Fix vllm in online serve. 2、Fix flagscale pd disaggregated on GPU Co-authored-by: MC952-arch <MC952-arch@qq.com>
1 parent 389cf1c commit 652ae9f

1 file changed

Lines changed: 11 additions & 7 deletions

File tree

flagscale/backends/vllm/vllm/inputs/preprocess.py.patch

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
11
diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py
2-
index 7518cd8fc..8ff83d805 100644
2+
index 7518cd8fc..f78b0ff04 100644
33
--- a/vllm/inputs/preprocess.py
44
+++ b/vllm/inputs/preprocess.py
5-
@@ -311,6 +311,8 @@ class InputPreprocessor:
5+
@@ -311,6 +311,12 @@ class InputPreprocessor:
66
) -> Union[TokenInputs, MultiModalInputs]:
77
prompt_token_ids = self._truncate_inputs(
88
parsed_content["prompt_token_ids"], tokenization_kwargs)
9-
+ uncond_prompt_token_ids = self._truncate_inputs(
10-
+ parsed_content["uncond_prompt_token_ids"], tokenization_kwargs)
9+
+ uncond_prompt_token_ids_raw = parsed_content.get("uncond_prompt_token_ids")
10+
+ if uncond_prompt_token_ids_raw is not None:
11+
+ uncond_prompt_token_ids = self._truncate_inputs(
12+
+ parsed_content["uncond_prompt_token_ids"], tokenization_kwargs)
13+
+ else:
14+
+ uncond_prompt_token_ids = None
1115

1216
inputs: Union[TokenInputs, MultiModalInputs]
1317
if multi_modal_data := parsed_content.get("multi_modal_data"):
14-
@@ -322,7 +324,10 @@ class InputPreprocessor:
18+
@@ -322,7 +328,10 @@ class InputPreprocessor:
1519
mm_uuids=mm_uuids,
1620
)
1721
else:
@@ -23,15 +27,15 @@ index 7518cd8fc..8ff83d805 100644
2327

2428
if cache_salt := parsed_content.get("cache_salt"):
2529
inputs["cache_salt"] = cache_salt
26-
@@ -337,6 +342,7 @@ class InputPreprocessor:
30+
@@ -337,6 +346,7 @@ class InputPreprocessor:
2731
mm_uuids: Optional[MultiModalUUIDDict] = None,
2832
) -> Union[TokenInputs, MultiModalInputs]:
2933
prompt_text = parsed_content["prompt"]
3034
+ uncond_prompt_text = parsed_content.get("uncond_prompt")
3135

3236
inputs: Union[TokenInputs, MultiModalInputs]
3337
if multi_modal_data := parsed_content.get("multi_modal_data"):
34-
@@ -352,9 +358,17 @@ class InputPreprocessor:
38+
@@ -352,9 +362,17 @@ class InputPreprocessor:
3539
prompt_text,
3640
tokenization_kwargs=tokenization_kwargs,
3741
)

0 commit comments

Comments
 (0)