Skip to content

Commit 66a37a7

Browse files
cifar10cifar10
authored
[MLU] Update vllm for Qwen infer (flagos-ai#950)
1 parent f566586 commit 66a37a7

422 files changed

Lines changed: 65119 additions & 15244 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎hardware/Cambricon_MLU/vllm/.buildkite/check-wheel-size.py.patch‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
diff --git a/.buildkite/check-wheel-size.py b/.buildkite/check-wheel-size.py
2+
index 68aff793a..76f6d7aec 100644
3+
--- a/.buildkite/check-wheel-size.py
4+
+++ b/.buildkite/check-wheel-size.py
5+
@@ -5,11 +5,11 @@ import os
6+
import sys
7+
import zipfile
8+
9+
-# Read the VLLM_MAX_SIZE_MB environment variable, defaulting to 400 MiB
10+
-# Note that we have 400 MiB quota, please use it wisely.
11+
-# See https://github.com/pypi/support/issues/3792 .
12+
+# Read the VLLM_MAX_SIZE_MB environment variable, defaulting to 450 MiB
13+
+# Note that we have 800 MiB quota, please use it wisely.
14+
+# See https://github.com/pypi/support/issues/6326 .
15+
# Please also sync the value with the one in Dockerfile.
16+
-VLLM_MAX_SIZE_MB = int(os.environ.get("VLLM_MAX_SIZE_MB", 400))
17+
+VLLM_MAX_SIZE_MB = int(os.environ.get("VLLM_MAX_SIZE_MB", 450))
18+
19+
20+
def print_top_10_largest_files(zip_file):
21+

‎hardware/Cambricon_MLU/vllm/.buildkite/generate_index.py.patch‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
diff --git a/.buildkite/generate_index.py b/.buildkite/generate_index.py
2+
index 7045d8810..bbed80ebe 100644
3+
--- a/.buildkite/generate_index.py
4+
+++ b/.buildkite/generate_index.py
5+
@@ -8,7 +8,8 @@ template = """<!DOCTYPE html>
6+
<html>
7+
<body>
8+
<h1>Links for vLLM</h1/>
9+
- <a href="../{wheel_html_escaped}">{wheel}</a><br/>
10+
+ <a href="../{x86_wheel_html_escaped}">{x86_wheel}</a><br/>
11+
+ <a href="../{arm_wheel_html_escaped}">{arm_wheel}</a><br/>
12+
</body>
13+
</html>
14+
"""
15+
@@ -21,7 +22,25 @@ filename = os.path.basename(args.wheel)
16+
17+
with open("index.html", "w") as f:
18+
print(f"Generated index.html for {args.wheel}")
19+
+ # sync the abi tag with .buildkite/scripts/upload-wheels.sh
20+
+ if "x86_64" in filename:
21+
+ x86_wheel = filename
22+
+ arm_wheel = filename.replace("x86_64", "aarch64").replace(
23+
+ "manylinux1", "manylinux2014"
24+
+ )
25+
+ elif "aarch64" in filename:
26+
+ x86_wheel = filename.replace("aarch64", "x86_64").replace(
27+
+ "manylinux2014", "manylinux1"
28+
+ )
29+
+ arm_wheel = filename
30+
+ else:
31+
+ raise ValueError(f"Unsupported wheel: {filename}")
32+
# cloudfront requires escaping the '+' character
33+
f.write(
34+
- template.format(wheel=filename, wheel_html_escaped=filename.replace("+", "%2B"))
35+
+ template.format(
36+
+ x86_wheel=x86_wheel,
37+
+ x86_wheel_html_escaped=x86_wheel.replace("+", "%2B"),
38+
+ arm_wheel=arm_wheel,
39+
+ arm_wheel_html_escaped=arm_wheel.replace("+", "%2B"),
40+
+ )
41+
)
42+

‎hardware/Cambricon_MLU/vllm/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml.patch‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml
2+
deleted file mode 100644
3+
index 56ec933c9..000000000
4+
--- a/.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml
5+
+++ /dev/null
6+
@@ -1,12 +0,0 @@
7+
-# For vllm script, with -t option (tensor parallel size).
8+
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m HandH1998/QQQ-Llama-3-8b-g128 -b 32 -l 1000 -f 5 -t 1
9+
-model_name: "HandH1998/QQQ-Llama-3-8b-g128"
10+
-tasks:
11+
-- name: "gsm8k"
12+
- metrics:
13+
- - name: "exact_match,strict-match"
14+
- value: 0.419
15+
- - name: "exact_match,flexible-extract"
16+
- value: 0.416
17+
-limit: 1000
18+
-num_fewshot: 5
19+

‎hardware/Cambricon_MLU/vllm/.buildkite/lm-eval-harness/configs/models-large.txt.patch‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
diff --git a/.buildkite/lm-eval-harness/configs/models-large.txt b/.buildkite/lm-eval-harness/configs/models-large.txt
2+
index 27a1a9a82..37eeac85c 100644
3+
--- a/.buildkite/lm-eval-harness/configs/models-large.txt
4+
+++ b/.buildkite/lm-eval-harness/configs/models-large.txt
5+
@@ -3,4 +3,3 @@ Meta-Llama-3-70B-Instruct.yaml
6+
Mixtral-8x7B-Instruct-v0.1.yaml
7+
Qwen2-57B-A14-Instruct.yaml
8+
DeepSeek-V2-Lite-Chat.yaml
9+
-Meta-Llama-3-8B-QQQ.yaml
10+

‎hardware/Cambricon_MLU/vllm/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh.patch‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
2+
index a67fc89d5..897f84d1e 100644
3+
--- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
4+
+++ b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
5+
@@ -2,7 +2,7 @@
6+
# We can use this script to compute baseline accuracy on GSM for transformers.
7+
#
8+
# Make sure you have lm-eval-harness installed:
9+
-# pip install lm-eval==0.4.4
10+
+# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
11+
12+
usage() {
13+
echo``
14+

‎hardware/Cambricon_MLU/vllm/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh.patch‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
2+
index 65be3c5d9..792f355c4 100644
3+
--- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
4+
+++ b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
5+
@@ -3,7 +3,7 @@
6+
# We use this for fp8, which HF does not support.
7+
#
8+
# Make sure you have lm-eval-harness installed:
9+
-# pip install lm-eval==0.4.4
10+
+# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
11+
12+
usage() {
13+
echo``
14+
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
15+
done
16+
17+
lm_eval --model vllm \
18+
- --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend=ray,trust_remote_code=true,max_model_len=4096" \
19+
+ --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \
20+
--tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
21+
--batch_size "$BATCH_SIZE"
22+

‎hardware/Cambricon_MLU/vllm/.buildkite/lm-eval-harness/test_lm_eval_correctness.py.patch‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py
2+
index 930adfaf3..ceea01166 100644
3+
--- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py
4+
+++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py
5+
@@ -18,12 +18,14 @@ RTOL = 0.08
6+
7+
def launch_lm_eval(eval_config, tp_size):
8+
trust_remote_code = eval_config.get("trust_remote_code", False)
9+
+ max_model_len = eval_config.get("max_model_len", 4096)
10+
model_args = (
11+
f"pretrained={eval_config['model_name']},"
12+
f"tensor_parallel_size={tp_size},"
13+
f"enforce_eager=true,"
14+
f"add_bos_token=true,"
15+
- f"trust_remote_code={trust_remote_code}"
16+
+ f"trust_remote_code={trust_remote_code},"
17+
+ f"max_model_len={max_model_len}"
18+
)
19+
results = lm_eval.simple_evaluate(
20+
model="vllm",
21+

‎hardware/Cambricon_MLU/vllm/.buildkite/nightly-benchmarks/README.md.patch‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
diff --git a/.buildkite/nightly-benchmarks/README.md b/.buildkite/nightly-benchmarks/README.md
2+
index 72c52d5bb..e6f5c8b60 100644
3+
--- a/.buildkite/nightly-benchmarks/README.md
4+
+++ b/.buildkite/nightly-benchmarks/README.md
5+
@@ -7,11 +7,11 @@ This directory contains two sets of benchmark for vllm.
6+
- Performance benchmark: benchmark vllm's performance under various workload, for **developers** to gain clarity on whether their PR improves/degrades vllm's performance
7+
- Nightly benchmark: compare vllm's performance against alternatives (tgi, trt-llm and lmdeploy), for **the public** to know when to choose vllm.
8+
9+
-See [vLLM performance dashboard](https://perf.vllm.ai) for the latest performance benchmark results and [vLLM GitHub README](https://github.com/vllm-project/vllm/blob/main/README.md) for latest nightly benchmark results.
10+
+See [vLLM performance dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm) for the latest performance benchmark results and [vLLM GitHub README](https://github.com/vllm-project/vllm/blob/main/README.md) for latest nightly benchmark results.
11+
12+
## Performance benchmark quick overview
13+
14+
-**Benchmarking Coverage**: latency, throughput and fix-qps serving on A100 (the support for FP8 benchmark on H100 is coming!), with different models.
15+
+**Benchmarking Coverage**: latency, throughput and fix-qps serving on A100 (the support for FP8 benchmark on H100 is coming!) and Intel® Xeon® Processors, with different models.
16+
17+
**Benchmarking Duration**: about 1hr.
18+
19+
@@ -28,16 +28,34 @@ See [vLLM performance dashboard](https://perf.vllm.ai) for the latest performanc
20+
## Trigger the benchmark
21+
22+
Performance benchmark will be triggered when:
23+
+
24+
- A PR being merged into vllm.
25+
- Every commit for those PRs with `perf-benchmarks` label AND `ready` label.
26+
27+
+Manually Trigger the benchmark
28+
+
29+
+```bash
30+
+bash .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
31+
+```
32+
+
33+
+Runtime environment variables:
34+
+
35+
+- `ON_CPU`: set the value to '1' on Intel® Xeon® Processors. Default value is 0.
36+
+- `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file).
37+
+- `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file).
38+
+- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file).
39+
+- `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string.
40+
+- `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string.
41+
+
42+
Nightly benchmark will be triggered when:
43+
+
44+
- Every commit for those PRs with `perf-benchmarks` label and `nightly-benchmarks` label.
45+
46+
## Performance benchmark details
47+
48+
See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
49+
-
50+
+> NOTE: For Intel® Xeon® Processors, use `tests/latency-tests-cpu.json`, `tests/throughput-tests-cpu.json`, `tests/serving-tests-cpu.json` instead.
51+
+>
52+
### Latency test
53+
54+
Here is an example of one test inside `latency-tests.json`:
55+
@@ -60,7 +78,7 @@ Here is an example of one test inside `latency-tests.json`:
56+
In this example:
57+
58+
- The `test_name` attributes is a unique identifier for the test. In `latency-tests.json`, it must start with `latency_`.
59+
-- The `parameters` attribute control the command line arguments to be used for `benchmark_latency.py`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-performance-benchmarks.sh` will convert the underline to dash when feeding the arguments to `benchmark_latency.py`. For example, the corresponding command line arguments for `benchmark_latency.py` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15`
60+
+- The `parameters` attribute control the command line arguments to be used for `vllm bench latency`. Note that please use underline `_` instead of the dash `-` when specifying the command line arguments, and `run-performance-benchmarks.sh` will convert the underline to dash when feeding the arguments to `vllm bench latency`. For example, the corresponding command line arguments for `vllm bench latency` will be `--model meta-llama/Meta-Llama-3-8B --tensor-parallel-size 1 --load-format dummy --num-iters-warmup 5 --num-iters 15`
61+
62+
Note that the performance numbers are highly sensitive to the value of the parameters. Please make sure the parameters are set correctly.
63+
64+
@@ -68,13 +86,13 @@ WARNING: The benchmarking script will save json results by itself, so please do
65+
66+
### Throughput test
67+
68+
-The tests are specified in `throughput-tests.json`. The syntax is similar to `latency-tests.json`, except for that the parameters will be fed forward to `benchmark_throughput.py`.
69+
+The tests are specified in `throughput-tests.json`. The syntax is similar to `latency-tests.json`, except for that the parameters will be fed forward to `vllm bench throughput`.
70+
71+
The number of this test is also stable -- a slight change on the value of this number might vary the performance numbers by a lot.
72+
73+
### Serving test
74+
75+
-We test the throughput by using `benchmark_serving.py` with request rate = inf to cover the online serving overhead. The corresponding parameters are in `serving-tests.json`, and here is an example:
76+
+We test the throughput by using `vllm bench serve` with request rate = inf to cover the online serving overhead. The corresponding parameters are in `serving-tests.json`, and here is an example:
77+
78+
```json
79+
[
80+
@@ -86,7 +104,6 @@ We test the throughput by using `benchmark_serving.py` with request rate = inf t
81+
"tensor_parallel_size": 1,
82+
"swap_space": 16,
83+
"disable_log_stats": "",
84+
- "disable_log_requests": "",
85+
"load_format": "dummy"
86+
},
87+
"client_parameters": {
88+
@@ -104,8 +121,8 @@ Inside this example:
89+
90+
- The `test_name` attribute is also a unique identifier for the test. It must start with `serving_`.
91+
- The `server-parameters` includes the command line arguments for vLLM server.
92+
-- The `client-parameters` includes the command line arguments for `benchmark_serving.py`.
93+
-- The `qps_list` controls the list of qps for test. It will be used to configure the `--request-rate` parameter in `benchmark_serving.py`
94+
+- The `client-parameters` includes the command line arguments for `vllm bench serve`.
95+
+- The `qps_list` controls the list of qps for test. It will be used to configure the `--request-rate` parameter in `vllm bench serve`
96+
97+
The number of this test is less stable compared to the delay and latency benchmarks (due to randomized sharegpt dataset sampling inside `benchmark_serving.py`), but a large change on this number (e.g. 5% change) still vary the output greatly.
98+
99+
@@ -119,6 +136,23 @@ If you do not see the table, please wait till the benchmark finish running.
100+
The json version of the table (together with the json version of the benchmark) will be also attached to the markdown file.
101+
The raw benchmarking results (in the format of json files) are in the `Artifacts` tab of the benchmarking.
102+
103+
+The `compare-json-results.py` helps to compare benchmark results JSON files converted using `convert-results-json-to-markdown.py`.
104+
+When run, benchmark script generates results under `benchmark/results` folder, along with the `benchmark_results.md` and `benchmark_results.json`.
105+
+`compare-json-results.py` compares two `benchmark_results.json` files and provides performance ratio e.g. for Output Tput, Median TTFT and Median TPOT.
106+
+If only one benchmark_results.json is passed, `compare-json-results.py` compares different TP and PP configurations in the benchmark_results.json instead.
107+
+
108+
+Here is an example using the script to compare result_a and result_b with Model, Dataset name, input/output length, max concurrency and qps.
109+
+`python3 compare-json-results.py -f results_a/benchmark_results.json -f results_b/benchmark_results.json`
110+
+
111+
+| | Model | Dataset Name | Input Len | Output Len | # of max concurrency | qps | results_a/benchmark_results.json | results_b/benchmark_results.json | perf_ratio |
112+
+|----|---------------------------------------|--------|-----|-----|------|-----|-----------|----------|----------|
113+
+| 0 | meta-llama/Meta-Llama-3.1-8B-Instruct | random | 128 | 128 | 1000 | 1 | 142.633982 | 156.526018 | 1.097396 |
114+
+| 1 | meta-llama/Meta-Llama-3.1-8B-Instruct | random | 128 | 128 | 1000 | inf| 241.620334 | 294.018783 | 1.216863 |
115+
+
116+
+A comparison diagram will be generated below the table.
117+
+Here is an example to compare between 96c/results_gnr_96c_091_tp2pp3 and 128c/results_gnr_128c_091_tp2pp3
118+
+<img width="1886" height="828" alt="image" src="https://github.com/user-attachments/assets/c02a43ef-25d0-4fd6-90e5-2169a28682dd" />
119+
+
120+
## Nightly test details
121+
122+
See [nightly-descriptions.md](nightly-descriptions.md) for the detailed description on test workload, models and docker containers of benchmarking other llm engines.
123+
@@ -126,9 +160,9 @@ See [nightly-descriptions.md](nightly-descriptions.md) for the detailed descript
124+
### Workflow
125+
126+
- The [nightly-pipeline.yaml](nightly-pipeline.yaml) specifies the docker containers for different LLM serving engines.
127+
-- Inside each container, we run [run-nightly-suite.sh](run-nightly-suite.sh), which will probe the serving engine of the current container.
128+
-- The `run-nightly-suite.sh` will redirect the request to `tests/run-[llm serving engine name]-nightly.sh`, which parses the workload described in [nightly-tests.json](tests/nightly-tests.json) and performs the benchmark.
129+
-- At last, we run [scripts/plot-nightly-results.py](scripts/plot-nightly-results.py) to collect and plot the final benchmarking results, and update the results to buildkite.
130+
+- Inside each container, we run [scripts/run-nightly-benchmarks.sh](scripts/run-nightly-benchmarks.sh), which will probe the serving engine of the current container.
131+
+- The `scripts/run-nightly-benchmarks.sh` will parse the workload described in [nightly-tests.json](tests/nightly-tests.json) and launch the right benchmark for the specified serving engine via `scripts/launch-server.sh`.
132+
+- At last, we run [scripts/summary-nightly-results.py](scripts/summary-nightly-results.py) to collect and plot the final benchmarking results, and update the results to buildkite.
133+
134+
### Nightly tests
135+
136+
@@ -138,6 +172,6 @@ In [nightly-tests.json](tests/nightly-tests.json), we include the command line a
137+
138+
The docker containers for benchmarking are specified in `nightly-pipeline.yaml`.
139+
140+
-WARNING: the docker versions are HARD-CODED and SHOULD BE ALIGNED WITH `nightly-descriptions.md`. The docker versions need to be hard-coded as there are several version-specific bug fixes inside `tests/run-[llm serving engine name]-nightly.sh`.
141+
+WARNING: the docker versions are HARD-CODED and SHOULD BE ALIGNED WITH `nightly-descriptions.md`. The docker versions need to be hard-coded as there are several version-specific bug fixes inside `scripts/run-nightly-benchmarks.sh` and `scripts/launch-server.sh`.
142+
143+
WARNING: populating `trt-llm` to latest version is not easy, as it requires updating several protobuf files in [tensorrt-demo](https://github.com/neuralmagic/tensorrt-demo.git).
144+

‎hardware/Cambricon_MLU/vllm/.buildkite/nightly-benchmarks/nightly-annotation.md.patch‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
diff --git a/.buildkite/nightly-benchmarks/nightly-annotation.md b/.buildkite/nightly-benchmarks/nightly-annotation.md
2+
index e43ea765f..466def07b 100644
3+
--- a/.buildkite/nightly-benchmarks/nightly-annotation.md
4+
+++ b/.buildkite/nightly-benchmarks/nightly-annotation.md
5+
@@ -1,3 +1,4 @@
6+
+# Nightly benchmark annotation
7+
8+
## Description
9+
10+
@@ -13,15 +14,15 @@ Please download the visualization scripts in the post
11+
12+
- Find the docker we use in `benchmarking pipeline`
13+
- Deploy the docker, and inside the docker:
14+
- - Download `nightly-benchmarks.zip`.
15+
- - In the same folder, run the following code:
16+
-
17+
- ```console
18+
- export HF_TOKEN=<your HF token>
19+
- apt update
20+
- apt install -y git
21+
- unzip nightly-benchmarks.zip
22+
- VLLM_SOURCE_CODE_LOC=./ bash .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
23+
- ```
24+
+ - Download `nightly-benchmarks.zip`.
25+
+ - In the same folder, run the following code:
26+
+
27+
+ ```bash
28+
+ export HF_TOKEN=<your HF token>
29+
+ apt update
30+
+ apt install -y git
31+
+ unzip nightly-benchmarks.zip
32+
+ VLLM_SOURCE_CODE_LOC=./ bash .buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
33+
+ ```
34+
35+
And the results will be inside `./benchmarks/results`.
36+

0 commit comments

Comments
 (0)