Skip to content

Commit 5842eca

Browse files
chen-commitschen-commits
authored
[CI] Move accuracy-group-2 to weekly single node configs (#10974)
### What this PR does / why we need it? This PR moves `accuracy-group-2` (containing `ERNIE-4.5-21B-A3B-PT`, `Molmo-7B-D-0924`, and `Llama-3.2-3B-Instruct`) from the nightly E2E models configuration to the weekly single-node E2E models configuration to optimize nightly test suite execution times. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? This is a configuration change for E2E tests. - vLLM version: v0.22.1 - vLLM main: vllm-project/vllm@967c5c3 --------- Signed-off-by: chen-commits <1636718796@qq.com> Signed-off-by: chen <1636718796@qq.com>
1 parent d541ed5 commit 5842eca

3 files changed

Lines changed: 237 additions & 17 deletions

File tree

‎.github/workflows/schedule_weekly_test_a2.yaml‎

Lines changed: 224 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,240 @@
1515
# This file is a part of the vllm-ascend project.
1616
#
1717

18+
# This workflow related to the resources atlas 800 A2
19+
# We will not limit the concurrency of jobs on A2
1820
name: Weekly-A2
1921

2022
on:
21-
schedule:
22-
# Run doctest at 00:00 Beijing time (UTC+8) every Sunday
23-
- cron: "0 16 * * 6"
2423
workflow_dispatch:
24+
inputs:
25+
vllm_ascend_branch:
26+
description: 'Branch to test'
27+
required: true
28+
default: 'main'
29+
type: string
30+
test_cases:
31+
description: 'Test cases to run (comma-separated, e.g., "test_custom_op,qwen3-32b,accuracy-group" or "all" for all tests)'
32+
required: false
33+
default: 'all'
34+
type: string
35+
request_id:
36+
description: 'Unique request ID from the triggering workflow for run identification'
37+
required: false
38+
default: ''
39+
type: string
40+
skip_build_image:
41+
description: 'Skip build-image job (used when triggered by /nightly command)'
42+
required: false
43+
default: 'false'
44+
type: string
45+
vllm_ascend_ref:
46+
description: 'PR commit SHA for PR-triggered tests'
47+
required: false
48+
default: ''
49+
type: string
2550

2651
permissions:
2752
contents: read
53+
pull-requests: read
54+
issues: read
2855

56+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
57+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
58+
# It's used to activate ascend-toolkit environment variables.
59+
defaults:
60+
run:
61+
shell: bash -el {0}
62+
63+
# only cancel in-progress runs of the same workflow
2964
concurrency:
30-
group: ascend-weekly-${{ github.ref }}-a2
65+
group: ascend-weekly-${{ github.ref }}-a2${{ inputs.request_id && format('-{0}', inputs.request_id) || '' }}
3166
cancel-in-progress: true
3267

68+
env:
69+
# Single source of truth for all shared variables across jobs.
70+
# NOTE: GitHub Actions does not support env context in reusable workflow `with:` inputs,
71+
# so these values are exported as job outputs via the `setup-vars` job below.
72+
CANN_IMAGE: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.12'
73+
ASCEND_LOG_PREFIX: '/root/.cache/ascend-logs'
74+
3375
jobs:
34-
doc-test:
35-
name: doc-test
36-
uses: ./.github/workflows/labeled_doctest.yaml
76+
parse-trigger:
77+
name: Parse trigger and determine test scope
78+
if: >-
79+
github.event_name == 'workflow_dispatch'
80+
runs-on: linux-aarch64-a2b3-0
81+
outputs:
82+
run: ${{ steps.parse.outputs.run }}
83+
filter: ${{ steps.parse.outputs.filter }}
84+
ref: ${{ steps.parse.outputs.ref }}
85+
steps:
86+
- name: Parse trigger
87+
id: parse
88+
uses: actions/github-script@v9
89+
with:
90+
script: |
91+
const eventName = context.eventName;
92+
const testCases = '${{ inputs.test_cases }}'.trim();
93+
94+
core.setOutput('ref', context.sha || 'unknown');
95+
96+
if (testCases) {
97+
core.setOutput('run', 'true');
98+
if (testCases === 'all') {
99+
core.setOutput('filter', 'all');
100+
} else {
101+
core.setOutput('filter', ',' + testCases.split(',').map(s => s.trim()).join(',') + ',');
102+
}
103+
return;
104+
}
105+
106+
core.setOutput('run', 'false');
107+
core.setOutput('filter', '');
108+
109+
setup-vars:
110+
name: Export global env vars as job outputs
111+
needs: parse-trigger
112+
runs-on: linux-aarch64-a2b3-0
113+
outputs:
114+
cann_image: ${{ steps.export.outputs.cann_image }}
115+
ascend_log_prefix: ${{ steps.export.outputs.ascend_log_prefix }}
116+
vllm_ascend_branches: ${{ steps.export.outputs.vllm_ascend_branches }}
117+
steps:
118+
- id: export
119+
run: |
120+
{
121+
echo "cann_image=${{ env.CANN_IMAGE }}"
122+
echo "ascend_log_prefix=${{ env.ASCEND_LOG_PREFIX }}"
123+
input_branch="${{ inputs.vllm_ascend_branch }}"
124+
normalized=$(echo "$input_branch" | tr '/' '-')
125+
echo "vllm_ascend_branches=[\"${normalized}\"]"
126+
} >> $GITHUB_OUTPUT
127+
128+
build-image:
129+
name: Build nightly-a2 image
130+
if: >-
131+
(github.event_name == 'workflow_dispatch' && inputs.skip_build_image != 'true')
132+
strategy:
133+
matrix:
134+
vllm_ascend_branch: >-
135+
${{ fromJSON(format('["{0}"]', inputs.vllm_ascend_branch)) }}
136+
uses: ./.github/workflows/_nightly_image_build.yaml
37137
with:
38-
doc_versions: '["v0.18.0","latest"]'
138+
target: a2
139+
vllm_ascend_branch: ${{ matrix.vllm_ascend_branch }}
140+
secrets:
141+
HW_USERNAME: ${{ secrets.HW_USERNAME }}
142+
HW_TOKEN: ${{ secrets.HW_TOKEN }}
143+
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
144+
145+
generate-accuracy-matrix:
146+
name: Generate accuracy test matrix
147+
needs: [parse-trigger]
148+
if: >-
149+
always() &&
150+
needs.parse-trigger.outputs.run == 'true' && (
151+
needs.parse-trigger.outputs.filter == 'all' ||
152+
contains(needs.parse-trigger.outputs.filter, 'accuracy-group')
153+
)
154+
runs-on: linux-aarch64-a2b3-0
155+
outputs:
156+
nightly_matrix: ${{ steps.set-matrix.outputs.nightly_matrix }}
157+
pr_only_matrix: ${{ steps.set-matrix.outputs.pr_only_matrix }}
158+
steps:
159+
- name: Checkout repository
160+
uses: actions/checkout@v7
161+
162+
- name: Read accuracy group config
163+
id: set-matrix
164+
run: |
165+
CONFIG_FILE="tests/e2e/weekly/single_node/configs/accuracy_groups_a2.json"
166+
NIGHTLY=$(jq -c '.nightly' "$CONFIG_FILE")
167+
PR_ONLY=$(jq -c '.pr_only' "$CONFIG_FILE")
168+
echo "nightly_matrix=${NIGHTLY}" >> "$GITHUB_OUTPUT"
169+
echo "pr_only_matrix=${PR_ONLY}" >> "$GITHUB_OUTPUT"
170+
171+
single-node-accuracy-tests:
172+
needs: [setup-vars, parse-trigger, build-image, generate-accuracy-matrix]
173+
if: >-
174+
always() &&
175+
needs.parse-trigger.outputs.run == 'true' &&
176+
needs.generate-accuracy-matrix.result != 'skipped' &&
177+
(needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
178+
strategy:
179+
fail-fast: false
180+
matrix:
181+
vllm_ascend_branch: ${{ fromJSON(needs.setup-vars.outputs.vllm_ascend_branches) }}
182+
test_config: ${{ fromJson(needs.generate-accuracy-matrix.outputs.nightly_matrix) }}
183+
uses: ./.github/workflows/_e2e_nightly_single_node_models.yaml
184+
with:
185+
runner: ${{ matrix.test_config.os }}
186+
model_list: ${{ toJson(matrix.test_config.model_list) }}
187+
model_filter: ${{ needs.parse-trigger.outputs.filter }}
188+
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-ci-${{ matrix.vllm_ascend_branch }}-a2'
189+
is_run: >-
190+
${{
191+
needs.parse-trigger.outputs.run == 'true' && (
192+
needs.parse-trigger.outputs.filter == 'all' ||
193+
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
194+
)
195+
}}
196+
request_id: ${{ inputs.request_id || '' }}
197+
vllm_ascend_ref: ${{ inputs.vllm_ascend_ref || '' }}
198+
upload: false
199+
200+
single-node-accuracy-tests-pr-only:
201+
needs: [setup-vars, parse-trigger, build-image, generate-accuracy-matrix]
202+
if: >-
203+
always() &&
204+
needs.parse-trigger.outputs.run == 'true' &&
205+
needs.parse-trigger.outputs.filter != 'all' &&
206+
needs.generate-accuracy-matrix.result != 'skipped' &&
207+
(needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
208+
strategy:
209+
fail-fast: false
210+
matrix:
211+
vllm_ascend_branch: ${{ fromJSON(needs.setup-vars.outputs.vllm_ascend_branches) }}
212+
test_config: ${{ fromJson(needs.generate-accuracy-matrix.outputs.pr_only_matrix) }}
213+
uses: ./.github/workflows/_e2e_nightly_single_node_models.yaml
214+
with:
215+
runner: ${{ matrix.test_config.os }}
216+
model_list: ${{ toJson(matrix.test_config.model_list) }}
217+
model_filter: ${{ needs.parse-trigger.outputs.filter }}
218+
image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-ci-${{ matrix.vllm_ascend_branch }}-a2'
219+
is_run: >-
220+
${{
221+
needs.parse-trigger.outputs.run == 'true' &&
222+
needs.parse-trigger.outputs.filter != 'all' &&
223+
contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
224+
}}
225+
request_id: ${{ inputs.request_id || '' }}
226+
vllm_ascend_ref: ${{ inputs.vllm_ascend_ref || '' }}
227+
upload: false
228+
229+
clear-pre-logs:
230+
needs: [parse-trigger]
231+
runs-on: linux-aarch64-a2b3-0
232+
steps:
233+
- name: Clear pre-logs on pvc
234+
run: |
235+
# Clear logs generated before the test to avoid confusion
236+
rm -rf "${ASCEND_LOG_PREFIX:?}"/* || true
237+
238+
merge-benchmark-artifacts:
239+
name: Merge benchmark artifacts into nightly-a2.zip
240+
needs: [parse-trigger]
241+
if: >-
242+
always() &&
243+
github.event_name == 'workflow_dispatch' &&
244+
needs.parse-trigger.result == 'success'
245+
runs-on: linux-aarch64-a2b3-0
246+
steps:
247+
- name: Merge all benchmark result artifacts
248+
continue-on-error: true
249+
uses: actions/upload-artifact/merge@v7
250+
with:
251+
name: weekly-a2
252+
pattern: nightly-test-benchmark-results-*
253+
compression-level: 0
254+
delete-merged: true

‎tests/e2e/models/configs/accuracy_groups_a2.json‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,6 @@
1212
"Minitron-8B-Base"
1313
]
1414
},
15-
{
16-
"name": "accuracy-group-2",
17-
"os": "linux-aarch64-a2b3-1",
18-
"model_list": [
19-
"ERNIE-4.5-21B-A3B-PT",
20-
"Molmo-7B-D-0924",
21-
"Llama-3.2-3B-Instruct"
22-
]
23-
},
2415
{
2516
"name": "accuracy-group-3",
2617
"os": "linux-aarch64-a2b3-2",

‎tests/e2e/weekly/single_node/configs/accuracy_groups_a2.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"nightly": [
3+
{
4+
"name": "accuracy-group-2",
5+
"os": "linux-aarch64-a2b3-1",
6+
"model_list": [
7+
"ERNIE-4.5-21B-A3B-PT",
8+
"Molmo-7B-D-0924",
9+
"Llama-3.2-3B-Instruct"
10+
]
11+
}
12+
]
13+
}

0 commit comments

Comments
 (0)