[Future] [P/D] support hybrid attention for mooncake connector #11876

Workflow file for this run

.github/workflows/schedule_nightly_test_a3.yaml at e38e282

	#
	# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	# This file is a part of the vllm-ascend project.
	#

	# This workflow related to the resources atlas 800 A3
	# Please note: current A3 resource pool's maximum allowed concurrency is 5*16 NPUs
	# We will limit the concurrency of jobs on A3 to avoid the risk of insufficient resources
	name: Nightly-A3

	on:
	schedule:
	# Run test at 23:45 Beijing time (UTC+8)
	- cron: "45 15 * * *"
	workflow_dispatch:
	inputs:
	vllm_ascend_branch:
	description: 'Branch to test (main or releases/v0.18.0)'
	required: true
	default: 'main'
	type: choice
	options:
	- main
	- releases/v0.18.0
	test_cases:
	description: 'Test cases to run (comma-separated, e.g., "test_custom_op,qwen3-32b,accuracy-group" or "all" for all tests)'
	required: false
	default: 'all'
	type: string
	pull_request:
	branches:
	- 'main'
	- 'releases/v*'
	- '*-dev'
	types: [ labeled, synchronize ]

	permissions:
	contents: read
	pull-requests: read
	issues: read

	# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
	# declared as "shell: bash -el {0}" on steps that need to be properly activated.
	# It's used to activate ascend-toolkit environment variables.
	defaults:
	run:
	shell: bash -el {0}

	concurrency:
	group: ascend-nightly-${{ github.ref }}-a3
	cancel-in-progress: true

	env:
	# Single source of truth for all shared variables across jobs.
	# NOTE: GitHub Actions does not support env context in reusable workflow `with:` inputs,
	# so these values are exported as job outputs via the `setup-vars` job below.
	ASCEND_LOG_PREFIX: '/root/.cache/ascend-logs'

	jobs:
	parse-trigger:
	name: Parse trigger and determine test scope
	if: >-
	github.event_name == 'schedule' \|\|
	github.event_name == 'workflow_dispatch' \|\|
	contains(github.event.pull_request.labels.*.name, 'nightly-test')
	uses: ./.github/workflows/_parse_trigger.yaml
	with:
	test_cases: ${{ github.event_name == 'workflow_dispatch' && inputs.test_cases \|\| '' }}

	setup-vars:
	name: Export global env vars as job outputs
	needs: parse-trigger
	runs-on: ubuntu-latest
	outputs:
	ascend_log_prefix: ${{ steps.export.outputs.ascend_log_prefix }}
	vllm_ascend_branches: ${{ steps.export.outputs.vllm_ascend_branches }}
	steps:
	- id: export
	run: \|
	{
	echo "ascend_log_prefix=${{ env.ASCEND_LOG_PREFIX }}"
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	base_ref="${{ github.base_ref }}"
	if [[ "$base_ref" == releases/* ]]; then
	normalized=$(echo "$base_ref" \| tr '/' '-')
	echo "vllm_ascend_branches=[\"${normalized}\"]"
	else
	echo 'vllm_ascend_branches=["main"]'
	fi
	elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
	input_branch="${{ inputs.vllm_ascend_branch }}"
	# Normalize branch name for image tag (replace '/' with '-')
	normalized=$(echo "$input_branch" \| tr '/' '-')
	echo "vllm_ascend_branches=[\"${normalized}\"]"
	else
	echo 'vllm_ascend_branches=["main"]'
	fi
	} >> $GITHUB_OUTPUT

	build-image:
	name: Build nightly-a3 image
	if: github.event_name == 'schedule' \|\| github.event_name == 'workflow_dispatch'
	strategy:
	matrix:
	vllm_ascend_branch: >-
	${{ github.event_name == 'workflow_dispatch'
	&& fromJSON(format('["{0}"]', inputs.vllm_ascend_branch))
	\|\| fromJSON('["main"]') }}
	uses: ./.github/workflows/_nightly_image_build.yaml
	with:
	target: a3
	vllm_ascend_branch: ${{ matrix.vllm_ascend_branch }}
	secrets:
	HW_USERNAME: ${{ secrets.HW_USERNAME }}
	HW_TOKEN: ${{ secrets.HW_TOKEN }}
	GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}

	multi-node-tests:
	name: multi-node
	needs: [setup-vars, parse-trigger, build-image]
	if: >-
	always() &&
	needs.parse-trigger.outputs.run == 'true' &&
	(needs.build-image.result == 'success' \|\| needs.build-image.result == 'skipped')
	strategy:
	fail-fast: false
	max-parallel: 1
	matrix:
	vllm_ascend_branch: ${{ fromJSON(needs.setup-vars.outputs.vllm_ascend_branches) }}
	test_config:
	- name: multi-node-deepseek-v3.2-W8A8-EP
	config_file_path: DeepSeek-V3_2-W8A8-EP.yaml
	size: 4
	- name: multi-node-deepseek-v3.2-W8A8-EP-aime2025
	config_file_path: DeepSeek-V3_2-W8A8-EP-aime2025.yaml
	size: 4
	- name: multi-node-GLM5_1-W8A8-EP
	config_file_path: GLM5_1-W8A8-EP.yaml
	size: 4
	uses: ./.github/workflows/_e2e_nightly_multi_node.yaml
	with:
	soc_version: a3
	runner: linux-aarch64-a3-0
	image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-ci-${{ matrix.vllm_ascend_branch }}-a3'
	ascend_log_prefix: ${{ needs.setup-vars.outputs.ascend_log_prefix }}/${{ matrix.vllm_ascend_branch }}
	vllm_ascend_branch: ${{ matrix.vllm_ascend_branch }}
	replicas: 1
	size: ${{ matrix.test_config.size }}
	config_file_path: ${{ matrix.test_config.config_file_path }}
	vllm_ascend_ref: ${{ needs.parse-trigger.outputs.ref }}
	should_run: >-
	${{
	needs.parse-trigger.outputs.run == 'true' && (
	needs.parse-trigger.outputs.filter == 'all' \|\|
	contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
	)
	}}
	secrets:
	KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
	OBS_ACCESS_KEY: ${{ secrets.OBS_ACCESS_KEY }}
	OBS_SECRET_KEY: ${{ secrets.OBS_SECRET_KEY }}


	double-node-tests:
	name: double-node
	needs: [setup-vars, parse-trigger, build-image, multi-node-tests]
	if: >-
	always() &&
	needs.parse-trigger.outputs.run == 'true' &&
	(needs.build-image.result == 'success' \|\| needs.build-image.result == 'skipped')
	strategy:
	fail-fast: false
	max-parallel: 3
	matrix:
	vllm_ascend_branch: ${{ fromJSON(needs.setup-vars.outputs.vllm_ascend_branches) }}
	test_config:
	- name: multi-node-deepseek-r1-w8a8-longseq
	config_file_path: DeepSeek-R1-W8A8-longseq.yaml
	size: 2
	- name: multi-node-deepseek-pd
	config_file_path: DeepSeek-V3.yaml
	size: 2
	- name: multi-node-qwen3-dp
	config_file_path: Qwen3-235B-A22B.yaml
	size: 2
	- name: multi-node-qwenw8a8-2node
	config_file_path: Qwen3-235B-W8A8.yaml
	size: 2
	- name: multi-node-qwenw8a8-2node-eplb
	config_file_path: Qwen3-235B-W8A8-EPLB.yaml
	size: 2
	- name: multi-node-dpsk3.2-2node
	config_file_path: DeepSeek-V3_2-W8A8-A3-dual-nodes.yaml
	size: 2
	- name: multi-node-qwen3-dp-mooncake-layerwise
	config_file_path: Qwen3-235B-A22B-Mooncake-Layerwise.yaml
	size: 2
	- name: multi-node-qwenw8a8-2node-longseq
	config_file_path: Qwen3-235B-W8A8-longseq.yaml
	size: 2
	- name: multi-node-qwen-disagg-pd
	config_file_path: Qwen3-235B-disagg-pd.yaml
	size: 2
	- name: multi-node-qwen-vl-disagg-pd
	config_file_path: Qwen3-VL-235B-disagg-pd.yaml
	size: 2
	- name: multi-node-kimi-k2-instruct-w8a8
	config_file_path: Kimi-K2-Instruct-W8A8.yaml
	size: 2
	- name: multi-node-deepseek-v3.1
	config_file_path: DeepSeek-V3.1-BF16.yaml
	size: 2
	- name: multi-node-glm4.7-w8a8c8-layerwise
	config_file_path: GLM-4.7-W8A8C8-Mooncake-Layerwise.yaml
	size: 2
	uses: ./.github/workflows/_e2e_nightly_multi_node.yaml
	with:
	soc_version: a3
	runner: linux-aarch64-a3-0
	image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-ci-${{ matrix.vllm_ascend_branch }}-a3'
	ascend_log_prefix: ${{ needs.setup-vars.outputs.ascend_log_prefix }}/${{ matrix.vllm_ascend_branch }}
	vllm_ascend_branch: ${{ matrix.vllm_ascend_branch }}
	replicas: 1
	size: ${{ matrix.test_config.size }}
	config_file_path: ${{ matrix.test_config.config_file_path }}
	name: ${{ matrix.test_config.name }}
	vllm_ascend_ref: ${{ needs.parse-trigger.outputs.ref }}
	should_run: >-
	${{
	needs.parse-trigger.outputs.run == 'true' && (
	needs.parse-trigger.outputs.filter == 'all' \|\|
	contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
	)
	}}
	secrets:
	KUBECONFIG_B64: ${{ secrets.KUBECONFIG_B64 }}
	OBS_ACCESS_KEY: ${{ secrets.OBS_ACCESS_KEY }}
	OBS_SECRET_KEY: ${{ secrets.OBS_SECRET_KEY }}

	single-node-tests:
	name: single-node
	needs: [setup-vars, parse-trigger, build-image, double-node-tests]
	if: >-
	always() &&
	needs.parse-trigger.outputs.run == 'true' &&
	(needs.build-image.result == 'success' \|\| needs.build-image.result == 'skipped')
	strategy:
	fail-fast: false
	max-parallel: 7
	matrix:
	vllm_ascend_branch: ${{ fromJSON(needs.setup-vars.outputs.vllm_ascend_branches) }}
	test_config:
	# YAML-driven tests
	- name: mtpx-deepseek-r1-0528-w8a8
	os: linux-aarch64-a3-16
	config_file_path: MTPX-DeepSeek-R1-0528-W8A8.yaml
	- name: deepseek-r1-0528-w8a8
	os: linux-aarch64-a3-16
	config_file_path: DeepSeek-R1-0528-W8A8.yaml
	- name: kimi-k2-thinking
	os: linux-aarch64-a3-16
	config_file_path: Kimi-K2-Thinking.yaml
	- name: qwen3-vl-235b-a22b-instruct-w8a8
	os: linux-aarch64-a3-16
	config_file_path: Qwen3-VL-235B-A22B-Instruct-W8A8.yaml
	- name: deepseek-r1-0528-w8a8-prefix-cache
	os: linux-aarch64-a3-16
	config_file_path: Prefix-Cache-DeepSeek-R1-0528-W8A8.yaml
	- name: deepseek-r1-w8a8-hbm
	os: linux-aarch64-a3-16
	config_file_path: DeepSeek-R1-W8A8-HBM.yaml
	- name: glm-5-w4a8
	os: linux-aarch64-a3-16
	config_file_path: GLM-5.yaml
	- name: glm-4.7-w8a8
	os: linux-aarch64-a3-16
	config_file_path: GLM-4.7.yaml
	- name: minimax-m2-5
	os: linux-aarch64-a3-16
	config_file_path: MiniMax-M2.5-A3.yaml
	- name: qwen3-235b-a22b-w8a8
	os: linux-aarch64-a3-16
	config_file_path: Qwen3-235B-A22B-W8A8.yaml
	- name: MiniMax-M2.5-w8a8
	os: linux-aarch64-a3-16
	config_file_path: MiniMax-M2.5-W8A8-A3.yaml
	# pytest-driven tests
	- name: custom-multi-ops
	os: linux-aarch64-a3-16
	tests: tests/e2e/nightly/single_node/ops/multicard_ops_a3/
	uses: ./.github/workflows/_e2e_nightly_single_node.yaml
	with:
	runner: ${{ matrix.test_config.os }}
	image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-ci-${{ matrix.vllm_ascend_branch }}-a3'
	tests: ${{ matrix.test_config.tests }}
	config_file_path: ${{ matrix.test_config.config_file_path }}
	name: ${{ matrix.test_config.name }}
	should_run: >-
	${{
	needs.parse-trigger.outputs.run == 'true' && (
	needs.parse-trigger.outputs.filter == 'all' \|\|
	contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
	)
	}}

	multi-card-tests:
	name: single-node
	needs: [setup-vars, parse-trigger, build-image, single-node-tests]
	if: >-
	always() &&
	needs.parse-trigger.outputs.run == 'true' &&
	(needs.build-image.result == 'success' \|\| needs.build-image.result == 'skipped')
	strategy:
	fail-fast: false
	matrix:
	vllm_ascend_branch: ${{ fromJSON(needs.setup-vars.outputs.vllm_ascend_branches) }}
	test_config:
	# pytest-driven tests
	- name: qwen3-30b-acc
	os: linux-aarch64-a3-4
	tests: tests/e2e/weekly/single_node/models/test_qwen3_30b_acc.py
	# YAML-driven tests
	- name: qwen3-30b-a3b-w8a8
	os: linux-aarch64-a3-4
	config_file_path: Qwen3-30B-A3B-W8A8.yaml
	- name: qwen3-next-80b-a3b-instruct
	os: linux-aarch64-a3-4
	config_file_path: Qwen3-Next-80B-A3B-Instruct.yaml
	- name: qwen3-next-80b-a3b-instruct-w8a8
	os: linux-aarch64-a3-4
	config_file_path: Qwen3-Next-80B-A3B-Instruct-W8A8.yaml
	- name: qwen3-32b-int8
	os: linux-aarch64-a3-4
	config_file_path: Qwen3-32B-Int8.yaml
	- name: qwen2-5-vl-7b-epd
	os: linux-aarch64-a3-4
	config_file_path: Qwen2.5-VL-7B-Instruct-EPD.yaml
	- name: qwen3-32b-int8-prefix-cache
	os: linux-aarch64-a3-4
	config_file_path: Prefix-Cache-Qwen3-32B-Int8.yaml
	- name: Qwen3.5-27B-w8a8-A3
	os: linux-aarch64-a3-2
	config_file_path: Qwen3.5-27B-w8a8-A3.yaml
	- name: Qwen3-30B-A3B-W4A8-llm-compressor
	os: linux-aarch64-a3-2
	config_file_path: Qwen3-30B-A3B-W4A8-llm-compressor.yaml
	uses: ./.github/workflows/_e2e_nightly_single_node.yaml
	with:
	runner: ${{ matrix.test_config.os }}
	image: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-ci-${{ matrix.vllm_ascend_branch }}-a3'
	tests: ${{ matrix.test_config.tests }}
	config_file_path: ${{ matrix.test_config.config_file_path }}
	name: ${{ matrix.test_config.name }}
	should_run: >-
	${{
	needs.parse-trigger.outputs.run == 'true' && (
	needs.parse-trigger.outputs.filter == 'all' \|\|
	contains(needs.parse-trigger.outputs.filter, format(',{0},', matrix.test_config.name))
	)
	}}
	secrets:
	OBS_ACCESS_KEY: ${{ secrets.OBS_ACCESS_KEY }}
	OBS_SECRET_KEY: ${{ secrets.OBS_SECRET_KEY }}


	clear-pre-logs:
	runs-on: linux-aarch64-a3-0
	needs: [multi-node-tests, double-node-tests]
	if: github.event_name == 'schedule' && (needs.multi-node-tests.result != 'skipped' \|\| needs.double-node-tests.result != 'skipped')
	steps:
	- name: Clear pre-logs on pvc
	run: \|
	# Clear logs generated before the test to avoid confusion
	rm -rf "${ASCEND_LOG_PREFIX:?}"/* \|\| true

	merge-benchmark-artifacts:
	name: Merge benchmark artifacts into nightly-a3.zip
	needs: [parse-trigger, single-node-tests, multi-node-tests, double-node-tests]
	if: >-
	always() &&
	(github.event_name == 'schedule' \|\| github.event_name == 'workflow_dispatch') &&
	needs.parse-trigger.result == 'success'
	runs-on: ubuntu-latest
	steps:
	- name: Merge all benchmark result artifacts
	continue-on-error: true
	uses: actions/upload-artifact/merge@v7
	with:
	name: nightly-a3
	pattern: nightly-test-benchmark-results-*
	compression-level: 0
	delete-merged: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Future] [P/D] support hybrid attention for mooncake connector #11876

Workflow file

[Future] [P/D] support hybrid attention for mooncake connector #11876

Uh oh!

Workflow file for this run