From 40362e46e2d04a93377cf542e6b5a303130dce88 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Tue, 16 Jun 2026 10:48:34 +0800 Subject: [PATCH 1/9] add run_op.sh --- tools/run_op.sh | 132 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 tools/run_op.sh diff --git a/tools/run_op.sh b/tools/run_op.sh new file mode 100755 index 0000000..b0e219a --- /dev/null +++ b/tools/run_op.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +# FlagBLAS operator test runner script +# Reference: FlagGems/tools/test-op.sh +# Usage: ./tools/run_op.sh [PR_ID] +# Environment variables: +# CHANGED_FILES - list of changed files (space-separated), or "__ALL__" for full test + +PR_ID=$1 + +# Leave this for debugging's purpose +echo "PR_ID=${PR_ID}" + +COLLECT_COVERAGE="" +FAIL_FAST=false + +if [[ "$CHANGED_FILES" == "__ALL__" ]]; then + # Replace "__ALL__" with all tests + CHANGED_FILES=$(find tests -name "test*.py") + # add options to generate summary report + EXTRA_OPTS="--md-report" + EXTRA_OPTS+=" --md-report-verbose=1" + EXTRA_OPTS+=" --md-report-output=${PR_ID}-summary.md" + SUFFIX="" + COLLECT_COVERAGE="yes" +else + # for per-PR test, fail early + FAIL_FAST=true + EXTRA_OPTS="-x" + SUFFIX="-${GITHUB_SHA::7}" +fi + +# Test cases that needs to run quick cpu tests +NO_QUICK_CPU_TESTS=( + "tests/conftest.py" + "tests/accuracy_utils.py" + "tests/__init__.py" +) + +# Extract test cases from CHANGED_FILES +TEST_CASES=() +PERF_TEST_CASES=() +TEST_CASES_CPU=() +for item in $CHANGED_FILES; do + file_name=$(basename "$item") + case $item in + tests/*.py) + if [[ "$file_name" == test*.py ]]; then + TEST_CASES+=($item) + fi + ;; + benchmark/test*) + PERF_TEST_CASES+=($item) + ;; + esac + + # filter out tests that do not need quick CPU mode tests + found=0 + for item_cpu in "${NO_QUICK_CPU_TESTS[@]}"; do + if [[ "$item" == "$item_cpu" ]]; then + found=1 + break + fi + done + if (( $found == 0 )); then + case $item in + tests/*.py) + if [[ "$file_name" == test*.py ]]; then + TEST_CASES_CPU+=($item) + fi + ;; + esac + fi +done + +# Skip tests if no tests file is found +if [[ ${#TEST_CASES[@]} -eq 0 && ${#PERF_TEST_CASES[@]} -eq 0 ]]; then + exit 0 +fi + +# Clear existing coverage data if any +coverage erase + +FAILURES=() +for item in "${TEST_CASES[@]}"; do + echo "Running unit tests for ${item}" + if ! coverage run -m pytest -s ${EXTRA_OPTS} ${item}; then + if $FAIL_FAST; then exit 1; fi + FAILURES+=("${item}") + fi +done + +# Run quick-cpu test if necessary +for item in "${TEST_CASES_CPU[@]}"; do + echo "Running quick-cpu mode unit tests for ${item}" + if ! coverage run -m pytest -s ${EXTRA_OPTS} ${item} --ref=cpu --quick; then + if $FAIL_FAST; then exit 1; fi + FAILURES+=("${item} (quick-cpu)") + fi +done + +# Run benchmark test if necessary +for item in "${PERF_TEST_CASES[@]}"; do + echo "Running benchmark tests for ${item}" + echo "pytest -s ${item} --level core --record log" + if ! pytest -s ${item} --level core --record log; then + if $FAIL_FAST; then exit 1; fi + FAILURES+=("${item} (benchmark)") + fi +done + +# Process coverage data only when full-range testing +# Coverage data HTML dumped to `htmlcov/` by default +if [ -n "$COLLECT_COVERAGE" ]; then + coverage combine + coverage html + rm -fr coverage + mkdir coverage + mv htmlcov coverage/ + echo "${PR_ID}${SUFFIX::7}" > coverage/COVERAGE_ID + mv ${PR_ID}-summary.md coverage/ut-summary.md +fi + +# Report failures +if [[ ${#FAILURES[@]} -gt 0 ]]; then + echo "" + echo "=== FAILED TESTS (${#FAILURES[@]}) ===" + for f in "${FAILURES[@]}"; do + echo " - ${f}" + done + exit 1 +fi \ No newline at end of file From ac0fe5343fd57589de731a597fb0d8dafc7408d5 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Tue, 16 Jun 2026 11:14:21 +0800 Subject: [PATCH 2/9] add run_op.sh in backend-test.yaml --- .github/workflows/backend-test.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index 0c58f15..addd036 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -17,9 +17,10 @@ on: type: string default: '' test_script: - description: 'Path to test script under tools/' - required: true + description: "Temporary parameter for alpha-ops test" + required: false type: string + default: '' changed_files: description: 'List of changed files in a PR' required: false @@ -75,9 +76,19 @@ jobs: run: | bash ${{ inputs.gpu_check_script }} - - name: Run backend tests + - name: Test alpha ops + if: ${{ inputs.test_script != '' }} env: CHANGED_FILES: ${{ inputs.changed_files }} shell: bash run: | - bash ${{ inputs.test_script }} ${{ inputs.vendor }} + bash ${{ inputs.test_script }} ${{ inputs.vendor }} ${{ inputs.pr_id }} + - name: Run backend tests + if: ${{ inputs.test_script == '' }} + shell: bash + env: + CHANGED_FILES: ${{ inputs.changed_files }} + run: | + source .venv/bin/activate + source tools/set-env.sh ${{ inputs.vendor }} + tools/run_op.sh ${{ inputs.pr_id }} From 4a1143a7f171c0bb1df144e760ba0dcb2966ecb3 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Tue, 16 Jun 2026 11:46:48 +0800 Subject: [PATCH 3/9] change sgemm test --- tests/test_sgemm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sgemm.py b/tests/test_sgemm.py index 1e9bf8f..315b616 100644 --- a/tests/test_sgemm.py +++ b/tests/test_sgemm.py @@ -203,7 +203,7 @@ def test_sgemm_empty(m, n, k): @pytest.mark.sgemm @pytest.mark.parametrize( - "alpha,beta", [(1.0, 0.0), (2.0, 0.0), (2.0, 0.5), (0.0, 1.0), (0.5, 1.5)] + "alpha,beta", [(1.0, 0.0), (2.0, 0.0), (2.0, 0.5), (0.0, 1.0), (0.5, 2.5)] ) def test_sgemm_alpha_beta(alpha, beta): m, n, k = 256, 256, 256 From 9be18b0097a42a34d8bef396b7f9f44d84053c70 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Tue, 16 Jun 2026 16:09:26 +0800 Subject: [PATCH 4/9] fix run_op.sh --- .github/workflows/unittest.yaml | 6 +++++- conf/operators.yaml | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index ed05bc5..ebc40fb 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -12,9 +12,12 @@ jobs: preprocess: runs-on: ubuntu-latest outputs: - labels: ${{ fromJSON(steps.get-labels.outputs.result) }} + changed_files: ${{ steps.changed-files.outputs.all_changed_files }} pr_id: ${{ steps.get-pr-id.outputs.PR_ID }} + labels: ${{ fromJSON(steps.get-labels.outputs.result) }} steps: + - id: changed-files + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - uses: actions/checkout@v4 - id: wait-for-triage @@ -75,3 +78,4 @@ jobs: runner_label: 'h100' gpu_check_script: 'tools/check_nvidia_gpu.sh' pr_id: ${{ needs.preprocess.outputs.pr_id }} + changed_files: ${{ join(needs.preprocess.outputs.changed_files, ' ') }} diff --git a/conf/operators.yaml b/conf/operators.yaml index 28f6912..90ce9a4 100644 --- a/conf/operators.yaml +++ b/conf/operators.yaml @@ -54,7 +54,7 @@ ops: for: - sabs labels: - - abs + - sabs kind: - BLAS stages: @@ -66,7 +66,7 @@ ops: for: - dabs labels: - - abs + - dabs kind: - BLAS stages: @@ -78,7 +78,7 @@ ops: for: - cabs labels: - - abs + - cabs kind: - BLAS stages: From ff987e93295f5c82bd3aff4f75e294c1cfadac07 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Thu, 25 Jun 2026 20:33:40 +0800 Subject: [PATCH 5/9] add coverage in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5d82555..a317da5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ [project.optional-dependencies] test = [ "pytest>=7.1.0", + "coverage==7.13.5", "numpy>=1.26", "scipy>=1.14", "cupy-cuda12x", From f20d1c101b4f5a477f9c089b6485bca505ed1e3b Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Sat, 27 Jun 2026 17:51:19 +0800 Subject: [PATCH 6/9] fix setup vendor --- tools/setup_vendor.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/setup_vendor.sh b/tools/setup_vendor.sh index a9c0349..185a467 100755 --- a/tools/setup_vendor.sh +++ b/tools/setup_vendor.sh @@ -29,16 +29,15 @@ echo "Installing FlagBLAS for ${VENDOR} ..." case $VENDOR in nvidia) # Install PyTorch and Triton with CUDA support - uv pip install --index ${FLAGOS_PYPI} \ - "torch==2.9.1+cu128" \ - "torchvision==0.24.1+cu128" + uv pip install torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1 \ + --index-url https://download.pytorch.org/whl/cu128 # Install FlagBLAS in editable mode - uv pip install -e . - uv pip install ".[test]" uv pip uninstall triton - uv pip install --index ${FLAGOS_PYPI} \ - flagtree==0.5.0 + RES="--index-url=https://resource.flagos.net/repository/flagos-pypi-hosted/simple" + python3.12 -m pip install flagtree===0.5.0 $RES + uv pip install -e . + uv pip install ".[test]" ;; iluvatar) From 62822c4d2a541c9c2fc0e9b9aeaa9e3a90193bb2 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Sat, 27 Jun 2026 18:03:50 +0800 Subject: [PATCH 7/9] fix setup_vendor.sh --- tools/setup_vendor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/setup_vendor.sh b/tools/setup_vendor.sh index 185a467..f547c44 100755 --- a/tools/setup_vendor.sh +++ b/tools/setup_vendor.sh @@ -35,7 +35,7 @@ case $VENDOR in uv pip uninstall triton RES="--index-url=https://resource.flagos.net/repository/flagos-pypi-hosted/simple" - python3.12 -m pip install flagtree===0.5.0 $RES + python3.12 -m uv pip install flagtree===0.5.0 $RES uv pip install -e . uv pip install ".[test]" ;; From 4756c0d44975f8ad784b308bab4462df4def426d Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Sat, 27 Jun 2026 20:21:32 +0800 Subject: [PATCH 8/9] fix setup_vendor.sh --- tools/setup_vendor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/setup_vendor.sh b/tools/setup_vendor.sh index f547c44..185a467 100755 --- a/tools/setup_vendor.sh +++ b/tools/setup_vendor.sh @@ -35,7 +35,7 @@ case $VENDOR in uv pip uninstall triton RES="--index-url=https://resource.flagos.net/repository/flagos-pypi-hosted/simple" - python3.12 -m uv pip install flagtree===0.5.0 $RES + python3.12 -m pip install flagtree===0.5.0 $RES uv pip install -e . uv pip install ".[test]" ;; From 062fc571d1f33bdb726e5db5f7c00dc1998a59f9 Mon Sep 17 00:00:00 2001 From: bin913 <842884726@qq.com> Date: Sat, 27 Jun 2026 21:47:03 +0800 Subject: [PATCH 9/9] fix quick mode error --- tests/conftest.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 08bca5e..25344c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,16 +36,11 @@ def pytest_addoption(parser): help="device to run reference tests on", ) parser.addoption( - ( - "--mode" - if not (flag_blas.vendor_name == "kunlunxin" and torch.__version__ < "2.5") - else "--fg_mode" - ), # TODO: fix pytest-* common --mode args, - action="store", - default="normal", + "--quick", + action="store_true", + default=False, required=False, - choices=["normal", "quick"], - help="run tests on normal or quick mode", + help="run tests on quick mode", ) parser.addoption( "--record", @@ -86,7 +81,7 @@ def pytest_configure(config): print(f"[correctness] reference backend: {ref_backend}", flush=True) global QUICK_MODE - QUICK_MODE = config.getoption("--mode") == "quick" + QUICK_MODE = config.getoption("--quick") global RECORD_LOG RECORD_LOG = config.getoption("--record") == "log"