Add run_op.sh in backend_test.yaml for cicd

.github/workflows/backend-test.yaml

@@ -17,9 +17,10 @@ on:
         type: string
         default: ''
       test_script:
-        description: 'Path to test script under tools/'
-        required: true
+        description: "Temporary parameter for alpha-ops test"
+        required: false
         type: string
+        default: ''
       changed_files:
         description: 'List of changed files in a PR'
         required: false
@@ -75,9 +76,19 @@ jobs:
         run: |
           bash ${{ inputs.gpu_check_script }}
 
-      - name: Run backend tests
+      - name: Test alpha ops
+        if: ${{ inputs.test_script != '' }}
         env:
           CHANGED_FILES: ${{ inputs.changed_files }}
         shell: bash
         run: |
-          bash ${{ inputs.test_script }} ${{ inputs.vendor }}
+          bash ${{ inputs.test_script }} ${{ inputs.vendor }} ${{ inputs.pr_id }}
+      - name: Run backend tests
+        if: ${{ inputs.test_script == '' }}
+        shell: bash
+        env:
+          CHANGED_FILES: ${{ inputs.changed_files }}
+        run: |
+          source .venv/bin/activate
+          source tools/set-env.sh ${{ inputs.vendor }}
+          tools/run_op.sh ${{ inputs.pr_id }}

.github/workflows/unittest.yaml

@@ -12,9 +12,12 @@ jobs:
   preprocess:
     runs-on: ubuntu-latest
     outputs:
-      labels: ${{ fromJSON(steps.get-labels.outputs.result) }}
+      changed_files: ${{ steps.changed-files.outputs.all_changed_files }}
       pr_id: ${{ steps.get-pr-id.outputs.PR_ID }}
+      labels: ${{ fromJSON(steps.get-labels.outputs.result) }}
     steps:
+      - id: changed-files
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96  # v47.0.6
       - uses: actions/checkout@v4
 
       - id: wait-for-triage
@@ -75,3 +78,4 @@ jobs:
       runner_label: 'h100'
       gpu_check_script: 'tools/check_nvidia_gpu.sh'
       pr_id: ${{ needs.preprocess.outputs.pr_id }}
+      changed_files: ${{ join(needs.preprocess.outputs.changed_files, ' ') }}

conf/operators.yaml

@@ -54,7 +54,7 @@ ops:
     for:
       - sabs
     labels:
-      - abs
+      - sabs
     kind:
       - BLAS
     stages:
@@ -66,7 +66,7 @@ ops:
     for:
       - dabs
     labels:
-      - abs
+      - dabs
     kind:
       - BLAS
     stages:
@@ -78,7 +78,7 @@ ops:
     for:
       - cabs
     labels:
-      - abs
+      - cabs
     kind:
       - BLAS
     stages:

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
 [project.optional-dependencies]
 test = [
     "pytest>=7.1.0",
+    "coverage==7.13.5",
     "numpy>=1.26",
     "scipy>=1.14",
     "cupy-cuda12x",

tests/conftest.py

@@ -36,16 +36,11 @@ def pytest_addoption(parser):
         help="device to run reference tests on",
     )
     parser.addoption(
-        (
-            "--mode"
-            if not (flag_blas.vendor_name == "kunlunxin" and torch.__version__ < "2.5")
-            else "--fg_mode"
-        ),  # TODO: fix pytest-* common --mode args,
-        action="store",
-        default="normal",
+        "--quick",
+        action="store_true",
+        default=False,
         required=False,
-        choices=["normal", "quick"],
-        help="run tests on normal or quick mode",
+        help="run tests on quick mode",
     )
     parser.addoption(
         "--record",
@@ -86,7 +81,7 @@ def pytest_configure(config):
     print(f"[correctness] reference backend: {ref_backend}", flush=True)
 
     global QUICK_MODE
-    QUICK_MODE = config.getoption("--mode") == "quick"
+    QUICK_MODE = config.getoption("--quick")
 
     global RECORD_LOG
     RECORD_LOG = config.getoption("--record") == "log"

tests/test_sgemm.py

@@ -203,7 +203,7 @@ def test_sgemm_empty(m, n, k):
 
 @pytest.mark.sgemm
 @pytest.mark.parametrize(
-    "alpha,beta", [(1.0, 0.0), (2.0, 0.0), (2.0, 0.5), (0.0, 1.0), (0.5, 1.5)]
+    "alpha,beta", [(1.0, 0.0), (2.0, 0.0), (2.0, 0.5), (0.0, 1.0), (0.5, 2.5)]
 )
 def test_sgemm_alpha_beta(alpha, beta):
     m, n, k = 256, 256, 256

tools/run_op.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+# FlagBLAS operator test runner script
+# Reference: FlagGems/tools/test-op.sh
+# Usage: ./tools/run_op.sh [PR_ID]
+# Environment variables:
+#   CHANGED_FILES - list of changed files (space-separated), or "__ALL__" for full test
+
+PR_ID=$1
+
+# Leave this for debugging's purpose
+echo "PR_ID=${PR_ID}"
+
+COLLECT_COVERAGE=""
+FAIL_FAST=false
+
+if [[ "$CHANGED_FILES" == "__ALL__" ]]; then
+  # Replace "__ALL__" with all tests
+  CHANGED_FILES=$(find tests -name "test*.py")
+  # add options to generate summary report
+  EXTRA_OPTS="--md-report"
+  EXTRA_OPTS+=" --md-report-verbose=1"
+  EXTRA_OPTS+=" --md-report-output=${PR_ID}-summary.md"
+  SUFFIX=""
+  COLLECT_COVERAGE="yes"
+else
+  # for per-PR test, fail early
+  FAIL_FAST=true
+  EXTRA_OPTS="-x"
+  SUFFIX="-${GITHUB_SHA::7}"
+fi
+
+# Test cases that needs to run quick cpu tests
+NO_QUICK_CPU_TESTS=(
+  "tests/conftest.py"
+  "tests/accuracy_utils.py"
+  "tests/__init__.py"
+)
+
+# Extract test cases from CHANGED_FILES
+TEST_CASES=()
+PERF_TEST_CASES=()
+TEST_CASES_CPU=()
+for item in $CHANGED_FILES; do
+  file_name=$(basename "$item")
+  case $item in
+    tests/*.py)
+      if [[ "$file_name" == test*.py ]]; then
+        TEST_CASES+=($item)
+      fi
+      ;;
+    benchmark/test*)
+      PERF_TEST_CASES+=($item)
+      ;;
+  esac
+
+  # filter out tests that do not need quick CPU mode tests
+  found=0
+  for item_cpu in "${NO_QUICK_CPU_TESTS[@]}"; do
+    if [[ "$item" == "$item_cpu" ]]; then
+      found=1
+      break
+    fi
+  done
+  if (( $found == 0 )); then
+    case $item in
+      tests/*.py)
+        if [[ "$file_name" == test*.py ]]; then
+          TEST_CASES_CPU+=($item)
+        fi
+        ;;
+    esac
+  fi
+done
+
+# Skip tests if no tests file is found
+if [[ ${#TEST_CASES[@]} -eq 0  && ${#PERF_TEST_CASES[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+# Clear existing coverage data if any
+coverage erase
+
+FAILURES=()
+for item in "${TEST_CASES[@]}"; do
+  echo "Running unit tests for ${item}"
+  if ! coverage run -m pytest -s ${EXTRA_OPTS} ${item}; then
+    if $FAIL_FAST; then exit 1; fi
+    FAILURES+=("${item}")
+  fi
+done
+
+# Run quick-cpu test if necessary
+for item in "${TEST_CASES_CPU[@]}"; do
+  echo "Running quick-cpu mode unit tests for ${item}"
+  if ! coverage run -m pytest -s ${EXTRA_OPTS} ${item} --ref=cpu --quick; then
+    if $FAIL_FAST; then exit 1; fi
+    FAILURES+=("${item} (quick-cpu)")
+  fi
+done
+
+# Run benchmark test if necessary
+for item in "${PERF_TEST_CASES[@]}"; do
+  echo "Running benchmark tests for ${item}"
+  echo "pytest -s ${item} --level core --record log"
+  if ! pytest -s ${item} --level core --record log; then
+    if $FAIL_FAST; then exit 1; fi
+    FAILURES+=("${item} (benchmark)")
+  fi
+done
+
+# Process coverage data only when full-range testing
+# Coverage data HTML dumped to `htmlcov/` by default
+if [ -n "$COLLECT_COVERAGE" ]; then
+  coverage combine
+  coverage html
+  rm -fr coverage
+  mkdir coverage
+  mv htmlcov coverage/
+  echo "${PR_ID}${SUFFIX::7}" > coverage/COVERAGE_ID
+  mv ${PR_ID}-summary.md coverage/ut-summary.md
+fi
+
+# Report failures
+if [[ ${#FAILURES[@]} -gt 0 ]]; then
+  echo ""
+  echo "=== FAILED TESTS (${#FAILURES[@]}) ==="
+  for f in "${FAILURES[@]}"; do
+    echo "  - ${f}"
+  done
+  exit 1
+fi

tools/setup_vendor.sh

@@ -29,16 +29,15 @@ echo "Installing FlagBLAS for ${VENDOR} ..."
 case $VENDOR in
   nvidia)
     # Install PyTorch and Triton with CUDA support
-    uv pip install --index ${FLAGOS_PYPI} \
-        "torch==2.9.1+cu128" \
-        "torchvision==0.24.1+cu128" 
+    uv pip install torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1 \
+        --index-url https://download.pytorch.org/whl/cu128
     # Install FlagBLAS in editable mode
-    uv pip install -e .
-    uv pip install ".[test]"
 
     uv pip uninstall triton
-    uv pip install --index ${FLAGOS_PYPI} \
-        flagtree==0.5.0
+    RES="--index-url=https://resource.flagos.net/repository/flagos-pypi-hosted/simple"
+    python3.12 -m pip install flagtree===0.5.0 $RES
+    uv pip install -e .
+    uv pip install ".[test]"
     ;;
 
   iluvatar)