qa-l0-te-cpp-unittest-pytorch-lint.yml

name: QA L0 - Core Unit & Lint Tests

on:
  push:
    branches: main
    paths:
      - '.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml'
      - 'qa/L0_pytorch_lint/**'
      - 'transformer_engine/**'
      - 'tests/pytorch/**'
  pull_request:
    branches: main
    paths:
      - '.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml'
      - 'qa/L0_pytorch_lint/**'
      - 'transformer_engine/**'
      - 'tests/pytorch/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
  cancel-in-progress: true

jobs:
  run-qa-l0-core-tests:
    runs-on: [ self-hosted, Linux, X64, nvidia, gpu-8 ]
    defaults:
      run:
        shell: bash
    container:
      image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
      ports:
        - 80:80
      options: >-
        --gpus all 
        --shm-size=500g 
        --privileged 
        --ipc=host 
        --ulimit memlock=-1 
        --ulimit stack=67108864 
        --ulimit nofile=65535:65535 
        --user root
        --pull always
    steps:
      - name: Checkout Code
        uses: actions/checkout@v6.0.1
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: recursive
          set-safe-directory: true

      - name: Install Dependencies & Build Transformer Engine
        # timeout-minutes: 40
        env:
          NVTE_FRAMEWORK: pytorch
          TE_WITH_NCCL: 1
        run: |
          # Activate conda environment
          echo "=== Activating Conda Environment ==="
          source /opt/miniconda3/etc/profile.d/conda.sh
          conda activate flagscale-train

          # Install Python dependencies with version pinning
          echo "=== Installing Python Dependencies ==="
          pip install transformers expecttest

          # Build and install transformer_engine with verbose output
          echo "=== Building & Installing Transformer Engine ==="
          pip install --no-build-isolation -vvv . --no-deps

          # Verify TE installation with version check
          echo "=== Verifying Transformer Engine Installation ==="
          python3 tests/pytorch/test_sanity_import.py

      - name: Verify GPU Availability & Health
        run: |
          # Execute GPU check
          echo "=== Checking GPU Status ==="
          source .github/workflows/scripts/gpu_check.sh
          wait_for_gpu

      - name: Run L0 C++ Unit Tests
        # timeout-minutes: 60
        env:
          TE_PATH: .
        run: |
          # Activate conda environment
          source /opt/miniconda3/etc/profile.d/conda.sh
          conda activate flagscale-train

          # Get TE library paths with robust detection
          TE_LIB_PATH=$(pip3 show transformer-engine | grep -E "Location:|Editable project location:" | tail -n 1 | awk '{print $NF}')
          TE_CPP_LIB_PATH="${TE_LIB_PATH}/transformer_engine"

          # Set environment variables for build
          export CMAKE_PREFIX_PATH="${TE_CPP_LIB_PATH}:${CMAKE_PREFIX_PATH}"
          export LD_LIBRARY_PATH="${TE_CPP_LIB_PATH}:${LD_LIBRARY_PATH}"
          NUM_PHYSICAL_CORES=$(nproc)
          NUM_PARALLEL_JOBS=4

          # Build and run C++ tests
          cd $TE_PATH/tests/cpp
          cmake -GNinja -Bbuild . -DTE_LIB_PATH="${TE_CPP_LIB_PATH}"
          cmake --build build
          export OMP_NUM_THREADS=$((NUM_PHYSICAL_CORES / NUM_PARALLEL_JOBS))

          # Run C++ tests with verbose output
          echo "=== Running C++ Unit Tests ==="
          # ctest --test-dir build -j$NUM_PARALLEL_JOBS

      - name: PyTorch C++ Lint
        # timeout-minutes: 5
        env:
          CPP_ONLY: 1
          TE_PATH: .
        run: |
          # Activate conda environment
          source /opt/miniconda3/etc/profile.d/conda.sh
          conda activate flagscale-train

          # Run C++ lint checks
          echo "=== Running C++ Lint Checks ==="
          bash ./qa/L0_pytorch_lint/test.sh || true

          echo ""
          echo "-----------------------------------------------------"
          echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
          echo "-----------------------------------------------------"
        continue-on-error: true

      - name: PyTorch Python Lint
        # timeout-minutes: 5
        env:
          PYTHON_ONLY: 1
          TE_PATH: .
        run: |
          # Activate conda environment
          source /opt/miniconda3/etc/profile.d/conda.sh
          conda activate flagscale-train

          # Run PyTorch lint checks
          echo "=== Running PyTorch Lint Checks ==="
          bash ./qa/L0_pytorch_lint/test.sh || true

          echo ""
          echo "-----------------------------------------------------"
          echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
          echo "-----------------------------------------------------"
        continue-on-error: true

      - name: Run L0 PyTorch Debug Unit Tests
        # timeout-minutes: 10
        env:
          TE_PATH: .
        run: |
          # Activate conda environment
          source /opt/miniconda3/etc/profile.d/conda.sh
          conda activate flagscale-train

          # Run debug unit tests
          echo "=== Running L0 PyTorch Debug Unit Tests ==="
          bash ./qa/L0_pytorch_debug_unittest/test.sh

      - name: Run L0 PyTorch Core Unit Tests
        # timeout-minutes: 10
        env:
          TE_PATH: .
        run: |
          # Activate conda environment
          source /opt/miniconda3/etc/profile.d/conda.sh
          conda activate flagscale-train

          # Run core unit tests
          echo "=== Running L0 PyTorch Core Unit Tests ==="
          bash ./qa/L0_pytorch_unittest/test.sh