qa_l1_pytorch_unittest.yml

name: QA L1 PYTHORCH Unittest

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
  cancel-in-progress: true

jobs:
  qa-l1-pytorch-unittest:
    runs-on: self-hosted
    defaults:
      run:
        shell: bash
    container:
      image: harbor.baai.ac.cn/flagscale/cuda12.8.1-cudnn9.10.2-torch2.9.1-py3.10-transformerengine_fl:20260204
      ports:
        - 80:80
      options: >-
        --gpus all 
        --shm-size=500g 
        --privileged 
        --ipc=host 
        --ulimit memlock=-1 
        --ulimit stack=67108864 
        --ulimit nofile=65535:65535 
        --user root
        --pull always

    steps:
      - name: Checkout Code
        uses: actions/checkout@v6.0.1
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: recursive
          set-safe-directory: true
      
      - name: Pre-Check Environment
        timeout-minutes: 5
        run: |
          set -euo pipefail
          echo "===== System Info ====="
          uname -a
          python3 --version || (echo "❌ python3 not found"; exit 1)
          pip3 --version || (echo "❌ pip3 not found"; exit 1)

          echo "===== GPU Info ====="
          if command -v nvidia-smi &>/dev/null; then
            nvidia-smi --query-gpu=index,name,memory.total,utilization.gpu --format=csv
          else
            echo "⚠️ GPU check skipped. nvidia-smi is not available in the current environment."
            exit 1
          fi

          nvcc --version || echo "⚠️ nvcc not found (non-fatal for python-only install)"

          echo "===== Disk Space ====="
          df -h

          echo "===== Available memory ====="
          free -h

      - name: Install dependencies and build transformer_engine
        timeout-minutes: 30
        run: |
          echo "\n=============== Install transformer_engine ==============="
          pip install --no-build-isolation -vvv . --no-deps
          cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/

      - name: Verify installation
        shell: bash
        run: |
          python3 tests/pytorch/test_sanity_import.py

      - name: GPU Usage Check / Verification
        run: |
          source .github/workflows/scripts/gpu_check.sh
          wait_for_gpu

      - name: L1 Pytorch Distributed Unittest
        id: L1_pytorch_distributed_unittest
        continue-on-error: true
        env:
          XML_LOG_DIR: "/logs/pytorch/distributed"
          TE_PATH: .
        run: |
          ignore_files=("tests/pytorch/distributed/test_sanity.py" "tests/pytorch/distributed/test_comm_gemm_overlap.py"
              "tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py" "tests/pytorch/attention/test_attention_with_cp.py"
              "tests/pytorch/debug/test_distributed.py" "tests/pytorch/distributed/test_numerics.py"
          )

          for file in "${ignore_files[@]}"; do
              escaped_file=$(echo "$file" | sed 's/\//\\\//g')
              sed -i "s/^.*\($escaped_file\).*$/#&/" ./qa/L1_pytorch_distributed_unittest/test.sh
          done

          cat ./qa/L1_pytorch_distributed_unittest/test.sh

          bash ./qa/L1_pytorch_distributed_unittest/test.sh

      - name: L1 Pytorch Onnx Unittest
        id: L1_pytorch_onnx_unittest
        env:
          XML_LOG_DIR: "/logs/pytorch/onnx"
          TE_PATH: .
        run: |
          # Test items that need to be skipped
          SKIP_TESTS=( 
            "test_trt_integration[None]"
          )

          # Execution file path
          TARGET_FILE="./qa/L1_pytorch_onnx_unittest/test.sh"
          OLD_CMD="python3 -m pytest --tb=auto --junitxml=\$XML_LOG_DIR/test_onnx_export.xml \$TE_PATH/tests/pytorch/test_onnx_export.py"

          SKIP_PARAM=""
          if [ ${#SKIP_TESTS[@]} -gt 0 ]; then
            SKIP_EXPR=""
            for test in "${SKIP_TESTS[@]}"; do
              if [ -z "$SKIP_EXPR" ]; then
                SKIP_EXPR="not $test"
              else
                SKIP_EXPR+=" and not $test"
              fi
            done
            SKIP_PARAM="-k \"${SKIP_EXPR}\""
          fi

          CMD="${OLD_CMD} ${SKIP_PARAM}"
          sed -i "s|${OLD_CMD}|${CMD}|g" "${TARGET_FILE}"

          echo ""
          cat ./qa/L1_pytorch_onnx_unittest/test.sh
          echo ""

          bash ./qa/L1_pytorch_onnx_unittest/test.sh

      - name: Upload Installation Logs
        if: always() && (steps.L1_pytorch_distributed_unittest.outcome == 'failure' || steps.L1_pytorch_onnx_unittest.outcome == 'failure')
        uses: actions/upload-artifact@v4
        with:
          name: L1_pytorch-logs-${{ github.run_id }}
          path: /logs/pytorch
          retention-days: 7
          if-no-files-found: warn