TransformerEngine-FL/.github/workflows/qa_l1_pytorch_unittest.yml at e2fee067ab91be9a47241c5b4f83387a7f21dfff · zihugithub/TransformerEngine-FL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
name: QA L1 PYTHORCH Unittest

on:
  push:
    branches:
      - add_qa_teat251230
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
  cancel-in-progress: true

jobs:
  qa-l1-pytorch-unittest:
    runs-on: self-hosted
    defaults:
      run:
        shell: bash
    container:
      image: localhost:5000/flagscale:cuda12.8.1-cudnn9.10.2-torch2.9.1-py3.10-transformerengine_fl
      ports:
        - 80:80
      options: >-
        --gpus all
        --shm-size=500g
        --privileged
        --ipc=host
        --ulimit memlock=-1
        --ulimit stack=67108864
        --ulimit nofile=65535:65535
        --user root

    steps:
      - name: Checkout Code
        uses: actions/checkout@v6.0.1
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          ssh-strict: true
          ssh-user: git
          persist-credentials: true
          clean: true
          sparse-checkout-cone-mode: true
          fetch-tags: false
          show-progress: true
          lfs: false
          submodules: recursive
          set-safe-directory: true

      - name: Pre-Check Environment
        timeout-minutes: 5
        run: |
          set -euo pipefail
          echo "===== System Info ====="
          uname -a
          python3 --version || (echo "❌ python3 not found"; exit 1)
          pip3 --version || (echo "❌ pip3 not found"; exit 1)

          echo "===== GPU Info ====="
          if command -v nvidia-smi &>/dev/null; then
            nvidia-smi --query-gpu=index,name,memory.total,utilization.gpu --format=csv
          else
            echo "⚠️ GPU check skipped. nvidia-smi is not available in the current environment."
            exit 1
          fi

          nvcc --version || echo "⚠️ nvcc not found (non-fatal for python-only install)"

          echo "===== Disk Space ====="
          df -h

          echo "===== Available memory ====="
          free -h

      - name: Install dependencies and build transformer_engine
        timeout-minutes: 30
        run: |
          echo "\n=============== Install transformer_engine ==============="
          pip install --no-build-isolation -vvv . --no-deps
          cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/

      - name: Verify installation
        shell: bash
        run: |
          python3 tests/pytorch/test_sanity_import.py

      - name: GPU Usage Check / Verification
        run: |
          source .github/workflows/scripts/gpu_check.sh
          wait_for_gpu

      - name: L1 Pytorch Distributed Unittest
        id: L1_pytorch_distributed_unittest
        continue-on-error: true
        env:
          XML_LOG_DIR: "/logs/pytorch/distributed"
          TE_PATH: .
        run: |
          ignore_files=("tests/pytorch/distributed/test_sanity.py" "tests/pytorch/distributed/test_comm_gemm_overlap.py"
              "tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py" "tests/pytorch/attention/test_attention_with_cp.py"
              "tests/pytorch/debug/test_distributed.py" "tests/pytorch/distributed/test_numerics.py"
          )

          for file in "${ignore_files[@]}"; do
              escaped_file=$(echo "$file" | sed 's/\//\\\//g')
              sed -i "s/^.*\($escaped_file\).*$/#&/" ./qa/L1_pytorch_distributed_unittest/test.sh
          done

          cat ./qa/L1_pytorch_distributed_unittest/test.sh

          bash ./qa/L1_pytorch_distributed_unittest/test.sh

      - name: L1 Pytorch Onnx Unittest
        id: L1_pytorch_onnx_unittest
        continue-on-error: true
        env:
          XML_LOG_DIR: "/logs/pytorch/onnx"
          TE_PATH: .
        run: bash ./qa/L1_pytorch_onnx_unittest/test.sh

      - name: Upload Installation Logs
        if: (failure() && (steps.L1_pytorch_distributed_unittest.outcome == 'failure' || steps.L1_pytorch_onnx_unittest.outcome == 'failure')) && always()
        uses: actions/upload-artifact@v4
        with:
          name: L0_pytorch-logs-${{ github.run_id }}
          path: /logs
          retention-days: 7
          if-no-files-found: warn
        continue-on-error: true