Skip to content

Commit 30d84c5

Browse files
author
yefubao
committed
ci: Add additional tests
1 parent 1090b15 commit 30d84c5

5 files changed

Lines changed: 399 additions & 38 deletions

File tree

‎.github/workflows/qa_format.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Format Check
2+
3+
on:
4+
pull_request:
5+
branches: [ "main" ]
6+
types: [opened, synchronize, reopened]
7+
push:
8+
branches:
9+
- add_qa_teat251230
10+
11+
jobs:
12+
format:
13+
runs-on: ubuntu-22.04
14+
steps:
15+
- name: Checkout Code
16+
uses: actions/checkout@v4
17+
with:
18+
ref: ${{ github.event.pull_request.base.ref }}
19+
20+
- name: Install dependencies
21+
run: |
22+
pip install black==24.4.2 isort==5.13.2 pre-commit==2.17.0
23+
24+
- name: Check Pre-Commit
25+
env:
26+
TE_PATH: .
27+
run: |
28+
set +e
29+
bash -x ./qa/format.sh;EXCODE=$?
30+
exit $EXCODE

‎.github/workflows/qa_l0_pytorch_lint.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
name: QA L0 PYTHORCH Unittest
2+
3+
on:
4+
push:
5+
branches:
6+
- add_qa_teat251230
7+
pull_request:
8+
branches:
9+
- main
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
qa-l0-pytorch-unittest:
17+
runs-on: self-hosted
18+
defaults:
19+
run:
20+
shell: bash
21+
container:
22+
image: localhost:5000/flagscale:cuda12.8.1-cudnn9.10.2-torch2.9.1-py3.10-transformerengine_fl
23+
ports:
24+
- 80:80
25+
options: >-
26+
--gpus all
27+
--shm-size=500g
28+
--privileged
29+
--ipc=host
30+
--ulimit memlock=-1
31+
--ulimit stack=67108864
32+
--ulimit nofile=65535:65535
33+
--user root
34+
35+
steps:
36+
- name: Checkout Code
37+
uses: actions/checkout@v6.0.1
38+
with:
39+
repository: ${{ github.event.pull_request.head.repo.full_name }}
40+
ref: ${{ github.event.pull_request.head.ref }}
41+
ssh-strict: true
42+
ssh-user: git
43+
persist-credentials: true
44+
clean: true
45+
sparse-checkout-cone-mode: true
46+
fetch-tags: false
47+
show-progress: true
48+
lfs: false
49+
submodules: recursive
50+
set-safe-directory: true
51+
52+
- name: Pre-Check Environment
53+
timeout-minutes: 5
54+
run: |
55+
set -euo pipefail
56+
echo "===== System Info ====="
57+
uname -a
58+
python3 --version || (echo "❌ python3 not found"; exit 1)
59+
pip3 --version || (echo "❌ pip3 not found"; exit 1)
60+
61+
echo "===== GPU Info ====="
62+
if command -v nvidia-smi &>/dev/null; then
63+
nvidia-smi --query-gpu=index,name,memory.total,utilization.gpu --format=csv
64+
else
65+
echo "⚠️ GPU check skipped. nvidia-smi is not available in the current environment."
66+
exit 1
67+
fi
68+
69+
nvcc --version || echo "⚠️ nvcc not found (non-fatal for python-only install)"
70+
71+
echo "===== Disk Space ====="
72+
df -h
73+
74+
echo "===== Available memory ====="
75+
free -h
76+
77+
- name: Verify installation
78+
shell: bash
79+
run: |
80+
python3 tests/pytorch/test_sanity_import.py
81+
82+
- name: PyTorch C++ Lint
83+
id: pytorch_cpplint
84+
env:
85+
CPP_ONLY: 1
86+
TE_PATH: .
87+
run: |
88+
bash ./qa/L0_pytorch_lint/test.sh
89+
90+
echo ""
91+
echo "-----------------------------------------------------"
92+
echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
93+
echo "-----------------------------------------------------"
94+
continue-on-error: true
95+
96+
- name: PyTorch Python Lint
97+
id: pytorch_pylint
98+
env:
99+
PYTHON_ONLY: 1
100+
TE_PATH: .
101+
run: |
102+
bash ./qa/L0_pytorch_lint/test.sh
103+
104+
echo ""
105+
echo "-----------------------------------------------------"
106+
echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
107+
echo "-----------------------------------------------------"
108+
continue-on-error: true

.github/workflows/qa_l0_pytorch.yml renamed to .github/workflows/qa_l0_pytorch_unittest.yml

Lines changed: 10 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: QA L0 PYTHORCH
1+
name: QA L0 PYTHORCH Unittest
22

33
on:
44
push:
@@ -13,7 +13,7 @@ concurrency:
1313
cancel-in-progress: true
1414

1515
jobs:
16-
qa-l0-pytorch:
16+
qa-l0-pytorch-unittest:
1717
runs-on: self-hosted
1818
defaults:
1919
run:
@@ -48,7 +48,7 @@ jobs:
4848
lfs: false
4949
submodules: recursive
5050
set-safe-directory: true
51-
51+
5252
- name: Pre-Check Environment
5353
timeout-minutes: 5
5454
run: |
@@ -74,45 +74,17 @@ jobs:
7474
echo "===== Available memory ====="
7575
free -h
7676
77-
# - name: Install dependencies and build transformer_engine
78-
# run: |
79-
# echo "\n=============== Install transformer_engine ==============="
80-
# pip install --no-build-isolation -vvv . --no-deps
81-
# cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/
77+
- name: Install dependencies and build transformer_engine
78+
run: |
79+
echo "\n=============== Install transformer_engine ==============="
80+
pip install --no-build-isolation -vvv . --no-deps
81+
cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/
8282
8383
- name: Verify installation
8484
shell: bash
8585
run: |
8686
python3 tests/pytorch/test_sanity_import.py
8787
88-
- name: PyTorch C++ Lint
89-
id: pytorch_cpplint
90-
env:
91-
CPP_ONLY: 1
92-
TE_PATH: .
93-
run: |
94-
bash ./qa/L0_pytorch_lint/test.sh
95-
96-
echo ""
97-
echo "-----------------------------------------------------"
98-
echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
99-
echo "-----------------------------------------------------"
100-
continue-on-error: true
101-
102-
- name: PyTorch Python Lint
103-
id: pytorch_pylint
104-
env:
105-
PYTHON_ONLY: 1
106-
TE_PATH: .
107-
run: |
108-
bash ./qa/L0_pytorch_lint/test.sh
109-
110-
echo ""
111-
echo "-----------------------------------------------------"
112-
echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
113-
echo "-----------------------------------------------------"
114-
continue-on-error: true
115-
11688
- name: GPU Usage Check / Verification
11789
run: |
11890
source .github/workflows/scripts/gpu_check.sh
@@ -149,15 +121,15 @@ jobs:
149121
150122
for file in "${ignore_files[@]}"; do
151123
escaped_file=$(echo "$file" | sed 's/\//\\\//g')
152-
sed -i 's/^.*\($file\).*$/#&/' ./qa/L0_pytorch_unittest/test.sh
124+
sed -i "s/^.*\($escaped_file\).*$/#&/" ./qa/L0_pytorch_unittest/test.sh
153125
done
154126
155127
cat ./qa/L0_pytorch_unittest/test.sh
156128
157129
bash ./qa/L0_pytorch_unittest/test.sh
158130
159131
- name: Upload Installation Logs
160-
if: failure()
132+
if: (failure() && (steps.L0_pytorch_debug_unittest.outcome == 'failure' || steps.L0_pytorch_unittest.outcome == 'failure')) && always()
161133
uses: actions/upload-artifact@v4
162134
with:
163135
name: L0_pytorch-logs-${{ github.run_id }}

‎.github/workflows/qa_l1_pytorch_unittest.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: QA L1 PYTHORCH Unittest
2+
3+
on:
4+
push:
5+
branches:
6+
- add_qa_teat251230
7+
pull_request:
8+
branches:
9+
- main
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
qa-l1-pytorch-unittest:
17+
runs-on: self-hosted
18+
defaults:
19+
run:
20+
shell: bash
21+
container:
22+
image: localhost:5000/flagscale:cuda12.8.1-cudnn9.10.2-torch2.9.1-py3.10-transformerengine_fl
23+
ports:
24+
- 80:80
25+
options: >-
26+
--gpus all
27+
--shm-size=500g
28+
--privileged
29+
--ipc=host
30+
--ulimit memlock=-1
31+
--ulimit stack=67108864
32+
--ulimit nofile=65535:65535
33+
--user root
34+
35+
steps:
36+
- name: Checkout Code
37+
uses: actions/checkout@v6.0.1
38+
with:
39+
repository: ${{ github.event.pull_request.head.repo.full_name }}
40+
ref: ${{ github.event.pull_request.head.ref }}
41+
ssh-strict: true
42+
ssh-user: git
43+
persist-credentials: true
44+
clean: true
45+
sparse-checkout-cone-mode: true
46+
fetch-tags: false
47+
show-progress: true
48+
lfs: false
49+
submodules: recursive
50+
set-safe-directory: true
51+
52+
- name: Pre-Check Environment
53+
timeout-minutes: 5
54+
run: |
55+
set -euo pipefail
56+
echo "===== System Info ====="
57+
uname -a
58+
python3 --version || (echo "❌ python3 not found"; exit 1)
59+
pip3 --version || (echo "❌ pip3 not found"; exit 1)
60+
61+
echo "===== GPU Info ====="
62+
if command -v nvidia-smi &>/dev/null; then
63+
nvidia-smi --query-gpu=index,name,memory.total,utilization.gpu --format=csv
64+
else
65+
echo "⚠️ GPU check skipped. nvidia-smi is not available in the current environment."
66+
exit 1
67+
fi
68+
69+
nvcc --version || echo "⚠️ nvcc not found (non-fatal for python-only install)"
70+
71+
echo "===== Disk Space ====="
72+
df -h
73+
74+
echo "===== Available memory ====="
75+
free -h
76+
77+
- name: Install dependencies and build transformer_engine
78+
run: |
79+
echo "\n=============== Install transformer_engine ==============="
80+
pip install --no-build-isolation -vvv . --no-deps
81+
cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/
82+
83+
- name: Verify installation
84+
shell: bash
85+
run: |
86+
python3 tests/pytorch/test_sanity_import.py
87+
88+
- name: GPU Usage Check / Verification
89+
run: |
90+
source .github/workflows/scripts/gpu_check.sh
91+
wait_for_gpu
92+
93+
- name: L1 Pytorch Distributed Unittest
94+
id: L1_pytorch_distributed_unittest
95+
continue-on-error: true
96+
env:
97+
XML_LOG_DIR: "/logs/pytorch/distributed"
98+
TE_PATH: .
99+
run: |
100+
ignore_files=("tests/pytorch/distributed/test_sanity.py" "tests/pytorch/distributed/test_comm_gemm_overlap.py"
101+
"tests/pytorch/distributed/test_fusible_ops_with_userbuffers.py" "tests/pytorch/attention/test_attention_with_cp.py"
102+
"tests/pytorch/debug/test_distributed.py" "tests/pytorch/distributed/test_numerics.py"
103+
)
104+
105+
for file in "${ignore_files[@]}"; do
106+
escaped_file=$(echo "$file" | sed 's/\//\\\//g')
107+
sed -i "s/^.*\($escaped_file\).*$/#&/" ./qa/L1_pytorch_distributed_unittest/test.sh
108+
done
109+
110+
cat ./qa/L1_pytorch_distributed_unittest/test.sh
111+
112+
bash ./qa/L1_pytorch_distributed_unittest/test.sh
113+
114+
- name: L1 Pytorch Onnx Unittest
115+
id: L1_pytorch_onnx_unittest
116+
# continue-on-error: true
117+
env:
118+
XML_LOG_DIR: "/logs/pytorch/onnx"
119+
TE_PATH: .
120+
run: bash ./qa/L1_pytorch_onnx_unittest/test.sh
121+
122+
- name: Upload Installation Logs
123+
if: (failure() && (steps.L1_pytorch_distributed_unittest.outcome == 'failure' || steps.L1_pytorch_onnx_unittest.outcome == 'failure')) && always()
124+
uses: actions/upload-artifact@v4
125+
with:
126+
name: L0_pytorch-logs-${{ github.run_id }}
127+
path: /logs
128+
retention-days: 7
129+
if-no-files-found: warn
130+
continue-on-error: true

0 commit comments

Comments
 (0)