Skip to content

Commit 6f9d881

Browse files
author
yefubao
committed
ci: update
1 parent e13a9a5 commit 6f9d881

8 files changed

Lines changed: 154 additions & 135 deletions

‎.github/workflows/qa_cpp_distributed.yml‎

Lines changed: 131 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,132 @@
1-
# name: QA CPP Distributed
2-
3-
# on:
4-
# push:
5-
# branches:
6-
# - main
7-
# pull_request:
8-
# branches:
9-
# - main
10-
11-
# concurrency:
12-
# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
13-
# cancel-in-progress: true
14-
15-
# jobs:
16-
# qa-l0-cpp_distributed:
17-
# runs-on: self-hosted
18-
# defaults:
19-
# run:
20-
# shell: bash
21-
# container:
22-
# image: localhost:5000/flagscale:cuda12.8.1-cudnn9.10.2-torch2.9.1-py3.10-transformerengine_fl
23-
# ports:
24-
# - 80:80
25-
# options: >-
26-
# --gpus all
27-
# --shm-size=500g
28-
# --privileged
29-
# --ipc=host
30-
# --ulimit memlock=-1
31-
# --ulimit stack=67108864
32-
# --ulimit nofile=65535:65535
33-
# --user root
34-
35-
# steps:
36-
# - name: Checkout Code
37-
# uses: actions/checkout@v6.0.1
38-
# with:
39-
# repository: ${{ github.event.pull_request.head.repo.full_name }}
40-
# ref: ${{ github.event.pull_request.head.ref }}
41-
# ssh-strict: true
42-
# ssh-user: git
43-
# persist-credentials: true
44-
# clean: true
45-
# sparse-checkout-cone-mode: true
46-
# fetch-tags: false
47-
# show-progress: true
48-
# lfs: false
49-
# submodules: recursive
50-
# set-safe-directory: true
51-
52-
# - name: Pre-Check Environment
53-
# timeout-minutes: 5
54-
# run: |
55-
# set -euo pipefail
56-
# echo "===== System Info ====="
57-
# uname -a
58-
# python3 --version || (echo "❌ python3 not found"; exit 1)
59-
# pip3 --version || (echo "❌ pip3 not found"; exit 1)
60-
61-
# echo "===== GPU Info ====="
62-
# if command -v nvidia-smi &>/dev/null; then
63-
# nvidia-smi --query-gpu=index,name,memory.total,utilization.gpu --format=csv
64-
# else
65-
# echo "⚠️ GPU check skipped. nvidia-smi is not available in the current environment."
66-
# exit 1
67-
# fi
68-
69-
# nvcc --version || echo "⚠️ nvcc not found (non-fatal for python-only install)"
70-
71-
# echo "===== Disk Space ====="
72-
# df -h
73-
74-
# echo "===== Available memory ====="
75-
# free -h
76-
77-
# - name: Install dependencies and build transformer_engine
78-
# # timeout-minutes: 30
79-
# env:
80-
# NVTE_FRAMEWORK: pytorch
81-
# TE_WITH_NCCL: 1
82-
# run: |
83-
# # =============== Install MPI ===============
84-
# apt update
85-
# apt install -y libopenmpi-dev openmpi-bin openmpi-common
86-
# apt install -y libmpich-dev mpich
87-
# # Verify the MPI header file
88-
# mpicxx -show | awk '{for(i=1;i<=NF;i++) if($i ~ /-I/) print substr($i,3)}'
89-
90-
# # Verify whether the MPI C++ environment is ready
91-
# # 1. Verify whether the MPI C++ compiler (mpicxx) exists
92-
# mpicxx --version
93-
# # 2. Verify if the MPI library file exists
94-
# ls /usr/lib/x86_64-linux-gnu/libmpi_cxx.so
95-
96-
# echo "=============== Install transformer_engine ==============="
97-
# pip install --no-build-isolation -vvv . --no-deps
98-
# cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/
99-
100-
# - name: Verify installation
101-
# shell: bash
102-
# run: |
103-
# python3 tests/pytorch/test_sanity_import.py
104-
105-
# - name: GPU Usage Check / Verification
106-
# run: |
107-
# source .github/workflows/scripts/gpu_check.sh
108-
# wait_for_gpu
1+
name: QA CPP Distributed
2+
3+
on:
4+
push:
5+
branches:
6+
- add_qa_teat251230
7+
pull_request:
8+
branches:
9+
- main
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
qa-l0-cpp_distributed:
17+
runs-on: self-hosted
18+
defaults:
19+
run:
20+
shell: bash
21+
container:
22+
image: localhost:5000/flagscale:cuda12.8.1-cudnn9.10.2-torch2.9.1-py3.10-transformerengine_fl
23+
ports:
24+
- 80:80
25+
options: >-
26+
--gpus all
27+
--shm-size=500g
28+
--privileged
29+
--ipc=host
30+
--ulimit memlock=-1
31+
--ulimit stack=67108864
32+
--ulimit nofile=65535:65535
33+
--user root
34+
35+
steps:
36+
- name: Checkout Code
37+
uses: actions/checkout@v6.0.1
38+
with:
39+
repository: ${{ github.event.pull_request.head.repo.full_name }}
40+
ref: ${{ github.event.pull_request.head.ref }}
41+
ssh-strict: true
42+
ssh-user: git
43+
persist-credentials: true
44+
clean: true
45+
sparse-checkout-cone-mode: true
46+
fetch-tags: false
47+
show-progress: true
48+
lfs: false
49+
submodules: recursive
50+
set-safe-directory: true
51+
52+
- name: Pre-Check Environment
53+
timeout-minutes: 5
54+
run: |
55+
set -euo pipefail
56+
echo "===== System Info ====="
57+
uname -a
58+
python3 --version || (echo "❌ python3 not found"; exit 1)
59+
pip3 --version || (echo "❌ pip3 not found"; exit 1)
60+
61+
echo "===== GPU Info ====="
62+
if command -v nvidia-smi &>/dev/null; then
63+
nvidia-smi --query-gpu=index,name,memory.total,utilization.gpu --format=csv
64+
else
65+
echo "⚠️ GPU check skipped. nvidia-smi is not available in the current environment."
66+
exit 1
67+
fi
68+
69+
nvcc --version || echo "⚠️ nvcc not found (non-fatal for python-only install)"
70+
71+
echo "===== Disk Space ====="
72+
df -h
73+
74+
echo "===== Available memory ====="
75+
free -h
76+
77+
- name: Install dependencies and build transformer_engine
78+
# timeout-minutes: 30
79+
env:
80+
NVTE_FRAMEWORK: pytorch
81+
TE_WITH_NCCL: 1
82+
run: |
83+
# =============== Install MPI ===============
84+
apt update
85+
apt install -y libopenmpi-dev openmpi-bin openmpi-common
86+
apt install -y libmpich-dev mpich
87+
# Verify the MPI header file
88+
mpicxx -show | awk '{for(i=1;i<=NF;i++) if($i ~ /-I/) print substr($i,3)}'
89+
90+
# Verify whether the MPI C++ environment is ready
91+
# 1. Verify whether the MPI C++ compiler (mpicxx) exists
92+
mpicxx --version
93+
# 2. Verify if the MPI library file exists
94+
ls /usr/lib/x86_64-linux-gnu/libmpi_cxx.so
95+
96+
echo "=============== Install transformer_engine ==============="
97+
pip install --no-build-isolation -vvv . --no-deps
98+
cp -rf transformer_engine/plugins/ /usr/local/lib/python3.10/dist-packages/transformer_engine/plugins/
99+
100+
- name: Verify installation
101+
shell: bash
102+
run: |
103+
python3 tests/pytorch/test_sanity_import.py
104+
105+
- name: GPU Usage Check / Verification
106+
run: |
107+
source .github/workflows/scripts/gpu_check.sh
108+
wait_for_gpu
109109
110-
# - name: L0 CPP Distributed
111-
# id: L0_cpp_distributed
112-
# continue-on-error: true
113-
# # timeout-minutes: 10
114-
# env:
115-
# TE_PATH: .
116-
# run: |
117-
# TE_LIB_PATH=$(pip3 show transformer-engine | grep -E "Location:|Editable project location:" | tail -n 1 | awk '{print $NF}')
118-
# TE_CPP_LIB_PATH="${TE_LIB_PATH}/transformer_engine"
119-
# export CMAKE_PREFIX_PATH="${TE_CPP_LIB_PATH}:${CMAKE_PREFIX_PATH}"
120-
# export LD_LIBRARY_PATH="${TE_CPP_LIB_PATH}:${LD_LIBRARY_PATH}"
121-
122-
# cd $TE_PATH/tests/cpp_distributed
123-
# cmake -GNinja -S. -Bbuild
110+
- name: L0 CPP Distributed
111+
id: L0_cpp_distributed
112+
# timeout-minutes: 10
113+
env:
114+
TE_PATH: .
115+
XML_LOG_DIR: "/logs/cpp/distributed
116+
117+
run: |
118+
TE_LIB_PATH=$(pip3 show transformer-engine | grep -E "Location:|Editable project location:" | tail -n 1 | awk '{print $NF}')
119+
TE_CPP_LIB_PATH="${TE_LIB_PATH}/transformer_engine"
120+
export CMAKE_PREFIX_PATH="${TE_CPP_LIB_PATH}:${CMAKE_PREFIX_PATH}"
121+
export LD_LIBRARY_PATH="${TE_CPP_LIB_PATH}:${LD_LIBRARY_PATH}"
122+
123+
bash ./qa/L1_cpp_distributed/test.sh | tee ${XML_LOG_DIR}/distributed-${{ github.run_id }}.log
124+
125+
- name: Upload Installation Logs
126+
if: always() && steps.L0_cpp_distributed.outcome == 'failure'
127+
uses: actions/upload-artifact@v4
128+
with:
129+
name: L0-cpp-logs-${{ github.run_id }}
130+
path: /logs/cpp/distributed
131+
retention-days: 7
132+
if-no-files-found: warn

‎.github/workflows/qa_cppunittes.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: QA CPP Unittest
33
on:
44
push:
55
branches:
6-
- main
6+
- add_qa_teat251230
77
pull_request:
88
branches:
99
- main

‎.github/workflows/qa_l0_pytorch_lint.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: QA L0 PYTHORCH Lint
33
on:
44
push:
55
branches:
6-
- main
6+
- add_qa_teat251230
77
pull_request:
88
branches:
99
- main
@@ -105,4 +105,3 @@ jobs:
105105
echo "-----------------------------------------------------"
106106
echo "Note: Pylint check ignores errors C0411 (incorrect import position) and W0611 (unused import), which can be achieved by adding the parameter --disable=C0411,W0611"
107107
echo "-----------------------------------------------------"
108-
continue-on-error: true

‎.github/workflows/qa_l0_pytorch_unittest.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: QA L0 PYTHORCH Unittest
33
on:
44
push:
55
branches:
6-
- main
6+
- add_qa_teat251230
77
pull_request:
88
branches:
99
- main
@@ -135,6 +135,6 @@ jobs:
135135
uses: actions/upload-artifact@v4
136136
with:
137137
name: L0_pytorch-logs-${{ github.run_id }}
138-
path: /logs
138+
path: /logs/pytorch
139139
retention-days: 7
140140
if-no-files-found: warn

‎.github/workflows/qa_l0_pytorch_wheel.yml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,14 +97,25 @@ jobs:
9797
wait_for_gpu
9898
9999
- name: L0 Pytorch Wheel
100+
id: L0_pytoech_wheel
100101
# timeout-minutes: 10
101102
env:
102103
TE_PATH: .
104+
RUN_LOG: /logs/pytorch/wheel
103105
run: |
104106
echo "TE_PATH: ${TE_PATH}"
105107
sed -i "s/^cd transformer_engine\/pytorch\s*$/pushd transformer_engine\/pytorch/" qa/L0_pytorch_wheel/test.sh
106108
sed -i '44 s/^cd \s*\$TE_PATH\s*$/popd/' qa/L0_pytorch_wheel/test.sh
107109
108110
cat qa/L0_pytorch_wheel/test.sh
109111
110-
bash qa/L0_pytorch_wheel/test.sh
112+
bash qa/L0_pytorch_wheel/test.sh | tee ${RUN_LOG}/pytorch_wheel-${{ github.run_id }}.log
113+
114+
- name: Upload Installation Logs
115+
if: always() && steps.L0_pytoech_wheel.outcome == 'failure'
116+
uses: actions/upload-artifact@v4
117+
with:
118+
name: L0-pytorch-logs-${{ github.run_id }}
119+
path: /logs/pytorch/wheel
120+
retention-days: 7
121+
if-no-files-found: warn

‎.github/workflows/qa_l1_pytorch_thunder_integration.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: QA L1 Pytorch Thunder Integration
33
on:
44
push:
55
branches:
6-
- main
6+
- add_qa_teat251230
77
pull_request:
88
branches:
99
- main
@@ -108,8 +108,8 @@ jobs:
108108
if: failure() && steps.L1_pytorch_thunder_integration.outcome == 'failure'
109109
uses: actions/upload-artifact@v4
110110
with:
111-
name: L0_pytorch-logs-${{ github.run_id }}
112-
path: /logs
111+
name: L1_pytorch-logs-${{ github.run_id }}
112+
path: /logs/pytorch/thunder
113113
retention-days: 7
114114
if-no-files-found: warn
115115
continue-on-error: true

‎.github/workflows/qa_l1_pytorch_unittest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: QA L1 PYTHORCH Unittest
33
on:
44
push:
55
branches:
6-
- main
6+
- add_qa_teat251230
77
pull_request:
88
branches:
99
- main
@@ -153,7 +153,7 @@ jobs:
153153
if: always() && (steps.L1_pytorch_distributed_unittest.outcome == 'failure' || steps.L1_pytorch_onnx_unittest.outcome == 'failure')
154154
uses: actions/upload-artifact@v4
155155
with:
156-
name: L0_pytorch-logs-${{ github.run_id }}
157-
path: /logs
156+
name: L1_pytorch-logs-${{ github.run_id }}
157+
path: /logs/pytorch
158158
retention-days: 7
159159
if-no-files-found: warn

‎.github/workflows/qa_l3_pytorch_fa_version_test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ jobs:
118118
if: always() && steps.L3_pytorch_FA_versions_test.outcome == 'failure'
119119
uses: actions/upload-artifact@v4
120120
with:
121-
name: L0_pytorch-logs-${{ github.run_id }}
121+
name: L3_pytorch-logs-${{ github.run_id }}
122122
path: /logs/pytorch/attention
123123
retention-days: 7
124124
if-no-files-found: warn

0 commit comments

Comments
 (0)