Skip to content

Commit ad1ff42

Browse files
author
zihugithub
committed
fix import error cased by _load_cuda_libs when running pytest
1 parent d2f5dc1 commit ad1ff42

5 files changed

Lines changed: 108 additions & 108 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,24 +10,25 @@ on:
1010
jobs:
1111
pytorch:
1212
name: 'PyTorch'
13-
runs-on: ubuntu-latest
13+
runs-on: [ self-hosted, Linux, X64, nvidia, gpu-8 ]
1414
container:
15-
image: nvcr.io/nvidia/cuda:12.8.0-devel-ubuntu22.04
15+
image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
1616
options: --user root
1717
steps:
18-
- name: 'Dependencies'
19-
run: |
20-
apt-get update
21-
apt-get install -y git python3.9 pip cudnn9-cuda-12
22-
pip install cmake torch ninja pydantic importlib-metadata>=1.0 packaging pybind11 numpy einops onnxscript nvidia-mathdx==25.1.1
2318
- name: 'Checkout'
2419
uses: actions/checkout@v3
2520
with:
2621
submodules: recursive
27-
# - name: 'Build'
28-
# run: pip install --no-build-isolation . -v --no-deps
29-
# env:
30-
# NVTE_FRAMEWORK: pytorch
31-
# MAX_JOBS: 1
32-
# - name: 'Sanity check'
33-
# run: python3 tests/pytorch/test_sanity_import.py
22+
- name: 'Build'
23+
run:
24+
source /opt/miniconda3/etc/profile.d/conda.sh
25+
conda activate flagscale-train
26+
pip install --no-build-isolation . -v --no-deps
27+
env:
28+
NVTE_FRAMEWORK: pytorch
29+
TE_WITH_NCCL: 1
30+
- name: 'Sanity check'
31+
run:
32+
source /opt/miniconda3/etc/profile.d/conda.sh
33+
conda activate flagscale-train
34+
python3 tests/pytorch/test_sanity_import.py

‎.github/workflows/qa-l0-pytorch-wheel.yml‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -50,29 +50,29 @@ jobs:
5050
submodules: recursive
5151
set-safe-directory: true
5252

53-
# - name: L0 Pytorch Wheel
54-
# id: L0_pytoech_wheel
55-
# # timeout-minutes: 50
56-
# env:
57-
# TE_PATH: .
58-
# RUN_LOG: /logs/pytorch/wheel
59-
# run: |
60-
# echo "TE_PATH: ${TE_PATH}"
61-
# sed -i "s/^cd transformer_engine\/pytorch\s*$/pushd transformer_engine\/pytorch/" qa/L0_pytorch_wheel/test.sh
62-
# sed -i '44 s/^cd \s*\$TE_PATH\s*$/popd/' qa/L0_pytorch_wheel/test.sh
53+
- name: L0 Pytorch Wheel
54+
id: L0_pytoech_wheel
55+
# timeout-minutes: 50
56+
env:
57+
TE_PATH: .
58+
RUN_LOG: /logs/pytorch/wheel
59+
run: |
60+
echo "TE_PATH: ${TE_PATH}"
61+
sed -i "s/^cd transformer_engine\/pytorch\s*$/pushd transformer_engine\/pytorch/" qa/L0_pytorch_wheel/test.sh
62+
sed -i '44 s/^cd \s*\$TE_PATH\s*$/popd/' qa/L0_pytorch_wheel/test.sh
6363
64-
# cat qa/L0_pytorch_wheel/test.sh
65-
# source /opt/miniconda3/etc/profile.d/conda.sh
66-
# conda activate flagscale-train
67-
# pip uninstall -y transformer_engine
64+
cat qa/L0_pytorch_wheel/test.sh
65+
source /opt/miniconda3/etc/profile.d/conda.sh
66+
conda activate flagscale-train
67+
pip uninstall -y transformer_engine
6868
69-
# bash qa/L0_pytorch_wheel/test.sh | tee ${RUN_LOG}/pytorch_wheel-${{ github.run_id }}.log
69+
bash qa/L0_pytorch_wheel/test.sh | tee ${RUN_LOG}/pytorch_wheel-${{ github.run_id }}.log
7070
71-
# - name: Upload Installation Logs
72-
# if: always() && steps.L0_pytoech_wheel.outcome == 'failure'
73-
# uses: actions/upload-artifact@v4
74-
# with:
75-
# name: L0-pytorch-logs-${{ github.run_id }}
76-
# path: /logs/pytorch/wheel
77-
# retention-days: 7
78-
# if-no-files-found: warn
71+
- name: Upload Installation Logs
72+
if: always() && steps.L0_pytoech_wheel.outcome == 'failure'
73+
uses: actions/upload-artifact@v4
74+
with:
75+
name: L0-pytorch-logs-${{ github.run_id }}
76+
path: /logs/pytorch/wheel
77+
retention-days: 7
78+
if-no-files-found: warn

‎.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,4 +181,3 @@ jobs:
181181
# Run core unit tests
182182
echo "=== Running L0 PyTorch Core Unit Tests ==="
183183
bash ./qa/L0_pytorch_unittest/test.sh
184-

‎.github/workflows/qa-l1-te-cpp-pytorch-tests.yml‎

Lines changed: 43 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -112,46 +112,46 @@ jobs:
112112
source .github/workflows/scripts/gpu_check.sh
113113
wait_for_gpu
114114
115-
# - name: Run L1 PyTorch Thunder Integration Tests
116-
# env:
117-
# XML_LOG_DIR: "/logs/pytorch/thunder"
118-
# THUNDER_PATH: "lightning-thunder"
119-
# TE_PATH: .
120-
# run: |
121-
# # Activate conda environment
122-
# source /opt/miniconda3/etc/profile.d/conda.sh
123-
# conda activate flagscale-train
124-
125-
# # Run thunder integration tests
126-
# echo "=== Running L1 PyTorch Thunder Integration Tests ==="
127-
# bash ./qa/L1_pytorch_thunder_integration/test.sh
128-
# # timeout-minutes: 5
129-
130-
# - name: Run L1 PyTorch Distributed Unit Tests
131-
# continue-on-error: true
132-
# env:
133-
# XML_LOG_DIR: "/logs/pytorch/distributed"
134-
# TE_PATH: .
135-
# run: |
136-
# # Activate conda environment
137-
# source /opt/miniconda3/etc/profile.d/conda.sh
138-
# conda activate flagscale-train
139-
140-
# # Run distributed unit tests
141-
# echo "=== Running L1 PyTorch Distributed Unit Tests ==="
142-
# bash ./qa/L1_pytorch_distributed_unittest/test.sh
143-
# # timeout-minutes: 5
144-
145-
# - name: Run L1 PyTorch ONNX Unit Tests
146-
# env:
147-
# XML_LOG_DIR: "/logs/pytorch/onnx"
148-
# TE_PATH: .
149-
# run: |
150-
# # Activate conda environment
151-
# source /opt/miniconda3/etc/profile.d/conda.sh
152-
# conda activate flagscale-train
153-
154-
# # Run ONNX unit tests
155-
# echo "=== Running L1 PyTorch ONNX Unit Tests ==="
156-
# bash ./qa/L1_pytorch_onnx_unittest/test.sh
157-
# # timeout-minutes: 30
115+
- name: Run L1 PyTorch Thunder Integration Tests
116+
env:
117+
XML_LOG_DIR: "/logs/pytorch/thunder"
118+
THUNDER_PATH: "lightning-thunder"
119+
TE_PATH: .
120+
run: |
121+
# Activate conda environment
122+
source /opt/miniconda3/etc/profile.d/conda.sh
123+
conda activate flagscale-train
124+
125+
# Run thunder integration tests
126+
echo "=== Running L1 PyTorch Thunder Integration Tests ==="
127+
bash ./qa/L1_pytorch_thunder_integration/test.sh
128+
# timeout-minutes: 5
129+
130+
- name: Run L1 PyTorch Distributed Unit Tests
131+
continue-on-error: true
132+
env:
133+
XML_LOG_DIR: "/logs/pytorch/distributed"
134+
TE_PATH: .
135+
run: |
136+
# Activate conda environment
137+
source /opt/miniconda3/etc/profile.d/conda.sh
138+
conda activate flagscale-train
139+
140+
# Run distributed unit tests
141+
echo "=== Running L1 PyTorch Distributed Unit Tests ==="
142+
bash ./qa/L1_pytorch_distributed_unittest/test.sh
143+
# timeout-minutes: 5
144+
145+
- name: Run L1 PyTorch ONNX Unit Tests
146+
env:
147+
XML_LOG_DIR: "/logs/pytorch/onnx"
148+
TE_PATH: .
149+
run: |
150+
# Activate conda environment
151+
source /opt/miniconda3/etc/profile.d/conda.sh
152+
conda activate flagscale-train
153+
154+
# Run ONNX unit tests
155+
echo "=== Running L1 PyTorch ONNX Unit Tests ==="
156+
bash ./qa/L1_pytorch_onnx_unittest/test.sh
157+
# timeout-minutes: 30

‎.github/workflows/qa-l3-te-pytorch-fa-versions-test.yml‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -103,31 +103,31 @@ jobs:
103103
source .github/workflows/scripts/gpu_check.sh
104104
wait_for_gpu
105105
106-
# - name: Run QA L3 PyTorch FlashAttention Versions Test
107-
# # timeout-minutes: 30
108-
# env:
109-
# XML_LOG_DIR: "/logs/pytorch/attention"
110-
# TE_PATH: .
111-
# MAX_JOBS: 32
112-
# run: |
113-
# # Activate conda environment
114-
# source /opt/miniconda3/etc/profile.d/conda.sh
115-
# conda activate flagscale-train
116-
117-
# # Create log directory with proper permissions
118-
# echo "=== Preparing Test Environment ==="
119-
# mkdir -p "$XML_LOG_DIR"
120-
# chmod 777 "$XML_LOG_DIR"
121-
122-
# # Download flash_attn_interface.py
123-
# pip3 install pytest==8.2.1
124-
# git clone https://github.com/Dao-AILab/flash-attention.git
125-
# cd flash-attention/ && git checkout 27f501d && cd hopper/ && python setup.py install
126-
# python_path=`python -c "import site; print(site.getsitepackages()[0])"`
127-
# mkdir -p $python_path/flash_attn_3
128-
# wget -P $python_path/flash_attn_3 https://raw.githubusercontent.com/Dao-AILab/flash-attention/27f501dbe011f4371bff938fe7e09311ab3002fa/hopper/flash_attn_interface.py
129-
# cd ../../
130-
131-
# # Run pytest with detailed output and error tracking
132-
# NVTE_TORCH_COMPILE=0 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest.xml $TE_PATH/tests/pytorch/attention/test_attention.py
106+
- name: Run QA L3 PyTorch FlashAttention Versions Test
107+
# timeout-minutes: 30
108+
env:
109+
XML_LOG_DIR: "/logs/pytorch/attention"
110+
TE_PATH: .
111+
MAX_JOBS: 32
112+
run: |
113+
# Activate conda environment
114+
source /opt/miniconda3/etc/profile.d/conda.sh
115+
conda activate flagscale-train
116+
117+
# Create log directory with proper permissions
118+
echo "=== Preparing Test Environment ==="
119+
mkdir -p "$XML_LOG_DIR"
120+
chmod 777 "$XML_LOG_DIR"
121+
122+
# Download flash_attn_interface.py
123+
pip3 install pytest==8.2.1
124+
git clone https://github.com/Dao-AILab/flash-attention.git
125+
cd flash-attention/ && git checkout 27f501d && cd hopper/ && python setup.py install
126+
python_path=`python -c "import site; print(site.getsitepackages()[0])"`
127+
mkdir -p $python_path/flash_attn_3
128+
wget -P $python_path/flash_attn_3 https://raw.githubusercontent.com/Dao-AILab/flash-attention/27f501dbe011f4371bff938fe7e09311ab3002fa/hopper/flash_attn_interface.py
129+
cd ../../
130+
131+
# Run pytest with detailed output and error tracking
132+
NVTE_TORCH_COMPILE=0 python3 -m pytest -v -s --junitxml=$XML_LOG_DIR/pytest.xml $TE_PATH/tests/pytorch/attention/test_attention.py
133133

0 commit comments

Comments
 (0)