Skip to content

Commit 6e5f543

Browse files
author
yefubao
committed
debug1
1 parent dde54e5 commit 6e5f543

1 file changed

Lines changed: 46 additions & 22 deletions

File tree

‎.github/workflows/qa_l0_pytorch.yml‎

Lines changed: 46 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -110,32 +110,56 @@ jobs:
110110
# ctest --test-dir build -j$NUM_PARALLEL_JOBS
111111

112112
- name: L0_pytorch_debug_unittest
113+
id: L0_pytorch_debug_unittest
113114
env:
114115
PYTEST_ADDOPTS: "-v -s"
115116
NVTE_TEST_NVINSPECT_FEATURE_DIRS: "transformer_engine/debug/features"
116117
NVTE_TEST_NVINSPECT_CONFIGS_DIR: "tests/pytorch/debug/test_configs/"
117118
NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE: "tests/pytorch/debug/test_configs/dummy_feature.yaml"
118119
XML_LOG_DIR: "/logs/pytorch/debug"
119120
run: |
120-
echo "XML_LOG_DIR: ${XML_LOG_DIR}"
121-
122-
123-
# mkdir -p "$XML_LOG_DIR"
124-
125-
# # It is not installed as a requirement,
126-
# # because it is not available on PyPI.
127-
# pip uninstall -y nvdlfw-inspect
128-
# pip install git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git
129-
# pip install pytest==8.2.1
130-
131-
# pytest -v -s --junitxml=$XML_LOG_DIR/test_sanity.xml tests/pytorch/debug/test_sanity.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS || FAIL=1
132-
# pytest -v -s --junitxml=$XML_LOG_DIR/test_config.xml tests/pytorch/debug/test_config.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS || FAIL=1
133-
# pytest -v -s --junitxml=$XML_LOG_DIR/test_numerics.xml tests/pytorch/debug/test_numerics.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS || FAIL=1
134-
# pytest -v -s --junitxml=$XML_LOG_DIR/test_log.xml tests/pytorch/debug/test_log.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR || FAIL=1
135-
# # NVTE_TORCH_COMPILE=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_api_features.xml tests/pytorch/debug/test_api_features.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR || FAIL=1
136-
# pytest -v -s --junitxml=$XML_LOG_DIR/test_perf.xml tests/pytorch/debug/test_perf.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR || FAIL=1
137-
138-
# # standard sanity and numerics tests with initialized debug
139-
# NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_sanity_2.xml tests/pytorch/test_sanity.py || FAIL=1
140-
# # NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_numerics_2.xml tests/pytorch/test_numerics.py || FAIL=1
141-
121+
# Build log storage directory
122+
mkdir -p "$XML_LOG_DIR"
123+
124+
# It is not installed as a requirement,
125+
# because it is not available on PyPI.
126+
pip uninstall -y nvdlfw-inspect
127+
pip install git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git
128+
pip install pytest==8.2.1
129+
130+
run_pytest() {
131+
local test_file=$1
132+
local xml_name=$2
133+
local extra_opts=${3:-""}
134+
echo "🚀 Running test: $test_file"
135+
pytest --junitxml="${XML_LOG_DIR}/${xml_name}" "${test_file}" "${extra_opts}"
136+
if [ $? -eq 0 ]; then
137+
echo "✅ Test $"xml_name" passed"
138+
else
139+
echo "❌ Test $"xml_name" failed"
140+
exit 1
141+
fi
142+
}
143+
144+
# tests/pytorch/debug
145+
run_pytest "tests/pytorch/debug/test_sanity.py" "test_sanity.xml" "--feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS"
146+
run_pytest "tests/pytorch/debug/test_config.py" "test_config.xml" "--feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS"
147+
run_pytest "tests/pytorch/debug/test_numerics.py" "test_numerics.xml" "--feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS"
148+
run_pytest "tests/pytorch/debug/test_log.py" "test_log.xml" "--feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR"
149+
run_pytest "tests/pytorch/debug/test_perf.py" "test_perf.xml" "--feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR"
150+
# NVTE_TORCH_COMPILE=0 run_pytest "tests/pytorch/debug/test_api_features.py" "test_api_features.xml" "--feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR"
151+
152+
153+
# standard sanity and numerics tests with initialized debug
154+
NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 pytest --junitxml=$XML_LOG_DIR/test_sanity_2.xml tests/pytorch/test_sanity.py
155+
# NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 pytest --junitxml=$XML_LOG_DIR/test_numerics_2.xml tests/pytorch/test_numerics.py
156+
157+
- name: Upload Installation Logs
158+
if: failure() && steps.L0_pytorch_debug_unittest.outcome == 'failure'
159+
uses: actions/upload-artifact@v4
160+
with:
161+
name: L0_pytorch-logs-${{ github.run_id }}
162+
path: /logs
163+
retention-days: 7
164+
if-no-files-found: warn
165+
continue-on-error: true

0 commit comments

Comments
 (0)