diff --git a/.github/configs/ascend.yml b/.github/configs/ascend.yml new file mode 100644 index 0000000000..03fc5acaf5 --- /dev/null +++ b/.github/configs/ascend.yml @@ -0,0 +1,15 @@ +# Huawei Ascend NPU configuration +image: ascend-infer:ubuntu18.04 +labels: + - npu + - ascend +docker_options: | + --device /dev/davinci0 + --device /dev/davinci1 + --device /dev/davinci2 + --device /dev/davinci3 + --device /dev/davinci_manager + --device /dev/devmm_svm + --device /dev/hisi_hdc + --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver + --volume /usr/local/Ascend/add-ons:/usr/local/Ascend/add-ons \ No newline at end of file diff --git a/.github/configs/cuda.yml b/.github/configs/cuda.yml new file mode 100644 index 0000000000..36373513de --- /dev/null +++ b/.github/configs/cuda.yml @@ -0,0 +1,65 @@ +# CUDA Hardware Configuration for TransformerEngine-FL +# Refactored for BAAI DGX A100 Nodes +# This file defines environment variables, volumes, and test filters for TE tests. + +hardware_name: cuda +display_name: 'NVIDIA CUDA (A100)' + +ci_image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209 + +# Runner labels for self-hosted A100 node +runner_labels: + - self-hosted + - Linux + - X64 + - nvidia + - gpu-8 + +# Container volumes +container_volumes: + - /home/flagscale_cicd/flask/static:/workspace/report + # - .:/opt/transformerengine + # - ./ci_logs:/logs + # - /home/flagscale_cicd/data:/opt/data + +# Container options +container_options: >- + --privileged + --gpus all + --shm-size=500g + --ipc=host + --ulimit memlock=-1 + --ulimit stack=67108864 + --user root + +# Device types +device_types: + - a100 + +# Build environment variables (platform-specific) +build_env: + TE_FL_SKIP_CUDA: '0' + SKIP_CUDA_BUILD: '0' + NVTE_WITH_CUDA: '1' + NVTE_WITH_MACA: '0' + TE_WITH_NCCL: '1' + NVTE_FRAMEWORK: pytorch + CUDA_HOME: /usr/local/cuda-12.8 + NVCC: /usr/local/cuda-12.8/bin/nvcc + +# Test matrix configuration +test_matrix: + l0_pytorch: + path: 'qa/L0_pytorch_unittest/test.sh' + ignored_tests: + - test_sanity_layernorm_mlp + - test_sanity_gpt + - test_sanity_bert + - test_sanity_T5 + - test_sanity_amp_and_nvfuser + - test_sanity_drop_path + - test_layernorm_mlp_accuracy + - test_grouped_linear_accuracy + - test_gpt_accuracy + - test_basic_linear + - test_layer_norm diff --git a/.github/configs/metax.yml b/.github/configs/metax.yml new file mode 100644 index 0000000000..e937189a55 --- /dev/null +++ b/.github/configs/metax.yml @@ -0,0 +1,68 @@ +# Metax Hardware Configuration for TE-FL +# This file defines CI/CD settings for Metax-based testing +# Test configurations are defined in tests/test_utils/config/platforms/metax.yaml + +hardware_name: metax +display_name: 'Metax Tests' + +ci_image: localhost:5000/megatron-lm-with-te:v1 + +runner_labels: + - self-hosted + - Linux + - X64 + - metax + - dev + +container_volumes: + - /nfs/metax_fs:/nfs/metax_fs + - /dev/dri:/dev/dri + - /dev/mxcd:/dev/mxcd + - /dev/infiniband:/dev/infiniband + +container_options: >- + --uts=host + --ipc=host + --privileged=true + --group-add video + --shm-size=100gb + --ulimit memlock=-1 + --security-opt seccomp=unconfined + --security-opt apparmor=unconfined + --device=/dev/dri + --device=/dev/mxcd + --device=/dev/infiniband + --user root + --ulimit nofile=65535:65535 + -e PLATFORM=metax + -e TORCH_DISTRIBUTED_BACKEND=mccl + -e LD_LIBRARY_PATH=/opt/maca/lib:/usr/local/lib:$LD_LIBRARY_PATH + +build_env: + TE_FL_SKIP_CUDA: '1' + NVTE_WITH_MACA: '1' + CUDA_HOME: /opt/maca + MACA_HOME: /opt/maca + +# Device types to run tests on +device_types: + - c500 + +# Test matrix configuration +test_matrix: + unit: + devices: + - c500 + # Ignored test files for unit tests + # These files will be skipped when running pytest + ignored_tests: + # example: tests/unit_tests/test_example.py + # - tests/unit_tests/test_inference.py + # - tests/unit_tests/test_rl_utils.py + + # functional: + # train: + # - device: c500 + # task: train + # model: deepseek + # case: tp2_pp2_ep2 diff --git a/.github/configs/template.yml b/.github/configs/template.yml new file mode 100644 index 0000000000..c7ec56b3e9 --- /dev/null +++ b/.github/configs/template.yml @@ -0,0 +1,16 @@ +# Configuration Template +# This file describes the structure for hardware-specific configurations. +# +# Fields: +# - image: Docker image to use for the runner +# - labels: List of labels for the runner +# - docker_options: Additional Docker options for mounting devices, volumes, etc. +# +# Example: +# image: +# labels: +# - +# - +# docker_options: | +# --option1 value1 +# --option2 value2 \ No newline at end of file diff --git a/.github/workflows/all_tests_ascend.yml b/.github/workflows/all_tests_ascend.yml new file mode 100644 index 0000000000..04e8f3cba0 --- /dev/null +++ b/.github/workflows/all_tests_ascend.yml @@ -0,0 +1,32 @@ +name: ascend_tests + +on: + # push: + # branches: ["main"] + # pull_request: + # branches: ["main"] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }} + cancel-in-progress: true + +jobs: + run_tests: + # Package manager and environment settings are read from .github/configs/ascend.yml + uses: ./.github/workflows/all_tests_common.yml + with: + platform: ascend + + all_tests: + needs: run_tests + runs-on: ubuntu-latest + if: always() + steps: + - name: Verify workflow status + run: | + if [ "${{ needs.run_tests.result }}" != "success" ]; then + echo "❌ Tests workflow failed" + exit 1 + fi + echo "✅ All tests passed!" diff --git a/.github/workflows/all_tests_common.yml b/.github/workflows/all_tests_common.yml new file mode 100644 index 0000000000..86a85a2d6a --- /dev/null +++ b/.github/workflows/all_tests_common.yml @@ -0,0 +1,150 @@ +name: Common All Tests + +on: + workflow_call: + inputs: + platform: + required: true + type: string + description: Platform name (e.g., cuda, default) + setup_commands: + required: false + type: string + default: '' + +jobs: + checkout_and_config: + defaults: + run: + shell: bash + runs-on: ubuntu-latest + outputs: + ci_image: ${{ steps.config.outputs.ci_image }} + runs_on: ${{ steps.config.outputs.runs_on }} + container_volumes: ${{ steps.config.outputs.container_volumes }} + container_options: ${{ steps.config.outputs.container_options }} + device_types: ${{ steps.config.outputs.device_types }} + train_test_matrix: ${{ steps.config.outputs.train_test_matrix }} + ignored_tests: ${{ steps.config.outputs.ignored_tests }} + build_env: ${{ steps.config.outputs.build_env }} + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Check if tests should run + id: should_run + run: | + + echo "should_run=true" >> $GITHUB_OUTPUT + + - name: Load platform configuration + id: config + run: | + set -euo pipefail + + PLATFORM="${{ inputs.platform }}" + CONFIG_FILE=".github/configs/${PLATFORM}.yml" + + # Install mikefarah/yq (v4) for YAML parsing + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + /usr/local/bin/yq --version + echo "Loading configuration from $CONFIG_FILE" + + # Read CI image + CI_IMAGE=$(yq '.ci_image' "$CONFIG_FILE") + echo "ci_image=$CI_IMAGE" >> $GITHUB_OUTPUT + + # Read runner labels and format as JSON array + RUNS_ON=$(yq '.runner_labels | tojson(0)' "$CONFIG_FILE") + echo "runs_on=$RUNS_ON" >> $GITHUB_OUTPUT + + # Read container volumes and format as JSON array + VOLUMES=$(yq '.container_volumes | tojson(0)' "$CONFIG_FILE") + echo "container_volumes=$VOLUMES" >> $GITHUB_OUTPUT + + # Read container options + OPTIONS=$(yq '.container_options' "$CONFIG_FILE") + echo "container_options=$OPTIONS" >> $GITHUB_OUTPUT + + # Read device types + DEVICE_TYPES=$(yq '.device_types | tojson(0)' "$CONFIG_FILE") + echo "device_types=$DEVICE_TYPES" >> $GITHUB_OUTPUT + + # Read test matrix for training + TRAIN_MATRIX=$(yq '.test_matrix.functional.train | tojson(0)' "$CONFIG_FILE") + echo "train_test_matrix=$TRAIN_MATRIX" >> $GITHUB_OUTPUT + + # Read ignored tests list from test_matrix.unit (default to empty array if not defined) + IGNORED_TESTS=$(yq '.test_matrix.unit.ignored_tests // [] | tojson(0)' "$CONFIG_FILE") + echo "ignored_tests=$IGNORED_TESTS" >> $GITHUB_OUTPUT + + # Read build environment variables (default to empty object if not defined) + BUILD_ENV=$(yq '.build_env // {} | tojson(0)' "$CONFIG_FILE") + echo "build_env=$BUILD_ENV" >> $GITHUB_OUTPUT + + unit_tests: + needs: checkout_and_config + strategy: + fail-fast: false + matrix: + device: ${{ fromJson(needs.checkout_and_config.outputs.device_types) }} + uses: ./.github/workflows/unit_tests_common.yml + name: unit_tests + with: + setup_commands: ${{ inputs.setup_commands }} + platform: ${{ inputs.platform }} + device: ${{ matrix.device }} + image: ${{ needs.checkout_and_config.outputs.ci_image }} + runs_on: ${{ needs.checkout_and_config.outputs.runs_on }} + container_volumes: ${{ needs.checkout_and_config.outputs.container_volumes }} + container_options: ${{ needs.checkout_and_config.outputs.container_options }} + ignored_tests: ${{ needs.checkout_and_config.outputs.ignored_tests }} + build_env: ${{ needs.checkout_and_config.outputs.build_env }} + + # arguments.py not compatible with megatron-core-fl + # functional_tests: + # needs: + # - checkout_and_config + # if: fromJson(needs.checkout_and_config.outputs.train_test_matrix)[0] != null + # uses: ./.github/workflows/functional_tests_common.yml + # with: + # platform: ${{ inputs.platform }} + # test_matrix: ${{ needs.checkout_and_config.outputs.train_test_matrix }} + # image: ${{ needs.checkout_and_config.outputs.ci_image }} + # runs_on: ${{ needs.checkout_and_config.outputs.runs_on }} + # container_volumes: ${{ needs.checkout_and_config.outputs.container_volumes }} + # container_options: ${{ needs.checkout_and_config.outputs.container_options }} + + + all_tests_complete: + defaults: + run: + shell: bash + needs: + - checkout_and_config + - unit_tests + # - functional_tests + runs-on: ubuntu-latest + if: always() + steps: + - name: Verify all tests passed + run: | + # Check all test jobs (skip if not run) + failed=false + + if [ "${{ needs.unit_tests.result }}" != "success" ]; then + echo "❌ Unit tests failed" + failed=true + fi + + # if [ "${{ needs.functional_tests.result }}" != "success" ]; then + # echo "❌ Training functional tests failed" + # failed=true + # fi + + if [ "$failed" = "true" ]; then + exit 1 + fi + + echo "✅ All tests completed successfully!" \ No newline at end of file diff --git a/.github/workflows/all_tests_cuda.yml b/.github/workflows/all_tests_cuda.yml new file mode 100644 index 0000000000..b78ddf35bb --- /dev/null +++ b/.github/workflows/all_tests_cuda.yml @@ -0,0 +1,32 @@ +name: cuda_tests + +on: + # push: + # branches: ["main"] + # pull_request: + # branches: ["main"] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }} + cancel-in-progress: true + +jobs: + run_tests: + # Package manager and environment settings are read from .github/configs/cuda.yml + uses: ./.github/workflows/all_tests_common.yml + with: + platform: cuda + + all_tests: + needs: run_tests + runs-on: ubuntu-latest + if: always() + steps: + - name: Verify workflow status + run: | + if [ "${{ needs.run_tests.result }}" != "success" ]; then + echo "❌ Tests workflow failed" + exit 1 + fi + echo "✅ All tests passed!" diff --git a/.github/workflows/all_tests_metax.yml b/.github/workflows/all_tests_metax.yml new file mode 100644 index 0000000000..d3e496c4b2 --- /dev/null +++ b/.github/workflows/all_tests_metax.yml @@ -0,0 +1,37 @@ +name: metax_tests + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }} + cancel-in-progress: true + +jobs: + run_tests: + uses: ./.github/workflows/all_tests_common.yml + with: + platform: metax + # Metax Environment Setup + setup_commands: | + export PATH=/opt/conda/bin:$PATH + export LD_LIBRARY_PATH=/usr/local/maca/lib:/opt/maca/lib:$LD_LIBRARY_PATH + which python3 + python3 -m pip --version + + all_tests: + needs: run_tests + runs-on: ubuntu-latest + if: always() + steps: + - name: Verify workflow status + run: | + if [ "${{ needs.run_tests.result }}" != "success" ]; then + echo "❌ Metax Tests workflow failed" + exit 1 + fi + echo "✅ All Metax tests passed!" \ No newline at end of file diff --git a/.github/workflows/functional_tests_common.yml b/.github/workflows/functional_tests_common.yml new file mode 100644 index 0000000000..aa6b734778 --- /dev/null +++ b/.github/workflows/functional_tests_common.yml @@ -0,0 +1,190 @@ +# Disabled for compatibility issues +name: Common Functional Tests - Training + +on: + workflow_call: + inputs: + platform: + required: true + type: string + description: Platform name (e.g., cuda, default) + test_matrix: + required: true + type: string + description: JSON array of test configurations + image: + required: true + type: string + runs_on: + required: true + type: string + container_volumes: + required: true + type: string + container_options: + required: true + type: string + +jobs: + functional_test_train: + defaults: + run: + shell: bash + env: + PROJECT_ROOT: ${{ github.workspace }} + runs-on: ${{ fromJson(inputs.runs_on) }} + strategy: + fail-fast: false + matrix: + test_config: ${{ fromJson(inputs.test_matrix) }} + container: + image: ${{ inputs.image }} + ports: + - 80 + volumes: ${{ fromJson(inputs.container_volumes) }} + options: ${{ inputs.container_options }} + + steps: + - name: Checkout source code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # - name: Set safe directory + # run: | + # git config --global --add safe.directory $PROJECT_ROOT + ## The above step is commented out because there is no git cli in the container, and it causes the step to fail. The safe directory is set in the next step with a conditional check. + - name: Set safe directory + run: | + command -v git && git config --global --add safe.directory $PROJECT_ROOT || true + + - name: Activate Python environment + run: | + source /opt/conda/etc/profile.d/conda.sh + conda activate base + echo "PATH=$PATH" >> $GITHUB_ENV + + - name: Setup Python environment + env: + NVTE_WITH_MACA: '1' + NVTE_WITH_CUDA: '0' + NVCC: /opt/maca/bin/mcc + CUDA_HOME: /opt/maca + + PATH: /opt/maca/bin:${{ env.PATH }} + LD_LIBRARY_PATH: /opt/maca/lib:${{ env.LD_LIBRARY_PATH }} + run: | + set -euo pipefail + cd $PROJECT_ROOT + pip install -e . --no-deps --no-build-isolation + timeout-minutes: 60 + + - name: L0 Pytorch Wheel + id: L0_pytoech_wheel + # timeout-minutes: 50 + env: + TE_PATH: . + RUN_LOG: /logs/pytorch/wheel + run: | + echo "TE_PATH: ${TE_PATH}" + sed -i "s/^cd transformer_engine\/pytorch\s*$/pushd transformer_engine\/pytorch/" qa/L0_pytorch_wheel/test.sh + sed -i '44 s/^cd \s*\$TE_PATH\s*$/popd/' qa/L0_pytorch_wheel/test.sh + + cat qa/L0_pytorch_wheel/test.sh + # source /opt/miniconda3/etc/profile.d/conda.sh + # conda activate flagscale-train + pip uninstall -y transformer_engine + + set -euo pipefail + cd $PROJECT_ROOT + + PLATFORM='${{ inputs.platform }}' + DEVICE='${{ matrix.test_config.device }}' + TASK='${{ matrix.test_config.task }}' + MODEL='${{ matrix.test_config.model }}' + CASE='${{ matrix.test_config.case }}' + + echo "Running functional tests for training" + echo "Platform: $PLATFORM" + echo "Device: $DEVICE" + echo "Task: $TASK" + echo "Model: $MODEL" + echo "Case: ${CASE:-all}" + + # Set environment variables + export PYTHONPATH=$PROJECT_ROOT:${PYTHONPATH:-} + + set +e + bash qa/L0_pytorch_wheel/test.sh | tee ${RUN_LOG}/pytorch_wheel-${{ github.run_id }}.log + exit_code=$? + set -e + + if [ $exit_code -eq 0 ]; then + echo "✅ Functional tests passed for $PLATFORM/$DEVICE/$TASK/$MODEL/$CASE" + else + echo "❌ Functional tests failed for $PLATFORM/$DEVICE/$TASK/$MODEL/$CASE (exit code: $exit_code)" + fi + + echo "exit_code=$exit_code" >> $GITHUB_OUTPUT + exit $exit_code + + - name: Upload Installation Logs + if: always() && steps.L0_pytoech_wheel.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: L0-pytorch-logs-${{ github.run_id }} + path: /logs/pytorch/wheel + retention-days: 7 + if-no-files-found: warn + + # - name: Run functional tests + # id: functional_test + # run: | + # set -euo pipefail + # cd $PROJECT_ROOT + + # PLATFORM='${{ inputs.platform }}' + # DEVICE='${{ matrix.test_config.device }}' + # TASK='${{ matrix.test_config.task }}' + # MODEL='${{ matrix.test_config.model }}' + # CASE='${{ matrix.test_config.case }}' + + # echo "Running functional tests for training" + # echo "Platform: $PLATFORM" + # echo "Device: $DEVICE" + # echo "Task: $TASK" + # echo "Model: $MODEL" + # echo "Case: ${CASE:-all}" + + # # Set environment variables + # export PYTHONPATH=$PROJECT_ROOT:${PYTHONPATH:-} + + # # Run functional tests via run_tests.sh with explicit platform/device/task/model/case + # set +e + # bash "$PROJECT_ROOT/tests/test_utils/runners/run_tests.sh" \ + # --platform "$PLATFORM" \ + # --device "$DEVICE" \ + # --type functional \ + # --task "$TASK" \ + # --model "$MODEL" \ + # --list "$CASE" + # exit_code=$? + # set -e + + # if [ $exit_code -eq 0 ]; then + # echo "✅ Functional tests passed for $PLATFORM/$DEVICE/$TASK/$MODEL/$CASE" + # else + # echo "❌ Functional tests failed for $PLATFORM/$DEVICE/$TASK/$MODEL/$CASE (exit code: $exit_code)" + # fi + + # echo "exit_code=$exit_code" >> $GITHUB_OUTPUT + # exit $exit_code + # timeout-minutes: 60 + + # - name: Debug - keep container alive on failure + # if: failure() + # run: | + # echo "Container sleeping for 60 minutes for debugging..." + # echo "On host, run: docker ps then docker exec -it bash" + # sleep 3600 + # timeout-minutes: 60 \ No newline at end of file diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index 3a2be6b1be..5a93e92b94 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -5,7 +5,8 @@ # A workflow to trigger the TE license check on GitHub name: 'License' on: - pull_request: [__disabled_do_not_remove__] + pull_request: + branches: [ "__disabled_do_not_remove__" ] workflow_dispatch: jobs: check: diff --git a/.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml b/.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml index 0ef8622c8a..52299cf411 100644 --- a/.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml +++ b/.github/workflows/qa-l0-te-cpp-unittest-pytorch-lint.yml @@ -16,6 +16,8 @@ on: - 'transformer_engine/**' - 'tests/pytorch/**' + workflow_dispatch: + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }} cancel-in-progress: true diff --git a/.github/workflows/qa-l1-te-cpp-pytorch-tests.yml b/.github/workflows/qa-l1-te-cpp-pytorch-tests.yml index d0d15d7cf8..51f071aa3b 100644 --- a/.github/workflows/qa-l1-te-cpp-pytorch-tests.yml +++ b/.github/workflows/qa-l1-te-cpp-pytorch-tests.yml @@ -26,6 +26,8 @@ on: - 'tests/pytorch/attention/**' - 'qa/L1_pytorch_onnx_unittest/**' - 'tests/pytorch/test_onnx_export.py' + + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }} diff --git a/.github/workflows/unit_tests_common.yml b/.github/workflows/unit_tests_common.yml new file mode 100644 index 0000000000..6bfe8fd311 --- /dev/null +++ b/.github/workflows/unit_tests_common.yml @@ -0,0 +1,334 @@ +name: Common Unit Tests + + +on: + workflow_call: + inputs: + platform: + required: true + type: string + device: + required: true + type: string + image: + required: true + type: string + runs_on: + required: true + type: string + container_volumes: + required: true + type: string + container_options: + required: true + type: string + ignored_tests: + required: false + type: string + default: '' + # New input for hardware-specific initialization (e.g., conda activate) + setup_commands: + required: false + type: string + default: '' + # Platform-specific build environment variables (JSON object from config) + build_env: + required: false + type: string + default: '{}' + # Whether to upload coverage report + upload_coverage: + description: "Whether to upload coverage report" + required: false + type: boolean + default: true + +jobs: + # 1. Change Detection + detect_changes: + runs-on: ubuntu-latest + outputs: + core: ${{ steps.filter.outputs.core }} + qa_l0: ${{ steps.filter.outputs.qa_l0 }} + steps: + - name: Checkout source code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed paths + id: filter + run: | + set -euo pipefail + BASE_REF="${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || 'HEAD~1' }}" + [ "${{ github.event_name }}" == "pull_request" ] && git fetch origin ${{ github.base_ref }} --depth=1 + + CHANGED_FILES=$(git diff --name-only $BASE_REF...HEAD 2>/dev/null || git diff --name-only $BASE_REF HEAD) + + echo "core=$(echo "$CHANGED_FILES" | grep -qE "^tests/unit_tests/|^megatron/core/|^.github/" && echo "true" || echo "false")" >> $GITHUB_OUTPUT + echo "qa_l0=$(echo "$CHANGED_FILES" | grep -qE "^qa/L0_|^transformer_engine/|^tests/pytorch/|^.github/" && echo "true" || echo "false")" >> $GITHUB_OUTPUT + + # 2. Unified Test Execution + unit_test: + needs: detect_changes + defaults: + run: + shell: bash + runs-on: ${{ fromJson(inputs.runs_on) }} + strategy: + fail-fast: false + matrix: + test_group: + - name: pytorch_lint + path: "qa/L0_pytorch_lint/test.sh" + test_type: "lint" + - name: pytorch_debug + path: "qa/L0_pytorch_debug_unittest/test.sh" + test_type: "debug" + - name: pytorch_unittest + path: "qa/L0_pytorch_unittest/test.sh" + test_type: "unittest" + + name: unit-${{ inputs.device }}-${{ matrix.test_group.name }} + container: + image: ${{ inputs.image }} + volumes: ${{ fromJson(inputs.container_volumes) }} + options: --pull never ${{ inputs.container_options }} + + steps: + - name: Check if tests should run + id: should_run + run: | + echo "should_run=true" >> $GITHUB_OUTPUT + GROUP='${{ matrix.test_group.name }}' + # Force run if 'full ci' label exists + if [ "${{ contains(github.event.pull_request.labels.*.name, 'full ci') }}" == "true" ]; then + echo "should_run=true" >> $GITHUB_OUTPUT; exit 0 + fi + + if [[ "$GROUP" == "pytorch_"* ]]; then + CHANGED='${{ needs.detect_changes.outputs.qa_l0 }}' + else + CHANGED='${{ needs.detect_changes.outputs.core }}' + fi + + # For debugging, you can force this to true + echo "should_run=true" >> $GITHUB_OUTPUT + + - name: Checkout Source Code + if: steps.should_run.outputs.should_run == 'true' + uses: actions/checkout@v4 + with: + set-safe-directory: true + + # - name: Activate Python environment + # run: | + # if [[ "$PLATFORM" == "cuda" ]] && [ -f /opt/miniconda3/etc/profile.d/conda.sh ]; then + # source /opt/miniconda3/etc/profile.d/conda.sh + # conda activate flagscale-train + # elif [ -f /opt/conda/etc/profile.d/conda.sh ]; then + # source /opt/conda/etc/profile.d/conda.sh + # conda activate base + # fi + # echo "PATH=$PATH" >> $GITHUB_ENV + # echo "Python: $(which python3) ($(python3 --version 2>&1))" + + - name: Environment Setup on Cuda + if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'cuda' + run: | + set -euo pipefail + + echo "===== Step 0: Activate Python environment =====" + source /opt/miniconda3/etc/profile.d/conda.sh + conda activate flagscale-train + echo "PATH=$PATH" >> $GITHUB_ENV + echo "Python: $(which python3) ($(python3 --version 2>&1))" + + echo "===== Step 1: Remove Existing TransformerEngine =====" + pip uninstall transformer_engine transformer_engine_torch -y || true + + echo "===== Step 2: Build & Install TransformerEngine =====" + cd $GITHUB_WORKSPACE + pip install nvdlfw-inspect --no-deps + pip install --no-build-isolation . -v --no-deps + + echo "===== Step 3: Verify Installation =====" + python3 tests/pytorch/test_sanity_import.py + + echo "===== Environment Setup Complete ===== " + + - name: Environment Setup on Metax + if: steps.should_run.outputs.should_run == 'true' && inputs.platform == 'metax' + run: | + set -euo pipefail + + echo "===== Step 0: Activate Python environment =====" + source /opt/conda/etc/profile.d/conda.sh + conda activate base + echo "PATH=$PATH" >> $GITHUB_ENV + echo "Python: $(which python3) ($(python3 --version 2>&1))" + + echo "===== Step 1: Base Environment Setup =====" + # Configure MACA toolchain paths + export PATH=/opt/maca/bin:$PATH + export LD_LIBRARY_PATH=/opt/maca/lib:$LD_LIBRARY_PATH + service ssh restart + + echo "===== Step 2: Create nvcc Symlink (cucc -> nvcc) =====" + # TransformerEngine expects nvcc, but MACA provides cucc + ln -sf /opt/maca/tools/cu-bridge/bin/cucc /opt/maca/tools/cu-bridge/bin/nvcc + which nvcc || true + + echo "===== Step 3: Install Required System Tools =====" + # Install essential build tools (avoid modifying Python dependencies) + apt-get update -qq && apt-get install -y -qq git cmake ninja-build curl + + echo "===== Step 4: Remove Existing TransformerEngine =====" + # Prevent conflicts with preinstalled or incompatible versions + python3 -m pip uninstall transformer_engine -y || true + python3 -m pip install nvdlfw-inspect --no-deps || true + + # echo "===== Step 5: Install Metax Binary Backend =====" + # # Install prebuilt Metax backend (required for MACA operators) + # WHL_PATH="/home/muxiuser/transformer_engine_metax-2.9.0-cp312-cp312-linux_x86_64.whl" + # if [ ! -f "$WHL_PATH" ]; then + # echo "ERROR: Wheel file not found at $WHL_PATH" + # echo "Please verify volume mount: -v /home/muxiuser:/home/muxiuser" + # exit 1 + # fi + + # # Use --no-deps to avoid overwriting Metax-optimized PyTorch + # python3 -m pip install "$WHL_PATH" --no-deps --force-reinstall + + # echo "===== Step 6: Verify Metax Backend =====" + # # Ensure transformer_engine_torch is correctly loaded + # python3 - <<'EOF' + # import transformer_engine_torch as te + # print("Backend loaded successfully:", te) + # EOF + + echo "===== Step 7: Install TE-FL Plugin Layer =====" + # Install TransformerEngine-FL Python layer (plugin logic) + # cd /workspace/TransformerEngine-FL + cd $GITHUB_WORKSPACE + TE_FL_SKIP_CUDA=1 python3 setup.py install + + echo "===== Step 8: Final Verification =====" + # Verify both TE Python API and backend are functional + python3 - <<'EOF' + import transformer_engine + import transformer_engine_torch as te + print("transformer_engine:", transformer_engine) + print("transformer_engine_torch:", te) + EOF + + echo "===== Environment Setup Complete ===== " + + - name: Execute Tests + if: steps.should_run.outputs.should_run == 'true' + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + ${{ inputs.setup_commands }} + + # Load platform-specific environment variables + while IFS='=' read -r key value; do + [ -n "$key" ] && export "$key=$value" + done < <(echo '${{ inputs.build_env }}' | python3 -c " + import json, sys + env = json.load(sys.stdin) + for k, v in env.items(): + print(f'{k}={v}') + ") + + export TE_PATH=$GITHUB_WORKSPACE + export TE_LIB_PATH=$(python3 -c "import site; print(site.getsitepackages()[0])") + export PYTHONPATH=$GITHUB_WORKSPACE:${PYTHONPATH:-} + export PATH=${CUDA_HOME:-/usr/local/cuda}/bin:$PATH + export LD_LIBRARY_PATH=${CUDA_HOME:-/usr/local/cuda}/lib:${LD_LIBRARY_PATH:-} + + # check envs before running tests + echo "TE_PATH=$TE_PATH" + echo "TE_LIB_PATH=$TE_LIB_PATH" + echo "PYTHONPATH=$PYTHONPATH" + echo "PATH=$PATH" + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + + # Ensure log directory exists regardless of volume mount state + mkdir -p /logs + + # Enable coverage collection for all pytest invocations in test.sh + # PYTEST_ADDOPTS is automatically appended to every pytest call + if [ "${{ inputs.upload_coverage }}" = "true" ]; then + pip3 install pytest-cov 2>/dev/null || true + export PYTEST_ADDOPTS="--cov=transformer_engine --cov-append --cov-report=" + fi + + if [[ "${{ matrix.test_group.name }}" == *"lint"* ]]; then + export CPP_ONLY=0 + export PYTHON_ONLY=0 + elif [[ "${{ matrix.test_group.name }}" != *"debug"* ]]; then + # Fail fast on backend/API mismatch before running the full test group. + # Skip for debug group (does not use FP8/optimizer symbols). + python3 -c "import sys, importlib; import transformer_engine.common as _te_common; tex = importlib.import_module('transformer_engine_torch'); required=['multi_tensor_scale','multi_tensor_compute_scale_and_scale_inv']; missing=[n for n in required if not hasattr(tex, n)]; print('[TE check] module:', tex); print('[TE check] file:', getattr(tex, '__file__', 'N/A')); print('[TE check] missing:', ', '.join(missing) if missing else 'none'); sys.exit(1 if missing else 0)" + fi + + bash ${{ matrix.test_group.path }} + timeout-minutes: 60 + + - name: Generate Coverage Report + if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest' + working-directory: ${{ github.workspace }} + env: + PLATFORM: ${{ inputs.platform }} + DEVICE: ${{ inputs.device }} + run: | + # Install coverage (may already be present) + pip3 install coverage pytest-cov 2>/dev/null || true + + # Merge all .coverage* files produced by sub-processes (torchrun spawns workers) + python3 -m coverage combine --keep 2>/dev/null || true + # Generate JSON coverage report (requires .coverage data from pytest --cov) + python3 -m coverage json -o "coverage-${PLATFORM}-${DEVICE}.json" \ + --include="transformer_engine/*" 2>/dev/null || echo "WARNING: No coverage data found, skipping coverage-${PLATFORM}-${DEVICE}.json" + continue-on-error: true + + - name: Upload Coverage Report + if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest' + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: coverage-${{ inputs.platform }}-${{ inputs.device }} + path: | + coverage-${{ inputs.platform }}-${{ inputs.device }}.json + + - name: Check FlagCICD Reachability + if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest' + id: check_flagcicd + continue-on-error: true + run: | + if curl -sf --max-time 3 --connect-timeout 2 \ + "http://flagcicd-inner.flagos.net:8000/" -o /dev/null 2>/dev/null; then + echo "reachable=true" >> $GITHUB_OUTPUT + else + echo "reachable=false" >> $GITHUB_OUTPUT + echo "INFO: flagcicd-inner.flagos.net unreachable from this runner, skipping report upload" + fi + + - name: Upload Coverage Report to FlagCICD + if: inputs.upload_coverage && matrix.test_group.test_type == 'unittest' && steps.check_flagcicd.outputs.reachable == 'true' + uses: flagos-ai/FlagOps/actions/post-pytest-report@v2 + continue-on-error: true + with: + backend_url: 'http://flagcicd-inner.flagos.net:8000/metrics/' + user_id: '000000000000000000' + report_path: 'coverage-${{ inputs.platform }}-${{ inputs.device }}.json' + fail_on_error: 'false' + + # - name: Debug - keep container alive on failure + # if: failure() + # run: | + # echo "Container sleeping for 200 minutes for debugging..." + # echo "On host, run: docker ps then docker exec -it bash" + # sleep 60000 + # timeout-minutes: 200 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1a9a04d72d..8ef9585fd3 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,6 @@ compile_commands.json tensor_dumps/ artifacts/ # Auto-generated build configuration (specific to each environment) -transformer_engine/plugin/core/_build_config.py \ No newline at end of file +transformer_engine/plugin/core/_build_config.py +# Mac OS +.DS_Store \ No newline at end of file diff --git a/3rdparty/cudnn-frontend b/3rdparty/cudnn-frontend index 0b1577c8c8..f0c638223e 160000 --- a/3rdparty/cudnn-frontend +++ b/3rdparty/cudnn-frontend @@ -1 +1 @@ -Subproject commit 0b1577c8c83401237d601d0d0db5210506705396 +Subproject commit f0c638223eac20a9676941a110c9ad9e9842941d diff --git a/3rdparty/cutlass b/3rdparty/cutlass index 57e3cfb47a..73c59c055c 160000 --- a/3rdparty/cutlass +++ b/3rdparty/cutlass @@ -1 +1 @@ -Subproject commit 57e3cfb47a2d9e0d46eb6335c3dc411498efa198 +Subproject commit 73c59c055c0fec87792470dbf33325158113db5e diff --git a/3rdparty/googletest b/3rdparty/googletest index f8d7d77c06..a35bc7693c 160000 --- a/3rdparty/googletest +++ b/3rdparty/googletest @@ -1 +1 @@ -Subproject commit f8d7d77c06936315286eb55f8de22cd23c188571 +Subproject commit a35bc7693c117a048152beeb34f6aac354b9423f diff --git a/SECURITY.md b/SECURITY.md index 35edb61b01..7a6de0d126 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,5 +20,5 @@ To report a potential security vulnerability in any NVIDIA product: While NVIDIA currently does not have a bug bounty program, we do offer acknowledgement when an externally reported security issue is addressed under our coordinated vulnerability disclosure policy. Please visit our [Product Security Incident Response Team (PSIRT)](https://www.nvidia.com/en-us/security/psirt-policies/) policies page for more information. ## NVIDIA Product Security - +## test For all security-related concerns, please visit NVIDIA's Product Security portal at https://www.nvidia.com/en-us/security diff --git a/qa/L0_pytorch_debug_unittest/test.sh b/qa/L0_pytorch_debug_unittest/test.sh index 18199258c1..acbb440e70 100644 --- a/qa/L0_pytorch_debug_unittest/test.sh +++ b/qa/L0_pytorch_debug_unittest/test.sh @@ -20,18 +20,68 @@ FAIL=0 # because it is not available on PyPI. pip uninstall -y nvdlfw-inspect pip install git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git - pip install pytest==8.2.1 -pytest -v -s --junitxml=$XML_LOG_DIR/test_sanity.xml $TE_PATH/tests/pytorch/debug/test_sanity.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS || FAIL=1 -pytest -v -s --junitxml=$XML_LOG_DIR/test_config.xml $TE_PATH/tests/pytorch/debug/test_config.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS || FAIL=1 -pytest -v -s --junitxml=$XML_LOG_DIR/test_numerics.xml $TE_PATH/tests/pytorch/debug/test_numerics.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS || FAIL=1 -pytest -v -s --junitxml=$XML_LOG_DIR/test_log.xml $TE_PATH/tests/pytorch/debug/test_log.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR || FAIL=1 -NVTE_TORCH_COMPILE=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_api_features.xml $TE_PATH/tests/pytorch/debug/test_api_features.py -k "not (test_per_tensor_scaling or test_fake_quant or test_statistics_collection or test_statistics_multi_run)" --no-header --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR || FAIL=1 -pytest -v -s --junitxml=$XML_LOG_DIR/test_perf.xml $TE_PATH/tests/pytorch/debug/test_perf.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR || FAIL=1 +run_test_step() { + local xml_file=$1 + local test_path=$2 + local cmd=$3 + + + if [ "$PLATFORM" = "metax" ]; then + case "$test_path" in + *"test_numerics.py" | *"test_api_features.py" | *"test_sanity.py") + echo "-------------------------------------------------------" + echo "[SKIP] Platform MetaX: Ignoring $test_path" + echo "-------------------------------------------------------" + return 0 + ;; + esac + fi + + + echo "-------------------------------------------------------" + echo "[RUN] Executing: $test_path" + eval "$cmd" || FAIL=1 +} + + + +# Step 1: Sanity +run_test_step "test_sanity.xml" "$TE_PATH/tests/pytorch/debug/test_sanity.py" \ +"pytest -v -s --junitxml=$XML_LOG_DIR/test_sanity.xml $TE_PATH/tests/pytorch/debug/test_sanity.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS" + +# Step 2: Config +run_test_step "test_config.xml" "$TE_PATH/tests/pytorch/debug/test_config.py" \ +"pytest -v -s --junitxml=$XML_LOG_DIR/test_config.xml $TE_PATH/tests/pytorch/debug/test_config.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS" + +# Step 3: Numerics +run_test_step "test_numerics.xml" "$TE_PATH/tests/pytorch/debug/test_numerics.py" \ +"pytest -v -s --junitxml=$XML_LOG_DIR/test_numerics.xml $TE_PATH/tests/pytorch/debug/test_numerics.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS" + +# Step 4: Log +run_test_step "test_log.xml" "$TE_PATH/tests/pytorch/debug/test_log.py" \ +"pytest -v -s --junitxml=$XML_LOG_DIR/test_log.xml $TE_PATH/tests/pytorch/debug/test_log.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR" + +# Step 5: API Features +run_test_step "test_api_features.xml" "$TE_PATH/tests/pytorch/debug/test_api_features.py" \ +"NVTE_TORCH_COMPILE=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_api_features.xml $TE_PATH/tests/pytorch/debug/test_api_features.py -k \"not (test_per_tensor_scaling or test_fake_quant or test_statistics_collection or test_statistics_multi_run)\" --no-header --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR" + +# Step 6: Performance +run_test_step "test_perf.xml" "$TE_PATH/tests/pytorch/debug/test_perf.py" \ +"pytest -v -s --junitxml=$XML_LOG_DIR/test_perf.xml $TE_PATH/tests/pytorch/debug/test_perf.py --feature_dirs=$NVTE_TEST_NVINSPECT_FEATURE_DIRS --configs_dir=$NVTE_TEST_NVINSPECT_CONFIGS_DIR" + + + + +# Step 7: Sanity 2 +run_test_step "test_sanity_2.xml" "$TE_PATH/tests/pytorch/test_sanity.py" \ +"NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 \ +pytest -v -s --junitxml=$XML_LOG_DIR/test_sanity_2.xml $TE_PATH/tests/pytorch/test_sanity.py -k \"not (test_sanity_grouped_linear or test_inference_mode)\" --no-header" -# standard sanity and numerics tests with initialized debug -NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_sanity_2.xml $TE_PATH/tests/pytorch/test_sanity.py -k "not (test_sanity_grouped_linear or test_inference_mode)" --no-header || FAIL=1 -NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 pytest -v -s --junitxml=$XML_LOG_DIR/test_numerics_2.xml $TE_PATH/tests/pytorch/test_numerics.py -k "not (test_linear_accuracy or test_layernorm_linear_accuracy or test_layernorm_mlp_accuracy or test_grouped_linear_accuracy or test_transformer_layer_hidden_states_format or test_grouped_gemm)" --no-header || FAIL=1 +# Step 8: Numerics 2 +run_test_step "test_numerics_2.xml" "$TE_PATH/tests/pytorch/test_numerics.py" \ +"NVTE_TEST_NVINSPECT_ENABLED=1 NVTE_TEST_NVINSPECT_CONFIG_FILE=$NVTE_TEST_NVINSPECT_DUMMY_CONFIG_FILE NVTE_TEST_NVINSPECT_FEATURE_DIRS=$NVTE_TEST_NVINSPECT_FEATURE_DIRS PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 \ +pytest -v -s --junitxml=$XML_LOG_DIR/test_numerics_2.xml $TE_PATH/tests/pytorch/test_numerics.py -k \"not (test_linear_accuracy or test_layernorm_linear_accuracy or test_layernorm_mlp_accuracy or test_grouped_linear_accuracy or test_transformer_layer_hidden_states_format or test_grouped_gemm)\" --no-header" exit $FAIL diff --git a/qa/L0_pytorch_unittest/test.sh b/qa/L0_pytorch_unittest/test.sh index 9c5d9ac86f..99a1370ac4 100644 --- a/qa/L0_pytorch_unittest/test.sh +++ b/qa/L0_pytorch_unittest/test.sh @@ -1,57 +1,132 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# See LICENSE for license information. +#!/bin/bash -function error_exit() { - echo "Error: $1" - exit 1 -} -function test_fail() { - RET=1 - FAILED_CASES="$FAILED_CASES $1" +: ${TE_PATH:=/opt/transformerengine} +: ${XML_LOG_DIR:=/logs} +mkdir -p "$XML_LOG_DIR" + +pip install pytest==8.2.1 +FAIL=0 + +IS_CUDA_BACKEND=$(python3 -c "import torch; print('cuda' if torch.cuda.is_available() else 'cpu')" 2>/dev/null) + +test_fail() { + FAIL=1 echo "Error: sub-test failed: $1" } -RET=0 -FAILED_CASES="" -set -x +run_test_step() { + local xml_file=$1 + local test_path=$2 + local cmd=$3 + local label=$4 + + + if [ "$PLATFORM" = "metax" ]; then + case "$test_path" in + *"test_numerics.py" | \ + *"test_sanity.py" | \ + *"test_parallel_cross_entropy.py" | \ + *"test_cuda_graphs.py" | \ + *"test_fused_rope.py" | \ + *"test_gqa.py" | \ + *"test_fused_optimizer.py" | \ + *"test_multi_tensor.py" | \ + *"test_cpu_offloading.py" | \ + *"test_attention.py" | \ + *"test_kv_cache.py" | \ + *"test_checkpoint.py" | \ + *"test_fused_router.py") + echo "-------------------------------------------------------" + echo "[SKIP] Platform MetaX: Ignoring $label" + echo "-------------------------------------------------------" + return 0 + ;; + esac + fi + + if [[ "$IS_CUDA_BACKEND" == *"cuda"* ]]; then + if [[ "$test_path" == *"test_checkpoint.py" || "$test_path" == *"test_cpu_offloading.py" || "$test_path" == *"test_attention.py" ]]; then + echo "-------------------------------------------------------" + echo "[SKIP] CUDA Backend detected: Ignoring $label" + echo "-------------------------------------------------------" + return 0 + fi + fi + + + echo "-------------------------------------------------------" + echo "[RUN] Executing: $label" + + eval "$cmd" || test_fail "$label" +} + + +# Step: Sanity +run_test_step "pytest_test_sanity.xml" "$TE_PATH/tests/pytorch/test_sanity.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/test_sanity.py -k \"not (test_sanity_layernorm_mlp or test_sanity_gpt or test_sanity_bert or test_sanity_T5 or test_sanity_amp_and_nvfuser or test_sanity_drop_path or test_sanity_fused_qkv_params or test_sanity_gradient_accumulation_fusion or test_inference_mode or test_sanity_normalization_amp or test_sanity_layernorm_linear or test_sanity_linear_with_zero_tokens or test_sanity_grouped_linear)\" --no-header" "test_sanity.py" + +# Step: Recipe +run_test_step "pytest_test_recipe.xml" "$TE_PATH/tests/pytorch/test_recipe.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_recipe.xml $TE_PATH/tests/pytorch/test_recipe.py" "test_recipe.py" + +# Step: Deferred Init +run_test_step "pytest_test_deferred_init.xml" "$TE_PATH/tests/pytorch/test_deferred_init.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_deferred_init.xml $TE_PATH/tests/pytorch/test_deferred_init.py" "test_deferred_init.py" + +# Step: Numerics +run_test_step "pytest_test_numerics.xml" "$TE_PATH/tests/pytorch/test_numerics.py" \ +"PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_numerics.xml $TE_PATH/tests/pytorch/test_numerics.py -k \"not (test_layernorm_mlp_accuracy or test_grouped_linear_accuracy or test_gpt_cuda_graph or test_transformer_layer_hidden_states_format or test_grouped_gemm or test_noncontiguous or test_gpt_checkpointing or test_gpt_accuracy or test_mha_accuracy or test_linear_accuracy or test_linear_accuracy_delay_wgrad_compute or test_rmsnorm_accuracy or test_layernorm_accuracy or test_layernorm_linear_accuracy)\" --no-header" "test_numerics.py" + +# Step: CUDA Graphs +run_test_step "pytest_test_cuda_graphs.xml" "$TE_PATH/tests/pytorch/test_cuda_graphs.py" \ +"PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cuda_graphs.xml $TE_PATH/tests/pytorch/test_cuda_graphs.py" "test_cuda_graphs.py" + +# Step: JIT +run_test_step "pytest_test_jit.xml" "$TE_PATH/tests/pytorch/test_jit.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_jit.xml $TE_PATH/tests/pytorch/test_jit.py -k \"not (test_torch_dynamo)\"" "test_jit.py" + +# Step: Fused Rope +run_test_step "pytest_test_fused_rope.xml" "$TE_PATH/tests/pytorch/test_fused_rope.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_rope.xml $TE_PATH/tests/pytorch/test_fused_rope.py" "test_fused_rope.py" + +# Step: NVFP4 (Directory) +run_test_step "pytest_test_nvfp4.xml" "$TE_PATH/tests/pytorch/nvfp4" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_nvfp4.xml $TE_PATH/tests/pytorch/nvfp4" "test_nvfp4" + +# Step: Float8 Tensors +run_test_step "pytest_test_float8tensor.xml" "$TE_PATH/tests/pytorch/test_float8tensor.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8tensor.xml $TE_PATH/tests/pytorch/test_float8tensor.py" "test_float8tensor.py" + +# Step: GQA +run_test_step "pytest_test_gqa.xml" "$TE_PATH/tests/pytorch/test_gqa.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gqa.xml $TE_PATH/tests/pytorch/test_gqa.py" "test_gqa.py" + +# Step: Fused Optimizer +run_test_step "pytest_test_fused_optimizer.xml" "$TE_PATH/tests/pytorch/test_fused_optimizer.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_optimizer.xml $TE_PATH/tests/pytorch/test_fused_optimizer.py" "test_fused_optimizer.py" + +# Step: Parallel Cross Entropy +run_test_step "pytest_test_parallel_cross_entropy.xml" "$TE_PATH/tests/pytorch/test_parallel_cross_entropy.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_parallel_cross_entropy.xml $TE_PATH/tests/pytorch/test_parallel_cross_entropy.py" "test_parallel_cross_entropy.py" + +# Step: CPU Offloading +run_test_step "pytest_test_cpu_offloading.xml" "$TE_PATH/tests/pytorch/test_cpu_offloading.py" \ +"NVTE_FLASH_ATTN=0 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cpu_offloading.xml $TE_PATH/tests/pytorch/test_cpu_offloading.py" "test_cpu_offloading.py" + +# Step: Attention +run_test_step "pytest_test_attention.xml" "$TE_PATH/tests/pytorch/attention/test_attention.py" \ +"python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention.xml $TE_PATH/tests/pytorch/attention/test_attention.py" "test_attention.py" + +# Step: Checkpoint +run_test_step "pytest_test_checkpoint.xml" "$TE_PATH/tests/pytorch/test_checkpoint.py" \ +"NVTE_TEST_CHECKPOINT_ARTIFACT_PATH=$TE_PATH/artifacts/tests/pytorch/test_checkpoint python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_checkpoint.xml $TE_PATH/tests/pytorch/test_checkpoint.py" "test_checkpoint.py" -: ${TE_PATH:=/opt/transformerengine} -: ${XML_LOG_DIR:=/logs} -mkdir -p "$XML_LOG_DIR" -pip3 install pytest==8.2.1 || error_exit "Failed to install pytest" - -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_sanity.xml $TE_PATH/tests/pytorch/test_sanity.py -k "not (test_sanity_layernorm_mlp or test_sanity_gpt or test_sanity_bert or test_sanity_T5 or test_sanity_amp_and_nvfuser or test_sanity_drop_path or test_sanity_fused_qkv_params or test_sanity_gradient_accumulation_fusion or test_inference_mode or test_sanity_normalization_amp or test_sanity_layernorm_linear or test_sanity_linear_with_zero_tokens or test_sanity_grouped_linear)" --no-header || test_fail "test_sanity.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_recipe.xml $TE_PATH/tests/pytorch/test_recipe.py || test_fail "test_recipe.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_deferred_init.xml $TE_PATH/tests/pytorch/test_deferred_init.py || test_fail "test_deferred_init.py" -PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_numerics.xml $TE_PATH/tests/pytorch/test_numerics.py -k "not (test_layernorm_mlp_accuracy or test_grouped_linear_accuracy or test_gpt_cuda_graph or test_transformer_layer_hidden_states_format or test_grouped_gemm or test_noncontiguous or test_gpt_checkpointing or test_gpt_accuracy or test_mha_accuracy or test_linear_accuracy or test_linear_accuracy_delay_wgrad_compute or test_rmsnorm_accuracy or test_layernorm_accuracy or test_layernorm_linear_accuracy)" --no-header || test_fail "test_numerics.py" -# PYTORCH_JIT=0 NVTE_TORCH_COMPILE=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 NVTE_FUSED_ATTN=0 python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cuda_graphs.xml $TE_PATH/tests/pytorch/test_cuda_graphs.py || test_fail "test_cuda_graphs.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_jit.xml $TE_PATH/tests/pytorch/test_jit.py -k "not (test_torch_dynamo)" || test_fail "test_jit.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_rope.xml $TE_PATH/tests/pytorch/test_fused_rope.py || test_fail "test_fused_rope.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_nvfp4.xml $TE_PATH/tests/pytorch/nvfp4 || test_fail "test_nvfp4" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8tensor.xml $TE_PATH/tests/pytorch/test_float8tensor.py || test_fail "test_float8tensor.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8blockwisetensor.xml $TE_PATH/tests/pytorch/test_float8blockwisetensor.py || test_fail "test_float8blockwisetensor.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8_blockwise_scaling_exact.xml $TE_PATH/tests/pytorch/test_float8_blockwise_scaling_exact.py || test_fail "test_float8_blockwise_scaling_exact.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_float8_blockwise_gemm_exact.xml $TE_PATH/tests/pytorch/test_float8_blockwise_gemm_exact.py || test_fail "test_float8_blockwise_gemm_exact.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_gqa.xml $TE_PATH/tests/pytorch/test_gqa.py || test_fail "test_gqa.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_optimizer.xml $TE_PATH/tests/pytorch/test_fused_optimizer.py || test_fail "test_fused_optimizer.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_multi_tensor.xml $TE_PATH/tests/pytorch/test_multi_tensor.py || test_fail "test_multi_tensor.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fusible_ops.xml $TE_PATH/tests/pytorch/test_fusible_ops.py -k "not (test_basic_linear or test_layer_norm or test_rmsnorm or test_forward_linear_bias_activation or test_backward_add_rmsnorm or test_layernorm_mlp or test_activation or test_clamped_swiglu or test_dropout or test_forward_linear_bias_add or test_forward_linear_scale_add or test_linear)" || test_fail "test_fusible_ops.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_permutation.xml $TE_PATH/tests/pytorch/test_permutation.py -k "not (test_permutation_index_map or test_permutation_single_case)" || test_fail "test_permutation.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_parallel_cross_entropy.xml $TE_PATH/tests/pytorch/test_parallel_cross_entropy.py || test_fail "test_parallel_cross_entropy.py" -# NVTE_FLASH_ATTN=0 python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_cpu_offloading.xml $TE_PATH/tests/pytorch/test_cpu_offloading.py || test_fail "test_cpu_offloading.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_attention.xml $TE_PATH/tests/pytorch/attention/test_attention.py || test_fail "test_attention.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_kv_cache.xml $TE_PATH/tests/pytorch/attention/test_kv_cache.py || test_fail "test_kv_cache.py" -python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_hf_integration.xml $TE_PATH/tests/pytorch/test_hf_integration.py || test_fail "test_hf_integration.py" -# NVTE_TEST_CHECKPOINT_ARTIFACT_PATH=$TE_PATH/artifacts/tests/pytorch/test_checkpoint python3 -m pytest --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_checkpoint.xml $TE_PATH/tests/pytorch/test_checkpoint.py || test_fail "test_checkpoint.py" -# python3 -m pytest -s -v --tb=auto --junitxml=$XML_LOG_DIR/pytest_test_fused_router.xml $TE_PATH/tests/pytorch/test_fused_router.py || test_fail "test_fused_router.py" - -if [ "$RET" -ne 0 ]; then - echo "Error in the following test cases:$FAILED_CASES" +if [ "$FAIL" -ne 0 ]; then + echo "Some tests failed." exit 1 fi -echo "All tests passed" +echo "All assigned tests passed (some might have been skipped)." exit 0