add L0 unit test

zhoujiamei · zhoujiamei · commit c0c75d1f2fd1 · 2026-03-19T03:21:25.000Z
diff --git a/.github/workflows/unit_tests_common.yml b/.github/workflows/unit_tests_common.yml
@@ -27,99 +27,37 @@ on:
         required: false
         type: string
         default: ''
-        description: JSON array of test files to ignore (e.g., '["tests/unit_tests/test_a.py", "tests/unit_tests/test_b.py"]')
+        description: JSON array of test files to ignore
 
 jobs:
-  # Detect which test groups need to run based on changed files
+  # 1. Change Detection: Monitors both new unit tests and legacy QA paths
   detect_changes:
     runs-on: ubuntu-latest
     outputs:
       core: ${{ steps.filter.outputs.core }}
-      transformer: ${{ steps.filter.outputs.transformer }}
-      models: ${{ steps.filter.outputs.models }}
-      distributed: ${{ steps.filter.outputs.distributed }}
-      dist_checkpointing: ${{ steps.filter.outputs.dist_checkpointing }}
-      tensor_parallel: ${{ steps.filter.outputs.tensor_parallel }}
-      pipeline_parallel: ${{ steps.filter.outputs.pipeline_parallel }}
-      data: ${{ steps.filter.outputs.data }}
-      fusions: ${{ steps.filter.outputs.fusions }}
-      others: ${{ steps.filter.outputs.others }}
+      qa_l0: ${{ steps.filter.outputs.qa_l0 }}
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Detect changed paths
         id: filter
         run: |
           set -euo pipefail
-
-          # Get the base ref for comparison
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            BASE_REF="origin/${{ github.base_ref }}"
-            git fetch origin ${{ github.base_ref }} --depth=1
-          else
-            # For push events, compare with the previous commit
-            BASE_REF="HEAD~1"
-          fi
-          # Get list of changed files
+          # Identify base reference for diff
+          BASE_REF="${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || 'HEAD~1' }}"
+          [ "${{ github.event_name }}" == "pull_request" ] && git fetch origin ${{ github.base_ref }} --depth=1
+          
           CHANGED_FILES=$(git diff --name-only $BASE_REF...HEAD 2>/dev/null || git diff --name-only $BASE_REF HEAD)
 
-          # Function to check if any pattern matches
-          check_patterns() {
-            local patterns="$1"
-            for pattern in $patterns; do
-              if echo "$CHANGED_FILES" | grep -qE "$pattern"; then
-                echo "true"
-                return
-              fi
-            done
-            echo "false"
-          }
-
-          # Core tests: root-level test files and core megatron files
-          CORE_PATTERNS="^tests/unit_tests/test_.*\.py$ ^megatron/core/[^/]*\.py$ ^megatron/core/__init__\.py$"
-          echo "core=$(check_patterns "$CORE_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Transformer tests
-          TRANSFORMER_PATTERNS="^tests/unit_tests/transformer/ ^megatron/core/transformer/"
-          echo "transformer=$(check_patterns "$TRANSFORMER_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Models tests
-          MODELS_PATTERNS="^tests/unit_tests/models/ ^megatron/core/models/"
-          echo "models=$(check_patterns "$MODELS_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Distributed tests
-          DISTRIBUTED_PATTERNS="^tests/unit_tests/distributed/ ^megatron/core/distributed/"
-          echo "distributed=$(check_patterns "$DISTRIBUTED_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Dist checkpointing tests
-          DIST_CKPT_PATTERNS="^tests/unit_tests/dist_checkpointing/ ^megatron/core/dist_checkpointing/"
-          echo "dist_checkpointing=$(check_patterns "$DIST_CKPT_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Tensor parallel tests
-          TP_PATTERNS="^tests/unit_tests/tensor_parallel/ ^megatron/core/tensor_parallel/"
-          echo "tensor_parallel=$(check_patterns "$TP_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Pipeline parallel tests
-          PP_PATTERNS="^tests/unit_tests/pipeline_parallel/ ^megatron/core/pipeline_parallel/"
-          echo "pipeline_parallel=$(check_patterns "$PP_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Data tests
-          DATA_PATTERNS="^tests/unit_tests/data/ ^megatron/core/datasets/"
-          echo "data=$(check_patterns "$DATA_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Fusions tests
-          FUSIONS_PATTERNS="^tests/unit_tests/fusions/ ^megatron/core/fusions/"
-          echo "fusions=$(check_patterns "$FUSIONS_PATTERNS")" >> $GITHUB_OUTPUT
-
-          # Others tests (export, post_training, tokenizers, utils)
-          OTHERS_PATTERNS="^tests/unit_tests/export/ ^tests/unit_tests/post_training/ ^tests/unit_tests/tokenizers/ ^tests/unit_tests/utils/ ^megatron/core/export/ ^megatron/post_training/ ^megatron/tokenizer/ ^megatron/core/utils/"
-          echo "others=$(check_patterns "$OTHERS_PATTERNS")" >> $GITHUB_OUTPUT
-
-          echo "=== Change detection complete ==="
+          # Logic for core framework and new pytest suite
+          echo "core=$(echo "$CHANGED_FILES" | grep -qE "^tests/unit_tests/|^megatron/core/" && echo "true" || echo "false")" >> $GITHUB_OUTPUT
+          # Logic for legacy QA L0 scripts and PyTorch specific paths
+          echo "qa_l0=$(echo "$CHANGED_FILES" | grep -qE "^qa/L0_|^transformer_engine/|^tests/pytorch/" && echo "true" || echo "false")" >> $GITHUB_OUTPUT
 
+  # 2. Unified Test Execution: Supports both pytest and legacy shell scripts
   unit_test:
     needs: detect_changes
     defaults:
@@ -130,153 +68,112 @@ jobs:
       fail-fast: false
       matrix:
         test_group:
+          # Modern Pytest architecture
           - name: core
             path: "tests/unit_tests/test_*.py"
-            description: "Core unit tests"
-          - name: transformer
-            path: "tests/unit_tests/transformer/"
-            description: "Transformer tests"
-          - name: models
-            path: "tests/unit_tests/models/"
-            description: "Model tests"
+            test_type: "pytest"
           - name: distributed
             path: "tests/unit_tests/distributed/"
-            description: "Distributed tests"
-          - name: dist_checkpointing
-            path: "tests/unit_tests/dist_checkpointing/"
-            description: "Distributed checkpointing tests"
-          - name: tensor_parallel
-            path: "tests/unit_tests/tensor_parallel/"
-            description: "Tensor parallel tests"
-          - name: pipeline_parallel
-            path: "tests/unit_tests/pipeline_parallel/"
-            description: "Pipeline parallel tests"
-          - name: data
-            path: "tests/unit_tests/data/"
-            description: "Data tests"
-          - name: fusions
-            path: "tests/unit_tests/fusions/"
-            description: "Fusion tests"
-          - name: others
-            path: "tests/unit_tests/export/ tests/unit_tests/post_training/ tests/unit_tests/tokenizers/ tests/unit_tests/utils/"
-            description: "Other tests (a2a_overlap, export, post_training, tokenizers, utils)"
+            test_type: "pytest"
+          
+          # Integrated legacy QA L0 groups
+          - name: pytorch_unittest
+            path: "qa/L0_pytorch_unittest/test.sh"
+            test_type: "sh"
+          - name: pytorch_debug
+            path: "qa/L0_pytorch_debug_unittest/test.sh"
+            test_type: "sh"
+          - name: pytorch_lint
+            path: "qa/L0_pytorch_lint/test.sh"
+            test_type: "sh"
+          - name: pytorch_wheel
+            path: "qa/L0_pytorch_wheel/test.sh"
+            test_type: "sh"
+
     name: unit-${{ inputs.device }}-${{ matrix.test_group.name }}
     container:
       image: ${{ inputs.image }}
-      
       ports:
         - 80
       volumes: ${{ fromJson(inputs.container_volumes) }}
-      # options: ${{ inputs.container_options }}
+      # Crucial: --pull never prevents internet access; container_options should include --privileged
       options: --pull never ${{ inputs.container_options }}
+
     steps:
-      # Check if this test group should run based on changed files or "full ci" label
       - name: Check if tests should run
         id: should_run
         run: |
-          TEST_GROUP='${{ matrix.test_group.name }}'
+          GROUP='${{ matrix.test_group.name }}'
           FULL_CI='${{ contains(github.event.pull_request.labels.*.name, 'full ci') }}'
-
-          # If "full ci" label is present, run all tests
-          if [ "$FULL_CI" == "true" ]; then
-            echo "should_run=true" >> $GITHUB_OUTPUT
-            exit 0
+          
+          if [ "$FULL_CI" == "true" ]; then 
+            echo "should_run=true" >> $GITHUB_OUTPUT; exit 0
           fi
 
-          # Check change detection output for this test group
-          case "$TEST_GROUP" in
-            core) CHANGED='${{ needs.detect_changes.outputs.core }}' ;;
-            transformer) CHANGED='${{ needs.detect_changes.outputs.transformer }}' ;;
-            models) CHANGED='${{ needs.detect_changes.outputs.models }}' ;;
-            distributed) CHANGED='${{ needs.detect_changes.outputs.distributed }}' ;;
-            dist_checkpointing) CHANGED='${{ needs.detect_changes.outputs.dist_checkpointing }}' ;;
-            tensor_parallel) CHANGED='${{ needs.detect_changes.outputs.tensor_parallel }}' ;;
-            pipeline_parallel) CHANGED='${{ needs.detect_changes.outputs.pipeline_parallel }}' ;;
-            data) CHANGED='${{ needs.detect_changes.outputs.data }}' ;;
-            fusions) CHANGED='${{ needs.detect_changes.outputs.fusions }}' ;;
-            others) CHANGED='${{ needs.detect_changes.outputs.others }}' ;;
-            *) CHANGED="true" ;;
-          esac
-
+          if [[ "$GROUP" == "pytorch_"* ]]; then
+            CHANGED='${{ needs.detect_changes.outputs.qa_l0 }}'
+          else
+            CHANGED='${{ needs.detect_changes.outputs.core }}'
+          fi
           echo "should_run=$CHANGED" >> $GITHUB_OUTPUT
-  
 
       - name: Checkout source code
         if: steps.should_run.outputs.should_run == 'true'
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
-          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
-          ref: ${{ github.event.pull_request.head.ref || github.ref }}
-          ssh-strict: true
-          ssh-user: git
-          ssh-key: ${{ secrets.RUNNER_SSH_KEY }}
-          persist-credentials: false
-          clean: true
-          sparse-checkout-cone-mode: true
-          fetch-tags: false
-          show-progress: true
-          lfs: false
-
-      - name: Set safe directory
-        if: steps.should_run.outputs.should_run == 'true'
-        run: |
-          git config --global --add safe.directory $GITHUB_WORKSPACE
+          set-safe-directory: true
 
-      - name: Setup Python environment
+      - name: Metax Environment Setup
         if: steps.should_run.outputs.should_run == 'true'
-        working-directory: ${{ github.workspace }}
         run: |
           set -euo pipefail
-          echo "Python location: $(which python)"
-          echo "Python version: $(python --version)"
-          # Install package in development mode
-          pip install boto3==1.42.1
+          # Dynamically set MACA library path if drivers are mounted
+          if [ -d "/usr/local/maca/lib" ]; then
+            echo "Setting MACA library path..."
+            echo "LD_LIBRARY_PATH=/usr/local/maca/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV
+          fi
+          
+          # Install dependencies required by legacy QA scripts
+          pip install --no-cache-dir transformers expecttest boto3==1.42.1
+          # Install current repo in editable mode without re-building
           pip install -e . --no-deps
-        timeout-minutes: 30
+        timeout-minutes: 20
 
-      - name: Run unit tests - ${{ matrix.test_group.name }}
+      - name: Execute Tests
         if: steps.should_run.outputs.should_run == 'true'
-        id: unit_test
         working-directory: ${{ github.workspace }}
         run: |
           set -euo pipefail
-
-          PLATFORM='${{ inputs.platform }}'
-          DEVICE='${{ inputs.device }}'
-          TEST_GROUP='${{ matrix.test_group.name }}'
-          TEST_PATH='${{ matrix.test_group.path }}'
-
-          echo "Running unit tests: $TEST_GROUP"
-          echo "Test path: $TEST_PATH"
-          echo "Platform: $PLATFORM"
-          echo "Device: $DEVICE"
-
-          # Set environment variables
+          
+          # Base environment variables (required for legacy scripts)
+          export TE_PATH=$GITHUB_WORKSPACE
           export PYTHONPATH=$GITHUB_WORKSPACE:${PYTHONPATH:-}
-          export TORCHINDUCTOR_CACHE_DIR=/tmp/.torch_inductor_cache
-          mkdir -p $TORCHINDUCTOR_CACHE_DIR
-
-          # Build ignore options from ignored_tests input
-          IGNORED_TESTS='${{ inputs.ignored_tests }}'
-          IGNORE_OPTS=""
-          if [ -n "$IGNORED_TESTS" ] && [ "$IGNORED_TESTS" != "[]" ]; then
-            echo "Parsing ignored tests list..."
-            # Parse JSON array and build --ignore options
-            IGNORE_OPTS=$(echo "$IGNORED_TESTS" | python3 -c "import sys, json; tests = json.load(sys.stdin); print(' '.join(['--deselect=' + t for t in tests])) if tests else None")
-            if [ -n "$IGNORE_OPTS" ]; then
-              echo "Ignoring tests: $IGNORE_OPTS"
+          export RUN_LOG=/tmp/te_logs && mkdir -p $RUN_LOG
+
+          # Detect local GPU count dynamically (e.g., 2 for C500)
+          NUM_GPUS=$(python3 -c "import torch; print(torch.cuda.device_count() if torch.cuda.is_available() else 0)")
+          echo "=== Detected $NUM_GPUS GPUs, starting execution ==="
+
+          if [ "${{ matrix.test_group.test_type }}" == "sh" ]; then
+            echo "::group::Executing Legacy Shell Script"
+            # Dynamically detect TE installation path to avoid "SO not found" errors
+            export TE_LIB_PATH=$(python3 -c "import transformer_engine; import os; print(os.path.dirname(transformer_engine.__file__))")
+            
+            # Special env for Lint group
+            if [[ "${{ matrix.test_group.name }}" == *"lint"* ]]; then
+               export CPP_ONLY=0
+               export PYTHON_ONLY=0
             fi
+            bash ${{ matrix.test_group.path }}
+            echo "::endgroup::"
+          else
+            echo "::group::Executing Modern Pytest"
+            # Build deselect options from ignored_tests input
+            IGNORED_TESTS='${{ inputs.ignored_tests }}'
+            IGNORE_OPTS=$(echo "$IGNORED_TESTS" | python3 -c "import sys, json; t = json.load(sys.stdin); print(' '.join(['--deselect=' + i for i in t])) if t else print('')")
+            
+            # Run distributed tests with torchrun
+            torchrun --nproc_per_node=$NUM_GPUS -m pytest -v ${{ matrix.test_group.path }} $IGNORE_OPTS -p no:randomly
+            echo "::endgroup::"
           fi
-
-          # Run unit tests with torchrun
-          # Each test group runs in isolation to avoid state pollution
-          NUM_GPUS=$(python3 -c "import torch; print(torch.cuda.device_count())")
-          echo "=== Detected $NUM_GPUS GPUs, starting torchrun ==="
-
-          # 使用动态获取的数量启动测试
-          torchrun --nproc_per_node=$NUM_GPUS -m pytest -v $TEST_PATH $IGNORE_OPTS -p no:randomly
-
-          exit_code=$?
-          echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
-          exit $exit_code
-        timeout-minutes: 60
+        timeout-minutes: 60
