fix:add image-pull-policy: never #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Common Unit Tests | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| platform: | ||
| required: true | ||
| type: string | ||
| description: Platform name (e.g., cuda, default) | ||
| device: | ||
| required: true | ||
| type: string | ||
| description: Device type (e.g., a100, a800, h100, generic) | ||
| image: | ||
| required: true | ||
| type: string | ||
| runs_on: | ||
| required: true | ||
| type: string | ||
| container_volumes: | ||
| required: true | ||
| type: string | ||
| container_options: | ||
| required: true | ||
| type: string | ||
| ignored_tests: | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| description: JSON array of test files to ignore (e.g., '["tests/unit_tests/test_a.py", "tests/unit_tests/test_b.py"]') | ||
| jobs: | ||
| # Detect which test groups need to run based on changed files | ||
| detect_changes: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| core: ${{ steps.filter.outputs.core }} | ||
| transformer: ${{ steps.filter.outputs.transformer }} | ||
| models: ${{ steps.filter.outputs.models }} | ||
| distributed: ${{ steps.filter.outputs.distributed }} | ||
| dist_checkpointing: ${{ steps.filter.outputs.dist_checkpointing }} | ||
| tensor_parallel: ${{ steps.filter.outputs.tensor_parallel }} | ||
| pipeline_parallel: ${{ steps.filter.outputs.pipeline_parallel }} | ||
| data: ${{ steps.filter.outputs.data }} | ||
| fusions: ${{ steps.filter.outputs.fusions }} | ||
| others: ${{ steps.filter.outputs.others }} | ||
| steps: | ||
| - name: Checkout source code | ||
| uses: actions/checkout@v6 | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: Detect changed paths | ||
| id: filter | ||
| run: | | ||
| set -euo pipefail | ||
| # Get the base ref for comparison | ||
| if [ "${{ github.event_name }}" == "pull_request" ]; then | ||
| BASE_REF="origin/${{ github.base_ref }}" | ||
| git fetch origin ${{ github.base_ref }} --depth=1 | ||
| else | ||
| # For push events, compare with the previous commit | ||
| BASE_REF="HEAD~1" | ||
| fi | ||
| # Get list of changed files | ||
| CHANGED_FILES=$(git diff --name-only $BASE_REF...HEAD 2>/dev/null || git diff --name-only $BASE_REF HEAD) | ||
| # Function to check if any pattern matches | ||
| check_patterns() { | ||
| local patterns="$1" | ||
| for pattern in $patterns; do | ||
| if echo "$CHANGED_FILES" | grep -qE "$pattern"; then | ||
| echo "true" | ||
| return | ||
| fi | ||
| done | ||
| echo "false" | ||
| } | ||
| # Core tests: root-level test files and core megatron files | ||
| CORE_PATTERNS="^tests/unit_tests/test_.*\.py$ ^megatron/core/[^/]*\.py$ ^megatron/core/__init__\.py$" | ||
| echo "core=$(check_patterns "$CORE_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Transformer tests | ||
| TRANSFORMER_PATTERNS="^tests/unit_tests/transformer/ ^megatron/core/transformer/" | ||
| echo "transformer=$(check_patterns "$TRANSFORMER_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Models tests | ||
| MODELS_PATTERNS="^tests/unit_tests/models/ ^megatron/core/models/" | ||
| echo "models=$(check_patterns "$MODELS_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Distributed tests | ||
| DISTRIBUTED_PATTERNS="^tests/unit_tests/distributed/ ^megatron/core/distributed/" | ||
| echo "distributed=$(check_patterns "$DISTRIBUTED_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Dist checkpointing tests | ||
| DIST_CKPT_PATTERNS="^tests/unit_tests/dist_checkpointing/ ^megatron/core/dist_checkpointing/" | ||
| echo "dist_checkpointing=$(check_patterns "$DIST_CKPT_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Tensor parallel tests | ||
| TP_PATTERNS="^tests/unit_tests/tensor_parallel/ ^megatron/core/tensor_parallel/" | ||
| echo "tensor_parallel=$(check_patterns "$TP_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Pipeline parallel tests | ||
| PP_PATTERNS="^tests/unit_tests/pipeline_parallel/ ^megatron/core/pipeline_parallel/" | ||
| echo "pipeline_parallel=$(check_patterns "$PP_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Data tests | ||
| DATA_PATTERNS="^tests/unit_tests/data/ ^megatron/core/datasets/" | ||
| echo "data=$(check_patterns "$DATA_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Fusions tests | ||
| FUSIONS_PATTERNS="^tests/unit_tests/fusions/ ^megatron/core/fusions/" | ||
| echo "fusions=$(check_patterns "$FUSIONS_PATTERNS")" >> $GITHUB_OUTPUT | ||
| # Others tests (export, post_training, tokenizers, utils) | ||
| OTHERS_PATTERNS="^tests/unit_tests/export/ ^tests/unit_tests/post_training/ ^tests/unit_tests/tokenizers/ ^tests/unit_tests/utils/ ^megatron/core/export/ ^megatron/post_training/ ^megatron/tokenizer/ ^megatron/core/utils/" | ||
| echo "others=$(check_patterns "$OTHERS_PATTERNS")" >> $GITHUB_OUTPUT | ||
| echo "=== Change detection complete ===" | ||
| unit_test: | ||
| needs: detect_changes | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| runs-on: ${{ fromJson(inputs.runs_on) }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| test_group: | ||
| - name: core | ||
| path: "tests/unit_tests/test_*.py" | ||
| description: "Core unit tests" | ||
| - name: transformer | ||
| path: "tests/unit_tests/transformer/" | ||
| description: "Transformer tests" | ||
| - name: models | ||
| path: "tests/unit_tests/models/" | ||
| description: "Model tests" | ||
| - name: distributed | ||
| path: "tests/unit_tests/distributed/" | ||
| description: "Distributed tests" | ||
| - name: dist_checkpointing | ||
| path: "tests/unit_tests/dist_checkpointing/" | ||
| description: "Distributed checkpointing tests" | ||
| - name: tensor_parallel | ||
| path: "tests/unit_tests/tensor_parallel/" | ||
| description: "Tensor parallel tests" | ||
| - name: pipeline_parallel | ||
| path: "tests/unit_tests/pipeline_parallel/" | ||
| description: "Pipeline parallel tests" | ||
| - name: data | ||
| path: "tests/unit_tests/data/" | ||
| description: "Data tests" | ||
| - name: fusions | ||
| path: "tests/unit_tests/fusions/" | ||
| description: "Fusion tests" | ||
| - name: others | ||
| path: "tests/unit_tests/export/ tests/unit_tests/post_training/ tests/unit_tests/tokenizers/ tests/unit_tests/utils/" | ||
| description: "Other tests (a2a_overlap, export, post_training, tokenizers, utils)" | ||
| name: unit-${{ inputs.device }}-${{ matrix.test_group.name }} | ||
| container: | ||
| image: ${{ inputs.image }} | ||
| image-pull-policy: never | ||
| ports: | ||
| - 80 | ||
| volumes: ${{ fromJson(inputs.container_volumes) }} | ||
| options: --pull never ${{ inputs.container_options }} | ||
| steps: | ||
| # Check if this test group should run based on changed files or "full ci" label | ||
| - name: Check if tests should run | ||
| id: should_run | ||
| run: | | ||
| TEST_GROUP='${{ matrix.test_group.name }}' | ||
| FULL_CI='${{ contains(github.event.pull_request.labels.*.name, 'full ci') }}' | ||
| # If "full ci" label is present, run all tests | ||
| if [ "$FULL_CI" == "true" ]; then | ||
| echo "should_run=true" >> $GITHUB_OUTPUT | ||
| exit 0 | ||
| fi | ||
| # Check change detection output for this test group | ||
| case "$TEST_GROUP" in | ||
| core) CHANGED='${{ needs.detect_changes.outputs.core }}' ;; | ||
| transformer) CHANGED='${{ needs.detect_changes.outputs.transformer }}' ;; | ||
| models) CHANGED='${{ needs.detect_changes.outputs.models }}' ;; | ||
| distributed) CHANGED='${{ needs.detect_changes.outputs.distributed }}' ;; | ||
| dist_checkpointing) CHANGED='${{ needs.detect_changes.outputs.dist_checkpointing }}' ;; | ||
| tensor_parallel) CHANGED='${{ needs.detect_changes.outputs.tensor_parallel }}' ;; | ||
| pipeline_parallel) CHANGED='${{ needs.detect_changes.outputs.pipeline_parallel }}' ;; | ||
| data) CHANGED='${{ needs.detect_changes.outputs.data }}' ;; | ||
| fusions) CHANGED='${{ needs.detect_changes.outputs.fusions }}' ;; | ||
| others) CHANGED='${{ needs.detect_changes.outputs.others }}' ;; | ||
| *) CHANGED="true" ;; | ||
| esac | ||
| echo "should_run=$CHANGED" >> $GITHUB_OUTPUT | ||
| - name: Checkout source code | ||
| if: steps.should_run.outputs.should_run == 'true' | ||
| uses: actions/checkout@v6 | ||
| with: | ||
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | ||
| ref: ${{ github.event.pull_request.head.ref || github.ref }} | ||
| ssh-strict: true | ||
| ssh-user: git | ||
| ssh-key: ${{ secrets.RUNNER_SSH_KEY }} | ||
| persist-credentials: false | ||
| clean: true | ||
| sparse-checkout-cone-mode: true | ||
| fetch-tags: false | ||
| show-progress: true | ||
| lfs: false | ||
| - name: Set safe directory | ||
| if: steps.should_run.outputs.should_run == 'true' | ||
| run: | | ||
| git config --global --add safe.directory $GITHUB_WORKSPACE | ||
| - name: Setup Python environment | ||
| if: steps.should_run.outputs.should_run == 'true' | ||
| working-directory: ${{ github.workspace }} | ||
| run: | | ||
| set -euo pipefail | ||
| echo "Python location: $(which python)" | ||
| echo "Python version: $(python --version)" | ||
| # Install package in development mode | ||
| pip install boto3==1.42.1 | ||
| pip install -e . --no-deps | ||
| timeout-minutes: 30 | ||
| - name: Run unit tests - ${{ matrix.test_group.name }} | ||
| if: steps.should_run.outputs.should_run == 'true' | ||
| id: unit_test | ||
| working-directory: ${{ github.workspace }} | ||
| run: | | ||
| set -euo pipefail | ||
| PLATFORM='${{ inputs.platform }}' | ||
| DEVICE='${{ inputs.device }}' | ||
| TEST_GROUP='${{ matrix.test_group.name }}' | ||
| TEST_PATH='${{ matrix.test_group.path }}' | ||
| echo "Running unit tests: $TEST_GROUP" | ||
| echo "Test path: $TEST_PATH" | ||
| echo "Platform: $PLATFORM" | ||
| echo "Device: $DEVICE" | ||
| # Set environment variables | ||
| export PYTHONPATH=$GITHUB_WORKSPACE:${PYTHONPATH:-} | ||
| export TORCHINDUCTOR_CACHE_DIR=/tmp/.torch_inductor_cache | ||
| mkdir -p $TORCHINDUCTOR_CACHE_DIR | ||
| # Build ignore options from ignored_tests input | ||
| IGNORED_TESTS='${{ inputs.ignored_tests }}' | ||
| IGNORE_OPTS="" | ||
| if [ -n "$IGNORED_TESTS" ] && [ "$IGNORED_TESTS" != "[]" ]; then | ||
| echo "Parsing ignored tests list..." | ||
| # Parse JSON array and build --ignore options | ||
| IGNORE_OPTS=$(echo "$IGNORED_TESTS" | python3 -c "import sys, json; tests = json.load(sys.stdin); print(' '.join(['--deselect=' + t for t in tests])) if tests else None") | ||
| if [ -n "$IGNORE_OPTS" ]; then | ||
| echo "Ignoring tests: $IGNORE_OPTS" | ||
| fi | ||
| fi | ||
| # Run unit tests with torchrun | ||
| # Each test group runs in isolation to avoid state pollution | ||
| NUM_GPUS=$(python3 -c "import torch; print(torch.cuda.device_count())") | ||
| echo "=== Detected $NUM_GPUS GPUs, starting torchrun ===" | ||
| # 使用动态获取的数量启动测试 | ||
| torchrun --nproc_per_node=$NUM_GPUS -m pytest -v $TEST_PATH $IGNORE_OPTS -p no:randomly | ||
| exit_code=$? | ||
| echo "exit_code=$exit_code" >> $GITHUB_OUTPUT | ||
| exit $exit_code | ||
| timeout-minutes: 60 | ||
