TransformerEngine-FL/.github/workflows/integration_tests_common.yml at main · ltllt1/TransformerEngine-FL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
name: Common Integration Tests

on:
  workflow_call:
    inputs:
      platform:
        required: true
        type: string
      device:
        required: true
        type: string
      image:
        required: true
        type: string
      runs_on:
        required: true
        type: string
      container_volumes:
        required: true
        type: string
      container_options:
        required: true
        type: string
      # Platform-specific environment setup script path (from platform config)
      setup_script:
        required: false
        type: string
        default: ''
      # Platform-specific build environment variables (JSON object from config)
      build_env:
        required: false
        type: string
        default: '{}'

jobs:
  integration_test:
    defaults:
      run:
        shell: bash
    runs-on: ${{ fromJson(inputs.runs_on) }}
    strategy:
      fail-fast: false
      matrix:
        test_group:
          - name: pytorch_mcore_integration
            path: "qa/L1_pytorch_mcore_integration/test.sh"
            test_type: "integration"
    name: integration-${{ inputs.device }}-${{ matrix.test_group.name }}
    container:
      image: ${{ inputs.image }}
      volumes: ${{ fromJson(inputs.container_volumes) }}
      options: --pull never ${{ inputs.container_options }}

    steps:
      # Cuda requires git safe.directory configuration and 3 checkout attempts to handle submodule-heavy repos
      - name: Configure Git Safe Directory on Cuda
        if: inputs.platform == 'cuda'
        run: /usr/bin/git config --global safe.directory '*'

      - name: Checkout Source Code on Cuda (attempt 1)
        id: checkout1
        if: inputs.platform == 'cuda'
        uses: actions/checkout@v4
        continue-on-error: true
        with:
          fetch-depth: 0
          submodules: recursive
          set-safe-directory: true

      - name: Checkout Source Code on Cuda (attempt 2)
        id: checkout2
        if: inputs.platform == 'cuda' && steps.checkout1.outcome == 'failure'
        uses: actions/checkout@v4
        continue-on-error: true
        with:
          fetch-depth: 0
          submodules: recursive
          set-safe-directory: true

      - name: Checkout Source Code on Cuda (attempt 3)
        id: checkout3
        if: inputs.platform == 'cuda' && steps.checkout2.outcome == 'failure'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
          set-safe-directory: true

      # Metax requires to clean vscode-remote-container
      - name: Configure Clean Git Env on Metax
        if: inputs.platform == 'metax'
        run: |
          git config --global --unset-all credential.helper 2>/dev/null || true
          git config --system --unset-all credential.helper 2>/dev/null || true

      # Metax no need submodules
      - name: Checkout Source Code on Metax
        if: inputs.platform == 'metax'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Environment Setup
        if: inputs.setup_script != ''
        run: |
          bash $GITHUB_WORKSPACE/${{ inputs.setup_script }}

      - name: Execute Tests
        env:
          TE_PATH: ${{ github.workspace }}
          TE_FL_PREFER: vendor
          MCORE_REPO_URL: https://github.com/flagos-ai/Megatron-LM-FL.git
          MCORE_REF: main
        run: |
          set -euo pipefail

          # Activate conda environment
          if ${{inputs.platform == 'metax'}}; then
            source /opt/conda/etc/profile.d/conda.sh
            conda activate base
          else
            source /opt/miniconda3/etc/profile.d/conda.sh
            conda activate flagscale-train
          fi
          echo "PATH=$PATH" >> $GITHUB_ENV
          export TE_LIB_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/transformer_engine

          echo "=== Running L1 PyTorch Megatron-FL MCore Integration Test ==="
          # python3 --version
          # pip list | grep -E "regex|six|torch" || true

          bash ${{ matrix.test_group.path }}
        timeout-minutes: 30