TE-FL Upgrade: Synchronization with TE Release V2.14

.github/configs/ascend.yml

@@ -0,0 +1,15 @@
+# Huawei Ascend NPU configuration
+image: ascend-infer:ubuntu18.04
+labels:
+  - npu
+  - ascend
+docker_options: |
+  --device /dev/davinci0
+  --device /dev/davinci1
+  --device /dev/davinci2
+  --device /dev/davinci3
+  --device /dev/davinci_manager
+  --device /dev/devmm_svm
+  --device /dev/hisi_hdc
+  --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver
+  --volume /usr/local/Ascend/add-ons:/usr/local/Ascend/add-ons

.github/configs/cuda.yml

@@ -0,0 +1,65 @@
+# CUDA Hardware Configuration for TransformerEngine-FL
+# Refactored for BAAI DGX A100 Nodes
+# This file defines environment variables, volumes, and test filters for TE tests.
+
+hardware_name: cuda
+display_name: 'NVIDIA CUDA (A100)'
+
+ci_image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
+
+# Runner labels for self-hosted A100 node
+# runner_labels:
+# - self-hosted
+# - Linux
+# - X64
+# - nvidia
+# - gpu-8
+runner_labels:
+  - nv-8g-cicd-te
+
+# Container volumes
+container_volumes:
+  - /home/flagscale_cicd/flask/static:/workspace/report
+  # - /home/flagscale_cicd/data:/opt/data
+
+# Container options
+container_options: >-
+  --privileged 
+  --gpus all 
+  --shm-size=500g 
+  --ipc=host 
+  --ulimit memlock=-1 
+  --ulimit stack=67108864 
+  --user root
+
+# Device types
+device_types:
+  - a100
+
+# Build environment variables (platform-specific)
+build_env:
+  TE_FL_SKIP_CUDA: '0'
+  SKIP_CUDA_BUILD: '0'
+  NVTE_WITH_CUDA: '1'
+  NVTE_WITH_MACA: '0'
+  TE_WITH_NCCL: '1'
+  NVTE_FRAMEWORK: pytorch
+  CUDA_HOME: /usr/local/cuda-12.8
+  NVCC: /usr/local/cuda-12.8/bin/nvcc
+
+# Test matrix configuration
+test_matrix:
+  l0_pytorch:
+    path: 'qa/L0_pytorch_unittest/test.sh'
+    ignored_tests:
+      - test_sanity_layernorm_mlp
+      - test_sanity_gpt
+      - test_sanity_bert
+      - test_sanity_T5
+      - test_sanity_amp_and_nvfuser
+      - test_sanity_drop_path
+      - test_layernorm_mlp_accuracy
+      - test_grouped_linear_accuracy
+      - test_gpt_accuracy
+      - test_basic_linear
+      - test_layer_norm

.github/configs/metax.yml

@@ -0,0 +1,71 @@
+# Metax Hardware Configuration for TE-FL
+# This file defines CI/CD settings for Metax-based testing
+# Test configurations are defined in tests/test_utils/config/platforms/metax.yaml
+
+hardware_name: metax
+display_name: 'Metax Tests'
+
+ci_image: localhost:5000/megatron-lm-with-te:v1
+# ci_image: harbor.baai.ac.cn/flagscale/megatron-lm-with-te:202603231839
+
+runner_labels:
+  - self-hosted
+  - Linux
+  - X64
+  - metax
+  - dev
+# runner_labels:
+#   - mx-4g-cicd-te
+
+container_volumes:
+  - /nfs/metax_fs:/nfs/metax_fs
+  - /dev/dri:/dev/dri
+  - /dev/mxcd:/dev/mxcd
+  - /dev/infiniband:/dev/infiniband
+
+container_options: >-
+  --uts=host
+  --ipc=host
+  --privileged=true
+  --group-add video
+  --shm-size=100gb
+  --ulimit memlock=-1
+  --security-opt seccomp=unconfined
+  --security-opt apparmor=unconfined
+  --device=/dev/dri
+  --device=/dev/mxcd
+  --device=/dev/infiniband
+  --user root
+  --ulimit nofile=65535:65535
+  -e PLATFORM=metax
+  -e TORCH_DISTRIBUTED_BACKEND=mccl
+  -e LD_LIBRARY_PATH=/opt/maca/lib:/usr/local/lib:$LD_LIBRARY_PATH
+
+build_env:
+  TE_FL_SKIP_CUDA: '1'
+  NVTE_WITH_MACA: '1'
+  CUDA_HOME: /opt/maca
+  MACA_HOME: /opt/maca
+
+# Device types to run tests on
+device_types:
+  - c500
+
+# Test matrix configuration
+test_matrix:
+  unit:
+    devices:
+      - c500
+    # Ignored test files for unit tests
+    # These files will be skipped when running pytest
+    ignored_tests:
+      # example: tests/unit_tests/test_example.py
+      # - tests/unit_tests/test_inference.py
+      # - tests/unit_tests/test_rl_utils.py
+
+  # functional:
+  #   train:
+  #     - device: c500
+  #       task: train
+  #       model: deepseek
+  #       case: tp2_pp2_ep2

.github/configs/template.yml

@@ -0,0 +1,16 @@
+# Configuration Template
+# This file describes the structure for hardware-specific configurations.
+#
+# Fields:
+# - image: Docker image to use for the runner
+# - labels: List of labels for the runner
+# - docker_options: Additional Docker options for mounting devices, volumes, etc.
+#
+# Example:
+# image: <docker_image>
+# labels:
+#   - <label1>
+#   - <label2>
+# docker_options: |
+#   --option1 value1
+#   --option2 value2

.github/workflows/all_tests_ascend.yml

@@ -0,0 +1,32 @@
+name: ascend_tests
+
+on:
+  # push:
+  #   branches: ["main"]
+  # pull_request:
+  #   branches: ["main"]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
+  cancel-in-progress: true
+
+jobs:
+  run_tests:
+    # Package manager and environment settings are read from .github/configs/ascend.yml
+    uses: ./.github/workflows/all_tests_common.yml
+    with:
+      platform: ascend
+
+  all_tests:
+    needs: run_tests
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Verify workflow status
+        run: |
+          if [ "${{ needs.run_tests.result }}" != "success" ]; then
+            echo "❌ Tests workflow failed"
+            exit 1
+          fi
+          echo "✅ All tests passed!"

.github/workflows/all_tests_common.yml

@@ -0,0 +1,150 @@
+name: Common All Tests
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+        description: Platform name (e.g., cuda, default)
+      setup_commands:
+        required: false
+        type: string
+        default: ''
+
+jobs:
+  checkout_and_config:
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-latest
+    outputs:
+      ci_image: ${{ steps.config.outputs.ci_image }}
+      runs_on: ${{ steps.config.outputs.runs_on }}
+      container_volumes: ${{ steps.config.outputs.container_volumes }}
+      container_options: ${{ steps.config.outputs.container_options }}
+      device_types: ${{ steps.config.outputs.device_types }}
+      train_test_matrix: ${{ steps.config.outputs.train_test_matrix }}
+      ignored_tests: ${{ steps.config.outputs.ignored_tests }}
+      build_env: ${{ steps.config.outputs.build_env }}
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Check if tests should run
+        id: should_run
+        run: |
+
+          echo "should_run=true" >> $GITHUB_OUTPUT
+
+      - name: Load platform configuration
+        id: config
+        run: |
+          set -euo pipefail
+
+          PLATFORM="${{ inputs.platform }}"
+          CONFIG_FILE=".github/configs/${PLATFORM}.yml"
+
+          # Install mikefarah/yq (v4) for YAML parsing
+          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_amd64
+          sudo chmod +x /usr/local/bin/yq
+          /usr/local/bin/yq --version
+          echo "Loading configuration from $CONFIG_FILE"
+
+          # Read CI image
+          CI_IMAGE=$(yq '.ci_image' "$CONFIG_FILE")
+          echo "ci_image=$CI_IMAGE" >> $GITHUB_OUTPUT
+
+          # Read runner labels and format as JSON array
+          RUNS_ON=$(yq '.runner_labels | tojson(0)' "$CONFIG_FILE")
+          echo "runs_on=$RUNS_ON" >> $GITHUB_OUTPUT
+
+          # Read container volumes and format as JSON array
+          VOLUMES=$(yq '.container_volumes | tojson(0)' "$CONFIG_FILE")
+          echo "container_volumes=$VOLUMES" >> $GITHUB_OUTPUT
+
+          # Read container options
+          OPTIONS=$(yq '.container_options' "$CONFIG_FILE")
+          echo "container_options=$OPTIONS" >> $GITHUB_OUTPUT
+
+          # Read device types
+          DEVICE_TYPES=$(yq '.device_types | tojson(0)' "$CONFIG_FILE")
+          echo "device_types=$DEVICE_TYPES" >> $GITHUB_OUTPUT
+
+          # Read test matrix for training
+          TRAIN_MATRIX=$(yq '.test_matrix.functional.train | tojson(0)' "$CONFIG_FILE")
+          echo "train_test_matrix=$TRAIN_MATRIX" >> $GITHUB_OUTPUT
+
+          # Read ignored tests list from test_matrix.unit (default to empty array if not defined)
+          IGNORED_TESTS=$(yq '.test_matrix.unit.ignored_tests // [] | tojson(0)' "$CONFIG_FILE")
+          echo "ignored_tests=$IGNORED_TESTS" >> $GITHUB_OUTPUT
+
+          # Read build environment variables (default to empty object if not defined)
+          BUILD_ENV=$(yq '.build_env // {} | tojson(0)' "$CONFIG_FILE")
+          echo "build_env=$BUILD_ENV" >> $GITHUB_OUTPUT
+
+  unit_tests:
+    needs: checkout_and_config
+    strategy:
+      fail-fast: false
+      matrix:
+        device: ${{ fromJson(needs.checkout_and_config.outputs.device_types) }}
+    uses: ./.github/workflows/unit_tests_common.yml
+    name: unit_tests
+    with:
+      platform: ${{ inputs.platform }}
+      device: ${{ matrix.device }}
+      image: ${{ needs.checkout_and_config.outputs.ci_image }}
+      runs_on: ${{ needs.checkout_and_config.outputs.runs_on }}
+      container_volumes: ${{ needs.checkout_and_config.outputs.container_volumes }}
+      container_options: ${{ needs.checkout_and_config.outputs.container_options }}
+      setup_commands: ${{ inputs.setup_commands }}
+      ignored_tests: ${{ needs.checkout_and_config.outputs.ignored_tests }}
+      build_env: ${{ needs.checkout_and_config.outputs.build_env }}
+
+  # arguments.py not compatible with megatron-core-fl
+  # functional_tests:
+  #   needs:
+  #     - checkout_and_config
+  #   if: fromJson(needs.checkout_and_config.outputs.train_test_matrix)[0] != null
+  #   uses: ./.github/workflows/functional_tests_common.yml
+  #   with:
+  #     platform: ${{ inputs.platform }}
+  #     test_matrix: ${{ needs.checkout_and_config.outputs.train_test_matrix }}
+  #     image: ${{ needs.checkout_and_config.outputs.ci_image }}
+  #     runs_on: ${{ needs.checkout_and_config.outputs.runs_on }}
+  #     container_volumes: ${{ needs.checkout_and_config.outputs.container_volumes }}
+  #     container_options: ${{ needs.checkout_and_config.outputs.container_options }}
+
+
+  all_tests_complete:
+    defaults:
+      run:
+        shell: bash
+    needs:
+      - checkout_and_config
+      - unit_tests
+      # - functional_tests
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Verify all tests passed
+        run: |
+          # Check all test jobs (skip if not run)
+          failed=false
+
+          if [ "${{ needs.unit_tests.result }}" != "success" ]; then
+            echo "❌ Unit tests failed"
+            failed=true
+          fi
+
+          # if [ "${{ needs.functional_tests.result }}" != "success" ]; then
+          #   echo "❌ Training functional tests failed"
+          #   failed=true
+          # fi
+
+          if [ "$failed" = "true" ]; then
+            exit 1
+          fi
+
+          echo "✅ All tests completed successfully!"

.github/workflows/all_tests_cuda.yml

@@ -0,0 +1,32 @@
+name: cuda_tests
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
+  cancel-in-progress: true
+
+jobs:
+  run_tests:
+    # Package manager and environment settings are read from .github/configs/cuda.yml
+    uses: ./.github/workflows/all_tests_common.yml
+    with:
+      platform: cuda
+
+  all_tests:
+    needs: run_tests
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Verify workflow status
+        run: |
+          if [ "${{ needs.run_tests.result }}" != "success" ]; then
+            echo "❌ Tests workflow failed"
+            exit 1
+          fi
+          echo "✅ All tests passed!"