all-tests-nvidia.yml

name: All Tests Nvidia

on:
  push:
    branches: ["main"]
    paths-ignore:
      - 'hardware/BI_V150/**'
      - 'hardware/Cambricon_MLU/**'
      - 'hardware/Huawei_Atlas800TA3/**'
      - 'hardware/Hygon_BW1000/**'
      - 'hardware/Kunlunxin_R310p/**'
      - 'hardware/MUSA_S5000/**'
      - 'hardware/Metax_C550/**'
      - 'hardware/Tsing_micro/**'
  pull_request:
    branches: ["main"]
    paths-ignore:
      - 'hardware/BI_V150/**'
      - 'hardware/Cambricon_MLU/**'
      - 'hardware/Huawei_Atlas800TA3/**'
      - 'hardware/Hygon_BW1000/**'
      - 'hardware/Kunlunxin_R310p/**'
      - 'hardware/MUSA_S5000/**'
      - 'hardware/Metax_C550/**'
      - 'hardware/Tsing_micro/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
  cancel-in-progress: true

jobs:
  set-env:
    runs-on: ubuntu-latest
    outputs:
      ci_image: ${{ steps.set-env.outputs.ci_image }} # Declare output variable
    steps:
      - name: Set Environment Variable
        id: set-env # Assign an ID to this step
        run: |
          echo "ci_image=localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.1-time2510131515" >> $GITHUB_OUTPUT  # Set output variable

  # Train Megatron Unit Tests with Matrix
  unit_tests_train_megatron:
    needs:
      - set-env
    uses: ./.github/workflows/unit-tests-nvidia.yml
    strategy:
      matrix:
        subset:
          - data
          - dist_checkpointing
          - distributed
          - export
          - fusions
          - inference
          - models
          - pipeline_parallel
          - post_training
          - ssm
          - tensor_parallel
          - transformer/moe
          - transformer
          - ./
    name: "train_megatron-${{ matrix.subset == './' && 'root' || matrix.subset }}"
    with:
      backend: train_megatron
      subset: ${{ matrix.subset }}
      image: ${{ needs.set-env.outputs.ci_image }}

  # Train Flagscale Unit Tests with Matrix
  unit_tests_train_flagscale:
    needs:
      - set-env
      - unit_tests_train_megatron
    uses: ./.github/workflows/unit-tests-nvidia.yml
    strategy:
      matrix:
        subset:
          - runner
          - ./
    name: "train_flagscale-${{ matrix.subset == './' && 'root' || matrix.subset }}"
    with:
      backend: train_flagscale
      subset: ${{ matrix.subset }}
      image: ${{ needs.set-env.outputs.ci_image }}

  # # Inference Flagscale Unit Tests with Matrix
  # unit_tests_inference_flagscale:
  #   needs:
  #     - set-env
  #     - unit_tests_train_flagscale
  #   uses: ./.github/workflows/unit-tests-nvidia.yml
  #   strategy:
  #     matrix:
  #       subset:
  #         - inference
  #         - transformations
  #   name: "inference_flagscale-${{ matrix.subset == './' && 'root' || matrix.subset }}"
  #   with:
  #     backend: inference_flagscale
  #     subset: ${{ matrix.subset }}
  #     image: ${{ needs.set-env.outputs.ci_image }}

  # Functional Tests with Mision and Type Matrix
  functional_tests_train:
    needs:
      - set-env
      - unit_tests_train_flagscale
    uses: ./.github/workflows/functional-tests-nvidia.yml
    strategy:
      matrix:
        task:
          - aquila
          - deepseek
          - mixtral
          # - llava_onevision # TODO: test need fix
    name: "train-${{ matrix.task }}"
    with:
      task: ${{ matrix.task }}
      type: train
      image: ${{ needs.set-env.outputs.ci_image }}

  functional_tests_hetero:
    needs:
      - set-env
      - functional_tests_train
    # TODO: test need fix
    uses: ./.github/workflows/functional-tests-nvidia.yml
    strategy:
      matrix:
        task:
          - aquila
    name: "hetero_train-${{ matrix.task }}"
    with:
      task: ${{ matrix.task }}
      type: hetero_train
      image: ${{ needs.set-env.outputs.ci_image }}

  # functional_tests_inference:
  #   needs:
  #     - set-env
  #   uses: ./.github/workflows/functional-tests-nvidia.yml
  #   strategy:
  #     matrix:
  #       task:
  #         - deepseek_r1_distill_qwen
  #         - deepseek_r1_distill_qwen-flaggems
  #         - qwen3
  #         - qwen3-flaggems
  #         - robobrain2
  #         # - robobrain2-flaggems
  #   name: "inference-${{ matrix.task }}"
  #   with:
  #     task: ${{ matrix.task }}
  #     type: inference
  #     image: ${{ needs.set-env.outputs.ci_image }}

  # functional_tests_serve:
  #   needs:
  #     - set-env
  #     - functional_tests_inference
  #   uses: ./.github/workflows/functional-tests-nvidia.yml
  #   strategy:
  #     matrix:
  #       task:
  #         - qwen2_5
  #         - base
  #   name: "serve-${{ matrix.task }}"
  #   with:
  #     task: ${{ matrix.task }}
  #     type: serve
  #     image: ${{ needs.set-env.outputs.ci_image }}

  functional_tests_rl:
    needs:
      - set-env
      - functional_tests_hetero
    uses: ./.github/workflows/functional-tests-nvidia.yml
    strategy:
      matrix:
        task:
          - qwen2_5
    name: "rl-${{ matrix.task }}"
    with:
      task: ${{ matrix.task }}
      type: rl
      image: ${{ needs.set-env.outputs.ci_image }}

  # Check All Tests
  all-tests:
    needs:
      - unit_tests_train_megatron
      - unit_tests_train_flagscale
      # - unit_tests_inference_flagscale
      - functional_tests_train
      - functional_tests_hetero
      # - functional_tests_inference
      # - functional_tests_serve
      - functional_tests_rl
    runs-on: ubuntu-latest
    steps:
      - name: All Tests Completed
        run: echo "All tests completed successfully!"