Merge branch 'main' of https://gitee.com/lxdcumt/FlagScale_dev

Author: lxd-cumt

.github/workflows/all-tests-nvidia.yml

@@ -3,8 +3,12 @@ name: All Tests Nvidia
 on:
   push:
     branches: ["main"]
+    paths-ignore:
+      - 'hardware/**'
   pull_request:
     branches: ["main"]
+    paths-ignore:
+      - 'hardware/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
@@ -21,8 +25,8 @@ jobs:
         run: |
           echo "ci_image=localhost:5000/flagscale:cuda12.8.1-cudnn9.7.1-python3.12-torch2.7.0-time2507111538" >> $GITHUB_OUTPUT  # Set output variable
 
-  # Megatron Unit Tests with Matrix
-  megatron-unit-tests:
+  # Train Megatron Unit Tests with Matrix
+  unit_tests_train_megatron:
     needs:
       - set-env
     uses: ./.github/workflows/unit-tests-nvidia.yml
@@ -43,34 +47,51 @@ jobs:
           - transformer/moe
           - transformer
           - ./
-    name: "megatron-${{ matrix.subset == './' && 'root' || matrix.subset }}"
+    name: "train_megatron-${{ matrix.subset == './' && 'root' || matrix.subset }}"
     with:
-      backend: megatron
+      backend: train_megatron
       subset: ${{ matrix.subset }}
       image: ${{ needs.set-env.outputs.ci_image }}
 
-  # Flagscale Unit Tests with Matrix
-  flagscale-unit-tests:
+  # Train Flagscale Unit Tests with Matrix
+  unit_tests_train_flagscale:
     needs:
       - set-env
-      - megatron-unit-tests
+      - unit_tests_train_megatron
     uses: ./.github/workflows/unit-tests-nvidia.yml
     strategy:
       matrix:
         subset:
           - runner
           - ./
-    name: "flagscale-${{ matrix.subset == './' && 'root' || matrix.subset }}"
+    name: "train_flagscale-${{ matrix.subset == './' && 'root' || matrix.subset }}"
     with:
-      backend: flagscale
+      backend: train_flagscale
+      subset: ${{ matrix.subset }}
+      image: ${{ needs.set-env.outputs.ci_image }}
+
+  # Inference Flagscale Unit Tests with Matrix
+  unit_tests_inference_flagscale:
+    needs:
+      - set-env
+      - unit_tests_train_flagscale
+    uses: ./.github/workflows/unit-tests-nvidia.yml
+    strategy:
+      matrix:
+        subset:
+          - inference
+          - transforms
+    name: "inference_flagscale-${{ matrix.subset == './' && 'root' || matrix.subset }}"
+    with:
+      backend: inference_flagscale
       subset: ${{ matrix.subset }}
       image: ${{ needs.set-env.outputs.ci_image }}
 
   # Functional Tests with Mision and Type Matrix
-  functional-tests-train:
+  functional_tests_train:
     needs:
       - set-env
-      - flagscale-unit-tests
+      - unit_tests_inference_flagscale
     uses: ./.github/workflows/functional-tests-nvidia.yml
     strategy:
       matrix:
@@ -85,10 +106,10 @@ jobs:
       type: train
       image: ${{ needs.set-env.outputs.ci_image }}
 
-  functional-tests-hetero:
+  functional_tests_hetero:
     needs:
       - set-env
-      - functional-tests-train
+      - functional_tests_train
     # TODO: test need fix
     uses: ./.github/workflows/functional-tests-nvidia.yml
     strategy:
@@ -101,10 +122,10 @@ jobs:
       type: hetero_train
       image: ${{ needs.set-env.outputs.ci_image }}
 
-  functional-tests-inference:
+  functional_tests_inference:
     needs:
       - set-env
-      - functional-tests-hetero
+      - functional_tests_hetero
     uses: ./.github/workflows/functional-tests-nvidia.yml
     strategy:
       matrix:
@@ -119,10 +140,10 @@ jobs:
       type: inference
       image: ${{ needs.set-env.outputs.ci_image }}
 
-  functional-tests-serve:
+  functional_tests_serve:
     needs:
       - set-env
-      - functional-tests-inference
+      - functional_tests_inference
     uses: ./.github/workflows/functional-tests-nvidia.yml
     strategy:
       matrix:
@@ -135,10 +156,10 @@ jobs:
       type: serve
       image: ${{ needs.set-env.outputs.ci_image }}
 
-  functional-tests-rl:
+  functional_tests_rl:
     needs:
       - set-env
-      - functional-tests-serve
+      - functional_tests_serve
     uses: ./.github/workflows/functional-tests-nvidia.yml
     strategy:
       matrix:
@@ -153,11 +174,12 @@ jobs:
   # Check All Tests
   all-tests:
     needs:
-      - megatron-unit-tests
-      - flagscale-unit-tests
-      - functional-tests-train
-      - functional-tests-hetero
-      - functional-tests-inference
+      - unit_tests_train_megatron
+      - unit_tests_train_flagscale
+      - unit_tests_inference_flagscale
+      - functional_tests_train
+      - functional_tests_hetero
+      - functional_tests_inference
       - functional-tests-serve
       - functional-tests-rl
     runs-on: ubuntu-latest

.github/workflows/unit-tests-nvidia.yml

@@ -53,10 +53,10 @@ jobs:
           pip install gitpython # TODO: Remove after updating the image
 
           git config --global --add safe.directory /__w/FlagScale/FlagScale
-          if [ "${{ inputs.backend }}" = "megatron" ] || [ "${{ inputs.backend }}" = "flagscale" ]; then
+          if [ "${{ inputs.backend }}" = "train_megatron" ] || [ "${{ inputs.backend }}" = "train_flagscale" ]; then
             echo ""
             # PYTHONPATH=./:$PYTHONPATH pip install . --config-settings=backend="Megatron-LM" --verbose --no-build-isolation
-          elif [ "${{ inputs.backend }}" = "vllm" ]; then
+          elif [ "${{ inputs.backend }}" = "vllm" ] || [ "${{ inputs.backend }}" = "inference_flagscale" ]; then
             source /root/miniconda3/bin/activate flagscale-inference
             pip install scikit-build scikit-build-core
             pip install --no-build-isolation git+https://github.com/FlagOpen/FlagGems.git@release_v1.0.0

examples/cloud/conf/serve.yaml

@@ -9,9 +9,9 @@ experiment:
     type: serve
   runner:
     type: cloud
-    hostfile: ${oc.env:HOSTFILE, /etc/hostfile}
-    master_addr: ${oc.env:MASTER_ADDR, 127.0.0.1}
-    master_port: 7396
+    nnodes: ${oc.env:LWS_GROUP_SIZE, 1}
+    master_addr: ${oc.env:LWS_LEADER_ADDRESS, 127.0.0.1}
+    master_port: ${oc.env:MASTER_PORT, 7396}
     device_type: ${oc.env:DEVICE_TYPE, gpu}
     nproc_per_node: ${oc.env:AIRS_ACCELERATOR_NUM, 1}
     deploy:

examples/cloud/conf/serve/cloud_model.yaml

@@ -1,7 +1,7 @@
 - serve_id: vllm_model
   engine: vllm
   engine_args:
-    model: null
+    model: /models/Qwen2.5-0.5B-Instruct
     port: 8000
     host: 0.0.0.0
     served_model_name: cloud

examples/openjourney/conf/inference.yaml

@@ -0,0 +1,26 @@
+defaults:
+  - _self_
+  - inference: t2i
+
+experiment:
+  exp_name: openjourney
+  exp_dir: outputs/${experiment.exp_name}
+  model: prompthero/openjourney
+  task:
+    type: inference
+    backend: vllm # TODO(yupu): Remove this restriction
+    entrypoint: flagscale/inference/diffusion_entrypoint.py
+  runner:
+    hostfile: null
+  cmds:
+    before_start: source /root/miniconda3/bin/activate flagscale-inference
+  envs:
+    CUDA_VISIBLE_DEVICES: 0
+    CUDA_DEVICE_MAX_CONNECTIONS: 1
+    # Optionally, set HF_HOME and HF_ENDPOINT
+
+action: run
+
+hydra:
+  run:
+    dir: ${experiment.exp_dir}/hydra

examples/openjourney/conf/inference/t2i.yaml

@@ -0,0 +1,18 @@
+engine:
+  model_config:
+    model: ${experiment.model}
+    loader: diffusers
+  engine_config:
+    results_path: ${experiment.exp_dir}/results
+    transformations:
+      LogIOTransformation:
+        log_level: "info"
+      StateScopeTransformation:
+        {}
+
+generate:
+  prompt: "retro serie of different cars with different colors and shapes, mdjrny-v4 style"
+  width: 512
+  height: 512
+  num_inference_steps: 50
+  guidance_scale: 7.0

flagscale/backends/Megatron-LM/examples/multimodal/mlp_converter.py

@@ -0,0 +1,81 @@
+diff --git a/examples/multimodal/mlp_converter.py b/examples/multimodal/mlp_converter.py
+new file mode 100644
+index 00000000..b105b93d
+--- /dev/null
++++ b/examples/multimodal/mlp_converter.py
+@@ -0,0 +1,75 @@
++# Copyright (c) 2024, FlagScale CORPORATION. All rights reserved.
++import argparse
++import os
++
++import torch
++
++
++def convert(input_path, output_path, tensor_parallel_size):
++    device = "cuda"
++
++    state_dict = torch.load(input_path, weights_only=False)
++
++    new_state_dicts = [{"model": dict()} for _ in range(tensor_parallel_size)]
++
++    for name, tensor in state_dict.items():
++
++        # Map parameter names to ones used in megatron.
++        new_name = ""
++        new_tensor = tensor
++        chunk_dim = None
++
++        # This is used for chunking some tensors to target tensor parallel size.
++        if name == "model.mm_projector.0.weight":
++            new_name = "encoder.linear_fc1.weight"
++            chunk_dim = 0
++        elif name == "model.mm_projector.0.bias":
++            new_name = "encoder.linear_fc1.bias"
++            chunk_dim = 0
++        elif name == "model.mm_projector.2.weight":
++            new_name = "encoder.linear_fc2.weight"
++            chunk_dim = 1
++        elif name == "model.mm_projector.2.bias":
++            new_name = "encoder.linear_fc2.bias"
++
++        assert new_name != "", f"unexpected name {name}"
++
++        if chunk_dim is None:
++            new_tensors = [new_tensor for _ in range(tensor_parallel_size)]
++        else:
++            new_tensors = torch.chunk(new_tensor, tensor_parallel_size, dim=chunk_dim)
++
++        for i in range(tensor_parallel_size):
++            # chunk() creates a view of a bigger tensor. clone() is used here to avoid excessive storage.
++            new_state_dicts[i]["model"][new_name] = new_tensors[i].clone()
++
++    for i in range(tensor_parallel_size):
++        output_path_tp = os.path.join(output_path, f"state_dict_tp_{i}.pt")
++        torch.save(new_state_dicts[i], output_path_tp)
++
++
++if __name__ == "__main__":
++    parser = argparse.ArgumentParser(
++        description="""
++Convert LLaVA MLP weights to megatron format.
++
++
++Example usage:
++python mlp_converter.py --input /some/input/folder/mm_projector.bin --output /some/output/folder --tensor-parallel-size 2
++""",
++        formatter_class=argparse.RawDescriptionHelpFormatter,
++    )
++
++    parser.add_argument("--input", type=str, required=True, help="The mlp weights with hf format")
++    parser.add_argument(
++        "--output", type=str, required=True, help="output directory for megatron state dict file(s)"
++    )
++    parser.add_argument(
++        "--tensor-parallel-size", type=int, default=1, help="model tensor parallel size"
++    )
++
++    args = parser.parse_args()
++
++    convert(args.input, args.output, args.tensor_parallel_size)
++
++    print("done.")

flagscale/backends/Megatron-LM/examples/multimodal/mlp_converter.py.patch

@@ -0,0 +1,3 @@
+diff --git a/megatron/__init__.py b/megatron/__init__.py
+new file mode 100644
+index 00000000..e69de29b