Merge branch 'develop' into qwen3-next

Author: lshpku

examples/experiments/paddlefleet/glm45.json

@@ -7,26 +7,26 @@
   "do_train": true,
   "do_eval": true,
   "per_device_eval_batch_size": 1,
-  "per_device_train_batch_size": 1,
+  "per_device_train_batch_size": 2,
   "num_train_epochs": 1,
   "max_steps": 50,
   "eval_steps": 100,
   "evaluation_strategy": "steps",
   "save_steps": 10,
   "save_strategy": "steps",
   "logging_steps": 1,
-  "gradient_accumulation_steps": 8,
+  "gradient_accumulation_steps": 1,
   "logging_dir": "./24_layer_tp4_ep8_pp4_sp2_glm_vdl_log",
   "output_dir": "./checkpoints/24layer_tp4_ep8_pp4_sp2_glm_pretrain_ckpts",
   "disable_tqdm": true,
   "eval_accumulation_steps": 16,
   "warmup_steps": 20,
   "learning_rate": 0.00001,
-  "tensor_parallel_degree": 1,
+  "tensor_parallel_degree": 4,
   "pipeline_parallel_degree": 1,
-  "use_expert_parallel": false,
-  "expert_parallel_degree": 8,
-  "sequence_parallel": false,
+  "use_expert_parallel": true,
+  "expert_parallel_degree": 16,
+  "sequence_parallel": true,
   "sharding_parallel_config": "split_param",
   "amp_master_grad": true,
   "sharding": "stage1",

examples/experiments/paddlefleet/glm45_provider.py

@@ -130,20 +130,16 @@ class GLM45AirModelProvider106B(GLMMoEModelProvider):
 
 
 @dataclass
-class GLM45AirModelDebugProvider(GLMMoEModelProvider):
+class GLM45AirModelDebugProvider(GLM45AirModelProvider106B):
     """
     Provider for GLM 4.5 Air 106B-A12B: https://huggingface.co/zai-org/GLM-4.5-Air
     """
 
-    num_layers: int = 1
-    # num_moe_experts: int = 128
-    hidden_size: int = 512
-    ffn_hidden_size: int = 512
-    moe_layer_freq: Union[int, List[int]] = field(
-        default_factory=lambda: [0] * 1 + [1] * 45
-    )  # first one layer is dense
-    moe_ffn_hidden_size: int = 1408
-    moe_shared_expert_intermediate_size: int = 1408
-    qk_layernorm: bool = False
-    moe_router_topk_scaling_factor: float = 1.0
+    num_layers: int = 10
+    moe_num_shared_experts: int = 1
+    hidden_size: int = 128
+    ffn_hidden_size: int = 128
+    moe_intermediate_size: int = 1408
     mtp_num_layers: Optional[int] = 0
+    use_bias: bool = False
+    vocab_size: int = 37888

examples/experiments/paddlefleet/run_glm45.sh

@@ -12,42 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#START_RANK=0 # 改成真正执行的机器号
-#END_RANK=4 # 改成真正执行的机器号
+START_RANK=0 # 改成真正执行的机器号
+END_RANK=2 # 改成真正执行的机器号
 
-#if [[ $rank -lt $START_RANK ]]; then
-#    exit 0
-#fi
+if [[ $rank -lt $START_RANK ]]; then
+   exit 0
+fi
 
-#if [[ $rank -ge $END_RANK ]]; then
-#    exit 0
-#fi
-#nnodes=$(($END_RANK-$START_RANK))
-#master=`cat /root/paddlejob/workspace/hostfile | head -n $(($START_RANK+1)) | tail -n 1 | awk '{print $1}'`
-# master=10.54.107.148
-#port=36677
+if [[ $rank -ge $END_RANK ]]; then
+   exit 0
+fi
+nnodes=$(($END_RANK-$START_RANK))
+master=`cat /root/paddlejob/workspace/hostfile | head -n $(($START_RANK+1)) | tail -n 1 | awk '{print $1}'`
+port=36677
 
-#rank=$(($rank-$START_RANK))
+rank=$(($rank-$START_RANK))
 #bash script/kill_process.sh 
 #sleep 5
 
-#rm core.* -rf
-# rank_id=$(echo "$LAUNCH_CMD" | sed -n 's/.*--rank \([0-9]*\).*/\1/p')
-#rm -rf /root/paddlejob/share-storage/gpfs/system-public/path/to/your/outputs # 改成自己的输出目录
-
-# ls /root/paddlejob/share-storage/gpfs/system-public/huggingface_model/GLM-4.5-Air
-
-export PYTHONPATH=/workspace/PaddleFleet:/workspace/PaddleFleet/examples/experiments/paddlefleet #修改为自己的paddlefleet路径
-export CUDA_VISIBLE_DEVICES=0
-
-python run_pretrain.py glm45.json \
-  --output_dir /workspace/PaddleFormers/examples/experiments/paddlefleet/outputs # 改成自己的保存模型目录
-
-#python3.10 -m paddle.distributed.launch \
-#    --log_dir /root/paddlejob/share-storage/gpfs/system-public/zhangyichen/outputs/output_$rank/paddle_distributed_logs \ # 改成自己的保存日志目录
-#    --master $master:$port \
-#    --nnodes $nnodes \
-#    --rank $rank \
-#    --run_mode=collective \
-#    ${script:-run_finetune.py}  \
-#    $@
+python -m paddle.distributed.launch \
+   --log_dir ./outputs/output_$rank/paddle_distributed_logs \
+   --master $master:$port \
+   --nnodes $nnodes \
+   --rank $rank \
+   --run_mode=collective \
+   run_pretrain.py glm45.json \
+   --output_dir . # 改成自己的保存模型目录

examples/experiments/paddlefleet/run_pretrain.py

@@ -14,12 +14,15 @@
 import copy
 import math
 import os
+import random
 import sys
 import time
 from dataclasses import dataclass, field
 from typing import Optional
 
+import numpy as np
 import paddle
+import paddlefleet
 
 from paddleformers.data.causal_dataset import (
     build_train_valid_test_datasets,
@@ -34,7 +37,6 @@
     StepFlexToken,
     TrainingArguments,
     get_last_checkpoint,
-    set_seed,
     speed_metrics,
 )
 from paddleformers.trainer.trainer import Trainer
@@ -350,6 +352,31 @@ def _get_train_sampler(self) -> Optional[paddle.io.Sampler]:
         )
 
 
+def _set_random_seed(
+    seed_: int,
+    data_parallel_random_init: bool = False,
+    te_rng_tracker: bool = False,
+    inference_rng_tracker: bool = False,
+    use_cudagraphable_rng: bool = False,
+):
+    """Set random seed for reproducability."""
+    if seed_ is not None and seed_ > 0:
+        # Ensure that different pipeline MP stages get different seeds.
+        seed = seed_ + (100 * paddlefleet.parallel_state.get_pipeline_model_parallel_rank())
+        # Ensure different data parallel ranks get different seeds
+        if data_parallel_random_init:
+            seed = seed + (10 * paddlefleet.parallel_state.get_data_parallel_rank())
+        random.seed(seed)
+        np.random.seed(seed)
+        paddle.manual_seed(seed)
+        if paddle.cuda.device_count() > 0:
+            paddlefleet.tensor_parallel.model_parallel_cuda_manual_seed(
+                seed, te_rng_tracker, inference_rng_tracker, use_cudagraphable_rng
+            )
+    else:
+        raise ValueError("Seed ({}) should be a positive integer.".format(seed_))
+
+
 def main():
     parser = PdArgumentParser((ModelArguments, DataArguments, PreTrainingArguments))
     # Support format as "args.json --arg1 value1 --arg2 value2.”
@@ -374,7 +401,7 @@ def main():
         os.makedirs(data_args.data_cache, exist_ok=True)
 
     paddle.set_device(training_args.device)
-    set_seed(seed=training_args.seed)
+    _set_random_seed(seed_=training_args.seed)
 
     training_args.eval_iters = 10
     training_args.test_iters = training_args.eval_iters * 10

paddleformers/trainer/training_args.py

@@ -1750,6 +1750,11 @@ def is_context_parallel_supported():
                     strategy = init_nccl_config(self.nccl_comm_group_config, strategy)
 
                 fleet.init(is_collective=True, strategy=strategy)
+
+                # In PaddleFleet, we should use the following code to initialize.
+
+                # from paddlefleet.training.initialize import initialize_fleet
+                # initialize_fleet(strategy)
                 logger.info(strategy)
 
                 if self.reorder_pipeline_priority: