FlagScale/examples/deepseek_v3/conf/train/16b_a3b_finetune.yaml at 42f45a93f5fa9a56753880d203fedf2abe2d16da · lxd-cumt/FlagScale · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
system:
  no_shared_fs: ${experiment.runner.no_shared_fs}
  num_workers: 2
  tensor_model_parallel_size: 2
  pipeline_model_parallel_size: 2
  decoder_first_pipeline_num_layers: 13
  expert_model_parallel_size: 2
  context_parallel_size: 1
  sequence_parallel: true
  use_distributed_optimizer: true
  overlap_grad_reduce: true
  overlap_param_gather: true
  precision:
    bf16: true
    attention_softmax_in_fp32: true
    accumulate_allreduce_grads_in_fp32: true
  logging:
    log_interval: 1
    tensorboard_log_interval: 1
    wandb_project: ${experiment.exp_name}
    wandb_exp_name: ${experiment.exp_name}
    log_timers_to_tensorboard: true
    log_validation_ppl_to_tensorboard: true
    log_throughput: true
    log_params_norm: true
    log_num_zeros_in_grad: true
    log_memory_to_tensorboard: true
  checkpoint:
    save_interval: ${experiment.save_steps}
    load: ${experiment.load}
    ckpt_format: ${experiment.ckpt_format}

model:
  transformer_impl: transformer_engine
  num_layers: 27
  hidden_size: 2048
  num_attention_heads: 16
  num_query_groups: 16 # num_key_value_heads
  seq_length: 4096
  max_position_embeddings: 4096
  norm_epsilon: 1e-6
  use_rotary_position_embeddings: true
  rotary_base: 1000000
  swiglu: true
  normalization: RMSNorm
  qk_layernorm: true
  init_method_std: 0.02
  attention_dropout: 0.0
  hidden_dropout: 0.0
  position_embedding_type: rope
  untie_embeddings_and_output_weights: true
  no_position_embedding: true
  no_rope_fusion: true
  disable_bias_linear: true

  # mla args ==================
  multi_latent_attention: true
  kv_lora_rank: 512
  qk_head_dim: 128
  qk_pos_emb_head_dim: 64
  v_head_dim: 128

  # moe args ===================
  ffn_hidden_size: 11264
  moe_ffn_hidden_size: 1408
  moe_grouped_gemm: true
  moe_shared_expert_intermediate_size: 2816
  num_experts: 64
  moe_router_load_balancing_type: "seq_aux_loss"
  moe_router_score_function: sigmoid
  moe_router_enable_expert_bias: true
  moe_router_bias_update_rate: 0.001
  moe_aux_loss_coeff: 0.02
  moe_layer_freq: "[0]+[1]*26"
  # node limited routing
  moe_router_num_groups: 1
  moe_router_group_topk: 1
  moe_router_topk: 6
  moe_router_topk_scaling_factor: 2.446
  moe_token_dispatcher_type: "alltoall"

  # mtp args ====================
  mtp_num_layers: 1
  mtp_loss_scaling_factor: 0.3

  # training
  seed: ${experiment.seed}
  micro_batch_size: 1
  global_batch_size: 128 #2048
  eval_iters: 0
  eval_interval: 1000
  train_iters: 102400

  # lora finetune
  finetune: true
  peft_type: lora
  lora_target_modules: ["linear_q_proj", "linear_kv_down_proj", "linear_kv_up_proj", "linear_proj", "linear_fc1", "linear_fc2"]
  lora_dim: 32
  lora_alpha: 64
  lora_dropout: 0.1
  lora_dropout_position: post
  lora_in_init_method: xavier
  lora_out_init_method: zero


  optimizer:
    clip_grad: 1.0
    weight_decay: 0.1
    adam_beta1: 0.9
    adam_beta2: 0.95
    lr_scheduler:
      lr: 3.0e-3
      min_lr: 3.0e-4
      lr_warmup_fraction: 0.01
      lr_decay_style: WSD
      lr_wsd_decay_style: cosine
      lr_wsd_decay_iters: 10

data:
  reset_position_ids: True
  reset_attention_mask: True
  data_path: /path
  split: 1
  no_mmap_bin_files: true
  tokenizer:
    tokenizer_type: QwenTokenizerFS
    tokenizer_path: qwentokenizer
    vocab_size: 151851
    make_vocab_size_divisible_by: 64