FlagScale/examples/qwen3_vl/conf/train/32b.yaml at 42f45a93f5fa9a56753880d203fedf2abe2d16da · lxd-cumt/FlagScale · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
system:
  num_workers: 1
  calculate_per_token_loss: true
  tensor_model_parallel_size: 4
  pipeline_model_parallel_size: 2
  context_parallel_size: 1
  decoder_first_pipeline_num_layers: 30
  use_flash_attn: True
  use_distributed_optimizer: True
  sequence_parallel: True
  tp_comm_overlap: False
  overlap_grad_reduce: False # if has text-only must be false
  overlap_param_gather: False # if has text-only must be false
  use_mcore_models: True
  transformer_impl: transformer_engine
  recompute_method: "uniform"
  recompute_granularity: "full"
  recompute_num_layers: 1
  use_te: True
  precision:
    bf16: True
    attention_softmax_in_fp32: True
  logging:
    log_interval: 1
    tensorboard_log_interval: 1
    log_throughput: True
    wandb_project: ${experiment.exp_name}
    wandb_exp_name: ${experiment.exp_name}
    log_params_norm: True
    log_num_zeros_in_grad: True
  checkpoint:
    save_interval: 1000
    pretrained_checkpoint: ${pretrained_checkpoint:??}
    dataloader_save: ${experiment.exp_dir}/checkpoints/dataloader
    # use_dist_ckpt: False
    ckpt_format: torch
    async_save: False

model:
  kv_channels: 128 # the weight out_size of prepare qkv
  qk_layernorm: True
  attention_backend: flash # don't use "auto(nvte_flash_attn)"
  disable_bias_linear: True
  # add_qkv_bias: True
  num_layers: 64
  hidden_size: 5120
  ffn_hidden_size: 25600
  num_attention_heads: 64
  num_query_groups: 8
  seq_length: 2048
  max_padding_length: 2048 # (cutoff_len)max 262144, change according the dataset
  # especial for qwen3-vl
  enable_variable_seq_lengths: True
  max_position_embeddings: 262144 # only useful for additional position embedding
  swiglu: True
  normalization: RMSNorm
  norm_epsilon: 1e-6
  init_method_std: 0.02
  attention_dropout: 0.0
  hidden_dropout: 0.0
  clip_grad: 1.0
  train_iters: 62
  eval_iters: 0 # no valid
  eval_interval: 1000
  micro_batch_size: 1
  global_batch_size: 16
  allow_missing_vision_projection_checkpoint: False
  apply_layernorm_1p: False
  group_query_attention: True
  no_masked_softmax_fusion: True
  untie_embeddings_and_output_weights: True

  # position embedding
  position_embedding_type: mrope
  rotary_percent: 1.0
  rotary_base: 5000000
  rotary_seq_len_interpolation_factor: 1
  no_rope_fusion: False
  mrope_section: [24, 20, 20]
  eod_mask_loss: False

  # vision model
  patch_size: 16
  freeze_LM: False
  freeze_ViT: False
  disable_vision_class_token: True
  seed: 42

  optimizer:
    weight_decay: 0.1
    adam_beta1: 0.9
    adam_beta2: 0.999
    lr_scheduler:
      lr: 1.0e-5
      min_lr: 1.0e-6
      # lr_warmup_fraction: .03
      lr_warmup_iters: 10
      lr_decay_style: cosine

data:
  no_use_system_prompt: True
  data_path: ${data_path:??}
  vision_root: ${vision_root:??}
  dataloader_type: external
  split: 100,0,0
  tokenizer:
    tokenizer_type: Qwen2VLTokenizer
    tokenizer_path: ${tokenizer_path:??}
    # vocab_size: 151936 #
    extra_vocab_size: 293 # Qwen3-VL specific. total vocab size: 151936 = 151643 + extra_vocab_size
    make_vocab_size_divisible_by: 64