Skip to content

Commit c8f6047

Browse files
wenqingqianzhaoyingliaCopilot
authored
add Qwen35 Model (flagos-ai#1196)
Qwen35 pretrain support --------- Co-authored-by: zhaoyingli <86812880+zhaoyinglia@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent b717a01 commit c8f6047

24 files changed

Lines changed: 6249 additions & 16 deletions

‎examples/qwen35/conf/train.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
defaults:
2+
- _self_
3+
- train: 4b
4+
5+
experiment:
6+
exp_name: outputs_qwen35_4b
7+
exp_dir: ./${experiment.exp_name}
8+
task:
9+
type: train
10+
backend: megatron
11+
entrypoint: ./flagscale/train/megatron/train_qwen35.py
12+
runner:
13+
nnodes: 1
14+
nproc_per_node: 8
15+
envs:
16+
CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7
17+
CUDA_DEVICE_MAX_CONNECTIONS: 1
18+
NVTE_APPLY_QK_LAYER_SCALING: 0
19+
NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
20+
21+
action: run
22+
23+
hydra:
24+
run:
25+
dir: ${experiment.exp_dir}/hydra

‎examples/qwen35/conf/train/0.8b.yaml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
system:
2+
vision_ration: 0.1
3+
num_workers: 1
4+
calculate_per_token_loss: true
5+
tensor_model_parallel_size: 2
6+
pipeline_model_parallel_size: 2
7+
context_parallel_size: 1
8+
use_flash_attn: True
9+
use_distributed_optimizer: True
10+
sequence_parallel: True
11+
tp_comm_overlap: False
12+
overlap_grad_reduce: False
13+
overlap_param_gather: False
14+
use_mcore_models: True
15+
transformer_impl: transformer_engine
16+
use_te: True
17+
precision:
18+
bf16: True
19+
attention_softmax_in_fp32: True
20+
logging:
21+
log_interval: 1
22+
checkpoint:
23+
save_interval: 5
24+
ckpt_format: torch
25+
26+
model:
27+
# Qwen3.5-0.8B architecture parameters
28+
kv_channels: 256
29+
qk_layernorm: True
30+
attention_backend: flash
31+
disable_bias_linear: True
32+
num_layers: 24
33+
hidden_size: 1024
34+
ffn_hidden_size: 3584
35+
num_attention_heads: 8
36+
num_query_groups: 2
37+
seq_length: 2048
38+
max_padding_length: 2048
39+
enable_variable_seq_lengths: True
40+
max_position_embeddings: 262144
41+
swiglu: True
42+
normalization: RMSNorm
43+
norm_epsilon: 1e-6
44+
init_method_std: 0.02
45+
attention_dropout: 0.0
46+
hidden_dropout: 0.0
47+
clip_grad: 1.0
48+
train_iters: 5000
49+
eval_iters: 0
50+
eval_interval: 1000
51+
micro_batch_size: 1
52+
global_batch_size: 4
53+
allow_missing_vision_projection_checkpoint: False
54+
apply_layernorm_1p: False
55+
group_query_attention: True
56+
no_masked_softmax_fusion: True
57+
untie_embeddings_and_output_weights: False
58+
59+
# Position embedding: Qwen3.5 uses mRoPE with partial rotary
60+
position_embedding_type: mrope
61+
rotary_percent: 0.25
62+
rotary_base: 10000000
63+
rotary_seq_len_interpolation_factor: 1
64+
no_rope_fusion: False
65+
mrope_section: [11, 11, 10]
66+
eod_mask_loss: False
67+
68+
# Vision model
69+
patch_size: 16
70+
freeze_LM: False
71+
freeze_ViT: False
72+
disable_vision_class_token: True
73+
74+
# Recompute
75+
vision_recompute_activations: True
76+
recompute:
77+
recompute_granularity: 'full'
78+
recompute_method: 'uniform'
79+
recompute_num_layers: 1
80+
81+
seed: 42
82+
83+
# Gated DeltaNet (GDN) parameters
84+
experimental_attention_variant: gated_delta_net
85+
linear_attention_freq: 4
86+
linear_conv_kernel_dim: 4
87+
linear_key_head_dim: 128
88+
linear_value_head_dim: 128
89+
linear_num_key_heads: 16
90+
linear_num_value_heads: 16
91+
layernorm_zero_centered_gamma: True
92+
attention_output_gate: True
93+
94+
# Vision encoder parameters
95+
vision_num_layers: 12
96+
vision_hidden_size: 768
97+
vision_ffn_hidden_size: 3072
98+
vision_num_attention_heads: 12
99+
100+
# mtp
101+
mtp_num_layers: 1
102+
103+
optimizer:
104+
weight_decay: 0.1
105+
adam_beta1: 0.9
106+
adam_beta2: 0.999
107+
lr_scheduler:
108+
lr: 1.0e-5
109+
min_lr: 1.0e-6
110+
lr_warmup_fraction: .03
111+
lr_decay_style: cosine
112+
113+
data:
114+
no_use_system_prompt: True
115+
vocab_size: 248320
116+
data_path: /workspace/data/qwen/qwen_wds/wds-1
117+
vision_root: /
118+
dataloader_type: external
119+
split: 100,0,0
120+
tokenizer:
121+
tokenizer_type: Qwen2VLTokenizer
122+
tokenizer_path: /workspace/data/qwen/qwen2vl_tokenizer
123+
extra_vocab_size: 0
124+
make_vocab_size_divisible_by: 64

‎examples/qwen35/conf/train/27b.yaml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
system:
2+
vision_ration: 0.1
3+
num_workers: 1
4+
calculate_per_token_loss: true
5+
tensor_model_parallel_size: 4
6+
pipeline_model_parallel_size: 2
7+
context_parallel_size: 1
8+
use_flash_attn: True
9+
use_distributed_optimizer: True
10+
sequence_parallel: True
11+
tp_comm_overlap: False
12+
overlap_grad_reduce: False
13+
overlap_param_gather: False
14+
use_mcore_models: True
15+
transformer_impl: transformer_engine
16+
use_te: True
17+
precision:
18+
bf16: True
19+
attention_softmax_in_fp32: True
20+
logging:
21+
log_interval: 1
22+
checkpoint:
23+
save_interval: 10000
24+
ckpt_format: torch
25+
26+
model:
27+
# Qwen3.5-27B architecture parameters
28+
kv_channels: 256
29+
qk_layernorm: True
30+
attention_backend: flash
31+
disable_bias_linear: True
32+
num_layers: 64
33+
hidden_size: 5120
34+
ffn_hidden_size: 17408 # intermediate size for dense MLP
35+
num_attention_heads: 24 # Q heads for gated attention
36+
num_query_groups: 4 # KV heads (GQA)
37+
seq_length: 2048
38+
max_padding_length: 2048
39+
enable_variable_seq_lengths: True
40+
max_position_embeddings: 262144
41+
swiglu: True
42+
normalization: RMSNorm
43+
norm_epsilon: 1e-6
44+
init_method_std: 0.02
45+
attention_dropout: 0.0
46+
hidden_dropout: 0.0
47+
clip_grad: 1.0
48+
train_iters: 5000
49+
eval_iters: 0
50+
eval_interval: 1000
51+
micro_batch_size: 1
52+
global_batch_size: 4
53+
allow_missing_vision_projection_checkpoint: False
54+
apply_layernorm_1p: False
55+
group_query_attention: True
56+
no_masked_softmax_fusion: True
57+
untie_embeddings_and_output_weights: True
58+
59+
# Position embedding: Qwen3.5 uses mRoPE with partial rotary
60+
position_embedding_type: mrope
61+
rotary_percent: 0.25 # partial rotary (rotary_dim = 256 * 0.25 = 64)
62+
rotary_base: 10000000
63+
rotary_seq_len_interpolation_factor: 1
64+
no_rope_fusion: False
65+
mrope_section: [11, 11, 10] # Qwen3.5 mRoPE sections
66+
eod_mask_loss: False
67+
68+
# Vision model
69+
patch_size: 16
70+
freeze_LM: False
71+
freeze_ViT: False
72+
disable_vision_class_token: True
73+
74+
# Recompute
75+
vision_recompute_activations: True
76+
recompute:
77+
recompute_granularity: 'full'
78+
recompute_method: 'uniform'
79+
recompute_num_layers: 1
80+
81+
seed: 42
82+
83+
# Gated DeltaNet (GDN) parameters (for hybrid GDN + Attention architecture)
84+
experimental_attention_variant: gated_delta_net
85+
linear_attention_freq: 4 # Pattern: [GDN, GDN, GDN, Attention] x 16
86+
linear_conv_kernel_dim: 4
87+
linear_key_head_dim: 128
88+
linear_value_head_dim: 128
89+
linear_num_key_heads: 16
90+
linear_num_value_heads: 48
91+
layernorm_zero_centered_gamma: True
92+
attention_output_gate: True
93+
94+
# Vision encoder parameters
95+
vision_num_layers: 27
96+
vision_hidden_size: 1152
97+
vision_ffn_hidden_size: 4304
98+
vision_num_attention_heads: 16
99+
100+
# mtp
101+
mtp_num_layers: 1
102+
103+
optimizer:
104+
weight_decay: 0.1
105+
adam_beta1: 0.9
106+
adam_beta2: 0.999
107+
lr_scheduler:
108+
lr: 1.0e-5
109+
min_lr: 1.0e-6
110+
lr_warmup_fraction: .03
111+
lr_decay_style: cosine
112+
113+
data:
114+
no_use_system_prompt: True
115+
vocab_size: 248320
116+
data_path: /workspace/data/qwen/qwen_wds/wds-1
117+
vision_root: /
118+
dataloader_type: external
119+
split: 100,0,0
120+
tokenizer:
121+
tokenizer_type: Qwen2VLTokenizer
122+
tokenizer_path: /workspace/data/qwen/qwen2vl_tokenizer
123+
extra_vocab_size: 0
124+
make_vocab_size_divisible_by: 64

0 commit comments

Comments
 (0)