Skip to content

Commit 2db6879

Browse files
committed
[train]: add apply_dsa_fusion and original_max_position_embeddings arguments.
1 parent 30f7024 commit 2db6879

1 file changed

Lines changed: 7 additions & 0 deletions

File tree

flagscale/train/megatron/training/arguments.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2073,6 +2073,7 @@ def _add_network_size_args(parser):
20732073
"persist_layer_norm",
20742074
"bias_dropout_fusion",
20752075
"apply_rope_fusion",
2076+
"apply_dsa_kernel_fusion",
20762077
# FlagScale-specific: manually added in arguments_fs.py
20772078
"enable_hetero",
20782079
"hetero_pipeline_layer_split",
@@ -3217,6 +3218,12 @@ def _add_mla_args(parser):
32173218
help="Dimension of the head in the V projection.")
32183219
group.add_argument('--rotary-scaling-factor', type=float, default=1.0,
32193220
help="Rotary scaling factor for the rotary embeddings.")
3221+
group.add_argument(
3222+
'--original-max-position-embeddings',
3223+
type=int,
3224+
default=4096,
3225+
help="Original maximum position embeddings for the original model, used by yarn.",
3226+
)
32203227
group.add_argument('--mscale', type=float, default=1.0,
32213228
help="Mscale for YaRN RoPE in multi-latent attention.")
32223229
group.add_argument('--mscale-all-dim', type=float, default=0.0,

0 commit comments

Comments
 (0)