File tree Expand file tree Collapse file tree
functional_tests/train/deepseek/conf/train Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -5,4 +5,3 @@ sentencepiece==0.2.1
55transformers==4.57.6
66tiktoken==0.12.0
77megatron_core @ https://resource.flagos.net/repository/flagos-pypi-hosted/packages/megatron-core/0.16.0rc0/megatron_core-0.16.0rc0-cp312-cp312-linux_x86_64.whl
8- emerging_optimizers @ https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git@v0.2.0
Original file line number Diff line number Diff line change @@ -18,8 +18,7 @@ system:
1818 reset_attention_mask : true
1919 qk_layernorm : true
2020 sequence_parallel : true
21- use_distributed_optimizer : false
22- use_padded_layerwise_optimizer : false
21+ use_distributed_optimizer : true
2322 num_distributed_optimizer_instances : 1
2423 overlap_grad_reduce : true
2524 overlap_param_gather : true
@@ -136,8 +135,6 @@ model:
136135 # pytorch_profiler_collect_callstack: true
137136
138137 optimizer :
139- optimizer : dist_muon
140- muon_tp_mode : distributed
141138 clip_grad : 1.0
142139 weight_decay : 0.1
143140 adam_beta1 : 0.9
Original file line number Diff line number Diff line change @@ -109,8 +109,7 @@ setup_cuda_training_env() {
109109 websockets==15.0.1 \
110110 msgpack==1.1.0 \
111111 datasets==4.5.0 \
112- fast-hadamard-transform==1.1.0 \
113- git+https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git@v0.2.0
112+ fast-hadamard-transform==1.1.0
114113}
115114
116115setup_metax_training_env () {
You can’t perform that action at this time.
0 commit comments