[train][DeepSeek-V4]: update deepseek-v4 test configs.

Author: LiJunscs

flagscale/train/megatron/training/arguments_fs.py

@@ -813,11 +813,6 @@ def _add_distributed_args(parser):
         action='store_true',
         help='Indicate whether not running on a shared file system.',
     )
-    group.add_argument(
-        '--use-padded-layerwise-optimizer',
-        action='store_true',
-        help='Enable pad when use layer-wise optimizer.'
-    )
     return parser
 
 

tests/functional_tests/train/deepseek/conf/tp1_pp2_ep2_v4.yaml

@@ -26,8 +26,9 @@ experiment:
     CUDNN_BENCHMARK: "false"
     CUDNN_DETERMINISTIC: "true"
     # Only for debug
-    # NVTE_DEBUG: 1
-    # NVTE_DEBUG_LEVEL: 2
+    NVTE_DEBUG: 1
+    NVTE_DEBUG_LEVEL: 2
+    TORCH_COMPILE_DISABLE: 1
     # CUDA_LAUNCH_BLOCKING: 1
     # NCCL_DEBUG: INFO
     # NCCL_NVLS_ENABLE: 0    # Some CUDA and NCCL versions maybe cause Cuda failure 401 'the operation cannot be performed in the present state'. Set it for debug is ok. But this will cause performance drop, please check your CUDA and NCCL versions.
@@ -36,6 +37,7 @@ experiment:
     # CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
     # CUDA_DEVICE_MAX_CONNECTIONS: 1
     # NVTE_TORCH_COMPILE: 0
+    MEGATRON_LOGGING_LEVEL: 40
   cmds:
     before_start: source /root/miniconda3/bin/activate flagscale-train
 action: run

tests/functional_tests/train/deepseek/conf/train/tp1_pp2_ep2_v4.yaml

@@ -18,14 +18,16 @@ system:
   reset_attention_mask: true
   qk_layernorm: true
   sequence_parallel: true
-  use_distributed_optimizer: false
-  use_padded_layerwise_optimizer: false
+  use_distributed_optimizer: true
   num_distributed_optimizer_instances: 1
   overlap_grad_reduce: true
   overlap_param_gather: true
-  attention_backend: auto
-  check_weight_hash_across_dp_replicas_interval: 1
+  attention_backend: unfused
+  check_weight_hash_across_dp_replicas_interval: 10
+  check_for_large_grads: true
   calculate_per_token_loss: true
+  deterministic_mode: true
+  recompute_granularity: selective
   recompute_modules:
     - mhc
   precision:
@@ -69,8 +71,7 @@ model:
   clip_grad: 1.0
   position_embedding_type: rope
   untie_embeddings_and_output_weights: true
-  no_position_embedding: true
-  no_rope_fusion: true
+  no_position_embedding: true   # no use
 
   # mla args ==================
   multi_latent_attention: true
@@ -114,6 +115,7 @@ model:
   o_lora_rank: 128
   q_lora_rank: 128
   csa_compress_ratios: "[128]*2+([128]+[4])*1+[128]"
+  activation_func_clamp_value: 10.0
 
   # mtp_args
   mtp_num_layers: 1
@@ -136,15 +138,13 @@ model:
   # pytorch_profiler_collect_callstack: true
 
   optimizer:
-    optimizer: dist_muon
-    muon_tp_mode: distributed
     clip_grad: 1.0
     weight_decay: 0.1
     adam_beta1: 0.9
     adam_beta2: 0.95
     lr_scheduler:
-      lr: 3.0e-3
-      min_lr: 3.0e-4
+      lr: 3.9e-6
+      min_lr: 3.9e-7
       lr_warmup_fraction: 0.01
       lr_decay_style: WSD
       lr_wsd_decay_style: cosine

tests/functional_tests/train/deepseek/gold_values/tp1_pp2_ep2_v4.json

@@ -0,0 +1 @@
+{"lm loss:": {"values": [12.32605, 12.32894, 18.66876, 13.79999, 9.770184, 10.72942, 10.59103, 10.65058, 10.09267, 10.82254]}}