disable cross entropy flags to avoid conv loss divergence

clairesonglee · wenxie-amd · commit 3b77c7ca417c · 2025-11-07T09:15:39.000+08:00
diff --git a/examples/megatron/configs/MI300X/deepseek_v2_lite-pretrain.yaml b/examples/megatron/configs/MI300X/deepseek_v2_lite-pretrain.yaml
@@ -99,5 +99,5 @@ modules:
       turbo_sync_free_moe_stage: 0
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/deepseek_v3-pretrain.yaml b/examples/megatron/configs/MI300X/deepseek_v3-pretrain.yaml
@@ -87,5 +87,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/grok1-pretrain.yaml b/examples/megatron/configs/MI300X/grok1-pretrain.yaml
@@ -61,8 +61,8 @@ modules:
       moe_permute_fusion: false
       gradient_accumulation_fusion: false
       moe_use_legacy_grouped_gemm: true
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # ckpt
       finetune: false
diff --git a/examples/megatron/configs/MI300X/grok2-pretrain.yaml b/examples/megatron/configs/MI300X/grok2-pretrain.yaml
@@ -61,8 +61,8 @@ modules:
       moe_permute_fusion: false
       gradient_accumulation_fusion: false
       moe_use_legacy_grouped_gemm: true
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # ckpt
       finetune: false
diff --git a/examples/megatron/configs/MI300X/llama2_70B-pretrain.yaml b/examples/megatron/configs/MI300X/llama2_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/llama2_7B-pretrain.yaml b/examples/megatron/configs/MI300X/llama2_7B-pretrain.yaml
@@ -80,5 +80,5 @@ modules:
       # sequence_parallel: 1
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/llama3.1_70B-pretrain.yaml b/examples/megatron/configs/MI300X/llama3.1_70B-pretrain.yaml
@@ -76,5 +76,5 @@ modules:
       use_turbo_attention: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/llama3.1_8B-pretrain.yaml b/examples/megatron/configs/MI300X/llama3.1_8B-pretrain.yaml
@@ -72,5 +72,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/llama3.3_70B-pretrain.yaml b/examples/megatron/configs/MI300X/llama3.3_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/llama3_70B-pretrain.yaml b/examples/megatron/configs/MI300X/llama3_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/llama3_8B-pretrain.yaml b/examples/megatron/configs/MI300X/llama3_8B-pretrain.yaml
@@ -73,5 +73,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/mixtral_8x22B_v0.1-pretrain.yaml b/examples/megatron/configs/MI300X/mixtral_8x22B_v0.1-pretrain.yaml
@@ -75,5 +75,5 @@ modules:
       ckpt_format: torch
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/mixtral_8x7B_v0.1-pretrain.yaml b/examples/megatron/configs/MI300X/mixtral_8x7B_v0.1-pretrain.yaml
@@ -70,5 +70,5 @@ modules:
       ckpt_format: torch
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/qwen2.5_72B-pretrain.yaml b/examples/megatron/configs/MI300X/qwen2.5_72B-pretrain.yaml
@@ -81,5 +81,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI300X/qwen2.5_7B-pretrain.yaml b/examples/megatron/configs/MI300X/qwen2.5_7B-pretrain.yaml
@@ -74,5 +74,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/deepseek_v2_lite-pretrain.yaml b/examples/megatron/configs/MI355X/deepseek_v2_lite-pretrain.yaml
@@ -96,5 +96,5 @@ modules:
       turbo_sync_free_moe_stage: 0
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/deepseek_v3-pretrain.yaml b/examples/megatron/configs/MI355X/deepseek_v3-pretrain.yaml
@@ -87,5 +87,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/grok1-pretrain.yaml b/examples/megatron/configs/MI355X/grok1-pretrain.yaml
@@ -61,8 +61,8 @@ modules:
       moe_permute_fusion: false
       gradient_accumulation_fusion: false
       moe_use_legacy_grouped_gemm: true
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # ckpt
       finetune: false
diff --git a/examples/megatron/configs/MI355X/grok2-pretrain.yaml b/examples/megatron/configs/MI355X/grok2-pretrain.yaml
@@ -61,8 +61,8 @@ modules:
       moe_permute_fusion: false
       gradient_accumulation_fusion: false
       moe_use_legacy_grouped_gemm: true
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # ckpt
       finetune: false
diff --git a/examples/megatron/configs/MI355X/llama2_70B-pretrain.yaml b/examples/megatron/configs/MI355X/llama2_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/llama2_7B-pretrain.yaml b/examples/megatron/configs/MI355X/llama2_7B-pretrain.yaml
@@ -80,5 +80,5 @@ modules:
       # sequence_parallel: 1
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/llama3.1_70B-pretrain.yaml b/examples/megatron/configs/MI355X/llama3.1_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/llama3.1_8B-pretrain.yaml b/examples/megatron/configs/MI355X/llama3.1_8B-pretrain.yaml
@@ -67,8 +67,8 @@ modules:
       ckpt_format: torch
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # Turbo
       enable_primus_turbo: true
diff --git a/examples/megatron/configs/MI355X/llama3.3_70B-pretrain.yaml b/examples/megatron/configs/MI355X/llama3.3_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/llama3_70B-pretrain.yaml b/examples/megatron/configs/MI355X/llama3_70B-pretrain.yaml
@@ -77,5 +77,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/llama3_8B-pretrain.yaml b/examples/megatron/configs/MI355X/llama3_8B-pretrain.yaml
@@ -73,5 +73,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/mixtral_8x22B_v0.1-pretrain.yaml b/examples/megatron/configs/MI355X/mixtral_8x22B_v0.1-pretrain.yaml
@@ -75,8 +75,8 @@ modules:
       ckpt_format: torch
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # Turbo
       enable_primus_turbo: true
diff --git a/examples/megatron/configs/MI355X/mixtral_8x7B_v0.1-pretrain.yaml b/examples/megatron/configs/MI355X/mixtral_8x7B_v0.1-pretrain.yaml
@@ -70,8 +70,8 @@ modules:
       ckpt_format: torch
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
 
       # Turbo
       enable_primus_turbo: true
diff --git a/examples/megatron/configs/MI355X/qwen2.5_72B-pretrain.yaml b/examples/megatron/configs/MI355X/qwen2.5_72B-pretrain.yaml
@@ -81,5 +81,5 @@ modules:
       use_turbo_grouped_mlp: false
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
diff --git a/examples/megatron/configs/MI355X/qwen2.5_7B-pretrain.yaml b/examples/megatron/configs/MI355X/qwen2.5_7B-pretrain.yaml
@@ -74,5 +74,5 @@ modules:
       use_turbo_grouped_mlp: true
 
       # Cross entropy flags
-      cross_entropy_fusion_impl: "te"
-      cross_entropy_loss_fusion: true
+      # cross_entropy_fusion_impl: "te"
+      # cross_entropy_loss_fusion: true
