fix moe freq of dpsk v2

Author: wenxie-amd

README.md

@@ -49,9 +49,6 @@ pre-commit install
 ## Examples
 ```bash
 cd workspace/Primus
-# deepseek pretrain (default use deepseek_v2_lite model)
-./examples/deepseek/run_pretrain.sh
+# megatron pretrain
+./examples/megatron/run_pretrain.sh
 ```
-
-
-

examples/megatron/run_pretrain.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 # shellcheck disable=SC2086
+###############################################################################
+# Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#################################################################################
 
 # available models: primus/configs/models/megatron
 export MODEL_CONFIG=${MODEL_CONFIG:-deepseek_v2_lite}

examples/megatron/run_slurm_pretrain.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 # shellcheck disable=SC2086
+###############################################################################
+# Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#################################################################################
 
 export RUN_ENV=slurm
 export MODEL_CONFIG=deepseek_v2_lite

primus/configs/models/megatron/deepseek_v2.yaml

@@ -21,6 +21,7 @@ qk_pos_emb_head_dim: 64
 v_head_dim: 128
 kv_channels: 128
 # moe
+moe_layer_freq: "([0]*1+[1]*59)"
 num_experts: 160
 # num_shared_experts: 2
 moe_ffn_hidden_size: 1536 # moe_intermediate_size

primus/configs/models/megatron/deepseek_v2_lite.yaml

@@ -21,6 +21,7 @@ qk_pos_emb_head_dim: 64
 v_head_dim: 128
 kv_channels: 128
 # moe
+moe_layer_freq: "([0]*1+[1]*26)"
 num_experts: 64
 # num_shared_experts: 2
 moe_ffn_hidden_size: 1408