Skip to content

Commit f53bbae

Browse files
committed
Optimize expert selection
Simplify the case when norm_topk_prob is set. Avoid extra normalization and smaller softmax computation. Results (Measured on M3 max): Before: 📊 Benchmark Summary: ================================================== Prompt: 1066.413 tokens/sec Generation: 62.131 tokens/sec ================================================== After: 📊 Benchmark Summary: ================================================== Prompt: 1103.736 tokens/sec Generation: 69.197 tokens/sec ================================================== Testing: presubmit
1 parent 34f0db3 commit f53bbae

1 file changed

Lines changed: 11 additions & 8 deletions

File tree

python/src/coreai_models/models/macos/qwen3_moe.py

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -125,16 +125,19 @@ def __init__(
125125
self.norm_topk_prob = norm_topk_prob
126126

127127
def forward(self, x: torch.Tensor) -> torch.Tensor:
128-
gates = self.gate(x)
129-
gates = torch.softmax(gates, dim=-1, dtype=torch.float32)
128+
router_logits = self.gate(x).to(torch.float32)
130129

131-
active_experts_scores, active_experts_indices = torch.topk(
132-
gates, self.top_k, dim=-1, largest=True
133-
)
134-
active_experts_indices = active_experts_indices.to(torch.uint16)
135130
if self.norm_topk_prob:
136-
partition = torch.sum(active_experts_scores, axis=-1, keepdims=True)
137-
active_experts_scores = active_experts_scores / partition
131+
top_logits, active_experts_indices = torch.topk(
132+
router_logits, self.top_k, dim=-1, largest=True
133+
)
134+
active_experts_scores = torch.softmax(top_logits, dim=-1)
135+
else:
136+
gates = torch.softmax(router_logits, dim=-1)
137+
active_experts_scores, active_experts_indices = torch.topk(
138+
gates, self.top_k, dim=-1, largest=True
139+
)
140+
active_experts_indices = active_experts_indices.to(torch.uint16)
138141

139142
y_active_experts = self.switch_mlp(x, active_experts_indices)
140143
active_experts_scores = active_experts_scores.unsqueeze(-1)

0 commit comments

Comments
 (0)