[ROCm][CI][MoE] Fix double-transpose of fused w3 expert weights (#47874)

Signed-off-by: stefankoncarevic <stefan.koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 3 people

vllm/model_executor/layers/fused_moe/routed_experts.py

@@ -886,17 +886,21 @@ def load_weights(
                 param_name = weight_name.removeprefix(f"{self.layer_name}.")
                 param = getattr(self, param_name)
                 if is_fused:
+                    # w1 and w3 share one fused tensor; use a local copy so the
+                    # transpose below doesn't mutate loaded_weight across
+                    # iterations (else w3 is transposed twice and wrongly chunked)
+                    fused_weight = loaded_weight
                     if shard_id in {"w1", "w3"}:
-                        if loaded_weight.shape[-1] != unpadded_hidden:
+                        if fused_weight.shape[-1] != unpadded_hidden:
                             # [..., hidden, intermediate] -> [..., intermediate, hidden]
-                            loaded_weight = loaded_weight.transpose(-1, -2)
+                            fused_weight = fused_weight.transpose(-1, -2)
                         # Repurpose expert_id for deconcatenating w1 and w3
-                        experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
+                        experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
                     else:
-                        if loaded_weight.shape[-2] != unpadded_hidden:
+                        if fused_weight.shape[-2] != unpadded_hidden:
                             # [..., intermediate, hidden] -> [..., hidden, intermediate]
-                            loaded_weight = loaded_weight.transpose(-1, -2)
-                        experts_shard = loaded_weight
+                            fused_weight = fused_weight.transpose(-1, -2)
+                        experts_shard = fused_weight
                     start = 0
                 else:
                     # loaded_weight is a single expert weight, so we add a dummy expert