From 7566981b079c7fffe7f684873461f90b867c65c7 Mon Sep 17 00:00:00 2001 From: YeonwooSung Date: Thu, 10 Sep 2026 11:58:44 +0900 Subject: [PATCH] Fail fast when GRPO combines Liger with the MoE auxiliary loss Liger fuses the GRPO loss without materializing router logits, so the MoE load-balancing term was silently dropped. Raise the same error DPO and KTO already use. --- tests/test_grpo_trainer.py | 19 +++++++++++++++++++ trl/trainer/grpo_trainer.py | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/tests/test_grpo_trainer.py b/tests/test_grpo_trainer.py index 9d2c6cc9e5f..003132ce41c 100644 --- a/tests/test_grpo_trainer.py +++ b/tests/test_grpo_trainer.py @@ -1006,6 +1006,25 @@ def test_liger_kernel_with_peft_prompt_learning_raises(self): peft_config=PromptTuningConfig(task_type=TaskType.CAUSAL_LM, num_virtual_tokens=8), ) + @require_liger_kernel + def test_init_fails_with_moe_aux_loss_and_liger(self): + dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train") + + # The MoE auxiliary loss is on by default; it is incompatible with the Liger fused loss. + training_args = GRPOConfig( + output_dir=self.tmp_dir, + use_liger_kernel=True, + report_to="none", + ) + + with pytest.raises(ValueError, match="does not support the Mixture-of-Experts load-balancing auxiliary loss"): + GRPOTrainer( + model="trl-internal-testing/tiny-Qwen3MoeForCausalLM", + reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5", + args=training_args, + train_dataset=dataset, + ) + @require_peft def test_train_peft_model(self): model = AutoModelForCausalLM.from_pretrained("trl-internal-testing/tiny-Qwen2ForCausalLM-2.5", dtype="float32") diff --git a/trl/trainer/grpo_trainer.py b/trl/trainer/grpo_trainer.py index f5c448601fb..2f3e5197940 100644 --- a/trl/trainer/grpo_trainer.py +++ b/trl/trainer/grpo_trainer.py @@ -792,6 +792,12 @@ def get_reward(environments, _env_type=env_type, **kwargs): is_moe = getattr(text_config, "output_router_logits", None) is not None self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0 self.router_aux_loss_coef = args.router_aux_loss_coef + if self.aux_loss_enabled and self.use_liger_kernel: + raise ValueError( + "Liger GRPO loss does not support the Mixture-of-Experts load-balancing auxiliary loss, because it " + "fuses the loss without materializing the router logits. Either set `router_aux_loss_coef` to `0.0` " + "to disable the auxiliary loss, or set `use_liger_kernel` to False." + ) self.scale_rewards = args.scale_rewards self.importance_sampling_level = args.importance_sampling_level self.off_policy_mask_threshold = args.off_policy_mask_threshold