Reject use_liger_kernel in GMPOTrainer

docs/source/gmpo.md

@@ -22,7 +22,7 @@ trainer = GMPOTrainer(
 trainer.train()
 ```
 
-In GMPO, clipping is applied to the per-token *log*-importance ratios (i.e. in log space) before the geometric mean is taken, so `epsilon` and `epsilon_high` are expressed in log space: the effective ratio clipping range is `(exp(-epsilon), exp(epsilon_high))`. The paper recommends a markedly wider range than GRPO/DAPO, `(exp(-0.4), exp(0.4))`, to encourage exploration.
+In GMPO, clipping is applied to the per-token *log*-importance ratios (i.e. in log space) before the geometric mean is taken, so `epsilon` and `epsilon_high` are expressed in log space: the effective ratio clipping range is `(exp(-epsilon), exp(epsilon_high))`. The paper recommends a markedly wider range than GRPO/DAPO, `(exp(-0.4), exp(0.4))`, to encourage exploration. `use_liger_kernel` is inherited from [`GRPOConfig`] but is not supported: the inherited Liger path would run the GRPO fused loss and skip the geometric-mean objective.
 
 ## GMPOTrainer
 

tests/experimental/test_gmpo_trainer.py

@@ -91,6 +91,24 @@ def test_init_with_eval_dataset(self, eval_dataset_type):
         else:
             assert trainer.eval_dataset is eval_dataset
 
+    def test_init_fails_with_liger_kernel(self):
+        # Raise before GRPOTrainer.__init__ so the inherited Liger path cannot silently run the GRPO fused loss.
+        dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train")
+
+        training_args = GMPOConfig(
+            output_dir=self.tmp_dir,
+            use_liger_kernel=True,
+            report_to="none",
+        )
+
+        with pytest.raises(ValueError, match="not supported with GMPOTrainer"):
+            GMPOTrainer(
+                model="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5",
+                reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5",
+                args=training_args,
+                train_dataset=dataset,
+            )
+
     @pytest.mark.parametrize("config_name", ["standard_prompt_only", "conversational_prompt_only"])
     def test_train_conversational(self, config_name):
         dataset = load_dataset("trl-internal-testing/zen", config_name, split="train")

trl/experimental/gmpo/gmpo_trainer.py

@@ -38,6 +38,12 @@ def __init__(self, model, reward_funcs, args=None, **kwargs):
             model_name = model if isinstance(model, str) else get_config_model_id(model.config)
             args = GMPOConfig(f"{model_name.split('/')[-1]}-GMPO")
 
+        if args.use_liger_kernel:
+            raise ValueError(
+                "`use_liger_kernel=True` is not supported with GMPOTrainer. The inherited Liger path runs the GRPO "
+                "fused loss and silently skips the geometric-mean objective. Set `use_liger_kernel=False`."
+            )
+
         super().__init__(model, reward_funcs, args=args, **kwargs)
 
     def _compute_loss(self, model, inputs):