diff --git a/docs/source/gmpo.md b/docs/source/gmpo.md index 8fc7f271616..f43f9fd8b21 100644 --- a/docs/source/gmpo.md +++ b/docs/source/gmpo.md @@ -22,7 +22,7 @@ trainer = GMPOTrainer( trainer.train() ``` -In GMPO, clipping is applied to the per-token *log*-importance ratios (i.e. in log space) before the geometric mean is taken, so `epsilon` and `epsilon_high` are expressed in log space: the effective ratio clipping range is `(exp(-epsilon), exp(epsilon_high))`. The paper recommends a markedly wider range than GRPO/DAPO, `(exp(-0.4), exp(0.4))`, to encourage exploration. +In GMPO, clipping is applied to the per-token *log*-importance ratios (i.e. in log space) before the geometric mean is taken, so `epsilon` and `epsilon_high` are expressed in log space: the effective ratio clipping range is `(exp(-epsilon), exp(epsilon_high))`. The paper recommends a markedly wider range than GRPO/DAPO, `(exp(-0.4), exp(0.4))`, to encourage exploration. `use_liger_kernel` is inherited from [`GRPOConfig`] but is not supported: the inherited Liger path would run the GRPO fused loss and skip the geometric-mean objective. ## GMPOTrainer diff --git a/tests/experimental/test_gmpo_trainer.py b/tests/experimental/test_gmpo_trainer.py index 811931aa249..b51988f348f 100644 --- a/tests/experimental/test_gmpo_trainer.py +++ b/tests/experimental/test_gmpo_trainer.py @@ -91,6 +91,24 @@ def test_init_with_eval_dataset(self, eval_dataset_type): else: assert trainer.eval_dataset is eval_dataset + def test_init_fails_with_liger_kernel(self): + # Raise before GRPOTrainer.__init__ so the inherited Liger path cannot silently run the GRPO fused loss. + dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train") + + training_args = GMPOConfig( + output_dir=self.tmp_dir, + use_liger_kernel=True, + report_to="none", + ) + + with pytest.raises(ValueError, match="not supported with GMPOTrainer"): + GMPOTrainer( + model="trl-internal-testing/tiny-Qwen2ForCausalLM-2.5", + reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5", + args=training_args, + train_dataset=dataset, + ) + @pytest.mark.parametrize("config_name", ["standard_prompt_only", "conversational_prompt_only"]) def test_train_conversational(self, config_name): dataset = load_dataset("trl-internal-testing/zen", config_name, split="train") diff --git a/trl/experimental/gmpo/gmpo_trainer.py b/trl/experimental/gmpo/gmpo_trainer.py index 929adce7d53..0f035fe4422 100644 --- a/trl/experimental/gmpo/gmpo_trainer.py +++ b/trl/experimental/gmpo/gmpo_trainer.py @@ -38,6 +38,12 @@ def __init__(self, model, reward_funcs, args=None, **kwargs): model_name = model if isinstance(model, str) else get_config_model_id(model.config) args = GMPOConfig(f"{model_name.split('/')[-1]}-GMPO") + if args.use_liger_kernel: + raise ValueError( + "`use_liger_kernel=True` is not supported with GMPOTrainer. The inherited Liger path runs the GRPO " + "fused loss and silently skips the geometric-mean objective. Set `use_liger_kernel=False`." + ) + super().__init__(model, reward_funcs, args=args, **kwargs) def _compute_loss(self, model, inputs):