fix(normalizations): exclude -1 padding from token-norm length + guard mismatch (#1170)

src/lighteval/metrics/normalizations.py

@@ -501,6 +501,19 @@ class LogProbCharNorm:
 LogProbNormalization = LogProbCharNorm | LogProbTokenNorm | LogProbPMINorm
 
 
+def _num_continuation_tokens(tokens: list[int]) -> int:
+    """Number of real continuation tokens for a choice.
+
+    When choices of differing lengths are stacked, the shorter continuations are
+    right-padded (with ``-1``) so they can share a tensor. Those padding tokens
+    must be excluded here; otherwise the per-choice length used to normalize the
+    log-probability by token count is inflated by padding (see #1170).
+    """
+    real = sum(1 for t in tokens if t >= 0)
+    # Fall back to the raw length (never 0) so normalization never divides by zero.
+    return real or len(tokens) or 1
+
+
 def normalize_log_probs(
     normalization: LogProbNormalization,
     choices_logprob: list[float],
@@ -523,8 +536,15 @@ def normalize_log_probs(
             normalized_log_probs = [choices_logprob[ix] / len(choice) for ix, choice in enumerate(choices_text)]
         case LogProbTokenNorm():
             assert choices_tokens is not None, "choices_tokens must be provided for token normalization"
+            if len(choices_tokens) != len(choices_logprob):
+                raise ValueError(
+                    "choices_tokens and choices_logprob must have the same length for "
+                    f"token normalization (got {len(choices_tokens)} and {len(choices_logprob)}); "
+                    "this usually means the model backend returned fewer output_tokens than choices."
+                )
             normalized_log_probs = [
-                choices_logprob[ix] / len(choices_tokens[ix]) for ix in range(len(choices_logprob))
+                choices_logprob[ix] / _num_continuation_tokens(choices_tokens[ix])
+                for ix in range(len(choices_logprob))
             ]
         case LogProbPMINorm():
             assert unconditioned_logprob is not None, "unconditioned_logprob must be provided for PMI normalization"

tests/unit/metrics/test_normalizations.py

@@ -44,6 +44,24 @@ def test_token_norm():
     assert result == pytest.approx([3.333333, 10.0])
 
 
+def test_token_norm_excludes_padding():
+    # continuations padded with -1 must not count padding toward the token length (#1170)
+    choices_logprob = [10.0, 20.0]
+    choices_tokens = [[1, 2, -1], [4, 5, 6]]  # 2 real tokens vs 3
+
+    result = normalize_log_probs(LogProbTokenNorm(), choices_logprob, None, None, choices_tokens)
+    assert result == pytest.approx([5.0, 6.666667])
+
+
+def test_token_norm_length_mismatch_raises():
+    # a backend returning fewer token lists than logprobs must fail loudly, not IndexError (#1170)
+    choices_logprob = [1.0, 2.0, 3.0, 4.0]
+    choices_tokens = [[1, 2], [3, 4], [5, 6]]  # 3 vs 4
+
+    with pytest.raises(ValueError):
+        normalize_log_probs(LogProbTokenNorm(), choices_logprob, None, None, choices_tokens)
+
+
 def test_pmi_norm():
     choices_logprob = [10.0, 20.0]
     unconditioned_logprob = [5.0, 8.0]