fix(lm_eval): don't build continuous metrics with a null bound

every_eval_ever/converters/lm_eval/adapter.py

@@ -1,6 +1,7 @@
 """Adapter for converting lm-evaluation-harness output to every_eval_ever format."""
 
 import json
+import logging
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
@@ -38,6 +39,8 @@
     parse_model_args,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class LMEvalAdapter(BaseEvaluationAdapter):
     """Converts lm-evaluation-harness results to every_eval_ever format."""
@@ -251,8 +254,19 @@ def _build_evaluation_results(
             is_higher_better = higher_is_better.get(metric_name, True)
 
             bounds = KNOWN_METRIC_BOUNDS.get(metric_name)
-            min_score = bounds[0] if bounds else None
-            max_score = bounds[1] if bounds else None
+            if bounds is None or bounds[0] is None or bounds[1] is None:
+                # `continuous` requires numeric min/max and EEE has no unbounded
+                # score_type, so a metric with no finite bound (e.g. perplexity)
+                # cannot be represented. Skip it rather than emit an invalid
+                # record with a null bound.
+                logger.warning(
+                    'lm_eval: skipping metric %r in task %r — no finite bound '
+                    'for continuous score_type',
+                    metric_name,
+                    task_name,
+                )
+                continue
+            min_score, max_score = bounds
 
             description = metric_name
             if filter_name != 'none':
@@ -386,6 +400,14 @@ def transform_from_file(
         for task_name in tasks:
             task_metadata = {**metadata_args, 'task_name': task_name}
             log = self._transform_single(raw_data, task_metadata)
+            if not log.evaluation_results:
+                # Every metric for this task was skipped (no finite bound); an
+                # EvaluationLog with no results conveys nothing, so drop it.
+                logger.warning(
+                    'lm_eval: skipping task %r — no representable metrics',
+                    task_name,
+                )
+                continue
             results.append(log)
 
         return results

every_eval_ever/converters/lm_eval/utils.py

@@ -71,6 +71,10 @@ def find_samples_file(output_dir: Path, task_name: str) -> Optional[Path]:
     'rouge2': (0.0, 1.0),
     'rougeL': (0.0, 1.0),
     'rougeLsum': (0.0, 1.0),
-    'ter': (0.0, None),
+    # TER (sacrebleu translation edit rate) is reported on a 0-100 scale, like
+    # `bleu`. It can exceed 100 in pathological cases, but 0-100 is the reported
+    # range; EEE has no unbounded score_type and does not range-check scores, so
+    # a finite nominal bound is used rather than None (None -> invalid record).
+    'ter': (0.0, 100.0),
     'brier_score': (0.0, 1.0),
 }

tests/test_lm_eval_adapter.py

@@ -1,3 +1,4 @@
+import json
 import tempfile
 from pathlib import Path
 
@@ -139,6 +140,70 @@ def test_transform_from_file_evaluation_results():
     assert rephrased_results[0].score_details.score == 0.0004
 
 
+def test_unbounded_metrics_do_not_produce_invalid_records(tmp_path):
+    """Regression: metrics with no finite bound must not build a `continuous`
+    MetricConfig with a null bound.
+
+    `ter` previously had `(0.0, None)` and any metric absent from the bounds
+    table fell through to `None`/`None`; either raises
+    `score_type 'continuous' requires max_score` at MetricConfig construction,
+    crashing the conversion. `ter` now takes a finite 0-100 bound, and a
+    genuinely-unbounded metric (perplexity) is skipped instead of emitted.
+    """
+    raw = {
+        'config': {
+            'model': 'hf',
+            'model_args': 'pretrained=EleutherAI/pythia-160m',
+        },
+        'model_name': 'EleutherAI/pythia-160m',
+        'lm_eval_version': '0.4.0',
+        'results': {
+            'wmt_ter': {
+                'alias': 'wmt_ter',
+                'ter,none': 42.0,
+                'ter_stderr,none': 1.5,
+            },
+            'mixed_task': {
+                'alias': 'mixed_task',
+                'acc,none': 0.5,
+                'perplexity,none': 12.3,
+            },
+            'ppl_only': {
+                'alias': 'ppl_only',
+                'word_perplexity,none': 30.0,
+            },
+        },
+        'higher_is_better': {
+            'wmt_ter': {'ter': False},
+            'mixed_task': {'acc': True, 'perplexity': False},
+            'ppl_only': {'word_perplexity': False},
+        },
+        'configs': {},
+    }
+    path = tmp_path / 'results_synth.json'
+    path.write_text(json.dumps(raw))
+
+    logs = LMEvalAdapter().transform_from_file(path, _make_metadata_args())
+    by_task = {log.evaluation_id.split('/')[0]: log for log in logs}
+
+    # ppl_only's only metric is unbounded -> the whole log is dropped.
+    assert set(by_task) == {'wmt_ter', 'mixed_task'}
+
+    # ter now has a finite 0-100 bound and yields a valid continuous record.
+    ter = by_task['wmt_ter'].evaluation_results
+    assert len(ter) == 1
+    assert ter[0].metric_config.min_score == 0.0
+    assert ter[0].metric_config.max_score == 100.0
+    assert ter[0].metric_config.lower_is_better is True
+    assert ter[0].score_details.score == 42.0
+
+    # perplexity is skipped; the bounded `acc` metric survives.
+    mixed = by_task['mixed_task'].evaluation_results
+    assert len(mixed) == 1
+    assert mixed[0].metric_config.evaluation_description == 'acc'
+    assert mixed[0].metric_config.max_score == 1.0
+
+
 def test_transform_from_file_uncertainty():
     adapter = LMEvalAdapter()
     logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args())