diff --git a/every_eval_ever/converters/lm_eval/adapter.py b/every_eval_ever/converters/lm_eval/adapter.py index 8b690e69e..7ef73a99c 100644 --- a/every_eval_ever/converters/lm_eval/adapter.py +++ b/every_eval_ever/converters/lm_eval/adapter.py @@ -1,6 +1,7 @@ """Adapter for converting lm-evaluation-harness output to every_eval_ever format.""" import json +import logging from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -38,6 +39,8 @@ parse_model_args, ) +logger = logging.getLogger(__name__) + class LMEvalAdapter(BaseEvaluationAdapter): """Converts lm-evaluation-harness results to every_eval_ever format.""" @@ -251,8 +254,19 @@ def _build_evaluation_results( is_higher_better = higher_is_better.get(metric_name, True) bounds = KNOWN_METRIC_BOUNDS.get(metric_name) - min_score = bounds[0] if bounds else None - max_score = bounds[1] if bounds else None + if bounds is None or bounds[0] is None or bounds[1] is None: + # `continuous` requires numeric min/max and EEE has no unbounded + # score_type, so a metric with no finite bound (e.g. perplexity) + # cannot be represented. Skip it rather than emit an invalid + # record with a null bound. + logger.warning( + 'lm_eval: skipping metric %r in task %r — no finite bound ' + 'for continuous score_type', + metric_name, + task_name, + ) + continue + min_score, max_score = bounds description = metric_name if filter_name != 'none': @@ -386,6 +400,14 @@ def transform_from_file( for task_name in tasks: task_metadata = {**metadata_args, 'task_name': task_name} log = self._transform_single(raw_data, task_metadata) + if not log.evaluation_results: + # Every metric for this task was skipped (no finite bound); an + # EvaluationLog with no results conveys nothing, so drop it. + logger.warning( + 'lm_eval: skipping task %r — no representable metrics', + task_name, + ) + continue results.append(log) return results diff --git a/every_eval_ever/converters/lm_eval/utils.py b/every_eval_ever/converters/lm_eval/utils.py index 92fad5d27..cf5c4cbcb 100644 --- a/every_eval_ever/converters/lm_eval/utils.py +++ b/every_eval_ever/converters/lm_eval/utils.py @@ -71,6 +71,10 @@ def find_samples_file(output_dir: Path, task_name: str) -> Optional[Path]: 'rouge2': (0.0, 1.0), 'rougeL': (0.0, 1.0), 'rougeLsum': (0.0, 1.0), - 'ter': (0.0, None), + # TER (sacrebleu translation edit rate) is reported on a 0-100 scale, like + # `bleu`. It can exceed 100 in pathological cases, but 0-100 is the reported + # range; EEE has no unbounded score_type and does not range-check scores, so + # a finite nominal bound is used rather than None (None -> invalid record). + 'ter': (0.0, 100.0), 'brier_score': (0.0, 1.0), } diff --git a/tests/test_lm_eval_adapter.py b/tests/test_lm_eval_adapter.py index 18d4a70c7..f7b5ceba0 100644 --- a/tests/test_lm_eval_adapter.py +++ b/tests/test_lm_eval_adapter.py @@ -1,3 +1,4 @@ +import json import tempfile from pathlib import Path @@ -139,6 +140,70 @@ def test_transform_from_file_evaluation_results(): assert rephrased_results[0].score_details.score == 0.0004 +def test_unbounded_metrics_do_not_produce_invalid_records(tmp_path): + """Regression: metrics with no finite bound must not build a `continuous` + MetricConfig with a null bound. + + `ter` previously had `(0.0, None)` and any metric absent from the bounds + table fell through to `None`/`None`; either raises + `score_type 'continuous' requires max_score` at MetricConfig construction, + crashing the conversion. `ter` now takes a finite 0-100 bound, and a + genuinely-unbounded metric (perplexity) is skipped instead of emitted. + """ + raw = { + 'config': { + 'model': 'hf', + 'model_args': 'pretrained=EleutherAI/pythia-160m', + }, + 'model_name': 'EleutherAI/pythia-160m', + 'lm_eval_version': '0.4.0', + 'results': { + 'wmt_ter': { + 'alias': 'wmt_ter', + 'ter,none': 42.0, + 'ter_stderr,none': 1.5, + }, + 'mixed_task': { + 'alias': 'mixed_task', + 'acc,none': 0.5, + 'perplexity,none': 12.3, + }, + 'ppl_only': { + 'alias': 'ppl_only', + 'word_perplexity,none': 30.0, + }, + }, + 'higher_is_better': { + 'wmt_ter': {'ter': False}, + 'mixed_task': {'acc': True, 'perplexity': False}, + 'ppl_only': {'word_perplexity': False}, + }, + 'configs': {}, + } + path = tmp_path / 'results_synth.json' + path.write_text(json.dumps(raw)) + + logs = LMEvalAdapter().transform_from_file(path, _make_metadata_args()) + by_task = {log.evaluation_id.split('/')[0]: log for log in logs} + + # ppl_only's only metric is unbounded -> the whole log is dropped. + assert set(by_task) == {'wmt_ter', 'mixed_task'} + + # ter now has a finite 0-100 bound and yields a valid continuous record. + ter = by_task['wmt_ter'].evaluation_results + assert len(ter) == 1 + assert ter[0].metric_config.min_score == 0.0 + assert ter[0].metric_config.max_score == 100.0 + assert ter[0].metric_config.lower_is_better is True + assert ter[0].score_details.score == 42.0 + + # perplexity is skipped; the bounded `acc` metric survives. + mixed = by_task['mixed_task'].evaluation_results + assert len(mixed) == 1 + assert mixed[0].metric_config.evaluation_description == 'acc' + assert mixed[0].metric_config.max_score == 1.0 + + def test_transform_from_file_uncertainty(): adapter = LMEvalAdapter() logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args())