[Fix] Evaluate MMMU open-ended answers by content

tests/test_mmmu_open_eval.py

@@ -0,0 +1,152 @@
+import importlib.util
+import logging
+import pickle
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import pandas as pd
+
+
+def _load_pickle(path):
+    with open(path, 'rb') as file:
+        return pickle.load(file)
+
+
+def _dump_pickle(value, path):
+    with open(path, 'wb') as file:
+        pickle.dump(value, file)
+
+
+def _load_multiple_choice():
+    vlmeval = types.ModuleType('vlmeval')
+    vlmeval.__path__ = ['vlmeval']
+    dataset = types.ModuleType('vlmeval.dataset')
+    dataset.__path__ = ['vlmeval/dataset']
+    utils_package = types.ModuleType('vlmeval.dataset.utils')
+    utils_package.__path__ = ['vlmeval/dataset/utils']
+
+    smp = types.ModuleType('vlmeval.smp')
+    smp.cn_string = lambda value: False
+    smp.d2df = lambda value: value
+    smp.dump = _dump_pickle
+    smp.get_logger = logging.getLogger
+    smp.get_pred_file_format = lambda: 'pkl'
+    smp.istype = lambda value, expected: isinstance(value, expected)
+    smp.load = _load_pickle
+    smp.timestr = lambda: 'test'
+
+    smp_vlm = types.ModuleType('vlmeval.smp.vlm')
+    smp_vlm.build_option_str = lambda choices: str(choices)
+
+    vlmeval_utils = types.ModuleType('vlmeval.utils')
+    vlmeval_utils.can_infer = lambda prediction, choices: prediction if prediction in choices else False
+    vlmeval_utils.can_infer_lego = vlmeval_utils.can_infer
+
+    def track_progress(function, tasks, save, keys, **kwargs):
+        del kwargs
+        results = [function(**task) for task in tasks]
+        _dump_pickle(dict(zip(keys, results)), save)
+        return results
+
+    vlmeval_utils.track_progress_rich = track_progress
+
+    modules = {
+        'vlmeval': vlmeval,
+        'vlmeval.dataset': dataset,
+        'vlmeval.dataset.utils': utils_package,
+        'vlmeval.smp': smp,
+        'vlmeval.smp.vlm': smp_vlm,
+        'vlmeval.utils': vlmeval_utils,
+    }
+    with mock.patch.dict(sys.modules, modules):
+        mmmu_spec = importlib.util.spec_from_file_location(
+            'vlmeval.dataset.utils.mmmu',
+            'vlmeval/dataset/utils/mmmu.py',
+        )
+        mmmu = importlib.util.module_from_spec(mmmu_spec)
+        sys.modules['vlmeval.dataset.utils.mmmu'] = mmmu
+        mmmu_spec.loader.exec_module(mmmu)
+
+        mcq_spec = importlib.util.spec_from_file_location(
+            'vlmeval.dataset.utils.multiple_choice',
+            'vlmeval/dataset/utils/multiple_choice.py',
+        )
+        multiple_choice = importlib.util.module_from_spec(mcq_spec)
+        mcq_spec.loader.exec_module(multiple_choice)
+        return mmmu, multiple_choice
+
+
+class TestMMMUOpenEvaluation(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.mmmu, cls.multiple_choice = _load_multiple_choice()
+
+    def test_open_match_handles_numeric_and_text_answers(self):
+        numeric = self.mmmu.parse_open_response('Therefore, the answer is 1,250.0')
+        text = self.mmmu.parse_open_response('The final answer is mitosis.')
+
+        self.assertTrue(self.mmmu.eval_open('1250', numeric))
+        self.assertTrue(self.mmmu.eval_open('mitosis', text))
+        self.assertFalse(self.mmmu.eval_open('meiosis', text))
+
+    def test_one_character_answer_does_not_match_inside_word(self):
+        prediction = self.mmmu.parse_open_response('The area is large')
+
+        self.assertFalse(self.mmmu.eval_open('a', prediction))
+
+    def test_mixed_mmmu_uses_content_for_open_and_choice_for_mcq(self):
+        meta = pd.DataFrame({
+            'index': [1, 2],
+            'answer': ['42', 'B'],
+            'question_type': ['open', 'multiple-choice'],
+            'A': [float('nan'), 'wrong'],
+            'B': [float('nan'), 'right'],
+        })
+        data = pd.DataFrame({
+            'index': [1, 2],
+            'question': ['What is six times seven?', 'Pick the correct option'],
+            'prediction': ['Work shown. Answer: 42', 'B'],
+            'A': [float('nan'), 'wrong'],
+            'B': [float('nan'), 'right'],
+        })
+
+        with tempfile.TemporaryDirectory() as directory:
+            result_file = str(Path(directory) / 'results.pkl')
+            evaluated = self.multiple_choice.mcq_vanilla_eval(
+                None, data, meta, 1, result_file, 'MMMU_DEV_VAL')
+
+        self.assertEqual(list(evaluated['hit']), [1, 1])
+        self.assertTrue(pd.isna(evaluated.iloc[0]['A']))
+        self.assertIn('MMMU open-ended match', evaluated.iloc[0]['log'])
+
+    def test_open_result_overrides_stale_pseudo_mcq_cache(self):
+        meta = pd.DataFrame({
+            'index': [1],
+            'answer': ['42'],
+            'question_type': ['open'],
+            'A': [float('nan')],
+        })
+        data = pd.DataFrame({
+            'index': [1],
+            'question': ['What is six times seven?'],
+            'prediction': ['42'],
+            'A': [float('nan')],
+        })
+
+        with tempfile.TemporaryDirectory() as directory:
+            result_file = str(Path(directory) / 'results.pkl')
+            _dump_pickle({1: {'hit': 0, 'log': 'stale pseudo-MCQ result'}}, result_file)
+            evaluated = self.multiple_choice.mcq_vanilla_eval(
+                None, data, meta, 1, result_file, 'MMMU_DEV_VAL')
+
+        self.assertEqual(evaluated.iloc[0]['hit'], 1)
+        self.assertNotIn('stale', evaluated.iloc[0]['log'])
+
+
+if __name__ == '__main__':
+    unittest.main()

vlmeval/dataset/utils/mmmu.py

@@ -0,0 +1,116 @@
+"""Shared MMMU open-answer parsing.
+
+Adapted from the official evaluator:
+https://github.com/MMMU-Benchmark/MMMU/blob/51ce7f3e829c16bb44bc5445782686b4c3508794/eval/eval_utils.py
+"""
+
+import re
+
+
+def check_is_number(value):
+    """Return whether a value can be interpreted as a number."""
+    try:
+        float(str(value).replace(',', ''))
+        return True
+    except ValueError:
+        return False
+
+
+def normalize_str(value):
+    """Normalize an MMMU open answer to strings or a rounded number."""
+    value = str(value).strip()
+    if check_is_number(value):
+        return [round(float(value.replace(',', '')), 2)]
+
+    value = value.lower()
+    if len(value) == 1:
+        return [' ' + value, value + ' ']
+    return [value]
+
+
+def extract_numbers(value):
+    """Extract comma-separated, scientific-notation, and simple numbers."""
+    pattern_commas = r'-?\b\d{1,3}(?:,\d{3})+\b'
+    pattern_scientific = r'-?\d+(?:\.\d+)?[eE][+-]?\d+'
+    pattern_simple = r'-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])'
+    return (
+        re.findall(pattern_commas, value)
+        + re.findall(pattern_scientific, value)
+        + re.findall(pattern_simple, value)
+    )
+
+
+def parse_open_response(response):
+    """Parse candidate strings and numbers from an MMMU open response."""
+    if response == 'API Error' or response == '':
+        return 'API Error'
+
+    def get_key_subresponses(value):
+        value = value.strip().strip('.').lower()
+        sub_responses = re.split(r'\.\s(?=[A-Z])|\n', value)
+        indicators = [
+            'could be ',
+            'so ',
+            'is ',
+            'thus ',
+            'therefore ',
+            'final ',
+            'answer ',
+            'result ',
+            'are ',
+            'in total ',
+            'total ',
+            'identify ',
+            'recognize ',
+            'calculated as ',
+            'counted as ',
+            'measured as ',
+            'observed as ',
+            'concluded as ',
+            'found to be ',
+            'equals ',
+            'determined to be ',
+            'number of ',
+            'value is ',
+            'adds up to ',
+            'have ',
+            'has ',
+        ]
+        key_responses = []
+        for index, item in enumerate(sub_responses):
+            current_indicators = indicators + (['='] if index == len(sub_responses) - 1 else [])
+            candidates = [item.split(indicator)[-1].strip() for indicator in current_indicators if indicator in item]
+            if candidates:
+                shortest = min(candidates, key=len)
+                if shortest not in {':', ',', '.', '!', '?', ';', "'"}:
+                    key_responses.append(shortest)
+        return key_responses or [value]
+
+    key_responses = get_key_subresponses(str(response))
+    predictions = key_responses.copy()
+    for item in key_responses:
+        predictions.extend(extract_numbers(item))
+
+    normalized = []
+    for item in predictions:
+        normalized.extend(normalize_str(item))
+    return list(dict.fromkeys(normalized))
+
+
+def eval_open(gold, predictions):
+    """Evaluate an MMMU open response using the official matching policy."""
+    if predictions == 'API Error':
+        return False
+
+    normalized_answers = []
+    answers = gold if isinstance(gold, list) else [gold]
+    for answer in answers:
+        normalized_answers.extend(normalize_str(answer))
+
+    for prediction in predictions:
+        if isinstance(prediction, str):
+            if any(isinstance(answer, str) and answer in prediction for answer in normalized_answers):
+                return True
+        elif prediction in normalized_answers:
+            return True
+    return False

vlmeval/dataset/utils/multiple_choice.py

@@ -11,6 +11,7 @@
                          timestr)
 from vlmeval.smp.vlm import build_option_str
 from vlmeval.utils import can_infer, can_infer_lego, track_progress_rich
+from .mmmu import eval_open, parse_open_response
 
 logger = get_logger(__name__)
 
@@ -59,21 +60,6 @@
 }
 
 
-def MMMU_preproc(data):
-    cnt = 0
-    As, Bs, Ans = list(data['A']), list(data['B']), list(data['answer'])
-    lt = len(data)
-    for i in range(lt):
-        if pd.isna(As[i]):
-            As[i] = Ans[i]
-            Bs[i] = 'Other Answers'
-            cnt += 1
-    logger.info(f'During MMMU_preproc in Evaluation, {cnt} open questions are re-formulated to multi-choice ones. ')
-    data['A'] = As
-    data['B'] = Bs
-    return data
-
-
 def report_acc(df):
     # assert group in [None, 'category', 'l2-category']
     res = defaultdict(list)
@@ -437,6 +423,14 @@ def eval_vanilla(model, item, dataset_name=None):
         return dict(hit=0, log=f'Match Log: {match_log}. ')
 
 
+def eval_mmmu_open(item):
+    prediction = str(item['prediction']).rpartition('Answer:')[-1].strip()
+    parsed_prediction = parse_open_response(prediction)
+    hit = int(eval_open(item['GT'], parsed_prediction))
+    log = f'MMMU open-ended match: answer={item["GT"]!r}; parsed_prediction={parsed_prediction!r}.'
+    return dict(hit=hit, log=log)
+
+
 # For Circular Evaluation
 def eval_circular_group(model, sub_data, dataset_name=None):
     prefetched = prefetch_circular_group(sub_data, verbose=True)
@@ -478,25 +472,41 @@ def mcq_vanilla_eval(model, data, meta, nproc, result_file, dataset_name=None):
         result = load(result_file)
     answer_map = {i: c for i, c in zip(meta['index'], meta['answer'])}
 
-    if 'MMMU' in dataset_name:
-        data = MMMU_preproc(data)
-        answer_map = {k: (v if v in list(string.ascii_uppercase) else 'A') for k, v in answer_map.items()}
+    mmmu_open_indices = set()
+    if dataset_name is not None and 'MMMU' in dataset_name:
+        open_mask = pd.Series(False, index=meta.index)
+        if 'question_type' in meta:
+            question_types = meta['question_type'].astype(str).str.lower()
+            open_mask |= question_types.isin(['open', 'open-ended', 'open_ended'])
+        if 'A' in meta:
+            open_mask |= meta['A'].isna()
+        mmmu_open_indices = set(meta.loc[open_mask, 'index'])
 
     data = data[data['index'].isin(answer_map)]
     data['GT'] = [answer_map[idx] for idx in data['index']]
     items = []
+    mmmu_open_results = {}
 
     for i in range(len(data)):
-        # Dealing with the normal part
         item = data.iloc[i]
-        if item['index'] not in result:
+        if item['index'] in mmmu_open_indices:
+            # Always recompute open answers so cached results from the old pseudo-MCQ
+            # evaluation cannot leak into corrected scores.
+            mmmu_open_results[item['index']] = eval_mmmu_open(item)
+            result[item['index']] = mmmu_open_results[item['index']]
+        elif item['index'] not in result:
             items.append(item)
 
+    if mmmu_open_indices:
+        dump(result, result_file)
+
     tups = [dict(model=model, item=x, dataset_name=dataset_name) for x in items]
     keys = [x['index'] for x in items]
     if len(tups):
         res = track_progress_rich(eval_vanilla, tups, nproc=nproc, chunksize=nproc, save=result_file, keys=keys)
         result = load(result_file)
+        result.update(mmmu_open_results)
+        dump(result, result_file)
         for k, v in zip(keys, res):
             if k not in result:
                 result[k] = v