fix: add HallusionBench Avg as the mean of aAcc, fAcc, and qAcc

Author: Kevin Li

vlmeval/dataset/image_yorn.py

@@ -2,6 +2,7 @@
 import warnings
 
 from vlmeval.smp import dump, get_intermediate_file_path, listinstr, load
+from vlmeval.smp.status_report import is_number, to_number
 from vlmeval.utils import track_progress_rich
 from .image_base import ImageBaseDataset
 from .utils import DEBUG_MESSAGE, build_judge
@@ -107,3 +108,20 @@ def evaluate(self, eval_file, **judge_kwargs):
         score_tgt = get_intermediate_file_path(eval_file, '_score', 'csv')
         dump(score, score_tgt)
         return score
+
+    @classmethod
+    def report_primary_metric(cls, metrics: dict | None) -> dict:
+        if not isinstance(metrics, dict) or not metrics:
+            return {}
+
+        avg = metrics.get('split=Overall|Avg')
+        if is_number(avg):
+            return {'Avg': to_number(avg)}
+
+        aacc = metrics.get('split=Overall|aAcc')
+        facc = metrics.get('split=Overall|fAcc')
+        qacc = metrics.get('split=Overall|qAcc')
+        if all(is_number(x) for x in (aacc, facc, qacc)):
+            return {'Avg': (to_number(aacc) + to_number(facc) + to_number(qacc)) / 3}
+
+        return super().report_primary_metric(metrics)

vlmeval/dataset/utils/yorn.py

@@ -118,29 +118,30 @@ def calc_aAcc(data):
     data['figure_id'] = [x.split('_')[4] for x in data['index']]
     data['question_id'] = [x.split('_')[5] for x in data['index']]
 
-    res = dict(split=[], aAcc=[], fAcc=[], qAcc=[])
-    res['split'].append('Overall')
-    res['aAcc'].append(calc_aAcc(data))
-    res['fAcc'].append(calc_fAcc(data))
-    res['qAcc'].append(calc_qAcc(data))
+    res = dict(split=[], aAcc=[], fAcc=[], qAcc=[], Avg=[])
+
+    def append_split(name, subset):
+        aacc = calc_aAcc(subset)
+        facc = calc_fAcc(subset)
+        qacc = calc_qAcc(subset)
+        res['split'].append(name)
+        res['aAcc'].append(aacc)
+        res['fAcc'].append(facc)
+        res['qAcc'].append(qacc)
+        # OpenCompass MM leaderboard overall: (aAcc + fAcc + qAcc) / 3
+        res['Avg'].append((aacc + facc + qacc) / 3)
+
+    append_split('Overall', data)
 
     if 'category' in data:
         cates = list(set(data['category']))
         for c in cates:
-            sub = data[data['category'] == c]
-            res['split'].append(c)
-            res['aAcc'].append(calc_aAcc(sub))
-            res['fAcc'].append(calc_fAcc(sub))
-            res['qAcc'].append(calc_qAcc(sub))
+            append_split(c, data[data['category'] == c])
 
     if 'l2-category' in data:
         cates = list(set(data['l2-category']))
         for c in cates:
-            sub = data[data['l2-category'] == c]
-            res['split'].append(c)
-            res['aAcc'].append(calc_aAcc(sub))
-            res['fAcc'].append(calc_fAcc(sub))
-            res['qAcc'].append(calc_qAcc(sub))
+            append_split(c, data[data['l2-category'] == c])
     ret = pd.DataFrame(res)
     return ret