From a9d4c0070a7d5a69b5ce602dd93c0eb88fd4d795 Mon Sep 17 00:00:00 2001
From: Kevin Li <kevinlee35.xx@gmail.com>
Date: Thu, 3 Sep 2026 19:40:43 +0800
Subject: [PATCH] fix: add HallusionBench Avg as the mean of aAcc, fAcc, and
 qAcc

---
 vlmeval/dataset/image_yorn.py | 18 ++++++++++++++++++
 vlmeval/dataset/utils/yorn.py | 31 ++++++++++++++++---------------
 2 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/vlmeval/dataset/image_yorn.py b/vlmeval/dataset/image_yorn.py
index c08694910..4e1e9c726 100644
--- a/vlmeval/dataset/image_yorn.py
+++ b/vlmeval/dataset/image_yorn.py
@@ -2,6 +2,7 @@
 import warnings
 
 from vlmeval.smp import dump, get_intermediate_file_path, listinstr, load
+from vlmeval.smp.status_report import is_number, to_number
 from vlmeval.utils import track_progress_rich
 from .image_base import ImageBaseDataset
 from .utils import DEBUG_MESSAGE, build_judge
@@ -107,3 +108,20 @@ def evaluate(self, eval_file, **judge_kwargs):
         score_tgt = get_intermediate_file_path(eval_file, '_score', 'csv')
         dump(score, score_tgt)
         return score
+
+    @classmethod
+    def report_primary_metric(cls, metrics: dict | None) -> dict:
+        if not isinstance(metrics, dict) or not metrics:
+            return {}
+
+        avg = metrics.get('split=Overall|Avg')
+        if is_number(avg):
+            return {'Avg': to_number(avg)}
+
+        aacc = metrics.get('split=Overall|aAcc')
+        facc = metrics.get('split=Overall|fAcc')
+        qacc = metrics.get('split=Overall|qAcc')
+        if all(is_number(x) for x in (aacc, facc, qacc)):
+            return {'Avg': (to_number(aacc) + to_number(facc) + to_number(qacc)) / 3}
+
+        return super().report_primary_metric(metrics)
diff --git a/vlmeval/dataset/utils/yorn.py b/vlmeval/dataset/utils/yorn.py
index 09db5479f..7038ca956 100644
--- a/vlmeval/dataset/utils/yorn.py
+++ b/vlmeval/dataset/utils/yorn.py
@@ -118,29 +118,30 @@ def calc_aAcc(data):
     data['figure_id'] = [x.split('_')[4] for x in data['index']]
     data['question_id'] = [x.split('_')[5] for x in data['index']]
 
-    res = dict(split=[], aAcc=[], fAcc=[], qAcc=[])
-    res['split'].append('Overall')
-    res['aAcc'].append(calc_aAcc(data))
-    res['fAcc'].append(calc_fAcc(data))
-    res['qAcc'].append(calc_qAcc(data))
+    res = dict(split=[], aAcc=[], fAcc=[], qAcc=[], Avg=[])
+
+    def append_split(name, subset):
+        aacc = calc_aAcc(subset)
+        facc = calc_fAcc(subset)
+        qacc = calc_qAcc(subset)
+        res['split'].append(name)
+        res['aAcc'].append(aacc)
+        res['fAcc'].append(facc)
+        res['qAcc'].append(qacc)
+        # OpenCompass MM leaderboard overall: (aAcc + fAcc + qAcc) / 3
+        res['Avg'].append((aacc + facc + qacc) / 3)
+
+    append_split('Overall', data)
 
     if 'category' in data:
         cates = list(set(data['category']))
         for c in cates:
-            sub = data[data['category'] == c]
-            res['split'].append(c)
-            res['aAcc'].append(calc_aAcc(sub))
-            res['fAcc'].append(calc_fAcc(sub))
-            res['qAcc'].append(calc_qAcc(sub))
+            append_split(c, data[data['category'] == c])
 
     if 'l2-category' in data:
         cates = list(set(data['l2-category']))
         for c in cates:
-            sub = data[data['l2-category'] == c]
-            res['split'].append(c)
-            res['aAcc'].append(calc_aAcc(sub))
-            res['fAcc'].append(calc_fAcc(sub))
-            res['qAcc'].append(calc_qAcc(sub))
+            append_split(c, data[data['l2-category'] == c])
     ret = pd.DataFrame(res)
     return ret