Add LFM2.5-VL-3B benchmark handling

Author: rshube

vlmeval/config.py

@@ -2424,10 +2424,17 @@
 }
 
 lfm2vl_series = {
-    "LFM2-VL-450M": partial(vlm.LFM2VL, model_path="LiquidAI/LFM2-VL-450M"),
-    "LFM2-VL-1.6B": partial(vlm.LFM2VL, model_path="LiquidAI/LFM2-VL-1.6B"),
-    "LFM2-VL-3B": partial(vlm.LFM2VL, model_path="LiquidAI/LFM2-VL-3B"),
-    "LFM2.5-VL-1.6B": partial(vlm.LFM2VL, model_path="LiquidAI/LFM2.5-VL-1.6B"),
+    "LFM2-VL-450M": partial(
+        vlm.LFM2VL, model_path="LiquidAI/LFM2-VL-450M", use_custom_prompt=False,
+    ),
+    "LFM2-VL-1.6B": partial(
+        vlm.LFM2VL, model_path="LiquidAI/LFM2-VL-1.6B", use_custom_prompt=False,
+    ),
+    "LFM2-VL-3B": partial(vlm.LFM2VL, model_path="LiquidAI/LFM2-VL-3B", use_custom_prompt=False),
+    "LFM2.5-VL-1.6B": partial(
+        vlm.LFM2VL, model_path="LiquidAI/LFM2.5-VL-1.6B", use_custom_prompt=False,
+    ),
+    "LFM2.5-VL-3B": partial(vlm.LFM2VL, model_path="LiquidAI/LFM2.5-VL-3B"),
 }
 
 covt_series = {

vlmeval/dataset/mmifeval.py

@@ -9,6 +9,7 @@
 from vlmeval.utils import track_progress_rich
 from .image_base import ImageBaseDataset
 from .utils import DEBUG_MESSAGE, build_judge
+from .utils.mmif import function_and_compare
 
 logger = get_logger(__name__)
 
@@ -293,7 +294,7 @@ def judge_one_item(item, retry=3):
                     score = 1.0
                     # breakpoint()
                     for func_dict in constraint["judge"]["verify_funcs"]:
-                        func = globals()[func_dict["func"]]
+                        func = getattr(function_and_compare, func_dict["func"])
                         # use * to unpack the list, ** is used for dict
                         judge_result = func(str(item["prediction"]), *func_dict["params"])
                         # breakpoint()

vlmeval/smp/file.py

@@ -73,8 +73,7 @@ def localize_df(data, dname, nproc=32):
     ret = pool.map(decode_img_omni, tups)
     pool.close()
     data.pop('image')
-    if 'image_path' not in data:
-        data['image_path'] = [x[0] if len(x) == 1 else x for x in ret]
+    data['image_path'] = [x[0] if len(x) == 1 else x for x in ret]
     return data