feat: add HiPhO benchmark task (#1186)

Luodian · web-flow · commit 5dee848c9437 · 2026-02-23T16:01:42.000+08:00
diff --git a/lmms_eval/tasks/hipho/_default_template_yaml b/lmms_eval/tasks/hipho/_default_template_yaml
@@ -0,0 +1,23 @@
+dataset_path: HY-Wan/HiPhO
+dataset_kwargs:
+  token: false
+output_type: generate_until
+doc_to_visual: !function utils.hipho_doc_to_visual
+doc_to_text: !function utils.hipho_doc_to_text
+doc_to_target: !function utils.hipho_doc_to_target
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  do_sample: false
+process_results: !function utils.hipho_process_results
+metric_list:
+  - metric: hipho_score
+    aggregation: !function utils.hipho_aggregate_results
+    higher_is_better: true
+metadata:
+  version: 0.0
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
diff --git a/lmms_eval/tasks/hipho/hipho.yaml b/lmms_eval/tasks/hipho/hipho.yaml
@@ -0,0 +1,17 @@
+group: hipho
+task:
+  - hipho_apho_2025
+  - hipho_eupho_2024
+  - hipho_eupho_2025
+  - hipho_f_ma_2024
+  - hipho_f_ma_2025
+  - hipho_ipho_2024
+  - hipho_ipho_2025
+  - hipho_nbpho_2024
+  - hipho_nbpho_2025
+  - hipho_panmechanics_2024
+  - hipho_panmechanics_2025
+  - hipho_panpho_2024
+  - hipho_panpho_2025
+metadata:
+  version: 0.0
diff --git a/lmms_eval/tasks/hipho/hipho_apho_2025.yaml b/lmms_eval/tasks/hipho/hipho_apho_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_apho_2025
+test_split: APhO_2025
diff --git a/lmms_eval/tasks/hipho/hipho_eupho_2024.yaml b/lmms_eval/tasks/hipho/hipho_eupho_2024.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_eupho_2024
+test_split: EuPhO_2024
diff --git a/lmms_eval/tasks/hipho/hipho_eupho_2025.yaml b/lmms_eval/tasks/hipho/hipho_eupho_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_eupho_2025
+test_split: EuPhO_2025
diff --git a/lmms_eval/tasks/hipho/hipho_f_ma_2024.yaml b/lmms_eval/tasks/hipho/hipho_f_ma_2024.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_f_ma_2024
+test_split: F_MA_2024
diff --git a/lmms_eval/tasks/hipho/hipho_f_ma_2025.yaml b/lmms_eval/tasks/hipho/hipho_f_ma_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_f_ma_2025
+test_split: F_MA_2025
diff --git a/lmms_eval/tasks/hipho/hipho_ipho_2024.yaml b/lmms_eval/tasks/hipho/hipho_ipho_2024.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_ipho_2024
+test_split: IPhO_2024
diff --git a/lmms_eval/tasks/hipho/hipho_ipho_2025.yaml b/lmms_eval/tasks/hipho/hipho_ipho_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_ipho_2025
+test_split: IPhO_2025
diff --git a/lmms_eval/tasks/hipho/hipho_nbpho_2024.yaml b/lmms_eval/tasks/hipho/hipho_nbpho_2024.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_nbpho_2024
+test_split: NBPhO_2024
diff --git a/lmms_eval/tasks/hipho/hipho_nbpho_2025.yaml b/lmms_eval/tasks/hipho/hipho_nbpho_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_nbpho_2025
+test_split: NBPhO_2025
diff --git a/lmms_eval/tasks/hipho/hipho_panmechanics_2024.yaml b/lmms_eval/tasks/hipho/hipho_panmechanics_2024.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_panmechanics_2024
+test_split: PanMechanics_2024
diff --git a/lmms_eval/tasks/hipho/hipho_panmechanics_2025.yaml b/lmms_eval/tasks/hipho/hipho_panmechanics_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_panmechanics_2025
+test_split: PanMechanics_2025
diff --git a/lmms_eval/tasks/hipho/hipho_panpho_2024.yaml b/lmms_eval/tasks/hipho/hipho_panpho_2024.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_panpho_2024
+test_split: PanPhO_2024
diff --git a/lmms_eval/tasks/hipho/hipho_panpho_2025.yaml b/lmms_eval/tasks/hipho/hipho_panpho_2025.yaml
@@ -0,0 +1,3 @@
+include: _default_template_yaml
+task: hipho_panpho_2025
+test_split: PanPhO_2025
diff --git a/lmms_eval/tasks/hipho/utils.py b/lmms_eval/tasks/hipho/utils.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_apho_2025`
	`3`	`+test_split: APhO_2025`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_eupho_2024`
	`3`	`+test_split: EuPhO_2024`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_eupho_2025`
	`3`	`+test_split: EuPhO_2025`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_f_ma_2024`
	`3`	`+test_split: F_MA_2024`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_f_ma_2025`
	`3`	`+test_split: F_MA_2025`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_ipho_2024`
	`3`	`+test_split: IPhO_2024`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_ipho_2025`
	`3`	`+test_split: IPhO_2025`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include: _default_template_yaml`
	`2`	`+task: hipho_nbpho_2024`
	`3`	`+test_split: NBPhO_2024`