add scibench(math) task

lmms_eval/tasks/medqa/medqa.yaml

@@ -0,0 +1,24 @@
+dataset_path: lmms-lab/MEDQA
+dataset_kwargs:
+  token: True
+
+task: "medqa"
+test_split: test
+doc_to_target: !function utils.medqa_doc_to_target
+doc_to_visual: null
+doc_to_text: !function utils.medqa_doc_to_text
+doc_to_choice: !function utils.medqa_doc_to_choice
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: "\nAnswer with the option's letter from the given choices directly: "
+metric_list:
+  - metric: accuracy
+    aggregation: mean
+    higher_is_better: true
+
+process_results: !function utils.medqa_process_results
+
+metadata:
+  version: 0.0

lmms_eval/tasks/medqa/utils.py

@@ -0,0 +1,122 @@
+import random
+from typing import Any, Dict, List
+
+import numpy as np
+
+
+def medqa_doc_to_text(doc: Dict[str, Any], lmms_eval_specific_kwargs: Dict[str, Any]):
+    """
+    Build the MCQ prompt from MEDQA sample.
+
+    Expected doc fields (from `lmms-lab/MEDQA` parquet):
+    - "question": str
+    - "options": dict mapping letters to option strings (e.g., {"A": "...", "B": "..."})
+    - Some samples may also expose choices as list-like; we normalize to a lettered block.
+    - We do not use visuals for MEDQA.
+    """
+    question = doc.get("question", "").strip()
+
+    # Normalize options into A..E style lines
+    options = doc.get("options")
+    if isinstance(options, dict):
+        # Keep only A-E in sorted letter order if present
+        ordered_keys = [k for k in ["A", "B", "C", "D", "E"] if k in options]
+        options_block = "\n".join([f"{k}. {str(options[k]).strip()}" for k in ordered_keys])
+    elif isinstance(options, list):
+        letters = ["A", "B", "C", "D", "E"]
+        options_block = "\n".join([f"{letters[i]}. {str(opt).strip()}" for i, opt in enumerate(options)])
+    else:
+        # Fallback: try to format if already string-like
+        options_block = str(options) if options is not None else ""
+
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    prompt = f"{question}\n{options_block}"
+    return f"{pre_prompt}{prompt}{post_prompt}"
+
+
+def medqa_doc_to_target(doc: Dict[str, Any]):
+    """
+    Return the ground-truth answer letter.
+
+    MEDQA on HF commonly provides either:
+    - "answer_idx": a letter like "A"/"B"/... OR
+    - "answer": a full string like "C" or the option text. We prioritize letter if available.
+    """
+    # Prefer explicit answer letter field when present
+    if "answer_idx" in doc and isinstance(doc["answer_idx"], str) and len(doc["answer_idx"]) == 1:
+        return doc["answer_idx"].strip()
+
+    # Some variants store the letter in "answer" directly
+    ans = doc.get("answer")
+    if isinstance(ans, str) and len(ans.strip()) == 1 and ans.strip().upper() in ["A", "B", "C", "D", "E"]:
+        return ans.strip().upper()
+
+    # If answer is provided as text, try to map back to a letter via options
+    options = doc.get("options")
+    if isinstance(options, dict) and isinstance(ans, str):
+        for k, v in options.items():
+            if isinstance(v, str) and v.strip() == ans.strip():
+                return k
+
+    # Fallback: unknown -> choose a dummy; evaluation will mark as incorrect
+    return "A"
+
+
+def medqa_doc_to_choice(doc: Dict[str, Any]) -> List[str]:
+    # Detect how many choices are present and return corresponding letters
+    if isinstance(doc.get("options"), dict):
+        present = [k for k in ["A", "B", "C", "D", "E"] if k in doc["options"]]
+        if present:
+            return present
+    if isinstance(doc.get("options"), list):
+        n = min(len(doc["options"]), 5)
+        return ["A", "B", "C", "D", "E"][:n]
+    # Default to 5-way if uncertain
+    return ["A", "B", "C", "D", "E"]
+
+
+def medqa_process_results(doc: Dict[str, Any], result: List[str]):
+    """
+    Parse model output and compute accuracy against the gold letter.
+    We robustly extract a single letter from the response.
+    """
+    response = result[0].strip()
+    all_choices = medqa_doc_to_choice(doc)
+    pred = _parse_multi_choice_response(response, all_choices)
+    gt_ans = medqa_doc_to_target(doc)
+    score = 1.0 if pred == gt_ans else 0.0
+    return {"accuracy": score}
+
+
+def _parse_multi_choice_response(response: str, all_choices: List[str]) -> str:
+    # Clean punctuation around the response
+    for ch in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(ch)
+    response = " " + response + " "
+
+    candidates = []
+    # (A) style
+    for c in all_choices:
+        if f"({c})" in response:
+            candidates.append(c)
+
+    # plain letter surrounded by spaces
+    if len(candidates) == 0:
+        for c in all_choices:
+            if f" {c} " in response:
+                candidates.append(c)
+
+    # A., B., etc.
+    if len(candidates) == 0:
+        for c in all_choices:
+            if f"{c}." in response:
+                candidates.append(c)
+
+    if len(candidates) == 0:
+        return random.choice(all_choices)
+    if len(candidates) > 1:
+        # choose the last occurrence to mitigate explanations mentioning multiple letters
+        start_indexes = [response.rfind(f" {can} ") for can in candidates]
+        return candidates[int(np.argmax(start_indexes))]
+    return candidates[0]

lmms_eval/tasks/scibench/scibench.yaml

@@ -0,0 +1,24 @@
+dataset_path: lmms-lab/SciBench
+dataset_kwargs:
+  token: True
+test_split: train
+task: "scibench"
+
+doc_to_text: !function utils.scibench_doc_to_text
+doc_to_target: "answer_number"
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: |
+      Please provide a clear and step-by-step solution for a scientific problem in the categories of Chemistry, Physics, or Mathematics. The problem will specify the unit of measurement, which should not be included in the answer. Express the final answer as a decimal number with three digits after the decimal point. Conclude the answer by stating "The answer is therefore \boxed{[ANSWER]}."
+    post_prompt: "\nLet's think step by step."
+
+metric_list:
+  - metric: accuracy
+    aggregation: mean
+    higher_is_better: true
+
+process_results: !function utils.scibench_process_results
+
+metadata:
+  version: 0.0

lmms_eval/tasks/scibench/scibench_multishot.yaml

@@ -0,0 +1,23 @@
+dataset_path: lmms-lab/SuperGPQA
+dataset_kwargs:
+  token: True
+test_split: test
+task: "scibench_multishot"
+
+doc_to_text: !function utils.scibench_multishot_doc_to_text
+doc_to_target: "answer_number"
+
+lmms_eval_specific_kwargs:
+  default:
+    pre_prompt: ""
+    post_prompt: ""
+
+metric_list:
+  - metric: accuracy
+    aggregation: mean
+    higher_is_better: true
+
+process_results: !function utils.scibench_process_results
+
+metadata:
+  version: 0.0

lmms_eval/tasks/scibench/utils.py

@@ -0,0 +1,131 @@
+import re
+from math import isclose
+from typing import Dict, List, Tuple
+
+FEWSHOT_PROMPT = """Problem:
+Suppose that $10.0 \mathrm{~mol} \mathrm{C}_2 \mathrm{H}_6(\mathrm{~g})$ is confined to $4.860 \mathrm{dm}^3$ at $27^{\circ} \mathrm{C}$. Predict the pressure exerted by the ethane from the perfect gas.
+
+Solution:
+To predict ethane's pressure using the ideal gas law (PV = nRT), I'll convert temperature from 27°C to Kelvin (300.15 K, rounded to 300 K for significant figures), then substitute the given values: 10.0 mol of ethane, volume of 4.860 L, gas constant R = 0.0821 L·atm/(mol·K), and T = 300 K. Rearranging the ideal gas equation to P = nRT/V and calculating: P = (10.0 mol × 0.0821 L·atm/(mol·K) × 300 K) ÷ 4.860 L = 246.3 L·atm ÷ 4.860 L ≈ 50.7 atm. Since temperature has two significant figures, the final pressure is \(\boxed{50.7}\) atm.
+Final Answer: The final answer is \(\boxed{50.7}\). I hope it is correct.
+
+Problem:
+Assume that all gases are perfect and that data refer to 298.15 K unless otherwise stated. Calculate the change in chemical potential of a perfect gas when its pressure is increased isothermally from $1.8 \mathrm{~atm}$ to $29.5 \mathrm{~atm}$ at $40^{\circ} \mathrm{C}$.
+
+Solution:
+To determine the change in chemical potential (Δμ) of a perfect gas during pressure change, I begin with the fundamental relation μ = μ° + RT ln(P/P°), which yields Δμ = RT ln(Pf/Pi) for changes between two states. Converting the given temperature of 40°C to 313.15K and using R = 8.314 J/(mol·K), I calculate Δμ = (8.314 J/(mol·K))(313.15K)ln(29.5/1.8). The pressure ratio 29.5/1.8 ≈ 16.39 gives ln(16.39) ≈ 2.797, so Δμ = 8.314 × 313.15 × 2.797 ≈ 7274.5 J/mol, which rounds to 7.3 kJ/mol.
+Final Answer: The final answer is \(\boxed{7.3}\). I hope it is correct.
+
+Problem:
+Show that the small angular deviation of $\epsilon$ of a plumb line from the true vertical (i.e., toward the center of Earth) at a point on Earth's surface at a latitude $\lambda$ is $\epsilon = \frac{R\omega^2sin\lambda cos\lambda}{g_0 - R\omega^2 cos^2\lambda}$ where R is the radius of Earth. What is the value (in seconds of arc) of the maximum deviation? Note that the entire denominator in the answer is actually the effective $g$, and $g_0$ denotes the pure gravitational component.
+
+Solution:
+To determine the small angular deviation (ε) of a plumb line from true vertical due to Earth's rotation, we analyze the balance of forces at latitude λ: gravitational force (Fg = mg0) toward Earth's center and centrifugal force (Fc = mRω²cosλ) perpendicular to the rotation axis. The centrifugal force resolves into a vertical component (Fc,v = mRω²cos²λ) that reduces effective gravity to g = g0 - Rω²cos²λ, and a horizontal component (Fc,h = mRω²sinλcosλ) pulling toward the equator. The angular deviation equals the ratio of horizontal force to effective gravity: ε = Rω²sinλcosλ/(g0 - Rω²cos²λ). To find maximum deviation, we differentiate with respect to λ and find it occurs at λ = 45°. Using Earth values (R = 6.371×10⁶ m, ω = 7.292×10⁻⁵ rad/s, g0 ≈ 9.81 m/s²), we calculate the numerator at 45° as 1.697×10⁻² m/s² and denominator as 9.793 m/s², yielding εmax = 1.733×10⁻³ rad or approximately 357 arcseconds (6 arcminutes).
+Final Answer: The final answer is \(\boxed{6}\). I hope it is correct."""
+
+
+def scibench_doc_to_text(doc: Dict, lmms_eval_specific_kwargs: Dict) -> str:
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    question = doc["problem_text"]
+    if doc["unit"].strip():
+        question = question + " The unit of the answer is " + doc["unit"] + "."
+    return f"{pre_prompt}{question}{post_prompt}"
+
+
+def extract_boxed_answers(text):
+    # Find all boxed contents
+    matches = re.findall(r"boxed{([^}]*)}", text)
+    for m in matches:
+        # Strip spaces
+        candidate = m.strip()
+        # Keep only the numeric ones (int or decimal, with optional sign)
+        if re.fullmatch(r"[-+]?\d*\.?\d+", candidate):
+            return candidate
+    return None
+
+
+def remove_not(x):
+    match_number = re.compile("[\$]?\ *10\^[{]?\ *-?[0-9]+\ *[}]?\ *[\$]?")
+    result = re.findall(match_number, x)
+    if len(result) != 0:
+        return re.split(match_number, x)[-1]
+    return None
+
+
+def cal_not(inputs):
+    try:
+        x, ab = list(inputs)
+        match_number = re.compile("10\^[{]?\ *-?[0-9]+\ *[}]?")
+        ab = re.findall(match_number, ab)[0]
+        ab = ab[ab.find("^") + 1 :]
+        if "{" in ab:
+            ab = ab[ab.find("{") + 1 :]
+        if "}" in ab:
+            ab = ab[: ab.find("}")]
+        x = x.strip()
+        out = float(x) * 10 ** float(ab)
+        # print(float(x)*10**float(ab))
+        return str(out)
+    except:
+        print("error")
+    return inputs
+
+
+def parse_not(inputs):
+    try:
+        if not inputs:
+            return "", ""
+        if "\\times" in inputs:
+            x, ab = inputs.split("\\times")
+        elif "\times" in inputs:
+            x, ab = inputs.split("\times")
+        elif "*" in inputs:
+            x, ab = inputs.split("*")
+        else:
+            return inputs
+        return x, ab
+    except:
+        return "", ""
+
+
+def equiv_with_unit(model_output, answer, unit):
+    model_output = model_output.replace(",", "")
+    print("Model_output: ", model_output)
+    try:
+        ans = float(answer.strip())
+        first = isclose(float(model_output.strip()), ans, rel_tol=0.05)
+    except:
+        first = False
+    try:
+        model = model_output.strip().split()[0]
+        second = isclose(float(model.strip()), ans, rel_tol=0.05)
+    except:
+        second = False
+    if first or second:
+        return True
+    return False
+
+
+def clean_number_string(s):
+    return s.replace(",", "").replace("−", "-").strip()
+
+
+def scibench_process_results(doc: Dict, result: List[str]) -> Dict[str, float]:
+    pred = result[0]
+    pred = extract_boxed_answers(pred)
+    if pred:
+        res_equiv = isclose(float(clean_number_string(pred)), float(clean_number_string(doc["answer_number"])), rel_tol=0.05)
+        score = 1 if res_equiv else 0
+    else:
+        score = 0
+    return {"accuracy": score}
+
+
+def scibench_multishot_doc_to_text(doc: Dict, lmms_eval_specific_kwargs: Dict) -> str:
+    pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
+    post_prompt = lmms_eval_specific_kwargs["post_prompt"]
+    question = doc["problem_text"]
+    if doc["unit"].strip():
+        question = question + " The unit of the answer is " + doc["unit"] + "."
+    return FEWSHOT_PROMPT + "\n" + question + "\nAnswer: Let's think step by step."