update

Author: zhulinJulia24

autotest/all/chat_obj_fullbench_v2.py

@@ -12,155 +12,162 @@
     from opencompass.configs.chatml_datasets.C_MHChem.C_MHChem_gen import \
         datasets as C_MHChem_chatml  # noqa: F401, E501
     from opencompass.configs.chatml_datasets.CPsyExam.CPsyExam_gen import \
-        datasets as CPsyExam_chatml  # noqa: F401
+        datasets as CPsyExam_chatml  # noqa: F401, E501
     from opencompass.configs.chatml_datasets.CS_Bench.CS_Bench_gen import \
-        datasets as CS_Bench_chatml  # noqa: F401
+        datasets as CS_Bench_chatml  # noqa: F401, E501
     from opencompass.configs.chatml_datasets.HMMT2025.HMMT2025_repeat32_gen import \
-        datasets as HMMT2025_chatml  # noqa: E501, F401
+        datasets as HMMT2025_chatml  # noqa: F401, E501
     from opencompass.configs.chatml_datasets.IMO_Bench_AnswerBench.IMO_Bench_AnswerBench_gen import \
-        datasets as IMO_Bench_AnswerBench_chatml  # noqa: E501, F401
+        datasets as IMO_Bench_AnswerBench_chatml  # noqa: F401, E501
+    # ChatML Datasets
     from opencompass.configs.chatml_datasets.MaScQA.MaScQA_gen import \
-        datasets as MaScQA_chatml  # noqa: F401
+        datasets as MaScQA_chatml  # noqa: F401, E501
     from opencompass.configs.chatml_datasets.UGD_hard.UGD_hard_repeat8_gen import \
-        datasets as UGD_hard_chatml  # noqa: E501, F401
+        datasets as UGD_hard_chatml  # noqa: F401, E501
     from opencompass.configs.chatml_datasets.UGPhysics.UGPhysics_gen import \
-        datasets as UGPhysics_chatml  # noqa: F401
-    # Math
+        datasets as UGPhysics_chatml  # noqa: F401, E501
+    # Math Calculation
     from opencompass.configs.datasets.aime2024.aime2024_cascade_eval_rawprompt_gen_2f2c96 import \
-        aime2024_datasets  # noqa: E501
+        aime2024_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.aime2025.aime2025_cascade_eval_rawprompt_gen_2f2c96 import \
-        aime2025_datasets  # noqa: E501
-    # CompassAcademic (same modules as chat_objective)
-    from opencompass.configs.datasets.aime2025.aime2025_llmjudge_academic import \
+        aime2025_datasets  # noqa: F401, E501
+    # CompassAcademic Extended
+    from opencompass.configs.datasets.aime2025.aime2025_llmjudge_academic_rawprompt import \
         aime2025_datasets as \
-        CompassAcademic_aime2025_datasets  # noqa: E501, F401
+        CompassAcademic_aime2025_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.aime2026.aime2026_cascade_eval_rawprompt_gen_0970dd import \
-        aime2026_datasets  # noqa: E501
+        aime2026_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.atlas.atlas_val_rawprompt_gen_277bee import \
-        atlas_datasets  # noqa: E501, F401
-    # General reasoning
+        atlas_datasets  # noqa: F401, E501
+    # General Reasoning
     from opencompass.configs.datasets.bbeh.bbeh_llmjudge_rawprompt_gen_36b5f4 import \
-        bbeh_datasets  # noqa: E501, F401
+        bbeh_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.bigcodebench.bigcodebench_hard_complete_rawprompt_gen_95140b import \
-        bigcodebench_hard_complete_datasets  # noqa: E501, F401
+        bigcodebench_hard_complete_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.bigcodebench.bigcodebench_hard_instruct_rawprompt_gen_5cbb9f import \
-        bigcodebench_hard_instruct_datasets  # noqa: E501, F401
+        bigcodebench_hard_instruct_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.biodata.biodata_task_rawprompt_gen import \
-        biodata_task_datasets  # noqa: E501, F401
+        biodata_task_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.CARDBiomedBench.CARDBiomedBench_llmjudge_rawprompt_gen_b4d90c import \
-        cardbiomedbench_datasets  # noqa: E501, F401
+        cardbiomedbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.chem_exam.competition_rawprompt_gen import \
-        chem_competition_instruct_datasets  # noqa: E501, F401
+        chem_competition_instruct_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.chem_exam.gaokao_rawprompt_gen import \
-        chem_gaokao_instruct_datasets  # noqa: F401
+        chem_gaokao_instruct_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.ChemBench.ChemBench_llmjudge_rawprompt_gen_fa3fc4 import \
-        chembench_datasets  # noqa: E501, F401
+        chembench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.ClimaQA.ClimaQA_Gold_llm_judge_rawprompt_gen_b3080f import \
-        climaqa_datasets  # noqa: E501, F401
+        climaqa_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.cmmlu.cmmlu_llmjudge_rawprompt_gen_9f9c31 import \
-        cmmlu_datasets  # noqa: E501, F401
+        cmmlu_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.CMPhysBench.cmphysbench_rawprompt_gen import \
-        cmphysbench_datasets  # noqa: E501, F401
+        cmphysbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.Earth_Silver.Earth_Silver_llmjudge_rawprompt_gen_a84bc6 import \
-        earth_silver_mcq_datasets  # noqa: E501, F401
-    from opencompass.configs.datasets.gpqa.gpqa_cascade_eval_academic import \
-        gpqa_datasets as CompassAcademic_gpqa_datasets  # noqa: F401
+        earth_silver_mcq_datasets  # noqa: F401, E501
+    from opencompass.configs.datasets.gpqa.gpqa_cascade_eval_academic_rawprompt import \
+        gpqa_datasets as CompassAcademic_gpqa_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.gpqa.gpqa_cascade_eval_rawprompt_gen_706039 import \
-        gpqa_datasets  # noqa: E501, F401
-    from opencompass.configs.datasets.HLE.hle_llmverify_academic import \
-        hle_datasets as CompassAcademic_hle_datasets  # noqa: F401
+        gpqa_datasets  # noqa: F401, E501
+    from opencompass.configs.datasets.HLE.hle_llmverify_academic_rawprompt import \
+        hle_datasets as CompassAcademic_hle_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.HLE.hle_llmverify_rawprompt_gen_0970dd import \
-        hle_datasets  # noqa: E501, F401
+        hle_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.hmmt2026.hmmt2026_cascade_eval_rawprompt_gen_0970dd import \
-        hmmt2026_datasets  # noqa: E501
+        hmmt2026_datasets  # noqa: F401, E501
     # Coding
     from opencompass.configs.datasets.humaneval.humaneval_openai_sample_evals_rawprompt_gen_6ce2ca import \
-        humaneval_datasets  # noqa: E501, F401
+        humaneval_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.IFBench.IFBench_rawprompt_gen import \
-        ifbench_datasets  # noqa: F401
-    # Instruct following
+        ifbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.IFEval.IFEval_rawprompt_gen_e7f781 import \
-        ifeval_datasets  # noqa: E501, F401
+        ifeval_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.kcle.kcle_llm_judge_rawprompt_gen_16e383 import \
-        kcle_datasets as kcle_fix_datasets  # noqa: E501, F401
+        kcle_datasets as kcle_fix_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.korbench.korbench_single_0shot_cascade_eval_rawprompt_gen_c048da import \
-        korbench_0shot_single_datasets  # noqa: E501
+        korbench_0shot_single_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.livecodebench.livecodebench_rawprompt_gen_c09673 import \
-        LCBCodeGeneration_dataset  # noqa: E501
-    from opencompass.configs.datasets.livecodebench.livecodebench_v6_academic import \
+        LCBCodeGeneration_dataset  # noqa: F401, E501
+    from opencompass.configs.datasets.livecodebench.livecodebench_v6_academic_rawprompt import \
         LCBCodeGeneration_dataset as \
-        CompassAcademic_LCBCodeGeneration_dataset  # noqa: E501, F401
+        CompassAcademic_LCBCodeGeneration_dataset  # noqa: F401, E501
     from opencompass.configs.datasets.livecodebench_pro.livecodebench_pro_rawprompt_gen import \
-        lcb_pro_datasets  # noqa: E501, F401
+        lcb_pro_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.livemathbench.livemathbench_hard_custom_cascade_eval_rawprompt_gen_e1ce64 import \
-        livemathbench_datasets  # noqa: E501
+        livemathbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.matbench.matbench_llm_judge_rawprompt_gen_c987b6 import \
-        matbench_datasets  # noqa: E501, F401
+        matbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.math.math_500_cascade_eval_rawprompt_gen_0970dd import \
-        math_datasets  # noqa: E501
+        math_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.mbpp.sanitized_mbpp_mdblock_0shot_nocot_rawprompt_gen_30c1e5 import \
-        sanitized_mbpp_datasets  # noqa: E501, F401
+        sanitized_mbpp_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.medmcqa.medmcqa_llmjudge_rawprompt_gen_015178 import \
-        medmcqa_datasets  # noqa: E501, F401
+        medmcqa_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.MedXpertQA.MedXpertQA_llmjudge_rawprompt_gen import \
-        medxpertqa_datasets  # noqa: E501, F401
+        medxpertqa_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.mmlu.mmlu_llmjudge_rawprompt_gen_af67f0 import \
-        mmlu_datasets  # noqa: E501, F401
+        mmlu_datasets  # noqa: F401, E501
     # Knowledge
     from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_nocot_genericllmeval_rawprompt_gen_0321fb import \
-        mmlu_pro_datasets  # noqa: E501, F401
+        mmlu_pro_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.molculariq.molculariq_rawprompt_gen import \
-        moleculariq_datasets  # noqa: E501, F401
+        moleculariq_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.MolInstructions_chem.mol_instructions_chem_rawprompt_gen import \
-        mol_gen_selfies_datasets  # noqa: E501, F401
+        mol_gen_selfies_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.OlymMATH.olymmath_llmverify_rawprompt_gen_9d3a8e import \
-        olymmath_datasets  # noqa: E501, F401
+        olymmath_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.OlympiadBench.OlympiadBench_0shot_llmverify_rawprompt_gen_d3e9e4 import \
-        olympiadbench_datasets  # noqa: E501, F401
+        olympiadbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.openswi.openswi_rawprompt_gen import \
-        openswi_datasets  # noqa: F401
+        openswi_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.PHYBench.phybench_rawprompt_gen import \
-        phybench_datasets  # noqa: F401
+        phybench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.PHYSICS.PHYSICS_llm_judge_rawprompt_gen_56ebc8 import \
-        physics_datasets  # noqa: E501, F401
+        physics_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.ProteinLMBench.ProteinLMBench_llmjudge_rawprompt_gen_9627a6 import \
-        proteinlmbench_datasets  # noqa: E501, F401
+        proteinlmbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.R_Bench.rbench_llmjudge_rawprompt_gen_c24221 import \
-        RBench_datasets  # noqa: E501, F401
+        RBench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.SimpleQA.simpleqa_verified_rawprompt_gen import \
-        simpleqa_verified_datasets  # noqa: E501, F401
+        simpleqa_verified_datasets  # noqa: F401, E501
     # AI4S
+    from opencompass.configs.datasets.SmolInstruct.smolinstruct_0shot_instruct_rawprompt_gen import \
+        mini_smolinstruct_datasets_0shot_instruct as \
+        mini_smolinstruct_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.SmolInstruct.smolinstruct_0shot_instruct_rawprompt_gen import \
         smolinstruct_datasets_0shot_instruct as \
-        smolinstruct_datasets  # noqa: E501, F401
+        smolinstruct_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.srbench.srbench_rawprompt_gen import \
-        srbench_datasets  # noqa: F401
+        srbench_datasets  # noqa: F401, E501
     from opencompass.configs.datasets.supergpqa.supergpqa_cascade_rawprompt_gen_ca8345 import \
-        supergpqa_datasets  # noqa: E501, F401
-    # Summary groups (aligned with chat_objective for eval / summarizer)
+        supergpqa_datasets  # noqa: F401, E501
+    # Summary Groups
     from opencompass.configs.summarizers.groups.bbeh import \
-        bbeh_summary_groups  # noqa: F401
+        bbeh_summary_groups  # noqa: F401, E501
     from opencompass.configs.summarizers.groups.biodata import \
-        biodata_summary_groups  # noqa: F401
+        biodata_summary_groups  # noqa: F401, E501
     from opencompass.configs.summarizers.groups.cmmlu import \
-        cmmlu_summary_groups  # noqa: F401
+        cmmlu_summary_groups  # noqa: F401, E501
     from opencompass.configs.summarizers.groups.korbench import \
-        korbench_summary_groups  # noqa: F401
+        korbench_summary_groups  # noqa: F401, E501
+    from opencompass.configs.summarizers.groups.matbench import \
+        matbench_summary_groups  # noqa: F401, E501
     from opencompass.configs.summarizers.groups.mmlu import \
-        mmlu_summary_groups  # noqa: F401
+        mmlu_summary_groups  # noqa: F401, E501
     from opencompass.configs.summarizers.groups.mmlu_pro import \
-        mmlu_pro_summary_groups  # noqa: F401
-    from opencompass.configs.summarizers.groups.OlympiadBench import (  # noqa: E501, F401
-        OlympiadBench_summary_groups, OlympiadBenchMath_summary_groups,
-        OlympiadBenchPhysics_summary_groups)
+        mmlu_pro_summary_groups  # noqa: F401, E501
+    from opencompass.configs.summarizers.groups.mol_instruct import \
+        mol_instruct_summary_groups  # noqa: F401, E501
+    from opencompass.configs.summarizers.groups.OlympiadBench import \
+        OlympiadBenchPhysics_summary_groups  # noqa: F401, E501
+    from opencompass.configs.summarizers.groups.OlympiadBench import (  # noqa: F401, E501
+        OlympiadBench_summary_groups, OlympiadBenchMath_summary_groups)
     from opencompass.configs.summarizers.groups.PHYSICS import \
-        physics_summary_groups  # noqa: F401
+        physics_summary_groups  # noqa: F401, E501
+    from opencompass.configs.summarizers.groups.smolinstruct import (  # noqa: F401, E501
+        smolinstruct_mini_summary_groups, smolinstruct_summary_groups)
     from opencompass.configs.summarizers.groups.supergpqa import \
-        supergpqa_summary_groups  # noqa: F401
+        supergpqa_summary_groups  # noqa: F401, E501
 
-# LiveCodeBench v5 / v6 (deepcopy so base LCB config is not mutated).
-# Avoid `import copy`: mmengine cfg.dump() serializes top-level names; a
-# `copy` module binding becomes invalid Python in the dumped config.
 LCBCodeGeneration_v6_datasets = __import__('copy').deepcopy(
     LCBCodeGeneration_dataset)
 LCBCodeGeneration_v6_datasets['abbr'] = 'lcb_code_generation_v6'