Merge branch 'open-compass:main' into more_case_into_pr

Author: zhulinJulia24

.github/workflows/pr-stage-check.yml

@@ -52,6 +52,7 @@ jobs:
       - name: Install opencompass dependencies
         run: |
           python -m pip install -r requirements.txt
+          python -m pip install huggingface-hub==1.15.0
       - name: Build and install
         run: python -m pip install -e .
       - name: Prepare dataset
@@ -105,6 +106,7 @@ jobs:
       - name: Install opencompass dependencies
         run: |
           python -m pip install -r requirements.txt
+          python -m pip install huggingface-hub==1.15.0
       - name: Build and install
         run: python -m pip install -e .
       - name: Prepare dataset
@@ -134,6 +136,7 @@ jobs:
       - name: Install opencompass dependencies
         run: |
           pip install -r requirements.txt
+          pip install huggingface-hub==1.15.0
       - name: Build and install
         run: pip install -e .
       - name: Prepare dataset

opencompass/cli/main.py

@@ -16,6 +16,7 @@
 from opencompass.utils import (HeartBeatManager, LarkReporter, get_logger,
                                pretty_print_config, read_from_station,
                                save_to_station)
+from opencompass.utils.repeat_analysis import analyze_repeat_predictions
 from opencompass.utils.run import (fill_eval_cfg, fill_infer_cfg,
                                    get_config_from_arg)
 
@@ -151,6 +152,10 @@ def parse_args():
                         help='dump the length of model responses',
                         action='store_true',
                         default=False)
+    parser.add_argument('--analysis-repeat',
+                        help='Analyze repeated predictions in viz stage.',
+                        action='store_true',
+                        default=False)
     parser.add_argument(
         '--dump-extract-rate',
         help='Whether to dump the evaluation details, including the '
@@ -500,6 +505,13 @@ def main():
             summarizer = build_from_cfg(summarizer_cfg)
             summarizer.summarize(time_str=cfg_time_str)
 
+        if args.analysis_repeat:
+            output_path = analyze_repeat_predictions(
+                cfg,
+                time_str=cfg_time_str,
+                show_progress=True,
+                print_summary=True)
+            logger.info(f'write repeat analysis to {osp.abspath(output_path)}')
 
 
 if __name__ == '__main__':

opencompass/datasets/SciReasoner/bio_instruction.py

@@ -1012,6 +1012,21 @@ def compute_AUC_for_Modification_task(task_name, task_entries, model_name):
                 over_len += 1
             else:
                 miss_len += 1
+        if not entry['model_output'].strip():
+            y_true.append(convert_to_binary_vector(entry['label'].split(',')))
+            y_pred.append(convert_to_binary_vector([]))
+            task_processed_data.append({
+                'input':
+                entry['input'],
+                'label':
+                entry['label'],
+                'processed_model_ouput': [],
+                'original_model_output':
+                entry['model_output']
+            })
+            print('label', entry['label'])
+            print('predication', [])
+            continue
         predicted_modifications = extract_modifications(entry['model_output'])
         # print(predicted_modifications)
         true_modifications = entry['label'].split(',')