Evalscope evaluator & MedEval evaluator for dj-sandbox

configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/demo.py

@@ -0,0 +1,19 @@
+import argparse
+
+from evalscope import TaskConfig, run_task
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--work_dir", type=str, default="outputs")
+args = parser.parse_args()
+
+task_cfg = TaskConfig(
+    model="qwen25-1.5b",
+    api_url="http://127.0.0.1:8901/v1/chat/completions",
+    api_key="EMPTY",  # pragma: allowlist secret
+    eval_type="service",
+    datasets=["gsm8k", "arc"],
+    work_dir=args.work_dir,
+    limit=20,
+)
+
+run_task(task_cfg=task_cfg)

configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/ifeval.py

@@ -0,0 +1,19 @@
+import argparse
+
+from evalscope import TaskConfig, run_task
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--work_dir", type=str, default="outputs")
+args = parser.parse_args()
+
+task_cfg = TaskConfig(
+    model="qwen25-1.5b",
+    api_url="http://127.0.0.1:8901/v1/chat/completions",
+    api_key="EMPTY",  # pragma: allowlist secret
+    eval_type="service",
+    datasets=["ifeval"],
+    work_dir=args.work_dir,
+    limit=20,
+)
+
+run_task(task_cfg=task_cfg)

configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/medagents.py

@@ -0,0 +1,39 @@
+import argparse
+
+from evalscope import TaskConfig, run_task
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--work_dir", type=str, default="outputs")
+args = parser.parse_args()
+
+task_cfg = TaskConfig(
+    model="qwen25-1.5b",
+    api_url="http://127.0.0.1:8901/v1/chat/completions",
+    api_key="EMPTY",  # pragma: allowlist secret
+    eval_type="service",
+    datasets=["general_mcq"],
+    dataset_args={
+        "general_mcq": {
+            "local_path": "medeval/data/med_data_sub/medagents",
+            "subset_list": [
+                "afrimedqa",
+                "medbullets",
+                "medexqa",
+                "medmcqa",
+                "medqa_5options",
+                "medqa",
+                "medxpertqa-r",
+                "medxpertqa-u",
+                "mmlu",
+                "mmlu-pro",
+                "pubmedqa",
+            ],
+            "prompt_template": "Please answer this medical question and select the correct answer\n{query}",
+            "query_template": "Question: {question}\n{choices}\nAnswer: {answer}\n\n",
+        }
+    },
+    work_dir=args.work_dir,
+    limit=20,
+)
+
+run_task(task_cfg=task_cfg)

configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/medjourney.py

@@ -0,0 +1,26 @@
+import argparse
+
+from evalscope import TaskConfig, run_task
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--work_dir", type=str, default="outputs")
+args = parser.parse_args()
+
+task_cfg = TaskConfig(
+    model="qwen25-1.5b",
+    api_url="http://127.0.0.1:8901/v1/chat/completions",
+    api_key="EMPTY",  # pragma: allowlist secret
+    eval_type="service",
+    datasets=["general_qa"],
+    dataset_args={
+        "general_qa": {
+            "local_path": "medeval/data/med_data_sub/medjourney",
+            "subset_list": ["dp", "dqa", "dr", "drg", "ep", "hqa", "iqa", "mp", "mqa", "pcds", "pdds", "tp"],
+            "prompt_template": "请回答下述问题\n{query}",
+        }
+    },
+    work_dir=args.work_dir,
+    limit=20,
+)
+
+run_task(task_cfg=task_cfg)

configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/perf.py

@@ -0,0 +1,25 @@
+import argparse
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.main import run_perf_benchmark
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--work_dir", type=str, default="outputs")
+args = parser.parse_args()
+
+task_cfg = Arguments(
+    parallel=[1, 100],
+    number=[10, 200],
+    model="qwen25-1.5b",
+    url="http://127.0.0.1:8901/v1/chat/completions",
+    api="openai",
+    dataset="openqa",
+    temperature=0.9,
+    max_tokens=1024,
+    min_prompt_length=10,
+    max_prompt_length=4096,
+    tokenizer_path="INFER_MODEL_PATH",
+    extra_args={"ignore_eos": True},
+    outputs_dir=args.work_dir,
+)
+results = run_perf_benchmark(task_cfg)

configs/data_juicer_recipes/sandbox/medeval/medeval_yaml/all_in_one.yaml

@@ -0,0 +1,25 @@
+type: med_evaluator
+med_task: 'all_in_one'
+
+input_path: "medeval/data/med_data_sub"
+output_root_path: 'medeval/res/sub/test'
+
+infer_model: 'qwen25-1.5b'
+infer_api_url: 'http://127.0.0.1:8901/v1'
+eval_model: 'qwen3-32b'
+eval_api_url: "http://127.0.0.1:8902/v1"
+flames_model_path: "CaasiHUANG/flames-scorer"
+
+infer_concurrency: 16
+eval_concurrency: 16
+flames_batch_size: 4
+
+env_name: 'dj-evalscope'
+env_manager: 'conda'
+evalscope_type: 'config'
+medjourney_config: 'configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/medjourney.py'
+medagents_config: 'configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/medagents.py'
+ifeval_config: 'configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/ifeval.py'
+perf_config: 'configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/perf.py'
+
+radar_parser: 'configs/data_juicer_recipes/sandbox/medeval/medeval_yaml/med_radar.yaml'

configs/data_juicer_recipes/sandbox/medeval/medeval_yaml/evalscope.yaml

@@ -0,0 +1,25 @@
+type: evalscope_evaluator
+
+# env related
+env_name: 'dj-evalscope'
+env_manager: 'conda'
+
+evalscope_type: 'config'
+config_path: 'configs/data_juicer_recipes/sandbox/medeval/evalscope_configs/demo.py'
+output_path: 'medeval/res/evalscope'
+
+# # For pt backend
+# evalscope_type: 'command'
+# model: INFER_MODEL_PATH
+# datasets: 'gsm8k'
+# output_path: 'medeval/res/evalscope/test'
+# limits: 10
+
+# For vllm backend
+# evalscope_type: 'command'
+# eval_service: 'service'
+# model: 'qwen25-1.5b'
+# datasets: 'arc'
+# api_url: 'http://127.0.0.1:8901/v1/chat/completions'
+# output_path: 'medeval/res/evalscope/test'
+# limits: 10

configs/data_juicer_recipes/sandbox/medeval/medeval_yaml/med_radar.yaml

@@ -0,0 +1,174 @@
+type: 'med_evaluator'
+med_task: 'parse_radar'
+
+input_path: "medeval/res/sub"
+output_path: "medeval/res/sub"
+title: "Med Evaluation Radar Chart"
+
+# Ultilized model for parsing
+model_dirs: ["raw", "ckpt"]
+model_order: ["raw", "ckpt"]
+model_colors:
+  raw: "#1f77b4"
+  ckpt: "#d62728"
+
+# Customized category color
+category_colors:
+  Domain Capability: "#1f77b4"
+  Risk & Security: "#ff7f0e"
+  Instruction Following: "#2ca02c"
+  Infer Performance: "#d62728"
+
+# Metrics value parser
+files:
+  - name: "MedAgents"
+    path: "medagents/stats.json"
+    metrics:
+      - name: "medagents_avg_score"
+        path: "results.overall_score"
+
+  - name: "MedJourney"
+    path: "medjourney/stats.json"
+    metrics:
+      - name: "medjourney_choice"
+        path: "results.choice"
+      - name: "medjourney_nlg"
+        path: "results.nlg"
+
+  - name: "MedSafety"
+    path: "medsafety/stats.json"
+    metrics:
+      - name: "medsafety_score"
+        path: "results.overall_score"
+
+  - name: "MedHallu"
+    path: "medhallu/stats.json"
+    metrics:
+      - name: "medhallu_with"
+        path: "results.overall.with_knowledge.f1"
+      - name: "medhallu_without"
+        path: "results.overall.without_knowledge.f1"
+
+  - name: "Flames"
+    path: "flames/stats.json"
+    metrics:
+      - name: "flames_harmless"
+        path: "result.harmless_rate"
+
+  - name: "IFEval"
+    path: "ifeval/stats.json"
+    metrics:
+      - name: "ifeval_prompt"
+        path: "results.prompt_level"
+      - name: "ifeval_instruction"
+        path: "results.inst_level"
+
+  - name: "InfoBench"
+    path: "infobench/stats.json"
+    metrics:
+      - name: "infobench_decomp"
+        path: "result.overall.accuracy"
+
+  - name: "Structflow"
+    path: "structflow/stats.json"
+    metrics:
+      - name: "structflow_wcsr"
+        path: "overall.WCSR"
+
+  - name: "Perf"
+    path: "perf/stats.json"
+    metrics:
+      - name: "perf_low_token"
+        path: "results.low_token"
+      - name: "perf_high_token"
+        path: "results.high_token"
+      - name: "perf_latency"
+        path: "results.low_latency"
+
+# Radar chart items
+items:
+  # Domain Capability
+  - category: "Domain Capability"
+    benchmark: "MedAgents"
+    metric: "Avg Score (Acc)"
+    result_key: "medagents_avg_score"
+
+  - category: "Domain Capability"
+    benchmark: "MedJourney"
+    metric: "Choice Question (bleu-1)"
+    result_key: "medjourney_choice"
+
+  - category: "Domain Capability"
+    benchmark: "MedJourney"
+    metric: "NLG Task (Rouge-L-F1)"
+    result_key: "medjourney_nlg"
+
+  # Risk & Security
+  - category: "Risk & Security"
+    benchmark: "MedSafety"
+    metric: "Avg Score"
+    result_key: "medsafety_score"
+    direction: "Minimize"
+    min: 1.0
+    max: 5.0
+
+  - category: "Risk & Security"
+    benchmark: "MedHallu"
+    metric: "w/ knowledge (F1)"
+    result_key: "medhallu_with"
+
+  - category: "Risk & Security"
+    benchmark: "MedHallu"
+    metric: "w/o knowledge (F1)"
+    result_key: "medhallu_without"
+
+  - category: "Risk & Security"
+    benchmark: "Flames"
+    metric: "Harmless (Rate)"
+    result_key: "flames_harmless"
+
+  # Instruction Following
+  - category: "Instruction Following"
+    benchmark: "IFEval"
+    metric: "Prompt (Acc)"
+    result_key: "ifeval_prompt"
+
+  - category: "Instruction Following"
+    benchmark: "IFEval"
+    metric: "Instruction (Acc)"
+    result_key: "ifeval_instruction"
+
+  - category: "Instruction Following"
+    benchmark: "InfoBench"
+    metric: "Decomposition (Acc)"
+    result_key: "infobench_decomp"
+    min: 0.0
+    max: 100.0
+
+  - category: "Instruction Following"
+    benchmark: "Structflow"
+    metric: "WCSR (Rate)"
+    result_key: "structflow_wcsr"
+
+  # Infer Performance
+  - category: "Infer Performance"
+    benchmark: "Perf"
+    metric: "Single Ouput (tok/s)"
+    result_key: "perf_low_token"
+    min: 0.0
+    max: 500.0
+
+  - category: "Infer Performance"
+    benchmark: "Perf"
+    metric: "Parallel Output (tok/s)"
+    result_key: "perf_high_token"
+    min: 1000.0
+    max: 10000.0
+
+  - category: "Infer Performance"
+    benchmark: "Perf"
+    metric: "Avg Latency (s)"
+    result_key: "perf_latency"
+    direction: "Minimize"
+    min: 3.0
+    max: 5.0