Skip to content

Commit 5d8815e

Browse files
helloml0326claude
helloml0326
and
claude
authored
Fix/aggregate missing tasks score zero (#10)
* fix(aggregate): score missing tasks as 0 instead of excluding from denominator Previously, tasks without metrics.json output were silently excluded from the mean calculation, inflating scores for models/harnesses that crashed on some tasks. Now all tasks in tasks.json are accounted for — missing ones receive score=0 and a new `tasks_missing` field tracks the count. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(aggregate): handle hermes summary name collisions in missing-task detection The previous approach re-resolved hermes task_ids to build seen_tids, but duplicate task names (zh/en variants) caused index collisions, incorrectly marking some tasks as missing for claude-opus hermes. Switch to count-based fill: compare per_task_score length against task_meta size to determine how many zeros to pad, avoiding the fragile name-resolution path entirely. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent b3b72de commit 5d8815e

29 files changed

Lines changed: 451 additions & 363 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
.DS_Store
22
.idea/
3+
.agents/
34

45
# Python bytecode / cache
56
__pycache__/
@@ -32,3 +33,4 @@ task_analysis/
3233
attribution/
3334
uv.lock
3435
run_batch.sh
36+
docs/

‎site/scripts/aggregate_results.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,9 @@ def _record_task(
172172
errored = 0
173173
missing_meta = 0
174174

175-
if grading_type == "error" or status == "error":
175+
if grading_type in ("error", "missing") or status in ("error", "missing"):
176176
errored = 1
177-
score = 0.0 # ensure error tasks count as 0 toward all means
177+
score = 0.0 # ensure error/missing tasks count as 0 toward all means
178178
else:
179179
a, j = score_partition(grading_type, score, breakdown)
180180
if a is not None:
@@ -467,6 +467,35 @@ def aggregate_pair(
467467
if not per_task_score:
468468
return None
469469

470+
# Fill missing tasks with score=0 so the denominator is always the full
471+
# task set, preventing harness crashes from inflating the mean.
472+
tasks_recorded = len(per_task_score)
473+
total_expected = len(task_meta)
474+
missing_count = max(0, total_expected - tasks_recorded)
475+
476+
# For per-task dirs we can identify exactly which tasks are missing.
477+
missing_tids: set[str] = set()
478+
if metrics_files and missing_count > 0:
479+
seen_tids = {mp.parent.parent.name.split("_", 1)[0] for mp in metrics_files}
480+
missing_tids = set(task_meta.keys()) - seen_tids
481+
elif not metrics_files and missing_count > 0:
482+
# Summary format: task_name resolution can have collisions (duplicate
483+
# names across zh/en variants), so fall back to count-based fill.
484+
missing_tids = set(f"_missing_{i}" for i in range(missing_count))
485+
for t_id in missing_tids:
486+
_record_task(
487+
t_id=t_id,
488+
score=0.0,
489+
grading_type="missing",
490+
breakdown={},
491+
status="missing",
492+
task_meta=task_meta,
493+
per_task_score=per_task_score,
494+
automated_scores=automated_scores,
495+
judge_scores=judge_scores,
496+
buckets=buckets,
497+
)
498+
470499
# Compute slice means
471500
by_dims = {dim: {k: safe_mean(v) for k, v in d.items()} for dim, d in buckets.items()}
472501

@@ -480,6 +509,7 @@ def aggregate_pair(
480509
"tasks": len(per_task_score),
481510
"tasks_total": len(task_meta) or len(per_task_score),
482511
"tasks_errored": errored,
512+
"tasks_missing": len(missing_tids),
483513
"missing_metadata": missing_meta,
484514
"_raw_model_dir": model_dir.name,
485515
"_raw_harness_dir": harness_dir.name,

‎submissions/pawbench-4models-opusjudge-20260529__deepseek-v4-pro__hermes.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"tasks": 150,
99
"tasks_total": 150,
1010
"tasks_errored": 0,
11+
"tasks_missing": 0,
1112
"missing_metadata": 0,
1213
"updated": "2026-05-29",
1314
"by_complexity": {

‎submissions/pawbench-4models-opusjudge-20260529__deepseek-v4-pro__openclaw.json‎

Lines changed: 34 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,23 @@
22
"run": "pawbench-4models-opusjudge-20260529",
33
"model": "deepseek-v4-pro",
44
"harness": "openclaw",
5-
"overall": 0.7693,
5+
"overall": 0.7539,
66
"automated": 0.8345,
77
"judge": 0.8072,
8-
"tasks": 147,
8+
"tasks": 150,
99
"tasks_total": 150,
1010
"tasks_errored": 0,
11+
"tasks_missing": 3,
1112
"missing_metadata": 0,
1213
"updated": "2026-05-29",
1314
"by_complexity": {
14-
"L3": 0.7636,
15-
"L2": 0.795,
15+
"L3": 0.7566,
16+
"L2": 0.7402,
1617
"L1": 0.7622
1718
},
1819
"by_environment": {
19-
"closed": 0.7657,
20-
"open": 0.7919
20+
"closed": 0.7539,
21+
"open": 0.7542
2122
},
2223
"by_scenario": {
2324
"Content_Creation/Design": 0.6484,
@@ -33,7 +34,7 @@
3334
"Office_Productivity/Document": 0.7979,
3435
"Safety_Alignment/Data_Protection": 0.7611,
3536
"Safety_Alignment/Prompt_Injection": 0.7447,
36-
"Software_Engineering/Code": 0.6331,
37+
"Software_Engineering/Code": 0.5958,
3738
"Software_Engineering/Database": 0.48,
3839
"Office_Productivity/Meeting": 0.9475,
3940
"Software_Engineering/DevOps": 0.7809,
@@ -46,10 +47,10 @@
4647
"Automation_Platform/Scheduling": 0.6505,
4748
"Content_Creation/Social_Media": 0.6248,
4849
"Finance_Investment/Stock": 0.8178,
49-
"Automation_Platform/Agent": 0.814,
50+
"Automation_Platform/Agent": 0.7326,
5051
"Finance_Investment/Quantitative_Trading": 0.0,
5152
"Information_Retrieval/Social_Platform": 0.78,
52-
"Information_Retrieval/Academic": 0.69,
53+
"Information_Retrieval/Academic": 0.345,
5354
"Safety_Alignment/Harmful_Content": 0.7978,
5455
"Manufacturing_Engineering/Quality_Control": 0.5635,
5556
"Data_Analytics/Text_Analytics": 0.2
@@ -59,36 +60,36 @@
5960
"Data_Analytics": 0.8238,
6061
"Knowledge": 0.6477,
6162
"Office_Productivity": 0.9038,
62-
"Information_Retrieval": 0.8253,
63+
"Information_Retrieval": 0.7428,
6364
"Safety_Alignment": 0.76,
64-
"Software_Engineering": 0.6317,
65+
"Software_Engineering": 0.6064,
6566
"Legal": 1.0,
6667
"Finance_Investment": 0.5893,
67-
"Automation_Platform": 0.7807,
68+
"Automation_Platform": 0.7396,
6869
"Manufacturing_Engineering": 0.5635
6970
},
7071
"by_modality": {
71-
"multimodal": 0.69,
72-
"text": 0.7855
72+
"multimodal": 0.6634,
73+
"text": 0.7729
7374
},
7475
"by_channel": {
75-
"image": 0.69
76+
"image": 0.6634
7677
},
7778
"by_capability": {
78-
"Tool_Use": 0.7746,
79-
"Code_Manipulation": 0.6443,
79+
"Tool_Use": 0.759,
80+
"Code_Manipulation": 0.6258,
8081
"Logic_Reasoning": 0.7565,
81-
"Planning": 0.7557,
82-
"Math_Computation": 0.72,
82+
"Planning": 0.7474,
83+
"Math_Computation": 0.6988,
8384
"Self_Verification": 0.723,
84-
"Skill_Use": 0.5208
85+
"Skill_Use": 0.4901
8586
},
8687
"by_source": {
8788
"claweval": 0.8728,
88-
"pinchbench": 0.888,
89+
"pinchbench": 0.8494,
8990
"qwenclawbench": 0.6335,
90-
"qwenpawbench": 0.8044,
91-
"skillsbench": 0.4577,
91+
"qwenpawbench": 0.7661,
92+
"skillsbench": 0.4271,
9293
"wildclawbench": 0.7298
9394
},
9495
"by_category": {
@@ -137,20 +138,22 @@
137138
"Safety_Compliance": 0.7281,
138139
"Education_Knowledge": 0.885,
139140
"Manufacturing_Engineering": 0.5635,
140-
"Software_Engineering": 0.3788,
141+
"Software_Engineering": 0.3157,
141142
"Healthcare": 0.906,
142143
"Office_Productivity": 0.2,
143144
"Data_Analytics": 0.2947,
144145
"01_Productivity_Flow": 0.99,
145146
"02_Code_Intelligence": 1.0,
146147
"03_Social_Interaction": 0.9405,
147148
"05_Creative_Synthesis": 0.0,
148-
"06_Safety_Alignment": 0.6216
149+
"06_Safety_Alignment": 0.6216,
150+
"Information Retrieval": 0.0,
151+
"Screenshot_Task": 0.0
149152
},
150153
"by_grading": {
151-
"hybrid": 0.7577,
154+
"hybrid": 0.7514,
152155
"automated": 0.8485,
153-
"llm_judge": 0.7842
156+
"llm_judge": 0.6797
154157
},
155158
"by_subcategory": {
156159
"Writing": 0.9095,
@@ -162,7 +165,7 @@
162165
"Meeting": 0.978,
163166
"Document": 0.6,
164167
"QA": 0.9444,
165-
"Code": 0.5461,
168+
"Code": 0.4779,
166169
"DevOps": 0.8333,
167170
"Web Research": 0.8383,
168171
"Meeting Transcript Analysis": 0.9164,
@@ -199,6 +202,8 @@
199202
"Quality_Control": 0.5635,
200203
"Clinical_Data_Harmonization": 0.906,
201204
"Text_Analytics": 0.2,
202-
"Business_Intelligence": 0.3895
205+
"Business_Intelligence": 0.3895,
206+
"Academic": 0.0,
207+
"Dashboard/Admin_Panel": 0.0
203208
}
204209
}

‎submissions/pawbench-4models-opusjudge-20260529__deepseek-v4-pro__qwenpaw.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"tasks": 150,
99
"tasks_total": 150,
1010
"tasks_errored": 0,
11+
"tasks_missing": 0,
1112
"missing_metadata": 0,
1213
"updated": "2026-05-29",
1314
"by_complexity": {

‎submissions/pawbench-4models-opusjudge-20260529__glm-5.1__hermes.json‎

Lines changed: 33 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,27 @@
22
"run": "pawbench-4models-opusjudge-20260529",
33
"model": "glm-5.1",
44
"harness": "hermes",
5-
"overall": 0.6675,
5+
"overall": 0.6319,
66
"automated": 0.753,
77
"judge": 0.7134,
8-
"tasks": 142,
8+
"tasks": 150,
99
"tasks_total": 150,
1010
"tasks_errored": 1,
11+
"tasks_missing": 8,
1112
"missing_metadata": 0,
1213
"updated": "2026-05-29",
1314
"by_complexity": {
14-
"L3": 0.629,
15+
"L3": 0.5828,
1516
"L2": 0.7637,
1617
"L1": 0.7588
1718
},
1819
"by_environment": {
19-
"closed": 0.6747,
20+
"closed": 0.6329,
2021
"open": 0.6259
2122
},
2223
"by_scenario": {
23-
"Content_Creation/Design": 0.0545,
24-
"Data_Analytics/Visualization": 0.9198,
24+
"Content_Creation/Design": 0.0205,
25+
"Data_Analytics/Visualization": 0.6132,
2526
"Data_Analytics/Business_Intelligence": 0.7483,
2627
"Knowledge/QA": 0.6797,
2728
"Office_Productivity/Email": 0.9302,
@@ -50,48 +51,49 @@
5051
"Finance_Investment/Quantitative_Trading": 0.358,
5152
"Information_Retrieval/Social_Platform": 0.57,
5253
"Safety_Alignment/Harmful_Content": 0.6773,
53-
"Manufacturing_Engineering/Quality_Control": 0.4544,
54-
"Data_Analytics/Text_Analytics": 0.2
54+
"Manufacturing_Engineering/Quality_Control": 0.3635,
55+
"Data_Analytics/Text_Analytics": 0.2,
56+
"Finance_Investment/Stock": 0.0
5557
},
5658
"by_scenario_top": {
57-
"Content_Creation": 0.5653,
58-
"Data_Analytics": 0.7362,
59+
"Content_Creation": 0.3768,
60+
"Data_Analytics": 0.6953,
5961
"Knowledge": 0.6797,
6062
"Office_Productivity": 0.7643,
6163
"Information_Retrieval": 0.6478,
6264
"Safety_Alignment": 0.5418,
6365
"Software_Engineering": 0.6392,
6466
"Legal": 1.0,
65-
"Finance_Investment": 0.579,
67+
"Finance_Investment": 0.386,
6668
"Automation_Platform": 0.7138,
67-
"Manufacturing_Engineering": 0.4544
69+
"Manufacturing_Engineering": 0.3635
6870
},
6971
"by_modality": {
70-
"multimodal": 0.5792,
71-
"text": 0.682
72+
"multimodal": 0.4455,
73+
"text": 0.671
7274
},
7375
"by_channel": {
74-
"image": 0.5792
76+
"image": 0.4455
7577
},
7678
"by_capability": {
77-
"Code_Manipulation": 0.5108,
78-
"Tool_Use": 0.6722,
79-
"Planning": 0.6267,
80-
"Math_Computation": 0.6449,
81-
"Self_Verification": 0.6059,
82-
"Logic_Reasoning": 0.661,
83-
"Skill_Use": 0.3989
79+
"Code_Manipulation": 0.4087,
80+
"Tool_Use": 0.6361,
81+
"Planning": 0.6129,
82+
"Math_Computation": 0.607,
83+
"Self_Verification": 0.5854,
84+
"Logic_Reasoning": 0.6313,
85+
"Skill_Use": 0.3755
8486
},
8587
"by_source": {
86-
"claweval": 0.8556,
88+
"claweval": 0.7568,
8789
"pinchbench": 0.6957,
88-
"qwenclawbench": 0.4747,
90+
"qwenclawbench": 0.4584,
8991
"qwenpawbench": 0.7325,
90-
"skillsbench": 0.3272,
92+
"skillsbench": 0.3054,
9193
"wildclawbench": 0.6169
9294
},
9395
"by_category": {
94-
"multimodal": 0.6095,
96+
"multimodal": 0.3657,
9597
"communication": 0.924,
9698
"productivity": 0.9875,
9799
"finance": 0.8509,
@@ -124,7 +126,7 @@
124126
"System Operations and Administration": 0.4368,
125127
"Knowledge and Memory Management": 0.5584,
126128
"Data Analysis and Modeling": 0.2635,
127-
"Finance and Quantitative Trading": 0.6156,
129+
"Finance and Quantitative Trading": 0.4104,
128130
"Security and Vulnerability Management": 0.248,
129131
"Research and Information Retrieval": 0.9407,
130132
"memory": 1.0,
@@ -137,7 +139,7 @@
137139
"Safety_Compliance": 0.6794,
138140
"Screenshot_Task": 0.455,
139141
"Education_Knowledge": 0.15,
140-
"Manufacturing_Engineering": 0.4544,
142+
"Manufacturing_Engineering": 0.3635,
141143
"Software_Engineering": 0.2804,
142144
"Healthcare": 0.2917,
143145
"Office_Productivity": 0.2,
@@ -149,7 +151,7 @@
149151
"06_Safety_Alignment": 0.5875
150152
},
151153
"by_grading": {
152-
"hybrid": 0.645,
154+
"hybrid": 0.602,
153155
"automated": 0.8935,
154156
"llm_judge": 0.6097
155157
},
@@ -179,7 +181,7 @@
179181
"Memory and Context Management": 0.3564,
180182
"Quality Auditing and Diagnosis": 0.2419,
181183
"Text Parsing": 0.525,
182-
"Trading Strategy and Backtesting": 0.8731,
184+
"Trading Strategy and Backtesting": 0.4366,
183185
"Security Detection and Hardening": 0.2452,
184186
"Privacy Compliance and Auditing": 0.0,
185187
"Identity Authentication and Access Control": 0.0,
@@ -199,7 +201,7 @@
199201
"Dashboard/Admin_Panel": 0.455,
200202
"Market/Industry_Report": 0.95,
201203
"Learning/Concept_Explanation": 0.15,
202-
"Quality_Control": 0.4544,
204+
"Quality_Control": 0.3635,
203205
"Clinical_Data_Harmonization": 0.2917,
204206
"Text_Analytics": 0.2,
205207
"Business_Intelligence": 0.3895

‎submissions/pawbench-4models-opusjudge-20260529__glm-5.1__openclaw.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"tasks": 150,
99
"tasks_total": 150,
1010
"tasks_errored": 0,
11+
"tasks_missing": 0,
1112
"missing_metadata": 0,
1213
"updated": "2026-05-29",
1314
"by_complexity": {

0 commit comments

Comments
 (0)