Skip to content

Commit 6e85bcc

Browse files
wangyuchao.404TRAE CLI
andcommitted
fix(benchmark): validate EdgeBench pass rates
Reject compact EdgeBench results whose best_pass_rate falls outside [0,1], and cover upper and lower out-of-range values. Co-authored-by: TRAE CLI <noreply@bytedance.com>
1 parent 8fd614a commit 6e85bcc

2 files changed

Lines changed: 18 additions & 0 deletions

File tree

examples/edgebench-benchmark-adapter-smoke.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,22 @@ def test_contracts() -> None:
162162
assert mismatched["first_blocker"] == "edgebench_submission_rounds_mismatch"
163163
assert_boundary(mismatched)
164164

165+
for invalid_pass_rate in (1.5, -0.1):
166+
invalid = reduce_edgebench_final_result(
167+
task_id="ad_placement_optimization",
168+
run_id=f"edgebench-smoke-pass-rate-{invalid_pass_rate}",
169+
result={
170+
"best_pass_rate": invalid_pass_rate,
171+
"total_rounds": 1,
172+
"agent_submissions": 1,
173+
"auto_submissions": 0,
174+
"runtime_seconds": 60.0,
175+
},
176+
)
177+
assert invalid["countable"] is False
178+
assert invalid["first_blocker"] == "edgebench_pass_rate_out_of_range"
179+
assert_boundary(invalid)
180+
165181

166182
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
167183
return subprocess.run(

loopx/benchmark_adapters/edgebench.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@ def reduce_edgebench_final_result(
244244
blockers: list[str] = []
245245
if best_score is None and best_pass_rate is None:
246246
blockers.append("edgebench_best_metric_missing")
247+
if best_pass_rate is not None and not 0.0 <= best_pass_rate <= 1.0:
248+
blockers.append("edgebench_pass_rate_out_of_range")
247249
if total_rounds is None or total_rounds <= 0:
248250
blockers.append("edgebench_submission_rounds_missing")
249251
elif (

0 commit comments

Comments
 (0)