File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -162,6 +162,22 @@ def test_contracts() -> None:
162162 assert mismatched ["first_blocker" ] == "edgebench_submission_rounds_mismatch"
163163 assert_boundary (mismatched )
164164
165+ for invalid_pass_rate in (1.5 , - 0.1 ):
166+ invalid = reduce_edgebench_final_result (
167+ task_id = "ad_placement_optimization" ,
168+ run_id = f"edgebench-smoke-pass-rate-{ invalid_pass_rate } " ,
169+ result = {
170+ "best_pass_rate" : invalid_pass_rate ,
171+ "total_rounds" : 1 ,
172+ "agent_submissions" : 1 ,
173+ "auto_submissions" : 0 ,
174+ "runtime_seconds" : 60.0 ,
175+ },
176+ )
177+ assert invalid ["countable" ] is False
178+ assert invalid ["first_blocker" ] == "edgebench_pass_rate_out_of_range"
179+ assert_boundary (invalid )
180+
165181
166182def run_cli (* args : str ) -> subprocess .CompletedProcess [str ]:
167183 return subprocess .run (
Original file line number Diff line number Diff line change @@ -244,6 +244,8 @@ def reduce_edgebench_final_result(
244244 blockers : list [str ] = []
245245 if best_score is None and best_pass_rate is None :
246246 blockers .append ("edgebench_best_metric_missing" )
247+ if best_pass_rate is not None and not 0.0 <= best_pass_rate <= 1.0 :
248+ blockers .append ("edgebench_pass_rate_out_of_range" )
247249 if total_rounds is None or total_rounds <= 0 :
248250 blockers .append ("edgebench_submission_rounds_missing" )
249251 elif (
You can’t perform that action at this time.
0 commit comments