Skip to content

Commit 95fdb95

Browse files
author
Shaw
committed
chore: add swe bench runtime gate
1 parent 8c28d49 commit 95fdb95

1 file changed

Lines changed: 9 additions & 0 deletions

File tree

packages/benchmarks/orchestrator/runtime_gates.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
GAIA_OFFICIAL_DATASET_UNAVAILABLE_REASON,
1010
HERMES_SANDBOX_UNAVAILABLE_REASON,
1111
HYPERLIQUID_LIVE_UNAVAILABLE_REASON,
12+
SWE_BENCH_DOCKER_UNAVAILABLE_REASON,
1213
TERMINAL_BENCH_DOCKER_UNAVAILABLE_REASON,
1314
VISION_LANGUAGE_HARNESS_RUNTIME_UNAVAILABLE_REASON,
1415
VISION_LANGUAGE_REAL_INPUTS_UNAVAILABLE_REASON,
@@ -69,6 +70,14 @@ def build_runtime_gate_report(_workspace_root: Path | None = None) -> RuntimeGat
6970
else TERMINAL_BENCH_DOCKER_UNAVAILABLE_REASON,
7071
benchmarks=("terminal_bench",),
7172
),
73+
RuntimeGate(
74+
id="swe_bench_docker",
75+
ok=adapter_module._has_swe_bench_docker_backend(),
76+
reason=None
77+
if adapter_module._has_swe_bench_docker_backend()
78+
else SWE_BENCH_DOCKER_UNAVAILABLE_REASON,
79+
benchmarks=("swe_bench", "swe_bench_orchestrated"),
80+
),
7281
RuntimeGate(
7382
id="hermes_sandbox",
7483
ok=adapter_module._has_hermes_sandbox_backend(),

0 commit comments

Comments
 (0)