From 1b1cd78f6d9001cb5e1bd0877239950591d33ffc Mon Sep 17 00:00:00 2001
From: burtenshaw <ben.burtenshaw@gmail.com>
Date: Fri, 3 Jul 2026 09:05:58 +0200
Subject: [PATCH] refactor inspect eval kwargs

---
 src/openenv/core/evals/inspect_harness.py     | 72 +++++++++----------
 tests/core/test_evals/test_inspect_harness.py |  6 ++
 2 files changed, 38 insertions(+), 40 deletions(-)

diff --git a/src/openenv/core/evals/inspect_harness.py b/src/openenv/core/evals/inspect_harness.py
index d75f5081e..7df42c51f 100644
--- a/src/openenv/core/evals/inspect_harness.py
+++ b/src/openenv/core/evals/inspect_harness.py
@@ -11,6 +11,18 @@
 
 from openenv.core.evals.base import EvalHarness
 
+_NON_EMPTY_EVAL_KWARGS = ("task_args", "model_args")
+_OPTIONAL_EVAL_KWARGS = (
+    "max_samples",
+    "max_connections",
+    "temperature",
+    "max_tokens",
+    "epochs",
+    "solver",
+    "scorer",
+    "model_base_url",
+)
+
 
 class InspectAIHarness(EvalHarness):
     """Evaluation harness wrapping Inspect AI's ``eval()`` function.
@@ -82,7 +94,6 @@ def run(
                 "Install it with: pip install 'inspect-ai>=0.3.0'"
             )
 
-        # Extract required model parameter
         model = eval_parameters.get("model")
         if model is None:
             raise ValueError(
@@ -90,48 +101,10 @@ def run(
                 "(e.g. 'openai/gpt-4o', 'hf/meta-llama/...')."
             )
 
-        # Task: explicit parameter or fall back to dataset
         task = eval_parameters.get("task", dataset)
-
-        # Build eval kwargs
-        eval_kwargs: Dict[str, Any] = {}
-
-        task_args = eval_parameters.get("task_args", {})
-        if task_args:
-            eval_kwargs["task_args"] = task_args
-
-        model_args = eval_parameters.get("model_args", {})
-        if model_args:
-            eval_kwargs["model_args"] = model_args
-
-        for key in (
-            "max_samples",
-            "max_connections",
-            "temperature",
-            "max_tokens",
-            "epochs",
-        ):
-            value = eval_parameters.get(key)
-            if value is not None:
-                eval_kwargs[key] = value
-
-        if eval_parameters.get("solver") is not None:
-            eval_kwargs["solver"] = eval_parameters["solver"]
-
-        if eval_parameters.get("scorer") is not None:
-            eval_kwargs["scorer"] = eval_parameters["scorer"]
-
-        if self.log_dir is not None:
-            eval_kwargs["log_dir"] = self.log_dir
-
-        model_base_url = eval_parameters.get("model_base_url")
-        if model_base_url is not None:
-            eval_kwargs["model_base_url"] = model_base_url
-
-        # Run evaluation
+        eval_kwargs = self._build_eval_kwargs(eval_parameters)
         logs = inspect_eval(task, model=model, **eval_kwargs)
 
-        # Extract results from the first log
         if not logs:
             raise RuntimeError(
                 "Inspect AI evaluation returned no logs. "
@@ -145,6 +118,25 @@ def run(
 
         return self._extract_scores(log)
 
+    def _build_eval_kwargs(self, eval_parameters: Dict[str, Any]) -> Dict[str, Any]:
+        """Build keyword arguments passed through to ``inspect_ai.eval``."""
+
+        eval_kwargs: Dict[str, Any] = {}
+        for key in _NON_EMPTY_EVAL_KWARGS:
+            value = eval_parameters.get(key, {})
+            if value:
+                eval_kwargs[key] = value
+
+        for key in _OPTIONAL_EVAL_KWARGS:
+            value = eval_parameters.get(key)
+            if value is not None:
+                eval_kwargs[key] = value
+
+        if self.log_dir is not None:
+            eval_kwargs["log_dir"] = self.log_dir
+
+        return eval_kwargs
+
     def _extract_scores(self, log: Any) -> Dict[str, Any]:
         """Parse an EvalLog's results into a flat score dictionary.
 
diff --git a/tests/core/test_evals/test_inspect_harness.py b/tests/core/test_evals/test_inspect_harness.py
index 60e4f2a42..0581330d7 100644
--- a/tests/core/test_evals/test_inspect_harness.py
+++ b/tests/core/test_evals/test_inspect_harness.py
@@ -182,24 +182,30 @@ def test_optional_kwargs_passed_through(self):
             {
                 "model": "openai/gpt-4o",
                 "max_samples": 100,
+                "max_connections": 4,
                 "temperature": 0.5,
                 "max_tokens": 256,
                 "epochs": 3,
+                "model_base_url": "https://example.test/v1",
             }
         )
         _, kwargs = mock_eval.call_args
         assert kwargs["max_samples"] == 100
+        assert kwargs["max_connections"] == 4
         assert kwargs["temperature"] == 0.5
         assert kwargs["max_tokens"] == 256
         assert kwargs["epochs"] == 3
+        assert kwargs["model_base_url"] == "https://example.test/v1"
 
     def test_none_optional_kwargs_omitted(self):
         _, mock_eval = self._run_harness({"model": "openai/gpt-4o"})
         _, kwargs = mock_eval.call_args
         assert "max_samples" not in kwargs
+        assert "max_connections" not in kwargs
         assert "temperature" not in kwargs
         assert "max_tokens" not in kwargs
         assert "epochs" not in kwargs
+        assert "model_base_url" not in kwargs
 
     def test_task_args_passed_through(self):
         _, mock_eval = self._run_harness(