Mock LLM tests: add llm_call marker and auto-mock fixture to prevent CI hangs

Tests in test_refiner.py, test_cache_evaluation_storage.py, and
test_best_example_evals.py called real OpenRouter APIs, causing
6-hour hangs in CI (no API keys). Add an autouse conftest fixture
that intercepts litellm.completion for @pytest.mark.llm_call tests,
returning randomized JSON candidates. All 404 tests now pass in ~20s.

Author: lukedhlee

pyproject.toml

@@ -80,6 +80,9 @@ gepa = ["py.typed"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+markers = [
+    "llm_call: tests that call LLM APIs (auto-mocked via conftest fixture)",
+]
 
 [tool.ruff]
 include = ["src/**/*.py"]

tests/conftest.py

@@ -1,7 +1,9 @@
 import json
 import os
 import random
+import re
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -98,3 +100,55 @@ def mocked_lms(recorder_dir):
 @pytest.fixture
 def rng():
     return random.Random(42)
+
+
+def _make_litellm_mock_response(content: str):
+    """Create a mock litellm.completion response."""
+    resp = MagicMock()
+    resp.choices = [MagicMock()]
+    resp.choices[0].message.content = content
+    resp.choices[0].finish_reason = "stop"
+    return resp
+
+
+@pytest.fixture(autouse=True)
+def _mock_litellm_for_llm_call_tests(request):
+    """Auto-mock litellm.completion for tests marked with @pytest.mark.llm_call.
+
+    Returns a JSON dict with randomized values that satisfies both the reflection
+    proposer (expects fenced code block) and the refiner (expects JSON dict).
+    """
+    marker = request.node.get_closest_marker("llm_call")
+    if marker is None:
+        yield
+        return
+
+    call_rng = random.Random(42)
+
+    def fake_completion(*_args, **kwargs):
+        messages = kwargs.get("messages", [])
+        last_content = messages[-1]["content"] if messages else ""
+
+        # Check for multi-param candidates
+        param_a_match = re.search(r'"param_a"\s*:\s*"(\d+)"', last_content)
+        param_b_match = re.search(r'"param_b"\s*:\s*"(\d+)"', last_content)
+        if param_a_match and param_b_match:
+            a = int(param_a_match.group(1)) + call_rng.randint(-5, 5)
+            b = 100 - a + call_rng.randint(-3, 3)
+            response_text = json.dumps({"param_a": str(a), "param_b": str(b)})
+        else:
+            # Each call returns a unique candidate so the cache can't stall budget
+            number_match = re.search(r'"number"\s*:\s*"(\d+)"', last_content)
+            if number_match:
+                old = int(number_match.group(1))
+                new_number = old + call_rng.randint(-10, 10)
+            else:
+                new_number = 42 + call_rng.randint(-10, 10)
+            response_text = json.dumps({"number": str(new_number)})
+
+        # Wrap in fenced block for reflection proposer compatibility
+        content = f"```\n{response_text}\n```"
+        return _make_litellm_mock_response(content)
+
+    with patch("litellm.completion", side_effect=fake_completion):
+        yield

tests/test_best_example_evals.py

@@ -53,6 +53,7 @@ def fitness_fn(
     return fitness_fn
 
 
+@pytest.mark.llm_call
 class TestExampleBestEvals:
     """Tests for OptimizationState / best_example_evals feature."""
 

tests/test_cache_evaluation_storage.py

@@ -43,6 +43,7 @@ def fitness_fn(candidate: dict[str, str], **kwargs) -> tuple[float, dict]:
     return fitness_fn
 
 
+@pytest.mark.llm_call
 class TestCacheEvaluationStorage:
     """Tests for cache_evaluation_storage parameter."""
 

tests/test_refiner.py

@@ -111,6 +111,7 @@ def fitness_fn(candidate: dict[str, str], example, **kwargs) -> tuple[float, dic
 DATASET = [{"golden": 40}, {"golden": 60}]
 
 
+@pytest.mark.llm_call
 class TestRefiner:
     """Tests for refiner functionality."""
 
@@ -605,6 +606,7 @@ def bad_refiner_lm(prompt: str) -> str:
         assert score == -abs(50 - GOLDEN_NUMBER)
 
 
+@pytest.mark.llm_call
 class TestRefinerWithDataset:
     """Test refiner with a dataset (per-instance evaluation)."""
 
@@ -668,6 +670,7 @@ def test_refiner_dataset_mode(self):
         print(f"Metric calls: {result.total_metric_calls}, Fitness calls: {call_counter['count']}")
 
 
+@pytest.mark.llm_call
 class TestRefinerFrontierTypes:
     """Test refiner with each frontier type using a dataset."""