[OPIK-8012] [SDK] fix: bound LLM judge provider calls and retry empty responses

An empty structured-output response raises BaseLLMError, which the judge's
retry policy did not cover (LLMJudgeParseError extends OpikException, not
BaseLLMError), so a transient blank cost the item its whole score.

LiteLLMChatModel also forwarded no timeout and no num_retries, leaving
litellm's 6000s default plus its ~3 internal HTTP retries stacked under two
tenacity layers — a stalled provider blocked effectively forever. Observed
stalls sit in TCP connect, hence the tighter connect ceiling.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: AndreiCautisanu

sdks/python/src/opik/evaluation/models/litellm/litellm_chat_model.py

@@ -7,6 +7,7 @@
 import tenacity
 
 if TYPE_CHECKING:
+    import httpx
     from litellm.types.utils import ModelResponse
 
 import opik.semantic_version as semantic_version
@@ -18,6 +19,35 @@
 
 LOGGER = logging.getLogger(__name__)
 
+# LiteLLM's own default request timeout is 6000s (100 minutes), so a provider
+# that stalls — most often on TCP connect, where no response ever arrives —
+# blocks the caller effectively forever.
+#
+# The connect timeout is deliberately much tighter than the read timeout: a
+# provider either accepts the connection promptly or it is unreachable, whereas
+# generating a long completion legitimately takes time. Splitting them lets a
+# stalled connect fail fast without truncating slow-but-healthy generations.
+#
+# `num_retries` is pinned alongside them because LiteLLM retries internally by
+# default (~3 HTTP attempts per call), which silently multiplies the timeout on
+# top of the tenacity retries this module and the LLM-judge metric already
+# apply. Leaving it unset makes the worst-case wall clock the product of three
+# independent retry layers. Retrying stays the callers' job.
+#
+# All are defaults only — an explicit `timeout`/`num_retries` still wins.
+DEFAULT_CONNECT_TIMEOUT_SECONDS = 10.0
+DEFAULT_READ_TIMEOUT_SECONDS = 60.0
+DEFAULT_NUM_RETRIES = 0
+
+
+def _default_timeout() -> "httpx.Timeout":
+    import httpx
+
+    return httpx.Timeout(
+        DEFAULT_READ_TIMEOUT_SECONDS,
+        connect=DEFAULT_CONNECT_TIMEOUT_SECONDS,
+    )
+
 
 def _log_warning(message: str, *args: Any) -> None:
     """Emit a warning to both this module logger and the root logger.
@@ -378,6 +408,8 @@ def generate_provider_response(
         # we need to pop messages first, and after we will check the rest params
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
         all_kwargs = {**self._completion_kwargs, **valid_litellm_params}
+        all_kwargs.setdefault("timeout", _default_timeout())
+        all_kwargs.setdefault("num_retries", DEFAULT_NUM_RETRIES)
         # Conflicts that can only be diagnosed on the merged dict
         # (constructor + per-call sources) run here — see the method's
         # docstring for the Anthropic reasoning_effort/temperature case.
@@ -469,6 +501,8 @@ async def agenerate_provider_response(
 
         valid_litellm_params = self._remove_unnecessary_not_supported_params(kwargs)
         all_kwargs = {**self._completion_kwargs, **valid_litellm_params}
+        all_kwargs.setdefault("timeout", _default_timeout())
+        all_kwargs.setdefault("num_retries", DEFAULT_NUM_RETRIES)
         # See sync `generate_provider_response` for why the merged
         # dict needs its own conflict-resolution pass.
         all_kwargs = self._resolve_provider_conflicts(all_kwargs)

sdks/python/src/opik/evaluation/models/litellm/response_parser.py

@@ -50,9 +50,14 @@ def parse_assistant_message(
         content = _extract_response_format_arguments(message)
 
     if content is None and not tool_calls:
+        # An authenticated-but-degraded provider also lands here (empty
+        # tool_use on the structured-output path), so report the response
+        # shape that was actually rejected instead of only blaming the key.
         raise exceptions.BaseLLMError(
-            "Received None as the output from the LLM. Please verify your environment "
-            "configuration and ensure that the API keys for the models in use "
+            "LLM returned no content and no tool calls "
+            f"(model={getattr(response, 'model', None)!r}, "
+            f"finish_reason={_finish_reason(response)!r}). "
+            "If this persists, verify the API keys for the models in use "
             "(e.g., OPENAI_API_KEY) are set correctly."
         )
 
@@ -64,6 +69,13 @@ def parse_assistant_message(
     return assistant
 
 
+def _finish_reason(response: "ModelResponse") -> Optional[str]:
+    choices = getattr(response, "choices", None)
+    if not isinstance(choices, list) or not choices:
+        return None
+    return _get_str(_as_mapping(util.normalise_choice(choices[0])), "finish_reason")
+
+
 def _normalise_message(response: "ModelResponse") -> Dict[str, Any]:
     choices = getattr(response, "choices", None)
     if not isinstance(choices, list) or not choices:

sdks/python/src/opik/evaluation/suite_evaluators/llm_judge/metric.py

@@ -13,7 +13,7 @@
 
 from opik.evaluation.models import base_model, models_factory
 from opik.evaluation.metrics import score_result
-from opik.exceptions import LLMJudgeParseError
+from opik.exceptions import BaseLLMError, LLMJudgeParseError
 
 from opik.evaluation.suite_evaluators import base
 from . import config as llm_judge_config
@@ -22,8 +22,13 @@
 
 LOGGER = logging.getLogger(__name__)
 
+# `BaseLLMError` covers the provider returning a structurally empty response
+# (no content and no tool_calls) — seen in practice when a provider is briefly
+# degraded and the structured-output tool_use call comes back blank. It is
+# transient in the same way a malformed parse is, so it earns the same retry
+# rather than costing the item its whole score.
 _RETRY_POLICY = tenacity.retry(
-    retry=tenacity.retry_if_exception_type(LLMJudgeParseError),
+    retry=tenacity.retry_if_exception_type((LLMJudgeParseError, BaseLLMError)),
     stop=tenacity.stop_after_attempt(3),
     before_sleep=tenacity.before_sleep_log(LOGGER, logging.WARNING),
     reraise=True,

sdks/python/tests/unit/evaluation/models/test_litellm_chat_model.py

@@ -876,3 +876,64 @@ def decorator(func):
 
     # Verify that track_completion decorator was applied the expected number of times
     assert decorator_calls == expected_calls
+
+
+def test_litellm_chat_model_bounds_request_by_default(monkeypatch):
+    """A stalled provider must not block forever.
+
+    litellm's own defaults are a 6000s timeout and internal retries, so an
+    unbounded call multiplied across the retry layers can outlive any caller.
+    Observed stalls sit in TCP connect, hence the separate connect ceiling.
+    """
+    stub = _install_litellm_stub(monkeypatch)
+
+    model = litellm_chat_model.LiteLLMChatModel(model_name="openai/gpt-4o-mini")
+    model.generate_string("hello")
+
+    assert stub._calls, "Expected completion to be invoked"
+    _, _, kwargs = stub._calls[-1]
+    assert (
+        kwargs["timeout"].connect == litellm_chat_model.DEFAULT_CONNECT_TIMEOUT_SECONDS
+    )
+    assert kwargs["timeout"].read == litellm_chat_model.DEFAULT_READ_TIMEOUT_SECONDS
+    assert kwargs["num_retries"] == litellm_chat_model.DEFAULT_NUM_RETRIES
+
+
+def test_litellm_chat_model_explicit_timeout_overrides_default(monkeypatch):
+    stub = _install_litellm_stub(monkeypatch)
+
+    model = litellm_chat_model.LiteLLMChatModel(
+        model_name="openai/gpt-4o-mini",
+        timeout=5.0,
+        num_retries=2,
+    )
+    model.generate_string("hello")
+
+    _, _, kwargs = stub._calls[-1]
+    assert kwargs["timeout"] == 5.0
+    assert kwargs["num_retries"] == 2
+
+
+@pytest.mark.asyncio
+async def test_litellm_chat_model_bounds_async_request_by_default(monkeypatch):
+    stub = _install_litellm_stub(monkeypatch)
+
+    captured = {}
+
+    async def acompletion(model, messages, **kwargs):
+        captured.update(kwargs)
+        return SimpleNamespace(
+            choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))]
+        )
+
+    stub.acompletion = acompletion
+
+    model = litellm_chat_model.LiteLLMChatModel(model_name="openai/gpt-4o-mini")
+    await model.agenerate_string("hello")
+
+    assert (
+        captured["timeout"].connect
+        == litellm_chat_model.DEFAULT_CONNECT_TIMEOUT_SECONDS
+    )
+    assert captured["timeout"].read == litellm_chat_model.DEFAULT_READ_TIMEOUT_SECONDS
+    assert captured["num_retries"] == litellm_chat_model.DEFAULT_NUM_RETRIES

sdks/python/tests/unit/evaluation/suite_evaluators/test_llm_judge.py

@@ -289,3 +289,46 @@ def test_from_config__config_model_params__temperature_and_seed_preserved(self):
             "seed": 123,
             "customParameters": {"reasoning_effort": "low"},
         }
+
+
+class TestLLMJudgeRetries:
+    """A transient provider hiccup must not cost the whole item's score.
+
+    An empty structured-output response surfaces as ``BaseLLMError`` from the
+    response parser, which is a different exception type than the parse errors
+    the retry policy was originally written for.
+    """
+
+    def _judge_with_model(self, model):
+        evaluator = llm_judge.LLMJudge(
+            assertions=["The response contains the literal text PASS."],
+            track=False,
+        )
+        evaluator._model = model
+        return evaluator
+
+    def test_score__transient_empty_llm_response__retries_and_succeeds(self):
+        from opik import exceptions
+
+        calls = {"n": 0}
+        valid = (
+            '{"assertion_1": {"score": true, "reason": "contains PASS", '
+            '"confidence": 1.0}}'
+        )
+
+        class FlakyModel:
+            def generate_chat_completion(self, messages, response_format=None):
+                calls["n"] += 1
+                if calls["n"] == 1:
+                    raise exceptions.BaseLLMError(
+                        "LLM infrastructure error: Received None as the output "
+                        "from the LLM."
+                    )
+                return {"role": "assistant", "content": valid}
+
+        evaluator = self._judge_with_model(FlakyModel())
+        results = evaluator.score(input="q", output="PASS")
+
+        assert calls["n"] == 2, "the empty first response should have been retried"
+        assert len(results) == 1
+        assert results[0].value == 1.0