fix(llm): add static provider capability lists and reasoning_effort fallback tests

Signed-off-by: Hasnaat Hussain <hasnaat.hussain.2@gmail.com>

Author: Hasnaathussain

lib/crewai/src/crewai/llm.py

@@ -165,6 +165,42 @@ def _ensure_litellm() -> bool:
 MAX_CONTEXT: Final[int] = 2097152  # Current max from gemini-1.5-pro
 ANTHROPIC_PREFIXES: Final[tuple[str, str, str]] = ("anthropic/", "claude-", "claude/")
 
+# Static provider capability lists.  These are checked *before* LiteLLM
+# introspection so that well-known providers always receive correct capability
+# detection even when LiteLLM's model-mapping table is unavailable or raises.
+RESPONSE_FORMAT_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset(
+    {
+        "openai",
+        "azure",
+        "azure_openai",
+        "google",
+        "gemini",
+        "anthropic",
+        "bedrock",
+        "deepseek",
+        "openrouter",
+        "mistral",
+        "groq",
+        "fireworks_ai",
+        "together_ai",
+        "anyscale",
+        "cerebras",
+        "dashscope",
+    }
+)
+
+REASONING_EFFORT_SUPPORTED_PROVIDERS: Final[frozenset[str]] = frozenset(
+    {
+        "openai",
+        "azure",
+        "azure_openai",
+        "anthropic",
+        "bedrock",
+        "deepseek",
+        "openrouter",
+    }
+)
+
 LLM_CONTEXT_WINDOW_SIZES: Final[dict[str, int]] = {
     "gpt-4": 8192,
     "gpt-4o": 128000,
@@ -2382,29 +2418,47 @@ def _get_custom_llm_provider(self) -> str | None:
     def _validate_call_params(self) -> None:
         """Validate call parameters before executing a completion request.
 
-        Checks whether the requested ``response_format`` is supported by the
-        target model/provider via LiteLLM introspection.  When introspection
-        raises an exception (e.g. an unmapped custom model ID or proxy
-        endpoint), the failure is logged at DEBUG level and validation is
-        skipped so the request can proceed — this mirrors the permissive
-        fallback used in :meth:`supports_function_calling`.
+        Performs two kinds of checks in order:
+
+        1. **Static capability checks** (always run): ``response_format`` and
+           ``reasoning_effort`` are tested against
+           :data:`RESPONSE_FORMAT_SUPPORTED_PROVIDERS` and
+           :data:`REASONING_EFFORT_SUPPORTED_PROVIDERS` respectively.  These
+           module-level frozensets encode well-known provider support and run
+           unconditionally — regardless of whether LiteLLM is available.
 
-        When no ``response_format`` is configured (including when only
-        ``reasoning_effort`` or other parameters are set) this method returns
-        immediately without performing any checks.
+        2. **LiteLLM introspection** (best-effort): When the provider is *not*
+           in the static lists, ``supports_response_schema`` is called for
+           additional coverage.  If introspection raises (e.g. an unmapped
+           custom model ID or proxy endpoint), the failure is logged at DEBUG
+           level and validation is skipped — permitting the request to proceed.
 
         Note: This validation only applies to the litellm fallback path.
         Native providers perform their own parameter validation.
         """
+        provider = self._get_custom_llm_provider() or "openai"
+
+        # --- Static reasoning_effort check (always runs) ---
+        if self.reasoning_effort is not None:
+            if provider not in REASONING_EFFORT_SUPPORTED_PROVIDERS:
+                logger.debug(
+                    f"Provider '{provider}' does not appear in the static reasoning_effort "
+                    "support list; the parameter will be forwarded and may be ignored."
+                )
+
+        # --- response_format checks ---
         if self.response_format is None:
             return
 
+        # 1. Static check: known-supported providers skip LiteLLM introspection.
+        if provider in RESPONSE_FORMAT_SUPPORTED_PROVIDERS:
+            return
+
+        # 2. LiteLLM introspection for providers not in the static list.
         if not _ensure_litellm() or supports_response_schema is None:
-            # When litellm is not available, skip validation
-            # (this path should only be reached for litellm fallback models)
+            # LiteLLM unavailable; static list was already consulted — allow.
             return
 
-        provider = self._get_custom_llm_provider()
         try:
             is_supported = supports_response_schema(
                 model=self.model,

lib/crewai/tests/test_llm.py

@@ -233,10 +233,12 @@ def test_validate_call_params_not_supported():
     class DummyResponse(BaseModel):
         a: int
 
-    # Patch supports_response_schema to simulate an unsupported model.
+    # Use an unknown provider (not in RESPONSE_FORMAT_SUPPORTED_PROVIDERS) so the
+    # LiteLLM introspection path is exercised and supports_response_schema can
+    # report False, which should raise ValueError.
     with patch("crewai.llm.supports_response_schema", return_value=False):
         llm = LLM(
-            model="gemini/gemini-1.5-pro",
+            model="unknown-llm-provider/some-model",
             response_format=DummyResponse,
             is_litellm=True,
         )
@@ -1211,19 +1213,24 @@ async def _ret(*args, **kwargs):
 
 
 def test_validate_call_params_handles_introspection_error():
-    """Verify that _validate_call_params gracefully handles LiteLLM introspection errors without crashing."""
+    """Verify _validate_call_params gracefully handles LiteLLM introspection errors without crashing.
+
+    When the provider is not in the static list, introspection is attempted.
+    An exception during introspection must be caught and treated as supported
+    so the request can proceed with custom/proxy models.
+    """
     llm = LLM(model="custom/unsupported-model", is_litellm=True, response_format={"type": "json_object"})
 
     with patch("crewai.llm.supports_response_schema", side_effect=Exception("Introspection error")):
-        # Should not raise exception — permissive fallback applies
+        # Should not raise exception -- permissive fallback applies
         llm._validate_call_params()
 
 
 def test_validate_call_params_raises_for_confirmed_unsupported_model():
     """Verify _validate_call_params still raises ValueError when introspection confirms no support.
 
-    This ensures the validation path is not silently skipped — only introspection
-    *errors* are swallowed; a clear ``False`` return still produces a ValueError.
+    This ensures the validation path is not silently skipped -- only introspection
+    errors are swallowed; a clear False return still produces a ValueError.
     """
     llm = LLM(model="my-provider/my-model", is_litellm=True, response_format={"type": "json_object"})
 
@@ -1233,17 +1240,44 @@ def test_validate_call_params_raises_for_confirmed_unsupported_model():
 
 
 def test_validate_call_params_skips_check_when_no_response_format():
-    """Verify _validate_call_params is a no-op when response_format is not set.
+    """Verify _validate_call_params is a no-op for params that require no validation.
+
+    When no response_format is set, the method must return without ever
+    calling the LiteLLM introspection function.
+    """
+    llm = LLM(model="gpt-4o", is_litellm=True, temperature=0.5)
+
+    with patch("crewai.llm.supports_response_schema", side_effect=Exception("Should not be called")):
+        # No exception -- response_format is None so introspection is never invoked
+        llm._validate_call_params()
+
 
-    This covers scenarios where only ``reasoning_effort`` or other parameters
-    are configured: the method must return immediately so those parameters are
-    forwarded to the provider without interference.
+def test_validate_call_params_reasoning_effort_supported_provider():
+    """Verify _validate_call_params passes without error for reasoning_effort on a supported provider.
+
+    openai is in REASONING_EFFORT_SUPPORTED_PROVIDERS, so no warning or
+    error should be raised and introspection must not be triggered.
     """
-    llm = LLM(model="gpt-4o", is_litellm=True, reasoning_effort="medium")
+    llm = LLM(model="openai/o3-mini", is_litellm=True, reasoning_effort="medium")
 
-    # If supports_response_schema were called, patching it to raise would surface the bug
     with patch("crewai.llm.supports_response_schema", side_effect=Exception("Should not be called")):
-        # No exception — response_format is None so introspection is never invoked
+        # No response_format set -- introspection is never invoked
+        llm._validate_call_params()
+
+
+def test_validate_call_params_reasoning_effort_unknown_provider_logs_debug(caplog):
+    """Verify _validate_call_params logs a debug message for unknown providers with reasoning_effort.
+
+    Unknown providers are not rejected outright: parameters are forwarded and
+    may be silently ignored by the underlying provider. The debug log signals
+    this to developers without breaking the call.
+    """
+    import logging
+
+    llm = LLM(model="unknown-provider/some-model", is_litellm=True, reasoning_effort="high")
+
+    with caplog.at_level(logging.DEBUG, logger="crewai.llm"):
         llm._validate_call_params()
 
+    assert any("reasoning_effort" in record.message for record in caplog.records)