Skip to content

Commit 53f6514

Browse files
krrishdholakiakrrishdholakia
authored
Support Anthropic web search tool + Add more google finish reason mapping (#10785)
* fix(vertex_and_google_ai_studio_gemini.py): handle additional google finish reasons Fixes #10768 * test: add more unit tests * fix(anthropic/chat/transformation.py): support anthropic web search Fixes #10664 * fix(anthropic/chat/transformation.py): add anthropic web search 'max uses' param support * docs(anthropic.md): add doc for web search tool calling Closes #10664 * build(model_prices_and_context_window.json): add search tool pricing for anthropic * fix: suppress linting error * test: update tests * fix: fix ruff check
1 parent fea3966 commit 53f6514

File tree

10 files changed

+355
-29
lines changed

10 files changed

+355
-29
lines changed

‎docs/my-website/docs/providers/anthropic.md

+130-1
Original file line numberDiff line numberDiff line change
@@ -750,7 +750,11 @@ except Exception as e:
750750

751751
s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this!
752752

753-
### Computer Tools
753+
### Anthropic Hosted Tools (Computer, Text Editor, Web Search)
754+
755+
756+
<Tabs>
757+
<TabItem value="computer" label="Computer">
754758

755759
```python
756760
from litellm import completion
@@ -781,6 +785,131 @@ resp = completion(
781785
print(resp)
782786
```
783787

788+
</TabItem>
789+
<TabItem value="text_editor" label="Text Editor">
790+
791+
<Tabs>
792+
<TabItem value="sdk" label="SDK">
793+
794+
```python
795+
from litellm import completion
796+
797+
tools = [{
798+
"type": "text_editor_20250124",
799+
"name": "str_replace_editor"
800+
}]
801+
model = "claude-3-5-sonnet-20241022"
802+
messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}]
803+
804+
resp = completion(
805+
model=model,
806+
messages=messages,
807+
tools=tools,
808+
)
809+
810+
print(resp)
811+
```
812+
813+
</TabItem>
814+
<TabItem value="proxy" label="PROXY">
815+
816+
1. Setup config.yaml
817+
818+
```yaml
819+
- model_name: claude-3-5-sonnet-latest
820+
litellm_params:
821+
model: anthropic/claude-3-5-sonnet-latest
822+
api_key: os.environ/ANTHROPIC_API_KEY
823+
```
824+
825+
2. Start proxy
826+
827+
```bash
828+
litellm --config /path/to/config.yaml
829+
```
830+
831+
3. Test it!
832+
833+
```bash
834+
curl http://0.0.0.0:4000/v1/chat/completions \
835+
-H "Content-Type: application/json" \
836+
-H "Authorization: Bearer $LITELLM_KEY" \
837+
-d '{
838+
"model": "claude-3-5-sonnet-latest",
839+
"messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}],
840+
"tools": [{"type": "text_editor_20250124", "name": "str_replace_editor"}]
841+
}'
842+
```
843+
</TabItem>
844+
</Tabs>
845+
846+
</TabItem>
847+
<TabItem value="web_search" label="Web Search">
848+
849+
:::info
850+
851+
Unified web search (same param across OpenAI + Anthropic) coming soon!
852+
:::
853+
854+
<Tabs>
855+
<TabItem value="sdk" label="SDK">
856+
857+
```python
858+
from litellm import completion
859+
860+
tools = [{
861+
"type": "web_search_20250305",
862+
"name": "web_search",
863+
"max_uses": 5
864+
}]
865+
model = "claude-3-5-sonnet-20241022"
866+
messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}]
867+
868+
resp = completion(
869+
model=model,
870+
messages=messages,
871+
tools=tools,
872+
)
873+
874+
print(resp)
875+
```
876+
877+
</TabItem>
878+
<TabItem value="proxy" label="PROXY">
879+
880+
1. Setup config.yaml
881+
882+
```yaml
883+
- model_name: claude-3-5-sonnet-latest
884+
litellm_params:
885+
model: anthropic/claude-3-5-sonnet-latest
886+
api_key: os.environ/ANTHROPIC_API_KEY
887+
```
888+
889+
2. Start proxy
890+
891+
```bash
892+
litellm --config /path/to/config.yaml
893+
```
894+
895+
3. Test it!
896+
897+
```bash
898+
curl http://0.0.0.0:4000/v1/chat/completions \
899+
-H "Content-Type: application/json" \
900+
-H "Authorization: Bearer $LITELLM_KEY" \
901+
-d '{
902+
"model": "claude-3-5-sonnet-latest",
903+
"messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}],
904+
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}]
905+
}'
906+
```
907+
</TabItem>
908+
</Tabs>
909+
910+
</TabItem>
911+
</Tabs>
912+
784913
## Usage - Vision
785914
786915
```python

‎litellm/llms/anthropic/chat/transformation.py

+12-7
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@
5858
LoggingClass = Any
5959

6060

61+
ANTHROPIC_HOSTED_TOOLS = ["web_search", "bash", "text_editor"]
62+
63+
6164
class AnthropicConfig(AnthropicModelInfo, BaseConfig):
6265
"""
6366
Reference: https://docs.anthropic.com/claude/reference/messages_post
@@ -212,16 +215,18 @@ def _map_tool_helper(
212215
_computer_tool["display_number"] = _display_number
213216

214217
returned_tool = _computer_tool
215-
elif tool["type"].startswith("bash_") or tool["type"].startswith(
216-
"text_editor_"
217-
):
218-
function_name = tool["function"].get("name")
219-
if function_name is None:
218+
elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS):
219+
function_name = tool.get("name", tool.get("function", {}).get("name"))
220+
if function_name is None or not isinstance(function_name, str):
220221
raise ValueError("Missing required parameter: name")
221222

223+
additional_tool_params = {}
224+
for k, v in tool.items():
225+
if k != "type" and k != "name":
226+
additional_tool_params[k] = v
227+
222228
returned_tool = AnthropicHostedTools(
223-
type=tool["type"],
224-
name=function_name,
229+
type=tool["type"], name=function_name, **additional_tool_params # type: ignore
225230
)
226231
if returned_tool is None:
227232
raise ValueError(f"Unsupported tool type: {tool['type']}")

‎litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

+32-9
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
3030
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
3131
)
32-
from litellm.litellm_core_utils.core_helpers import map_finish_reason
3332
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
3433
from litellm.llms.custom_httpx.http_handler import (
3534
AsyncHTTPHandler,
@@ -571,6 +570,28 @@ def get_flagged_finish_reasons(self) -> Dict[str, str]:
571570
"BLOCKLIST": "The token generation was stopped as the response was flagged for the terms which are included from the terminology blocklist.",
572571
"PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for the prohibited contents.",
573572
"SPII": "The token generation was stopped as the response was flagged for Sensitive Personally Identifiable Information (SPII) contents.",
573+
"IMAGE_SAFETY": "The token generation was stopped as the response was flagged for image safety reasons.",
574+
}
575+
576+
def get_finish_reason_mapping(self) -> Dict[str, OpenAIChatCompletionFinishReason]:
577+
"""
578+
Return Dictionary of finish reasons which indicate response was flagged
579+
580+
and what it means
581+
"""
582+
return {
583+
"FINISH_REASON_UNSPECIFIED": "stop", # openai doesn't have a way of representing this
584+
"STOP": "stop",
585+
"MAX_TOKENS": "length",
586+
"SAFETY": "content_filter",
587+
"RECITATION": "content_filter",
588+
"LANGUAGE": "content_filter",
589+
"OTHER": "content_filter",
590+
"BLOCKLIST": "content_filter",
591+
"PROHIBITED_CONTENT": "content_filter",
592+
"SPII": "content_filter",
593+
"MALFORMED_FUNCTION_CALL": "stop", # openai doesn't have a way of representing this
594+
"IMAGE_SAFETY": "content_filter",
574595
}
575596

576597
def translate_exception_str(self, exception_string: str):
@@ -820,17 +841,18 @@ def _calculate_usage(
820841

821842
def _check_finish_reason(
822843
self,
823-
chat_completion_message: ChatCompletionResponseMessage,
844+
chat_completion_message: Optional[ChatCompletionResponseMessage],
824845
finish_reason: Optional[str],
825846
) -> OpenAIChatCompletionFinishReason:
826-
if chat_completion_message.get("function_call"):
847+
mapped_finish_reason = self.get_finish_reason_mapping()
848+
if chat_completion_message and chat_completion_message.get("function_call"):
827849
return "function_call"
828-
elif chat_completion_message.get("tool_calls"):
850+
elif chat_completion_message and chat_completion_message.get("tool_calls"):
829851
return "tool_calls"
830-
elif finish_reason and (
831-
finish_reason == "SAFETY" or finish_reason == "RECITATION"
852+
elif (
853+
finish_reason and finish_reason in mapped_finish_reason.keys()
832854
): # vertex ai
833-
return "content_filter"
855+
return mapped_finish_reason[finish_reason]
834856
else:
835857
return "stop"
836858

@@ -1586,8 +1608,9 @@ def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
15861608
)
15871609

15881610
if gemini_chunk and "finishReason" in gemini_chunk:
1589-
finish_reason = map_finish_reason(
1590-
finish_reason=gemini_chunk["finishReason"]
1611+
finish_reason = VertexGeminiConfig()._check_finish_reason(
1612+
chat_completion_message=None,
1613+
finish_reason=gemini_chunk["finishReason"],
15911614
)
15921615
## DO NOT SET 'is_finished' = True
15931616
## GEMINI SETS FINISHREASON ON EVERY CHUNK!

‎litellm/model_prices_and_context_window_backup.json

+40-5
Original file line numberDiff line numberDiff line change
@@ -4390,6 +4390,11 @@
43904390
"output_cost_per_token": 0.000004,
43914391
"cache_creation_input_token_cost": 0.000001,
43924392
"cache_read_input_token_cost": 0.00000008,
4393+
"search_context_cost_per_query": {
4394+
"search_context_size_low": 1e-2,
4395+
"search_context_size_medium": 1e-2,
4396+
"search_context_size_high": 1e-2
4397+
},
43934398
"litellm_provider": "anthropic",
43944399
"mode": "chat",
43954400
"supports_function_calling": true,
@@ -4400,7 +4405,8 @@
44004405
"supports_prompt_caching": true,
44014406
"supports_response_schema": true,
44024407
"deprecation_date": "2025-10-01",
4403-
"supports_tool_choice": true
4408+
"supports_tool_choice": true,
4409+
"supports_web_search": true
44044410
},
44054411
"claude-3-5-haiku-latest": {
44064412
"max_tokens": 8192,
@@ -4410,6 +4416,11 @@
44104416
"output_cost_per_token": 0.000005,
44114417
"cache_creation_input_token_cost": 0.00000125,
44124418
"cache_read_input_token_cost": 0.0000001,
4419+
"search_context_cost_per_query": {
4420+
"search_context_size_low": 1e-2,
4421+
"search_context_size_medium": 1e-2,
4422+
"search_context_size_high": 1e-2
4423+
},
44134424
"litellm_provider": "anthropic",
44144425
"mode": "chat",
44154426
"supports_function_calling": true,
@@ -4420,7 +4431,8 @@
44204431
"supports_prompt_caching": true,
44214432
"supports_response_schema": true,
44224433
"deprecation_date": "2025-10-01",
4423-
"supports_tool_choice": true
4434+
"supports_tool_choice": true,
4435+
"supports_web_search": true
44244436
},
44254437
"claude-3-opus-latest": {
44264438
"max_tokens": 4096,
@@ -4485,6 +4497,11 @@
44854497
"output_cost_per_token": 0.000015,
44864498
"cache_creation_input_token_cost": 0.00000375,
44874499
"cache_read_input_token_cost": 0.0000003,
4500+
"search_context_cost_per_query": {
4501+
"search_context_size_low": 1e-2,
4502+
"search_context_size_medium": 1e-2,
4503+
"search_context_size_high": 1e-2
4504+
},
44884505
"litellm_provider": "anthropic",
44894506
"mode": "chat",
44904507
"supports_function_calling": true,
@@ -4495,7 +4512,8 @@
44954512
"supports_prompt_caching": true,
44964513
"supports_response_schema": true,
44974514
"deprecation_date": "2025-06-01",
4498-
"supports_tool_choice": true
4515+
"supports_tool_choice": true,
4516+
"supports_web_search": true
44994517
},
45004518
"claude-3-5-sonnet-20240620": {
45014519
"max_tokens": 8192,
@@ -4523,6 +4541,11 @@
45234541
"max_output_tokens": 128000,
45244542
"input_cost_per_token": 0.000003,
45254543
"output_cost_per_token": 0.000015,
4544+
"search_context_cost_per_query": {
4545+
"search_context_size_low": 1e-2,
4546+
"search_context_size_medium": 1e-2,
4547+
"search_context_size_high": 1e-2
4548+
},
45264549
"cache_creation_input_token_cost": 0.00000375,
45274550
"cache_read_input_token_cost": 0.0000003,
45284551
"litellm_provider": "anthropic",
@@ -4546,6 +4569,11 @@
45464569
"output_cost_per_token": 0.000015,
45474570
"cache_creation_input_token_cost": 0.00000375,
45484571
"cache_read_input_token_cost": 0.0000003,
4572+
"search_context_cost_per_query": {
4573+
"search_context_size_low": 1e-2,
4574+
"search_context_size_medium": 1e-2,
4575+
"search_context_size_high": 1e-2
4576+
},
45494577
"litellm_provider": "anthropic",
45504578
"mode": "chat",
45514579
"supports_function_calling": true,
@@ -4557,7 +4585,8 @@
45574585
"supports_response_schema": true,
45584586
"deprecation_date": "2026-02-01",
45594587
"supports_tool_choice": true,
4560-
"supports_reasoning": true
4588+
"supports_reasoning": true,
4589+
"supports_web_search": true
45614590
},
45624591
"claude-3-5-sonnet-20241022": {
45634592
"max_tokens": 8192,
@@ -4567,6 +4596,11 @@
45674596
"output_cost_per_token": 0.000015,
45684597
"cache_creation_input_token_cost": 0.00000375,
45694598
"cache_read_input_token_cost": 0.0000003,
4599+
"search_context_cost_per_query": {
4600+
"search_context_size_low": 1e-2,
4601+
"search_context_size_medium": 1e-2,
4602+
"search_context_size_high": 1e-2
4603+
},
45704604
"litellm_provider": "anthropic",
45714605
"mode": "chat",
45724606
"supports_function_calling": true,
@@ -4577,7 +4611,8 @@
45774611
"supports_prompt_caching": true,
45784612
"supports_response_schema": true,
45794613
"deprecation_date": "2025-10-01",
4580-
"supports_tool_choice": true
4614+
"supports_tool_choice": true,
4615+
"supports_web_search": true
45814616
},
45824617
"text-bison": {
45834618
"max_tokens": 2048,

0 commit comments

Comments
 (0)