scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Support Anthropic web search tool + Add more google finish reason mapping (#10785)

Browse source 
* fix(vertex_and_google_ai_studio_gemini.py): handle additional google finish reasons

Fixes https://github.com/BerriAI/litellm/issues/10768

* test: add more unit tests

* fix(anthropic/chat/transformation.py): support anthropic web search

Fixes https://github.com/BerriAI/litellm/issues/10664

* fix(anthropic/chat/transformation.py): add anthropic web search 'max uses' param support

* docs(anthropic.md): add doc for web search tool calling

Closes https://github.com/BerriAI/litellm/issues/10664

* build(model_prices_and_context_window.json): add search tool pricing for anthropic

* fix: suppress linting error

* test: update tests

* fix: fix ruff check
This commit is contained in:
Krish Dholakia 2025-05-12 22:45:51 -07:00 • committed by GitHub
parent fea3966d8e
commit 53f6514f12
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 355 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
docs/my-website/docs/providers/anthropic.md
View file

@ -750,7 +750,11 @@ except Exception as e:
s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this!
### Computer Tools
### Anthropic Hosted Tools (Computer, Text Editor, Web Search)
<Tabs>
<TabItem value="computer" label="Computer">
```python
from litellm import completion
@ -781,6 +785,131 @@ resp = completion(
print(resp)
```
</TabItem>
<TabItem value="text_editor" label="Text Editor">
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
tools = [{
"type": "text_editor_20250124",
"name": "str_replace_editor"
}]
model = "claude-3-5-sonnet-20241022"
messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}]
resp = completion(
model=model,
messages=messages,
tools=tools,
)
print(resp)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "claude-3-5-sonnet-latest",
"messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}],
"tools": [{"type": "text_editor_20250124", "name": "str_replace_editor"}]
}'
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="web_search" label="Web Search">
:::info
Unified web search (same param across OpenAI + Anthropic) coming soon!
:::
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
tools = [{
"type": "web_search_20250305",
"name": "web_search",
"max_uses": 5
}]
model = "claude-3-5-sonnet-20241022"
messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}]
resp = completion(
model=model,
messages=messages,
tools=tools,
)
print(resp)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
- model_name: claude-3-5-sonnet-latest
litellm_params:
model: anthropic/claude-3-5-sonnet-latest
api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "claude-3-5-sonnet-latest",
"messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}],
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}]
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
## Usage - Vision
```python

19
litellm/llms/anthropic/chat/transformation.py
View file

@ -58,6 +58,9 @@ else:
LoggingClass = Any
ANTHROPIC_HOSTED_TOOLS = ["web_search", "bash", "text_editor"]
class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
Reference: https://docs.anthropic.com/claude/reference/messages_post
@ -212,16 +215,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_computer_tool["display_number"] = _display_number
returned_tool = _computer_tool
elif tool["type"].startswith("bash_") or tool["type"].startswith(
"text_editor_"
):
function_name = tool["function"].get("name")
if function_name is None:
elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS):
function_name = tool.get("name", tool.get("function", {}).get("name"))
if function_name is None or not isinstance(function_name, str):
raise ValueError("Missing required parameter: name")
additional_tool_params = {}
for k, v in tool.items():
if k != "type" and k != "name":
additional_tool_params[k] = v
returned_tool = AnthropicHostedTools(
type=tool["type"],
name=function_name,
type=tool["type"], name=function_name, **additional_tool_params # type: ignore
)
if returned_tool is None:
raise ValueError(f"Unsupported tool type: {tool['type']}")

41
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -29,7 +29,6 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -571,6 +570,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"BLOCKLIST": "The token generation was stopped as the response was flagged for the terms which are included from the terminology blocklist.",
"PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for the prohibited contents.",
"SPII": "The token generation was stopped as the response was flagged for Sensitive Personally Identifiable Information (SPII) contents.",
"IMAGE_SAFETY": "The token generation was stopped as the response was flagged for image safety reasons.",
}
def get_finish_reason_mapping(self) -> Dict[str, OpenAIChatCompletionFinishReason]:
"""
Return Dictionary of finish reasons which indicate response was flagged
and what it means
"""
return {
"FINISH_REASON_UNSPECIFIED": "stop", # openai doesn't have a way of representing this
"STOP": "stop",
"MAX_TOKENS": "length",
"SAFETY": "content_filter",
"RECITATION": "content_filter",
"LANGUAGE": "content_filter",
"OTHER": "content_filter",
"BLOCKLIST": "content_filter",
"PROHIBITED_CONTENT": "content_filter",
"SPII": "content_filter",
"MALFORMED_FUNCTION_CALL": "stop", # openai doesn't have a way of representing this
"IMAGE_SAFETY": "content_filter",
}
def translate_exception_str(self, exception_string: str):
@ -820,17 +841,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
def _check_finish_reason(
self,
chat_completion_message: ChatCompletionResponseMessage,
chat_completion_message: Optional[ChatCompletionResponseMessage],
finish_reason: Optional[str],
) -> OpenAIChatCompletionFinishReason:
if chat_completion_message.get("function_call"):
mapped_finish_reason = self.get_finish_reason_mapping()
if chat_completion_message and chat_completion_message.get("function_call"):
return "function_call"
elif chat_completion_message.get("tool_calls"):
elif chat_completion_message and chat_completion_message.get("tool_calls"):
return "tool_calls"
elif finish_reason and (
finish_reason == "SAFETY" or finish_reason == "RECITATION"
elif (
finish_reason and finish_reason in mapped_finish_reason.keys()
): # vertex ai
return "content_filter"
return mapped_finish_reason[finish_reason]
else:
return "stop"
@ -1586,8 +1608,9 @@ class ModelResponseIterator:
)
if gemini_chunk and "finishReason" in gemini_chunk:
finish_reason = map_finish_reason(
finish_reason=gemini_chunk["finishReason"]
finish_reason = VertexGeminiConfig()._check_finish_reason(
chat_completion_message=None,
finish_reason=gemini_chunk["finishReason"],
)
## DO NOT SET 'is_finished' = True
## GEMINI SETS FINISHREASON ON EVERY CHUNK!

45
litellm/model_prices_and_context_window_backup.json
View file

@ -4390,6 +4390,11 @@
"output_cost_per_token": 0.000004,
"cache_creation_input_token_cost": 0.000001,
"cache_read_input_token_cost": 0.00000008,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4400,7 +4405,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-10-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"claude-3-5-haiku-latest": {
"max_tokens": 8192,
@ -4410,6 +4416,11 @@
"output_cost_per_token": 0.000005,
"cache_creation_input_token_cost": 0.00000125,
"cache_read_input_token_cost": 0.0000001,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4420,7 +4431,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-10-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"claude-3-opus-latest": {
"max_tokens": 4096,
@ -4485,6 +4497,11 @@
"output_cost_per_token": 0.000015,
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4495,7 +4512,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-06-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"claude-3-5-sonnet-20240620": {
"max_tokens": 8192,
@ -4523,6 +4541,11 @@
"max_output_tokens": 128000,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000015,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"litellm_provider": "anthropic",
@ -4546,6 +4569,11 @@
"output_cost_per_token": 0.000015,
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4557,7 +4585,8 @@
"supports_response_schema": true,
"deprecation_date": "2026-02-01",
"supports_tool_choice": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_web_search": true
},
"claude-3-5-sonnet-20241022": {
"max_tokens": 8192,
@ -4567,6 +4596,11 @@
"output_cost_per_token": 0.000015,
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4577,7 +4611,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-10-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"text-bison": {
"max_tokens": 2048,

45
model_prices_and_context_window.json
View file

@ -4390,6 +4390,11 @@
"output_cost_per_token": 0.000004,
"cache_creation_input_token_cost": 0.000001,
"cache_read_input_token_cost": 0.00000008,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4400,7 +4405,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-10-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"claude-3-5-haiku-latest": {
"max_tokens": 8192,
@ -4410,6 +4416,11 @@
"output_cost_per_token": 0.000005,
"cache_creation_input_token_cost": 0.00000125,
"cache_read_input_token_cost": 0.0000001,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4420,7 +4431,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-10-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"claude-3-opus-latest": {
"max_tokens": 4096,
@ -4485,6 +4497,11 @@
"output_cost_per_token": 0.000015,
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4495,7 +4512,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-06-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"claude-3-5-sonnet-20240620": {
"max_tokens": 8192,
@ -4523,6 +4541,11 @@
"max_output_tokens": 128000,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000015,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"litellm_provider": "anthropic",
@ -4546,6 +4569,11 @@
"output_cost_per_token": 0.000015,
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4557,7 +4585,8 @@
"supports_response_schema": true,
"deprecation_date": "2026-02-01",
"supports_tool_choice": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_web_search": true
},
"claude-3-5-sonnet-20241022": {
"max_tokens": 8192,
@ -4567,6 +4596,11 @@
"output_cost_per_token": 0.000015,
"cache_creation_input_token_cost": 0.00000375,
"cache_read_input_token_cost": 0.0000003,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
@ -4577,7 +4611,8 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"deprecation_date": "2025-10-01",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_web_search": true
},
"text-bison": {
"max_tokens": 2048,

11
tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -111,3 +111,14 @@ def test_extract_response_content_with_citations():
_, citations, _, _, _ = config.extract_response_content(completion_response)
assert citations is not None
def test_map_tool_helper():
config = AnthropicConfig()
tool = {"type": "web_search_20250305", "name": "web_search", "max_uses": 5}
result = config._map_tool_helper(tool)
assert result is not None
assert result["name"] == "web_search"
assert result["max_uses"] == 5

12
tests/litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -1,6 +1,6 @@
import asyncio
from typing import List, cast
from copy import deepcopy
from typing import List, cast
from unittest.mock import MagicMock
import pytest
@ -315,3 +315,13 @@ def test_vertex_ai_candidate_token_count_inclusive(
assert usage.prompt_tokens == expected_usage.prompt_tokens
assert usage.completion_tokens == expected_usage.completion_tokens
assert usage.total_tokens == expected_usage.total_tokens
def test_check_finish_reason():
config = VertexGeminiConfig()
finish_reason_mappings = config.get_finish_reason_mapping()
for k, v in finish_reason_mappings.items():
assert (
config._check_finish_reason(chat_completion_message=None, finish_reason=k)
== v
)

24
tests/litellm/test_utils.py
View file

@ -395,3 +395,27 @@ def test_all_model_configs():
optional_params={},
drop_params=False,
) == {"max_output_tokens": 10}
def test_anthropic_web_search_in_model_info():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
supported_models = [
"anthropic/claude-3-7-sonnet-20250219",
"anthropic/claude-3-5-sonnet-latest",
"anthropic/claude-3-5-sonnet-20241022",
"anthropic/claude-3-5-haiku-20241022",
"anthropic/claude-3-5-haiku-latest",
]
for model in supported_models:
from litellm.utils import get_model_info
model_info = get_model_info(model)
assert model_info is not None
assert (
model_info["supports_web_search"] is True
), f"Model {model} should support web search"
assert (
model_info["search_context_cost_per_query"] is not None
), f"Model {model} should have a search context cost per query"

44
tests/llm_translation/test_anthropic_completion.py
View file

@ -1170,3 +1170,47 @@ def test_just_system_message():
response = litellm.completion(**params)
assert response is not None
def test_anthropic_websearch():
litellm._turn_on_debug()
params = {
"model": "anthropic/claude-3-5-sonnet-latest",
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"tools": [{
"type": "web_search_20250305",
"name": "web_search",
"max_uses": 5
}]
}
try:
response = litellm.completion(**params)
except litellm.InternalServerError as e:
print(e)
assert response is not None
def test_anthropic_text_editor():
litellm._turn_on_debug()
params = {
"model": "anthropic/claude-3-5-sonnet-latest",
"messages": [
{
"role": "user",
"content": "There'\''s a syntax error in my primes.py file. Can you help me fix it?"
}
],
"tools": [{
"type": "text_editor_20250124",
"name": "str_replace_editor"
}]
}
try:
response = litellm.completion(**params)
except litellm.InternalServerError as e:
print(e)
assert response is not None

12
tests/llm_translation/test_gemini.py
View file

@ -134,4 +134,14 @@ def test_gemini_thinking_budget_0():
}
)
print(raw_request)
assert "0" in json.dumps(raw_request["raw_request_body"])
assert "0" in json.dumps(raw_request["raw_request_body"])
def test_gemini_finish_reason():
import os
from litellm import completion
litellm._turn_on_debug()
response = completion(model="gemini/gemini-1.5-pro", messages=[{"role": "user", "content": "give me 3 random words"}], max_tokens=2)
print(response)
assert response.choices[0].finish_reason is not None
assert response.choices[0].finish_reason == "length"