diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 95323719f0a..8b5c800eaf2 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -750,7 +750,11 @@ except Exception as e: s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this! -### Computer Tools +### Anthropic Hosted Tools (Computer, Text Editor, Web Search) + + + + ```python from litellm import completion @@ -781,6 +785,131 @@ resp = completion( print(resp) ``` + + + + + + +```python +from litellm import completion + +tools = [{ + "type": "text_editor_20250124", + "name": "str_replace_editor" +}] +model = "claude-3-5-sonnet-20241022" +messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}] + +resp = completion( + model=model, + messages=messages, + tools=tools, +) + +print(resp) +``` + + + + +1. Setup config.yaml + +```yaml +- model_name: claude-3-5-sonnet-latest + litellm_params: + model: anthropic/claude-3-5-sonnet-latest + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "claude-3-5-sonnet-latest", + "messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}], + "tools": [{"type": "text_editor_20250124", "name": "str_replace_editor"}] + }' +``` + + + + + + +:::info + +Unified web search (same param across OpenAI + Anthropic) coming soon! +::: + + + + +```python +from litellm import completion + +tools = [{ + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 +}] +model = "claude-3-5-sonnet-20241022" +messages = [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}] + +resp = completion( + model=model, + messages=messages, + tools=tools, +) + +print(resp) +``` + + + + +1. Setup config.yaml + +```yaml +- model_name: claude-3-5-sonnet-latest + litellm_params: + model: anthropic/claude-3-5-sonnet-latest + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "claude-3-5-sonnet-latest", + "messages": [{"role": "user", "content": "There's a syntax error in my primes.py file. Can you help me fix it?"}], + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] + }' +``` + + + + + + ## Usage - Vision ```python diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 06e0553f8d5..3cf62bc7f77 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -58,6 +58,9 @@ else: LoggingClass = Any +ANTHROPIC_HOSTED_TOOLS = ["web_search", "bash", "text_editor"] + + class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ Reference: https://docs.anthropic.com/claude/reference/messages_post @@ -212,16 +215,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _computer_tool["display_number"] = _display_number returned_tool = _computer_tool - elif tool["type"].startswith("bash_") or tool["type"].startswith( - "text_editor_" - ): - function_name = tool["function"].get("name") - if function_name is None: + elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS): + function_name = tool.get("name", tool.get("function", {}).get("name")) + if function_name is None or not isinstance(function_name, str): raise ValueError("Missing required parameter: name") + additional_tool_params = {} + for k, v in tool.items(): + if k != "type" and k != "name": + additional_tool_params[k] = v + returned_tool = AnthropicHostedTools( - type=tool["type"], - name=function_name, + type=tool["type"], name=function_name, **additional_tool_params # type: ignore ) if returned_tool is None: raise ValueError(f"Unsupported tool type: {tool['type']}") diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index e57015a22e5..caf591585b2 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -29,7 +29,6 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, ) -from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -571,6 +570,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "BLOCKLIST": "The token generation was stopped as the response was flagged for the terms which are included from the terminology blocklist.", "PROHIBITED_CONTENT": "The token generation was stopped as the response was flagged for the prohibited contents.", "SPII": "The token generation was stopped as the response was flagged for Sensitive Personally Identifiable Information (SPII) contents.", + "IMAGE_SAFETY": "The token generation was stopped as the response was flagged for image safety reasons.", + } + + def get_finish_reason_mapping(self) -> Dict[str, OpenAIChatCompletionFinishReason]: + """ + Return Dictionary of finish reasons which indicate response was flagged + + and what it means + """ + return { + "FINISH_REASON_UNSPECIFIED": "stop", # openai doesn't have a way of representing this + "STOP": "stop", + "MAX_TOKENS": "length", + "SAFETY": "content_filter", + "RECITATION": "content_filter", + "LANGUAGE": "content_filter", + "OTHER": "content_filter", + "BLOCKLIST": "content_filter", + "PROHIBITED_CONTENT": "content_filter", + "SPII": "content_filter", + "MALFORMED_FUNCTION_CALL": "stop", # openai doesn't have a way of representing this + "IMAGE_SAFETY": "content_filter", } def translate_exception_str(self, exception_string: str): @@ -820,17 +841,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): def _check_finish_reason( self, - chat_completion_message: ChatCompletionResponseMessage, + chat_completion_message: Optional[ChatCompletionResponseMessage], finish_reason: Optional[str], ) -> OpenAIChatCompletionFinishReason: - if chat_completion_message.get("function_call"): + mapped_finish_reason = self.get_finish_reason_mapping() + if chat_completion_message and chat_completion_message.get("function_call"): return "function_call" - elif chat_completion_message.get("tool_calls"): + elif chat_completion_message and chat_completion_message.get("tool_calls"): return "tool_calls" - elif finish_reason and ( - finish_reason == "SAFETY" or finish_reason == "RECITATION" + elif ( + finish_reason and finish_reason in mapped_finish_reason.keys() ): # vertex ai - return "content_filter" + return mapped_finish_reason[finish_reason] else: return "stop" @@ -1586,8 +1608,9 @@ class ModelResponseIterator: ) if gemini_chunk and "finishReason" in gemini_chunk: - finish_reason = map_finish_reason( - finish_reason=gemini_chunk["finishReason"] + finish_reason = VertexGeminiConfig()._check_finish_reason( + chat_completion_message=None, + finish_reason=gemini_chunk["finishReason"], ) ## DO NOT SET 'is_finished' = True ## GEMINI SETS FINISHREASON ON EVERY CHUNK! diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 42a61b1b3be..edf912d22f7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4390,6 +4390,11 @@ "output_cost_per_token": 0.000004, "cache_creation_input_token_cost": 0.000001, "cache_read_input_token_cost": 0.00000008, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4400,7 +4405,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-10-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "claude-3-5-haiku-latest": { "max_tokens": 8192, @@ -4410,6 +4416,11 @@ "output_cost_per_token": 0.000005, "cache_creation_input_token_cost": 0.00000125, "cache_read_input_token_cost": 0.0000001, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4420,7 +4431,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-10-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "claude-3-opus-latest": { "max_tokens": 4096, @@ -4485,6 +4497,11 @@ "output_cost_per_token": 0.000015, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4495,7 +4512,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-06-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "claude-3-5-sonnet-20240620": { "max_tokens": 8192, @@ -4523,6 +4541,11 @@ "max_output_tokens": 128000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, "litellm_provider": "anthropic", @@ -4546,6 +4569,11 @@ "output_cost_per_token": 0.000015, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4557,7 +4585,8 @@ "supports_response_schema": true, "deprecation_date": "2026-02-01", "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_web_search": true }, "claude-3-5-sonnet-20241022": { "max_tokens": 8192, @@ -4567,6 +4596,11 @@ "output_cost_per_token": 0.000015, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4577,7 +4611,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-10-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "text-bison": { "max_tokens": 2048, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 42a61b1b3be..edf912d22f7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4390,6 +4390,11 @@ "output_cost_per_token": 0.000004, "cache_creation_input_token_cost": 0.000001, "cache_read_input_token_cost": 0.00000008, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4400,7 +4405,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-10-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "claude-3-5-haiku-latest": { "max_tokens": 8192, @@ -4410,6 +4416,11 @@ "output_cost_per_token": 0.000005, "cache_creation_input_token_cost": 0.00000125, "cache_read_input_token_cost": 0.0000001, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4420,7 +4431,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-10-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "claude-3-opus-latest": { "max_tokens": 4096, @@ -4485,6 +4497,11 @@ "output_cost_per_token": 0.000015, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4495,7 +4512,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-06-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "claude-3-5-sonnet-20240620": { "max_tokens": 8192, @@ -4523,6 +4541,11 @@ "max_output_tokens": 128000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, "litellm_provider": "anthropic", @@ -4546,6 +4569,11 @@ "output_cost_per_token": 0.000015, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4557,7 +4585,8 @@ "supports_response_schema": true, "deprecation_date": "2026-02-01", "supports_tool_choice": true, - "supports_reasoning": true + "supports_reasoning": true, + "supports_web_search": true }, "claude-3-5-sonnet-20241022": { "max_tokens": 8192, @@ -4567,6 +4596,11 @@ "output_cost_per_token": 0.000015, "cache_creation_input_token_cost": 0.00000375, "cache_read_input_token_cost": 0.0000003, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, @@ -4577,7 +4611,8 @@ "supports_prompt_caching": true, "supports_response_schema": true, "deprecation_date": "2025-10-01", - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_web_search": true }, "text-bison": { "max_tokens": 2048, diff --git a/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 6c98c87cd76..708cb3abee2 100644 --- a/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -111,3 +111,14 @@ def test_extract_response_content_with_citations(): _, citations, _, _, _ = config.extract_response_content(completion_response) assert citations is not None + + +def test_map_tool_helper(): + config = AnthropicConfig() + + tool = {"type": "web_search_20250305", "name": "web_search", "max_uses": 5} + + result = config._map_tool_helper(tool) + assert result is not None + assert result["name"] == "web_search" + assert result["max_uses"] == 5 diff --git a/tests/litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 9abcd57b29a..133d4aebb37 100644 --- a/tests/litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,6 +1,6 @@ import asyncio -from typing import List, cast from copy import deepcopy +from typing import List, cast from unittest.mock import MagicMock import pytest @@ -315,3 +315,13 @@ def test_vertex_ai_candidate_token_count_inclusive( assert usage.prompt_tokens == expected_usage.prompt_tokens assert usage.completion_tokens == expected_usage.completion_tokens assert usage.total_tokens == expected_usage.total_tokens + + +def test_check_finish_reason(): + config = VertexGeminiConfig() + finish_reason_mappings = config.get_finish_reason_mapping() + for k, v in finish_reason_mappings.items(): + assert ( + config._check_finish_reason(chat_completion_message=None, finish_reason=k) + == v + ) diff --git a/tests/litellm/test_utils.py b/tests/litellm/test_utils.py index 8dd0b170d0a..a6d2565066c 100644 --- a/tests/litellm/test_utils.py +++ b/tests/litellm/test_utils.py @@ -395,3 +395,27 @@ def test_all_model_configs(): optional_params={}, drop_params=False, ) == {"max_output_tokens": 10} + + +def test_anthropic_web_search_in_model_info(): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + supported_models = [ + "anthropic/claude-3-7-sonnet-20250219", + "anthropic/claude-3-5-sonnet-latest", + "anthropic/claude-3-5-sonnet-20241022", + "anthropic/claude-3-5-haiku-20241022", + "anthropic/claude-3-5-haiku-latest", + ] + for model in supported_models: + from litellm.utils import get_model_info + + model_info = get_model_info(model) + assert model_info is not None + assert ( + model_info["supports_web_search"] is True + ), f"Model {model} should support web search" + assert ( + model_info["search_context_cost_per_query"] is not None + ), f"Model {model} should have a search context cost per query" diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 3f6f17a8a45..3f940e2675b 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1170,3 +1170,47 @@ def test_just_system_message(): response = litellm.completion(**params) assert response is not None + + +def test_anthropic_websearch(): + litellm._turn_on_debug() + params = { + "model": "anthropic/claude-3-5-sonnet-latest", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "tools": [{ + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 + }] + } + + try: + response = litellm.completion(**params) + except litellm.InternalServerError as e: + print(e) + + assert response is not None + + +def test_anthropic_text_editor(): + litellm._turn_on_debug() + params = { + "model": "anthropic/claude-3-5-sonnet-latest", + "messages": [ + { + "role": "user", + "content": "There'\''s a syntax error in my primes.py file. Can you help me fix it?" + } + ], + "tools": [{ + "type": "text_editor_20250124", + "name": "str_replace_editor" + }] + } + + try: + response = litellm.completion(**params) + except litellm.InternalServerError as e: + print(e) + + assert response is not None \ No newline at end of file diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 475c4f03b72..8d2de1ea04e 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -134,4 +134,14 @@ def test_gemini_thinking_budget_0(): } ) print(raw_request) - assert "0" in json.dumps(raw_request["raw_request_body"]) \ No newline at end of file + assert "0" in json.dumps(raw_request["raw_request_body"]) + + +def test_gemini_finish_reason(): + import os + from litellm import completion + litellm._turn_on_debug() + response = completion(model="gemini/gemini-1.5-pro", messages=[{"role": "user", "content": "give me 3 random words"}], max_tokens=2) + print(response) + assert response.choices[0].finish_reason is not None + assert response.choices[0].finish_reason == "length" \ No newline at end of file