diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 5d6c1ac0e4b..9f293ccfe14 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1815,7 +1815,8 @@ def convert_function_to_anthropic_tool_invoke( raise e -ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" +def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool: + return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES def find_anthropic_server_tool_result( @@ -1910,6 +1911,8 @@ def convert_to_anthropic_tool_invoke( } ) anthropic_tool_invoke.append(server_tool_result) + elif is_anthropic_tool_search_use(tool_id, tool_name): + continue else: sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index b12d4d280d3..51942fa5fff 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -22,6 +22,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.exceptions import BadRequestError from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( ChatCompletionToolParam, @@ -159,14 +160,24 @@ class _CustomToolFormat(LiteLLMBaseModel): definition: str = "" -_ALLOWED_CALLERS_ADAPTER: Final = TypeAdapter(list[str] | None) +_ALLOWED_CALLERS_ADAPTER: Final = TypeAdapter[list[str] | None](list[str] | None) def validated_allowed_callers(value: object) -> list[str] | None: try: return _ALLOWED_CALLERS_ADAPTER.validate_python(value, strict=True) except ValidationError as exc: - raise ValueError("allowed_callers must be a list of strings") from exc + raise BadRequestError(message="allowed_callers must be a list of strings", model="", llm_provider="") from exc + + +_DEFER_LOADING_ADAPTER: Final = TypeAdapter[bool | None](bool | None) + + +def validated_defer_loading(value: object) -> bool: + try: + return bool(_DEFER_LOADING_ADAPTER.validate_python(value, strict=True)) + except ValidationError as exc: + raise BadRequestError(message="defer_loading must be a boolean", model="", llm_provider="") from exc def custom_tool_grammar_suffix(fmt: object) -> str: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 3a8514434fd..bdc4796b2ea 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -17,7 +17,7 @@ from litellm.responses.litellm_completion_transformation.reasoning_items import from litellm.responses.litellm_completion_transformation.tool_search import ( build_tool_search_call, has_server_tool_search, - is_tool_search_call_name, + is_anthropic_tool_search_use, ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, @@ -170,14 +170,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids self._tool_search_requested: bool = has_server_tool_search(request_tools) - self._tool_search_call_ids: set[str] = set() # mutable-ok: provider call ids seen as tool search calls + self._tool_search_call_ids: frozenset[str] = frozenset() def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool: if call_id in self._tool_search_call_ids: return True - if not self._tool_search_requested or not is_tool_search_call_name(fn_name): + if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name): return False - self._tool_search_call_ids.add(call_id) + self._tool_search_call_ids = self._tool_search_call_ids | {call_id} return True def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None: diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 07f6a558cd1..13d8191d6d5 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -3,12 +3,27 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.responses.main import ResponseToolSearchCall +from litellm.types.utils import LlmProviders from .custom_tools import openai_shaped_tool_call_item_id +__all__ = [ + "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", + "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", + "ToolSearchCallStatus", + "anthropic_tool_search_chat_tool", + "build_tool_search_call", + "has_server_tool_search", + "is_anthropic_tool_search_chat_tool", + "is_anthropic_tool_search_use", + "is_server_tool_search_tool", + "provider_builds_anthropic_request", +] + ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] @@ -43,8 +58,18 @@ def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool: return tool.get("type") == ANTHROPIC_TOOL_SEARCH_TOOL_TYPE -def is_tool_search_call_name(tool_name: str) -> bool: - return tool_name == ANTHROPIC_TOOL_SEARCH_TOOL_NAME +def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.utils import ProviderConfigManager + + _, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return False + chat_config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) + return isinstance(chat_config, AnthropicConfig) def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3901e2f15f8..9fbbe116a98 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -111,6 +111,7 @@ from .custom_tools import ( serialize_tool_call_arguments, unwrap_custom_tool_arguments, validated_allowed_callers, + validated_defer_loading, ) from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( @@ -118,8 +119,9 @@ from .tool_search import ( build_tool_search_call, has_server_tool_search, is_anthropic_tool_search_chat_tool, + is_anthropic_tool_search_use, is_server_tool_search_tool, - is_tool_search_call_name, + provider_builds_anthropic_request, ) NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] @@ -388,11 +390,13 @@ class LiteLLMCompletionResponsesConfig: if not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools - if supports_tool_search(model=model, custom_llm_provider=custom_llm_provider): + if supports_tool_search( + model=model, custom_llm_provider=custom_llm_provider + ) and provider_builds_anthropic_request(model, custom_llm_provider): return tools verbose_logger.warning( - "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') does not support " - "tool search, so the target provider would reject the request.", + "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search " + "on this route, so the target provider would reject or ignore it.", model, custom_llm_provider, ) @@ -1984,7 +1988,7 @@ class LiteLLMCompletionResponsesConfig: strict=bool(namespace_tool.get("strict", False)), ) allowed_callers: Final = validated_allowed_callers(namespace_tool.get("allowed_callers")) - deferred: Final = bool(namespace_tool.get("defer_loading")) + deferred: Final = validated_defer_loading(namespace_tool.get("defer_loading")) if allowed_callers is None and not deferred: return ChatCompletionToolParam(type="function", function=function) if allowed_callers is None: @@ -2076,8 +2080,8 @@ class LiteLLMCompletionResponsesConfig: } if tool.get("cache_control"): chat_completion_tool["cache_control"] = tool.get("cache_control") - if tool.get("defer_loading"): - chat_completion_tool["defer_loading"] = tool.get("defer_loading") + if validated_defer_loading(tool.get("defer_loading")): + chat_completion_tool["defer_loading"] = True if tool.get("allowed_callers"): chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") if tool.get("input_examples"): @@ -2325,7 +2329,7 @@ class LiteLLMCompletionResponsesConfig: web_search_call: Final = web_search_calls.get(tool_id) if web_search_call is not None: return web_search_call - if tool_search_requested and is_tool_search_call_name(tool_name): + if tool_search_requested and is_anthropic_tool_search_use(tool_id, tool_name): return build_tool_search_call(tool_id, tool_arguments, "completed") return None diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f0b5a1bb84e..cf4946aaa24 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -772,6 +772,8 @@ ANTHROPIC_TOOL_SEARCH_BETA_HEADER: Final = "advanced-tool-use-2025-11-20" ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( {"tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"} ) +ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"}) +ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" # Effort beta header constant ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index bb7af3f1c06..75e4a7ab2ae 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -7077,3 +7077,34 @@ def test_completion_bedrock_invalid_role_exception(monkeypatch): str(exc_info.value) == "litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}" ) + + +def test_anthropic_messages_pt_drops_replayed_tool_search_call_without_its_result(): + messages: Final = [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, + }, + { + "id": "toolu_01Weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }, + ], + }, + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C and cloudy"}, + {"role": "user", "content": "And tomorrow?"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + + assert [(block["type"], block["id"]) for block in result[1]["content"]] == [("tool_use", "toolu_01Weather")] + assert [block["tool_use_id"] for block in result[2]["content"] if block["type"] == "tool_result"] == [ + "toolu_01Weather" + ] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 5c0fa79fd05..fc8d6e0de4b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1694,7 +1694,9 @@ def _deferred_tools_with_server_tool_search() -> list[dict[str, object]]: ] -def _chat_response_with_tool_search_call(search_arguments: str) -> ModelResponse: +def _chat_response_with_tool_search_call( + search_arguments: str, search_call_id: str = "srvtoolu_01Search" +) -> ModelResponse: return ModelResponse( id="chatcmpl-tool-search", created=1748575031, @@ -1709,7 +1711,7 @@ def _chat_response_with_tool_search_call(search_arguments: str) -> ModelResponse "content": None, "tool_calls": [ { - "id": "srvtoolu_01Search", + "id": search_call_id, "type": "function", "function": {"name": "tool_search_tool_regex", "arguments": search_arguments}, }, @@ -1883,18 +1885,75 @@ class TestToolTransformation: assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True] def test_server_tool_search_dropped_for_models_without_tool_search(self) -> None: - model: Final = "gemini/gemini-2.5-flash" + model: Final = "openai/model-with-no-cost-map-entry" assert litellm.supports_tool_search(model) is False result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input="What is the weather in Paris right now?", responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, - custom_llm_provider="gemini", + custom_llm_provider="openai", ) assert [tool["type"] for tool in result["tools"]] == ["function", "function"] + def test_server_tool_search_follows_the_bedrock_route(self) -> None: + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request + converse: Final = transform( + model="bedrock/anthropic.claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, + custom_llm_provider="bedrock", + ) + invoke: Final = transform( + model="bedrock/invoke/anthropic.claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, + custom_llm_provider="bedrock", + ) + + assert [tool["type"] for tool in converse["tools"]] == ["function", "function"] + assert invoke["tools"][0] == {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} + + def test_caller_function_named_like_the_search_tool_stays_a_function_call(self) -> None: + items: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( + chat_completion_response=_chat_response_with_tool_search_call( + '{"pattern": "weather"}', search_call_id="call_01Mine" + ), + responses_api_request={ + "tools": [ + *_deferred_tools_with_server_tool_search(), + {"type": "function", "name": "tool_search_tool_regex", "parameters": {"type": "object"}}, + ] + }, + ) + + assert [(item.type, item.name) for item in items] == [ + ("function_call", "tool_search_tool_regex"), + ("function_call", "get_weather"), + ] + + def test_defer_loading_must_be_a_boolean(self) -> None: + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools + namespace_tool: Final = { + "type": "namespace", + "name": "finance", + "tools": [{"type": "function", "name": "get_stock_price", "parameters": {"type": "object"}}], + } + + with pytest.raises(litellm.BadRequestError, match="defer_loading must be a boolean"): + transform(tools=[{**namespace_tool, "tools": [{**namespace_tool["tools"][0], "defer_loading": "false"}]}]) + with pytest.raises(litellm.BadRequestError, match="defer_loading must be a boolean"): + transform(tools=[{"type": "function", "name": "get_weather", "parameters": {}, "defer_loading": "false"}]) + not_deferred, _ = transform( + tools=[ + {**namespace_tool, "tools": [{**namespace_tool["tools"][0], "defer_loading": False}]}, + {"type": "function", "name": "get_weather", "parameters": {}, "defer_loading": False}, + ] + ) + assert [tool["function"]["name"] for tool in not_deferred] == ["finance__get_stock_price", "get_weather"] + assert all("defer_loading" not in tool for tool in not_deferred) + def test_anthropic_tool_search_call_becomes_tool_search_call_item(self) -> None: response: Final = _chat_response_with_tool_search_call('{"pattern": "weather|forecast"}') with_tool_search: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( @@ -2103,7 +2162,7 @@ class TestToolTransformation: tools = [custom_tool] - with pytest.raises(ValueError, match="allowed_callers must be a list of strings"): + with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( tools=tools ) @@ -2696,7 +2755,7 @@ class TestToolTransformation: else {**function_tool, "type": "namespace"} ) - with pytest.raises(ValueError, match="allowed_callers must be a list of strings"): + with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( tools=[namespace_tool] ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index ec702703577..4117c303940 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -1418,3 +1418,52 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ assert {event["output_index"] for event in search_events}.isdisjoint( {event["output_index"] for event in function_events} ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_caller_function_named_like_the_search_tool_streams_as_function_call(sync_mode: bool): + chunk: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-sonnet-5-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + finish_reason="tool_calls", + delta=Delta( + content=None, + tool_calls=[ + { + "index": 0, + "id": "call_01Mine", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, + } + ], + ), + ) + ], + ) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper((chunk,)), + request_input="What is the weather in Paris right now?", + responses_api_request={ + "tools": [ + {"type": "tool_search", "execution": "server"}, + {"type": "function", "name": "tool_search_tool_regex", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + events: Final = ( + [event.model_dump(exclude_none=True) for event in iterator] + if sync_mode + else [event.model_dump(exclude_none=True) async for event in iterator] + ) + output: Final = events[-1]["response"]["output"] + + assert [(item["type"], item["name"]) for item in output] == [("function_call", "tool_search_tool_regex")] + assert "response.function_call_arguments.done" in [event["type"] for event in events]