From e7b738585b19ebb2ee390b161fd44df206670f0a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:47:35 -0700 Subject: [PATCH] fix(responses): gate tool search on the Anthropic request builder and validate deferred tool fields Bedrock Converse builds its own request and has no tool search, so the bridge now drops the tool_search tool there with a warning instead of sending a tool the transport loses, while Bedrock invoke, Anthropic, and Vertex keep it. A replayed tool_search_call with no stored result is dropped from the Anthropic request so a previous_response_id follow-up no longer answers 400. Only srvtoolu_ ids whose name is one of Anthropic's search tools count as search calls, so a caller function named like one stays a function call. A non-boolean defer_loading or a non-list allowed_callers now answers 400 instead of 500 --- .../prompt_templates/factory.py | 5 +- .../custom_tools.py | 15 +++- .../streaming_iterator.py | 8 +-- .../tool_search.py | 29 +++++++- .../transformation.py | 20 +++--- litellm/types/llms/anthropic.py | 2 + ...llm_core_utils_prompt_templates_factory.py | 31 ++++++++ .../test_litellm_completion_responses.py | 71 +++++++++++++++++-- .../test_streaming_iterator_transformation.py | 49 +++++++++++++ 9 files changed, 207 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 5d6c1ac0e4b..9f293ccfe14 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1815,7 +1815,8 @@ def convert_function_to_anthropic_tool_invoke( raise e -ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" +def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool: + return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES def find_anthropic_server_tool_result( @@ -1910,6 +1911,8 @@ def convert_to_anthropic_tool_invoke( } ) anthropic_tool_invoke.append(server_tool_result) + elif is_anthropic_tool_search_use(tool_id, tool_name): + continue else: sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index b12d4d280d3..51942fa5fff 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -22,6 +22,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.exceptions import BadRequestError from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( ChatCompletionToolParam, @@ -159,14 +160,24 @@ class _CustomToolFormat(LiteLLMBaseModel): definition: str = "" -_ALLOWED_CALLERS_ADAPTER: Final = TypeAdapter(list[str] | None) +_ALLOWED_CALLERS_ADAPTER: Final = TypeAdapter[list[str] | None](list[str] | None) def validated_allowed_callers(value: object) -> list[str] | None: try: return _ALLOWED_CALLERS_ADAPTER.validate_python(value, strict=True) except ValidationError as exc: - raise ValueError("allowed_callers must be a list of strings") from exc + raise BadRequestError(message="allowed_callers must be a list of strings", model="", llm_provider="") from exc + + +_DEFER_LOADING_ADAPTER: Final = TypeAdapter[bool | None](bool | None) + + +def validated_defer_loading(value: object) -> bool: + try: + return bool(_DEFER_LOADING_ADAPTER.validate_python(value, strict=True)) + except ValidationError as exc: + raise BadRequestError(message="defer_loading must be a boolean", model="", llm_provider="") from exc def custom_tool_grammar_suffix(fmt: object) -> str: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 3a8514434fd..bdc4796b2ea 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -17,7 +17,7 @@ from litellm.responses.litellm_completion_transformation.reasoning_items import from litellm.responses.litellm_completion_transformation.tool_search import ( build_tool_search_call, has_server_tool_search, - is_tool_search_call_name, + is_anthropic_tool_search_use, ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, @@ -170,14 +170,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._web_search_calls: dict[str, object] = {} # mutable-ok: latest call by provider id self._queued_web_search_call_ids: set[str] = set() # mutable-ok: emitted call ids self._tool_search_requested: bool = has_server_tool_search(request_tools) - self._tool_search_call_ids: set[str] = set() # mutable-ok: provider call ids seen as tool search calls + self._tool_search_call_ids: frozenset[str] = frozenset() def _is_tool_search_call(self, call_id: str, fn_name: str) -> bool: if call_id in self._tool_search_call_ids: return True - if not self._tool_search_requested or not is_tool_search_call_name(fn_name): + if not self._tool_search_requested or not is_anthropic_tool_search_use(call_id, fn_name): return False - self._tool_search_call_ids.add(call_id) + self._tool_search_call_ids = self._tool_search_call_ids | {call_id} return True def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None: diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 07f6a558cd1..13d8191d6d5 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -3,12 +3,27 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm.litellm_core_utils.prompt_templates.factory import is_anthropic_tool_search_use from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.responses.main import ResponseToolSearchCall +from litellm.types.utils import LlmProviders from .custom_tools import openai_shaped_tool_call_item_id +__all__ = [ + "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", + "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", + "ToolSearchCallStatus", + "anthropic_tool_search_chat_tool", + "build_tool_search_call", + "has_server_tool_search", + "is_anthropic_tool_search_chat_tool", + "is_anthropic_tool_search_use", + "is_server_tool_search_tool", + "provider_builds_anthropic_request", +] + ANTHROPIC_TOOL_SEARCH_TOOL_TYPE: Final = "tool_search_tool_regex_20251119" ANTHROPIC_TOOL_SEARCH_TOOL_NAME: Final = "tool_search_tool_regex" ToolSearchCallStatus: TypeAlias = Literal["in_progress", "completed", "incomplete"] @@ -43,8 +58,18 @@ def is_anthropic_tool_search_chat_tool(tool: Mapping[str, object]) -> bool: return tool.get("type") == ANTHROPIC_TOOL_SEARCH_TOOL_TYPE -def is_tool_search_call_name(tool_name: str) -> bool: - return tool_name == ANTHROPIC_TOOL_SEARCH_TOOL_NAME +def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.utils import ProviderConfigManager + + _, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return False + chat_config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) + return isinstance(chat_config, AnthropicConfig) def _parsed_arguments(arguments_json: str) -> Mapping[str, object] | None: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 3901e2f15f8..9fbbe116a98 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -111,6 +111,7 @@ from .custom_tools import ( serialize_tool_call_arguments, unwrap_custom_tool_arguments, validated_allowed_callers, + validated_defer_loading, ) from .reasoning_items import decode_thinking_blocks, encode_thinking_blocks, mint_reasoning_item_id from .tool_search import ( @@ -118,8 +119,9 @@ from .tool_search import ( build_tool_search_call, has_server_tool_search, is_anthropic_tool_search_chat_tool, + is_anthropic_tool_search_use, is_server_tool_search_tool, - is_tool_search_call_name, + provider_builds_anthropic_request, ) NamespaceNameMap: TypeAlias = Mapping[str, tuple[str, str]] @@ -388,11 +390,13 @@ class LiteLLMCompletionResponsesConfig: if not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): return tools - if supports_tool_search(model=model, custom_llm_provider=custom_llm_provider): + if supports_tool_search( + model=model, custom_llm_provider=custom_llm_provider + ) and provider_builds_anthropic_request(model, custom_llm_provider): return tools verbose_logger.warning( - "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') does not support " - "tool search, so the target provider would reject the request.", + "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search " + "on this route, so the target provider would reject or ignore it.", model, custom_llm_provider, ) @@ -1984,7 +1988,7 @@ class LiteLLMCompletionResponsesConfig: strict=bool(namespace_tool.get("strict", False)), ) allowed_callers: Final = validated_allowed_callers(namespace_tool.get("allowed_callers")) - deferred: Final = bool(namespace_tool.get("defer_loading")) + deferred: Final = validated_defer_loading(namespace_tool.get("defer_loading")) if allowed_callers is None and not deferred: return ChatCompletionToolParam(type="function", function=function) if allowed_callers is None: @@ -2076,8 +2080,8 @@ class LiteLLMCompletionResponsesConfig: } if tool.get("cache_control"): chat_completion_tool["cache_control"] = tool.get("cache_control") - if tool.get("defer_loading"): - chat_completion_tool["defer_loading"] = tool.get("defer_loading") + if validated_defer_loading(tool.get("defer_loading")): + chat_completion_tool["defer_loading"] = True if tool.get("allowed_callers"): chat_completion_tool["allowed_callers"] = tool.get("allowed_callers") if tool.get("input_examples"): @@ -2325,7 +2329,7 @@ class LiteLLMCompletionResponsesConfig: web_search_call: Final = web_search_calls.get(tool_id) if web_search_call is not None: return web_search_call - if tool_search_requested and is_tool_search_call_name(tool_name): + if tool_search_requested and is_anthropic_tool_search_use(tool_id, tool_name): return build_tool_search_call(tool_id, tool_arguments, "completed") return None diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f0b5a1bb84e..cf4946aaa24 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -772,6 +772,8 @@ ANTHROPIC_TOOL_SEARCH_BETA_HEADER: Final = "advanced-tool-use-2025-11-20" ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: Final = frozenset( {"tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"} ) +ANTHROPIC_TOOL_SEARCH_TOOL_NAMES: Final = frozenset({"tool_search_tool_regex", "tool_search_tool_bm25"}) +ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX: Final = "srvtoolu_" # Effort beta header constant ANTHROPIC_EFFORT_BETA_HEADER: Final = "effort-2025-11-24" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index bb7af3f1c06..75e4a7ab2ae 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -7077,3 +7077,34 @@ def test_completion_bedrock_invalid_role_exception(monkeypatch): str(exc_info.value) == "litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}" ) + + +def test_anthropic_messages_pt_drops_replayed_tool_search_call_without_its_result(): + messages: Final = [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, + }, + { + "id": "toolu_01Weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }, + ], + }, + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C and cloudy"}, + {"role": "user", "content": "And tomorrow?"}, + ] + + result: Final = anthropic_messages_pt(messages=messages, model="claude-sonnet-5-5", llm_provider="anthropic") + + assert [(block["type"], block["id"]) for block in result[1]["content"]] == [("tool_use", "toolu_01Weather")] + assert [block["tool_use_id"] for block in result[2]["content"] if block["type"] == "tool_result"] == [ + "toolu_01Weather" + ] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 5c0fa79fd05..fc8d6e0de4b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1694,7 +1694,9 @@ def _deferred_tools_with_server_tool_search() -> list[dict[str, object]]: ] -def _chat_response_with_tool_search_call(search_arguments: str) -> ModelResponse: +def _chat_response_with_tool_search_call( + search_arguments: str, search_call_id: str = "srvtoolu_01Search" +) -> ModelResponse: return ModelResponse( id="chatcmpl-tool-search", created=1748575031, @@ -1709,7 +1711,7 @@ def _chat_response_with_tool_search_call(search_arguments: str) -> ModelResponse "content": None, "tool_calls": [ { - "id": "srvtoolu_01Search", + "id": search_call_id, "type": "function", "function": {"name": "tool_search_tool_regex", "arguments": search_arguments}, }, @@ -1883,18 +1885,75 @@ class TestToolTransformation: assert [tool["defer_loading"] for tool in chat_tools[1:]] == [True, True] def test_server_tool_search_dropped_for_models_without_tool_search(self) -> None: - model: Final = "gemini/gemini-2.5-flash" + model: Final = "openai/model-with-no-cost-map-entry" assert litellm.supports_tool_search(model) is False result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input="What is the weather in Paris right now?", responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, - custom_llm_provider="gemini", + custom_llm_provider="openai", ) assert [tool["type"] for tool in result["tools"]] == ["function", "function"] + def test_server_tool_search_follows_the_bedrock_route(self) -> None: + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request + converse: Final = transform( + model="bedrock/anthropic.claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, + custom_llm_provider="bedrock", + ) + invoke: Final = transform( + model="bedrock/invoke/anthropic.claude-sonnet-5-5", + input="What is the weather in Paris right now?", + responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, + custom_llm_provider="bedrock", + ) + + assert [tool["type"] for tool in converse["tools"]] == ["function", "function"] + assert invoke["tools"][0] == {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} + + def test_caller_function_named_like_the_search_tool_stays_a_function_call(self) -> None: + items: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( + chat_completion_response=_chat_response_with_tool_search_call( + '{"pattern": "weather"}', search_call_id="call_01Mine" + ), + responses_api_request={ + "tools": [ + *_deferred_tools_with_server_tool_search(), + {"type": "function", "name": "tool_search_tool_regex", "parameters": {"type": "object"}}, + ] + }, + ) + + assert [(item.type, item.name) for item in items] == [ + ("function_call", "tool_search_tool_regex"), + ("function_call", "get_weather"), + ] + + def test_defer_loading_must_be_a_boolean(self) -> None: + transform: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools + namespace_tool: Final = { + "type": "namespace", + "name": "finance", + "tools": [{"type": "function", "name": "get_stock_price", "parameters": {"type": "object"}}], + } + + with pytest.raises(litellm.BadRequestError, match="defer_loading must be a boolean"): + transform(tools=[{**namespace_tool, "tools": [{**namespace_tool["tools"][0], "defer_loading": "false"}]}]) + with pytest.raises(litellm.BadRequestError, match="defer_loading must be a boolean"): + transform(tools=[{"type": "function", "name": "get_weather", "parameters": {}, "defer_loading": "false"}]) + not_deferred, _ = transform( + tools=[ + {**namespace_tool, "tools": [{**namespace_tool["tools"][0], "defer_loading": False}]}, + {"type": "function", "name": "get_weather", "parameters": {}, "defer_loading": False}, + ] + ) + assert [tool["function"]["name"] for tool in not_deferred] == ["finance__get_stock_price", "get_weather"] + assert all("defer_loading" not in tool for tool in not_deferred) + def test_anthropic_tool_search_call_becomes_tool_search_call_item(self) -> None: response: Final = _chat_response_with_tool_search_call('{"pattern": "weather|forecast"}') with_tool_search: Final = LiteLLMCompletionResponsesConfig.transform_chat_completion_tools_to_responses_tools( @@ -2103,7 +2162,7 @@ class TestToolTransformation: tools = [custom_tool] - with pytest.raises(ValueError, match="allowed_callers must be a list of strings"): + with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( tools=tools ) @@ -2696,7 +2755,7 @@ class TestToolTransformation: else {**function_tool, "type": "namespace"} ) - with pytest.raises(ValueError, match="allowed_callers must be a list of strings"): + with pytest.raises(litellm.BadRequestError, match="allowed_callers must be a list of strings"): LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( tools=[namespace_tool] ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index ec702703577..4117c303940 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -1418,3 +1418,52 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ assert {event["output_index"] for event in search_events}.isdisjoint( {event["output_index"] for event in function_events} ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_caller_function_named_like_the_search_tool_streams_as_function_call(sync_mode: bool): + chunk: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-sonnet-5-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + finish_reason="tool_calls", + delta=Delta( + content=None, + tool_calls=[ + { + "index": 0, + "id": "call_01Mine", + "type": "function", + "function": {"name": "tool_search_tool_regex", "arguments": '{"pattern": "weather"}'}, + } + ], + ), + ) + ], + ) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper((chunk,)), + request_input="What is the weather in Paris right now?", + responses_api_request={ + "tools": [ + {"type": "tool_search", "execution": "server"}, + {"type": "function", "name": "tool_search_tool_regex", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + events: Final = ( + [event.model_dump(exclude_none=True) for event in iterator] + if sync_mode + else [event.model_dump(exclude_none=True) async for event in iterator] + ) + output: Final = events[-1]["response"]["output"] + + assert [(item["type"], item["name"]) for item in output] == [("function_call", "tool_search_tool_regex")] + assert "response.function_call_arguments.done" in [event["type"] for event in events]