From 24bc04e65be752075229b9e3954d5b9bfaa1bd56 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:44:47 -0800 Subject: [PATCH] fix(responses): keep a replayed search when the caller sent Anthropic's own search tool A Claude model the cost map does not flag for tool search still runs it when the request carries Anthropic's own tool_search_tool_regex or bm25 tool, which is how turn 1 already decides to keep it. Both replay gates now follow that rule, so a stored follow-up on such a model keeps Claude's search as it did before, and a stateless one replays it --- .../tool_search.py | 7 +++++ .../transformation.py | 13 +++++--- .../test_anthropic_responses_bridge.py | 18 ++++++++++- .../test_litellm_completion_responses.py | 31 +++++++++++++++++++ 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index dfec900042c..261ba0361dc 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -41,6 +41,7 @@ __all__ = [ "is_server_tool_search_tool", "provider_builds_anthropic_request", "provider_chat_config", + "request_runs_tool_search", "route_runs_tool_search", "runs_tool_search_on_the_server", "tool_search_call_ids", @@ -226,6 +227,12 @@ def route_runs_tool_search(model: str, custom_llm_provider: str | None) -> bool: ) and provider_builds_anthropic_request(model, custom_llm_provider) +def request_runs_tool_search(tools: Iterable[object], model: str, custom_llm_provider: str | None) -> bool: + if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools): + return provider_builds_anthropic_request(model, custom_llm_provider) + return route_runs_tool_search(model, custom_llm_provider) + + def chat_config_builds_anthropic_request(chat_config: object) -> bool: from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9791c809827..cd50d461770 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -163,6 +163,7 @@ from .tool_search import ( is_anthropic_tool_search_chat_tool, is_server_tool_search_tool, provider_builds_anthropic_request, + request_runs_tool_search, route_runs_tool_search, tool_search_call_ids, tool_search_routing, @@ -1191,7 +1192,7 @@ class LiteLLMCompletionResponsesConfig: native_compaction=native_compaction, only_since_compaction=not native_compaction, replay_tool_search=holds_tool_search_items(request_input) - and route_runs_tool_search(model, custom_llm_provider), + and request_runs_tool_search(request_tools or (), model, custom_llm_provider), ) litellm_completion_request: dict = { @@ -1325,10 +1326,10 @@ class LiteLLMCompletionResponsesConfig: return messages @staticmethod - def _without_tool_search_uses_the_route_cannot_run( - messages: Sequence[SessionMessage], route: CompletionRoute + def _without_tool_search_uses_the_request_cannot_run( + messages: Sequence[SessionMessage], route: CompletionRoute, tools: Iterable[object] ) -> tuple[SessionMessage, ...]: - if route_runs_tool_search(route.model, route.custom_llm_provider): + if request_runs_tool_search(tools, route.model, route.custom_llm_provider): return tuple(messages) search_call_ids: Final = tool_search_call_ids(messages) return tuple( @@ -1390,7 +1391,9 @@ class LiteLLMCompletionResponsesConfig: request_fields.get("context_management"), ) replayed_messages: Final = ( - LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_route_cannot_run(stored_messages, route) + LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_request_cannot_run( + stored_messages, route, _OBJECT_LIST_ADAPTER.validate_python(request_fields.get("tools") or []) + ) if any(holds_tool_search_use(message) for message in stored_messages) else stored_messages ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index e0d77a230b1..c7dbabb7f0b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -377,7 +377,10 @@ async def _follow_up_messages( async def _follow_up_messages_after( - spend_log: Mapping[str, object], model: str, custom_llm_provider: str + spend_log: Mapping[str, object], + model: str, + custom_llm_provider: str, + tools: Sequence[Mapping[str, object]] = (), ) -> Sequence[object]: with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([spend_log])): request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( @@ -386,6 +389,7 @@ async def _follow_up_messages_after( "model": model, "custom_llm_provider": custom_llm_provider, "messages": [{"role": "user", "content": "And in Osaka?"}], + **({"tools": [dict(tool) for tool in tools]} if tools else {}), }, instructions=None, ) @@ -1801,3 +1805,15 @@ async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_ messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, model, custom_llm_provider) assert _history_tool_ids(messages) == (["toolu_01Weather"], ["toolu_01Weather"]) + + +@pytest.mark.asyncio +async def test_previous_response_id_keeps_a_stored_search_for_a_claude_sent_anthropics_own_search_tool() -> None: + messages: Final = await _follow_up_messages_after( + _STATELESS_SEARCH_TURN, + "anthropic/claude-3-7-sonnet-20250219", + "anthropic", + tools=({"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},), + ) + + assert _history_tool_ids(messages)[0] == ["srvtoolu_01Search", "toolu_01Weather"] diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index e72dadb985a..9ce9d68f48c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -2429,6 +2429,37 @@ class TestToolTransformation: assert [message["role"] for message in request["messages"]] == ["user", "assistant", "tool"] assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids + @pytest.mark.parametrize( + ("model", "custom_llm_provider", "replayed_call_ids"), + [ + pytest.param( + "anthropic/claude-3-7-sonnet-20250219", + "anthropic", + ["srvtoolu_01Search", "toolu_01Weather"], + id="on_anthropic", + ), + pytest.param("mistral/mistral-small-latest", "mistral", ["toolu_01Weather"], id="off_anthropic"), + ], + ) + def test_stateless_tool_search_items_follow_anthropics_own_search_tool_on_a_claude_the_cost_map_does_not_flag( + self, model: str, custom_llm_provider: str, replayed_call_ids: Sequence[str] + ) -> None: + request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts + ResponseInputParam, list(_stateless_tool_search_input("srvtoolu_01Search", "completed")) + ), + responses_api_request={ + "tools": [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + *_deferred_tools_with_server_tool_search()[1:], + ] + }, + custom_llm_provider=custom_llm_provider, + ) + + assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids + def test_transform_local_shell_tools_dropped(self) -> None: ( result_tools,