fix(responses): keep a replayed search when the caller sent Anthropic's own search tool

A Claude model the cost map does not flag for tool search still runs it when the request carries Anthropic's own tool_search_tool_regex or bm25 tool, which is how turn 1 already decides to keep it. Both replay gates now follow that rule, so a stored follow-up on such a model keeps Claude's search as it did before, and a stateless one replays it
This commit is contained in:
mateo-berri 2026-11-06 14:44:47 -08:00
parent 3e61760198
commit 24bc04e65b
4 changed files with 63 additions and 6 deletions

View file

@ -41,6 +41,7 @@ __all__ = [
"is_server_tool_search_tool",
"provider_builds_anthropic_request",
"provider_chat_config",
"request_runs_tool_search",
"route_runs_tool_search",
"runs_tool_search_on_the_server",
"tool_search_call_ids",
@ -226,6 +227,12 @@ def route_runs_tool_search(model: str, custom_llm_provider: str | None) -> bool:
) and provider_builds_anthropic_request(model, custom_llm_provider)
def request_runs_tool_search(tools: Iterable[object], model: str, custom_llm_provider: str | None) -> bool:
if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools):
return provider_builds_anthropic_request(model, custom_llm_provider)
return route_runs_tool_search(model, custom_llm_provider)
def chat_config_builds_anthropic_request(chat_config: object) -> bool:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig

View file

@ -163,6 +163,7 @@ from .tool_search import (
is_anthropic_tool_search_chat_tool,
is_server_tool_search_tool,
provider_builds_anthropic_request,
request_runs_tool_search,
route_runs_tool_search,
tool_search_call_ids,
tool_search_routing,
@ -1191,7 +1192,7 @@ class LiteLLMCompletionResponsesConfig:
native_compaction=native_compaction,
only_since_compaction=not native_compaction,
replay_tool_search=holds_tool_search_items(request_input)
and route_runs_tool_search(model, custom_llm_provider),
and request_runs_tool_search(request_tools or (), model, custom_llm_provider),
)
litellm_completion_request: dict = {
@ -1325,10 +1326,10 @@ class LiteLLMCompletionResponsesConfig:
return messages
@staticmethod
def _without_tool_search_uses_the_route_cannot_run(
messages: Sequence[SessionMessage], route: CompletionRoute
def _without_tool_search_uses_the_request_cannot_run(
messages: Sequence[SessionMessage], route: CompletionRoute, tools: Iterable[object]
) -> tuple[SessionMessage, ...]:
if route_runs_tool_search(route.model, route.custom_llm_provider):
if request_runs_tool_search(tools, route.model, route.custom_llm_provider):
return tuple(messages)
search_call_ids: Final = tool_search_call_ids(messages)
return tuple(
@ -1390,7 +1391,9 @@ class LiteLLMCompletionResponsesConfig:
request_fields.get("context_management"),
)
replayed_messages: Final = (
LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_route_cannot_run(stored_messages, route)
LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_request_cannot_run(
stored_messages, route, _OBJECT_LIST_ADAPTER.validate_python(request_fields.get("tools") or [])
)
if any(holds_tool_search_use(message) for message in stored_messages)
else stored_messages
)

View file

@ -377,7 +377,10 @@ async def _follow_up_messages(
async def _follow_up_messages_after(
spend_log: Mapping[str, object], model: str, custom_llm_provider: str
spend_log: Mapping[str, object],
model: str,
custom_llm_provider: str,
tools: Sequence[Mapping[str, object]] = (),
) -> Sequence[object]:
with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([spend_log])):
request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
@ -386,6 +389,7 @@ async def _follow_up_messages_after(
"model": model,
"custom_llm_provider": custom_llm_provider,
"messages": [{"role": "user", "content": "And in Osaka?"}],
**({"tools": [dict(tool) for tool in tools]} if tools else {}),
},
instructions=None,
)
@ -1801,3 +1805,15 @@ async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_
messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, model, custom_llm_provider)
assert _history_tool_ids(messages) == (["toolu_01Weather"], ["toolu_01Weather"])
@pytest.mark.asyncio
async def test_previous_response_id_keeps_a_stored_search_for_a_claude_sent_anthropics_own_search_tool() -> None:
messages: Final = await _follow_up_messages_after(
_STATELESS_SEARCH_TURN,
"anthropic/claude-3-7-sonnet-20250219",
"anthropic",
tools=({"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},),
)
assert _history_tool_ids(messages)[0] == ["srvtoolu_01Search", "toolu_01Weather"]

View file

@ -2429,6 +2429,37 @@ class TestToolTransformation:
assert [message["role"] for message in request["messages"]] == ["user", "assistant", "tool"]
assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids
@pytest.mark.parametrize(
("model", "custom_llm_provider", "replayed_call_ids"),
[
pytest.param(
"anthropic/claude-3-7-sonnet-20250219",
"anthropic",
["srvtoolu_01Search", "toolu_01Weather"],
id="on_anthropic",
),
pytest.param("mistral/mistral-small-latest", "mistral", ["toolu_01Weather"], id="off_anthropic"),
],
)
def test_stateless_tool_search_items_follow_anthropics_own_search_tool_on_a_claude_the_cost_map_does_not_flag(
self, model: str, custom_llm_provider: str, replayed_call_ids: Sequence[str]
) -> None:
request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts
ResponseInputParam, list(_stateless_tool_search_input("srvtoolu_01Search", "completed"))
),
responses_api_request={
"tools": [
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
*_deferred_tools_with_server_tool_search()[1:],
]
},
custom_llm_provider=custom_llm_provider,
)
assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids
def test_transform_local_shell_tools_dropped(self) -> None:
(
result_tools,