mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(responses): keep a replayed search when the caller sent Anthropic's own search tool
A Claude model the cost map does not flag for tool search still runs it when the request carries Anthropic's own tool_search_tool_regex or bm25 tool, which is how turn 1 already decides to keep it. Both replay gates now follow that rule, so a stored follow-up on such a model keeps Claude's search as it did before, and a stateless one replays it
This commit is contained in:
parent
3e61760198
commit
24bc04e65b
4 changed files with 63 additions and 6 deletions
|
|
@ -41,6 +41,7 @@ __all__ = [
|
|||
"is_server_tool_search_tool",
|
||||
"provider_builds_anthropic_request",
|
||||
"provider_chat_config",
|
||||
"request_runs_tool_search",
|
||||
"route_runs_tool_search",
|
||||
"runs_tool_search_on_the_server",
|
||||
"tool_search_call_ids",
|
||||
|
|
@ -226,6 +227,12 @@ def route_runs_tool_search(model: str, custom_llm_provider: str | None) -> bool:
|
|||
) and provider_builds_anthropic_request(model, custom_llm_provider)
|
||||
|
||||
|
||||
def request_runs_tool_search(tools: Iterable[object], model: str, custom_llm_provider: str | None) -> bool:
|
||||
if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools):
|
||||
return provider_builds_anthropic_request(model, custom_llm_provider)
|
||||
return route_runs_tool_search(model, custom_llm_provider)
|
||||
|
||||
|
||||
def chat_config_builds_anthropic_request(chat_config: object) -> bool:
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
|
|
|||
|
|
@ -163,6 +163,7 @@ from .tool_search import (
|
|||
is_anthropic_tool_search_chat_tool,
|
||||
is_server_tool_search_tool,
|
||||
provider_builds_anthropic_request,
|
||||
request_runs_tool_search,
|
||||
route_runs_tool_search,
|
||||
tool_search_call_ids,
|
||||
tool_search_routing,
|
||||
|
|
@ -1191,7 +1192,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
native_compaction=native_compaction,
|
||||
only_since_compaction=not native_compaction,
|
||||
replay_tool_search=holds_tool_search_items(request_input)
|
||||
and route_runs_tool_search(model, custom_llm_provider),
|
||||
and request_runs_tool_search(request_tools or (), model, custom_llm_provider),
|
||||
)
|
||||
|
||||
litellm_completion_request: dict = {
|
||||
|
|
@ -1325,10 +1326,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _without_tool_search_uses_the_route_cannot_run(
|
||||
messages: Sequence[SessionMessage], route: CompletionRoute
|
||||
def _without_tool_search_uses_the_request_cannot_run(
|
||||
messages: Sequence[SessionMessage], route: CompletionRoute, tools: Iterable[object]
|
||||
) -> tuple[SessionMessage, ...]:
|
||||
if route_runs_tool_search(route.model, route.custom_llm_provider):
|
||||
if request_runs_tool_search(tools, route.model, route.custom_llm_provider):
|
||||
return tuple(messages)
|
||||
search_call_ids: Final = tool_search_call_ids(messages)
|
||||
return tuple(
|
||||
|
|
@ -1390,7 +1391,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
request_fields.get("context_management"),
|
||||
)
|
||||
replayed_messages: Final = (
|
||||
LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_route_cannot_run(stored_messages, route)
|
||||
LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_request_cannot_run(
|
||||
stored_messages, route, _OBJECT_LIST_ADAPTER.validate_python(request_fields.get("tools") or [])
|
||||
)
|
||||
if any(holds_tool_search_use(message) for message in stored_messages)
|
||||
else stored_messages
|
||||
)
|
||||
|
|
|
|||
|
|
@ -377,7 +377,10 @@ async def _follow_up_messages(
|
|||
|
||||
|
||||
async def _follow_up_messages_after(
|
||||
spend_log: Mapping[str, object], model: str, custom_llm_provider: str
|
||||
spend_log: Mapping[str, object],
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
tools: Sequence[Mapping[str, object]] = (),
|
||||
) -> Sequence[object]:
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([spend_log])):
|
||||
request: Final = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
|
||||
|
|
@ -386,6 +389,7 @@ async def _follow_up_messages_after(
|
|||
"model": model,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"messages": [{"role": "user", "content": "And in Osaka?"}],
|
||||
**({"tools": [dict(tool) for tool in tools]} if tools else {}),
|
||||
},
|
||||
instructions=None,
|
||||
)
|
||||
|
|
@ -1801,3 +1805,15 @@ async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_
|
|||
messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, model, custom_llm_provider)
|
||||
|
||||
assert _history_tool_ids(messages) == (["toolu_01Weather"], ["toolu_01Weather"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_previous_response_id_keeps_a_stored_search_for_a_claude_sent_anthropics_own_search_tool() -> None:
|
||||
messages: Final = await _follow_up_messages_after(
|
||||
_STATELESS_SEARCH_TURN,
|
||||
"anthropic/claude-3-7-sonnet-20250219",
|
||||
"anthropic",
|
||||
tools=({"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},),
|
||||
)
|
||||
|
||||
assert _history_tool_ids(messages)[0] == ["srvtoolu_01Search", "toolu_01Weather"]
|
||||
|
|
|
|||
|
|
@ -2429,6 +2429,37 @@ class TestToolTransformation:
|
|||
assert [message["role"] for message in request["messages"]] == ["user", "assistant", "tool"]
|
||||
assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "custom_llm_provider", "replayed_call_ids"),
|
||||
[
|
||||
pytest.param(
|
||||
"anthropic/claude-3-7-sonnet-20250219",
|
||||
"anthropic",
|
||||
["srvtoolu_01Search", "toolu_01Weather"],
|
||||
id="on_anthropic",
|
||||
),
|
||||
pytest.param("mistral/mistral-small-latest", "mistral", ["toolu_01Weather"], id="off_anthropic"),
|
||||
],
|
||||
)
|
||||
def test_stateless_tool_search_items_follow_anthropics_own_search_tool_on_a_claude_the_cost_map_does_not_flag(
|
||||
self, model: str, custom_llm_provider: str, replayed_call_ids: Sequence[str]
|
||||
) -> None:
|
||||
request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
|
||||
model=model,
|
||||
input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts
|
||||
ResponseInputParam, list(_stateless_tool_search_input("srvtoolu_01Search", "completed"))
|
||||
),
|
||||
responses_api_request={
|
||||
"tools": [
|
||||
{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"},
|
||||
*_deferred_tools_with_server_tool_search()[1:],
|
||||
]
|
||||
},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
assert [call["id"] for call in request["messages"][1]["tool_calls"]] == replayed_call_ids
|
||||
|
||||
def test_transform_local_shell_tools_dropped(self) -> None:
|
||||
(
|
||||
result_tools,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue