scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(responses): replay Claude's tool search only on routes that run it

Browse source 
A follow-up sent to a route that cannot run tool search, a Claude model without it included, now leaves out the replayed search the way it did before, instead of sending Claude search blocks that model cannot take. The stateless gate and the stored-turn filter now use the same check that already decides whether the tool_search tool itself is kept

A streamed tool_search_call now gets its output_item.done when Claude's search result arrives, so it finishes before its tool_search_output. Clients that build history from done events, like Codex, kept the output ahead of its call
This commit is contained in:
mateo-berri 2026-11-06 14:18:30 -08:00
parent 5d56e180ca
commit 3e61760198
5 changed files with 59 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -205,6 +205,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self._tool_search = tool_search_routing(request_tools)
self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({})
self._tool_search_output_index_by_call_id: Mapping[str, int] = MappingProxyType({})
self._done_tool_search_call_ids: frozenset[str] = frozenset()
async def aclose(self) -> None:
await self.litellm_custom_stream_wrapper.aclose()
@ -245,19 +246,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def _queue_tool_search_call_done(
self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution
) -> None:
self._queue_tool_search_call_added(call_id, output_index, "", execution)
item: Final = build_tool_search_call(
call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution
)
self._sequence_number += 1
self._pending_tool_events.append(
OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=output_index,
sequence_number=self._sequence_number,
item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()),
if call_id not in self._done_tool_search_call_ids:
self._done_tool_search_call_ids = self._done_tool_search_call_ids | {call_id}
self._queue_tool_search_call_added(call_id, output_index, "", execution)
item: Final = build_tool_search_call(
call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution
)
self._sequence_number += 1
self._pending_tool_events.append(
OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=output_index,
sequence_number=self._sequence_number,
item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()),
)
)
)
self._queue_tool_search_output(call_id)
def _queue_arrived_tool_search_outputs(self) -> None:
@ -265,7 +268,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return
for call_id in tool_search_results_by_call_id((self._accumulated_provider_specific_fields,)):
if self._tool_search_executions.get(call_id) == "server":
self._queue_tool_search_output(call_id)
self._queue_tool_search_call_done(call_id, self._get_or_assign_tool_output_index(call_id), "", "server")
def _queue_tool_search_output(self, call_id: str) -> None:
if call_id in self._tool_search_output_index_by_call_id:

9
litellm/responses/litellm_completion_transformation/tool_search.py
View file

@ -41,6 +41,7 @@ __all__ = [
"is_server_tool_search_tool",
"provider_builds_anthropic_request",
"provider_chat_config",
"route_runs_tool_search",
"runs_tool_search_on_the_server",
"tool_search_call_ids",
"tool_search_routing",
@ -217,6 +218,14 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non
return chat_config_builds_anthropic_request(provider_chat_config(model, custom_llm_provider))
def route_runs_tool_search(model: str, custom_llm_provider: str | None) -> bool:
from litellm.utils import supports_tool_search
return supports_tool_search(
model=model, custom_llm_provider=custom_llm_provider
) and provider_builds_anthropic_request(model, custom_llm_provider)
def chat_config_builds_anthropic_request(chat_config: object) -> bool:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig

16
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -163,6 +163,7 @@ from .tool_search import (
is_anthropic_tool_search_chat_tool,
is_server_tool_search_tool,
provider_builds_anthropic_request,
route_runs_tool_search,
tool_search_call_ids,
tool_search_routing,
without_tool_search_calls,
@ -1015,16 +1016,11 @@ class LiteLLMCompletionResponsesConfig:
def _without_unsupported_tool_search(
tools: ResponseTools, model: str, custom_llm_provider: str | None
) -> ResponseTools:
from litellm.utils import supports_tool_search
if not has_server_tool_search(tools):
return tools
if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()):
return [tool for tool in tools or () if not is_server_tool_search_tool(tool)]
route_runs_tool_search: Final = supports_tool_search(
model=model, custom_llm_provider=custom_llm_provider
) and provider_builds_anthropic_request(model, custom_llm_provider)
if route_runs_tool_search:
if route_runs_tool_search(model, custom_llm_provider):
return tools
verbose_logger.warning(
"Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search "
@ -1195,7 +1191,7 @@ class LiteLLMCompletionResponsesConfig:
native_compaction=native_compaction,
only_since_compaction=not native_compaction,
replay_tool_search=holds_tool_search_items(request_input)
and provider_builds_anthropic_request(model, custom_llm_provider),
and route_runs_tool_search(model, custom_llm_provider),
)
litellm_completion_request: dict = {
@ -1329,10 +1325,10 @@ class LiteLLMCompletionResponsesConfig:
return messages
@staticmethod
def _without_tool_search_uses_off_anthropic(
def _without_tool_search_uses_the_route_cannot_run(
messages: Sequence[SessionMessage], route: CompletionRoute
) -> tuple[SessionMessage, ...]:
if provider_builds_anthropic_request(route.model, route.custom_llm_provider):
if route_runs_tool_search(route.model, route.custom_llm_provider):
return tuple(messages)
search_call_ids: Final = tool_search_call_ids(messages)
return tuple(
@ -1394,7 +1390,7 @@ class LiteLLMCompletionResponsesConfig:
request_fields.get("context_management"),
)
replayed_messages: Final = (
LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic(stored_messages, route)
LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_route_cannot_run(stored_messages, route)
if any(holds_tool_search_use(message) for message in stored_messages)
else stored_messages
)

20
tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py
View file

@ -490,9 +490,13 @@ def _history_tool_ids(messages: Sequence[object]) -> tuple[Sequence[str], Sequen
@pytest.mark.parametrize("search_answered", [False, True])
@pytest.mark.parametrize(
("model", "custom_llm_provider", "expect_search"),
[("mistral/mistral-small-latest", "mistral", False), (_MODEL, "anthropic", True)],
[
("mistral/mistral-small-latest", "mistral", False),
("anthropic/claude-3-7-sonnet-20250219", "anthropic", False),
(_MODEL, "anthropic", True),
],
)
async def test_previous_response_id_drops_a_client_replayed_search_call_only_off_anthropic(
async def test_previous_response_id_drops_a_client_replayed_search_call_only_where_it_cannot_run(
model: str, custom_llm_provider: str, expect_search: bool, search_answered: bool
) -> None:
"""
@ -1709,7 +1713,7 @@ def test_streamed_responses_over_anthropic_emit_the_tool_search_output_in_order_
1,
2,
]
assert sorted((event.output_index, event.item.id) for event in done) == [
assert [(event.output_index, event.item.id) for event in done] == [
(index, item.id) for index, item in enumerate(completed)
]
assert {event.item.id: event.output_index for event in done} == added
@ -1787,7 +1791,13 @@ async def test_previous_response_id_sends_back_a_search_the_stored_turn_got_as_i
@pytest.mark.asyncio
async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_items_off_anthropic() -> None:
messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, "mistral/mistral-small-latest", "mistral")
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[("mistral/mistral-small-latest", "mistral"), ("anthropic/claude-3-7-sonnet-20250219", "anthropic")],
)
async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_items_where_it_cannot_run(
model: str, custom_llm_provider: str
) -> None:
messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, model, custom_llm_provider)
assert _history_tool_ids(messages) == (["toolu_01Weather"], ["toolu_01Weather"])

16
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -1737,7 +1737,7 @@ def _chat_response_with_tool_search_result(result_content: Mapping[str, object])
def _stateless_tool_search_input(
call_id: str | None, output_status: Literal["completed", "incomplete"]
) -> list[dict[str, object]]:
) -> Sequence[Mapping[str, object]]:
return [
{"type": "message", "role": "user", "content": "What is the weather in Paris right now?"},
{
@ -2399,6 +2399,14 @@ class TestToolTransformation:
["toolu_01Weather"],
id="claude_search_off_anthropic",
),
pytest.param(
"anthropic/claude-3-7-sonnet-20250219",
"anthropic",
"srvtoolu_01Search",
"completed",
["toolu_01Weather"],
id="claude_search_sent_to_a_claude_without_tool_search",
),
],
)
def test_stateless_tool_search_items_are_replayed_only_for_a_completed_claude_search_on_anthropic(
@ -2407,11 +2415,13 @@ class TestToolTransformation:
custom_llm_provider: str,
call_id: str | None,
output_status: Literal["completed", "incomplete"],
replayed_call_ids: list[str],
replayed_call_ids: Sequence[str],
) -> None:
request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=cast(ResponseInputParam, _stateless_tool_search_input(call_id, output_status)),
input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts
ResponseInputParam, list(_stateless_tool_search_input(call_id, output_status))
),
responses_api_request={"tools": _deferred_tools_with_server_tool_search()},
custom_llm_provider=custom_llm_provider,
)