diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index f865d3adc34..566bff3f60f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -205,6 +205,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_search = tool_search_routing(request_tools) self._tool_search_executions: Mapping[str, ToolSearchExecution] = MappingProxyType({}) self._tool_search_output_index_by_call_id: Mapping[str, int] = MappingProxyType({}) + self._done_tool_search_call_ids: frozenset[str] = frozenset() async def aclose(self) -> None: await self.litellm_custom_stream_wrapper.aclose() @@ -245,19 +246,21 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _queue_tool_search_call_done( self, call_id: str, output_index: int, final_args: str, execution: ToolSearchExecution ) -> None: - self._queue_tool_search_call_added(call_id, output_index, "", execution) - item: Final = build_tool_search_call( - call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution - ) - self._sequence_number += 1 - self._pending_tool_events.append( - OutputItemDoneEvent( - type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, - output_index=output_index, - sequence_number=self._sequence_number, - item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()), + if call_id not in self._done_tool_search_call_ids: + self._done_tool_search_call_ids = self._done_tool_search_call_ids | {call_id} + self._queue_tool_search_call_added(call_id, output_index, "", execution) + item: Final = build_tool_search_call( + call_id, final_args or self._tool_args_by_call_id.get(call_id, ""), "completed", execution + ) + self._sequence_number += 1 + self._pending_tool_events.append( + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=output_index, + sequence_number=self._sequence_number, + item=BaseLiteLLMOpenAIResponseObject(**item.model_dump()), + ) ) - ) self._queue_tool_search_output(call_id) def _queue_arrived_tool_search_outputs(self) -> None: @@ -265,7 +268,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return for call_id in tool_search_results_by_call_id((self._accumulated_provider_specific_fields,)): if self._tool_search_executions.get(call_id) == "server": - self._queue_tool_search_output(call_id) + self._queue_tool_search_call_done(call_id, self._get_or_assign_tool_output_index(call_id), "", "server") def _queue_tool_search_output(self, call_id: str) -> None: if call_id in self._tool_search_output_index_by_call_id: diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 5a1b27f9092..dfec900042c 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -41,6 +41,7 @@ __all__ = [ "is_server_tool_search_tool", "provider_builds_anthropic_request", "provider_chat_config", + "route_runs_tool_search", "runs_tool_search_on_the_server", "tool_search_call_ids", "tool_search_routing", @@ -217,6 +218,14 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non return chat_config_builds_anthropic_request(provider_chat_config(model, custom_llm_provider)) +def route_runs_tool_search(model: str, custom_llm_provider: str | None) -> bool: + from litellm.utils import supports_tool_search + + return supports_tool_search( + model=model, custom_llm_provider=custom_llm_provider + ) and provider_builds_anthropic_request(model, custom_llm_provider) + + def chat_config_builds_anthropic_request(chat_config: object) -> bool: from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index beeecf8eef6..9791c809827 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -163,6 +163,7 @@ from .tool_search import ( is_anthropic_tool_search_chat_tool, is_server_tool_search_tool, provider_builds_anthropic_request, + route_runs_tool_search, tool_search_call_ids, tool_search_routing, without_tool_search_calls, @@ -1015,16 +1016,11 @@ class LiteLLMCompletionResponsesConfig: def _without_unsupported_tool_search( tools: ResponseTools, model: str, custom_llm_provider: str | None ) -> ResponseTools: - from litellm.utils import supports_tool_search - if not has_server_tool_search(tools): return tools if any(is_anthropic_tool_search_chat_tool(tool) for tool in tools or ()): return [tool for tool in tools or () if not is_server_tool_search_tool(tool)] - route_runs_tool_search: Final = supports_tool_search( - model=model, custom_llm_provider=custom_llm_provider - ) and provider_builds_anthropic_request(model, custom_llm_provider) - if route_runs_tool_search: + if route_runs_tool_search(model, custom_llm_provider): return tools verbose_logger.warning( "Dropping Responses API tool of type 'tool_search': model '%s' (provider '%s') cannot run tool search " @@ -1195,7 +1191,7 @@ class LiteLLMCompletionResponsesConfig: native_compaction=native_compaction, only_since_compaction=not native_compaction, replay_tool_search=holds_tool_search_items(request_input) - and provider_builds_anthropic_request(model, custom_llm_provider), + and route_runs_tool_search(model, custom_llm_provider), ) litellm_completion_request: dict = { @@ -1329,10 +1325,10 @@ class LiteLLMCompletionResponsesConfig: return messages @staticmethod - def _without_tool_search_uses_off_anthropic( + def _without_tool_search_uses_the_route_cannot_run( messages: Sequence[SessionMessage], route: CompletionRoute ) -> tuple[SessionMessage, ...]: - if provider_builds_anthropic_request(route.model, route.custom_llm_provider): + if route_runs_tool_search(route.model, route.custom_llm_provider): return tuple(messages) search_call_ids: Final = tool_search_call_ids(messages) return tuple( @@ -1394,7 +1390,7 @@ class LiteLLMCompletionResponsesConfig: request_fields.get("context_management"), ) replayed_messages: Final = ( - LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic(stored_messages, route) + LiteLLMCompletionResponsesConfig._without_tool_search_uses_the_route_cannot_run(stored_messages, route) if any(holds_tool_search_use(message) for message in stored_messages) else stored_messages ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 44132c7a56a..e0d77a230b1 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -490,9 +490,13 @@ def _history_tool_ids(messages: Sequence[object]) -> tuple[Sequence[str], Sequen @pytest.mark.parametrize("search_answered", [False, True]) @pytest.mark.parametrize( ("model", "custom_llm_provider", "expect_search"), - [("mistral/mistral-small-latest", "mistral", False), (_MODEL, "anthropic", True)], + [ + ("mistral/mistral-small-latest", "mistral", False), + ("anthropic/claude-3-7-sonnet-20250219", "anthropic", False), + (_MODEL, "anthropic", True), + ], ) -async def test_previous_response_id_drops_a_client_replayed_search_call_only_off_anthropic( +async def test_previous_response_id_drops_a_client_replayed_search_call_only_where_it_cannot_run( model: str, custom_llm_provider: str, expect_search: bool, search_answered: bool ) -> None: """ @@ -1709,7 +1713,7 @@ def test_streamed_responses_over_anthropic_emit_the_tool_search_output_in_order_ 1, 2, ] - assert sorted((event.output_index, event.item.id) for event in done) == [ + assert [(event.output_index, event.item.id) for event in done] == [ (index, item.id) for index, item in enumerate(completed) ] assert {event.item.id: event.output_index for event in done} == added @@ -1787,7 +1791,13 @@ async def test_previous_response_id_sends_back_a_search_the_stored_turn_got_as_i @pytest.mark.asyncio -async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_items_off_anthropic() -> None: - messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, "mistral/mistral-small-latest", "mistral") +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [("mistral/mistral-small-latest", "mistral"), ("anthropic/claude-3-7-sonnet-20250219", "anthropic")], +) +async def test_previous_response_id_drops_a_search_the_stored_turn_got_as_input_items_where_it_cannot_run( + model: str, custom_llm_provider: str +) -> None: + messages: Final = await _follow_up_messages_after(_STATELESS_SEARCH_TURN, model, custom_llm_provider) assert _history_tool_ids(messages) == (["toolu_01Weather"], ["toolu_01Weather"]) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 76e177c8b05..e72dadb985a 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1737,7 +1737,7 @@ def _chat_response_with_tool_search_result(result_content: Mapping[str, object]) def _stateless_tool_search_input( call_id: str | None, output_status: Literal["completed", "incomplete"] -) -> list[dict[str, object]]: +) -> Sequence[Mapping[str, object]]: return [ {"type": "message", "role": "user", "content": "What is the weather in Paris right now?"}, { @@ -2399,6 +2399,14 @@ class TestToolTransformation: ["toolu_01Weather"], id="claude_search_off_anthropic", ), + pytest.param( + "anthropic/claude-3-7-sonnet-20250219", + "anthropic", + "srvtoolu_01Search", + "completed", + ["toolu_01Weather"], + id="claude_search_sent_to_a_claude_without_tool_search", + ), ], ) def test_stateless_tool_search_items_are_replayed_only_for_a_completed_claude_search_on_anthropic( @@ -2407,11 +2415,13 @@ class TestToolTransformation: custom_llm_provider: str, call_id: str | None, output_status: Literal["completed", "incomplete"], - replayed_call_ids: list[str], + replayed_call_ids: Sequence[str], ) -> None: request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, - input=cast(ResponseInputParam, _stateless_tool_search_input(call_id, output_status)), + input=cast( # cast-ok: tool search items are not in the SDK's input TypedDicts + ResponseInputParam, list(_stateless_tool_search_input(call_id, output_status)) + ), responses_api_request={"tools": _deferred_tools_with_server_tool_search()}, custom_llm_provider=custom_llm_provider, )