diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e500c5f426f..80ab0ca5a9f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2029,6 +2029,14 @@ def _assistant_tool_calls_with_server_tool_calls( ) +def _server_tool_call_ids(message: object) -> frozenset[str]: + from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params + + return frozenset( + tool_call["id"] for tool_call in anthropic_server_tool_call_params(message) if tool_call["id"] is not None + ) + + def _server_tool_answers_replaced_after(messages: Sequence[object], index: int) -> frozenset[str]: kept_server_result_ids: Final = _tool_call_ids_with_a_kept_server_result(messages[index]) if not kept_server_result_ids: @@ -2092,6 +2100,7 @@ def convert_to_anthropic_tool_invoke( web_search_results: Sequence[object] | None = None, tool_results: Sequence[object] | None = None, answered_tool_ids: Container[str] = frozenset(), + server_tool_call_ids: Container[str] = frozenset(), ) -> list[_AnthropicToolInvokeItem]: """ OpenAI tool invokes: @@ -2169,6 +2178,10 @@ def convert_to_anthropic_tool_invoke( anthropic_tool_invoke.append(server_tool_result) elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids: continue + elif tool_id in server_tool_call_ids and tool_id not in answered_tool_ids: + anthropic_tool_invoke.append( + {"type": "server_tool_use", "id": tool_id, "name": tool_name, "input": tool_input} + ) else: sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id) _anthropic_tool_use_param = AnthropicMessagesToolUseParam( @@ -3161,6 +3174,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results_tc, tool_results=_tool_results_tc, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Group tool invoke results into (server_tool_use, result) pairs @@ -3326,6 +3340,7 @@ def anthropic_messages_pt( web_search_results=_web_search_results, tool_results=_tool_results, answered_tool_ids=client_answered_tool_ids, + server_tool_call_ids=_server_tool_call_ids(assistant_content_block), ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 93c01ec6ae0..e1caa7c4b4a 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -1128,6 +1128,15 @@ class ModelResponseIterator: except (json.JSONDecodeError, TypeError): pass server_tool_input: Final = self._server_tool_inputs.get(self._current_server_tool_id, {}) + if self._current_server_tool_id in self._web_search_calls: + self._web_search_calls[self._current_server_tool_id] = build_web_search_call( + self._current_server_tool_id, + server_tool_input, + {"content": []}, + status="in_progress", + tool_name=self._current_server_tool_name, + ) + provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot() self._server_tool_calls = ( *self._server_tool_calls, ChatCompletionToolCallChunk( diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..e924934637a 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -45,6 +45,25 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, (list, tuple)) else None + + +def _server_tool_blocks(message: object) -> tuple[Mapping[str, object], ...]: + server_tool_calls: Final = anthropic_server_tool_call_params(message) + if not server_tool_calls: + return () + fields: Final = _as_string_mapping(_optional_attr(message, "provider_specific_fields")) or {} + return tuple( + convert_to_anthropic_tool_invoke( + server_tool_calls, + web_search_results=_object_sequence(fields.get("web_search_results")), + tool_results=_object_sequence(fields.get("tool_results")), + server_tool_call_ids=frozenset(tool_call["id"] for tool_call in server_tool_calls), + ) + ) + + def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: fields: Final = _as_string_mapping(provider_specific_fields) raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None @@ -129,11 +148,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, + convert_to_anthropic_tool_invoke, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) from litellm.llms.anthropic.common_utils import ( + anthropic_server_tool_call_params, eager_input_streaming_flag, is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, @@ -1410,6 +1431,7 @@ class LiteLLMAnthropicMessagesAdapter: ).model_dump() ) + new_content.extend(dict(block) for block in _server_tool_blocks(choice.message)) # Handle text content if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index fa92a0c90e0..541e49c6751 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -19,6 +19,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.llms.anthropic.common_utils import anthropic_server_tool_call_params, anthropic_server_tool_calls from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( add_guardrail_to_applied_guardrails_header, @@ -515,7 +516,7 @@ class ToolPermissionGuardrail(CustomGuardrail): for choice_index, choice in enumerate(response.choices): if isinstance(choice, Choices): - for tool in choice.message.tool_calls or []: + for tool in (*(choice.message.tool_calls or ()), *anthropic_server_tool_calls(choice.message)): tool_calls.append(tool) legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index @@ -784,6 +785,23 @@ class ToolPermissionGuardrail(CustomGuardrail): choice.message.tool_calls = filtered_tool_calls if filtered_tool_calls else None + if (server_tool_calls := anthropic_server_tool_call_params(choice.message)) and any( + server_tool_call["id"] in error_results for server_tool_call in server_tool_calls + ): + error_messages.extend( + error_results[server_tool_call["id"]].content + for server_tool_call in server_tool_calls + if server_tool_call["id"] in error_results + ) + choice.message.provider_specific_fields = { + **(choice.message.provider_specific_fields or {}), + "server_tool_calls": [ + server_tool_call + for server_tool_call in server_tool_calls + if server_tool_call["id"] not in error_results + ], + } + legacy_tool_call = self._legacy_function_call_to_tool_call( getattr(choice.message, "function_call", None), choice_index ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c9b398b054a..6d0cbec8929 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -471,16 +471,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: message: Final = litellm_complete_object.choices[0].message - tool_calls = [ - *anthropic_server_tool_calls(message), - *(getattr(message, "tool_calls", None) or ()), - ] + server_tool_calls: Final = anthropic_server_tool_calls(message) + tool_calls: Final = [*server_tool_calls, *(getattr(message, "tool_calls", None) or ())] except Exception: - tool_calls = None + return if not tool_calls: return + server_tool_call_ids: Final = frozenset(tool_call.id for tool_call in server_tool_calls) + for tc in tool_calls: call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None) if not call_id_raw: @@ -506,6 +506,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._is_tool_search_call(call_id, fn_name): self._queue_tool_search_call_done(call_id, output_index, fn_args) continue + if call_id in server_tool_call_ids: + continue # Track if this is a new tool call that wasn't streamed is_new_tool_call = call_id not in self._tool_item_id_by_call_id @@ -580,10 +582,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ), ) + finished_events: Final = ( + () + if item.status == "in_progress" + else ((ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),) + ) for event_type, event_class in ( (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent), (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent), - (ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent), + *finished_events, ): self._sequence_number += 1 event = event_class(type=event_type, output_index=output_index, item_id=item.id) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 03a1411bba1..7c6d99b9253 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3370,7 +3370,22 @@ class LiteLLMCompletionResponsesConfig: for item in responses_output ] - return responses_output + server_tool_call_ids: Final = LiteLLMCompletionResponsesConfig._server_tool_call_ids(choices) + return [ + item + for item in responses_output + if not ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in server_tool_call_ids + ) + ] + + @staticmethod + def _server_tool_call_ids(choices: Sequence[Choices]) -> frozenset[str]: + server_tool_calls: Final = chain.from_iterable( + anthropic_server_tool_calls(choice.message) for choice in choices + ) + return frozenset(tool_call.id for tool_call in server_tool_calls) @staticmethod def _extract_tool_result_output_items( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index e274a998533..255162b3cd9 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3735,6 +3735,33 @@ def test_server_tool_result_round_trips_right_after_its_call( assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] +_FETCH_STILL_WAITING_ON_ITS_RESULT: Final = { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, +} + + +@pytest.mark.parametrize("modify_params", [False, True]) +def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call( + modify_params: bool, monkeypatch: pytest.MonkeyPatch +): + content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL] + message: Final = _parsed_assistant_message(content, "tool_use") + monkeypatch.setattr(litellm, "modify_params", modify_params) + + replayed: Final = _replayed( + {"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"}, + message.model_dump(), + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ) + + assert replayed[1]["content"] == content + assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] + assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] + + _PARIS_TEXT: Final = "Paris is the capital of France." _PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT} diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 4e4b89b764b..ed7e9c8772b 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -80,6 +80,59 @@ def test_native_compaction_response_roundtrip(text: str | None, count: int, expe assert result["stop_reason"] == expected_stop +_SERVER_AND_CLIENT_TOOL_TURN: Final[tuple[dict[str, object], ...]] = ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Fetch", + "name": "web_fetch", + "input": {"url": "https://example.com/"}, + }, + { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + {"type": "server_tool_use", "id": "srvtoolu_02Search", "name": "web_search", "input": {"query": "Paris"}}, + {"type": "text", "text": "The page is titled Example."}, + {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, +) + + +def test_claude_server_tool_blocks_survive_the_round_trip_through_chat_completions() -> None: + from unittest.mock import MagicMock + + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + anthropic_response: Final = { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(_SERVER_AND_CLIENT_TOOL_TURN), + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + raw_response: Final = MagicMock(spec=httpx.Response) + raw_response.json.return_value = anthropic_response + raw_response.status_code = 200 + raw_response.headers = {} + chat_response: Final = AnthropicConfig().transform_parsed_response( + completion_response=anthropic_response, + raw_response=raw_response, + model_response=ModelResponse(), + json_mode=False, + prefix_prompt=None, + ) + + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(chat_response) + + assert [(block["type"], block.get("id") or block.get("tool_use_id")) for block in result["content"]] == [ + (block["type"], block.get("id") or block.get("tool_use_id")) for block in _SERVER_AND_CLIENT_TOOL_TURN + ] + assert result["stop_reason"] == "tool_use" + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 2ee4c8c72c0..1fac7e7a4c9 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -27,11 +27,21 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( from litellm.types.utils import ( ChatCompletionMessageToolCall, Choices, + Delta, ModelResponse, ModelResponseStream, + StreamingChoices, ) +_SERVER_WEB_SEARCH_CALL: Final = { + "id": "srvtoolu_01Search", + "type": "function", + "function": {"name": "web_search", "arguments": '{"query": "weather in Paris"}'}, +} +_ALLOWED_BASH_CALL: Final = {"id": "toolu_01Bash", "type": "function", "function": {"name": "Bash", "arguments": "{}"}} + + class TestToolPermissionGuardrail: """Test class for Tool Permission Guardrail functionality""" @@ -952,6 +962,71 @@ class TestToolPermissionGuardrail: assert "Permission denied" in choice.message.content + def _server_search_response(self) -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message={ + "content": "Searching.", + "tool_calls": [_ALLOWED_BASH_CALL], + "provider_specific_fields": {"server_tool_calls": [_SERVER_WEB_SEARCH_CALL]}, + }, + ) + ] + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_is_blocked(self): + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + await self.guardrail.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + @pytest.mark.asyncio + async def test_denied_server_tool_call_in_a_chat_stream_is_blocked(self): + async def _stream(): + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + content="Searching.", + provider_specific_fields={"server_tool_calls": [{**_SERVER_WEB_SEARCH_CALL, "index": 0}]}, + ), + finish_reason="stop", + ) + ] + ) + + with patch.object(self.guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(GuardrailRaisedException): + async for _ in self.guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={} + ): + pass + + @pytest.mark.asyncio + async def test_rewrite_mode_removes_the_denied_server_tool_call(self): + rewriting: Final = ToolPermissionGuardrail( + guardrail_name="test-tool-permission", + rules=self.test_rules, + default_action="deny", + on_disallowed_action="rewrite", + ) + + with patch.object(rewriting, "should_run_guardrail", return_value=True): + result: Final = await rewriting.async_post_call_success_hook( + data={}, user_api_key_dict=UserAPIKeyAuth(), response=self._server_search_response() + ) + + message: Final = result.choices[0].message + assert message.provider_specific_fields["server_tool_calls"] == [] + assert [tool_call.id for tool_call in message.tool_calls] == [_ALLOWED_BASH_CALL["id"]] + assert "Permission denied" in message.content + + class TestToolPermissionGuardrailIntegration: """Integration tests for Tool Permission Guardrail""" diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2c827588668..618a33339bf 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5064,6 +5064,47 @@ def test_anthropic_server_web_call_without_a_result_is_an_in_progress_hosted_cal assert [item.name for item in response.output if item.type == "function_call"] == ["get_weather"] +def test_anthropic_code_execution_is_never_a_client_function_call_on_responses() -> None: + response: Final = _anthropic_turn_on_responses( + ( + { + "type": "server_tool_use", + "id": "srvtoolu_01Bash", + "name": "bash_code_execution", + "input": {"command": "ls"}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01Bash", + "content": { + "type": "bash_code_execution_result", + "stdout": "notes.txt\n", + "stderr": "", + "return_code": 0, + }, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_02Edit", + "name": "text_editor_code_execution", + "input": {"command": "view", "path": "notes.txt"}, + }, + { + "type": "server_tool_use", + "id": "srvtoolu_03Bash", + "name": "bash_code_execution", + "input": {"command": "pwd"}, + }, + _WEATHER_TOOL_USE, + ) + ) + + assert [(item.type, getattr(item, "call_id", item.id)) for item in response.output if item.type != "message"] == [ + ("code_interpreter_call", "srvtoolu_01Bash"), + ("function_call", "toolu_01Weather"), + ] + + class TestHostedWebSearchReplay: def test_emitted_hosted_search_output_round_trips_with_client_tool_result(self) -> None: search_result: Final = { diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index db7c54c52ca..f1f6597130b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -355,53 +355,59 @@ async def test_web_search_stream_preserves_hosted_and_client_calls(sync_mode, to ) -def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]: +_STREAMED_WEB_FETCH_CALL: Final[tuple[dict[str, object], ...]] = ( + { + "type": "message_start", + "message": { + "id": "msg_01Fetch", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, + }, + {"type": "content_block_stop", "index": 0}, +) +_STREAMED_WEB_FETCH_RESULT: Final[tuple[dict[str, object], ...]] = ( + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_01Fetch", + "content": {"type": "web_fetch_result", "url": "https://example.com/"}, + }, + }, + {"type": "content_block_stop", "index": 1}, +) + + +def _streamed_text_then_weather_call(first_index: int) -> tuple[dict[str, object], ...]: return ( - { - "type": "message_start", - "message": { - "id": "msg_01Fetch", - "type": "message", - "role": "assistant", - "content": [], - "usage": {"input_tokens": 10, "output_tokens": 1}, - }, - }, + {"type": "content_block_start", "index": first_index, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": first_index, "delta": {"type": "text_delta", "text": "Fetched."}}, + {"type": "content_block_stop", "index": first_index}, { "type": "content_block_start", - "index": 0, - "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Fetch", "name": "web_fetch"}, - }, - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com/"}'}, - }, - {"type": "content_block_stop", "index": 0}, - { - "type": "content_block_start", - "index": 1, - "content_block": { - "type": "web_fetch_tool_result", - "tool_use_id": "srvtoolu_01Fetch", - "content": {"type": "web_fetch_result", "url": "https://example.com/"}, - }, - }, - {"type": "content_block_stop", "index": 1}, - {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, - {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Fetched."}}, - {"type": "content_block_stop", "index": 2}, - { - "type": "content_block_start", - "index": 3, + "index": first_index + 1, "content_block": {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {}}, }, { "type": "content_block_delta", - "index": 3, + "index": first_index + 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Paris"}'}, }, - {"type": "content_block_stop", "index": 3}, + {"type": "content_block_stop", "index": first_index + 1}, { "type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": None}, @@ -410,13 +416,52 @@ def _anthropic_web_fetch_stream_events() -> tuple[dict[str, object], ...]: ) +def _anthropic_web_fetch_stream_events(fetch_finished: bool) -> tuple[dict[str, object], ...]: + if fetch_finished: + return (*_STREAMED_WEB_FETCH_CALL, *_STREAMED_WEB_FETCH_RESULT, *_streamed_text_then_weather_call(2)) + return (*_STREAMED_WEB_FETCH_CALL, *_streamed_text_then_weather_call(1)) + + @pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + ("fetch_finished", "expected_status", "expected_search_events"), + [ + pytest.param( + True, + "completed", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_finished_in_this_turn", + ), + pytest.param( + False, + "in_progress", + [ + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, + ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ], + id="fetch_still_waiting_on_its_result", + ), + ], +) @pytest.mark.asyncio -async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode: bool) -> None: +async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call( + sync_mode: bool, + fetch_finished: bool, + expected_status: str, + expected_search_events: list[ResponsesAPIStreamEvents], +) -> None: from litellm.llms.anthropic.chat.handler import ModelResponseIterator parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) - chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events()] + chunks: Final = [parser.chunk_parser(event) for event in _anthropic_web_fetch_stream_events(fetch_finished)] iterator: Final = LiteLLMCompletionStreamingIterator( model="claude-sonnet-5-5", litellm_custom_stream_wrapper=_FakeStreamWrapper(chunks), @@ -439,7 +484,12 @@ async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode (item.id, item.status, item.action.model_dump()) for item in completed.response.output if item.type == "web_search_call" - ] == [("ws_srvtoolu_01Fetch", "completed", {"type": "open_page", "url": "https://example.com/"})] + ] == [("ws_srvtoolu_01Fetch", expected_status, {"type": "open_page", "url": "https://example.com/"})] + assert [ + event.type + for event in events + if getattr(event, "item_id", getattr(getattr(event, "item", None), "id", None)) == "ws_srvtoolu_01Fetch" + ] == expected_search_events assert [item.name for item in completed.response.output if item.type == "function_call"] == ["get_weather"] assert not any( event.item.name == "web_fetch" @@ -449,6 +499,54 @@ async def test_streamed_anthropic_web_fetch_is_a_hosted_open_page_call(sync_mode ) +_STREAMED_PENDING_CODE_EXECUTION: Final[tuple[dict[str, object], ...]] = ( + _STREAMED_WEB_FETCH_CALL[0], + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_01Bash", "name": "bash_code_execution"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"command": "ls"}'}, + }, + {"type": "content_block_stop", "index": 0}, + *_streamed_text_then_weather_call(1), +) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_streamed_anthropic_code_execution_is_never_a_client_function_call(sync_mode: bool) -> None: + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + parser: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=sync_mode, json_mode=False) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-sonnet-5-5", + litellm_custom_stream_wrapper=_FakeStreamWrapper( + [parser.chunk_parser(event) for event in _STREAMED_PENDING_CODE_EXECUTION] + ), + request_input="List the files, then check the weather in Paris.", + responses_api_request={ + "tools": [ + {"type": "code_execution_20250825", "name": "code_execution"}, + {"type": "function", "name": "get_weather", "parameters": {"type": "object"}}, + ] + }, + custom_llm_provider="anthropic", + ) + + events: Final = await _collect_events(iterator, sync_mode) + + item_events: Final = [event for event in events if getattr(event, "item", None) is not None] + assert {event.item.call_id for event in item_events if event.item.type == "function_call"} == {"toolu_01Weather"} + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.call_id for item in completed.response.output if item.type == "function_call"] == ["toolu_01Weather"] + + def test_tool_calls_present_only_in_final_response_are_emitted_before_completed(): iterator = LiteLLMCompletionStreamingIterator( model="test-model",