diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d02f953001e..b96ceb936ae 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2127,6 +2127,12 @@ def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozen ) +def _paused_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]: + if any(block.get("type") == "tool_use" for block in blocks): + return frozenset() + return _unrun_server_tool_use_ids(blocks) + + def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object: return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids) @@ -2140,7 +2146,7 @@ def _paused_turn_replay( tuple[_AnthropicToolInvokeItem, ...], tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...], ]: - unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes) + unrun_ids: Final = _paused_server_tool_use_ids(tool_invokes) if not unrun_ids: return tuple(tool_invokes), tuple(thinking_blocks) dropped_ids: Final = ( @@ -3226,7 +3232,7 @@ def anthropic_messages_pt( if assistant_tool_calls is not None else () ) - turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results)) + turn_paused = bool(_paused_server_tool_use_ids(tool_invoke_results)) replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay( tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1 ) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 393ff39b148..23e050eb896 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3825,7 +3825,7 @@ _FETCH_STILL_WAITING_ON_ITS_RESULT: Final = { @pytest.mark.parametrize("modify_params", [False, True]) -def test_server_call_still_waiting_on_its_result_is_never_replayed_as_a_client_call( +def test_server_call_still_waiting_on_its_result_is_replayed_as_a_server_call( modify_params: bool, monkeypatch: pytest.MonkeyPatch ): content: Final = [_FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL] @@ -3838,11 +3838,45 @@ def test_server_call_still_waiting_on_its_result_is_never_replayed_as_a_client_c {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, ) - assert replayed[1]["content"] == [_WEATHER_CALL] + assert replayed[1]["content"] == content assert [turn["role"] for turn in replayed] == ["user", "assistant", "user"] assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] +_FETCH_AND_WEATHER_THINKING: Final = { + "type": "thinking", + "thinking": "Fetch the page, then check the weather.", + "signature": "sig_fetch_weather", +} +_CHECKING_BOTH: Final = {"type": "text", "text": "Checking both."} + + +@pytest.mark.parametrize("thinking_inline", [False, True], ids=["thinking_blocks", "inline_thinking"]) +def test_tool_use_turn_waiting_on_a_server_call_keeps_its_thinking_and_both_calls(thinking_inline: bool): + parsed: Final = _parsed_assistant_message( + [_FETCH_AND_WEATHER_THINKING, _CHECKING_BOTH, _FETCH_STILL_WAITING_ON_ITS_RESULT, _WEATHER_CALL], "tool_use" + ).model_dump() + message: Final = ( + {**parsed, "thinking_blocks": None, "content": [_FETCH_AND_WEATHER_THINKING, _CHECKING_BOTH]} + if thinking_inline + else parsed + ) + + replayed: Final = _replayed( + {"role": "user", "content": "Fetch example.com, then tell me the weather in Paris"}, + message, + {"role": "tool", "tool_call_id": "toolu_01Weather", "content": "18C"}, + ) + + assistant_content: Final = replayed[1]["content"] + assert assistant_content[:2] == [_FETCH_AND_WEATHER_THINKING, _CHECKING_BOTH] + assert sorted(assistant_content[2:], key=lambda block: block["id"]) == [ + _FETCH_STILL_WAITING_ON_ITS_RESULT, + _WEATHER_CALL, + ] + assert [block["tool_use_id"] for block in replayed[2]["content"]] == ["toolu_01Weather"] + + _PARIS_TEXT: Final = "Paris is the capital of France." _PARIS_ANSWER: Final[Mapping[str, str]] = {"type": "text", "text": _PARIS_TEXT} diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index f3dce45e3b0..f48723aec12 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -5252,17 +5252,24 @@ class TestHostedWebSearchReplay: "leading_text", ((), ({"type": "text", "text": "Let me look that up."},)), ids=("alone", "after_text") ) @pytest.mark.parametrize( - "server_tool_use", + ("server_tool_use", "expected_input"), ( - {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, - {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_fetch", "input": {"url": _FETCHED_URL}}, + {"url": _FETCHED_URL}, + ), + ( + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": "web_search", "input": {"query": "python"}}, + {"query": "python"}, + ), ), ids=("web_fetch", "web_search"), ) - def test_pending_claude_web_call_is_never_replayed_to_anthropic_as_a_client_tool_use( + def test_pending_claude_web_call_replays_to_anthropic_as_its_own_server_tool_use( self, leading_text: tuple[Mapping[str, object], ...], server_tool_use: Mapping[str, object], + expected_input: Mapping[str, object], ) -> None: response: Final = _anthropic_turn_on_responses((*leading_text, server_tool_use, _WEATHER_TOOL_USE)) history: Final = [ @@ -5281,6 +5288,7 @@ class TestHostedWebSearchReplay: assert [turn["role"] for turn in anthropic_turns] == ["user", "assistant", "user"] assert [block for block in anthropic_turns[1]["content"] if block["type"] != "text"] == [ + {"type": "server_tool_use", "id": "srvtoolu_01Pending", "name": server_tool_use["name"], "input": expected_input}, {"type": "tool_use", "id": "toolu_01Weather", "name": "get_weather", "input": {"city": "Paris"}}, ] assert [(block["type"], block["tool_use_id"]) for block in anthropic_turns[2]["content"]] == [