From fdea5713418960d96ff89824685b54b85e6c651d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:05:04 -0700 Subject: [PATCH] fix(router): let Responses fallbacks continue under allowed_tools auto and after server-run tool calls A Responses tool_choice of allowed_tools in auto mode no longer blocks the continuation, since the model may still answer in text. Hosted MCP calls and the bridge's server-run tool search no longer count as a started client tool call. --- .../streaming_iterator.py | 2 +- litellm/responses/streaming_iterator.py | 14 +- litellm/router.py | 3 +- .../continuation_prefill_check.py | 26 +++- .../base_llm/responses/test_transformation.py | 130 +++++++++++++++++- .../test_continuation_prefill_check.py | 32 +++++ 6 files changed, 190 insertions(+), 17 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 87e991f0c5a..219944de521 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -408,7 +408,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): continue if call_id in self._web_search_calls: continue - self._started_tool_call = True fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) fn_name = "" @@ -424,6 +423,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._is_tool_search_call(call_id, fn_name): self._queue_tool_search_call_added(call_id, output_index, fn_args_delta) continue + self._started_tool_call = True if call_id not in self._tool_args_by_call_id: self._tool_args_by_call_id[call_id] = "" diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index ec5dbc09069..c1f422c7d7b 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -486,17 +486,18 @@ class BaseResponsesAPIStreamingIterator: if isinstance(_delta, str): self._generated_content += _delta elif _event_type in _TOOL_ARGUMENTS_DELTA_EVENTS: - self._started_tool_call = True _args_delta: Final = getattr(openai_responses_api_chunk, "delta", None) if isinstance(_args_delta, str): self._generated_tool_arguments += _args_delta + if _event_type in _CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS: + self._started_tool_call = True _stream_model_id: Final = _model_id_from_metadata(self.litellm_metadata) if _event_type in ( ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, ): _item: Final[object] = getattr(openai_responses_api_chunk, "item", None) - if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES: + if getattr(_item, "type", None) in _CLIENT_RUN_TOOL_CALL_ITEM_TYPES: self._started_tool_call = True if _item is not None: ResponsesAPIRequestUtils.encode_container_id_on_output_item( @@ -1699,11 +1700,10 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None: return None -_TOOL_CALL_ITEM_TYPES: Final = frozenset( +_CLIENT_RUN_TOOL_CALL_ITEM_TYPES: Final = frozenset( { "function_call", "custom_tool_call", - "mcp_call", "mcp_approval_request", "computer_call", "local_shell_call", @@ -1711,13 +1711,15 @@ _TOOL_CALL_ITEM_TYPES: Final = frozenset( "apply_patch_call", } ) -_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset( +_CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset( { ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA, - ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA, } ) +_TOOL_ARGUMENTS_DELTA_EVENTS: Final = _CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS | { + ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA +} def _estimate_usage_from_text( diff --git a/litellm/router.py b/litellm/router.py index 6b0aca9d9ec..30448f2feff 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -310,6 +310,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( ResponsesContinuation, ResponsesContinuationSeam, constrains_continuation, + constrains_responses_continuation, continuation_deployments, continuation_output_ceilings, continued_text, @@ -4183,7 +4184,7 @@ class Router: cut_deployment: Final = ( self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None ) - if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): + if constrains_responses_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): Router._raise_original_mid_stream_error(e) earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG) if isinstance(earlier, ResponsesContinuation): diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 9abad288cad..d2a46fc316d 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -2,7 +2,7 @@ the output unconstrained, and shapes a Responses API continuation for the deployment that serves it.""" import json -from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping, Sequence from dataclasses import dataclass from itertools import accumulate from os.path import commonprefix @@ -57,23 +57,39 @@ def deployment_supports_prefill(deployment: object) -> bool: return model is not None and supports_assistant_prefill(model=model) -def constrains_continuation(params: Mapping[str, object]) -> bool: +def _forces_tool_call(tool_choice: object) -> bool: + return tool_choice == "required" or isinstance(tool_choice, Mapping) + + +def _forces_responses_tool_call(tool_choice: object) -> bool: + choice: Final = _str_keyed(tool_choice) + leaves_text_open: Final = choice.get("type") == "allowed_tools" and choice.get("mode") == "auto" + return _forces_tool_call(tool_choice) and not leaves_text_open + + +def _constrains(params: Mapping[str, object], forces_tool_call: Callable[[object], bool]) -> bool: sent: Final = {**params, **_str_keyed(params.get("extra_body"))} choice_count: Final = sent.get("n") response_format: Final = sent.get("response_format") text_format: Final = _str_keyed(sent.get("text")).get("format") - tool_choice: Final = sent.get("tool_choice") return ( (isinstance(choice_count, int) and choice_count > 1) or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT) or sent.get("text_format") is not None - or tool_choice == "required" - or isinstance(tool_choice, Mapping) + or forces_tool_call(sent.get("tool_choice")) or sent.get("merge_reasoning_content_in_choices") is True ) +def constrains_continuation(params: Mapping[str, object]) -> bool: + return _constrains(params, _forces_tool_call) + + +def constrains_responses_continuation(params: Mapping[str, object]) -> bool: + return _constrains(params, _forces_responses_tool_call) + + def deployment_can_continue(deployment: object) -> bool: litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")) return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params) diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 8ef420219ae..50d918fa5be 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -1599,7 +1599,7 @@ def _openai_stream_cut_mid_tool_call(preamble: str, *, announce_item: bool = Tru return _sse(*((text, item_added, arguments, error) if announce_item else (text, arguments, error))) -def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response: +def _anthropic_stream_cut_mid_tool_call(preamble: str, tool_block: Mapping[str, object] | None = None) -> httpx.Response: return _sse( {"type": "message_start", "message": { "id": "msg_primary", "type": "message", "role": "assistant", @@ -1608,7 +1608,7 @@ def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response: {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}}, {"type": "content_block_stop", "index": 0}, - {"type": "content_block_start", "index": 1, "content_block": { + {"type": "content_block_start", "index": 1, "content_block": tool_block or { "type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {}, }}, {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}}, @@ -1643,13 +1643,13 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream router.discard() -def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response: +def _openai_stream_cut_after_event(preamble: str, event: Mapping[str, object]) -> httpx.Response: return _sse( { "type": "response.output_text.delta", "sequence_number": 0, "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble, }, - {"type": "response.output_item.added", "sequence_number": 1, "output_index": 1, "item": item}, + {**event, "sequence_number": 1}, { "type": "error", "sequence_number": 2, "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, @@ -1657,6 +1657,12 @@ def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, objec ) +def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response: + return _openai_stream_cut_after_event( + preamble, {"type": "response.output_item.added", "output_index": 1, "item": item} + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "item", @@ -1698,6 +1704,83 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream router.discard() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "server_run_event", + [ + { + "type": "response.output_item.added", "output_index": 1, + "item": { + "type": "mcp_call", "id": "mcp_1", "name": "ask_question", "arguments": "{}", + "server_label": "docs", "status": "in_progress", + }, + }, + {"type": "response.mcp_call_arguments.delta", "output_index": 1, "item_id": "mcp_1", "delta": '{"question": "set'}, + ], + ids=["mcp_call_item", "mcp_call_arguments"], +) +async def test_responses_fallback_continues_after_a_server_run_mcp_call( + server_run_event: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_after_event("Let me check the docs. ", server_run_event) + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("The docs cover setup.") + ) + router: Final = _primary_with_fallback("anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses(model="primary", input="What do the docs cover?", stream=True) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events).endswith("The docs cover setup.") + assert fallback.call_count == 1 + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_continues_when_allowed_tools_in_auto_mode_leaves_the_reply_open( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + openai: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[ + _openai_stream_cut_after("1, 2, "), + _sse( + { + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_fallback", "output_index": 0, "content_index": 0, "delta": "3, 4, 5", + }, + { + "type": "response.completed", "sequence_number": 1, + "response": { + "id": "resp_fallback", "object": "response", "created_at": 0, "status": "completed", + "model": "gpt-6.1-terra", "output": [], + "usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10}, + }, + }, + ), + ] + ) + router: Final = _primary_with_fallback("openai/gpt-6.1-terra") + try: + stream: Final = await router.aresponses( + model="primary", + input="Print the integers 1 through 5.", + stream=True, + tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}], + tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "lookup"}]}, + ) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert openai.call_count == 2 + continued_input: Final = json.loads(openai.calls.last.request.content)["input"] + assert [item["role"] for item in continued_input[-2:]] == ["assistant", "user"] + finally: + router.discard() + + @pytest.mark.asyncio async def test_responses_fallback_surfaces_a_content_filter_that_cut_the_stream_mid_answer( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch @@ -1772,6 +1855,45 @@ async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cu router.discard() +@pytest.mark.asyncio +async def test_bridged_responses_fallback_continues_after_a_server_run_tool_search( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + side_effect=[ + _anthropic_stream_cut_mid_tool_call( + "Let me find the right tool. ", + {"type": "server_tool_use", "id": "srvtoolu_primary", "name": "tool_search_tool_regex", "input": {}}, + ), + _anthropic_stream("It is sunny in Paris."), + ] + ) + router: Final = Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + try: + stream: Final = await router.aresponses( + model="primary", + input="What is the weather in Paris?", + stream=True, + tools=[ + {"type": "tool_search", "execution": "server"}, + {"type": "function", "name": "lookup", "parameters": {"type": "object"}, "defer_loading": True}, + ], + ) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events).endswith("It is sunny in Paris.") + assert provider.call_count == 2 + finally: + router.discard() + + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_client_error_event_skips_fallback(): diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py index bbd9395f8c5..f79c8a6fadb 100644 --- a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -10,6 +10,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( ResponsesContinuation, ResponsesContinuationSeam, constrains_continuation, + constrains_responses_continuation, continuation_output_ceilings, deployment_supports_prefill, deployment_takes_responses_prefill, @@ -346,6 +347,37 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected) assert constrains_continuation(sent) is expected +@pytest.mark.parametrize( + "sent,expected", + [ + ({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False), + ({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True), + ({"tool_choice": {"type": "function", "name": "lookup"}}, True), + ({"tool_choice": {"type": "web_search_preview"}}, True), + ({"tool_choice": "required"}, True), + ({"tool_choice": "auto"}, False), + ( + { + "tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": []}, + "text": {"format": {"type": "json_object"}}, + }, + True, + ), + ], + ids=[ + "allowed_tools_auto", + "allowed_tools_required", + "forced_function", + "hosted_tool", + "required_tool", + "auto_tool", + "allowed_tools_auto_with_json", + ], +) +def test_constrains_responses_continuation_lets_allowed_tools_in_auto_mode_answer_in_text(sent, expected): + assert constrains_responses_continuation(sent) is expected + + @pytest.mark.parametrize( "deployment,expected", [