fix(router): let Responses fallbacks continue under allowed_tools auto and after server-run tool calls

A Responses tool_choice of allowed_tools in auto mode no longer blocks the continuation, since the model may still answer in text. Hosted MCP calls and the bridge's server-run tool search no longer count as a started client tool call.
This commit is contained in:
mateo-berri 2026-10-28 04:05:04 -07:00
parent fc7c3e368e
commit fdea571341
6 changed files with 190 additions and 17 deletions

View file

@ -408,7 +408,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
continue
if call_id in self._web_search_calls:
continue
self._started_tool_call = True
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)
fn_name = ""
@ -424,6 +423,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if self._is_tool_search_call(call_id, fn_name):
self._queue_tool_search_call_added(call_id, output_index, fn_args_delta)
continue
self._started_tool_call = True
if call_id not in self._tool_args_by_call_id:
self._tool_args_by_call_id[call_id] = ""

View file

@ -486,17 +486,18 @@ class BaseResponsesAPIStreamingIterator:
if isinstance(_delta, str):
self._generated_content += _delta
elif _event_type in _TOOL_ARGUMENTS_DELTA_EVENTS:
self._started_tool_call = True
_args_delta: Final = getattr(openai_responses_api_chunk, "delta", None)
if isinstance(_args_delta, str):
self._generated_tool_arguments += _args_delta
if _event_type in _CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS:
self._started_tool_call = True
_stream_model_id: Final = _model_id_from_metadata(self.litellm_metadata)
if _event_type in (
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
):
_item: Final[object] = getattr(openai_responses_api_chunk, "item", None)
if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES:
if getattr(_item, "type", None) in _CLIENT_RUN_TOOL_CALL_ITEM_TYPES:
self._started_tool_call = True
if _item is not None:
ResponsesAPIRequestUtils.encode_container_id_on_output_item(
@ -1699,11 +1700,10 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
return None
_TOOL_CALL_ITEM_TYPES: Final = frozenset(
_CLIENT_RUN_TOOL_CALL_ITEM_TYPES: Final = frozenset(
{
"function_call",
"custom_tool_call",
"mcp_call",
"mcp_approval_request",
"computer_call",
"local_shell_call",
@ -1711,13 +1711,15 @@ _TOOL_CALL_ITEM_TYPES: Final = frozenset(
"apply_patch_call",
}
)
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
_CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
{
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA,
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA,
}
)
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = _CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS | {
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA
}
def _estimate_usage_from_text(

View file

@ -310,6 +310,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
ResponsesContinuation,
ResponsesContinuationSeam,
constrains_continuation,
constrains_responses_continuation,
continuation_deployments,
continuation_output_ceilings,
continued_text,
@ -4183,7 +4184,7 @@ class Router:
cut_deployment: Final = (
self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None
)
if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
if constrains_responses_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
Router._raise_original_mid_stream_error(e)
earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG)
if isinstance(earlier, ResponsesContinuation):

View file

@ -2,7 +2,7 @@
the output unconstrained, and shapes a Responses API continuation for the deployment that serves it."""
import json
from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence
from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate
from os.path import commonprefix
@ -57,23 +57,39 @@ def deployment_supports_prefill(deployment: object) -> bool:
return model is not None and supports_assistant_prefill(model=model)
def constrains_continuation(params: Mapping[str, object]) -> bool:
def _forces_tool_call(tool_choice: object) -> bool:
return tool_choice == "required" or isinstance(tool_choice, Mapping)
def _forces_responses_tool_call(tool_choice: object) -> bool:
choice: Final = _str_keyed(tool_choice)
leaves_text_open: Final = choice.get("type") == "allowed_tools" and choice.get("mode") == "auto"
return _forces_tool_call(tool_choice) and not leaves_text_open
def _constrains(params: Mapping[str, object], forces_tool_call: Callable[[object], bool]) -> bool:
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
choice_count: Final = sent.get("n")
response_format: Final = sent.get("response_format")
text_format: Final = _str_keyed(sent.get("text")).get("format")
tool_choice: Final = sent.get("tool_choice")
return (
(isinstance(choice_count, int) and choice_count > 1)
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or sent.get("text_format") is not None
or tool_choice == "required"
or isinstance(tool_choice, Mapping)
or forces_tool_call(sent.get("tool_choice"))
or sent.get("merge_reasoning_content_in_choices") is True
)
def constrains_continuation(params: Mapping[str, object]) -> bool:
return _constrains(params, _forces_tool_call)
def constrains_responses_continuation(params: Mapping[str, object]) -> bool:
return _constrains(params, _forces_responses_tool_call)
def deployment_can_continue(deployment: object) -> bool:
litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params"))
return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params)

View file

@ -1599,7 +1599,7 @@ def _openai_stream_cut_mid_tool_call(preamble: str, *, announce_item: bool = Tru
return _sse(*((text, item_added, arguments, error) if announce_item else (text, arguments, error)))
def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
def _anthropic_stream_cut_mid_tool_call(preamble: str, tool_block: Mapping[str, object] | None = None) -> httpx.Response:
return _sse(
{"type": "message_start", "message": {
"id": "msg_primary", "type": "message", "role": "assistant",
@ -1608,7 +1608,7 @@ def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}},
{"type": "content_block_stop", "index": 0},
{"type": "content_block_start", "index": 1, "content_block": {
{"type": "content_block_start", "index": 1, "content_block": tool_block or {
"type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {},
}},
{"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}},
@ -1643,13 +1643,13 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream
router.discard()
def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response:
def _openai_stream_cut_after_event(preamble: str, event: Mapping[str, object]) -> httpx.Response:
return _sse(
{
"type": "response.output_text.delta", "sequence_number": 0,
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble,
},
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 1, "item": item},
{**event, "sequence_number": 1},
{
"type": "error", "sequence_number": 2,
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
@ -1657,6 +1657,12 @@ def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, objec
)
def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response:
return _openai_stream_cut_after_event(
preamble, {"type": "response.output_item.added", "output_index": 1, "item": item}
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"item",
@ -1698,6 +1704,83 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream
router.discard()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_run_event",
[
{
"type": "response.output_item.added", "output_index": 1,
"item": {
"type": "mcp_call", "id": "mcp_1", "name": "ask_question", "arguments": "{}",
"server_label": "docs", "status": "in_progress",
},
},
{"type": "response.mcp_call_arguments.delta", "output_index": 1, "item_id": "mcp_1", "delta": '{"question": "set'},
],
ids=["mcp_call_item", "mcp_call_arguments"],
)
async def test_responses_fallback_continues_after_a_server_run_mcp_call(
server_run_event: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=_openai_stream_cut_after_event("Let me check the docs. ", server_run_event)
)
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("The docs cover setup.")
)
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
try:
stream: Final = await router.aresponses(model="primary", input="What do the docs cover?", stream=True)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events).endswith("The docs cover setup.")
assert fallback.call_count == 1
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_continues_when_allowed_tools_in_auto_mode_leaves_the_reply_open(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
openai: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
side_effect=[
_openai_stream_cut_after("1, 2, "),
_sse(
{
"type": "response.output_text.delta", "sequence_number": 0,
"item_id": "msg_fallback", "output_index": 0, "content_index": 0, "delta": "3, 4, 5",
},
{
"type": "response.completed", "sequence_number": 1,
"response": {
"id": "resp_fallback", "object": "response", "created_at": 0, "status": "completed",
"model": "gpt-6.1-terra", "output": [],
"usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10},
},
},
),
]
)
router: Final = _primary_with_fallback("openai/gpt-6.1-terra")
try:
stream: Final = await router.aresponses(
model="primary",
input="Print the integers 1 through 5.",
stream=True,
tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}],
tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "lookup"}]},
)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events) == "1, 2, 3, 4, 5"
assert openai.call_count == 2
continued_input: Final = json.loads(openai.calls.last.request.content)["input"]
assert [item["role"] for item in continued_input[-2:]] == ["assistant", "user"]
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_surfaces_a_content_filter_that_cut_the_stream_mid_answer(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
@ -1772,6 +1855,45 @@ async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cu
router.discard()
@pytest.mark.asyncio
async def test_bridged_responses_fallback_continues_after_a_server_run_tool_search(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
side_effect=[
_anthropic_stream_cut_mid_tool_call(
"Let me find the right tool. ",
{"type": "server_tool_use", "id": "srvtoolu_primary", "name": "tool_search_tool_regex", "input": {}},
),
_anthropic_stream("It is sunny in Paris."),
]
)
router: Final = Router(
model_list=[
{"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}},
{"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}},
],
fallbacks=[{"primary": ["fallback"]}],
num_retries=0,
)
try:
stream: Final = await router.aresponses(
model="primary",
input="What is the weather in Paris?",
stream=True,
tools=[
{"type": "tool_search", "execution": "server"},
{"type": "function", "name": "lookup", "parameters": {"type": "object"}, "defer_loading": True},
],
)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events).endswith("It is sunny in Paris.")
assert provider.call_count == 2
finally:
router.discard()
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
@pytest.mark.asyncio
async def test_aresponses_client_error_event_skips_fallback():

View file

@ -10,6 +10,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
ResponsesContinuation,
ResponsesContinuationSeam,
constrains_continuation,
constrains_responses_continuation,
continuation_output_ceilings,
deployment_supports_prefill,
deployment_takes_responses_prefill,
@ -346,6 +347,37 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected)
assert constrains_continuation(sent) is expected
@pytest.mark.parametrize(
"sent,expected",
[
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
({"tool_choice": {"type": "web_search_preview"}}, True),
({"tool_choice": "required"}, True),
({"tool_choice": "auto"}, False),
(
{
"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": []},
"text": {"format": {"type": "json_object"}},
},
True,
),
],
ids=[
"allowed_tools_auto",
"allowed_tools_required",
"forced_function",
"hosted_tool",
"required_tool",
"auto_tool",
"allowed_tools_auto_with_json",
],
)
def test_constrains_responses_continuation_lets_allowed_tools_in_auto_mode_answer_in_text(sent, expected):
assert constrains_responses_continuation(sent) is expected
@pytest.mark.parametrize(
"deployment,expected",
[