mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): let Responses fallbacks continue under allowed_tools auto and after server-run tool calls
A Responses tool_choice of allowed_tools in auto mode no longer blocks the continuation, since the model may still answer in text. Hosted MCP calls and the bridge's server-run tool search no longer count as a started client tool call.
This commit is contained in:
parent
fc7c3e368e
commit
fdea571341
6 changed files with 190 additions and 17 deletions
|
|
@ -408,7 +408,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
continue
|
||||
if call_id in self._web_search_calls:
|
||||
continue
|
||||
self._started_tool_call = True
|
||||
|
||||
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)
|
||||
fn_name = ""
|
||||
|
|
@ -424,6 +423,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if self._is_tool_search_call(call_id, fn_name):
|
||||
self._queue_tool_search_call_added(call_id, output_index, fn_args_delta)
|
||||
continue
|
||||
self._started_tool_call = True
|
||||
|
||||
if call_id not in self._tool_args_by_call_id:
|
||||
self._tool_args_by_call_id[call_id] = ""
|
||||
|
|
|
|||
|
|
@ -486,17 +486,18 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if isinstance(_delta, str):
|
||||
self._generated_content += _delta
|
||||
elif _event_type in _TOOL_ARGUMENTS_DELTA_EVENTS:
|
||||
self._started_tool_call = True
|
||||
_args_delta: Final = getattr(openai_responses_api_chunk, "delta", None)
|
||||
if isinstance(_args_delta, str):
|
||||
self._generated_tool_arguments += _args_delta
|
||||
if _event_type in _CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS:
|
||||
self._started_tool_call = True
|
||||
_stream_model_id: Final = _model_id_from_metadata(self.litellm_metadata)
|
||||
if _event_type in (
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
):
|
||||
_item: Final[object] = getattr(openai_responses_api_chunk, "item", None)
|
||||
if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES:
|
||||
if getattr(_item, "type", None) in _CLIENT_RUN_TOOL_CALL_ITEM_TYPES:
|
||||
self._started_tool_call = True
|
||||
if _item is not None:
|
||||
ResponsesAPIRequestUtils.encode_container_id_on_output_item(
|
||||
|
|
@ -1699,11 +1700,10 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
|
|||
return None
|
||||
|
||||
|
||||
_TOOL_CALL_ITEM_TYPES: Final = frozenset(
|
||||
_CLIENT_RUN_TOOL_CALL_ITEM_TYPES: Final = frozenset(
|
||||
{
|
||||
"function_call",
|
||||
"custom_tool_call",
|
||||
"mcp_call",
|
||||
"mcp_approval_request",
|
||||
"computer_call",
|
||||
"local_shell_call",
|
||||
|
|
@ -1711,13 +1711,15 @@ _TOOL_CALL_ITEM_TYPES: Final = frozenset(
|
|||
"apply_patch_call",
|
||||
}
|
||||
)
|
||||
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
|
||||
_CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
|
||||
{
|
||||
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
|
||||
ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA,
|
||||
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA,
|
||||
}
|
||||
)
|
||||
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = _CLIENT_RUN_TOOL_ARGUMENTS_DELTA_EVENTS | {
|
||||
ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA
|
||||
}
|
||||
|
||||
|
||||
def _estimate_usage_from_text(
|
||||
|
|
|
|||
|
|
@ -310,6 +310,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
|
|||
ResponsesContinuation,
|
||||
ResponsesContinuationSeam,
|
||||
constrains_continuation,
|
||||
constrains_responses_continuation,
|
||||
continuation_deployments,
|
||||
continuation_output_ceilings,
|
||||
continued_text,
|
||||
|
|
@ -4183,7 +4184,7 @@ class Router:
|
|||
cut_deployment: Final = (
|
||||
self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None
|
||||
)
|
||||
if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
|
||||
if constrains_responses_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG)
|
||||
if isinstance(earlier, ResponsesContinuation):
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
the output unconstrained, and shapes a Responses API continuation for the deployment that serves it."""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate
|
||||
from os.path import commonprefix
|
||||
|
|
@ -57,23 +57,39 @@ def deployment_supports_prefill(deployment: object) -> bool:
|
|||
return model is not None and supports_assistant_prefill(model=model)
|
||||
|
||||
|
||||
def constrains_continuation(params: Mapping[str, object]) -> bool:
|
||||
def _forces_tool_call(tool_choice: object) -> bool:
|
||||
return tool_choice == "required" or isinstance(tool_choice, Mapping)
|
||||
|
||||
|
||||
def _forces_responses_tool_call(tool_choice: object) -> bool:
|
||||
choice: Final = _str_keyed(tool_choice)
|
||||
leaves_text_open: Final = choice.get("type") == "allowed_tools" and choice.get("mode") == "auto"
|
||||
return _forces_tool_call(tool_choice) and not leaves_text_open
|
||||
|
||||
|
||||
def _constrains(params: Mapping[str, object], forces_tool_call: Callable[[object], bool]) -> bool:
|
||||
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
|
||||
choice_count: Final = sent.get("n")
|
||||
response_format: Final = sent.get("response_format")
|
||||
text_format: Final = _str_keyed(sent.get("text")).get("format")
|
||||
tool_choice: Final = sent.get("tool_choice")
|
||||
return (
|
||||
(isinstance(choice_count, int) and choice_count > 1)
|
||||
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or sent.get("text_format") is not None
|
||||
or tool_choice == "required"
|
||||
or isinstance(tool_choice, Mapping)
|
||||
or forces_tool_call(sent.get("tool_choice"))
|
||||
or sent.get("merge_reasoning_content_in_choices") is True
|
||||
)
|
||||
|
||||
|
||||
def constrains_continuation(params: Mapping[str, object]) -> bool:
|
||||
return _constrains(params, _forces_tool_call)
|
||||
|
||||
|
||||
def constrains_responses_continuation(params: Mapping[str, object]) -> bool:
|
||||
return _constrains(params, _forces_responses_tool_call)
|
||||
|
||||
|
||||
def deployment_can_continue(deployment: object) -> bool:
|
||||
litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params"))
|
||||
return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params)
|
||||
|
|
|
|||
|
|
@ -1599,7 +1599,7 @@ def _openai_stream_cut_mid_tool_call(preamble: str, *, announce_item: bool = Tru
|
|||
return _sse(*((text, item_added, arguments, error) if announce_item else (text, arguments, error)))
|
||||
|
||||
|
||||
def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
|
||||
def _anthropic_stream_cut_mid_tool_call(preamble: str, tool_block: Mapping[str, object] | None = None) -> httpx.Response:
|
||||
return _sse(
|
||||
{"type": "message_start", "message": {
|
||||
"id": "msg_primary", "type": "message", "role": "assistant",
|
||||
|
|
@ -1608,7 +1608,7 @@ def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
|
|||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "content_block_start", "index": 1, "content_block": {
|
||||
{"type": "content_block_start", "index": 1, "content_block": tool_block or {
|
||||
"type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {},
|
||||
}},
|
||||
{"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}},
|
||||
|
|
@ -1643,13 +1643,13 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream
|
|||
router.discard()
|
||||
|
||||
|
||||
def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response:
|
||||
def _openai_stream_cut_after_event(preamble: str, event: Mapping[str, object]) -> httpx.Response:
|
||||
return _sse(
|
||||
{
|
||||
"type": "response.output_text.delta", "sequence_number": 0,
|
||||
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble,
|
||||
},
|
||||
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 1, "item": item},
|
||||
{**event, "sequence_number": 1},
|
||||
{
|
||||
"type": "error", "sequence_number": 2,
|
||||
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
|
||||
|
|
@ -1657,6 +1657,12 @@ def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, objec
|
|||
)
|
||||
|
||||
|
||||
def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response:
|
||||
return _openai_stream_cut_after_event(
|
||||
preamble, {"type": "response.output_item.added", "output_index": 1, "item": item}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
|
|
@ -1698,6 +1704,83 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream
|
|||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server_run_event",
|
||||
[
|
||||
{
|
||||
"type": "response.output_item.added", "output_index": 1,
|
||||
"item": {
|
||||
"type": "mcp_call", "id": "mcp_1", "name": "ask_question", "arguments": "{}",
|
||||
"server_label": "docs", "status": "in_progress",
|
||||
},
|
||||
},
|
||||
{"type": "response.mcp_call_arguments.delta", "output_index": 1, "item_id": "mcp_1", "delta": '{"question": "set'},
|
||||
],
|
||||
ids=["mcp_call_item", "mcp_call_arguments"],
|
||||
)
|
||||
async def test_responses_fallback_continues_after_a_server_run_mcp_call(
|
||||
server_run_event: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=_openai_stream_cut_after_event("Let me check the docs. ", server_run_event)
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("The docs cover setup.")
|
||||
)
|
||||
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
|
||||
try:
|
||||
stream: Final = await router.aresponses(model="primary", input="What do the docs cover?", stream=True)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events).endswith("The docs cover setup.")
|
||||
assert fallback.call_count == 1
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_continues_when_allowed_tools_in_auto_mode_leaves_the_reply_open(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
openai: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
side_effect=[
|
||||
_openai_stream_cut_after("1, 2, "),
|
||||
_sse(
|
||||
{
|
||||
"type": "response.output_text.delta", "sequence_number": 0,
|
||||
"item_id": "msg_fallback", "output_index": 0, "content_index": 0, "delta": "3, 4, 5",
|
||||
},
|
||||
{
|
||||
"type": "response.completed", "sequence_number": 1,
|
||||
"response": {
|
||||
"id": "resp_fallback", "object": "response", "created_at": 0, "status": "completed",
|
||||
"model": "gpt-6.1-terra", "output": [],
|
||||
"usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10},
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
router: Final = _primary_with_fallback("openai/gpt-6.1-terra")
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary",
|
||||
input="Print the integers 1 through 5.",
|
||||
stream=True,
|
||||
tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}],
|
||||
tool_choice={"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "lookup"}]},
|
||||
)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events) == "1, 2, 3, 4, 5"
|
||||
assert openai.call_count == 2
|
||||
continued_input: Final = json.loads(openai.calls.last.request.content)["input"]
|
||||
assert [item["role"] for item in continued_input[-2:]] == ["assistant", "user"]
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_surfaces_a_content_filter_that_cut_the_stream_mid_answer(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
@ -1772,6 +1855,45 @@ async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cu
|
|||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_fallback_continues_after_a_server_run_tool_search(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
side_effect=[
|
||||
_anthropic_stream_cut_mid_tool_call(
|
||||
"Let me find the right tool. ",
|
||||
{"type": "server_tool_use", "id": "srvtoolu_primary", "name": "tool_search_tool_regex", "input": {}},
|
||||
),
|
||||
_anthropic_stream("It is sunny in Paris."),
|
||||
]
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}},
|
||||
{"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}},
|
||||
],
|
||||
fallbacks=[{"primary": ["fallback"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary",
|
||||
input="What is the weather in Paris?",
|
||||
stream=True,
|
||||
tools=[
|
||||
{"type": "tool_search", "execution": "server"},
|
||||
{"type": "function", "name": "lookup", "parameters": {"type": "object"}, "defer_loading": True},
|
||||
],
|
||||
)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events).endswith("It is sunny in Paris.")
|
||||
assert provider.call_count == 2
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_client_error_event_skips_fallback():
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
|
|||
ResponsesContinuation,
|
||||
ResponsesContinuationSeam,
|
||||
constrains_continuation,
|
||||
constrains_responses_continuation,
|
||||
continuation_output_ceilings,
|
||||
deployment_supports_prefill,
|
||||
deployment_takes_responses_prefill,
|
||||
|
|
@ -346,6 +347,37 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected)
|
|||
assert constrains_continuation(sent) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sent,expected",
|
||||
[
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
|
||||
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
|
||||
({"tool_choice": {"type": "web_search_preview"}}, True),
|
||||
({"tool_choice": "required"}, True),
|
||||
({"tool_choice": "auto"}, False),
|
||||
(
|
||||
{
|
||||
"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": []},
|
||||
"text": {"format": {"type": "json_object"}},
|
||||
},
|
||||
True,
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"allowed_tools_auto",
|
||||
"allowed_tools_required",
|
||||
"forced_function",
|
||||
"hosted_tool",
|
||||
"required_tool",
|
||||
"auto_tool",
|
||||
"allowed_tools_auto_with_json",
|
||||
],
|
||||
)
|
||||
def test_constrains_responses_continuation_lets_allowed_tools_in_auto_mode_answer_in_text(sent, expected):
|
||||
assert constrains_responses_continuation(sent) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment,expected",
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue