mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): stop Responses fallbacks from continuing content-filter cuts and client-run tool calls
A content filter that cuts a /v1/responses stream mid-answer now returns its error, as on chat, instead of letting a content_policy_fallbacks backup finish the filtered text. Shell, apply_patch, computer, local_shell, and MCP approval items count as a started tool call. Chat continuation goes back to treating any tool_choice object as constraining, as on main
This commit is contained in:
parent
6473cedf67
commit
c6056dbacd
5 changed files with 116 additions and 14 deletions
|
|
@ -1698,7 +1698,18 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
|
|||
return None
|
||||
|
||||
|
||||
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"})
|
||||
_TOOL_CALL_ITEM_TYPES: Final = frozenset(
|
||||
{
|
||||
"function_call",
|
||||
"custom_tool_call",
|
||||
"mcp_call",
|
||||
"mcp_approval_request",
|
||||
"computer_call",
|
||||
"local_shell_call",
|
||||
"shell_call",
|
||||
"apply_patch_call",
|
||||
}
|
||||
)
|
||||
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
|
||||
{
|
||||
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
|
||||
|
|
|
|||
|
|
@ -4174,10 +4174,10 @@ class Router:
|
|||
) -> ResponsesContinuation | None:
|
||||
"""What a fallback continues from once a Responses stream broke after text, raising the primary's error
|
||||
when the request constrains its output or the stream began a tool call, since a continued JSON document or
|
||||
tool call would not parse."""
|
||||
tool call would not parse, and when a content filter cut it, so no other model finishes filtered text."""
|
||||
if not e.generated_content or e.is_pre_first_chunk:
|
||||
return None
|
||||
if source_iterator.started_tool_call:
|
||||
if source_iterator.started_tool_call or isinstance(e.original_exception, litellm.ContentPolicyViolationError):
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id")
|
||||
cut_deployment: Final = (
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
|||
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
|
||||
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens")
|
||||
_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"})
|
||||
_QUOTED_ENDING_CHARS: Final = 100
|
||||
RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation"
|
||||
|
||||
|
|
@ -58,23 +57,19 @@ def deployment_supports_prefill(deployment: object) -> bool:
|
|||
return model is not None and supports_assistant_prefill(model=model)
|
||||
|
||||
|
||||
def _forces_tool_call(tool_choice: object) -> bool:
|
||||
choice: Final = _str_keyed(tool_choice)
|
||||
mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode")
|
||||
return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES
|
||||
|
||||
|
||||
def constrains_continuation(params: Mapping[str, object]) -> bool:
|
||||
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
|
||||
choice_count: Final = sent.get("n")
|
||||
response_format: Final = sent.get("response_format")
|
||||
text_format: Final = _str_keyed(sent.get("text")).get("format")
|
||||
tool_choice: Final = sent.get("tool_choice")
|
||||
return (
|
||||
(isinstance(choice_count, int) and choice_count > 1)
|
||||
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or sent.get("text_format") is not None
|
||||
or _forces_tool_call(sent.get("tool_choice"))
|
||||
or tool_choice == "required"
|
||||
or isinstance(tool_choice, Mapping)
|
||||
or sent.get("merge_reasoning_content_in_choices") is True
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1643,6 +1643,102 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream
|
|||
router.discard()
|
||||
|
||||
|
||||
def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response:
|
||||
return _sse(
|
||||
{
|
||||
"type": "response.output_text.delta", "sequence_number": 0,
|
||||
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble,
|
||||
},
|
||||
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 1, "item": item},
|
||||
{
|
||||
"type": "error", "sequence_number": 2,
|
||||
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
[
|
||||
{"type": "shell_call", "id": "sh_1", "call_id": "call_1", "action": {"commands": ["ls"]}, "status": "in_progress"},
|
||||
{
|
||||
"type": "apply_patch_call", "id": "apc_1", "call_id": "call_1", "status": "in_progress",
|
||||
"operation": {"type": "update_file", "path": "a.py", "diff": "@@"},
|
||||
},
|
||||
{
|
||||
"type": "local_shell_call", "id": "lsh_1", "call_id": "call_1", "status": "in_progress",
|
||||
"action": {"type": "exec", "command": ["ls"], "env": {}},
|
||||
},
|
||||
{
|
||||
"type": "computer_call", "id": "cu_1", "call_id": "call_1", "status": "in_progress",
|
||||
"action": {"type": "screenshot"}, "pending_safety_checks": [],
|
||||
},
|
||||
{"type": "mcp_approval_request", "id": "mcpr_1", "name": "lookup", "arguments": "{}", "server_label": "docs"},
|
||||
],
|
||||
ids=lambda item: item["type"],
|
||||
)
|
||||
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_client_run_tool_call(
|
||||
item: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=_openai_stream_cut_after_output_item("Let me check the repo. ", item)
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("The repo has two files.")
|
||||
)
|
||||
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
|
||||
try:
|
||||
stream: Final = await router.aresponses(model="primary", input="What is in the repo?", stream=True)
|
||||
with pytest.raises(litellm.InternalServerError, match="interrupted"):
|
||||
[event async for event in stream]
|
||||
assert fallback.call_count == 0
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_surfaces_a_content_filter_that_cut_the_stream_mid_answer(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=_sse(
|
||||
{
|
||||
"type": "response.output_text.delta", "sequence_number": 0,
|
||||
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": "Step one is to obtain the",
|
||||
},
|
||||
{
|
||||
"type": "error", "sequence_number": 1,
|
||||
"error": {
|
||||
"type": "invalid_request_error", "code": "content_policy_violation",
|
||||
"message": "The response was halted mid-stream.", "param": None,
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream(" reagents.")
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}},
|
||||
{"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}},
|
||||
],
|
||||
fallbacks=[{"primary": ["fallback"]}],
|
||||
content_policy_fallbacks=[{"primary": ["fallback"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
try:
|
||||
stream: Final = await router.aresponses(model="primary", input="How do I make it?", stream=True)
|
||||
with pytest.raises(litellm.ContentPolicyViolationError, match="halted mid-stream"):
|
||||
[event async for event in stream]
|
||||
assert fallback.call_count == 0
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
|
|||
|
|
@ -324,9 +324,9 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(
|
|||
({"tool_choice": "auto"}, False),
|
||||
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
|
||||
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False),
|
||||
({"tool_choice": {"type": "web_search_preview"}}, False),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, True),
|
||||
({"tool_choice": {"type": "web_search_preview"}}, True),
|
||||
],
|
||||
ids=[
|
||||
"json_schema",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue