fix(router): stop Responses fallbacks from continuing content-filter cuts and client-run tool calls

A content filter that cuts a /v1/responses stream mid-answer now returns its error, as on chat, instead of letting a content_policy_fallbacks backup finish the filtered text. Shell, apply_patch, computer, local_shell, and MCP approval items count as a started tool call. Chat continuation goes back to treating any tool_choice object as constraining, as on main
This commit is contained in:
mateo-berri 2026-10-28 01:38:36 -07:00
parent 6473cedf67
commit c6056dbacd
5 changed files with 116 additions and 14 deletions

View file

@ -1698,7 +1698,18 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
return None
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"})
_TOOL_CALL_ITEM_TYPES: Final = frozenset(
{
"function_call",
"custom_tool_call",
"mcp_call",
"mcp_approval_request",
"computer_call",
"local_shell_call",
"shell_call",
"apply_patch_call",
}
)
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
{
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,

View file

@ -4174,10 +4174,10 @@ class Router:
) -> ResponsesContinuation | None:
"""What a fallback continues from once a Responses stream broke after text, raising the primary's error
when the request constrains its output or the stream began a tool call, since a continued JSON document or
tool call would not parse."""
tool call would not parse, and when a content filter cut it, so no other model finishes filtered text."""
if not e.generated_content or e.is_pre_first_chunk:
return None
if source_iterator.started_tool_call:
if source_iterator.started_tool_call or isinstance(e.original_exception, litellm.ContentPolicyViolationError):
Router._raise_original_mid_stream_error(e)
cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id")
cut_deployment: Final = (

View file

@ -30,7 +30,6 @@ _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens")
_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"})
_QUOTED_ENDING_CHARS: Final = 100
RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation"
@ -58,23 +57,19 @@ def deployment_supports_prefill(deployment: object) -> bool:
return model is not None and supports_assistant_prefill(model=model)
def _forces_tool_call(tool_choice: object) -> bool:
choice: Final = _str_keyed(tool_choice)
mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode")
return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES
def constrains_continuation(params: Mapping[str, object]) -> bool:
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
choice_count: Final = sent.get("n")
response_format: Final = sent.get("response_format")
text_format: Final = _str_keyed(sent.get("text")).get("format")
tool_choice: Final = sent.get("tool_choice")
return (
(isinstance(choice_count, int) and choice_count > 1)
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or sent.get("text_format") is not None
or _forces_tool_call(sent.get("tool_choice"))
or tool_choice == "required"
or isinstance(tool_choice, Mapping)
or sent.get("merge_reasoning_content_in_choices") is True
)

View file

@ -1643,6 +1643,102 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream
router.discard()
def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response:
return _sse(
{
"type": "response.output_text.delta", "sequence_number": 0,
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble,
},
{"type": "response.output_item.added", "sequence_number": 1, "output_index": 1, "item": item},
{
"type": "error", "sequence_number": 2,
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"item",
[
{"type": "shell_call", "id": "sh_1", "call_id": "call_1", "action": {"commands": ["ls"]}, "status": "in_progress"},
{
"type": "apply_patch_call", "id": "apc_1", "call_id": "call_1", "status": "in_progress",
"operation": {"type": "update_file", "path": "a.py", "diff": "@@"},
},
{
"type": "local_shell_call", "id": "lsh_1", "call_id": "call_1", "status": "in_progress",
"action": {"type": "exec", "command": ["ls"], "env": {}},
},
{
"type": "computer_call", "id": "cu_1", "call_id": "call_1", "status": "in_progress",
"action": {"type": "screenshot"}, "pending_safety_checks": [],
},
{"type": "mcp_approval_request", "id": "mcpr_1", "name": "lookup", "arguments": "{}", "server_label": "docs"},
],
ids=lambda item: item["type"],
)
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_client_run_tool_call(
item: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=_openai_stream_cut_after_output_item("Let me check the repo. ", item)
)
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("The repo has two files.")
)
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
try:
stream: Final = await router.aresponses(model="primary", input="What is in the repo?", stream=True)
with pytest.raises(litellm.InternalServerError, match="interrupted"):
[event async for event in stream]
assert fallback.call_count == 0
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_surfaces_a_content_filter_that_cut_the_stream_mid_answer(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=_sse(
{
"type": "response.output_text.delta", "sequence_number": 0,
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": "Step one is to obtain the",
},
{
"type": "error", "sequence_number": 1,
"error": {
"type": "invalid_request_error", "code": "content_policy_violation",
"message": "The response was halted mid-stream.", "param": None,
},
},
)
)
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream(" reagents.")
)
router: Final = Router(
model_list=[
{"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}},
{"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}},
],
fallbacks=[{"primary": ["fallback"]}],
content_policy_fallbacks=[{"primary": ["fallback"]}],
num_retries=0,
)
try:
stream: Final = await router.aresponses(model="primary", input="How do I make it?", stream=True)
with pytest.raises(litellm.ContentPolicyViolationError, match="halted mid-stream"):
[event async for event in stream]
assert fallback.call_count == 0
finally:
router.discard()
@pytest.mark.asyncio
async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch

View file

@ -324,9 +324,9 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(
({"tool_choice": "auto"}, False),
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False),
({"tool_choice": {"type": "web_search_preview"}}, False),
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, True),
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, True),
({"tool_choice": {"type": "web_search_preview"}}, True),
],
ids=[
"json_schema",