diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0e0d0ddade3..86d0019d02e 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1698,7 +1698,18 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None: return None -_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"}) +_TOOL_CALL_ITEM_TYPES: Final = frozenset( + { + "function_call", + "custom_tool_call", + "mcp_call", + "mcp_approval_request", + "computer_call", + "local_shell_call", + "shell_call", + "apply_patch_call", + } +) _TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset( { ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, diff --git a/litellm/router.py b/litellm/router.py index f52884d25ca..6b0aca9d9ec 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4174,10 +4174,10 @@ class Router: ) -> ResponsesContinuation | None: """What a fallback continues from once a Responses stream broke after text, raising the primary's error when the request constrains its output or the stream began a tool call, since a continued JSON document or - tool call would not parse.""" + tool call would not parse, and when a content filter cut it, so no other model finishes filtered text.""" if not e.generated_content or e.is_pre_first_chunk: return None - if source_iterator.started_tool_call: + if source_iterator.started_tool_call or isinstance(e.original_exception, litellm.ContentPolicyViolationError): Router._raise_original_mid_stream_error(e) cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") cut_deployment: Final = ( diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 37e10dd7b4f..9abad288cad 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -30,7 +30,6 @@ _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) _OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens") -_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"}) _QUOTED_ENDING_CHARS: Final = 100 RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation" @@ -58,23 +57,19 @@ def deployment_supports_prefill(deployment: object) -> bool: return model is not None and supports_assistant_prefill(model=model) -def _forces_tool_call(tool_choice: object) -> bool: - choice: Final = _str_keyed(tool_choice) - mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode") - return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES - - def constrains_continuation(params: Mapping[str, object]) -> bool: sent: Final = {**params, **_str_keyed(params.get("extra_body"))} choice_count: Final = sent.get("n") response_format: Final = sent.get("response_format") text_format: Final = _str_keyed(sent.get("text")).get("format") + tool_choice: Final = sent.get("tool_choice") return ( (isinstance(choice_count, int) and choice_count > 1) or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT) or sent.get("text_format") is not None - or _forces_tool_call(sent.get("tool_choice")) + or tool_choice == "required" + or isinstance(tool_choice, Mapping) or sent.get("merge_reasoning_content_in_choices") is True ) diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 343524e3b66..5b1672ce34e 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -1643,6 +1643,102 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream router.discard() +def _openai_stream_cut_after_output_item(preamble: str, item: Mapping[str, object]) -> httpx.Response: + return _sse( + { + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble, + }, + {"type": "response.output_item.added", "sequence_number": 1, "output_index": 1, "item": item}, + { + "type": "error", "sequence_number": 2, + "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "item", + [ + {"type": "shell_call", "id": "sh_1", "call_id": "call_1", "action": {"commands": ["ls"]}, "status": "in_progress"}, + { + "type": "apply_patch_call", "id": "apc_1", "call_id": "call_1", "status": "in_progress", + "operation": {"type": "update_file", "path": "a.py", "diff": "@@"}, + }, + { + "type": "local_shell_call", "id": "lsh_1", "call_id": "call_1", "status": "in_progress", + "action": {"type": "exec", "command": ["ls"], "env": {}}, + }, + { + "type": "computer_call", "id": "cu_1", "call_id": "call_1", "status": "in_progress", + "action": {"type": "screenshot"}, "pending_safety_checks": [], + }, + {"type": "mcp_approval_request", "id": "mcpr_1", "name": "lookup", "arguments": "{}", "server_label": "docs"}, + ], + ids=lambda item: item["type"], +) +async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_client_run_tool_call( + item: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_after_output_item("Let me check the repo. ", item) + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("The repo has two files.") + ) + router: Final = _primary_with_fallback("anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses(model="primary", input="What is in the repo?", stream=True) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_surfaces_a_content_filter_that_cut_the_stream_mid_answer( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_sse( + { + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": "Step one is to obtain the", + }, + { + "type": "error", "sequence_number": 1, + "error": { + "type": "invalid_request_error", "code": "content_policy_violation", + "message": "The response was halted mid-stream.", "param": None, + }, + }, + ) + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream(" reagents.") + ) + router: Final = Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + content_policy_fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + try: + stream: Final = await router.aresponses(model="primary", input="How do I make it?", stream=True) + with pytest.raises(litellm.ContentPolicyViolationError, match="halted mid-stream"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + @pytest.mark.asyncio async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py index 4492f8512ae..bbd9395f8c5 100644 --- a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -324,9 +324,9 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left( ({"tool_choice": "auto"}, False), ({"tool_choice": {"type": "function", "name": "lookup"}}, True), ({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True), - ({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False), - ({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False), - ({"tool_choice": {"type": "web_search_preview"}}, False), + ({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, True), + ({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, True), + ({"tool_choice": {"type": "web_search_preview"}}, True), ], ids=[ "json_schema",