diff --git a/litellm/router.py b/litellm/router.py index a10acc63fe8..3de4a70d5c4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3067,7 +3067,9 @@ class Router: return False # Any structured-output request (response_format, or a forced tool call) # produces a partial that cannot be resumed from an arbitrary cut point. - if request_kwargs.get("response_format") is not None: + # `{"type": "text"}` is the unconstrained default and stays eligible. + response_format: Final = request_kwargs.get("response_format") + if response_format is not None and response_format != {"type": "text"}: return False tool_choice: Final = request_kwargs.get("tool_choice") if tool_choice == "required" or isinstance(tool_choice, Mapping): @@ -3087,7 +3089,16 @@ class Router: regenerating text already delivered to the caller. The deployment filter guarantees the target supports ``prefix: True`` (parity with ``_build_responses_continuation_input`` for the Responses-API path). + + A nested mid-stream break can re-enter here with a prefill already + appended; the new partial is folded into that trailing assistant turn so + the request keeps a single prefill rather than two consecutive assistant + messages a non-merging provider would reject. """ + if messages and messages[-1].get("role") == "assistant" and messages[-1].get("prefix"): + last: Final = messages[-1] + merged: dict[str, object] = {**last, "content": str(last.get("content") or "") + generated_content} + return [*messages[:-1], merged] prefill: dict[str, object] = {"role": "assistant", "content": generated_content, "prefix": True} return [*messages, prefill] diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 75d1ecb225c..14e637280d8 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -28,12 +28,11 @@ _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) def _deployment_supports_prefill(deployment: object) -> bool: try: - params: Final = _STR_KEYED_DICT_ADAPTER.validate_python( - _STR_KEYED_DICT_ADAPTER.validate_python(deployment).get("litellm_params") - ) + deployment_map: Final = _STR_KEYED_DICT_ADAPTER.validate_python(deployment) + litellm_params: Final = _STR_KEYED_DICT_ADAPTER.validate_python(deployment_map.get("litellm_params")) except ValidationError: return False - model: Final = params.get("model") + model: Final = litellm_params.get("model") return isinstance(model, str) and bool(model) and supports_assistant_prefill(model=model) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 1a1ca08b9b2..0c3bf43182d 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -2276,8 +2276,9 @@ async def test_acompletion_streaming_iterator_continues_after_content_when_eligi ({"emitted_disqualifying_content": True}, {}), ({"emitted_disqualifying_content": False}, {"response_format": {"type": "json_object"}}), ({"emitted_disqualifying_content": False}, {"tool_choice": "required"}), + ({"emitted_disqualifying_content": False}, {"merge_reasoning_content_in_choices": True}), ], - ids=["tool_or_thinking_emitted", "json_mode", "forced_tool_choice"], + ids=["tool_or_thinking_emitted", "json_mode", "forced_tool_choice", "merged_reasoning"], ) async def test_acompletion_streaming_iterator_declines_ineligible_after_content(error_kwargs, request_kwargs): """Flag on but the break is not continuation-safe: the router re-raises and @@ -2352,6 +2353,32 @@ def test_build_completion_continuation_input_appends_assistant_prefill(): assert built[-1] == {"role": "assistant", "content": "partial answer", "prefix": True} +def test_build_completion_continuation_input_folds_into_existing_prefill(): + """A nested break must not leave two trailing assistant turns: the new partial + folds into the prior prefill so a non-merging provider still gets one.""" + once = litellm.Router._build_completion_continuation_input([{"role": "user", "content": "hi"}], "part one ") + twice = litellm.Router._build_completion_continuation_input(list(once), "part two") + assert [m["role"] for m in twice] == ["user", "assistant"] + assert twice[-1] == {"role": "assistant", "content": "part one part two", "prefix": True} + + +def test_mid_stream_continuation_eligible_allows_text_response_format(): + """response_format={"type": "text"} is the unconstrained default and must stay + eligible, unlike json_object / json_schema.""" + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[{"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k"}}], + enable_mid_stream_fallback_continuation=True, + ) + e = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + assert router._mid_stream_continuation_eligible(e=e, request_kwargs={"response_format": {"type": "text"}}) is True + assert router._mid_stream_continuation_eligible(e=e, request_kwargs={"response_format": {"type": "json_object"}}) is False + + @pytest.mark.asyncio async def test_acompletion_streaming_iterator_reraises_original_exception_when_available(): """Async: when the mid-stream MidStreamFallbackError wraps a real provider