From cea8228b2c1c7cf56cafe2a6d4efac7648d4d443 Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 20 Oct 2026 15:36:06 -0700 Subject: [PATCH 1/4] fix(router): avoid assistant prefill in Responses fallbacks --- litellm/router.py | 33 ++---- .../base_llm/responses/test_transformation.py | 106 ++++++++++++++++-- tests/unit/test_router/test_router.py | 13 +-- 3 files changed, 110 insertions(+), 42 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 1c5a9176207..7a59099ff0d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3799,25 +3799,8 @@ class Router: input_val: Union[str, "ResponseInputParam"] | None, generated_content: str, ) -> "ResponseInputParam": - """ - Convert Responses-API input + partial assistant output into a - continuation input that asks the fallback model to pick up where the - prior assistant message stopped. - - Best effort across providers. The chat-completions path uses - Anthropic's `prefix: True` prefill trick on the assistant message; - the Responses-API input schema has no direct equivalent, so we - append an instruction (developer role) plus a prior assistant - message containing the partial output. Providers without prefill - semantics (OpenAI, Vertex) treat this as conversational context - and may regenerate — same trade-off as the chat-completions path - for non-Anthropic fallbacks. - """ - # base/continuation are List[Any] because ResponseInputParam items - # are a wide Union of TypedDicts (EasyInputMessageParam, Message, - # ResponseOutputMessageParam, ...) — annotating as List[Dict[str, Any]] - # rejects the list() spread of input_val. We cast the combined list to - # ResponseInputParam at the return. + """Request a best-effort continuation using assistant history followed by a user turn, so the fallback + does not need to support assistant prefill.""" base: list[object] if isinstance(input_val, str): base = [ @@ -3834,7 +3817,12 @@ class Router: continuation: Final[list[object]] = [ { "type": "message", - "role": "developer", + "role": "assistant", + "content": [{"type": "output_text", "text": generated_content}], + }, + { + "type": "message", + "role": "user", "content": [ { "type": "input_text", @@ -3847,11 +3835,6 @@ class Router: } ], }, - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": generated_content}], - }, ] return cast("ResponseInputParam", base + continuation) diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 8d21af83668..e5a685fc805 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -1,9 +1,11 @@ """The shared Responses API config contract.""" import asyncio, importlib, json +from collections.abc import Mapping import httpx import pytest +import respx import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -885,9 +887,9 @@ def test_build_responses_continuation_input_from_string(): assert len(out) == 3 assert out[0]["role"] == "user" assert out[0]["content"][0]["text"] == "Hello world" - assert out[1]["role"] == "developer" - assert out[2]["role"] == "assistant" - assert out[2]["content"][0]["text"] == "partial assistant text" + assert out[1]["role"] == "assistant" + assert out[1]["content"][0]["text"] == "partial assistant text" + assert out[2]["role"] == "user" @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") def test_build_responses_continuation_input_from_list_preserves_items(): @@ -901,15 +903,16 @@ def test_build_responses_continuation_input_from_list_preserves_items(): out = Router._build_responses_continuation_input(existing, "partial") assert len(out) == 3 assert out[0]["content"][0]["text"] == "msg1" - assert out[1]["role"] == "developer" - assert out[2]["role"] == "assistant" + assert out[1]["role"] == "assistant" + assert out[2]["role"] == "user" + assert existing == out[:1] @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") def test_build_responses_continuation_input_from_none(): out = Router._build_responses_continuation_input(None, "partial") assert len(out) == 2 - assert out[0]["role"] == "developer" - assert out[1]["role"] == "assistant" + assert out[0]["role"] == "assistant" + assert out[1]["role"] == "user" @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio @@ -1355,9 +1358,92 @@ async def test_aresponses_fallback_uses_continuation_input_after_partial_content continuation = mock_fallback.await_args.kwargs["kwargs"]["input"] assert isinstance(continuation, list) assert continuation[0]["content"][0]["text"] == "original question" - assert continuation[-2]["role"] == "developer" - assert continuation[-1]["role"] == "assistant" - assert continuation[-1]["content"][0]["text"] == "partial answer" + assert continuation[-2]["role"] == "assistant" + assert continuation[-2]["content"][0]["text"] == "partial answer" + assert continuation[-1]["role"] == "user" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("partial", ["", "1, 2, "]) +async def test_responses_fallback_can_continue_on_a_provider_without_prefill( + partial: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + prompt: Final = "Print the integers 1 through 5, separated by commas." + + def sse(*events: Mapping[str, object]) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events), + ) + + primary: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=sse( + *( + ({ + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": partial, + },) + if partial else () + ), + { + "type": "error", "sequence_number": 1, + "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, + }, + ) + ) + + def no_prefill_provider(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + if body["messages"][-1]["role"] != "user": + return httpx.Response( + 400, + json={"type": "error", "error": { + "type": "invalid_request_error", "message": "The conversation must end with a user message", + }}, + ) + return sse( + {"type": "message_start", "message": { + "id": "msg_fallback", "type": "message", "role": "assistant", + "model": "claude-opus-5-5", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": { + "type": "text_delta", "text": "3, 4, 5" if partial else "1, 2, 3, 4, 5", + }}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}, + {"type": "message_stop"}, + ) + + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=no_prefill_provider) + router: Final = Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + try: + stream: Final = await router.aresponses(model="primary", input=prompt, stream=True, max_output_tokens=100) + events: Final = tuple([event async for event in stream]) + assert "".join( + event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + ) == "1, 2, 3, 4, 5" + assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + assert primary.call_count == fallback.call_count == 1 + messages: Final = json.loads(fallback.calls.last.request.content)["messages"] + assert messages[0] == {"role": "user", "content": [{"type": "text", "text": prompt}]} + if partial: + assert [message["role"] for message in messages] == ["user", "assistant", "user"] + assert messages[-2] == {"role": "assistant", "content": [{"type": "text", "text": partial}]} + else: + assert len(messages) == 1 + finally: + router.discard() + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d645ab4566d..ebddb71a7f6 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -6895,8 +6895,7 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events @pytest.mark.asyncio async def test_aresponses_streaming_iterator_partial_content_injects_continuation(): - """Mid-stream error: input is rewritten to include user prompt + - developer instruction + prior assistant message with partial output.""" + """Mid-stream error: preserve partial assistant output and request continuation in a user turn.""" from litellm.exceptions import MidStreamFallbackError router = _make_router_with_fallback() @@ -6932,11 +6931,11 @@ async def test_aresponses_streaming_iterator_partial_content_injects_continuatio assert isinstance(new_input, list) assert new_input[0]["role"] == "user" assert new_input[0]["content"][0]["text"] == "What's the capital of France?" - assert new_input[1]["role"] == "developer" - assert "do not repeat" in new_input[1]["content"][0]["text"].lower() - assert new_input[2]["role"] == "assistant" - assert new_input[2]["content"][0]["type"] == "output_text" - assert new_input[2]["content"][0]["text"] == "The capital of France is" + assert new_input[1]["role"] == "assistant" + assert new_input[1]["content"][0]["type"] == "output_text" + assert new_input[1]["content"][0]["text"] == "The capital of France is" + assert new_input[2]["role"] == "user" + assert "do not repeat" in new_input[2]["content"][0]["text"].lower() @pytest.mark.asyncio From ccda4e11abf383bf4e17a50d79b99a3ba97b406f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:13:48 -0700 Subject: [PATCH 2/4] fix(router): keep prefill for Claude fallbacks that take it and skip constrained continuations A Responses fallback to a Claude model that takes assistant prefill now gets the partial response as prefill again, so it continues mid-word without the prompt-shape drift. Every other fallback keeps the request to continue. A stream cut after text no longer continues when the request constrains its output (a json_schema or json_object text format, or a forced tool_choice), since a continued JSON document or tool call would not parse. The client gets the primary's error, as on the chat path. The fallback's re-sent leading space at the seam is dropped, and the prefill shaping runs per deployment after the encrypted reasoning strip, so stripped reasoning is never replayed. --- litellm/router.py | 100 ++++---- .../continuation_prefill_check.py | 102 +++++++- .../router_code_coverage.py | 1 + .../base_llm/responses/test_transformation.py | 236 ++++++++++++------ .../test_continuation_prefill_check.py | 197 ++++++++++++++- 5 files changed, 492 insertions(+), 144 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 7a59099ff0d..6b36ff824b5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -299,14 +299,19 @@ from litellm.router_utils.kubernetes_pod_discovery import ( ) from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, + RESPONSES_CONTINUATION_KWARG, ContinuationOutput, ContinuationPrefillDeploymentCheck, ContinuedPartial, + ResponsesContinuation, + ResponsesContinuationSeam, constrains_continuation, continuation_deployments, continuation_output_ceilings, continued_text, deployment_can_continue, + responses_continuation_input, + responses_continuation_input_for, ) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, @@ -480,7 +485,6 @@ if TYPE_CHECKING: from litellm.types.llms.openai import ( ResponseAPIUsage, ResponseIncompleteEvent, - ResponseInputParam, ResponsesAPIResponse, ) @@ -3794,50 +3798,6 @@ class Router: extended: Final = _extend_trailing_text_block(blocks, new_text) return [*messages[:-1], {**last, "content": extended, "prefix": True}] - @staticmethod - def _build_responses_continuation_input( - input_val: Union[str, "ResponseInputParam"] | None, - generated_content: str, - ) -> "ResponseInputParam": - """Request a best-effort continuation using assistant history followed by a user turn, so the fallback - does not need to support assistant prefill.""" - base: list[object] - if isinstance(input_val, str): - base = [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": input_val}], - } - ] - elif isinstance(input_val, list): - base = list(input_val) - else: - base = [] - continuation: Final[list[object]] = [ - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": generated_content}], - }, - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": ( - "The previous assistant response was interrupted " - "mid-stream. Continue exactly where it stopped — " - "do not repeat any of its content. Your response " - "must read as a seamless continuation." - ), - } - ], - }, - ] - return cast("ResponseInputParam", base + continuation) - async def _aresponses_streaming_iterator( self, response: "BaseResponsesAPIStreamingIterator", @@ -3859,9 +3819,11 @@ class Router: Full parity with the chat-completions path: - Pre-first-chunk: retry with the original input unchanged. - - Partial content: inject a developer instruction + prior - assistant message carrying the generated text so the fallback - model continues rather than restarts. + - Partial content: continue from the generated text, as an + assistant prefill on a fallback that takes one and as assistant + history plus a user request to continue on any other. A request + whose output is constrained (a JSON schema, a forced tool) is not + continued, and the primary's error surfaces instead. - Usage combining: merge partial-stream usage onto the fallback's response.completed event so accounting reflects both attempts. - Stream cleanup: shielded aclose() on both source and fallback @@ -4050,6 +4012,28 @@ class Router: wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks()) return wrapper + def _responses_continuation( + self, + e: "MidStreamFallbackError", + source_iterator: "BaseResponsesAPIStreamingIterator", + request_kwargs: Mapping[str, object], + ) -> ResponsesContinuation | None: + """What a fallback continues from once a Responses stream broke after text, raising the primary's error + when the request constrains its output, since a continued JSON document or forced tool call would not + parse.""" + if not e.generated_content or e.is_pre_first_chunk: + return None + cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") + cut_deployment: Final = ( + self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None + ) + if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): + Router._raise_original_mid_stream_error(e) + earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG) + if isinstance(earlier, ResponsesContinuation): + return earlier.extended(e.generated_content) + return ResponsesContinuation(request_kwargs.get("input"), e.generated_content) + async def _aresponses_fallback_attempt( self, e: "MidStreamFallbackError", @@ -4068,6 +4052,8 @@ class Router: """ from litellm.exceptions import MidStreamFallbackError + continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs) + seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "") partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement @@ -4085,11 +4071,9 @@ class Router: initial_kwargs["original_function"] = ( # rebind-ok: the fallback chain re-enters on the same kwargs self._ageneric_api_call_with_fallbacks_responses_attempt ) - if e.generated_content and not e.is_pre_first_chunk: - initial_kwargs["input"] = Router._build_responses_continuation_input( # rebind-ok: fallback hop input - initial_kwargs.get("input"), - e.generated_content, - ) + if continuation is not None: + initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment + initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input # The Responses-API path stores observability metadata # under "litellm_metadata" (not the default "metadata") — # see _ageneric_api_call_with_fallbacks. Mirroring that @@ -4118,7 +4102,7 @@ class Router: ) prepared_fallback_hidden_params: Final = adopt_headers(fallback_response) if hasattr(fallback_response, "__aiter__"): - async for fallback_item in fallback_response: + async for fallback_item in seam.splice(fallback_response): Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) if partial_usage is not None: Router._combine_responses_fallback_usage(fallback_item, partial_usage) @@ -4641,11 +4625,12 @@ class Router: splits_stream_timeout: bool = False, ) -> None: """ - 4 jobs: + 5 jobs: - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging) - Adds default litellm params to kwargs, if set. - Merges tools from deployment with request (proxy-configured tools + request tools). - On a fallback hop, drops the encrypted reasoning this deployment cannot decrypt, keeping its summary. + - On a Responses continuation, sends the partial response as prefill where this deployment takes one. """ for key in self._forwarded_alias_marker_keys_the_deployment_sets( deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) @@ -4805,6 +4790,11 @@ class Router: (_FALLBACK_HOP_ADAPTER.validate_python(deployment),), unmarked_origin=deployment_that_just_failed, ) + if RESPONSES_CONTINUATION_KWARG in hop_kwargs: + del kwargs[RESPONSES_CONTINUATION_KWARG] # rebind-ok: this method prepares the call's kwargs in place + kwargs["input"] = responses_continuation_input_for( # rebind-ok: the deployment's continuation input + hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment) + ) def _get_async_openai_model_client(self, deployment: dict, kwargs: dict): """ diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 8f211ef67db..6811d9dc79d 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -1,12 +1,12 @@ """Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave -the output unconstrained.""" +the output unconstrained, and shapes a Responses API continuation for the deployment that serves it.""" from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence from dataclasses import dataclass from itertools import accumulate from os.path import commonprefix from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar from pydantic import TypeAdapter, ValidationError @@ -14,7 +14,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.rules import Rules from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from litellm.utils import supports_assistant_prefill @@ -29,6 +29,7 @@ _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) _OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens") +RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation" def _str_keyed(value: object) -> Mapping[str, object]: @@ -54,10 +55,12 @@ def constrains_continuation(params: Mapping[str, object]) -> bool: sent: Final = {**params, **_str_keyed(params.get("extra_body"))} choice_count: Final = sent.get("n") response_format: Final = sent.get("response_format") + text_format: Final = _str_keyed(sent.get("text")).get("format") tool_choice: Final = sent.get("tool_choice") return ( (isinstance(choice_count, int) and choice_count > 1) or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) + or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT) or tool_choice == "required" or isinstance(tool_choice, Mapping) or sent.get("merge_reasoning_content_in_choices") is True @@ -99,6 +102,7 @@ def _delta_text(chunk: ModelResponseStream) -> str: _CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) +_INPUT_ITEMS_ADAPTER: Final = TypeAdapter(list[object]) def _prefill_text(prefill_message: Mapping[str, object]) -> str: @@ -114,7 +118,7 @@ def _prefill_text(prefill_message: Mapping[str, object]) -> str: ) -def _resendable_whitespace(prefill: str) -> str: +def resendable_whitespace(prefill: str) -> str: return prefill[len(prefill.rstrip()) :] if prefill.strip() else "" @@ -124,7 +128,7 @@ def continued_text(prefill_message: Mapping[str, object], generated_content: str return None echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0 after_echo: Final = generated_content[echo:] - return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :] + return after_echo[len(commonprefix((after_echo, resendable_whitespace(prefill)))) :] def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]: @@ -211,7 +215,7 @@ class ContinuationOutput: self._rules: Final = Rules() self._held: tuple[ModelResponseStream, ...] = () self._echo_settled = not self._prefill_text - self._seam = _resendable_whitespace(self._prefill_text) + self._seam = resendable_whitespace(self._prefill_text) async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]: async for item in stream: @@ -262,3 +266,89 @@ class ContinuationOutput: self._text += _delta_text(chunk) self._rules.post_call_rules(input=self._text, model=self._model) return chunks + + +@dataclass(frozen=True, slots=True) +class ResponsesContinuation: + prior_input: object + emitted_text: str + + def extended(self, generated_content: str) -> "ResponsesContinuation": + return ResponsesContinuation(self.prior_input, f"{self.emitted_text}{generated_content}") + + +def deployment_takes_responses_prefill(deployment: object) -> bool: + """Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot + carry.""" + model: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")).get("model") + return isinstance(model, str) and "claude" in model.lower() and deployment_supports_prefill(deployment) + + +def _continuation_request(emitted_text: str) -> Mapping[str, object]: + seam: Final = ( + "It already ends with whitespace, so do not add another space before your first word." + if emitted_text[-1:].isspace() + else "It ends without trailing whitespace, so begin with a space if your first word starts a new word." + ) + instruction: Final = ( + "The previous assistant response was interrupted mid-stream. Continue exactly where it stopped and do not " + f"repeat any of its content. {seam} Your response must read as a seamless continuation." + ) + return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": instruction}]} + + +def _input_items(request_input: object) -> tuple[object, ...]: + if isinstance(request_input, str): + return ({"type": "message", "role": "user", "content": [{"type": "input_text", "text": request_input}]},) + try: + return tuple(_INPUT_ITEMS_ADAPTER.validate_python(request_input)) + except ValidationError: + return () + + +def responses_continuation_input(continuation: ResponsesContinuation) -> Sequence[object]: + """The prior input, the partial response as assistant history, and a user request to continue it.""" + partial: Final = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": continuation.emitted_text}], + } + return [*_input_items(continuation.prior_input), partial, _continuation_request(continuation.emitted_text)] + + +def responses_continuation_input_for(request_kwargs: Mapping[str, object], deployment: Mapping[str, object]) -> object: + """The continuation input as the deployment receives it, without the request to continue where the deployment + takes the partial response as prefill. It reads the input the router already stripped of reasoning this + deployment cannot decrypt, never the continuation's prior input.""" + request_input: Final = request_kwargs.get("input") + continuation: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG) + if not isinstance(continuation, ResponsesContinuation): + return request_input + items: Final = _input_items(request_input) + if items[-1:] != (_continuation_request(continuation.emitted_text),) or not deployment_takes_responses_prefill( + deployment + ): + return request_input + return list(items[:-1]) + + +_StreamItem = TypeVar("_StreamItem") + + +class ResponsesContinuationSeam: + """Drops whitespace a fallback sends again at the start of its text when the client already has it, since + Anthropic strips a prefill's trailing whitespace and the model then writes it back.""" + + def __init__(self, emitted_text: str) -> None: + self._seam = resendable_whitespace(emitted_text) + + async def splice(self, stream: AsyncIterable[_StreamItem]) -> AsyncIterator[_StreamItem]: + async for item in stream: + yield self._past(item) + + def _past(self, item: _StreamItem) -> _StreamItem: + if not self._seam or not isinstance(item, OutputTextDeltaEvent): + return item + matched: Final = len(commonprefix((item.delta, self._seam))) + self._seam = self._seam[matched:] if matched == len(item.delta) else "" + return item.model_copy(update={"delta": item.delta[matched:]}) if matched else item diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 7809a23e955..2f460e29a06 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -155,6 +155,7 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py ] diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index e5a685fc805..9f18b377770 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -881,39 +881,6 @@ def test_combine_responses_fallback_usage_passthrough_for_unknown_event(): partial = ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2) Router._combine_responses_fallback_usage(other, partial) -@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") -def test_build_responses_continuation_input_from_string(): - out = Router._build_responses_continuation_input("Hello world", "partial assistant text") - assert len(out) == 3 - assert out[0]["role"] == "user" - assert out[0]["content"][0]["text"] == "Hello world" - assert out[1]["role"] == "assistant" - assert out[1]["content"][0]["text"] == "partial assistant text" - assert out[2]["role"] == "user" - -@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") -def test_build_responses_continuation_input_from_list_preserves_items(): - existing: List[Any] = [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "msg1"}], - } - ] - out = Router._build_responses_continuation_input(existing, "partial") - assert len(out) == 3 - assert out[0]["content"][0]["text"] == "msg1" - assert out[1]["role"] == "assistant" - assert out[2]["role"] == "user" - assert existing == out[:1] - -@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") -def test_build_responses_continuation_input_from_none(): - out = Router._build_responses_continuation_input(None, "partial") - assert len(out) == 2 - assert out[0]["role"] == "assistant" - assert out[1]["role"] == "user" - @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_streaming_iterator_passthrough(): @@ -1363,6 +1330,70 @@ async def test_aresponses_fallback_uses_continuation_input_after_partial_content assert continuation[-1]["role"] == "user" +def _sse(*events: Mapping[str, object]) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events), + ) + + +def _openai_stream_cut_after(partial: str) -> httpx.Response: + return _sse( + *( + ({ + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": partial, + },) + if partial else () + ), + { + "type": "error", "sequence_number": 1, + "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, + }, + ) + + +def _anthropic_stream(text: str) -> httpx.Response: + return _sse( + {"type": "message_start", "message": { + "id": "msg_fallback", "type": "message", "role": "assistant", + "model": "claude-fallback", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}, + {"type": "message_stop"}, + ) + + +def _primary_with_fallback(fallback_model: str) -> Router: + return Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": fallback_model, "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + + +def _streamed_text(events: tuple[Any, ...]) -> str: + return "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) + + +@pytest.fixture +def claude_with_prefill() -> str: + model: Final = "anthropic/claude-prefill-capable-test-model" + litellm.register_model( + {model: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}}, + persist_across_reloads=False, + ) + litellm.get_model_info.cache_clear() + return model + + @pytest.mark.asyncio @pytest.mark.parametrize("partial", ["", "1, 2, "]) async def test_responses_fallback_can_continue_on_a_provider_without_prefill( @@ -1370,28 +1401,8 @@ async def test_responses_fallback_can_continue_on_a_provider_without_prefill( ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) prompt: Final = "Print the integers 1 through 5, separated by commas." - - def sse(*events: Mapping[str, object]) -> httpx.Response: - return httpx.Response( - 200, - headers={"content-type": "text/event-stream"}, - content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events), - ) - primary: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( - return_value=sse( - *( - ({ - "type": "response.output_text.delta", "sequence_number": 0, - "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": partial, - },) - if partial else () - ), - { - "type": "error", "sequence_number": 1, - "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, - }, - ) + return_value=_openai_stream_cut_after(partial) ) def no_prefill_provider(request: httpx.Request) -> httpx.Response: @@ -1403,35 +1414,14 @@ async def test_responses_fallback_can_continue_on_a_provider_without_prefill( "type": "invalid_request_error", "message": "The conversation must end with a user message", }}, ) - return sse( - {"type": "message_start", "message": { - "id": "msg_fallback", "type": "message", "role": "assistant", - "model": "claude-opus-5-5", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0}, - }}, - {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, - {"type": "content_block_delta", "index": 0, "delta": { - "type": "text_delta", "text": "3, 4, 5" if partial else "1, 2, 3, 4, 5", - }}, - {"type": "content_block_stop", "index": 0}, - {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}, - {"type": "message_stop"}, - ) + return _anthropic_stream("3, 4, 5" if partial else "1, 2, 3, 4, 5") fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=no_prefill_provider) - router: Final = Router( - model_list=[ - {"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}}, - {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}}, - ], - fallbacks=[{"primary": ["fallback"]}], - num_retries=0, - ) + router: Final = _primary_with_fallback("anthropic/claude-opus-5-5") try: stream: Final = await router.aresponses(model="primary", input=prompt, stream=True, max_output_tokens=100) events: Final = tuple([event async for event in stream]) - assert "".join( - event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA - ) == "1, 2, 3, 4, 5" + assert _streamed_text(events) == "1, 2, 3, 4, 5" assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED assert primary.call_count == fallback.call_count == 1 messages: Final = json.loads(fallback.calls.last.request.content)["messages"] @@ -1445,6 +1435,96 @@ async def test_responses_fallback_can_continue_on_a_provider_without_prefill( router.discard() +@pytest.mark.asyncio +async def test_responses_fallback_continues_a_claude_that_takes_prefill_from_its_own_partial( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream(" 3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Print the integers 1 through 5.", stream=True, max_output_tokens=100 + ) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + messages: Final = json.loads(fallback.calls.last.request.content)["messages"] + assert [message["role"] for message in messages] == ["user", "assistant"] + assert messages[-1]["content"][0]["text"].startswith("1, 2,") + finally: + router.discard() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fallback_takes_prefill", [True, False], ids=["prefill", "request_to_continue"]) +async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cannot_decrypt( + fallback_takes_prefill: bool, + claude_with_prefill: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream(" 3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill if fallback_takes_prefill else "anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses( + model="primary", + input=[ + {"role": "user", "content": "Print the integers 1 through 5."}, + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "primary-only-ciphertext"}, + ], + stream=True, + max_output_tokens=100, + ) + _ = [event async for event in stream] + assert fallback.call_count == 1 + assert "primary-only-ciphertext" not in fallback.calls.last.request.content.decode() + finally: + router.discard() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "constraint", + [ + {"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}}, + {"text": {"format": {"type": "json_object"}}}, + {"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]}, + ], + ids=["json_schema", "json_object", "required_tool"], +) +async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed( + constraint: dict[str, object], + claude_with_prefill: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_after('{"numbers": [1, 2, ') + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3]}") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Numbers 1 to 3 as JSON.", stream=True, max_output_tokens=100, **constraint + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_client_error_event_skips_fallback(): diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py index 8fa1c799863..dd12ad0bf2b 100644 --- a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -4,13 +4,23 @@ import litellm from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_KWARG, MID_STREAM_CONTINUATION_MARKER, + RESPONSES_CONTINUATION_KWARG, ContinuationOutput, ContinuationPrefillDeploymentCheck, + ResponsesContinuation, + ResponsesContinuationSeam, + constrains_continuation, continuation_output_ceilings, deployment_supports_prefill, + deployment_takes_responses_prefill, + responses_continuation_input, + responses_continuation_input_for, ) +from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents PREFILL_MODEL = "anthropic/prefill-capable-test-model" +CLAUDE_PREFILL_MODEL = "anthropic/claude-prefill-capable-test-model" +CLAUDE_NON_PREFILL_MODEL = "anthropic/claude-prefill-rejecting-test-model" NON_PREFILL_MODEL = "openai/prefill-unknown-test-model" UNMAPPED_MODEL = "openai/unmapped-test-model" @@ -21,6 +31,12 @@ def _synthetic_cost_map_entries() -> None: { PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}, NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"}, + CLAUDE_PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}, + CLAUDE_NON_PREFILL_MODEL: { + "litellm_provider": "anthropic", + "mode": "chat", + "supports_assistant_prefill": False, + }, }, persist_across_reloads=False, ) @@ -191,7 +207,11 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream: ("Hello", (" world",), (" world",)), ("Hello", ("Hel", "p me"), ("Hel", "p me")), ("Hello", ("Hel",), ("Hel",)), - ([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")), + ( + [{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], + ("Sure: Hello", " world"), + ("", " world"), + ), ], ids=[ "echo_split_across_chunks_is_dropped", @@ -206,9 +226,7 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream: async def test_continuation_output_drops_only_a_full_echo_of_the_prefill( prefill_content, fallback_texts, expected_texts ): - output = ContinuationOutput( - "Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", () - ) + output = ContinuationOutput("Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ()) async def fallback_stream(): for text in fallback_texts: @@ -266,7 +284,14 @@ async def test_continuation_output_drops_only_whitespace_the_client_already_has( ({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}), ({"extra_body": {"top_k": 3}}, {}), ], - ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"], + ids=[ + "max_tokens", + "both_ceilings", + "no_ceiling", + "extra_body_max_tokens", + "extra_body_only", + "extra_body_no_ceiling", + ], ) def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends( request_kwargs, expected @@ -286,3 +311,165 @@ def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_ ) def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs): assert continuation_output_ceilings(request_kwargs, 30) is None + + +@pytest.mark.parametrize( + "sent,expected", + [ + ({"text": {"format": {"type": "json_schema", "name": "n", "schema": {"type": "object"}}}}, True), + ({"text": {"format": {"type": "json_object"}}}, True), + ({"text": {"format": {"type": "text"}}}, False), + ({"text": {"verbosity": "low"}}, False), + ({"tool_choice": "required"}, True), + ({"tool_choice": "auto"}, False), + ], + ids=["json_schema", "json_object", "text_format", "verbosity_only", "required_tool", "auto_tool"], +) +def test_constrains_continuation_reads_the_responses_text_format(sent, expected): + assert constrains_continuation(sent) is expected + + +@pytest.mark.parametrize( + "deployment,expected", + [ + (_deployment(CLAUDE_PREFILL_MODEL, "a"), True), + (_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False), + (_deployment(PREFILL_MODEL, "c"), False), + ], + ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude"], +) +def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected): + assert deployment_takes_responses_prefill(deployment) is expected + + +def _continuation_kwargs() -> dict[str, object]: + continuation = ResponsesContinuation("Count to 5.", "1, 2, ") + return {RESPONSES_CONTINUATION_KWARG: continuation, "input": responses_continuation_input(continuation)} + + +@pytest.mark.parametrize( + "prior_input,expected_prior", + [ + ( + "Count to 5.", + [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Count to 5."}]}], + ), + ( + [ + {"role": "user", "content": "Count to 5."}, + {"type": "function_call_output", "call_id": "c", "output": "x"}, + ], + [ + {"role": "user", "content": "Count to 5."}, + {"type": "function_call_output", "call_id": "c", "output": "x"}, + ], + ), + (None, []), + ], + ids=["string_input", "item_list_input", "no_input"], +) +def test_responses_continuation_input_keeps_the_prior_input_then_the_partial_then_a_request_to_continue( + prior_input, expected_prior +): + built = responses_continuation_input(ResponsesContinuation(prior_input, "1, 2, ")) + + partial = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "1, 2, "}]} + assert built[:-1] == [*expected_prior, partial] + assert built[-1]["role"] == "user" + assert "already ends with whitespace" in built[-1]["content"][0]["text"] + + +def test_responses_continuation_request_asks_for_a_leading_space_only_after_a_word(): + built = responses_continuation_input(ResponsesContinuation("Hi", "Transformers use")) + + assert "begin with a space" in built[-1]["content"][0]["text"] + + +def test_responses_continuation_accumulates_across_hops(): + continuation = ResponsesContinuation("Count to 5.", "1, ").extended("2, ") + + assert continuation == ResponsesContinuation("Count to 5.", "1, 2, ") + + +@pytest.mark.parametrize( + "deployment,expected_roles", + [ + (_deployment(CLAUDE_PREFILL_MODEL, "a"), ["user", "assistant"]), + (_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), ["user", "assistant", "user"]), + (_deployment(NON_PREFILL_MODEL, "c"), ["user", "assistant", "user"]), + ], + ids=["claude_prefill", "claude_without_prefill", "openai"], +) +def test_responses_continuation_input_for_shapes_the_input_for_the_deployment_that_serves_it( + deployment, expected_roles +): + shaped = responses_continuation_input_for(_continuation_kwargs(), deployment) + + assert [item["role"] for item in shaped] == expected_roles + + +def test_responses_continuation_input_for_keeps_what_the_router_stripped_from_the_input(): + continuation = ResponsesContinuation( + [ + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "opaque"}, + {"role": "user", "content": "Count to 5."}, + ], + "1, 2, ", + ) + stripped_input = responses_continuation_input(continuation)[1:] + + shaped = responses_continuation_input_for( + {RESPONSES_CONTINUATION_KWARG: continuation, "input": stripped_input}, _deployment(CLAUDE_PREFILL_MODEL, "a") + ) + + assert shaped == stripped_input[:-1] + + +def test_responses_continuation_input_for_leaves_an_input_without_its_request_to_continue_alone(): + kwargs = {**_continuation_kwargs(), "input": [{"role": "user", "content": "Count to 5."}]} + + assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == kwargs["input"] + + +def test_responses_continuation_input_for_leaves_a_request_without_a_continuation_alone(): + kwargs = {"input": "Count to 5."} + + assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == "Count to 5." + + +def _delta(text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="m", output_index=0, content_index=0, delta=text + ) + + +@pytest.mark.parametrize( + "emitted,fallback_texts,expected_texts", + [ + ("1, 2, ", (" 3, 4", ", 5"), ("3, 4", ", 5")), + ("1, 2, ", (" ", " 3"), ("", " 3")), + ("1, 2,", (" 3",), (" 3",)), + ("1, 2, ", ("3",), ("3",)), + ("A.\n\n", ("\n Hi",), (" Hi",)), + ("", (" Hi",), (" Hi",)), + ], + ids=[ + "re_sent_space_is_dropped", + "seam_split_across_events", + "no_trailing_whitespace_keeps_the_space", + "no_re_sent_space_passes_through", + "seam_ends_at_the_first_divergence", + "fresh_start_passes_through", + ], +) +@pytest.mark.asyncio +async def test_responses_continuation_seam_drops_only_whitespace_the_client_already_has( + emitted, fallback_texts, expected_texts +): + async def fallback_stream(): + for text in fallback_texts: + yield _delta(text) + + released = [event async for event in ResponsesContinuationSeam(emitted).splice(fallback_stream())] + + assert tuple(event.delta for event in released) == expected_texts From f9116cc3f1a689c3a3fcd1d3fcd7b029beddc5cd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:27:37 -0700 Subject: [PATCH 3/4] fix(router): guard SDK text_format, narrow forced tool_choice, read Claude base_model, and cap Responses continuation output Cut Responses streams that asked for structured output through the SDK's text_format now surface the primary's error instead of stitching two models' JSON. Only a required or named tool_choice counts as forced, so allowed_tools in auto mode and hosted tool choices still continue. A Claude deployment named only through base_model keeps its prefill continuation, and the continuation's max_output_tokens is lowered by what the cut stream already emitted. --- litellm/router.py | 20 ++++++- .../continuation_prefill_check.py | 32 ++++++++---- .../router_code_coverage.py | 1 + .../base_llm/responses/test_transformation.py | 52 ++++++++++++++++++- .../test_continuation_prefill_check.py | 25 ++++++++- 5 files changed, 115 insertions(+), 15 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6b36ff824b5..40b555cffbd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -454,6 +454,7 @@ from litellm.utils import ( provider_rejectable_params, resolve_api_key_secret_reference, set_live_deployment_replay, + token_counter, ) from .router_utils.pattern_match_deployments import PatternMatchRouter @@ -3719,7 +3720,8 @@ class Router: ) -> Mapping[str, object]: """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} - defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None} + default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params) + defaults: Final = {key: value for key, value in default_params.items() if value is not None} deployment_tool_choice: Final = ( {"tool_choice": deployment_params["tool_choice"]} if deployment_params.get("tool_choice") is not None @@ -4034,6 +4036,18 @@ class Router: return earlier.extended(e.generated_content) return ResponsesContinuation(request_kwargs.get("input"), e.generated_content) + @staticmethod + def _responses_continuation_ceilings( + e: "MidStreamFallbackError", request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + """The caller's output ceilings less what the cut stream already emitted, raising the primary's error once + nothing is left.""" + emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content) + reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens) + if reduced is None: + Router._raise_original_mid_stream_error(e) + return reduced + async def _aresponses_fallback_attempt( self, e: "MidStreamFallbackError", @@ -4053,6 +4067,9 @@ class Router: from litellm.exceptions import MidStreamFallbackError continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs) + reduced_ceilings: Final[Mapping[str, object]] = ( + Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {} + ) seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "") partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted @@ -4072,6 +4089,7 @@ class Router: self._ageneric_api_call_with_fallbacks_responses_attempt ) if continuation is not None: + initial_kwargs.update(reduced_ceilings) initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input # The Responses-API path stores observability metadata diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 6811d9dc79d..fc360a9df7b 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -28,7 +28,8 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker() _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) -_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens") +_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens") +_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"}) RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation" @@ -39,30 +40,39 @@ def _str_keyed(value: object) -> Mapping[str, object]: return _NO_FIELDS -def deployment_supports_prefill(deployment: object) -> bool: +def _underlying_model(deployment: object) -> str | None: deployment_map: Final = _str_keyed(deployment) model_info: Final = _str_keyed(deployment_map.get("model_info")) - declared: Final = model_info.get("supports_assistant_prefill") - if isinstance(declared, bool): - return declared litellm_params: Final = _str_keyed(deployment_map.get("litellm_params")) candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model")) - model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + + +def deployment_supports_prefill(deployment: object) -> bool: + declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill") + if isinstance(declared, bool): + return declared + model: Final = _underlying_model(deployment) return model is not None and supports_assistant_prefill(model=model) +def _forces_tool_call(tool_choice: object) -> bool: + choice: Final = _str_keyed(tool_choice) + mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode") + return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES + + def constrains_continuation(params: Mapping[str, object]) -> bool: sent: Final = {**params, **_str_keyed(params.get("extra_body"))} choice_count: Final = sent.get("n") response_format: Final = sent.get("response_format") text_format: Final = _str_keyed(sent.get("text")).get("format") - tool_choice: Final = sent.get("tool_choice") return ( (isinstance(choice_count, int) and choice_count > 1) or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT) - or tool_choice == "required" - or isinstance(tool_choice, Mapping) + or sent.get("text_format") is not None + or _forces_tool_call(sent.get("tool_choice")) or sent.get("merge_reasoning_content_in_choices") is True ) @@ -280,8 +290,8 @@ class ResponsesContinuation: def deployment_takes_responses_prefill(deployment: object) -> bool: """Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot carry.""" - model: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")).get("model") - return isinstance(model, str) and "claude" in model.lower() and deployment_supports_prefill(deployment) + model: Final = _underlying_model(deployment) + return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment) def _continuation_request(emitted_text: str) -> Mapping[str, object]: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 2f460e29a06..f76193f9a89 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -156,6 +156,7 @@ ignored_function_names = [ "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py + "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py ] diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 9f18b377770..d1992e106d8 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -11,6 +11,7 @@ import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.router import GenericLiteLLMParams from litellm import Router +from pydantic import BaseModel, ConfigDict from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -1490,6 +1491,12 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann router.discard() +class _Numbers(BaseModel): + model_config = ConfigDict(frozen=True) + + numbers: tuple[int, ...] + + @pytest.mark.asyncio @pytest.mark.parametrize( "constraint", @@ -1497,8 +1504,9 @@ async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cann {"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}}, {"text": {"format": {"type": "json_object"}}}, {"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]}, + {"text_format": _Numbers}, ], - ids=["json_schema", "json_object", "required_tool"], + ids=["json_schema", "json_object", "required_tool", "sdk_text_format"], ) async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed( constraint: dict[str, object], @@ -1525,6 +1533,48 @@ async def test_responses_fallback_surfaces_the_primary_error_once_constrained_ou router.discard() +@pytest.mark.asyncio +async def test_responses_fallback_continues_within_the_callers_output_ceiling( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter( + text="1, 2, " + ) + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ") + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_client_error_event_skips_fallback(): diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py index dd12ad0bf2b..4492f8512ae 100644 --- a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -322,8 +322,25 @@ def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left( ({"text": {"verbosity": "low"}}, False), ({"tool_choice": "required"}, True), ({"tool_choice": "auto"}, False), + ({"tool_choice": {"type": "function", "name": "lookup"}}, True), + ({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True), + ({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False), + ({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False), + ({"tool_choice": {"type": "web_search_preview"}}, False), + ], + ids=[ + "json_schema", + "json_object", + "text_format", + "verbosity_only", + "required_tool", + "auto_tool", + "forced_function", + "allowed_tools_required", + "allowed_tools_auto", + "chat_allowed_tools_auto", + "hosted_tool", ], - ids=["json_schema", "json_object", "text_format", "verbosity_only", "required_tool", "auto_tool"], ) def test_constrains_continuation_reads_the_responses_text_format(sent, expected): assert constrains_continuation(sent) is expected @@ -335,8 +352,12 @@ def test_constrains_continuation_reads_the_responses_text_format(sent, expected) (_deployment(CLAUDE_PREFILL_MODEL, "a"), True), (_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False), (_deployment(PREFILL_MODEL, "c"), False), + ( + {"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}}, + True, + ), ], - ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude"], + ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"], ) def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected): assert deployment_takes_responses_prefill(deployment) is expected From 81ada872f5dbd0736bed5e991395a4f906a8b3d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:15:12 -0700 Subject: [PATCH 4/4] fix(router): quote the partial's ending in the continue request and surface the primary's error after a cut tool call --- .../streaming_iterator.py | 1 + litellm/responses/streaming_iterator.py | 9 ++ litellm/router.py | 11 +- .../continuation_prefill_check.py | 8 +- .../base_llm/responses/test_transformation.py | 101 ++++++++++++++++++ 5 files changed, 124 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 84d4648f75f..e9db0a7aa27 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -349,6 +349,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): continue if call_id in self._web_search_calls: continue + self._started_tool_call = True fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) fn_name = "" diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index c5dc45566a9..1387b05b11c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -363,6 +363,7 @@ class BaseResponsesAPIStreamingIterator: """ _pending_logging_tasks: tuple[asyncio.Task[object], ...] = () + _started_tool_call: bool = False def __init__( self, @@ -491,6 +492,8 @@ class BaseResponsesAPIStreamingIterator: ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, ): _item: Final[object] = getattr(openai_responses_api_chunk, "item", None) + if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES: + self._started_tool_call = True if _item is not None: ResponsesAPIRequestUtils.encode_container_id_on_output_item( item=_item, @@ -1037,6 +1040,11 @@ class BaseResponsesAPIStreamingIterator: if pending: await asyncio.wait(pending) + @property + def started_tool_call(self) -> bool: + """Whether the stream began a tool call, which a text continuation could never finish.""" + return self._started_tool_call + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: """ The response.incomplete event to bill when the client went away mid-stream, @@ -1683,6 +1691,7 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None: return None +_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"}) _TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset( { ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, diff --git a/litellm/router.py b/litellm/router.py index 40b555cffbd..6b3136fcb35 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3824,8 +3824,9 @@ class Router: - Partial content: continue from the generated text, as an assistant prefill on a fallback that takes one and as assistant history plus a user request to continue on any other. A request - whose output is constrained (a JSON schema, a forced tool) is not - continued, and the primary's error surfaces instead. + whose output is constrained (a JSON schema, a forced tool), or a + stream that began a tool call, is not continued, and the primary's + error surfaces instead. - Usage combining: merge partial-stream usage onto the fallback's response.completed event so accounting reflects both attempts. - Stream cleanup: shielded aclose() on both source and fallback @@ -4021,10 +4022,12 @@ class Router: request_kwargs: Mapping[str, object], ) -> ResponsesContinuation | None: """What a fallback continues from once a Responses stream broke after text, raising the primary's error - when the request constrains its output, since a continued JSON document or forced tool call would not - parse.""" + when the request constrains its output or the stream began a tool call, since a continued JSON document or + tool call would not parse.""" if not e.generated_content or e.is_pre_first_chunk: return None + if source_iterator.started_tool_call: + Router._raise_original_mid_stream_error(e) cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") cut_deployment: Final = ( self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index fc360a9df7b..37e10dd7b4f 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -1,6 +1,7 @@ """Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave the output unconstrained, and shapes a Responses API continuation for the deployment that serves it.""" +import json from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence from dataclasses import dataclass from itertools import accumulate @@ -30,6 +31,7 @@ _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) _OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens") _TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"}) +_QUOTED_ENDING_CHARS: Final = 100 RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation" @@ -300,9 +302,11 @@ def _continuation_request(emitted_text: str) -> Mapping[str, object]: if emitted_text[-1:].isspace() else "It ends without trailing whitespace, so begin with a space if your first word starts a new word." ) + ending: Final = json.dumps(emitted_text[-_QUOTED_ENDING_CHARS:], ensure_ascii=False) instruction: Final = ( - "The previous assistant response was interrupted mid-stream. Continue exactly where it stopped and do not " - f"repeat any of its content. {seam} Your response must read as a seamless continuation." + f"The previous assistant response was interrupted mid-stream. It ended with {ending}. Continue exactly where " + "it stopped, so your first characters are the ones that would have come right after it, and do not repeat " + f"any of its content. {seam} Your response must read as a seamless continuation." ) return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": instruction}]} diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index d1992e106d8..343524e3b66 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -1430,6 +1430,7 @@ async def test_responses_fallback_can_continue_on_a_provider_without_prefill( if partial: assert [message["role"] for message in messages] == ["user", "assistant", "user"] assert messages[-2] == {"role": "assistant", "content": [{"type": "text", "text": partial}]} + assert json.dumps(partial) in messages[-1]["content"][0]["text"] else: assert len(messages) == 1 finally: @@ -1575,6 +1576,106 @@ async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream router.discard() +def _openai_stream_cut_mid_tool_call(preamble: str) -> httpx.Response: + return _sse( + { + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble, + }, + { + "type": "response.output_item.added", "sequence_number": 1, "output_index": 1, + "item": { + "type": "function_call", "id": "fc_primary", "call_id": "call_primary", + "name": "lookup", "arguments": "", "status": "in_progress", + }, + }, + { + "type": "response.function_call_arguments.delta", "sequence_number": 2, + "item_id": "fc_primary", "output_index": 1, "delta": '{"city": "Par', + }, + { + "type": "error", "sequence_number": 3, + "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, + }, + ) + + +def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response: + return _sse( + {"type": "message_start", "message": { + "id": "msg_primary", "type": "message", "role": "assistant", + "model": "claude-primary", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}}, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": { + "type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {}, + }}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}}, + {"type": "error", "error": {"type": "overloaded_error", "message": "interrupted"}}, + ) + + +@pytest.mark.asyncio +async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_mid_tool_call("Let me look that up. ") + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("It is sunny in Paris.") + ) + router: Final = _primary_with_fallback("anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses( + model="primary", + input="What is the weather in Paris?", + stream=True, + tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}], + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + side_effect=[ + _anthropic_stream_cut_mid_tool_call("Let me look that up. "), + _anthropic_stream("It is sunny in Paris."), + ] + ) + router: Final = Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + try: + stream: Final = await router.aresponses( + model="primary", + input="What is the weather in Paris?", + stream=True, + tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}], + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert provider.call_count == 1 + finally: + router.discard() + + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_client_error_event_skips_fallback():