diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index b5ef9e95741..87e991f0c5a 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -408,6 +408,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): continue if call_id in self._web_search_calls: continue + self._started_tool_call = True fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None) fn_name = "" diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 50ab3e4f75a..0e0d0ddade3 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -365,6 +365,7 @@ class BaseResponsesAPIStreamingIterator: """ _pending_logging_tasks: tuple[asyncio.Task[object], ...] = () + _started_tool_call: bool = False _earlier_rounds_spend: BilledRoundsSpend | None = None def __init__( @@ -494,6 +495,8 @@ class BaseResponsesAPIStreamingIterator: ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, ): _item: Final[object] = getattr(openai_responses_api_chunk, "item", None) + if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES: + self._started_tool_call = True if _item is not None: ResponsesAPIRequestUtils.encode_container_id_on_output_item( item=_item, @@ -1044,6 +1047,11 @@ class BaseResponsesAPIStreamingIterator: if pending: await asyncio.wait(pending) + @property + def started_tool_call(self) -> bool: + """Whether the stream began a tool call, which a text continuation could never finish.""" + return self._started_tool_call + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: """ The response.incomplete event to bill when the client went away mid-stream, @@ -1690,6 +1698,7 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None: return None +_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"}) _TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset( { ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..94f6b0c10fc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -301,14 +301,19 @@ from litellm.router_utils.kubernetes_pod_discovery import ( ) from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, + RESPONSES_CONTINUATION_KWARG, ContinuationOutput, ContinuationPrefillDeploymentCheck, ContinuedPartial, + ResponsesContinuation, + ResponsesContinuationSeam, constrains_continuation, continuation_deployments, continuation_output_ceilings, continued_text, deployment_can_continue, + responses_continuation_input, + responses_continuation_input_for, ) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, @@ -453,6 +458,7 @@ from litellm.utils import ( provider_rejectable_params, resolve_api_key_secret_reference, set_live_deployment_replay, + token_counter, ) from .router_utils.pattern_match_deployments import PatternMatchRouter @@ -484,7 +490,6 @@ if TYPE_CHECKING: from litellm.types.llms.openai import ( ResponseAPIUsage, ResponseIncompleteEvent, - ResponseInputParam, ResponsesAPIResponse, ) @@ -3838,7 +3843,8 @@ class Router: ) -> Mapping[str, object]: """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} - defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None} + default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params) + defaults: Final = {key: value for key, value in default_params.items() if value is not None} deployment_tool_choice: Final = ( {"tool_choice": deployment_params["tool_choice"]} if deployment_params.get("tool_choice") is not None @@ -3917,67 +3923,6 @@ class Router: extended: Final = _extend_trailing_text_block(blocks, new_text) return [*messages[:-1], {**last, "content": extended, "prefix": True}] - @staticmethod - def _build_responses_continuation_input( - input_val: Union[str, "ResponseInputParam"] | None, - generated_content: str, - ) -> "ResponseInputParam": - """ - Convert Responses-API input + partial assistant output into a - continuation input that asks the fallback model to pick up where the - prior assistant message stopped. - - Best effort across providers. The chat-completions path uses - Anthropic's `prefix: True` prefill trick on the assistant message; - the Responses-API input schema has no direct equivalent, so we - append an instruction (developer role) plus a prior assistant - message containing the partial output. Providers without prefill - semantics (OpenAI, Vertex) treat this as conversational context - and may regenerate — same trade-off as the chat-completions path - for non-Anthropic fallbacks. - """ - # base/continuation are List[Any] because ResponseInputParam items - # are a wide Union of TypedDicts (EasyInputMessageParam, Message, - # ResponseOutputMessageParam, ...) — annotating as List[Dict[str, Any]] - # rejects the list() spread of input_val. We cast the combined list to - # ResponseInputParam at the return. - base: list[object] - if isinstance(input_val, str): - base = [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": input_val}], - } - ] - elif isinstance(input_val, list): - base = list(input_val) - else: - base = [] - continuation: Final[list[object]] = [ - { - "type": "message", - "role": "developer", - "content": [ - { - "type": "input_text", - "text": ( - "The previous assistant response was interrupted " - "mid-stream. Continue exactly where it stopped — " - "do not repeat any of its content. Your response " - "must read as a seamless continuation." - ), - } - ], - }, - { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": generated_content}], - }, - ] - return cast("ResponseInputParam", base + continuation) - async def _aresponses_streaming_iterator( self, response: "BaseResponsesAPIStreamingIterator", @@ -3999,9 +3944,12 @@ class Router: Full parity with the chat-completions path: - Pre-first-chunk: retry with the original input unchanged. - - Partial content: inject a developer instruction + prior - assistant message carrying the generated text so the fallback - model continues rather than restarts. + - Partial content: continue from the generated text, as an + assistant prefill on a fallback that takes one and as assistant + history plus a user request to continue on any other. A request + whose output is constrained (a JSON schema, a forced tool), or a + stream that began a tool call, is not continued, and the primary's + error surfaces instead. - Usage combining: merge partial-stream usage onto the fallback's response.completed event so accounting reflects both attempts. - Stream cleanup: shielded aclose() on both source and fallback @@ -4190,6 +4138,42 @@ class Router: wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks()) return wrapper + def _responses_continuation( + self, + e: "MidStreamFallbackError", + source_iterator: "BaseResponsesAPIStreamingIterator", + request_kwargs: Mapping[str, object], + ) -> ResponsesContinuation | None: + """What a fallback continues from once a Responses stream broke after text, raising the primary's error + when the request constrains its output or the stream began a tool call, since a continued JSON document or + tool call would not parse.""" + if not e.generated_content or e.is_pre_first_chunk: + return None + if source_iterator.started_tool_call: + Router._raise_original_mid_stream_error(e) + cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") + cut_deployment: Final = ( + self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None + ) + if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): + Router._raise_original_mid_stream_error(e) + earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG) + if isinstance(earlier, ResponsesContinuation): + return earlier.extended(e.generated_content) + return ResponsesContinuation(request_kwargs.get("input"), e.generated_content) + + @staticmethod + def _responses_continuation_ceilings( + e: "MidStreamFallbackError", request_kwargs: Mapping[str, object] + ) -> Mapping[str, object]: + """The caller's output ceilings less what the cut stream already emitted, raising the primary's error once + nothing is left.""" + emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content) + reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens) + if reduced is None: + Router._raise_original_mid_stream_error(e) + return reduced + async def _aresponses_fallback_attempt( self, e: "MidStreamFallbackError", @@ -4208,6 +4192,11 @@ class Router: """ from litellm.exceptions import MidStreamFallbackError + continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs) + reduced_ceilings: Final[Mapping[str, object]] = ( + Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {} + ) + seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "") partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement @@ -4225,11 +4214,10 @@ class Router: initial_kwargs["original_function"] = ( # rebind-ok: the fallback chain re-enters on the same kwargs self._ageneric_api_call_with_fallbacks_responses_attempt ) - if e.generated_content and not e.is_pre_first_chunk: - initial_kwargs["input"] = Router._build_responses_continuation_input( # rebind-ok: fallback hop input - initial_kwargs.get("input"), - e.generated_content, - ) + if continuation is not None: + initial_kwargs.update(reduced_ceilings) + initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment + initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input # The Responses-API path stores observability metadata # under "litellm_metadata" (not the default "metadata") — # see _ageneric_api_call_with_fallbacks. Mirroring that @@ -4258,7 +4246,7 @@ class Router: ) prepared_fallback_hidden_params: Final = adopt_headers(fallback_response) if hasattr(fallback_response, "__aiter__"): - async for fallback_item in fallback_response: + async for fallback_item in seam.splice(fallback_response): Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) if partial_usage is not None: Router._combine_responses_fallback_usage(fallback_item, partial_usage) @@ -4781,11 +4769,12 @@ class Router: splits_stream_timeout: bool = False, ) -> None: """ - 4 jobs: + 5 jobs: - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging) - Adds default litellm params to kwargs, if set. - Merges tools from deployment with request (proxy-configured tools + request tools). - On a fallback hop, drops the encrypted reasoning this deployment cannot decrypt, keeping its summary. + - On a Responses continuation, sends the partial response as prefill where this deployment takes one. """ for key in self._forwarded_alias_marker_keys_the_deployment_sets( deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) @@ -4959,6 +4948,11 @@ class Router: (_FALLBACK_HOP_ADAPTER.validate_python(deployment),), unmarked_origin=deployment_that_just_failed, ) + if RESPONSES_CONTINUATION_KWARG in hop_kwargs: + del kwargs[RESPONSES_CONTINUATION_KWARG] # rebind-ok: this method prepares the call's kwargs in place + kwargs["input"] = responses_continuation_input_for( # rebind-ok: the deployment's continuation input + hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment) + ) def _get_async_openai_model_client(self, deployment: dict, kwargs: dict): """ diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 8f211ef67db..37e10dd7b4f 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -1,12 +1,13 @@ """Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave -the output unconstrained.""" +the output unconstrained, and shapes a Responses API continuation for the deployment that serves it.""" +import json from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence from dataclasses import dataclass from itertools import accumulate from os.path import commonprefix from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar from pydantic import TypeAdapter, ValidationError @@ -14,7 +15,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.rules import Rules from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from litellm.utils import supports_assistant_prefill @@ -28,7 +29,10 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker() _STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) -_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens") +_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens") +_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"}) +_QUOTED_ENDING_CHARS: Final = 100 +RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation" def _str_keyed(value: object) -> Mapping[str, object]: @@ -38,28 +42,39 @@ def _str_keyed(value: object) -> Mapping[str, object]: return _NO_FIELDS -def deployment_supports_prefill(deployment: object) -> bool: +def _underlying_model(deployment: object) -> str | None: deployment_map: Final = _str_keyed(deployment) model_info: Final = _str_keyed(deployment_map.get("model_info")) - declared: Final = model_info.get("supports_assistant_prefill") - if isinstance(declared, bool): - return declared litellm_params: Final = _str_keyed(deployment_map.get("litellm_params")) candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model")) - model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + + +def deployment_supports_prefill(deployment: object) -> bool: + declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill") + if isinstance(declared, bool): + return declared + model: Final = _underlying_model(deployment) return model is not None and supports_assistant_prefill(model=model) +def _forces_tool_call(tool_choice: object) -> bool: + choice: Final = _str_keyed(tool_choice) + mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode") + return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES + + def constrains_continuation(params: Mapping[str, object]) -> bool: sent: Final = {**params, **_str_keyed(params.get("extra_body"))} choice_count: Final = sent.get("n") response_format: Final = sent.get("response_format") - tool_choice: Final = sent.get("tool_choice") + text_format: Final = _str_keyed(sent.get("text")).get("format") return ( (isinstance(choice_count, int) and choice_count > 1) or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) - or tool_choice == "required" - or isinstance(tool_choice, Mapping) + or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT) + or sent.get("text_format") is not None + or _forces_tool_call(sent.get("tool_choice")) or sent.get("merge_reasoning_content_in_choices") is True ) @@ -99,6 +114,7 @@ def _delta_text(chunk: ModelResponseStream) -> str: _CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) +_INPUT_ITEMS_ADAPTER: Final = TypeAdapter(list[object]) def _prefill_text(prefill_message: Mapping[str, object]) -> str: @@ -114,7 +130,7 @@ def _prefill_text(prefill_message: Mapping[str, object]) -> str: ) -def _resendable_whitespace(prefill: str) -> str: +def resendable_whitespace(prefill: str) -> str: return prefill[len(prefill.rstrip()) :] if prefill.strip() else "" @@ -124,7 +140,7 @@ def continued_text(prefill_message: Mapping[str, object], generated_content: str return None echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0 after_echo: Final = generated_content[echo:] - return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :] + return after_echo[len(commonprefix((after_echo, resendable_whitespace(prefill)))) :] def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]: @@ -211,7 +227,7 @@ class ContinuationOutput: self._rules: Final = Rules() self._held: tuple[ModelResponseStream, ...] = () self._echo_settled = not self._prefill_text - self._seam = _resendable_whitespace(self._prefill_text) + self._seam = resendable_whitespace(self._prefill_text) async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]: async for item in stream: @@ -262,3 +278,91 @@ class ContinuationOutput: self._text += _delta_text(chunk) self._rules.post_call_rules(input=self._text, model=self._model) return chunks + + +@dataclass(frozen=True, slots=True) +class ResponsesContinuation: + prior_input: object + emitted_text: str + + def extended(self, generated_content: str) -> "ResponsesContinuation": + return ResponsesContinuation(self.prior_input, f"{self.emitted_text}{generated_content}") + + +def deployment_takes_responses_prefill(deployment: object) -> bool: + """Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot + carry.""" + model: Final = _underlying_model(deployment) + return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment) + + +def _continuation_request(emitted_text: str) -> Mapping[str, object]: + seam: Final = ( + "It already ends with whitespace, so do not add another space before your first word." + if emitted_text[-1:].isspace() + else "It ends without trailing whitespace, so begin with a space if your first word starts a new word." + ) + ending: Final = json.dumps(emitted_text[-_QUOTED_ENDING_CHARS:], ensure_ascii=False) + instruction: Final = ( + f"The previous assistant response was interrupted mid-stream. It ended with {ending}. Continue exactly where " + "it stopped, so your first characters are the ones that would have come right after it, and do not repeat " + f"any of its content. {seam} Your response must read as a seamless continuation." + ) + return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": instruction}]} + + +def _input_items(request_input: object) -> tuple[object, ...]: + if isinstance(request_input, str): + return ({"type": "message", "role": "user", "content": [{"type": "input_text", "text": request_input}]},) + try: + return tuple(_INPUT_ITEMS_ADAPTER.validate_python(request_input)) + except ValidationError: + return () + + +def responses_continuation_input(continuation: ResponsesContinuation) -> Sequence[object]: + """The prior input, the partial response as assistant history, and a user request to continue it.""" + partial: Final = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": continuation.emitted_text}], + } + return [*_input_items(continuation.prior_input), partial, _continuation_request(continuation.emitted_text)] + + +def responses_continuation_input_for(request_kwargs: Mapping[str, object], deployment: Mapping[str, object]) -> object: + """The continuation input as the deployment receives it, without the request to continue where the deployment + takes the partial response as prefill. It reads the input the router already stripped of reasoning this + deployment cannot decrypt, never the continuation's prior input.""" + request_input: Final = request_kwargs.get("input") + continuation: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG) + if not isinstance(continuation, ResponsesContinuation): + return request_input + items: Final = _input_items(request_input) + if items[-1:] != (_continuation_request(continuation.emitted_text),) or not deployment_takes_responses_prefill( + deployment + ): + return request_input + return list(items[:-1]) + + +_StreamItem = TypeVar("_StreamItem") + + +class ResponsesContinuationSeam: + """Drops whitespace a fallback sends again at the start of its text when the client already has it, since + Anthropic strips a prefill's trailing whitespace and the model then writes it back.""" + + def __init__(self, emitted_text: str) -> None: + self._seam = resendable_whitespace(emitted_text) + + async def splice(self, stream: AsyncIterable[_StreamItem]) -> AsyncIterator[_StreamItem]: + async for item in stream: + yield self._past(item) + + def _past(self, item: _StreamItem) -> _StreamItem: + if not self._seam or not isinstance(item, OutputTextDeltaEvent): + return item + matched: Final = len(commonprefix((item.delta, self._seam))) + self._seam = self._seam[matched:] if matched == len(item.delta) else "" + return item.model_copy(update={"delta": item.delta[matched:]}) if matched else item diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index b5e36055354..3e740302f54 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -171,6 +171,8 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py + "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py ] diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index 8d21af83668..343524e3b66 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -1,14 +1,17 @@ """The shared Responses API config contract.""" import asyncio, importlib, json +from collections.abc import Mapping import httpx import pytest +import respx import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.router import GenericLiteLLMParams from litellm import Router +from pydantic import BaseModel, ConfigDict from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -879,38 +882,6 @@ def test_combine_responses_fallback_usage_passthrough_for_unknown_event(): partial = ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2) Router._combine_responses_fallback_usage(other, partial) -@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") -def test_build_responses_continuation_input_from_string(): - out = Router._build_responses_continuation_input("Hello world", "partial assistant text") - assert len(out) == 3 - assert out[0]["role"] == "user" - assert out[0]["content"][0]["text"] == "Hello world" - assert out[1]["role"] == "developer" - assert out[2]["role"] == "assistant" - assert out[2]["content"][0]["text"] == "partial assistant text" - -@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") -def test_build_responses_continuation_input_from_list_preserves_items(): - existing: List[Any] = [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "msg1"}], - } - ] - out = Router._build_responses_continuation_input(existing, "partial") - assert len(out) == 3 - assert out[0]["content"][0]["text"] == "msg1" - assert out[1]["role"] == "developer" - assert out[2]["role"] == "assistant" - -@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") -def test_build_responses_continuation_input_from_none(): - out = Router._build_responses_continuation_input(None, "partial") - assert len(out) == 2 - assert out[0]["role"] == "developer" - assert out[1]["role"] == "assistant" - @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio async def test_aresponses_streaming_iterator_passthrough(): @@ -1355,9 +1326,355 @@ async def test_aresponses_fallback_uses_continuation_input_after_partial_content continuation = mock_fallback.await_args.kwargs["kwargs"]["input"] assert isinstance(continuation, list) assert continuation[0]["content"][0]["text"] == "original question" - assert continuation[-2]["role"] == "developer" - assert continuation[-1]["role"] == "assistant" - assert continuation[-1]["content"][0]["text"] == "partial answer" + assert continuation[-2]["role"] == "assistant" + assert continuation[-2]["content"][0]["text"] == "partial answer" + assert continuation[-1]["role"] == "user" + + +def _sse(*events: Mapping[str, object]) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events), + ) + + +def _openai_stream_cut_after(partial: str) -> httpx.Response: + return _sse( + *( + ({ + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": partial, + },) + if partial else () + ), + { + "type": "error", "sequence_number": 1, + "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, + }, + ) + + +def _anthropic_stream(text: str) -> httpx.Response: + return _sse( + {"type": "message_start", "message": { + "id": "msg_fallback", "type": "message", "role": "assistant", + "model": "claude-fallback", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}}, + {"type": "message_stop"}, + ) + + +def _primary_with_fallback(fallback_model: str) -> Router: + return Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": fallback_model, "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + + +def _streamed_text(events: tuple[Any, ...]) -> str: + return "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) + + +@pytest.fixture +def claude_with_prefill() -> str: + model: Final = "anthropic/claude-prefill-capable-test-model" + litellm.register_model( + {model: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}}, + persist_across_reloads=False, + ) + litellm.get_model_info.cache_clear() + return model + + +@pytest.mark.asyncio +@pytest.mark.parametrize("partial", ["", "1, 2, "]) +async def test_responses_fallback_can_continue_on_a_provider_without_prefill( + partial: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + prompt: Final = "Print the integers 1 through 5, separated by commas." + primary: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_after(partial) + ) + + def no_prefill_provider(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + if body["messages"][-1]["role"] != "user": + return httpx.Response( + 400, + json={"type": "error", "error": { + "type": "invalid_request_error", "message": "The conversation must end with a user message", + }}, + ) + return _anthropic_stream("3, 4, 5" if partial else "1, 2, 3, 4, 5") + + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=no_prefill_provider) + router: Final = _primary_with_fallback("anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses(model="primary", input=prompt, stream=True, max_output_tokens=100) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + assert primary.call_count == fallback.call_count == 1 + messages: Final = json.loads(fallback.calls.last.request.content)["messages"] + assert messages[0] == {"role": "user", "content": [{"type": "text", "text": prompt}]} + if partial: + assert [message["role"] for message in messages] == ["user", "assistant", "user"] + assert messages[-2] == {"role": "assistant", "content": [{"type": "text", "text": partial}]} + assert json.dumps(partial) in messages[-1]["content"][0]["text"] + else: + assert len(messages) == 1 + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_continues_a_claude_that_takes_prefill_from_its_own_partial( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream(" 3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Print the integers 1 through 5.", stream=True, max_output_tokens=100 + ) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + messages: Final = json.loads(fallback.calls.last.request.content)["messages"] + assert [message["role"] for message in messages] == ["user", "assistant"] + assert messages[-1]["content"][0]["text"].startswith("1, 2,") + finally: + router.discard() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fallback_takes_prefill", [True, False], ids=["prefill", "request_to_continue"]) +async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cannot_decrypt( + fallback_takes_prefill: bool, + claude_with_prefill: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream(" 3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill if fallback_takes_prefill else "anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses( + model="primary", + input=[ + {"role": "user", "content": "Print the integers 1 through 5."}, + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "primary-only-ciphertext"}, + ], + stream=True, + max_output_tokens=100, + ) + _ = [event async for event in stream] + assert fallback.call_count == 1 + assert "primary-only-ciphertext" not in fallback.calls.last.request.content.decode() + finally: + router.discard() + + +class _Numbers(BaseModel): + model_config = ConfigDict(frozen=True) + + numbers: tuple[int, ...] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "constraint", + [ + {"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}}, + {"text": {"format": {"type": "json_object"}}}, + {"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]}, + {"text_format": _Numbers}, + ], + ids=["json_schema", "json_object", "required_tool", "sdk_text_format"], +) +async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed( + constraint: dict[str, object], + claude_with_prefill: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_after('{"numbers": [1, 2, ') + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3]}") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Numbers 1 to 3 as JSON.", stream=True, max_output_tokens=100, **constraint + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_continues_within_the_callers_output_ceiling( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100) + events: Final = tuple([event async for event in stream]) + assert _streamed_text(events) == "1, 2, 3, 4, 5" + assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter( + text="1, 2, " + ) + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling( + claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, ")) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("3, 4, 5") + ) + router: Final = _primary_with_fallback(claude_with_prefill) + try: + stream: Final = await router.aresponses( + model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ") + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + +def _openai_stream_cut_mid_tool_call(preamble: str) -> httpx.Response: + return _sse( + { + "type": "response.output_text.delta", "sequence_number": 0, + "item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble, + }, + { + "type": "response.output_item.added", "sequence_number": 1, "output_index": 1, + "item": { + "type": "function_call", "id": "fc_primary", "call_id": "call_primary", + "name": "lookup", "arguments": "", "status": "in_progress", + }, + }, + { + "type": "response.function_call_arguments.delta", "sequence_number": 2, + "item_id": "fc_primary", "output_index": 1, "delta": '{"city": "Par', + }, + { + "type": "error", "sequence_number": 3, + "error": {"type": "server_error", "code": "internal_error", "message": "interrupted"}, + }, + ) + + +def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response: + return _sse( + {"type": "message_start", "message": { + "id": "msg_primary", "type": "message", "role": "assistant", + "model": "claude-primary", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}}, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": { + "type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {}, + }}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}}, + {"type": "error", "error": {"type": "overloaded_error", "message": "interrupted"}}, + ) + + +@pytest.mark.asyncio +async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=_openai_stream_cut_mid_tool_call("Let me look that up. ") + ) + fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=_anthropic_stream("It is sunny in Paris.") + ) + router: Final = _primary_with_fallback("anthropic/claude-opus-5-5") + try: + stream: Final = await router.aresponses( + model="primary", + input="What is the weather in Paris?", + stream=True, + tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}], + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert fallback.call_count == 0 + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + side_effect=[ + _anthropic_stream_cut_mid_tool_call("Let me look that up. "), + _anthropic_stream("It is sunny in Paris."), + ] + ) + router: Final = Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}}, + {"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}}, + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + try: + stream: Final = await router.aresponses( + model="primary", + input="What is the weather in Paris?", + stream=True, + tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}], + ) + with pytest.raises(litellm.InternalServerError, match="interrupted"): + [event async for event in stream] + assert provider.call_count == 1 + finally: + router.discard() + @pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit") @pytest.mark.asyncio diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py index 8fa1c799863..4492f8512ae 100644 --- a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -4,13 +4,23 @@ import litellm from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_KWARG, MID_STREAM_CONTINUATION_MARKER, + RESPONSES_CONTINUATION_KWARG, ContinuationOutput, ContinuationPrefillDeploymentCheck, + ResponsesContinuation, + ResponsesContinuationSeam, + constrains_continuation, continuation_output_ceilings, deployment_supports_prefill, + deployment_takes_responses_prefill, + responses_continuation_input, + responses_continuation_input_for, ) +from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents PREFILL_MODEL = "anthropic/prefill-capable-test-model" +CLAUDE_PREFILL_MODEL = "anthropic/claude-prefill-capable-test-model" +CLAUDE_NON_PREFILL_MODEL = "anthropic/claude-prefill-rejecting-test-model" NON_PREFILL_MODEL = "openai/prefill-unknown-test-model" UNMAPPED_MODEL = "openai/unmapped-test-model" @@ -21,6 +31,12 @@ def _synthetic_cost_map_entries() -> None: { PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}, NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"}, + CLAUDE_PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}, + CLAUDE_NON_PREFILL_MODEL: { + "litellm_provider": "anthropic", + "mode": "chat", + "supports_assistant_prefill": False, + }, }, persist_across_reloads=False, ) @@ -191,7 +207,11 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream: ("Hello", (" world",), (" world",)), ("Hello", ("Hel", "p me"), ("Hel", "p me")), ("Hello", ("Hel",), ("Hel",)), - ([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")), + ( + [{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], + ("Sure: Hello", " world"), + ("", " world"), + ), ], ids=[ "echo_split_across_chunks_is_dropped", @@ -206,9 +226,7 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream: async def test_continuation_output_drops_only_a_full_echo_of_the_prefill( prefill_content, fallback_texts, expected_texts ): - output = ContinuationOutput( - "Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", () - ) + output = ContinuationOutput("Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ()) async def fallback_stream(): for text in fallback_texts: @@ -266,7 +284,14 @@ async def test_continuation_output_drops_only_whitespace_the_client_already_has( ({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}), ({"extra_body": {"top_k": 3}}, {}), ], - ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"], + ids=[ + "max_tokens", + "both_ceilings", + "no_ceiling", + "extra_body_max_tokens", + "extra_body_only", + "extra_body_no_ceiling", + ], ) def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends( request_kwargs, expected @@ -286,3 +311,186 @@ def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_ ) def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs): assert continuation_output_ceilings(request_kwargs, 30) is None + + +@pytest.mark.parametrize( + "sent,expected", + [ + ({"text": {"format": {"type": "json_schema", "name": "n", "schema": {"type": "object"}}}}, True), + ({"text": {"format": {"type": "json_object"}}}, True), + ({"text": {"format": {"type": "text"}}}, False), + ({"text": {"verbosity": "low"}}, False), + ({"tool_choice": "required"}, True), + ({"tool_choice": "auto"}, False), + ({"tool_choice": {"type": "function", "name": "lookup"}}, True), + ({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True), + ({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False), + ({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False), + ({"tool_choice": {"type": "web_search_preview"}}, False), + ], + ids=[ + "json_schema", + "json_object", + "text_format", + "verbosity_only", + "required_tool", + "auto_tool", + "forced_function", + "allowed_tools_required", + "allowed_tools_auto", + "chat_allowed_tools_auto", + "hosted_tool", + ], +) +def test_constrains_continuation_reads_the_responses_text_format(sent, expected): + assert constrains_continuation(sent) is expected + + +@pytest.mark.parametrize( + "deployment,expected", + [ + (_deployment(CLAUDE_PREFILL_MODEL, "a"), True), + (_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False), + (_deployment(PREFILL_MODEL, "c"), False), + ( + {"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}}, + True, + ), + ], + ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"], +) +def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected): + assert deployment_takes_responses_prefill(deployment) is expected + + +def _continuation_kwargs() -> dict[str, object]: + continuation = ResponsesContinuation("Count to 5.", "1, 2, ") + return {RESPONSES_CONTINUATION_KWARG: continuation, "input": responses_continuation_input(continuation)} + + +@pytest.mark.parametrize( + "prior_input,expected_prior", + [ + ( + "Count to 5.", + [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Count to 5."}]}], + ), + ( + [ + {"role": "user", "content": "Count to 5."}, + {"type": "function_call_output", "call_id": "c", "output": "x"}, + ], + [ + {"role": "user", "content": "Count to 5."}, + {"type": "function_call_output", "call_id": "c", "output": "x"}, + ], + ), + (None, []), + ], + ids=["string_input", "item_list_input", "no_input"], +) +def test_responses_continuation_input_keeps_the_prior_input_then_the_partial_then_a_request_to_continue( + prior_input, expected_prior +): + built = responses_continuation_input(ResponsesContinuation(prior_input, "1, 2, ")) + + partial = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "1, 2, "}]} + assert built[:-1] == [*expected_prior, partial] + assert built[-1]["role"] == "user" + assert "already ends with whitespace" in built[-1]["content"][0]["text"] + + +def test_responses_continuation_request_asks_for_a_leading_space_only_after_a_word(): + built = responses_continuation_input(ResponsesContinuation("Hi", "Transformers use")) + + assert "begin with a space" in built[-1]["content"][0]["text"] + + +def test_responses_continuation_accumulates_across_hops(): + continuation = ResponsesContinuation("Count to 5.", "1, ").extended("2, ") + + assert continuation == ResponsesContinuation("Count to 5.", "1, 2, ") + + +@pytest.mark.parametrize( + "deployment,expected_roles", + [ + (_deployment(CLAUDE_PREFILL_MODEL, "a"), ["user", "assistant"]), + (_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), ["user", "assistant", "user"]), + (_deployment(NON_PREFILL_MODEL, "c"), ["user", "assistant", "user"]), + ], + ids=["claude_prefill", "claude_without_prefill", "openai"], +) +def test_responses_continuation_input_for_shapes_the_input_for_the_deployment_that_serves_it( + deployment, expected_roles +): + shaped = responses_continuation_input_for(_continuation_kwargs(), deployment) + + assert [item["role"] for item in shaped] == expected_roles + + +def test_responses_continuation_input_for_keeps_what_the_router_stripped_from_the_input(): + continuation = ResponsesContinuation( + [ + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "opaque"}, + {"role": "user", "content": "Count to 5."}, + ], + "1, 2, ", + ) + stripped_input = responses_continuation_input(continuation)[1:] + + shaped = responses_continuation_input_for( + {RESPONSES_CONTINUATION_KWARG: continuation, "input": stripped_input}, _deployment(CLAUDE_PREFILL_MODEL, "a") + ) + + assert shaped == stripped_input[:-1] + + +def test_responses_continuation_input_for_leaves_an_input_without_its_request_to_continue_alone(): + kwargs = {**_continuation_kwargs(), "input": [{"role": "user", "content": "Count to 5."}]} + + assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == kwargs["input"] + + +def test_responses_continuation_input_for_leaves_a_request_without_a_continuation_alone(): + kwargs = {"input": "Count to 5."} + + assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == "Count to 5." + + +def _delta(text: str) -> OutputTextDeltaEvent: + return OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="m", output_index=0, content_index=0, delta=text + ) + + +@pytest.mark.parametrize( + "emitted,fallback_texts,expected_texts", + [ + ("1, 2, ", (" 3, 4", ", 5"), ("3, 4", ", 5")), + ("1, 2, ", (" ", " 3"), ("", " 3")), + ("1, 2,", (" 3",), (" 3",)), + ("1, 2, ", ("3",), ("3",)), + ("A.\n\n", ("\n Hi",), (" Hi",)), + ("", (" Hi",), (" Hi",)), + ], + ids=[ + "re_sent_space_is_dropped", + "seam_split_across_events", + "no_trailing_whitespace_keeps_the_space", + "no_re_sent_space_passes_through", + "seam_ends_at_the_first_divergence", + "fresh_start_passes_through", + ], +) +@pytest.mark.asyncio +async def test_responses_continuation_seam_drops_only_whitespace_the_client_already_has( + emitted, fallback_texts, expected_texts +): + async def fallback_stream(): + for text in fallback_texts: + yield _delta(text) + + released = [event async for event in ResponsesContinuationSeam(emitted).splice(fallback_stream())] + + assert tuple(event.delta for event in released) == expected_texts diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 568a05e8ff4..7966326bc8d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -7026,8 +7026,7 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events @pytest.mark.asyncio async def test_aresponses_streaming_iterator_partial_content_injects_continuation(): - """Mid-stream error: input is rewritten to include user prompt + - developer instruction + prior assistant message with partial output.""" + """Mid-stream error: preserve partial assistant output and request continuation in a user turn.""" from litellm.exceptions import MidStreamFallbackError router = _make_router_with_fallback() @@ -7063,11 +7062,11 @@ async def test_aresponses_streaming_iterator_partial_content_injects_continuatio assert isinstance(new_input, list) assert new_input[0]["role"] == "user" assert new_input[0]["content"][0]["text"] == "What's the capital of France?" - assert new_input[1]["role"] == "developer" - assert "do not repeat" in new_input[1]["content"][0]["text"].lower() - assert new_input[2]["role"] == "assistant" - assert new_input[2]["content"][0]["type"] == "output_text" - assert new_input[2]["content"][0]["text"] == "The capital of France is" + assert new_input[1]["role"] == "assistant" + assert new_input[1]["content"][0]["type"] == "output_text" + assert new_input[1]["content"][0]["text"] == "The capital of France is" + assert new_input[2]["role"] == "user" + assert "do not repeat" in new_input[2]["content"][0]["text"].lower() @pytest.mark.asyncio