This commit is contained in:
tin-berri 2026-10-26 16:23:42 +03:00 • committed by GitHub
commit 9be2eb70d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 769 additions and 135 deletions

View file

@ -408,6 +408,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
continue
if call_id in self._web_search_calls:
continue
self._started_tool_call = True
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)
fn_name = ""

View file

@ -365,6 +365,7 @@ class BaseResponsesAPIStreamingIterator:
"""
_pending_logging_tasks: tuple[asyncio.Task[object], ...] = ()
_started_tool_call: bool = False
_earlier_rounds_spend: BilledRoundsSpend | None = None
def __init__(
@ -494,6 +495,8 @@ class BaseResponsesAPIStreamingIterator:
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
):
_item: Final[object] = getattr(openai_responses_api_chunk, "item", None)
if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES:
self._started_tool_call = True
if _item is not None:
ResponsesAPIRequestUtils.encode_container_id_on_output_item(
item=_item,
@ -1044,6 +1047,11 @@ class BaseResponsesAPIStreamingIterator:
if pending:
await asyncio.wait(pending)
@property
def started_tool_call(self) -> bool:
"""Whether the stream began a tool call, which a text continuation could never finish."""
return self._started_tool_call
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
"""
The response.incomplete event to bill when the client went away mid-stream,
@ -1690,6 +1698,7 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
return None
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"})
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
{
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,

View file

@ -301,14 +301,19 @@ from litellm.router_utils.kubernetes_pod_discovery import (
)
from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
MID_STREAM_CONTINUATION_MARKER,
RESPONSES_CONTINUATION_KWARG,
ContinuationOutput,
ContinuationPrefillDeploymentCheck,
ContinuedPartial,
ResponsesContinuation,
ResponsesContinuationSeam,
constrains_continuation,
continuation_deployments,
continuation_output_ceilings,
continued_text,
deployment_can_continue,
responses_continuation_input,
responses_continuation_input_for,
)
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,
@ -453,6 +458,7 @@ from litellm.utils import (
provider_rejectable_params,
resolve_api_key_secret_reference,
set_live_deployment_replay,
token_counter,
)
from .router_utils.pattern_match_deployments import PatternMatchRouter
@ -484,7 +490,6 @@ if TYPE_CHECKING:
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseIncompleteEvent,
ResponseInputParam,
ResponsesAPIResponse,
)
@ -3838,7 +3843,8 @@ class Router:
) -> Mapping[str, object]:
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None}
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params)
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
deployment_tool_choice: Final = (
{"tool_choice": deployment_params["tool_choice"]}
if deployment_params.get("tool_choice") is not None
@ -3917,67 +3923,6 @@ class Router:
extended: Final = _extend_trailing_text_block(blocks, new_text)
return [*messages[:-1], {**last, "content": extended, "prefix": True}]
@staticmethod
def _build_responses_continuation_input(
input_val: Union[str, "ResponseInputParam"] | None,
generated_content: str,
) -> "ResponseInputParam":
"""
Convert Responses-API input + partial assistant output into a
continuation input that asks the fallback model to pick up where the
prior assistant message stopped.
Best effort across providers. The chat-completions path uses
Anthropic's `prefix: True` prefill trick on the assistant message;
the Responses-API input schema has no direct equivalent, so we
append an instruction (developer role) plus a prior assistant
message containing the partial output. Providers without prefill
semantics (OpenAI, Vertex) treat this as conversational context
and may regenerate — same trade-off as the chat-completions path
for non-Anthropic fallbacks.
"""
# base/continuation are List[Any] because ResponseInputParam items
# are a wide Union of TypedDicts (EasyInputMessageParam, Message,
# ResponseOutputMessageParam, ...) — annotating as List[Dict[str, Any]]
# rejects the list() spread of input_val. We cast the combined list to
# ResponseInputParam at the return.
base: list[object]
if isinstance(input_val, str):
base = [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": input_val}],
}
]
elif isinstance(input_val, list):
base = list(input_val)
else:
base = []
continuation: Final[list[object]] = [
{
"type": "message",
"role": "developer",
"content": [
{
"type": "input_text",
"text": (
"The previous assistant response was interrupted "
"mid-stream. Continue exactly where it stopped — "
"do not repeat any of its content. Your response "
"must read as a seamless continuation."
),
}
],
},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": generated_content}],
},
]
return cast("ResponseInputParam", base + continuation)
async def _aresponses_streaming_iterator(
self,
response: "BaseResponsesAPIStreamingIterator",
@ -3999,9 +3944,12 @@ class Router:
Full parity with the chat-completions path:
- Pre-first-chunk: retry with the original input unchanged.
- Partial content: inject a developer instruction + prior
assistant message carrying the generated text so the fallback
model continues rather than restarts.
- Partial content: continue from the generated text, as an
assistant prefill on a fallback that takes one and as assistant
history plus a user request to continue on any other. A request
whose output is constrained (a JSON schema, a forced tool), or a
stream that began a tool call, is not continued, and the primary's
error surfaces instead.
- Usage combining: merge partial-stream usage onto the fallback's
response.completed event so accounting reflects both attempts.
- Stream cleanup: shielded aclose() on both source and fallback
@ -4190,6 +4138,42 @@ class Router:
wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks())
return wrapper
def _responses_continuation(
self,
e: "MidStreamFallbackError",
source_iterator: "BaseResponsesAPIStreamingIterator",
request_kwargs: Mapping[str, object],
) -> ResponsesContinuation | None:
"""What a fallback continues from once a Responses stream broke after text, raising the primary's error
when the request constrains its output or the stream began a tool call, since a continued JSON document or
tool call would not parse."""
if not e.generated_content or e.is_pre_first_chunk:
return None
if source_iterator.started_tool_call:
Router._raise_original_mid_stream_error(e)
cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id")
cut_deployment: Final = (
self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None
)
if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
Router._raise_original_mid_stream_error(e)
earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG)
if isinstance(earlier, ResponsesContinuation):
return earlier.extended(e.generated_content)
return ResponsesContinuation(request_kwargs.get("input"), e.generated_content)
@staticmethod
def _responses_continuation_ceilings(
e: "MidStreamFallbackError", request_kwargs: Mapping[str, object]
) -> Mapping[str, object]:
"""The caller's output ceilings less what the cut stream already emitted, raising the primary's error once
nothing is left."""
emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content)
reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens)
if reduced is None:
Router._raise_original_mid_stream_error(e)
return reduced
async def _aresponses_fallback_attempt(
self,
e: "MidStreamFallbackError",
@ -4208,6 +4192,11 @@ class Router:
"""
from litellm.exceptions import MidStreamFallbackError
continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs)
reduced_ceilings: Final[Mapping[str, object]] = (
Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {}
)
seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "")
partial_usage: Final = Router._extract_partial_responses_usage(source_iterator)
fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement
@ -4225,11 +4214,10 @@ class Router:
initial_kwargs["original_function"] = ( # rebind-ok: the fallback chain re-enters on the same kwargs
self._ageneric_api_call_with_fallbacks_responses_attempt
)
if e.generated_content and not e.is_pre_first_chunk:
initial_kwargs["input"] = Router._build_responses_continuation_input( # rebind-ok: fallback hop input
initial_kwargs.get("input"),
e.generated_content,
)
if continuation is not None:
initial_kwargs.update(reduced_ceilings)
initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment
initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input
# The Responses-API path stores observability metadata
# under "litellm_metadata" (not the default "metadata") —
# see _ageneric_api_call_with_fallbacks. Mirroring that
@ -4258,7 +4246,7 @@ class Router:
)
prepared_fallback_hidden_params: Final = adopt_headers(fallback_response)
if hasattr(fallback_response, "__aiter__"):
async for fallback_item in fallback_response:
async for fallback_item in seam.splice(fallback_response):
Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params)
if partial_usage is not None:
Router._combine_responses_fallback_usage(fallback_item, partial_usage)
@ -4781,11 +4769,12 @@ class Router:
splits_stream_timeout: bool = False,
) -> None:
"""
4 jobs:
5 jobs:
- Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
- Adds default litellm params to kwargs, if set.
- Merges tools from deployment with request (proxy-configured tools + request tools).
- On a fallback hop, drops the encrypted reasoning this deployment cannot decrypt, keeping its summary.
- On a Responses continuation, sends the partial response as prefill where this deployment takes one.
"""
for key in self._forwarded_alias_marker_keys_the_deployment_sets(
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
@ -4959,6 +4948,11 @@ class Router:
(_FALLBACK_HOP_ADAPTER.validate_python(deployment),),
unmarked_origin=deployment_that_just_failed,
)
if RESPONSES_CONTINUATION_KWARG in hop_kwargs:
del kwargs[RESPONSES_CONTINUATION_KWARG] # rebind-ok: this method prepares the call's kwargs in place
kwargs["input"] = responses_continuation_input_for( # rebind-ok: the deployment's continuation input
hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment)
)
def _get_async_openai_model_client(self, deployment: dict, kwargs: dict):
"""

View file

@ -1,12 +1,13 @@
"""Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave
the output unconstrained."""
the output unconstrained, and shapes a Responses API continuation for the deployment that serves it."""
import json
from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate
from os.path import commonprefix
from types import MappingProxyType
from typing import Final
from typing import Final, TypeVar
from pydantic import TypeAdapter, ValidationError
@ -14,7 +15,7 @@ import litellm
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.rules import Rules
from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
from litellm.utils import supports_assistant_prefill
@ -28,7 +29,10 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens")
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens")
_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"})
_QUOTED_ENDING_CHARS: Final = 100
RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation"
def _str_keyed(value: object) -> Mapping[str, object]:
@ -38,28 +42,39 @@ def _str_keyed(value: object) -> Mapping[str, object]:
return _NO_FIELDS
def deployment_supports_prefill(deployment: object) -> bool:
def _underlying_model(deployment: object) -> str | None:
deployment_map: Final = _str_keyed(deployment)
model_info: Final = _str_keyed(deployment_map.get("model_info"))
declared: Final = model_info.get("supports_assistant_prefill")
if isinstance(declared, bool):
return declared
litellm_params: Final = _str_keyed(deployment_map.get("litellm_params"))
candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model"))
model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
def deployment_supports_prefill(deployment: object) -> bool:
declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill")
if isinstance(declared, bool):
return declared
model: Final = _underlying_model(deployment)
return model is not None and supports_assistant_prefill(model=model)
def _forces_tool_call(tool_choice: object) -> bool:
choice: Final = _str_keyed(tool_choice)
mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode")
return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES
def constrains_continuation(params: Mapping[str, object]) -> bool:
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
choice_count: Final = sent.get("n")
response_format: Final = sent.get("response_format")
tool_choice: Final = sent.get("tool_choice")
text_format: Final = _str_keyed(sent.get("text")).get("format")
return (
(isinstance(choice_count, int) and choice_count > 1)
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or tool_choice == "required"
or isinstance(tool_choice, Mapping)
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
or sent.get("text_format") is not None
or _forces_tool_call(sent.get("tool_choice"))
or sent.get("merge_reasoning_content_in_choices") is True
)
@ -99,6 +114,7 @@ def _delta_text(chunk: ModelResponseStream) -> str:
_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
_INPUT_ITEMS_ADAPTER: Final = TypeAdapter(list[object])
def _prefill_text(prefill_message: Mapping[str, object]) -> str:
@ -114,7 +130,7 @@ def _prefill_text(prefill_message: Mapping[str, object]) -> str:
)
def _resendable_whitespace(prefill: str) -> str:
def resendable_whitespace(prefill: str) -> str:
return prefill[len(prefill.rstrip()) :] if prefill.strip() else ""
@ -124,7 +140,7 @@ def continued_text(prefill_message: Mapping[str, object], generated_content: str
return None
echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0
after_echo: Final = generated_content[echo:]
return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :]
return after_echo[len(commonprefix((after_echo, resendable_whitespace(prefill)))) :]
def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]:
@ -211,7 +227,7 @@ class ContinuationOutput:
self._rules: Final = Rules()
self._held: tuple[ModelResponseStream, ...] = ()
self._echo_settled = not self._prefill_text
self._seam = _resendable_whitespace(self._prefill_text)
self._seam = resendable_whitespace(self._prefill_text)
async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]:
async for item in stream:
@ -262,3 +278,91 @@ class ContinuationOutput:
self._text += _delta_text(chunk)
self._rules.post_call_rules(input=self._text, model=self._model)
return chunks
@dataclass(frozen=True, slots=True)
class ResponsesContinuation:
prior_input: object
emitted_text: str
def extended(self, generated_content: str) -> "ResponsesContinuation":
return ResponsesContinuation(self.prior_input, f"{self.emitted_text}{generated_content}")
def deployment_takes_responses_prefill(deployment: object) -> bool:
"""Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot
carry."""
model: Final = _underlying_model(deployment)
return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment)
def _continuation_request(emitted_text: str) -> Mapping[str, object]:
seam: Final = (
"It already ends with whitespace, so do not add another space before your first word."
if emitted_text[-1:].isspace()
else "It ends without trailing whitespace, so begin with a space if your first word starts a new word."
)
ending: Final = json.dumps(emitted_text[-_QUOTED_ENDING_CHARS:], ensure_ascii=False)
instruction: Final = (
f"The previous assistant response was interrupted mid-stream. It ended with {ending}. Continue exactly where "
"it stopped, so your first characters are the ones that would have come right after it, and do not repeat "
f"any of its content. {seam} Your response must read as a seamless continuation."
)
return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": instruction}]}
def _input_items(request_input: object) -> tuple[object, ...]:
if isinstance(request_input, str):
return ({"type": "message", "role": "user", "content": [{"type": "input_text", "text": request_input}]},)
try:
return tuple(_INPUT_ITEMS_ADAPTER.validate_python(request_input))
except ValidationError:
return ()
def responses_continuation_input(continuation: ResponsesContinuation) -> Sequence[object]:
"""The prior input, the partial response as assistant history, and a user request to continue it."""
partial: Final = {
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": continuation.emitted_text}],
}
return [*_input_items(continuation.prior_input), partial, _continuation_request(continuation.emitted_text)]
def responses_continuation_input_for(request_kwargs: Mapping[str, object], deployment: Mapping[str, object]) -> object:
"""The continuation input as the deployment receives it, without the request to continue where the deployment
takes the partial response as prefill. It reads the input the router already stripped of reasoning this
deployment cannot decrypt, never the continuation's prior input."""
request_input: Final = request_kwargs.get("input")
continuation: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG)
if not isinstance(continuation, ResponsesContinuation):
return request_input
items: Final = _input_items(request_input)
if items[-1:] != (_continuation_request(continuation.emitted_text),) or not deployment_takes_responses_prefill(
deployment
):
return request_input
return list(items[:-1])
_StreamItem = TypeVar("_StreamItem")
class ResponsesContinuationSeam:
"""Drops whitespace a fallback sends again at the start of its text when the client already has it, since
Anthropic strips a prefill's trailing whitespace and the model then writes it back."""
def __init__(self, emitted_text: str) -> None:
self._seam = resendable_whitespace(emitted_text)
async def splice(self, stream: AsyncIterable[_StreamItem]) -> AsyncIterator[_StreamItem]:
async for item in stream:
yield self._past(item)
def _past(self, item: _StreamItem) -> _StreamItem:
if not self._seam or not isinstance(item, OutputTextDeltaEvent):
return item
matched: Final = len(commonprefix((item.delta, self._seam)))
self._seam = self._seam[matched:] if matched == len(item.delta) else ""
return item.model_copy(update={"delta": item.delta[matched:]}) if matched else item

View file

@ -171,6 +171,8 @@ ignored_function_names = [
"_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
]

View file

@ -1,14 +1,17 @@
"""The shared Responses API config contract."""
import asyncio, importlib, json
from collections.abc import Mapping
import httpx
import pytest
import respx
import litellm
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.router import GenericLiteLLMParams
from litellm import Router
from pydantic import BaseModel, ConfigDict
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
@ -879,38 +882,6 @@ def test_combine_responses_fallback_usage_passthrough_for_unknown_event():
partial = ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2)
Router._combine_responses_fallback_usage(other, partial)
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
def test_build_responses_continuation_input_from_string():
out = Router._build_responses_continuation_input("Hello world", "partial assistant text")
assert len(out) == 3
assert out[0]["role"] == "user"
assert out[0]["content"][0]["text"] == "Hello world"
assert out[1]["role"] == "developer"
assert out[2]["role"] == "assistant"
assert out[2]["content"][0]["text"] == "partial assistant text"
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
def test_build_responses_continuation_input_from_list_preserves_items():
existing: List[Any] = [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "msg1"}],
}
]
out = Router._build_responses_continuation_input(existing, "partial")
assert len(out) == 3
assert out[0]["content"][0]["text"] == "msg1"
assert out[1]["role"] == "developer"
assert out[2]["role"] == "assistant"
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
def test_build_responses_continuation_input_from_none():
out = Router._build_responses_continuation_input(None, "partial")
assert len(out) == 2
assert out[0]["role"] == "developer"
assert out[1]["role"] == "assistant"
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_passthrough():
@ -1355,9 +1326,355 @@ async def test_aresponses_fallback_uses_continuation_input_after_partial_content
continuation = mock_fallback.await_args.kwargs["kwargs"]["input"]
assert isinstance(continuation, list)
assert continuation[0]["content"][0]["text"] == "original question"
assert continuation[-2]["role"] == "developer"
assert continuation[-1]["role"] == "assistant"
assert continuation[-1]["content"][0]["text"] == "partial answer"
assert continuation[-2]["role"] == "assistant"
assert continuation[-2]["content"][0]["text"] == "partial answer"
assert continuation[-1]["role"] == "user"
def _sse(*events: Mapping[str, object]) -> httpx.Response:
return httpx.Response(
200,
headers={"content-type": "text/event-stream"},
content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events),
)
def _openai_stream_cut_after(partial: str) -> httpx.Response:
return _sse(
*(
({
"type": "response.output_text.delta", "sequence_number": 0,
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": partial,
},)
if partial else ()
),
{
"type": "error", "sequence_number": 1,
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
},
)
def _anthropic_stream(text: str) -> httpx.Response:
return _sse(
{"type": "message_start", "message": {
"id": "msg_fallback", "type": "message", "role": "assistant",
"model": "claude-fallback", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0},
}},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}},
{"type": "message_stop"},
)
def _primary_with_fallback(fallback_model: str) -> Router:
return Router(
model_list=[
{"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}},
{"model_name": "fallback", "litellm_params": {"model": fallback_model, "api_key": "synthetic"}},
],
fallbacks=[{"primary": ["fallback"]}],
num_retries=0,
)
def _streamed_text(events: tuple[Any, ...]) -> str:
return "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA)
@pytest.fixture
def claude_with_prefill() -> str:
model: Final = "anthropic/claude-prefill-capable-test-model"
litellm.register_model(
{model: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}},
persist_across_reloads=False,
)
litellm.get_model_info.cache_clear()
return model
@pytest.mark.asyncio
@pytest.mark.parametrize("partial", ["", "1, 2, "])
async def test_responses_fallback_can_continue_on_a_provider_without_prefill(
partial: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
prompt: Final = "Print the integers 1 through 5, separated by commas."
primary: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=_openai_stream_cut_after(partial)
)
def no_prefill_provider(request: httpx.Request) -> httpx.Response:
body: Final = json.loads(request.content)
if body["messages"][-1]["role"] != "user":
return httpx.Response(
400,
json={"type": "error", "error": {
"type": "invalid_request_error", "message": "The conversation must end with a user message",
}},
)
return _anthropic_stream("3, 4, 5" if partial else "1, 2, 3, 4, 5")
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=no_prefill_provider)
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
try:
stream: Final = await router.aresponses(model="primary", input=prompt, stream=True, max_output_tokens=100)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events) == "1, 2, 3, 4, 5"
assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
assert primary.call_count == fallback.call_count == 1
messages: Final = json.loads(fallback.calls.last.request.content)["messages"]
assert messages[0] == {"role": "user", "content": [{"type": "text", "text": prompt}]}
if partial:
assert [message["role"] for message in messages] == ["user", "assistant", "user"]
assert messages[-2] == {"role": "assistant", "content": [{"type": "text", "text": partial}]}
assert json.dumps(partial) in messages[-1]["content"][0]["text"]
else:
assert len(messages) == 1
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_continues_a_claude_that_takes_prefill_from_its_own_partial(
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream(" 3, 4, 5")
)
router: Final = _primary_with_fallback(claude_with_prefill)
try:
stream: Final = await router.aresponses(
model="primary", input="Print the integers 1 through 5.", stream=True, max_output_tokens=100
)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events) == "1, 2, 3, 4, 5"
assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
messages: Final = json.loads(fallback.calls.last.request.content)["messages"]
assert [message["role"] for message in messages] == ["user", "assistant"]
assert messages[-1]["content"][0]["text"].startswith("1, 2,")
finally:
router.discard()
@pytest.mark.asyncio
@pytest.mark.parametrize("fallback_takes_prefill", [True, False], ids=["prefill", "request_to_continue"])
async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cannot_decrypt(
fallback_takes_prefill: bool,
claude_with_prefill: str,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream(" 3, 4, 5")
)
router: Final = _primary_with_fallback(claude_with_prefill if fallback_takes_prefill else "anthropic/claude-opus-5-5")
try:
stream: Final = await router.aresponses(
model="primary",
input=[
{"role": "user", "content": "Print the integers 1 through 5."},
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "primary-only-ciphertext"},
],
stream=True,
max_output_tokens=100,
)
_ = [event async for event in stream]
assert fallback.call_count == 1
assert "primary-only-ciphertext" not in fallback.calls.last.request.content.decode()
finally:
router.discard()
class _Numbers(BaseModel):
model_config = ConfigDict(frozen=True)
numbers: tuple[int, ...]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"constraint",
[
{"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}},
{"text": {"format": {"type": "json_object"}}},
{"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]},
{"text_format": _Numbers},
],
ids=["json_schema", "json_object", "required_tool", "sdk_text_format"],
)
async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed(
constraint: dict[str, object],
claude_with_prefill: str,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=_openai_stream_cut_after('{"numbers": [1, 2, ')
)
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("3]}")
)
router: Final = _primary_with_fallback(claude_with_prefill)
try:
stream: Final = await router.aresponses(
model="primary", input="Numbers 1 to 3 as JSON.", stream=True, max_output_tokens=100, **constraint
)
with pytest.raises(litellm.InternalServerError, match="interrupted"):
[event async for event in stream]
assert fallback.call_count == 0
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_continues_within_the_callers_output_ceiling(
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("3, 4, 5")
)
router: Final = _primary_with_fallback(claude_with_prefill)
try:
stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100)
events: Final = tuple([event async for event in stream])
assert _streamed_text(events) == "1, 2, 3, 4, 5"
assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter(
text="1, 2, "
)
finally:
router.discard()
@pytest.mark.asyncio
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling(
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("3, 4, 5")
)
router: Final = _primary_with_fallback(claude_with_prefill)
try:
stream: Final = await router.aresponses(
model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ")
)
with pytest.raises(litellm.InternalServerError, match="interrupted"):
[event async for event in stream]
assert fallback.call_count == 0
finally:
router.discard()
def _openai_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
return _sse(
{
"type": "response.output_text.delta", "sequence_number": 0,
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble,
},
{
"type": "response.output_item.added", "sequence_number": 1, "output_index": 1,
"item": {
"type": "function_call", "id": "fc_primary", "call_id": "call_primary",
"name": "lookup", "arguments": "", "status": "in_progress",
},
},
{
"type": "response.function_call_arguments.delta", "sequence_number": 2,
"item_id": "fc_primary", "output_index": 1, "delta": '{"city": "Par',
},
{
"type": "error", "sequence_number": 3,
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
},
)
def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
return _sse(
{"type": "message_start", "message": {
"id": "msg_primary", "type": "message", "role": "assistant",
"model": "claude-primary", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0},
}},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}},
{"type": "content_block_stop", "index": 0},
{"type": "content_block_start", "index": 1, "content_block": {
"type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {},
}},
{"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}},
{"type": "error", "error": {"type": "overloaded_error", "message": "interrupted"}},
)
@pytest.mark.asyncio
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=_openai_stream_cut_mid_tool_call("Let me look that up. ")
)
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
return_value=_anthropic_stream("It is sunny in Paris.")
)
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
try:
stream: Final = await router.aresponses(
model="primary",
input="What is the weather in Paris?",
stream=True,
tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}],
)
with pytest.raises(litellm.InternalServerError, match="interrupted"):
[event async for event in stream]
assert fallback.call_count == 0
finally:
router.discard()
@pytest.mark.asyncio
async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
side_effect=[
_anthropic_stream_cut_mid_tool_call("Let me look that up. "),
_anthropic_stream("It is sunny in Paris."),
]
)
router: Final = Router(
model_list=[
{"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}},
{"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}},
],
fallbacks=[{"primary": ["fallback"]}],
num_retries=0,
)
try:
stream: Final = await router.aresponses(
model="primary",
input="What is the weather in Paris?",
stream=True,
tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}],
)
with pytest.raises(litellm.InternalServerError, match="interrupted"):
[event async for event in stream]
assert provider.call_count == 1
finally:
router.discard()
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
@pytest.mark.asyncio

View file

@ -4,13 +4,23 @@ import litellm
from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
MID_STREAM_CONTINUATION_KWARG,
MID_STREAM_CONTINUATION_MARKER,
RESPONSES_CONTINUATION_KWARG,
ContinuationOutput,
ContinuationPrefillDeploymentCheck,
ResponsesContinuation,
ResponsesContinuationSeam,
constrains_continuation,
continuation_output_ceilings,
deployment_supports_prefill,
deployment_takes_responses_prefill,
responses_continuation_input,
responses_continuation_input_for,
)
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
PREFILL_MODEL = "anthropic/prefill-capable-test-model"
CLAUDE_PREFILL_MODEL = "anthropic/claude-prefill-capable-test-model"
CLAUDE_NON_PREFILL_MODEL = "anthropic/claude-prefill-rejecting-test-model"
NON_PREFILL_MODEL = "openai/prefill-unknown-test-model"
UNMAPPED_MODEL = "openai/unmapped-test-model"
@ -21,6 +31,12 @@ def _synthetic_cost_map_entries() -> None:
{
PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True},
NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"},
CLAUDE_PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True},
CLAUDE_NON_PREFILL_MODEL: {
"litellm_provider": "anthropic",
"mode": "chat",
"supports_assistant_prefill": False,
},
},
persist_across_reloads=False,
)
@ -191,7 +207,11 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream:
("Hello", (" world",), (" world",)),
("Hello", ("Hel", "p me"), ("Hel", "p me")),
("Hello", ("Hel",), ("Hel",)),
([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")),
(
[{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}],
("Sure: Hello", " world"),
("", " world"),
),
],
ids=[
"echo_split_across_chunks_is_dropped",
@ -206,9 +226,7 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream:
async def test_continuation_output_drops_only_a_full_echo_of_the_prefill(
prefill_content, fallback_texts, expected_texts
):
output = ContinuationOutput(
"Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ()
)
output = ContinuationOutput("Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ())
async def fallback_stream():
for text in fallback_texts:
@ -266,7 +284,14 @@ async def test_continuation_output_drops_only_whitespace_the_client_already_has(
({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}),
({"extra_body": {"top_k": 3}}, {}),
],
ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"],
ids=[
"max_tokens",
"both_ceilings",
"no_ceiling",
"extra_body_max_tokens",
"extra_body_only",
"extra_body_no_ceiling",
],
)
def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends(
request_kwargs, expected
@ -286,3 +311,186 @@ def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_
)
def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs):
assert continuation_output_ceilings(request_kwargs, 30) is None
@pytest.mark.parametrize(
"sent,expected",
[
({"text": {"format": {"type": "json_schema", "name": "n", "schema": {"type": "object"}}}}, True),
({"text": {"format": {"type": "json_object"}}}, True),
({"text": {"format": {"type": "text"}}}, False),
({"text": {"verbosity": "low"}}, False),
({"tool_choice": "required"}, True),
({"tool_choice": "auto"}, False),
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False),
({"tool_choice": {"type": "web_search_preview"}}, False),
],
ids=[
"json_schema",
"json_object",
"text_format",
"verbosity_only",
"required_tool",
"auto_tool",
"forced_function",
"allowed_tools_required",
"allowed_tools_auto",
"chat_allowed_tools_auto",
"hosted_tool",
],
)
def test_constrains_continuation_reads_the_responses_text_format(sent, expected):
assert constrains_continuation(sent) is expected
@pytest.mark.parametrize(
"deployment,expected",
[
(_deployment(CLAUDE_PREFILL_MODEL, "a"), True),
(_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False),
(_deployment(PREFILL_MODEL, "c"), False),
(
{"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}},
True,
),
],
ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"],
)
def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected):
assert deployment_takes_responses_prefill(deployment) is expected
def _continuation_kwargs() -> dict[str, object]:
continuation = ResponsesContinuation("Count to 5.", "1, 2, ")
return {RESPONSES_CONTINUATION_KWARG: continuation, "input": responses_continuation_input(continuation)}
@pytest.mark.parametrize(
"prior_input,expected_prior",
[
(
"Count to 5.",
[{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Count to 5."}]}],
),
(
[
{"role": "user", "content": "Count to 5."},
{"type": "function_call_output", "call_id": "c", "output": "x"},
],
[
{"role": "user", "content": "Count to 5."},
{"type": "function_call_output", "call_id": "c", "output": "x"},
],
),
(None, []),
],
ids=["string_input", "item_list_input", "no_input"],
)
def test_responses_continuation_input_keeps_the_prior_input_then_the_partial_then_a_request_to_continue(
prior_input, expected_prior
):
built = responses_continuation_input(ResponsesContinuation(prior_input, "1, 2, "))
partial = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "1, 2, "}]}
assert built[:-1] == [*expected_prior, partial]
assert built[-1]["role"] == "user"
assert "already ends with whitespace" in built[-1]["content"][0]["text"]
def test_responses_continuation_request_asks_for_a_leading_space_only_after_a_word():
built = responses_continuation_input(ResponsesContinuation("Hi", "Transformers use"))
assert "begin with a space" in built[-1]["content"][0]["text"]
def test_responses_continuation_accumulates_across_hops():
continuation = ResponsesContinuation("Count to 5.", "1, ").extended("2, ")
assert continuation == ResponsesContinuation("Count to 5.", "1, 2, ")
@pytest.mark.parametrize(
"deployment,expected_roles",
[
(_deployment(CLAUDE_PREFILL_MODEL, "a"), ["user", "assistant"]),
(_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), ["user", "assistant", "user"]),
(_deployment(NON_PREFILL_MODEL, "c"), ["user", "assistant", "user"]),
],
ids=["claude_prefill", "claude_without_prefill", "openai"],
)
def test_responses_continuation_input_for_shapes_the_input_for_the_deployment_that_serves_it(
deployment, expected_roles
):
shaped = responses_continuation_input_for(_continuation_kwargs(), deployment)
assert [item["role"] for item in shaped] == expected_roles
def test_responses_continuation_input_for_keeps_what_the_router_stripped_from_the_input():
continuation = ResponsesContinuation(
[
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "opaque"},
{"role": "user", "content": "Count to 5."},
],
"1, 2, ",
)
stripped_input = responses_continuation_input(continuation)[1:]
shaped = responses_continuation_input_for(
{RESPONSES_CONTINUATION_KWARG: continuation, "input": stripped_input}, _deployment(CLAUDE_PREFILL_MODEL, "a")
)
assert shaped == stripped_input[:-1]
def test_responses_continuation_input_for_leaves_an_input_without_its_request_to_continue_alone():
kwargs = {**_continuation_kwargs(), "input": [{"role": "user", "content": "Count to 5."}]}
assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == kwargs["input"]
def test_responses_continuation_input_for_leaves_a_request_without_a_continuation_alone():
kwargs = {"input": "Count to 5."}
assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == "Count to 5."
def _delta(text: str) -> OutputTextDeltaEvent:
return OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="m", output_index=0, content_index=0, delta=text
)
@pytest.mark.parametrize(
"emitted,fallback_texts,expected_texts",
[
("1, 2, ", (" 3, 4", ", 5"), ("3, 4", ", 5")),
("1, 2, ", (" ", " 3"), ("", " 3")),
("1, 2,", (" 3",), (" 3",)),
("1, 2, ", ("3",), ("3",)),
("A.\n\n", ("\n Hi",), (" Hi",)),
("", (" Hi",), (" Hi",)),
],
ids=[
"re_sent_space_is_dropped",
"seam_split_across_events",
"no_trailing_whitespace_keeps_the_space",
"no_re_sent_space_passes_through",
"seam_ends_at_the_first_divergence",
"fresh_start_passes_through",
],
)
@pytest.mark.asyncio
async def test_responses_continuation_seam_drops_only_whitespace_the_client_already_has(
emitted, fallback_texts, expected_texts
):
async def fallback_stream():
for text in fallback_texts:
yield _delta(text)
released = [event async for event in ResponsesContinuationSeam(emitted).splice(fallback_stream())]
assert tuple(event.delta for event in released) == expected_texts

View file

@ -7026,8 +7026,7 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events
@pytest.mark.asyncio
async def test_aresponses_streaming_iterator_partial_content_injects_continuation():
"""Mid-stream error: input is rewritten to include user prompt +
developer instruction + prior assistant message with partial output."""
"""Mid-stream error: preserve partial assistant output and request continuation in a user turn."""
from litellm.exceptions import MidStreamFallbackError
router = _make_router_with_fallback()
@ -7063,11 +7062,11 @@ async def test_aresponses_streaming_iterator_partial_content_injects_continuatio
assert isinstance(new_input, list)
assert new_input[0]["role"] == "user"
assert new_input[0]["content"][0]["text"] == "What's the capital of France?"
assert new_input[1]["role"] == "developer"
assert "do not repeat" in new_input[1]["content"][0]["text"].lower()
assert new_input[2]["role"] == "assistant"
assert new_input[2]["content"][0]["type"] == "output_text"
assert new_input[2]["content"][0]["text"] == "The capital of France is"
assert new_input[1]["role"] == "assistant"
assert new_input[1]["content"][0]["type"] == "output_text"
assert new_input[1]["content"][0]["text"] == "The capital of France is"
assert new_input[2]["role"] == "user"
assert "do not repeat" in new_input[2]["content"][0]["text"].lower()
@pytest.mark.asyncio