mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 2143c6c263 into dddbaa803d
This commit is contained in:
commit
9be2eb70d0
8 changed files with 769 additions and 135 deletions
|
|
@ -408,6 +408,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
continue
|
||||
if call_id in self._web_search_calls:
|
||||
continue
|
||||
self._started_tool_call = True
|
||||
|
||||
fn = tc.get("function") if isinstance(tc, dict) else getattr(tc, "function", None)
|
||||
fn_name = ""
|
||||
|
|
|
|||
|
|
@ -365,6 +365,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
"""
|
||||
|
||||
_pending_logging_tasks: tuple[asyncio.Task[object], ...] = ()
|
||||
_started_tool_call: bool = False
|
||||
_earlier_rounds_spend: BilledRoundsSpend | None = None
|
||||
|
||||
def __init__(
|
||||
|
|
@ -494,6 +495,8 @@ class BaseResponsesAPIStreamingIterator:
|
|||
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
):
|
||||
_item: Final[object] = getattr(openai_responses_api_chunk, "item", None)
|
||||
if getattr(_item, "type", None) in _TOOL_CALL_ITEM_TYPES:
|
||||
self._started_tool_call = True
|
||||
if _item is not None:
|
||||
ResponsesAPIRequestUtils.encode_container_id_on_output_item(
|
||||
item=_item,
|
||||
|
|
@ -1044,6 +1047,11 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if pending:
|
||||
await asyncio.wait(pending)
|
||||
|
||||
@property
|
||||
def started_tool_call(self) -> bool:
|
||||
"""Whether the stream began a tool call, which a text continuation could never finish."""
|
||||
return self._started_tool_call
|
||||
|
||||
def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None:
|
||||
"""
|
||||
The response.incomplete event to bill when the client went away mid-stream,
|
||||
|
|
@ -1690,6 +1698,7 @@ def _usage_as_model(usage: object) -> ResponseAPIUsage | None:
|
|||
return None
|
||||
|
||||
|
||||
_TOOL_CALL_ITEM_TYPES: Final = frozenset({"function_call", "custom_tool_call", "mcp_call"})
|
||||
_TOOL_ARGUMENTS_DELTA_EVENTS: Final = frozenset(
|
||||
{
|
||||
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
|
||||
|
|
|
|||
|
|
@ -301,14 +301,19 @@ from litellm.router_utils.kubernetes_pod_discovery import (
|
|||
)
|
||||
from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
|
||||
MID_STREAM_CONTINUATION_MARKER,
|
||||
RESPONSES_CONTINUATION_KWARG,
|
||||
ContinuationOutput,
|
||||
ContinuationPrefillDeploymentCheck,
|
||||
ContinuedPartial,
|
||||
ResponsesContinuation,
|
||||
ResponsesContinuationSeam,
|
||||
constrains_continuation,
|
||||
continuation_deployments,
|
||||
continuation_output_ceilings,
|
||||
continued_text,
|
||||
deployment_can_continue,
|
||||
responses_continuation_input,
|
||||
responses_continuation_input_for,
|
||||
)
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
|
||||
DeploymentAffinityCheck,
|
||||
|
|
@ -453,6 +458,7 @@ from litellm.utils import (
|
|||
provider_rejectable_params,
|
||||
resolve_api_key_secret_reference,
|
||||
set_live_deployment_replay,
|
||||
token_counter,
|
||||
)
|
||||
|
||||
from .router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
|
@ -484,7 +490,6 @@ if TYPE_CHECKING:
|
|||
from litellm.types.llms.openai import (
|
||||
ResponseAPIUsage,
|
||||
ResponseIncompleteEvent,
|
||||
ResponseInputParam,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
|
||||
|
|
@ -3838,7 +3843,8 @@ class Router:
|
|||
) -> Mapping[str, object]:
|
||||
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
|
||||
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
|
||||
defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None}
|
||||
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params)
|
||||
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
|
||||
deployment_tool_choice: Final = (
|
||||
{"tool_choice": deployment_params["tool_choice"]}
|
||||
if deployment_params.get("tool_choice") is not None
|
||||
|
|
@ -3917,67 +3923,6 @@ class Router:
|
|||
extended: Final = _extend_trailing_text_block(blocks, new_text)
|
||||
return [*messages[:-1], {**last, "content": extended, "prefix": True}]
|
||||
|
||||
@staticmethod
|
||||
def _build_responses_continuation_input(
|
||||
input_val: Union[str, "ResponseInputParam"] | None,
|
||||
generated_content: str,
|
||||
) -> "ResponseInputParam":
|
||||
"""
|
||||
Convert Responses-API input + partial assistant output into a
|
||||
continuation input that asks the fallback model to pick up where the
|
||||
prior assistant message stopped.
|
||||
|
||||
Best effort across providers. The chat-completions path uses
|
||||
Anthropic's `prefix: True` prefill trick on the assistant message;
|
||||
the Responses-API input schema has no direct equivalent, so we
|
||||
append an instruction (developer role) plus a prior assistant
|
||||
message containing the partial output. Providers without prefill
|
||||
semantics (OpenAI, Vertex) treat this as conversational context
|
||||
and may regenerate — same trade-off as the chat-completions path
|
||||
for non-Anthropic fallbacks.
|
||||
"""
|
||||
# base/continuation are List[Any] because ResponseInputParam items
|
||||
# are a wide Union of TypedDicts (EasyInputMessageParam, Message,
|
||||
# ResponseOutputMessageParam, ...) — annotating as List[Dict[str, Any]]
|
||||
# rejects the list() spread of input_val. We cast the combined list to
|
||||
# ResponseInputParam at the return.
|
||||
base: list[object]
|
||||
if isinstance(input_val, str):
|
||||
base = [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": input_val}],
|
||||
}
|
||||
]
|
||||
elif isinstance(input_val, list):
|
||||
base = list(input_val)
|
||||
else:
|
||||
base = []
|
||||
continuation: Final[list[object]] = [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": (
|
||||
"The previous assistant response was interrupted "
|
||||
"mid-stream. Continue exactly where it stopped — "
|
||||
"do not repeat any of its content. Your response "
|
||||
"must read as a seamless continuation."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": generated_content}],
|
||||
},
|
||||
]
|
||||
return cast("ResponseInputParam", base + continuation)
|
||||
|
||||
async def _aresponses_streaming_iterator(
|
||||
self,
|
||||
response: "BaseResponsesAPIStreamingIterator",
|
||||
|
|
@ -3999,9 +3944,12 @@ class Router:
|
|||
|
||||
Full parity with the chat-completions path:
|
||||
- Pre-first-chunk: retry with the original input unchanged.
|
||||
- Partial content: inject a developer instruction + prior
|
||||
assistant message carrying the generated text so the fallback
|
||||
model continues rather than restarts.
|
||||
- Partial content: continue from the generated text, as an
|
||||
assistant prefill on a fallback that takes one and as assistant
|
||||
history plus a user request to continue on any other. A request
|
||||
whose output is constrained (a JSON schema, a forced tool), or a
|
||||
stream that began a tool call, is not continued, and the primary's
|
||||
error surfaces instead.
|
||||
- Usage combining: merge partial-stream usage onto the fallback's
|
||||
response.completed event so accounting reflects both attempts.
|
||||
- Stream cleanup: shielded aclose() on both source and fallback
|
||||
|
|
@ -4190,6 +4138,42 @@ class Router:
|
|||
wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks())
|
||||
return wrapper
|
||||
|
||||
def _responses_continuation(
|
||||
self,
|
||||
e: "MidStreamFallbackError",
|
||||
source_iterator: "BaseResponsesAPIStreamingIterator",
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> ResponsesContinuation | None:
|
||||
"""What a fallback continues from once a Responses stream broke after text, raising the primary's error
|
||||
when the request constrains its output or the stream began a tool call, since a continued JSON document or
|
||||
tool call would not parse."""
|
||||
if not e.generated_content or e.is_pre_first_chunk:
|
||||
return None
|
||||
if source_iterator.started_tool_call:
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id")
|
||||
cut_deployment: Final = (
|
||||
self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None
|
||||
)
|
||||
if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG)
|
||||
if isinstance(earlier, ResponsesContinuation):
|
||||
return earlier.extended(e.generated_content)
|
||||
return ResponsesContinuation(request_kwargs.get("input"), e.generated_content)
|
||||
|
||||
@staticmethod
|
||||
def _responses_continuation_ceilings(
|
||||
e: "MidStreamFallbackError", request_kwargs: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
"""The caller's output ceilings less what the cut stream already emitted, raising the primary's error once
|
||||
nothing is left."""
|
||||
emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content)
|
||||
reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens)
|
||||
if reduced is None:
|
||||
Router._raise_original_mid_stream_error(e)
|
||||
return reduced
|
||||
|
||||
async def _aresponses_fallback_attempt(
|
||||
self,
|
||||
e: "MidStreamFallbackError",
|
||||
|
|
@ -4208,6 +4192,11 @@ class Router:
|
|||
"""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs)
|
||||
reduced_ceilings: Final[Mapping[str, object]] = (
|
||||
Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {}
|
||||
)
|
||||
seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "")
|
||||
partial_usage: Final = Router._extract_partial_responses_usage(source_iterator)
|
||||
fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
|
||||
fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement
|
||||
|
|
@ -4225,11 +4214,10 @@ class Router:
|
|||
initial_kwargs["original_function"] = ( # rebind-ok: the fallback chain re-enters on the same kwargs
|
||||
self._ageneric_api_call_with_fallbacks_responses_attempt
|
||||
)
|
||||
if e.generated_content and not e.is_pre_first_chunk:
|
||||
initial_kwargs["input"] = Router._build_responses_continuation_input( # rebind-ok: fallback hop input
|
||||
initial_kwargs.get("input"),
|
||||
e.generated_content,
|
||||
)
|
||||
if continuation is not None:
|
||||
initial_kwargs.update(reduced_ceilings)
|
||||
initial_kwargs[RESPONSES_CONTINUATION_KWARG] = continuation # rebind-ok: shaped per fallback deployment
|
||||
initial_kwargs["input"] = responses_continuation_input(continuation) # rebind-ok: fallback hop input
|
||||
# The Responses-API path stores observability metadata
|
||||
# under "litellm_metadata" (not the default "metadata") —
|
||||
# see _ageneric_api_call_with_fallbacks. Mirroring that
|
||||
|
|
@ -4258,7 +4246,7 @@ class Router:
|
|||
)
|
||||
prepared_fallback_hidden_params: Final = adopt_headers(fallback_response)
|
||||
if hasattr(fallback_response, "__aiter__"):
|
||||
async for fallback_item in fallback_response:
|
||||
async for fallback_item in seam.splice(fallback_response):
|
||||
Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params)
|
||||
if partial_usage is not None:
|
||||
Router._combine_responses_fallback_usage(fallback_item, partial_usage)
|
||||
|
|
@ -4781,11 +4769,12 @@ class Router:
|
|||
splits_stream_timeout: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
4 jobs:
|
||||
5 jobs:
|
||||
- Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
|
||||
- Adds default litellm params to kwargs, if set.
|
||||
- Merges tools from deployment with request (proxy-configured tools + request tools).
|
||||
- On a fallback hop, drops the encrypted reasoning this deployment cannot decrypt, keeping its summary.
|
||||
- On a Responses continuation, sends the partial response as prefill where this deployment takes one.
|
||||
"""
|
||||
for key in self._forwarded_alias_marker_keys_the_deployment_sets(
|
||||
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
|
||||
|
|
@ -4959,6 +4948,11 @@ class Router:
|
|||
(_FALLBACK_HOP_ADAPTER.validate_python(deployment),),
|
||||
unmarked_origin=deployment_that_just_failed,
|
||||
)
|
||||
if RESPONSES_CONTINUATION_KWARG in hop_kwargs:
|
||||
del kwargs[RESPONSES_CONTINUATION_KWARG] # rebind-ok: this method prepares the call's kwargs in place
|
||||
kwargs["input"] = responses_continuation_input_for( # rebind-ok: the deployment's continuation input
|
||||
hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment)
|
||||
)
|
||||
|
||||
def _get_async_openai_model_client(self, deployment: dict, kwargs: dict):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
"""Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave
|
||||
the output unconstrained."""
|
||||
the output unconstrained, and shapes a Responses API continuation for the deployment that serves it."""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate
|
||||
from os.path import commonprefix
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -14,7 +15,7 @@ import litellm
|
|||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.rules import Rules
|
||||
from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
|
||||
from litellm.utils import supports_assistant_prefill
|
||||
|
||||
|
|
@ -28,7 +29,10 @@ MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
|
|||
_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
|
||||
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens")
|
||||
_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens", "max_output_tokens")
|
||||
_TOOL_FORCING_CHOICE_TYPES: Final = frozenset({"function", "custom", "tool", "any"})
|
||||
_QUOTED_ENDING_CHARS: Final = 100
|
||||
RESPONSES_CONTINUATION_KWARG: Final = "_responses_continuation"
|
||||
|
||||
|
||||
def _str_keyed(value: object) -> Mapping[str, object]:
|
||||
|
|
@ -38,28 +42,39 @@ def _str_keyed(value: object) -> Mapping[str, object]:
|
|||
return _NO_FIELDS
|
||||
|
||||
|
||||
def deployment_supports_prefill(deployment: object) -> bool:
|
||||
def _underlying_model(deployment: object) -> str | None:
|
||||
deployment_map: Final = _str_keyed(deployment)
|
||||
model_info: Final = _str_keyed(deployment_map.get("model_info"))
|
||||
declared: Final = model_info.get("supports_assistant_prefill")
|
||||
if isinstance(declared, bool):
|
||||
return declared
|
||||
litellm_params: Final = _str_keyed(deployment_map.get("litellm_params"))
|
||||
candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model"))
|
||||
model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
|
||||
return next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
|
||||
|
||||
|
||||
def deployment_supports_prefill(deployment: object) -> bool:
|
||||
declared: Final = _str_keyed(_str_keyed(deployment).get("model_info")).get("supports_assistant_prefill")
|
||||
if isinstance(declared, bool):
|
||||
return declared
|
||||
model: Final = _underlying_model(deployment)
|
||||
return model is not None and supports_assistant_prefill(model=model)
|
||||
|
||||
|
||||
def _forces_tool_call(tool_choice: object) -> bool:
|
||||
choice: Final = _str_keyed(tool_choice)
|
||||
mode: Final = choice.get("mode") or _str_keyed(choice.get("allowed_tools")).get("mode")
|
||||
return tool_choice == "required" or mode == "required" or choice.get("type") in _TOOL_FORCING_CHOICE_TYPES
|
||||
|
||||
|
||||
def constrains_continuation(params: Mapping[str, object]) -> bool:
|
||||
sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
|
||||
choice_count: Final = sent.get("n")
|
||||
response_format: Final = sent.get("response_format")
|
||||
tool_choice: Final = sent.get("tool_choice")
|
||||
text_format: Final = _str_keyed(sent.get("text")).get("format")
|
||||
return (
|
||||
(isinstance(choice_count, int) and choice_count > 1)
|
||||
or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or tool_choice == "required"
|
||||
or isinstance(tool_choice, Mapping)
|
||||
or (text_format is not None and text_format != _UNCONSTRAINED_RESPONSE_FORMAT)
|
||||
or sent.get("text_format") is not None
|
||||
or _forces_tool_call(sent.get("tool_choice"))
|
||||
or sent.get("merge_reasoning_content_in_choices") is True
|
||||
)
|
||||
|
||||
|
|
@ -99,6 +114,7 @@ def _delta_text(chunk: ModelResponseStream) -> str:
|
|||
|
||||
|
||||
_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
|
||||
_INPUT_ITEMS_ADAPTER: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _prefill_text(prefill_message: Mapping[str, object]) -> str:
|
||||
|
|
@ -114,7 +130,7 @@ def _prefill_text(prefill_message: Mapping[str, object]) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _resendable_whitespace(prefill: str) -> str:
|
||||
def resendable_whitespace(prefill: str) -> str:
|
||||
return prefill[len(prefill.rstrip()) :] if prefill.strip() else ""
|
||||
|
||||
|
||||
|
|
@ -124,7 +140,7 @@ def continued_text(prefill_message: Mapping[str, object], generated_content: str
|
|||
return None
|
||||
echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0
|
||||
after_echo: Final = generated_content[echo:]
|
||||
return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :]
|
||||
return after_echo[len(commonprefix((after_echo, resendable_whitespace(prefill)))) :]
|
||||
|
||||
|
||||
def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]:
|
||||
|
|
@ -211,7 +227,7 @@ class ContinuationOutput:
|
|||
self._rules: Final = Rules()
|
||||
self._held: tuple[ModelResponseStream, ...] = ()
|
||||
self._echo_settled = not self._prefill_text
|
||||
self._seam = _resendable_whitespace(self._prefill_text)
|
||||
self._seam = resendable_whitespace(self._prefill_text)
|
||||
|
||||
async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]:
|
||||
async for item in stream:
|
||||
|
|
@ -262,3 +278,91 @@ class ContinuationOutput:
|
|||
self._text += _delta_text(chunk)
|
||||
self._rules.post_call_rules(input=self._text, model=self._model)
|
||||
return chunks
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResponsesContinuation:
|
||||
prior_input: object
|
||||
emitted_text: str
|
||||
|
||||
def extended(self, generated_content: str) -> "ResponsesContinuation":
|
||||
return ResponsesContinuation(self.prior_input, f"{self.emitted_text}{generated_content}")
|
||||
|
||||
|
||||
def deployment_takes_responses_prefill(deployment: object) -> bool:
|
||||
"""Claude takes a trailing assistant turn as prefill with no `prefix` flag, which a Responses input item cannot
|
||||
carry."""
|
||||
model: Final = _underlying_model(deployment)
|
||||
return model is not None and "claude" in model.lower() and deployment_supports_prefill(deployment)
|
||||
|
||||
|
||||
def _continuation_request(emitted_text: str) -> Mapping[str, object]:
|
||||
seam: Final = (
|
||||
"It already ends with whitespace, so do not add another space before your first word."
|
||||
if emitted_text[-1:].isspace()
|
||||
else "It ends without trailing whitespace, so begin with a space if your first word starts a new word."
|
||||
)
|
||||
ending: Final = json.dumps(emitted_text[-_QUOTED_ENDING_CHARS:], ensure_ascii=False)
|
||||
instruction: Final = (
|
||||
f"The previous assistant response was interrupted mid-stream. It ended with {ending}. Continue exactly where "
|
||||
"it stopped, so your first characters are the ones that would have come right after it, and do not repeat "
|
||||
f"any of its content. {seam} Your response must read as a seamless continuation."
|
||||
)
|
||||
return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": instruction}]}
|
||||
|
||||
|
||||
def _input_items(request_input: object) -> tuple[object, ...]:
|
||||
if isinstance(request_input, str):
|
||||
return ({"type": "message", "role": "user", "content": [{"type": "input_text", "text": request_input}]},)
|
||||
try:
|
||||
return tuple(_INPUT_ITEMS_ADAPTER.validate_python(request_input))
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def responses_continuation_input(continuation: ResponsesContinuation) -> Sequence[object]:
|
||||
"""The prior input, the partial response as assistant history, and a user request to continue it."""
|
||||
partial: Final = {
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": continuation.emitted_text}],
|
||||
}
|
||||
return [*_input_items(continuation.prior_input), partial, _continuation_request(continuation.emitted_text)]
|
||||
|
||||
|
||||
def responses_continuation_input_for(request_kwargs: Mapping[str, object], deployment: Mapping[str, object]) -> object:
|
||||
"""The continuation input as the deployment receives it, without the request to continue where the deployment
|
||||
takes the partial response as prefill. It reads the input the router already stripped of reasoning this
|
||||
deployment cannot decrypt, never the continuation's prior input."""
|
||||
request_input: Final = request_kwargs.get("input")
|
||||
continuation: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG)
|
||||
if not isinstance(continuation, ResponsesContinuation):
|
||||
return request_input
|
||||
items: Final = _input_items(request_input)
|
||||
if items[-1:] != (_continuation_request(continuation.emitted_text),) or not deployment_takes_responses_prefill(
|
||||
deployment
|
||||
):
|
||||
return request_input
|
||||
return list(items[:-1])
|
||||
|
||||
|
||||
_StreamItem = TypeVar("_StreamItem")
|
||||
|
||||
|
||||
class ResponsesContinuationSeam:
|
||||
"""Drops whitespace a fallback sends again at the start of its text when the client already has it, since
|
||||
Anthropic strips a prefill's trailing whitespace and the model then writes it back."""
|
||||
|
||||
def __init__(self, emitted_text: str) -> None:
|
||||
self._seam = resendable_whitespace(emitted_text)
|
||||
|
||||
async def splice(self, stream: AsyncIterable[_StreamItem]) -> AsyncIterator[_StreamItem]:
|
||||
async for item in stream:
|
||||
yield self._past(item)
|
||||
|
||||
def _past(self, item: _StreamItem) -> _StreamItem:
|
||||
if not self._seam or not isinstance(item, OutputTextDeltaEvent):
|
||||
return item
|
||||
matched: Final = len(commonprefix((item.delta, self._seam)))
|
||||
self._seam = self._seam[matched:] if matched == len(item.delta) else ""
|
||||
return item.model_copy(update={"delta": item.delta[matched:]}) if matched else item
|
||||
|
|
|
|||
|
|
@ -171,6 +171,8 @@ ignored_function_names = [
|
|||
"_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py
|
||||
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
|
||||
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
|
||||
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
|
||||
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,17 @@
|
|||
"""The shared Responses API config contract."""
|
||||
|
||||
import asyncio, importlib, json
|
||||
from collections.abc import Mapping
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm import Router
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from litellm.constants import STREAM_SSE_DONE_STRING
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
|
|
@ -879,38 +882,6 @@ def test_combine_responses_fallback_usage_passthrough_for_unknown_event():
|
|||
partial = ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2)
|
||||
Router._combine_responses_fallback_usage(other, partial)
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
def test_build_responses_continuation_input_from_string():
|
||||
out = Router._build_responses_continuation_input("Hello world", "partial assistant text")
|
||||
assert len(out) == 3
|
||||
assert out[0]["role"] == "user"
|
||||
assert out[0]["content"][0]["text"] == "Hello world"
|
||||
assert out[1]["role"] == "developer"
|
||||
assert out[2]["role"] == "assistant"
|
||||
assert out[2]["content"][0]["text"] == "partial assistant text"
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
def test_build_responses_continuation_input_from_list_preserves_items():
|
||||
existing: List[Any] = [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "msg1"}],
|
||||
}
|
||||
]
|
||||
out = Router._build_responses_continuation_input(existing, "partial")
|
||||
assert len(out) == 3
|
||||
assert out[0]["content"][0]["text"] == "msg1"
|
||||
assert out[1]["role"] == "developer"
|
||||
assert out[2]["role"] == "assistant"
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
def test_build_responses_continuation_input_from_none():
|
||||
out = Router._build_responses_continuation_input(None, "partial")
|
||||
assert len(out) == 2
|
||||
assert out[0]["role"] == "developer"
|
||||
assert out[1]["role"] == "assistant"
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_passthrough():
|
||||
|
|
@ -1355,9 +1326,355 @@ async def test_aresponses_fallback_uses_continuation_input_after_partial_content
|
|||
continuation = mock_fallback.await_args.kwargs["kwargs"]["input"]
|
||||
assert isinstance(continuation, list)
|
||||
assert continuation[0]["content"][0]["text"] == "original question"
|
||||
assert continuation[-2]["role"] == "developer"
|
||||
assert continuation[-1]["role"] == "assistant"
|
||||
assert continuation[-1]["content"][0]["text"] == "partial answer"
|
||||
assert continuation[-2]["role"] == "assistant"
|
||||
assert continuation[-2]["content"][0]["text"] == "partial answer"
|
||||
assert continuation[-1]["role"] == "user"
|
||||
|
||||
|
||||
def _sse(*events: Mapping[str, object]) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events),
|
||||
)
|
||||
|
||||
|
||||
def _openai_stream_cut_after(partial: str) -> httpx.Response:
|
||||
return _sse(
|
||||
*(
|
||||
({
|
||||
"type": "response.output_text.delta", "sequence_number": 0,
|
||||
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": partial,
|
||||
},)
|
||||
if partial else ()
|
||||
),
|
||||
{
|
||||
"type": "error", "sequence_number": 1,
|
||||
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_stream(text: str) -> httpx.Response:
|
||||
return _sse(
|
||||
{"type": "message_start", "message": {
|
||||
"id": "msg_fallback", "type": "message", "role": "assistant",
|
||||
"model": "claude-fallback", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0},
|
||||
}},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 5}},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
|
||||
|
||||
def _primary_with_fallback(fallback_model: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/gpt-6-astra", "api_key": "synthetic"}},
|
||||
{"model_name": "fallback", "litellm_params": {"model": fallback_model, "api_key": "synthetic"}},
|
||||
],
|
||||
fallbacks=[{"primary": ["fallback"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
|
||||
def _streamed_text(events: tuple[Any, ...]) -> str:
|
||||
return "".join(event.delta for event in events if event.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def claude_with_prefill() -> str:
|
||||
model: Final = "anthropic/claude-prefill-capable-test-model"
|
||||
litellm.register_model(
|
||||
{model: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("partial", ["", "1, 2, "])
|
||||
async def test_responses_fallback_can_continue_on_a_provider_without_prefill(
|
||||
partial: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
prompt: Final = "Print the integers 1 through 5, separated by commas."
|
||||
primary: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=_openai_stream_cut_after(partial)
|
||||
)
|
||||
|
||||
def no_prefill_provider(request: httpx.Request) -> httpx.Response:
|
||||
body: Final = json.loads(request.content)
|
||||
if body["messages"][-1]["role"] != "user":
|
||||
return httpx.Response(
|
||||
400,
|
||||
json={"type": "error", "error": {
|
||||
"type": "invalid_request_error", "message": "The conversation must end with a user message",
|
||||
}},
|
||||
)
|
||||
return _anthropic_stream("3, 4, 5" if partial else "1, 2, 3, 4, 5")
|
||||
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=no_prefill_provider)
|
||||
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
|
||||
try:
|
||||
stream: Final = await router.aresponses(model="primary", input=prompt, stream=True, max_output_tokens=100)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events) == "1, 2, 3, 4, 5"
|
||||
assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
assert primary.call_count == fallback.call_count == 1
|
||||
messages: Final = json.loads(fallback.calls.last.request.content)["messages"]
|
||||
assert messages[0] == {"role": "user", "content": [{"type": "text", "text": prompt}]}
|
||||
if partial:
|
||||
assert [message["role"] for message in messages] == ["user", "assistant", "user"]
|
||||
assert messages[-2] == {"role": "assistant", "content": [{"type": "text", "text": partial}]}
|
||||
assert json.dumps(partial) in messages[-1]["content"][0]["text"]
|
||||
else:
|
||||
assert len(messages) == 1
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_continues_a_claude_that_takes_prefill_from_its_own_partial(
|
||||
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream(" 3, 4, 5")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill)
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary", input="Print the integers 1 through 5.", stream=True, max_output_tokens=100
|
||||
)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events) == "1, 2, 3, 4, 5"
|
||||
assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
|
||||
messages: Final = json.loads(fallback.calls.last.request.content)["messages"]
|
||||
assert [message["role"] for message in messages] == ["user", "assistant"]
|
||||
assert messages[-1]["content"][0]["text"].startswith("1, 2,")
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fallback_takes_prefill", [True, False], ids=["prefill", "request_to_continue"])
|
||||
async def test_responses_fallback_continuation_drops_reasoning_the_fallback_cannot_decrypt(
|
||||
fallback_takes_prefill: bool,
|
||||
claude_with_prefill: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream(" 3, 4, 5")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill if fallback_takes_prefill else "anthropic/claude-opus-5-5")
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary",
|
||||
input=[
|
||||
{"role": "user", "content": "Print the integers 1 through 5."},
|
||||
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "primary-only-ciphertext"},
|
||||
],
|
||||
stream=True,
|
||||
max_output_tokens=100,
|
||||
)
|
||||
_ = [event async for event in stream]
|
||||
assert fallback.call_count == 1
|
||||
assert "primary-only-ciphertext" not in fallback.calls.last.request.content.decode()
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
class _Numbers(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
numbers: tuple[int, ...]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"constraint",
|
||||
[
|
||||
{"text": {"format": {"type": "json_schema", "name": "numbers", "strict": True, "schema": {"type": "object"}}}},
|
||||
{"text": {"format": {"type": "json_object"}}},
|
||||
{"tool_choice": "required", "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}]},
|
||||
{"text_format": _Numbers},
|
||||
],
|
||||
ids=["json_schema", "json_object", "required_tool", "sdk_text_format"],
|
||||
)
|
||||
async def test_responses_fallback_surfaces_the_primary_error_once_constrained_output_was_streamed(
|
||||
constraint: dict[str, object],
|
||||
claude_with_prefill: str,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=_openai_stream_cut_after('{"numbers": [1, 2, ')
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("3]}")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill)
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary", input="Numbers 1 to 3 as JSON.", stream=True, max_output_tokens=100, **constraint
|
||||
)
|
||||
with pytest.raises(litellm.InternalServerError, match="interrupted"):
|
||||
[event async for event in stream]
|
||||
assert fallback.call_count == 0
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_continues_within_the_callers_output_ceiling(
|
||||
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("3, 4, 5")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill)
|
||||
try:
|
||||
stream: Final = await router.aresponses(model="primary", input="Count to 5.", stream=True, max_output_tokens=100)
|
||||
events: Final = tuple([event async for event in stream])
|
||||
assert _streamed_text(events) == "1, 2, 3, 4, 5"
|
||||
assert json.loads(fallback.calls.last.request.content)["max_tokens"] == 100 - litellm.token_counter(
|
||||
text="1, 2, "
|
||||
)
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_used_the_output_ceiling(
|
||||
claude_with_prefill: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(return_value=_openai_stream_cut_after("1, 2, "))
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("3, 4, 5")
|
||||
)
|
||||
router: Final = _primary_with_fallback(claude_with_prefill)
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary", input="Count to 5.", stream=True, max_output_tokens=litellm.token_counter(text="1, 2, ")
|
||||
)
|
||||
with pytest.raises(litellm.InternalServerError, match="interrupted"):
|
||||
[event async for event in stream]
|
||||
assert fallback.call_count == 0
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
def _openai_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
|
||||
return _sse(
|
||||
{
|
||||
"type": "response.output_text.delta", "sequence_number": 0,
|
||||
"item_id": "msg_primary", "output_index": 0, "content_index": 0, "delta": preamble,
|
||||
},
|
||||
{
|
||||
"type": "response.output_item.added", "sequence_number": 1, "output_index": 1,
|
||||
"item": {
|
||||
"type": "function_call", "id": "fc_primary", "call_id": "call_primary",
|
||||
"name": "lookup", "arguments": "", "status": "in_progress",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "response.function_call_arguments.delta", "sequence_number": 2,
|
||||
"item_id": "fc_primary", "output_index": 1, "delta": '{"city": "Par',
|
||||
},
|
||||
{
|
||||
"type": "error", "sequence_number": 3,
|
||||
"error": {"type": "server_error", "code": "internal_error", "message": "interrupted"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_stream_cut_mid_tool_call(preamble: str) -> httpx.Response:
|
||||
return _sse(
|
||||
{"type": "message_start", "message": {
|
||||
"id": "msg_primary", "type": "message", "role": "assistant",
|
||||
"model": "claude-primary", "content": [], "usage": {"input_tokens": 10, "output_tokens": 0},
|
||||
}},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": preamble}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{"type": "content_block_start", "index": 1, "content_block": {
|
||||
"type": "tool_use", "id": "toolu_primary", "name": "lookup", "input": {},
|
||||
}},
|
||||
{"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": "Par'}},
|
||||
{"type": "error", "error": {"type": "overloaded_error", "message": "interrupted"}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=_openai_stream_cut_mid_tool_call("Let me look that up. ")
|
||||
)
|
||||
fallback: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
return_value=_anthropic_stream("It is sunny in Paris.")
|
||||
)
|
||||
router: Final = _primary_with_fallback("anthropic/claude-opus-5-5")
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary",
|
||||
input="What is the weather in Paris?",
|
||||
stream=True,
|
||||
tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}],
|
||||
)
|
||||
with pytest.raises(litellm.InternalServerError, match="interrupted"):
|
||||
[event async for event in stream]
|
||||
assert fallback.call_count == 0
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridged_responses_fallback_surfaces_the_primary_error_once_the_cut_stream_began_a_tool_call(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
side_effect=[
|
||||
_anthropic_stream_cut_mid_tool_call("Let me look that up. "),
|
||||
_anthropic_stream("It is sunny in Paris."),
|
||||
]
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "primary", "litellm_params": {"model": "anthropic/claude-primary", "api_key": "synthetic"}},
|
||||
{"model_name": "fallback", "litellm_params": {"model": "anthropic/claude-opus-5-5", "api_key": "synthetic"}},
|
||||
],
|
||||
fallbacks=[{"primary": ["fallback"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
try:
|
||||
stream: Final = await router.aresponses(
|
||||
model="primary",
|
||||
input="What is the weather in Paris?",
|
||||
stream=True,
|
||||
tools=[{"type": "function", "name": "lookup", "parameters": {"type": "object"}}],
|
||||
)
|
||||
with pytest.raises(litellm.InternalServerError, match="interrupted"):
|
||||
[event async for event in stream]
|
||||
assert provider.call_count == 1
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_router_unit", "setup_and_teardown_router_unit")
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -4,13 +4,23 @@ import litellm
|
|||
from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
MID_STREAM_CONTINUATION_MARKER,
|
||||
RESPONSES_CONTINUATION_KWARG,
|
||||
ContinuationOutput,
|
||||
ContinuationPrefillDeploymentCheck,
|
||||
ResponsesContinuation,
|
||||
ResponsesContinuationSeam,
|
||||
constrains_continuation,
|
||||
continuation_output_ceilings,
|
||||
deployment_supports_prefill,
|
||||
deployment_takes_responses_prefill,
|
||||
responses_continuation_input,
|
||||
responses_continuation_input_for,
|
||||
)
|
||||
from litellm.types.llms.openai import OutputTextDeltaEvent, ResponsesAPIStreamEvents
|
||||
|
||||
PREFILL_MODEL = "anthropic/prefill-capable-test-model"
|
||||
CLAUDE_PREFILL_MODEL = "anthropic/claude-prefill-capable-test-model"
|
||||
CLAUDE_NON_PREFILL_MODEL = "anthropic/claude-prefill-rejecting-test-model"
|
||||
NON_PREFILL_MODEL = "openai/prefill-unknown-test-model"
|
||||
UNMAPPED_MODEL = "openai/unmapped-test-model"
|
||||
|
||||
|
|
@ -21,6 +31,12 @@ def _synthetic_cost_map_entries() -> None:
|
|||
{
|
||||
PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True},
|
||||
NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"},
|
||||
CLAUDE_PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True},
|
||||
CLAUDE_NON_PREFILL_MODEL: {
|
||||
"litellm_provider": "anthropic",
|
||||
"mode": "chat",
|
||||
"supports_assistant_prefill": False,
|
||||
},
|
||||
},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
|
|
@ -191,7 +207,11 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream:
|
|||
("Hello", (" world",), (" world",)),
|
||||
("Hello", ("Hel", "p me"), ("Hel", "p me")),
|
||||
("Hello", ("Hel",), ("Hel",)),
|
||||
([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")),
|
||||
(
|
||||
[{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}],
|
||||
("Sure: Hello", " world"),
|
||||
("", " world"),
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"echo_split_across_chunks_is_dropped",
|
||||
|
|
@ -206,9 +226,7 @@ def _text_chunk(text: str) -> litellm.ModelResponseStream:
|
|||
async def test_continuation_output_drops_only_a_full_echo_of_the_prefill(
|
||||
prefill_content, fallback_texts, expected_texts
|
||||
):
|
||||
output = ContinuationOutput(
|
||||
"Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ()
|
||||
)
|
||||
output = ContinuationOutput("Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ())
|
||||
|
||||
async def fallback_stream():
|
||||
for text in fallback_texts:
|
||||
|
|
@ -266,7 +284,14 @@ async def test_continuation_output_drops_only_whitespace_the_client_already_has(
|
|||
({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}),
|
||||
({"extra_body": {"top_k": 3}}, {}),
|
||||
],
|
||||
ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"],
|
||||
ids=[
|
||||
"max_tokens",
|
||||
"both_ceilings",
|
||||
"no_ceiling",
|
||||
"extra_body_max_tokens",
|
||||
"extra_body_only",
|
||||
"extra_body_no_ceiling",
|
||||
],
|
||||
)
|
||||
def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends(
|
||||
request_kwargs, expected
|
||||
|
|
@ -286,3 +311,186 @@ def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_
|
|||
)
|
||||
def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs):
|
||||
assert continuation_output_ceilings(request_kwargs, 30) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sent,expected",
|
||||
[
|
||||
({"text": {"format": {"type": "json_schema", "name": "n", "schema": {"type": "object"}}}}, True),
|
||||
({"text": {"format": {"type": "json_object"}}}, True),
|
||||
({"text": {"format": {"type": "text"}}}, False),
|
||||
({"text": {"verbosity": "low"}}, False),
|
||||
({"tool_choice": "required"}, True),
|
||||
({"tool_choice": "auto"}, False),
|
||||
({"tool_choice": {"type": "function", "name": "lookup"}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "f"}]}}, True),
|
||||
({"tool_choice": {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "f"}]}}, False),
|
||||
({"tool_choice": {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}}, False),
|
||||
({"tool_choice": {"type": "web_search_preview"}}, False),
|
||||
],
|
||||
ids=[
|
||||
"json_schema",
|
||||
"json_object",
|
||||
"text_format",
|
||||
"verbosity_only",
|
||||
"required_tool",
|
||||
"auto_tool",
|
||||
"forced_function",
|
||||
"allowed_tools_required",
|
||||
"allowed_tools_auto",
|
||||
"chat_allowed_tools_auto",
|
||||
"hosted_tool",
|
||||
],
|
||||
)
|
||||
def test_constrains_continuation_reads_the_responses_text_format(sent, expected):
|
||||
assert constrains_continuation(sent) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment,expected",
|
||||
[
|
||||
(_deployment(CLAUDE_PREFILL_MODEL, "a"), True),
|
||||
(_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), False),
|
||||
(_deployment(PREFILL_MODEL, "c"), False),
|
||||
(
|
||||
{"litellm_params": {"model": "anthropic/support-model-v1"}, "model_info": {"id": "d", "base_model": CLAUDE_PREFILL_MODEL}},
|
||||
True,
|
||||
),
|
||||
],
|
||||
ids=["claude_with_prefill", "claude_without_prefill", "prefill_capable_but_not_claude", "claude_named_by_base_model"],
|
||||
)
|
||||
def test_deployment_takes_responses_prefill_only_on_claude_with_prefill(deployment, expected):
|
||||
assert deployment_takes_responses_prefill(deployment) is expected
|
||||
|
||||
|
||||
def _continuation_kwargs() -> dict[str, object]:
|
||||
continuation = ResponsesContinuation("Count to 5.", "1, 2, ")
|
||||
return {RESPONSES_CONTINUATION_KWARG: continuation, "input": responses_continuation_input(continuation)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prior_input,expected_prior",
|
||||
[
|
||||
(
|
||||
"Count to 5.",
|
||||
[{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Count to 5."}]}],
|
||||
),
|
||||
(
|
||||
[
|
||||
{"role": "user", "content": "Count to 5."},
|
||||
{"type": "function_call_output", "call_id": "c", "output": "x"},
|
||||
],
|
||||
[
|
||||
{"role": "user", "content": "Count to 5."},
|
||||
{"type": "function_call_output", "call_id": "c", "output": "x"},
|
||||
],
|
||||
),
|
||||
(None, []),
|
||||
],
|
||||
ids=["string_input", "item_list_input", "no_input"],
|
||||
)
|
||||
def test_responses_continuation_input_keeps_the_prior_input_then_the_partial_then_a_request_to_continue(
|
||||
prior_input, expected_prior
|
||||
):
|
||||
built = responses_continuation_input(ResponsesContinuation(prior_input, "1, 2, "))
|
||||
|
||||
partial = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "1, 2, "}]}
|
||||
assert built[:-1] == [*expected_prior, partial]
|
||||
assert built[-1]["role"] == "user"
|
||||
assert "already ends with whitespace" in built[-1]["content"][0]["text"]
|
||||
|
||||
|
||||
def test_responses_continuation_request_asks_for_a_leading_space_only_after_a_word():
|
||||
built = responses_continuation_input(ResponsesContinuation("Hi", "Transformers use"))
|
||||
|
||||
assert "begin with a space" in built[-1]["content"][0]["text"]
|
||||
|
||||
|
||||
def test_responses_continuation_accumulates_across_hops():
|
||||
continuation = ResponsesContinuation("Count to 5.", "1, ").extended("2, ")
|
||||
|
||||
assert continuation == ResponsesContinuation("Count to 5.", "1, 2, ")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment,expected_roles",
|
||||
[
|
||||
(_deployment(CLAUDE_PREFILL_MODEL, "a"), ["user", "assistant"]),
|
||||
(_deployment(CLAUDE_NON_PREFILL_MODEL, "b"), ["user", "assistant", "user"]),
|
||||
(_deployment(NON_PREFILL_MODEL, "c"), ["user", "assistant", "user"]),
|
||||
],
|
||||
ids=["claude_prefill", "claude_without_prefill", "openai"],
|
||||
)
|
||||
def test_responses_continuation_input_for_shapes_the_input_for_the_deployment_that_serves_it(
|
||||
deployment, expected_roles
|
||||
):
|
||||
shaped = responses_continuation_input_for(_continuation_kwargs(), deployment)
|
||||
|
||||
assert [item["role"] for item in shaped] == expected_roles
|
||||
|
||||
|
||||
def test_responses_continuation_input_for_keeps_what_the_router_stripped_from_the_input():
|
||||
continuation = ResponsesContinuation(
|
||||
[
|
||||
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "opaque"},
|
||||
{"role": "user", "content": "Count to 5."},
|
||||
],
|
||||
"1, 2, ",
|
||||
)
|
||||
stripped_input = responses_continuation_input(continuation)[1:]
|
||||
|
||||
shaped = responses_continuation_input_for(
|
||||
{RESPONSES_CONTINUATION_KWARG: continuation, "input": stripped_input}, _deployment(CLAUDE_PREFILL_MODEL, "a")
|
||||
)
|
||||
|
||||
assert shaped == stripped_input[:-1]
|
||||
|
||||
|
||||
def test_responses_continuation_input_for_leaves_an_input_without_its_request_to_continue_alone():
|
||||
kwargs = {**_continuation_kwargs(), "input": [{"role": "user", "content": "Count to 5."}]}
|
||||
|
||||
assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == kwargs["input"]
|
||||
|
||||
|
||||
def test_responses_continuation_input_for_leaves_a_request_without_a_continuation_alone():
|
||||
kwargs = {"input": "Count to 5."}
|
||||
|
||||
assert responses_continuation_input_for(kwargs, _deployment(CLAUDE_PREFILL_MODEL, "a")) == "Count to 5."
|
||||
|
||||
|
||||
def _delta(text: str) -> OutputTextDeltaEvent:
|
||||
return OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id="m", output_index=0, content_index=0, delta=text
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"emitted,fallback_texts,expected_texts",
|
||||
[
|
||||
("1, 2, ", (" 3, 4", ", 5"), ("3, 4", ", 5")),
|
||||
("1, 2, ", (" ", " 3"), ("", " 3")),
|
||||
("1, 2,", (" 3",), (" 3",)),
|
||||
("1, 2, ", ("3",), ("3",)),
|
||||
("A.\n\n", ("\n Hi",), (" Hi",)),
|
||||
("", (" Hi",), (" Hi",)),
|
||||
],
|
||||
ids=[
|
||||
"re_sent_space_is_dropped",
|
||||
"seam_split_across_events",
|
||||
"no_trailing_whitespace_keeps_the_space",
|
||||
"no_re_sent_space_passes_through",
|
||||
"seam_ends_at_the_first_divergence",
|
||||
"fresh_start_passes_through",
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_continuation_seam_drops_only_whitespace_the_client_already_has(
|
||||
emitted, fallback_texts, expected_texts
|
||||
):
|
||||
async def fallback_stream():
|
||||
for text in fallback_texts:
|
||||
yield _delta(text)
|
||||
|
||||
released = [event async for event in ResponsesContinuationSeam(emitted).splice(fallback_stream())]
|
||||
|
||||
assert tuple(event.delta for event in released) == expected_texts
|
||||
|
|
|
|||
|
|
@ -7026,8 +7026,7 @@ async def test_aresponses_fallback_attempt_does_not_replay_held_lifecycle_events
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aresponses_streaming_iterator_partial_content_injects_continuation():
|
||||
"""Mid-stream error: input is rewritten to include user prompt +
|
||||
developer instruction + prior assistant message with partial output."""
|
||||
"""Mid-stream error: preserve partial assistant output and request continuation in a user turn."""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
router = _make_router_with_fallback()
|
||||
|
|
@ -7063,11 +7062,11 @@ async def test_aresponses_streaming_iterator_partial_content_injects_continuatio
|
|||
assert isinstance(new_input, list)
|
||||
assert new_input[0]["role"] == "user"
|
||||
assert new_input[0]["content"][0]["text"] == "What's the capital of France?"
|
||||
assert new_input[1]["role"] == "developer"
|
||||
assert "do not repeat" in new_input[1]["content"][0]["text"].lower()
|
||||
assert new_input[2]["role"] == "assistant"
|
||||
assert new_input[2]["content"][0]["type"] == "output_text"
|
||||
assert new_input[2]["content"][0]["text"] == "The capital of France is"
|
||||
assert new_input[1]["role"] == "assistant"
|
||||
assert new_input[1]["content"][0]["type"] == "output_text"
|
||||
assert new_input[1]["content"][0]["text"] == "The capital of France is"
|
||||
assert new_input[2]["role"] == "user"
|
||||
assert "do not repeat" in new_input[2]["content"][0]["text"].lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue