This commit is contained in:
HootRock 2026-10-31 19:57:12 +00:00 • committed by GitHub
commit cf149e5df0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 2466 additions and 81 deletions

View file

@ -269,6 +269,7 @@ _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call"
_OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE}
_SCANNED_TOOL_SEARCH_NAME: Final = "tool_search"
_TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str)
_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call": "arguments", "custom_tool_call": "input"}
)
@ -1950,7 +1951,6 @@ class OpenAIResponsesHandler(BaseTranslation):
new output item under the same response id, and close with a
``response.completed`` carrying only the replacement item.
The proxy's data generator appends ``data: [DONE]`` itself.
"""
events: Final = (
self._block_continuation_events(exc, responses_so_far or ())
@ -1958,6 +1958,7 @@ class OpenAIResponsesHandler(BaseTranslation):
else self._standalone_block_events(exc)
)
return tuple(
f"event: {_STREAM_EVENT_TYPE_ADAPTER.validate_python(stream_item_field(event, 'type'))}\n"
f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode()
for event in events
)

View file

@ -7,6 +7,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
from dataclasses import dataclass
from datetime import datetime, timezone
from functools import lru_cache
from http import HTTPStatus
from itertools import chain
from types import MappingProxyType
from typing import (
@ -51,7 +52,7 @@ from litellm.constants import (
STREAM_SSE_KEEPALIVE_PING_BYTES,
UNSAFE_PROXY_RESPONSE_HEADERS,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.integrations.otel.runtime import phase_event
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.bug_report import (
@ -92,6 +93,7 @@ from litellm.litellm_core_utils.streaming_handler import (
backfill_missing_cache_usage_fields,
)
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import (
can_key_call_resolved_model,
@ -119,6 +121,11 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_type,
relayed_provider_specific_fields,
)
from litellm.proxy.common_utils.responses_stream_errors import (
SSE_LINE_BREAK,
ResponsesStreamErrorState,
sse_error_object,
)
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING_BYTES,
SSE_STREAM_START_TAIL,
@ -1005,11 +1012,32 @@ async def _enforce_guardrail_added_tag_budgets(
) from e
async def _parse_event_data_for_error(event_line: str | bytes) -> int | None:
SseDataPayload: TypeAlias = Callable[[str | bytes], str | None]
def _sse_text(event_line: str | bytes) -> str:
return event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
def _sse_data_payload(event_line: str | bytes) -> str | None:
text: Final = _sse_text(event_line)
return text.removeprefix("data: ").strip() if text.startswith("data: ") else None
def _responses_event_data_payload(event_line: str | bytes) -> str | None:
match SSE_LINE_BREAK.split(_sse_text(event_line).strip()):
case ["event: error", data_line] if data_line.startswith("data: "):
return data_line.removeprefix("data: ").strip()
case _:
return _sse_data_payload(event_line)
async def _parse_event_data_for_error(
event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload
) -> int | None:
"""Parses an event line and returns an error code if present, else None."""
event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
if event_line.startswith("data: "):
json_str: Final = event_line[len("data: ") :].strip()
json_str: Final = data_payload(event_line)
if json_str is not None:
if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message
return None
try:
@ -1040,18 +1068,17 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None:
return None
def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict:
def _extract_error_from_sse_chunk(event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload) -> dict:
"""
Extract error dictionary from SSE format chunk.
Args:
event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n"
data_payload: Reads the JSON payload out of the event line
Returns:
Error dictionary in OpenAI API format
"""
event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
# Default error format
default_error: Final = {
"message": "Unknown error",
@ -1060,8 +1087,8 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict:
"code": "500",
}
if event_line.startswith("data: "):
json_str: Final = event_line[len("data: ") :].strip()
json_str: Final = data_payload(event_line)
if json_str is not None:
if not json_str or json_str == "[DONE]":
return default_error
@ -1269,6 +1296,75 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n"
def _responses_error_frames(error_obj: Mapping[str, object]) -> tuple[str, ...]:
return (ResponsesStreamErrorState().format_error(error_obj),)
@dataclass(frozen=True, slots=True)
class LateSseFraming:
"""How a response the keepalive already opened ends once its upstream fails late."""
failure: Callable[[Exception], tuple[str, ...]]
body: Callable[[Response], tuple[bytes, ...]]
def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]:
_, error_obj = sse_error_payload(exc)
return _sse_error_frames(error_obj)
def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]:
return b"data: " + (bytes(produced.body) or b"{}") + b"\n\n", b"data: [DONE]\n\n"
def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]:
if isinstance(exc, ModifyResponseException):
return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc))
return _responses_error_frames(sse_error_payload(exc)[1])
def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]:
body: Final = bytes(produced.body)
error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode()}
with_status: Final = {**error_obj, "code": error_obj.get("code") or str(produced.status_code)}
return tuple(frame.encode() for frame in _responses_error_frames(with_status))
OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames)
RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming(
failure=_responses_late_failure_frames, body=_responses_late_body_frames
)
@dataclass(frozen=True, slots=True)
class FirstChunkSseFraming:
"""How a stream's first chunk is read for an error, and how the stream ends when reading it fails."""
data_payload: SseDataPayload
error_frames: Callable[[Mapping[str, object]], tuple[str, ...]]
lowest_error_status: int
OPENAI_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(
data_payload=_sse_data_payload, error_frames=_sse_error_frames, lowest_error_status=HTTPStatus.CONTINUE
)
RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(
data_payload=_responses_event_data_payload,
error_frames=_responses_error_frames,
lowest_error_status=HTTPStatus.BAD_REQUEST,
)
@dataclass(frozen=True, slots=True)
class SseFraming:
first_chunk: FirstChunkSseFraming
late: LateSseFraming
OPENAI_SSE_FRAMING: Final = SseFraming(first_chunk=OPENAI_FIRST_CHUNK_FRAMING, late=OPENAI_LATE_SSE_FRAMING)
RESPONSES_SSE_FRAMING: Final = SseFraming(first_chunk=RESPONSES_FIRST_CHUNK_FRAMING, late=RESPONSES_LATE_SSE_FRAMING)
def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
"""`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384)."""
return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS})
@ -1297,6 +1393,7 @@ async def create_response(
request: Request | None = None,
refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None,
general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS,
framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING,
) -> StreamingResponse | JSONResponse:
"""
Create streaming response, checking if the first chunk is an error.
@ -1332,8 +1429,10 @@ async def create_response(
if first_chunk_value is not None:
try:
error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value)
if error_code_from_chunk is not None:
error_code_from_chunk: Final = await _parse_event_data_for_error(
first_chunk_value, framing.data_payload
)
if error_code_from_chunk is not None and error_code_from_chunk >= framing.lowest_error_status:
# First chunk is an error, stream hasn't really started yet
# Should return standard JSON error response instead of SSE format
final_status_code = error_code_from_chunk
@ -1344,7 +1443,9 @@ async def create_response(
# Parse error content
error_dict: Final = with_call_id(
JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)),
JSON_OBJECT.validate_python(
_extract_error_from_sse_chunk(first_chunk_value, framing.data_payload)
),
error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)),
)
@ -1386,7 +1487,7 @@ async def create_response(
error_status, error_obj = sse_error_payload(e)
async def error_gen_message() -> AsyncGenerator[str, None]:
for frame in _sse_error_frames(error_obj):
for frame in framing.error_frames(error_obj):
yield frame
return StreamingResponse(
@ -1469,13 +1570,13 @@ async def _aclose_late_response(produced: Response) -> None:
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]:
async def _relay_late_response(produced: Response, framing: LateSseFraming) -> AsyncGenerator[bytes, None]:
"""Replay a Response that was built after a keepalive had already opened the wire."""
if not isinstance(produced, StreamingResponse):
# The status line is already on the wire, so a non-streaming body, an error
# body included, can only reach the client as an SSE frame.
yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n"
yield b"data: [DONE]\n\n"
for frame in framing.body(produced):
yield frame
return
try:
@ -1517,6 +1618,7 @@ async def open_sse_before_first_byte(
ping_interval_seconds: float | str | None,
media_type: str = "text/event-stream",
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING,
) -> _LateResponseT | StreamingResponse:
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
@ -1562,11 +1664,10 @@ async def open_sse_before_first_byte(
# would never fire and the failure would go unaudited. The hook
# also gets to sanitize what reaches the client, by returning or
# raising a replacement, so its answer decides the frame.
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
for frame in _sse_error_frames(error_obj):
for frame in late_framing.failure(await _sanitized_late_failure(exc, on_late_failure)):
yield frame.encode()
return
async for chunk in _relay_late_response(produced):
async for chunk in _relay_late_response(produced, late_framing):
yield chunk
finally:
if not produce_task.done():
@ -2964,6 +3065,7 @@ class ProxyBaseLLMRequestProcessing:
is_streaming_request: bool | None = False,
contents: list[object] | None = None,
skip_pre_call_logic: bool = False,
sse_framing: SseFraming = OPENAI_SSE_FRAMING,
) -> object:
"""Run the request, sending SSE keepalives while the upstream is still silent.
@ -3004,9 +3106,11 @@ class ProxyBaseLLMRequestProcessing:
is_streaming_request=is_streaming_request,
contents=contents,
skip_pre_call_logic=skip_pre_call_logic,
first_chunk_framing=sse_framing.first_chunk,
),
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
on_late_failure=_audit_late_failure,
late_framing=sse_framing.late,
)
async def _process_llm_request(
@ -3030,6 +3134,7 @@ class ProxyBaseLLMRequestProcessing:
is_streaming_request: bool | None = False,
contents: list[object] | None = None, # Add contents parameter
skip_pre_call_logic: bool = False,
first_chunk_framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING,
) -> Any:
"""
Common request processing logic for both chat completions and responses API endpoints
@ -3313,6 +3418,7 @@ class ProxyBaseLLMRequestProcessing:
request=request,
refresh_headers=refresh_stream_headers,
general_settings=general_settings,
framing=first_chunk_framing,
)
### CALL HOOKS ### - modify outgoing data
@ -3555,8 +3661,8 @@ class ProxyBaseLLMRequestProcessing:
Streaming ``/v1/responses`` short-circuits out of
``base_process_llm_request`` before the non-streaming ownership
tail runs. The OpenAI SDK closes the connection at ``data: [DONE]``
and starlette cancels the body task on disconnect, so a write that
tail runs. The OpenAI SDK can close the connection right after the terminal
event and starlette cancels the body task on disconnect, so a write that
waits for the generator to finish never lands. The iterator sets
``completed_response`` before it hands over its terminal chunk, so
the ``LiteLLM_ManagedObjectTable`` row is written the moment it

View file

@ -1,10 +1,18 @@
import gc
import json
import re
import time
from collections.abc import Mapping
from collections import Counter, OrderedDict, defaultdict
from collections.abc import Iterable, Iterator, Mapping, Sequence
from contextlib import suppress
from enum import Enum
from http import HTTPStatus
from itertools import chain, groupby, takewhile, zip_longest
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, field_validator
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
from typing_extensions import TypeIs # noqa: TID251 # the guards narrow hook values to object items, isinstance cannot
from litellm._logging import redact_internal_details_from_client_message
from litellm._uuid import uuid
@ -29,6 +37,13 @@ class _StreamEvent(LiteLLMBaseModel):
response: _ResponseIdentity | None = None
class _FramedEvent(_StreamEvent):
error: object = None
_LAX_INT: Final = TypeAdapter(int)
class _FailureDetails(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
@ -37,6 +52,16 @@ class _FailureDetails(LiteLLMBaseModel):
type: str | None = None
status_code: int | None = None
@field_validator("status_code", mode="before")
@classmethod
def normalize_status_code(cls, value: object) -> int | None:
if isinstance(value, bool):
return None
try:
return _LAX_INT.validate_python(value)
except ValidationError:
return None
@field_validator("message", mode="before")
@classmethod
def normalize_message(cls, value: object) -> str | None:
@ -57,6 +82,265 @@ class _FailureDetails(LiteLLMBaseModel):
return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields})
class _ErrorPayload(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
error: Mapping[str, object] | str
def error_object(self) -> Mapping[str, object]:
return {"message": self.error} if isinstance(self.error, str) else self.error
_DROPPED: Final = object()
class _BaseForm(Enum):
PRINTED = "printed"
DUMPED = "dumped"
def _repr_owner(kind: type[object]) -> type[object]:
return next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object)
def _repr_prints_every_item(kind: type[object]) -> bool:
return kind in (OrderedDict, Counter) or _repr_owner(kind) in (dict, list, tuple, defaultdict)
def _format_prints_every_item(kind: type[object]) -> bool:
return kind.__str__ is object.__str__ and kind.__format__ is object.__format__ and _repr_prints_every_item(kind)
def _str_prints_every_item(kind: type[object]) -> bool:
return kind.__str__ is object.__str__ and _repr_owner(kind) in (
dict,
list,
tuple,
OrderedDict,
defaultdict,
Counter,
)
def _is_dict(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object
return isinstance(value, dict)
def _is_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: items as object
return isinstance(value, (list, tuple))
def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed
return isinstance(value, MappingProxyType)
def _is_defaultdict(value: object) -> TypeIs[defaultdict[object, object]]: # guard-ok: dict items as object
return isinstance(value, defaultdict)
def _proxied(proxy: Mapping[object, object]) -> object:
return gc.get_referents(proxy)[0]
def _repr_prints_its_fields(model: BaseModel) -> bool:
kind: Final = type(model)
return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__
def _plain_scalar(value: str | float) -> str | float:
if isinstance(value, str):
return str.__str__(value)
if isinstance(value, int):
return int.__int__(value)
return float.__float__(value)
def _repr_shows_its_value(value: str | float, printed: str) -> bool:
plain: Final = _plain_scalar(value)
if isinstance(value, Enum):
return printed == f"<{type(value).__name__}.{value.name}: {plain!r}>"
return printed == repr(plain)
def _printed_text(value: object) -> object:
printed: Final = repr(value)
if isinstance(value, (str, int, float)) and _repr_shows_its_value(value, printed):
return _plain_scalar(value)
return printed
def _dumped_text(value: object) -> object:
return _plain_scalar(value) if isinstance(value, (str, int, float)) else str(value)
def _text(value: object, form: _BaseForm) -> object:
try:
return _printed_text(value) if form is _BaseForm.PRINTED else _dumped_text(value)
except Exception: # noqa: BLE001 # hook values run their own __repr__ and __str__, so a failure drops the value
return _DROPPED
def _json_key(key: object, form: _BaseForm) -> object:
return key if key is None or type(key) in (str, int, float, bool) else _text(key, form)
def _stored_sequence(value: Sequence[object]) -> Iterator[object]:
if isinstance(value, list):
return list[object].__iter__(value)
if isinstance(value, tuple):
return tuple[object, ...].__iter__(value)
return iter(value)
def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, object]]:
return dict[object, object].items(value) if isinstance(value, dict) else value.items()
def _live_items(value: Mapping[object, object]) -> Iterator[tuple[object, object]]:
shown: Final = dict[int, object]()
while True:
with suppress(RuntimeError):
for key, item in _stored_items(value):
if id(key) not in shown:
shown[id(key)] = key
yield key, item
return
def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool:
return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored))
def _shown_sequence(value: object, form: _BaseForm) -> Iterable[object] | None:
if not _is_sequence(value):
return None
if form is _BaseForm.PRINTED:
return _stored_sequence(value) if _repr_prints_every_item(type(value)) else None
dumped: Final = tuple(value)
return dumped if _dumps_shown(type(value), dumped, _stored_sequence(value)) else None
def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None:
if not _is_dict(value):
return None
if form is _BaseForm.DUMPED:
dumped: Final = tuple(value.items())
stored: Final = chain.from_iterable(_stored_items(value))
return dumped if _dumps_shown(type(value), chain.from_iterable(dumped), stored) else None
if not _repr_prints_every_item(type(value)):
return None
if isinstance(value, OrderedDict):
return tuple(OrderedDict[object, object].items(value))
if isinstance(value, Counter):
return Counter[object].most_common(value)
return _live_items(value)
def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None:
mapping: Final = _proxied(proxy)
if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)):
return None
return _shown_items(mapping, _BaseForm.PRINTED)
def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None:
if _is_proxy(value):
proxy_items: Final = _proxy_items(value, form)
return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED)
shown_items: Final = _shown_items(value, form)
if shown_items is not None:
return shown_items, form
if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value):
return value.__repr_args__(), _BaseForm.PRINTED
return None
def _walked_ids(value: object) -> frozenset[int]:
return frozenset((id(value), id(_proxied(value)))) if _is_proxy(value) else frozenset((id(value),))
def _container_marker(value: object) -> object:
if isinstance(value, list):
return "[...]"
if isinstance(value, tuple):
return "(...)"
if isinstance(value, OrderedDict):
return "..."
if _is_defaultdict(value):
factory: Final = _text(value.default_factory, _BaseForm.PRINTED)
return factory if factory is _DROPPED else f"{type(value).__name__}({factory}, {{...}})"
return "{...}" if _is_dict(value) and not isinstance(value, Counter) else None
def _cycle_marker(value: object) -> object:
if not _is_proxy(value):
return _container_marker(value)
inner: Final = _container_marker(_proxied(value))
return inner if inner is None or inner is _DROPPED else f"mappingproxy({inner})"
def _cycle_text(value: object, form: _BaseForm) -> object:
marker: Final = _cycle_marker(value)
return _text(value, form) if marker is None else marker
def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
if value is None or type(value) in (str, int, float, bool):
return value
walked: Final = _walked_ids(value)
if not walked.isdisjoint(path):
return _cycle_text(value, form)
inner_path: Final = path | walked
try:
sequence: Final = _shown_sequence(value, form)
if sequence is not None:
shown_items: Final = [_json_safe(item, form, inner_path) for item in sequence]
return [item for item in shown_items if item is not _DROPPED]
expansion: Final = _expansion(value, form)
if expansion is None:
return _text(value, form)
items, inner = expansion
pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items]
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit
return _text(value, form)
def _chunk_json(shown: object) -> str | None:
if isinstance(shown, str):
return shown
return None if shown is _DROPPED else json.dumps(shown)
def responses_chunk_data(chunk: object) -> str | None:
if _is_proxy(chunk):
expandable: Final = _proxy_items(chunk, _BaseForm.DUMPED) is not None
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if expandable else None
if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)):
return None
return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset()))
def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]:
path: Final = frozenset((id(error),))
shown: Final = [(key, _json_safe(value, _BaseForm.DUMPED, path)) for key, value in error.items()]
return {key: value for key, value in shown if value is not _DROPPED}
def sse_error_object(data: str | bytes) -> Mapping[str, object] | None:
try:
return _ErrorPayload.model_validate_json(data).error_object()
except ValidationError:
return None
def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None:
try:
return _ErrorPayload.model_validate(event).error_object() or None
except ValidationError:
return None
def _original_failure(exception: Exception) -> Exception:
current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal
while isinstance(current, MidStreamFallbackError) and current.original_exception is not None:
@ -95,6 +379,16 @@ def _status_error_code(status_code: int | None) -> str:
return _CLIENT_ERROR_CODES.get(status_code, "invalid_request_error")
def _status_of(code: str | int | None) -> int | None:
if isinstance(code, int):
return code
if code is None or not code.isdecimal():
return None
with suppress(ValueError):
return int(code)
return None
def _response_error_code(details: _FailureDetails) -> str:
for value in (details.code, details.type):
if value == "insufficient_quota":
@ -106,6 +400,115 @@ def _response_error_code(details: _FailureDetails) -> str:
return _status_error_code(details.status_code)
_STATUS_PHRASES: Final = MappingProxyType({int(status): status.phrase for status in HTTPStatus})
def _status_phrase(status_code: int | None) -> str:
return _STATUS_PHRASES.get(status_code, "Unknown error") if status_code is not None else "Unknown error"
def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]:
details: Final = _FailureDetails.model_validate(error)
status_code: Final = _status_of(details.code)
return {
"code": _response_error_code(details.model_copy(update={"status_code": status_code})),
"message": details.message or _status_phrase(status_code),
}
def _error_event(
fields: Mapping[str, object], error: Mapping[str, object], sequence_number: int
) -> Mapping[str, object]:
return {
"type": ResponsesAPIStreamEvents.ERROR.value,
**fields,
"param": error.get("param"),
"sequence_number": sequence_number,
"error": error,
}
SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
_TERMINAL_EVENT_TYPES: Final = frozenset(("response.completed", "response.failed", "response.incomplete", "error"))
_MAX_SEQUENCE_NUMBER: Final = 2**63 - 1
def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]:
prefix: Final = f"{name}:"
return tuple(line.removeprefix(prefix).removeprefix(" ") for line in lines if line.startswith(prefix))
def _parsed_event(data: str) -> _FramedEvent:
try:
return _FramedEvent.model_validate_json(data)
except ValidationError:
return _FramedEvent()
def _in_range_sequence(event: _FramedEvent) -> _FramedEvent:
sequence_number: Final = event.sequence_number
if sequence_number is None or 0 <= sequence_number < _MAX_SEQUENCE_NUMBER:
return event
return event.model_copy(update={"sequence_number": None})
def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool:
return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",)
def _block_data(lines: Sequence[str]) -> str:
return "\n".join(_field_values(lines, "data"))
def _block_event(lines: Sequence[str]) -> _StreamEvent:
parsed: Final = _in_range_sequence(_parsed_event(_block_data(lines)))
if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines):
return parsed
return parsed.model_copy(update={"type": "error"})
def _blocks(frames: str) -> Iterator[tuple[str, ...]]:
return (tuple(lines) for filled, lines in groupby(SSE_LINE_BREAK.split(frames), key=bool) if filled)
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
return tuple(_block_event(lines) for lines in _blocks(frames))
def _before_done(lines: Sequence[str]) -> bool:
return _block_data(lines).strip() != "[DONE]"
def _terminal_blocks(frame: str) -> tuple[tuple[tuple[str, ...], _StreamEvent], ...]:
read: Final = ((lines, _block_event(lines)) for lines in takewhile(_before_done, _blocks(frame)))
return tuple((lines, event) for lines, event in read if event.type in _TERMINAL_EVENT_TYPES)
def frame_error_data(frame: str) -> str | None:
terminal: Final = _terminal_blocks(frame)
if not terminal:
return None
lines, event = terminal[-1]
return _block_data(lines) if event.type == "error" else None
def _observed_event(chunk: object) -> _StreamEvent | None:
if not isinstance(chunk, (BaseModel, Mapping)):
return None
try:
return _StreamEvent.model_validate(chunk)
except ValidationError:
return None
def _wire_event(frame: str | bytes) -> _StreamEvent | None:
text: Final = frame.decode("utf-8", errors="replace") if isinstance(frame, bytes) else frame
try:
return _StreamEvent.model_validate_json(text.removeprefix("data: "))
except ValidationError:
return None
class ResponsesStreamErrorState:
def __init__(self) -> None:
self.response_id: str | None = None
@ -114,14 +517,13 @@ class ResponsesStreamErrorState:
self.sequence_number = -1
self.terminal_emitted = False
self._pending_event: _StreamEvent | None = None
self._pending_on_wire = False
def observe_chunk(self, chunk: object) -> None:
self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None
self._pending_event = _observed_event(chunk)
self._pending_on_wire = isinstance(chunk, Mapping)
def mark_emitted(self, frame: str | bytes) -> str | bytes:
event: Final = self._pending_event
if event is None:
return frame
def _record(self, event: _StreamEvent) -> None:
if event.sequence_number is not None:
self.sequence_number = max(self.sequence_number, event.sequence_number)
if event.response is not None:
@ -129,9 +531,34 @@ class ResponsesStreamErrorState:
self.model = event.response.model or self.model
if event.response.created_at is not None:
self.created_at = event.response.created_at
if event.type in ("response.completed", "response.failed", "response.incomplete"):
if event.type in _TERMINAL_EVENT_TYPES:
self.terminal_emitted = True
return frame
def observe_frames(self, frames: str) -> str:
for event in _frame_events(frames):
self._record(event)
return frames
def mark_emitted(self, frame: str | bytes) -> str | bytes:
observed: Final = self._pending_event
if observed is None:
return frame
event: Final = _wire_event(frame) if self._pending_on_wire else observed
self._record(observed if event is None else event)
if event is None or not event.type or "\n" in event.type or "\r" in event.type:
return frame
prefix: Final = f"event: {event.type}\n"
return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame
def format_error(self, error: Mapping[str, object]) -> str:
fields: Final = responses_error_fields(error)
sequence_number: Final = self.sequence_number + 1
payload: Final = json.dumps(_error_event(fields, _dumped_detail(error), sequence_number))
return f"event: error\ndata: {payload}\n\n"
def format_error_message(self, frame: str) -> str:
error: Final = sse_error_object(frame.removeprefix("data: "))
return frame if error is None else self.format_error(error)
def format_failure(self, exception: Exception) -> str | None:
if self.terminal_emitted:

View file

@ -554,7 +554,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import (
from litellm.proxy.common_utils.proxy_state import ProxyState
from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_data
from litellm.proxy.common_utils.scheduled_job_stagger import (
apply_scheduled_job_stagger,
attach_job_timing_logger,
@ -10309,6 +10309,15 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes:
return f"data: {chunk}\n\n"
def _format_responses_sse_chunk(chunk: object) -> str | bytes:
data: Final = responses_chunk_data(chunk)
return _format_streaming_sse_chunk(chunk=chunk) if data is None else f"data: {data}\n\n"
def _observed_frames(frames: str, error_state: ResponsesStreamErrorState | None) -> str:
return frames if error_state is None else error_state.observe_frames(frames)
_SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r")
_OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
_MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024
@ -10548,7 +10557,9 @@ async def async_data_generator(
stream_completed = False
client_disconnected = False
error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None
ends_with_openai_done: Final = not request_data.get("_litellm_skip_openai_stream_done")
ends_with_openai_done: Final = not responses_stream_errors and not request_data.get(
"_litellm_skip_openai_stream_done"
)
needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap()
stream_iterator: Final = _object_stream(
proxy_logging_obj.async_post_call_streaming_iterator_hook(
@ -10683,12 +10694,14 @@ async def async_data_generator(
frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer)
if frame is None:
break
yield frame
yield _observed_frames(frame, error_state)
if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS:
raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter")
raw_passthrough = True
elif chunk.startswith(("data:", "event:", ":")):
yield (chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n")
yield _observed_frames(
chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n", error_state
)
raw_passthrough = True
elif isinstance(chunk, str) and is_raw_sse_stream:
raw_sse_buffer += chunk
@ -10696,7 +10709,7 @@ async def async_data_generator(
frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer)
if frame is None:
break
yield frame
yield _observed_frames(frame, error_state)
if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS:
raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter")
raw_passthrough = True
@ -10707,7 +10720,7 @@ async def async_data_generator(
if not raw_passthrough:
try:
if error_state is not None:
yield error_state.mark_emitted(_format_streaming_sse_chunk(chunk=chunk))
yield error_state.mark_emitted(_format_responses_sse_chunk(chunk=chunk))
else:
yield _format_streaming_sse_chunk(chunk=chunk)
except Exception as e:
@ -10730,11 +10743,14 @@ async def async_data_generator(
ProxyLogging.fire_deferred_stream_logging(request_data)
if raw_sse_buffer:
yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n")
yield _observed_frames(
raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n",
error_state,
)
if error_message is not None:
yield error_message
# OpenAI-compatible streams terminate with data: [DONE]; Google GenAI (?alt=sse) does not.
yield error_message if error_state is None else error_state.format_error_message(error_message)
# Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not.
if ends_with_openai_done:
yield _OPENAI_STREAM_DONE_FRAME
except (asyncio.CancelledError, GeneratorExit):

View file

@ -61,6 +61,7 @@ from litellm.proxy.auth.user_api_key_auth import (
user_api_key_auth_websocket,
)
from litellm.proxy.common_request_processing import (
RESPONSES_SSE_FRAMING,
ProxyBaseLLMRequestProcessing,
cost_headers,
create_response,
@ -483,6 +484,7 @@ async def responses_api(
if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data))
else native_data_generator
),
sse_framing=RESPONSES_SSE_FRAMING,
model=None,
user_model=user_model,
user_temperature=user_temperature,
@ -541,7 +543,6 @@ async def responses_api(
async def _blocked_stream() -> AsyncGenerator[str, None]:
for chunk in block_chunks:
yield chunk.decode()
yield "data: [DONE]\n\n"
return await create_response(generator=_blocked_stream(), media_type="text/event-stream", headers={})
return build_blocked_response(e)
@ -908,6 +909,7 @@ async def get_response(
response_id,
read_settles_billing,
),
sse_framing=RESPONSES_SSE_FRAMING,
model=None,
user_model=user_model,
user_temperature=user_temperature,

View file

@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
import asyncio
import json
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
from http import HTTPStatus
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
from fastapi import Request, Response
@ -20,7 +21,13 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.responses_stream_errors import (
event_error_object,
frame_error_data,
responses_error_fields,
sse_error_object,
)
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
from litellm.types.llms.openai import ResponsesAPIStatus
@ -69,6 +76,7 @@ class _StreamEvent(TypedDict, total=False):
delta: ReadOnly[str]
part: ReadOnly[object]
response: ReadOnly[_TerminalResponse]
error: ReadOnly[_JsonDict]
class _StreamEventParser:
@ -85,6 +93,22 @@ def _sse_frame_data(frame: str) -> str | None:
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
def _event_error(event: Mapping[str, object]) -> Mapping[str, object]:
match event.get("error"):
case str() as message if message:
return {**event, "message": message}
case _:
return event_error_object(event) or event
def _first_chunk_error(response: object) -> Mapping[str, object] | None:
if isinstance(response, _StreamsBody) or not isinstance(response, Response):
return None
if response.status_code < HTTPStatus.BAD_REQUEST:
return None
return responses_error_fields(sse_error_object(bytes(response.body)) or {"code": response.status_code})
async def _never_receive() -> Message:
await asyncio.Event().wait()
raise AssertionError("unreachable")
@ -165,6 +189,7 @@ async def background_streaming_task(
user_api_base=user_api_base,
version=version,
skip_pre_call_logic=True,
sse_framing=RESPONSES_SSE_FRAMING,
)
# Process streaming response following OpenAI events format
@ -199,6 +224,7 @@ async def background_streaming_task(
None # Will be set by response.completed/failed/incomplete/cancelled
)
terminal_error = None
frame_failure: Mapping[str, object] | None = None # rebind-ok: set once by the first error frame
_event_to_status: Final[Mapping[str, ResponsesAPIStatus]] = {
"response.completed": "completed",
"response.failed": "failed",
@ -234,6 +260,13 @@ async def background_streaming_task(
if isinstance(chunk, bytes):
chunk = chunk.decode("utf-8")
if (
isinstance(chunk, str)
and frame_failure is None
and (error_data := frame_error_data(chunk)) is not None
):
frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data)))
if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None:
if chunk_data == "[DONE]":
break
@ -373,23 +406,25 @@ async def background_streaming_task(
output_items[item_id] = item
state_dirty = True
# Flush state to Redis if interval elapsed
await flush_state_if_needed()
except json.JSONDecodeError as e:
except (json.JSONDecodeError, AttributeError, TypeError) as e:
verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e)
else:
await flush_state_if_needed()
# Final flush to ensure all accumulated state is saved
await flush_state_if_needed(force=True)
# Use the terminal status from the stream, default to "completed"
final_status: Final = terminal_status or "completed"
early_error: Final = _first_chunk_error(response)
stream_failure: Final = frame_failure if terminal_status is None else None
failure: Final = stream_failure if early_error is None else early_error
final_error: Final = terminal_error if failure is None else failure
final_status: Final = (terminal_status or "completed") if failure is None else "failed"
await polling_handler.update_state(
polling_id=polling_id,
status=final_status,
usage=usage_data,
error=terminal_error,
error=final_error,
reasoning=reasoning_data,
tool_choice=tool_choice_data,
tools=tools_data,
@ -411,7 +446,7 @@ async def background_streaming_task(
"Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s",
polling_id,
final_status,
terminal_error,
final_error,
incomplete_details_data,
len(output_items),
)

View file

@ -98,7 +98,7 @@ class ResponsePollingHandler:
polling_id: str,
status: ResponsesAPIStatus | None = None,
usage: dict | None = None,
error: dict | None = None,
error: Mapping[str, object] | None = None,
incomplete_details: dict | None = None,
reasoning: dict | None = None,
tool_choice: object | None = None,

View file

@ -1200,9 +1200,7 @@ def test_responses_pre_call_denial_streams_sse_with_typed_message_item(gateway:
response.headers["content-type"],
response.text,
)
lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: "))
assert lines[-1] == "data: [DONE]", response.text
events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1])
events: Final = _blocked_stream_events(response.text)
kinds: Final = tuple(event["type"] for event in events)
assert tuple(kind for kind in kinds if kind != "response.output_text.delta") == (
"response.created",
@ -1289,9 +1287,13 @@ def _responses_output_denial_config(tmp_path: Path, identity: str, model: str) -
def _blocked_stream_events(text: str) -> tuple[dict[str, object], ...]:
lines: Final = tuple(line for line in text.split("\n") if line.startswith("data: "))
assert lines[-1] == "data: [DONE]", text
return tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1])
assert "data: [DONE]" not in text, text
lines: Final = tuple(line for line in text.split("\n") if line.startswith(("event: ", "data: ")))
events: Final[tuple[dict[str, object], ...]] = tuple(
json.loads(line.removeprefix("data: ")) for line in lines[1::2]
)
assert lines[::2] == tuple(f"event: {event['type']}" for event in events), text
return events
def _dead_api_base() -> str:
@ -2798,7 +2800,7 @@ def test_client_disconnect_after_a_finished_responses_tool_call_scans_the_text_a
with _disconnect_rig(gateway, tmp_path, shape="text_then_tool_call") as rig:
try:
received: Final = _stream_and_close(
rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), "response.output_item.done"
rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), '"type":"response.output_item.done"'
)
assert rig.token() in received, received
scans: Final = eventually(

View file

@ -261,7 +261,7 @@ async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once(
assert len(served) == 10
for item in served:
_assert_answered_with_its_own_marker(item)
assert item.text.rstrip().endswith("data: [DONE]"), item.text
assert item.text.rstrip().endswith("data: [DONE]") == (item.call.endpoint == "chat"), item.text
_assert_native_requests_for(wire.drain(), calls)
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}

View file

@ -846,7 +846,12 @@ def test_responses_api_stream_on_a_mantle_claude_id_bridges_to_the_native_stream
"POST", "/v1/responses", {"model": model, "input": _prompt(marker), "stream": True}
)
assert response.status_code == 200, response.text
events: Final = _sse_objects(response.text)
assert "data: [DONE]" not in response.text, response.text
events: Final = tuple(
_JSON_OBJECT.validate_json(line.removeprefix("data: "))
for line in response.text.splitlines()
if line.startswith("data: ")
)
kinds: Final = [event["type"] for event in events]
assert (kinds[0], kinds[-1]) == ("response.created", "response.completed"), kinds
assert "".join(

View file

@ -482,8 +482,7 @@ def _responses_body(marker: str, *directives: str) -> dict[str, JsonValue]:
def _failed_response_error(response: httpx.Response) -> dict[str, JsonValue]:
assert response.status_code == 200, response.text
lines: Final = _data_lines(response.text)
assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text
assert "[DONE]" not in _data_lines(response.text), response.text
events: Final = _payloads(response.text)
kinds: Final = tuple(string_value(event["type"]) for event in events)
assert kinds[0] == "response.created" and kinds[-1] == "response.failed", response.text

View file

@ -474,9 +474,10 @@ def test_native_responses_stream_forwards_the_failed_response_on_both_legs(gatew
assert response.status_code == 200, response.text
frames: Final = data_frames(response.text)
assert [frame["type"] for frame in frames] == ["response.created", "response.failed"], response.text
assert [name for name, _ in sse_events(response.text)] == ["response.created", "response.failed"], response.text
failed: Final = object_value(frames[-1]["response"])
assert failed["status"] == "failed", failed
assert object_value(failed["error"])["code"] == "rate_limit_exceeded", failed
assert response.text.rstrip().endswith("data: [DONE]"), response.text
assert "data: [DONE]" not in response.text, response.text
assert len(wire.drain()) == 1
assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"

View file

@ -256,8 +256,10 @@ def _responses_frames(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[http
)
_assert_one_attempt(served.wire, marker)
assert served.response.status_code == 200, served.response.text
assert served.response.text.rstrip().endswith("data: [DONE]"), served.response.text
assert "data: [DONE]" not in served.response.text, served.response.text
frames: Final = data_frames(served.response.text)
names: Final = tuple(name for name, _ in sse_events(served.response.text))
assert names == tuple(item["type"] for item in frames), served.response.text
assert frames[-1]["type"] == "response.failed", served.response.text
return served.response, object_value(object_value(frames[-1]["response"])["error"])

View file

@ -4461,7 +4461,14 @@ class TestBuildBlockSseChunks:
def _payloads(self, chunks):
import json
return [json.loads(chunk.decode().removeprefix("data: ").strip()) for chunk in chunks]
payloads: Final = [
json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: ")))
for chunk in chunks
]
assert [chunk.decode().splitlines()[0] for chunk in chunks] == [
f"event: {payload['type']}" for payload in payloads
]
return payloads
def test_standalone_block_emits_complete_synthetic_stream(self):
handler = OpenAIResponsesHandler()

File diff suppressed because it is too large Load diff

View file

@ -9,7 +9,7 @@ import datetime
import json
import re
import unittest
from collections.abc import Callable, Mapping
from collections.abc import AsyncIterator, Callable, Mapping
from typing import Any, Final, Literal, cast
from unittest.mock import AsyncMock, MagicMock, call, patch
@ -26,6 +26,7 @@ import litellm
from litellm.caching.caching import DualCache
from litellm.constants import EMPTY_MAPPING
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.proxy_server import app
@ -36,6 +37,7 @@ from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.router import RouterErrors
from litellm.types.utils import CallTypesLiteral
from tests._master_key import MASTER_KEY
from tests._support.clock_loops import AutojumpClockLoop
@pytest.mark.asyncio
@ -154,8 +156,9 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
assert result.status_code == 200, result.text
assert message in result.text
if path == "/v1/responses":
assert frames[-1] == "data: [DONE]", result.text
assert frames[-2].startswith("event: response.failed\n"), result.text
assert all("data: [DONE]" not in frame for frame in frames), result.text
assert frames[-1].startswith("event: response.failed\n"), result.text
assert [frame.splitlines()[0] for frame in frames] == [f"event: {event['type']}" for event in events]
if partial:
assert [event["type"] for event in events] == [
"response.created",
@ -188,6 +191,173 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
assert "error" in events[-1]
class _FirstFrameHook(CustomLogger):
def __init__(self, first: str | bytes | Exception) -> None:
super().__init__()
self.first: Final = first
async def async_post_call_streaming_iterator_hook(
self, user_api_key_dict: object, response: object, request_data: object
) -> AsyncIterator[str | bytes]:
if isinstance(self.first, Exception):
raise self.first
yield self.first
_DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n'
_NAMED_ERROR_FRAME: Final = (
b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "denied", "code": "529"}}\n\n'
)
async def _post_through_first_frame_hook(
monkeypatch: pytest.MonkeyPatch, path: str, first: str | bytes | Exception
) -> httpx.Response:
import litellm.proxy.proxy_server as ps
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
model: Final = "gpt-6-astra"
upstream_url: Final = "https://streaming.example/v1"
is_chat: Final = path == "/v1/chat/completions"
router: Final = litellm.Router(
model_list=[
{
"model_name": model,
"litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"},
}
],
num_retries=0,
)
monkeypatch.setattr(ps, "llm_router", router)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first)])
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override)
with respx.mock as transport:
transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond(
200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"}
)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client:
return await client.post(
path,
json={
"model": model,
"stream": True,
**({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}),
},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"path,first_frame,expected_status",
[
("/v1/responses", _DATA_ERROR_FRAME, 403),
("/v1/chat/completions", _DATA_ERROR_FRAME, 403),
("/v1/chat/completions", _NAMED_ERROR_FRAME, 200),
("/cursor/chat/completions", _NAMED_ERROR_FRAME, 200),
("/v1/responses", _NAMED_ERROR_FRAME, 529),
("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200),
("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200),
("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200),
],
)
async def test_only_a_lone_first_error_frame_answers_with_its_status(
monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int
) -> None:
result: Final = await _post_through_first_frame_hook(monkeypatch, path, first_frame)
assert result.status_code == expected_status, result.text
if expected_status == 200:
assert result.headers["content-type"].startswith("text/event-stream"), result.text
else:
assert result.json()["error"]["message"] == "denied"
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["/cursor/chat/completions", "/v1/chat/completions"])
async def test_a_hook_failure_before_the_first_chunk_answers_in_chat_completions_shape(
monkeypatch: pytest.MonkeyPatch, path: str
) -> None:
message: Final = "rate limited before the first chunk"
result: Final = await _post_through_first_frame_hook(
monkeypatch, path, HTTPException(status_code=429, detail=message)
)
frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame)
assert result.status_code == 429, result.text
assert frames[-1] == "data: [DONE]", result.text
assert message in json.loads(frames[0].removeprefix("data: "))["error"]["message"]
assert len(frames) == 2, result.text
@pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"])
def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None:
async def request_and_check() -> None:
import litellm.proxy.proxy_server as ps
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
model: Final = "gpt-6-astra"
message: Final = "Upstream is rate limiting this key"
upstream_url: Final = "https://streaming.example/v1"
is_chat: Final = path == "/v1/chat/completions"
async def slow_rate_limit(_: httpx.Request) -> httpx.Response:
await asyncio.sleep(0.3)
return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}})
router: Final = litellm.Router(
model_list=[
{
"model_name": model,
"litellm_params": {
"model": "openai/" + model,
"api_base": upstream_url,
"api_key": "fixture-key",
"max_retries": 0,
},
}
],
num_retries=0,
)
monkeypatch.setattr(ps, "llm_router", router)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05)
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override)
with respx.mock as transport:
transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock(
side_effect=slow_rate_limit
)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client:
result: Final = await client.post(
path,
json={
"model": model,
"stream": True,
**({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}),
},
)
frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame)
assert result.status_code == 200, result.text
assert frames[0] == ": ping", result.text
if is_chat:
assert frames[-1] == "data: [DONE]", result.text
assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"]
else:
assert "data: [DONE]" not in result.text
assert frames[-1].startswith("event: error\ndata: "), result.text
error: Final = json.loads(frames[-1].split("data: ", 1)[1])["error"]
assert error["code"] == "429"
assert message in error["message"]
loop: Final = AutojumpClockLoop()
try:
loop.run_until_complete(request_and_check())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
_COMPACTION_ITEM: Final = {
"type": "compaction",
"encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}),
@ -1478,6 +1648,25 @@ async def test_a_streamed_read_cut_before_its_terminal_event_ends_in_a_responses
assert "before any terminal event" in events[-1]["response"]["error"]["message"]
def _resumed_wire() -> str:
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in _resumed_events("resp_read"))
@pytest.mark.asyncio
async def test_a_streamed_read_whose_first_event_is_a_named_error_answers_with_its_status(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(_NAMED_ERROR_FRAME)])
result, _ = await _read_through_proxy(
monkeypatch,
{"stream": "true"},
httpx.Response(200, content=_resumed_wire(), headers={"Content-Type": "text/event-stream"}),
)
assert result.status_code == 529, result.text
assert result.json()["error"]["message"] == "denied"
@pytest.mark.asyncio
async def test_a_plain_read_through_the_proxy_still_answers_json(monkeypatch: pytest.MonkeyPatch):
result, upstream_requests = await _read_through_proxy(
@ -5825,9 +6014,11 @@ class TestGuardrailBlockedResponsesShape:
assert response.status_code == 200, response.text
assert response.headers["content-type"].startswith("text/event-stream")
frames = _sse_data_frames(response.text)
assert frames[-1] == "[DONE]"
events = [json.loads(frame) for frame in frames[:-1]]
assert "[DONE]" not in frames
events = [json.loads(frame) for frame in frames]
types = [event["type"] for event in events]
assert types[-1] == "response.completed"
assert [line[7:] for line in response.text.splitlines() if line.startswith("event: ")] == types
assert "response.created" in types
completed = [event for event in events if event["type"] == "response.completed"]
assert len(completed) == 1
@ -5854,7 +6045,8 @@ class TestGuardrailBlockedResponsesShape:
assert response.status_code == 200, response.text
frames = _sse_data_frames(response.text)
completed = [json.loads(frame) for frame in frames[:-1] if json.loads(frame)["type"] == "response.completed"]
assert "[DONE]" not in frames
completed = [json.loads(frame) for frame in frames if json.loads(frame)["type"] == "response.completed"]
usage = completed[0]["response"]["usage"]
assert usage["input_tokens"] == 14
assert usage["output_tokens"] == 20

View file

@ -16,12 +16,13 @@ following the OpenAI Response API format.
import asyncio
import json
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Dict, Final, Optional
from unittest.mock import AsyncMock, Mock, patch
import pytest
from fastapi import Request
from fastapi import HTTPException, Request
from pydantic import ValidationError
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
@ -1524,6 +1525,272 @@ class TestBackgroundStreamingTerminalEvents:
assert final_call.kwargs["status"] == "failed"
assert final_call.kwargs["error"] == error_payload
@pytest.mark.asyncio
async def test_a_late_failure_after_keepalive_pings_sets_failed_status_and_error(self):
from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
late_frames: Final = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited"))
async def _body_iterator():
yield b": ping\n\n"
for frame in late_frames:
yield frame.encode()
mock_response: Final = Mock()
mock_response.body_iterator = _body_iterator()
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_late_failure", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call: Final = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "failed"
assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("data_type", "nested_error", "expected_error"),
[
(
{"type": "error"},
{"type": "guardrail_error", "code": "400", "message": "output blocked by policy"},
{"code": "invalid_request_error", "message": "output blocked by policy"},
),
({"type": "error"}, "output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}),
(
{},
{"code": "400", "message": "output blocked by policy"},
{"code": "invalid_request_error", "message": "output blocked by policy"},
),
(
{"type": "response.output_text.delta"},
{"code": "400", "message": "output blocked by policy"},
{"code": "invalid_request_error", "message": "output blocked by policy"},
),
],
ids=["typed_object", "typed_string", "untyped_object", "delta_typed_object"],
)
async def test_a_nested_only_error_event_keeps_its_reason(
self,
data_type: Mapping[str, str],
nested_error: Mapping[str, str] | str,
expected_error: Mapping[str, str],
):
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
error_event: Final = {**data_type, "sequence_number": 9, "error": nested_error}
async def _body_iterator():
yield b'data: {"type": "response.in_progress"}\n\n'
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
mock_response: Final = Mock()
mock_response.body_iterator = _body_iterator()
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_nested_error", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call: Final = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "failed"
assert final_call.kwargs["error"] == expected_error
@pytest.mark.asyncio
@pytest.mark.parametrize(
("chunks", "expected_status", "expected_error", "expected_usage"),
[
(
(
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n'
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
),
"failed",
{"code": "invalid_request_error", "message": "output blocked by policy"},
None,
),
(
(
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
(
(
'data:{"type": "response.output_text.delta", "delta": "hi"}\n\n'
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
(
(
'event: error\ndata: {"error": {"code": "' + "9" * 4400 + '", "message": "huge code"}}\n\n',
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
(
(
'data: {"type": "response.output_item.done", "output_index": 0, "item": "hook-text"}\n\n',
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
(
(
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n'
'event: response.completed\ndata: {"type": "response.completed", "response": {"status": "completed"}}\n\n',
),
"completed",
None,
None,
),
(
(
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n',
'data: [DONE]\n\nevent: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
),
"completed",
None,
None,
),
],
ids=[
"error_frame_after_a_delta",
"error_frame_then_completed",
"unspaced_delta_then_completed",
"error_code_past_the_int_digit_limit_then_completed",
"output_item_done_with_a_string_item_then_completed",
"error_frame_then_completed_in_one_chunk",
"error_frame_after_done_in_one_chunk",
],
)
async def test_a_poll_stores_what_its_stream_ended_with(
self,
chunks: tuple[str, ...],
expected_status: str,
expected_error: Mapping[str, str] | None,
expected_usage: Mapping[str, int] | None,
):
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
async def _body_iterator():
yield b'data: {"type": "response.in_progress"}\n\n'
for chunk in chunks:
yield chunk.encode()
mock_response: Final = Mock()
mock_response.body_iterator = _body_iterator()
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_stream_end", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call: Final = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == expected_status
assert final_call.kwargs["error"] == expected_error
assert final_call.kwargs["usage"] == expected_usage
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status_code", "body", "expected_error"),
[
(
404,
json.dumps(
{"error": {"message": "model not found", "type": "invalid_request_error", "param": None, "code": "404"}}
),
{"code": "not_found_error", "message": "model not found"},
),
(429, "rate limited", {"code": "rate_limit_exceeded", "message": "Too Many Requests"}),
(
403,
json.dumps({"error": {"type": "guardrail_scan_error", "code": 403}}),
{"code": "permission_error", "message": "Forbidden"},
),
],
)
async def test_a_first_chunk_error_polls_failed_like_a_late_one(
self, status_code: int, body: str, expected_error: Mapping[str, str]
):
from fastapi import Response
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
first_chunk_error: Final = Response(status_code=status_code, content=body)
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_first_chunk_error", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=first_chunk_error
)
await background_streaming_task(**kwargs)
final_call: Final = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "failed"
assert final_call.kwargs["error"] == expected_error
@pytest.mark.asyncio
async def test_a_non_streamed_success_response_still_polls_completed(self):
from fastapi.responses import JSONResponse
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_non_streamed_success", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=JSONResponse(status_code=200, content={"id": "resp_123", "status": "completed"})
)
await background_streaming_task(**kwargs)
final_call: Final = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == "completed"
assert final_call.kwargs["error"] is None
@pytest.mark.asyncio
async def test_response_incomplete_sets_incomplete_status_and_details(self):
"""Test that a response.incomplete stream event results in incomplete status"""

View file

@ -4,7 +4,7 @@ import copy
import datetime
import gzip
import json
from collections.abc import AsyncIterable, Mapping
from collections.abc import AsyncIterable, Awaitable, Mapping
from types import MappingProxyType, SimpleNamespace
from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence
from urllib.parse import unquote_plus
@ -40,6 +40,7 @@ from litellm.constants import (
STREAM_SSE_KEEPALIVE_PING_BYTES,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.integrations.opentelemetry import UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_request_processing import (
@ -52,11 +53,15 @@ from litellm.proxy.common_request_processing import (
_ClientDisconnectedBeforeFirstChunk,
attach_guardrail_information,
_extract_error_from_sse_chunk,
FirstChunkSseFraming,
_get_cost_breakdown_from_logging_obj,
CostBreakdownHeaderValues,
_has_attribute_error_in_chain,
OPENAI_FIRST_CHUNK_FRAMING,
RESPONSES_FIRST_CHUNK_FRAMING,
include_guardrail_response_requested,
is_azure_model_router_request,
RESPONSES_LATE_SSE_FRAMING,
open_sse_before_first_byte,
resolve_litellm_call_id,
ttft_keepalive_interval,
@ -89,6 +94,7 @@ from litellm.router import Router
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.utils import ModelResponseListIterator
from litellm.litellm_core_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
from tests._support.clock_loops import AutojumpClockLoop
def test_attach_guardrail_information_copies_recorded_entries_onto_model_response():
@ -2257,6 +2263,14 @@ class TestProxyBaseLLMRequestProcessing:
)
def _fail_stream_setup() -> str:
raise RuntimeError("stream setup failed")
async def _stream_failing_before_its_first_chunk() -> AsyncGenerator[str, None]:
yield _fail_stream_setup()
@pytest.mark.asyncio
class TestCommonRequestProcessingHelpers:
async def consume_stream(self, streaming_response: StreamingResponse) -> list:
@ -2335,6 +2349,74 @@ class TestCommonRequestProcessingHelpers:
assert body["error"]["code"] == 403
assert body["error"]["message"] == "forbidden"
@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self, newline: str):
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None}
event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error})
async def responses_stream():
yield f"event: error{newline}data: {event}{newline}{newline}"
response: Final = await create_response(
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert json.loads(response.body) == {"error": error}
async def test_a_first_event_error_with_a_unicode_line_separator_in_its_message_answers_as_json(self):
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked\u2028by policy", "param": None}
async def responses_stream():
event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False)
yield f"event: error\ndata: {event}\n\n"
response: Final = await create_response(
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
)
assert isinstance(response, JSONResponse)
assert response.status_code == 400
assert json.loads(response.body) == {"error": error}
@pytest.mark.parametrize(
"framing,frame_starts",
[
(OPENAI_FIRST_CHUNK_FRAMING, ('data: {"error": ', "data: [DONE]")),
(RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)),
],
)
async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(
self, framing: FirstChunkSseFraming, frame_starts: tuple[str, ...]
) -> None:
response: Final = await create_response(
_stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing
)
frames: Final = await self.consume_stream(response)
assert response.status_code == 500
assert len(frames) == len(frame_starts)
assert all(frame.startswith(start) for frame, start in zip(frames, frame_starts, strict=True))
async def test_a_responses_stream_whose_first_frame_names_a_success_code_streams_on(self):
frames: Final = (
'data: {"error": {"message": "note", "type": "guardrail_scan_error", "code": 200}}\n\n',
'event: response.completed\ndata: {"type": "response.completed", "sequence_number": 1}\n\n',
)
async def responses_stream():
for frame in frames:
yield frame
response: Final = await create_response(
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
)
assert isinstance(response, StreamingResponse)
assert response.status_code == status.HTTP_200_OK
assert tuple(await self.consume_stream(response)) == frames
async def test_create_streaming_response_first_chunk_not_error(self):
async def mock_generator():
yield 'data: {"content": "first part"}\n\n'
@ -10677,6 +10759,104 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
assert collected[-1] == b"data: [DONE]\n\n"
async def _slow_rate_limit_failure() -> Response:
await asyncio.sleep(0.2)
raise HTTPException(status_code=429, detail="rate limited")
async def _slow_guardrail_block_body() -> Response:
await asyncio.sleep(0.2)
return JSONResponse(
status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}
)
_POLICY_BLOCK_DETAIL: Final = MappingProxyType(
{"error": {"message": "blocked by policy", "type": "guardrail_error"}, "guardrail": "content-filter"}
)
async def _slow_codeless_rate_limit_body() -> Response:
await asyncio.sleep(0.2)
return JSONResponse(status_code=429, content={"error": {"message": "slow down", "code": None}})
async def _slow_dict_detail_failure() -> Response:
await asyncio.sleep(0.2)
raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL))
@pytest.mark.parametrize(
"produce,expected_code,expected_error",
[
(_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}),
(_slow_codeless_rate_limit_body, "rate_limit_exceeded", {"message": "slow down", "code": "429"}),
(
_slow_guardrail_block_body,
"invalid_request_error",
{"message": "blocked", "type": "guardrail_error", "code": "400"},
),
(
_slow_dict_detail_failure,
"invalid_request_error",
{"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)},
),
],
)
def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done(
produce: Callable[[], Awaitable[Response]], expected_code: str, expected_error: Mapping[str, object]
) -> None:
async def collect_and_check() -> None:
response: Final = await open_sse_before_first_byte(
produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING
)
collected: Final = await _drain(response)
assert collected[0] == TTFT_PING
assert b"data: [DONE]\n\n" not in collected
event_line, data_line = collected[-1].decode().strip().split("\n")
event: Final = json.loads(data_line.removeprefix("data: "))
assert event_line == "event: error"
assert event["type"] == "error"
assert (event["code"], event["message"]) == (expected_code, expected_error["message"])
assert {key: event["error"].get(key) for key in expected_error} == expected_error
loop: Final = AutojumpClockLoop()
try:
loop.run_until_complete(collect_and_check())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
async def _slow_passthrough_block() -> Response:
await asyncio.sleep(0.2)
raise ModifyResponseException(message="blocked by policy", model="gpt-4o", request_data={})
def test_ttft_keepalive_ends_a_late_responses_passthrough_block_with_the_block_response() -> None:
async def collect_and_check() -> None:
response: Final = await open_sse_before_first_byte(
_slow_passthrough_block(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING
)
collected: Final = await _drain(response)
frames: Final = tuple(chunk.decode() for chunk in collected if chunk != TTFT_PING)
names: Final = tuple(frame.split("\n", 1)[0] for frame in frames)
completed: Final = json.loads(frames[-1].split("\n", 1)[1].removeprefix("data: "))
assert collected[0] == TTFT_PING
assert (names[0], names[-1]) == ("event: response.created", "event: response.completed")
assert "event: error" not in names
assert completed["response"]["output"][0]["content"][0]["text"] == "blocked by policy"
loop: Final = AutojumpClockLoop()
try:
loop.run_until_complete(collect_and_check())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
@pytest.mark.asyncio
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
"""Starlette never calls the produced response, so its own cleanup never runs

View file

@ -763,7 +763,10 @@ def test_restamp_streaming_chunk_rewrites_nested_model_in_guardrail_block_frames
)[0]
for frame in frames
]
events = [json.loads(frame.decode().removeprefix("data: ")) for frame in client_frames]
events = [
json.loads(next(line[6:] for line in frame.decode().splitlines() if line.startswith("data: ")))
for frame in client_frames
]
assert all(frame.endswith(b"\n\n") for frame in client_frames)
assert events[-1]["type"] == "response.completed"