mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge eab9d81cf2 into c43a1addf2
This commit is contained in:
commit
cf149e5df0
19 changed files with 2466 additions and 81 deletions
|
|
@ -269,6 +269,7 @@ _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call"
|
|||
_OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE}
|
||||
_SCANNED_TOOL_SEARCH_NAME: Final = "tool_search"
|
||||
_TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str)
|
||||
_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"function_call": "arguments", "custom_tool_call": "input"}
|
||||
)
|
||||
|
|
@ -1950,7 +1951,6 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
new output item under the same response id, and close with a
|
||||
``response.completed`` carrying only the replacement item.
|
||||
|
||||
The proxy's data generator appends ``data: [DONE]`` itself.
|
||||
"""
|
||||
events: Final = (
|
||||
self._block_continuation_events(exc, responses_so_far or ())
|
||||
|
|
@ -1958,6 +1958,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
else self._standalone_block_events(exc)
|
||||
)
|
||||
return tuple(
|
||||
f"event: {_STREAM_EVENT_TYPE_ADAPTER.validate_python(stream_item_field(event, 'type'))}\n"
|
||||
f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode()
|
||||
for event in events
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache
|
||||
from http import HTTPStatus
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
|
|
@ -51,7 +52,7 @@ from litellm.constants import (
|
|||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
UNSAFE_PROXY_RESPONSE_HEADERS,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
|
|
@ -92,6 +93,7 @@ from litellm.litellm_core_utils.streaming_handler import (
|
|||
backfill_missing_cache_usage_fields,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler
|
||||
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_key_call_resolved_model,
|
||||
|
|
@ -119,6 +121,11 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_type,
|
||||
relayed_provider_specific_fields,
|
||||
)
|
||||
from litellm.proxy.common_utils.responses_stream_errors import (
|
||||
SSE_LINE_BREAK,
|
||||
ResponsesStreamErrorState,
|
||||
sse_error_object,
|
||||
)
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING_BYTES,
|
||||
SSE_STREAM_START_TAIL,
|
||||
|
|
@ -1005,11 +1012,32 @@ async def _enforce_guardrail_added_tag_budgets(
|
|||
) from e
|
||||
|
||||
|
||||
async def _parse_event_data_for_error(event_line: str | bytes) -> int | None:
|
||||
SseDataPayload: TypeAlias = Callable[[str | bytes], str | None]
|
||||
|
||||
|
||||
def _sse_text(event_line: str | bytes) -> str:
|
||||
return event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
|
||||
|
||||
|
||||
def _sse_data_payload(event_line: str | bytes) -> str | None:
|
||||
text: Final = _sse_text(event_line)
|
||||
return text.removeprefix("data: ").strip() if text.startswith("data: ") else None
|
||||
|
||||
|
||||
def _responses_event_data_payload(event_line: str | bytes) -> str | None:
|
||||
match SSE_LINE_BREAK.split(_sse_text(event_line).strip()):
|
||||
case ["event: error", data_line] if data_line.startswith("data: "):
|
||||
return data_line.removeprefix("data: ").strip()
|
||||
case _:
|
||||
return _sse_data_payload(event_line)
|
||||
|
||||
|
||||
async def _parse_event_data_for_error(
|
||||
event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload
|
||||
) -> int | None:
|
||||
"""Parses an event line and returns an error code if present, else None."""
|
||||
event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
|
||||
if event_line.startswith("data: "):
|
||||
json_str: Final = event_line[len("data: ") :].strip()
|
||||
json_str: Final = data_payload(event_line)
|
||||
if json_str is not None:
|
||||
if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message
|
||||
return None
|
||||
try:
|
||||
|
|
@ -1040,18 +1068,17 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict:
|
||||
def _extract_error_from_sse_chunk(event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload) -> dict:
|
||||
"""
|
||||
Extract error dictionary from SSE format chunk.
|
||||
|
||||
Args:
|
||||
event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n"
|
||||
data_payload: Reads the JSON payload out of the event line
|
||||
|
||||
Returns:
|
||||
Error dictionary in OpenAI API format
|
||||
"""
|
||||
event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
|
||||
|
||||
# Default error format
|
||||
default_error: Final = {
|
||||
"message": "Unknown error",
|
||||
|
|
@ -1060,8 +1087,8 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict:
|
|||
"code": "500",
|
||||
}
|
||||
|
||||
if event_line.startswith("data: "):
|
||||
json_str: Final = event_line[len("data: ") :].strip()
|
||||
json_str: Final = data_payload(event_line)
|
||||
if json_str is not None:
|
||||
if not json_str or json_str == "[DONE]":
|
||||
return default_error
|
||||
|
||||
|
|
@ -1269,6 +1296,75 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
|
|||
return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _responses_error_frames(error_obj: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return (ResponsesStreamErrorState().format_error(error_obj),)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LateSseFraming:
|
||||
"""How a response the keepalive already opened ends once its upstream fails late."""
|
||||
|
||||
failure: Callable[[Exception], tuple[str, ...]]
|
||||
body: Callable[[Response], tuple[bytes, ...]]
|
||||
|
||||
|
||||
def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]:
|
||||
_, error_obj = sse_error_payload(exc)
|
||||
return _sse_error_frames(error_obj)
|
||||
|
||||
|
||||
def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]:
|
||||
return b"data: " + (bytes(produced.body) or b"{}") + b"\n\n", b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]:
|
||||
if isinstance(exc, ModifyResponseException):
|
||||
return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc))
|
||||
return _responses_error_frames(sse_error_payload(exc)[1])
|
||||
|
||||
|
||||
def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]:
|
||||
body: Final = bytes(produced.body)
|
||||
error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode()}
|
||||
with_status: Final = {**error_obj, "code": error_obj.get("code") or str(produced.status_code)}
|
||||
return tuple(frame.encode() for frame in _responses_error_frames(with_status))
|
||||
|
||||
|
||||
OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames)
|
||||
RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming(
|
||||
failure=_responses_late_failure_frames, body=_responses_late_body_frames
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FirstChunkSseFraming:
|
||||
"""How a stream's first chunk is read for an error, and how the stream ends when reading it fails."""
|
||||
|
||||
data_payload: SseDataPayload
|
||||
error_frames: Callable[[Mapping[str, object]], tuple[str, ...]]
|
||||
lowest_error_status: int
|
||||
|
||||
|
||||
OPENAI_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(
|
||||
data_payload=_sse_data_payload, error_frames=_sse_error_frames, lowest_error_status=HTTPStatus.CONTINUE
|
||||
)
|
||||
RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(
|
||||
data_payload=_responses_event_data_payload,
|
||||
error_frames=_responses_error_frames,
|
||||
lowest_error_status=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SseFraming:
|
||||
first_chunk: FirstChunkSseFraming
|
||||
late: LateSseFraming
|
||||
|
||||
|
||||
OPENAI_SSE_FRAMING: Final = SseFraming(first_chunk=OPENAI_FIRST_CHUNK_FRAMING, late=OPENAI_LATE_SSE_FRAMING)
|
||||
RESPONSES_SSE_FRAMING: Final = SseFraming(first_chunk=RESPONSES_FIRST_CHUNK_FRAMING, late=RESPONSES_LATE_SSE_FRAMING)
|
||||
|
||||
|
||||
def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
"""`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384)."""
|
||||
return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS})
|
||||
|
|
@ -1297,6 +1393,7 @@ async def create_response(
|
|||
request: Request | None = None,
|
||||
refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None,
|
||||
general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS,
|
||||
framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING,
|
||||
) -> StreamingResponse | JSONResponse:
|
||||
"""
|
||||
Create streaming response, checking if the first chunk is an error.
|
||||
|
|
@ -1332,8 +1429,10 @@ async def create_response(
|
|||
|
||||
if first_chunk_value is not None:
|
||||
try:
|
||||
error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value)
|
||||
if error_code_from_chunk is not None:
|
||||
error_code_from_chunk: Final = await _parse_event_data_for_error(
|
||||
first_chunk_value, framing.data_payload
|
||||
)
|
||||
if error_code_from_chunk is not None and error_code_from_chunk >= framing.lowest_error_status:
|
||||
# First chunk is an error, stream hasn't really started yet
|
||||
# Should return standard JSON error response instead of SSE format
|
||||
final_status_code = error_code_from_chunk
|
||||
|
|
@ -1344,7 +1443,9 @@ async def create_response(
|
|||
|
||||
# Parse error content
|
||||
error_dict: Final = with_call_id(
|
||||
JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)),
|
||||
JSON_OBJECT.validate_python(
|
||||
_extract_error_from_sse_chunk(first_chunk_value, framing.data_payload)
|
||||
),
|
||||
error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)),
|
||||
)
|
||||
|
||||
|
|
@ -1386,7 +1487,7 @@ async def create_response(
|
|||
error_status, error_obj = sse_error_payload(e)
|
||||
|
||||
async def error_gen_message() -> AsyncGenerator[str, None]:
|
||||
for frame in _sse_error_frames(error_obj):
|
||||
for frame in framing.error_frames(error_obj):
|
||||
yield frame
|
||||
|
||||
return StreamingResponse(
|
||||
|
|
@ -1469,13 +1570,13 @@ async def _aclose_late_response(produced: Response) -> None:
|
|||
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
|
||||
|
||||
|
||||
async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]:
|
||||
async def _relay_late_response(produced: Response, framing: LateSseFraming) -> AsyncGenerator[bytes, None]:
|
||||
"""Replay a Response that was built after a keepalive had already opened the wire."""
|
||||
if not isinstance(produced, StreamingResponse):
|
||||
# The status line is already on the wire, so a non-streaming body, an error
|
||||
# body included, can only reach the client as an SSE frame.
|
||||
yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n"
|
||||
yield b"data: [DONE]\n\n"
|
||||
for frame in framing.body(produced):
|
||||
yield frame
|
||||
return
|
||||
|
||||
try:
|
||||
|
|
@ -1517,6 +1618,7 @@ async def open_sse_before_first_byte(
|
|||
ping_interval_seconds: float | str | None,
|
||||
media_type: str = "text/event-stream",
|
||||
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
|
||||
late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING,
|
||||
) -> _LateResponseT | StreamingResponse:
|
||||
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
|
||||
|
||||
|
|
@ -1562,11 +1664,10 @@ async def open_sse_before_first_byte(
|
|||
# would never fire and the failure would go unaudited. The hook
|
||||
# also gets to sanitize what reaches the client, by returning or
|
||||
# raising a replacement, so its answer decides the frame.
|
||||
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
|
||||
for frame in _sse_error_frames(error_obj):
|
||||
for frame in late_framing.failure(await _sanitized_late_failure(exc, on_late_failure)):
|
||||
yield frame.encode()
|
||||
return
|
||||
async for chunk in _relay_late_response(produced):
|
||||
async for chunk in _relay_late_response(produced, late_framing):
|
||||
yield chunk
|
||||
finally:
|
||||
if not produce_task.done():
|
||||
|
|
@ -2964,6 +3065,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
is_streaming_request: bool | None = False,
|
||||
contents: list[object] | None = None,
|
||||
skip_pre_call_logic: bool = False,
|
||||
sse_framing: SseFraming = OPENAI_SSE_FRAMING,
|
||||
) -> object:
|
||||
"""Run the request, sending SSE keepalives while the upstream is still silent.
|
||||
|
||||
|
|
@ -3004,9 +3106,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
is_streaming_request=is_streaming_request,
|
||||
contents=contents,
|
||||
skip_pre_call_logic=skip_pre_call_logic,
|
||||
first_chunk_framing=sse_framing.first_chunk,
|
||||
),
|
||||
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
|
||||
on_late_failure=_audit_late_failure,
|
||||
late_framing=sse_framing.late,
|
||||
)
|
||||
|
||||
async def _process_llm_request(
|
||||
|
|
@ -3030,6 +3134,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
is_streaming_request: bool | None = False,
|
||||
contents: list[object] | None = None, # Add contents parameter
|
||||
skip_pre_call_logic: bool = False,
|
||||
first_chunk_framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING,
|
||||
) -> Any:
|
||||
"""
|
||||
Common request processing logic for both chat completions and responses API endpoints
|
||||
|
|
@ -3313,6 +3418,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request=request,
|
||||
refresh_headers=refresh_stream_headers,
|
||||
general_settings=general_settings,
|
||||
framing=first_chunk_framing,
|
||||
)
|
||||
|
||||
### CALL HOOKS ### - modify outgoing data
|
||||
|
|
@ -3555,8 +3661,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
Streaming ``/v1/responses`` short-circuits out of
|
||||
``base_process_llm_request`` before the non-streaming ownership
|
||||
tail runs. The OpenAI SDK closes the connection at ``data: [DONE]``
|
||||
and starlette cancels the body task on disconnect, so a write that
|
||||
tail runs. The OpenAI SDK can close the connection right after the terminal
|
||||
event and starlette cancels the body task on disconnect, so a write that
|
||||
waits for the generator to finish never lands. The iterator sets
|
||||
``completed_response`` before it hands over its terminal chunk, so
|
||||
the ``LiteLLM_ManagedObjectTable`` row is written the moment it
|
||||
|
|
|
|||
|
|
@ -1,10 +1,18 @@
|
|||
import gc
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections import Counter, OrderedDict, defaultdict
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
from itertools import chain, groupby, takewhile, zip_longest
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
|
||||
from typing_extensions import TypeIs # noqa: TID251 # the guards narrow hook values to object items, isinstance cannot
|
||||
|
||||
from litellm._logging import redact_internal_details_from_client_message
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -29,6 +37,13 @@ class _StreamEvent(LiteLLMBaseModel):
|
|||
response: _ResponseIdentity | None = None
|
||||
|
||||
|
||||
class _FramedEvent(_StreamEvent):
|
||||
error: object = None
|
||||
|
||||
|
||||
_LAX_INT: Final = TypeAdapter(int)
|
||||
|
||||
|
||||
class _FailureDetails(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
@ -37,6 +52,16 @@ class _FailureDetails(LiteLLMBaseModel):
|
|||
type: str | None = None
|
||||
status_code: int | None = None
|
||||
|
||||
@field_validator("status_code", mode="before")
|
||||
@classmethod
|
||||
def normalize_status_code(cls, value: object) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return _LAX_INT.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
@field_validator("message", mode="before")
|
||||
@classmethod
|
||||
def normalize_message(cls, value: object) -> str | None:
|
||||
|
|
@ -57,6 +82,265 @@ class _FailureDetails(LiteLLMBaseModel):
|
|||
return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields})
|
||||
|
||||
|
||||
class _ErrorPayload(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
error: Mapping[str, object] | str
|
||||
|
||||
def error_object(self) -> Mapping[str, object]:
|
||||
return {"message": self.error} if isinstance(self.error, str) else self.error
|
||||
|
||||
|
||||
_DROPPED: Final = object()
|
||||
|
||||
|
||||
class _BaseForm(Enum):
|
||||
PRINTED = "printed"
|
||||
DUMPED = "dumped"
|
||||
|
||||
|
||||
def _repr_owner(kind: type[object]) -> type[object]:
|
||||
return next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object)
|
||||
|
||||
|
||||
def _repr_prints_every_item(kind: type[object]) -> bool:
|
||||
return kind in (OrderedDict, Counter) or _repr_owner(kind) in (dict, list, tuple, defaultdict)
|
||||
|
||||
|
||||
def _format_prints_every_item(kind: type[object]) -> bool:
|
||||
return kind.__str__ is object.__str__ and kind.__format__ is object.__format__ and _repr_prints_every_item(kind)
|
||||
|
||||
|
||||
def _str_prints_every_item(kind: type[object]) -> bool:
|
||||
return kind.__str__ is object.__str__ and _repr_owner(kind) in (
|
||||
dict,
|
||||
list,
|
||||
tuple,
|
||||
OrderedDict,
|
||||
defaultdict,
|
||||
Counter,
|
||||
)
|
||||
|
||||
|
||||
def _is_dict(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: items as object
|
||||
return isinstance(value, (list, tuple))
|
||||
|
||||
|
||||
def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed
|
||||
return isinstance(value, MappingProxyType)
|
||||
|
||||
|
||||
def _is_defaultdict(value: object) -> TypeIs[defaultdict[object, object]]: # guard-ok: dict items as object
|
||||
return isinstance(value, defaultdict)
|
||||
|
||||
|
||||
def _proxied(proxy: Mapping[object, object]) -> object:
|
||||
return gc.get_referents(proxy)[0]
|
||||
|
||||
|
||||
def _repr_prints_its_fields(model: BaseModel) -> bool:
|
||||
kind: Final = type(model)
|
||||
return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__
|
||||
|
||||
|
||||
def _plain_scalar(value: str | float) -> str | float:
|
||||
if isinstance(value, str):
|
||||
return str.__str__(value)
|
||||
if isinstance(value, int):
|
||||
return int.__int__(value)
|
||||
return float.__float__(value)
|
||||
|
||||
|
||||
def _repr_shows_its_value(value: str | float, printed: str) -> bool:
|
||||
plain: Final = _plain_scalar(value)
|
||||
if isinstance(value, Enum):
|
||||
return printed == f"<{type(value).__name__}.{value.name}: {plain!r}>"
|
||||
return printed == repr(plain)
|
||||
|
||||
|
||||
def _printed_text(value: object) -> object:
|
||||
printed: Final = repr(value)
|
||||
if isinstance(value, (str, int, float)) and _repr_shows_its_value(value, printed):
|
||||
return _plain_scalar(value)
|
||||
return printed
|
||||
|
||||
|
||||
def _dumped_text(value: object) -> object:
|
||||
return _plain_scalar(value) if isinstance(value, (str, int, float)) else str(value)
|
||||
|
||||
|
||||
def _text(value: object, form: _BaseForm) -> object:
|
||||
try:
|
||||
return _printed_text(value) if form is _BaseForm.PRINTED else _dumped_text(value)
|
||||
except Exception: # noqa: BLE001 # hook values run their own __repr__ and __str__, so a failure drops the value
|
||||
return _DROPPED
|
||||
|
||||
|
||||
def _json_key(key: object, form: _BaseForm) -> object:
|
||||
return key if key is None or type(key) in (str, int, float, bool) else _text(key, form)
|
||||
|
||||
|
||||
def _stored_sequence(value: Sequence[object]) -> Iterator[object]:
|
||||
if isinstance(value, list):
|
||||
return list[object].__iter__(value)
|
||||
if isinstance(value, tuple):
|
||||
return tuple[object, ...].__iter__(value)
|
||||
return iter(value)
|
||||
|
||||
|
||||
def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, object]]:
|
||||
return dict[object, object].items(value) if isinstance(value, dict) else value.items()
|
||||
|
||||
|
||||
def _live_items(value: Mapping[object, object]) -> Iterator[tuple[object, object]]:
|
||||
shown: Final = dict[int, object]()
|
||||
while True:
|
||||
with suppress(RuntimeError):
|
||||
for key, item in _stored_items(value):
|
||||
if id(key) not in shown:
|
||||
shown[id(key)] = key
|
||||
yield key, item
|
||||
return
|
||||
|
||||
|
||||
def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool:
|
||||
return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored))
|
||||
|
||||
|
||||
def _shown_sequence(value: object, form: _BaseForm) -> Iterable[object] | None:
|
||||
if not _is_sequence(value):
|
||||
return None
|
||||
if form is _BaseForm.PRINTED:
|
||||
return _stored_sequence(value) if _repr_prints_every_item(type(value)) else None
|
||||
dumped: Final = tuple(value)
|
||||
return dumped if _dumps_shown(type(value), dumped, _stored_sequence(value)) else None
|
||||
|
||||
|
||||
def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None:
|
||||
if not _is_dict(value):
|
||||
return None
|
||||
if form is _BaseForm.DUMPED:
|
||||
dumped: Final = tuple(value.items())
|
||||
stored: Final = chain.from_iterable(_stored_items(value))
|
||||
return dumped if _dumps_shown(type(value), chain.from_iterable(dumped), stored) else None
|
||||
if not _repr_prints_every_item(type(value)):
|
||||
return None
|
||||
if isinstance(value, OrderedDict):
|
||||
return tuple(OrderedDict[object, object].items(value))
|
||||
if isinstance(value, Counter):
|
||||
return Counter[object].most_common(value)
|
||||
return _live_items(value)
|
||||
|
||||
|
||||
def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None:
|
||||
mapping: Final = _proxied(proxy)
|
||||
if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)):
|
||||
return None
|
||||
return _shown_items(mapping, _BaseForm.PRINTED)
|
||||
|
||||
|
||||
def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None:
|
||||
if _is_proxy(value):
|
||||
proxy_items: Final = _proxy_items(value, form)
|
||||
return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED)
|
||||
shown_items: Final = _shown_items(value, form)
|
||||
if shown_items is not None:
|
||||
return shown_items, form
|
||||
if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value):
|
||||
return value.__repr_args__(), _BaseForm.PRINTED
|
||||
return None
|
||||
|
||||
|
||||
def _walked_ids(value: object) -> frozenset[int]:
|
||||
return frozenset((id(value), id(_proxied(value)))) if _is_proxy(value) else frozenset((id(value),))
|
||||
|
||||
|
||||
def _container_marker(value: object) -> object:
|
||||
if isinstance(value, list):
|
||||
return "[...]"
|
||||
if isinstance(value, tuple):
|
||||
return "(...)"
|
||||
if isinstance(value, OrderedDict):
|
||||
return "..."
|
||||
if _is_defaultdict(value):
|
||||
factory: Final = _text(value.default_factory, _BaseForm.PRINTED)
|
||||
return factory if factory is _DROPPED else f"{type(value).__name__}({factory}, {{...}})"
|
||||
return "{...}" if _is_dict(value) and not isinstance(value, Counter) else None
|
||||
|
||||
|
||||
def _cycle_marker(value: object) -> object:
|
||||
if not _is_proxy(value):
|
||||
return _container_marker(value)
|
||||
inner: Final = _container_marker(_proxied(value))
|
||||
return inner if inner is None or inner is _DROPPED else f"mappingproxy({inner})"
|
||||
|
||||
|
||||
def _cycle_text(value: object, form: _BaseForm) -> object:
|
||||
marker: Final = _cycle_marker(value)
|
||||
return _text(value, form) if marker is None else marker
|
||||
|
||||
|
||||
def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
||||
if value is None or type(value) in (str, int, float, bool):
|
||||
return value
|
||||
walked: Final = _walked_ids(value)
|
||||
if not walked.isdisjoint(path):
|
||||
return _cycle_text(value, form)
|
||||
inner_path: Final = path | walked
|
||||
try:
|
||||
sequence: Final = _shown_sequence(value, form)
|
||||
if sequence is not None:
|
||||
shown_items: Final = [_json_safe(item, form, inner_path) for item in sequence]
|
||||
return [item for item in shown_items if item is not _DROPPED]
|
||||
expansion: Final = _expansion(value, form)
|
||||
if expansion is None:
|
||||
return _text(value, form)
|
||||
items, inner = expansion
|
||||
pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items]
|
||||
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit
|
||||
return _text(value, form)
|
||||
|
||||
|
||||
def _chunk_json(shown: object) -> str | None:
|
||||
if isinstance(shown, str):
|
||||
return shown
|
||||
return None if shown is _DROPPED else json.dumps(shown)
|
||||
|
||||
|
||||
def responses_chunk_data(chunk: object) -> str | None:
|
||||
if _is_proxy(chunk):
|
||||
expandable: Final = _proxy_items(chunk, _BaseForm.DUMPED) is not None
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if expandable else None
|
||||
if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)):
|
||||
return None
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset()))
|
||||
|
||||
|
||||
def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]:
|
||||
path: Final = frozenset((id(error),))
|
||||
shown: Final = [(key, _json_safe(value, _BaseForm.DUMPED, path)) for key, value in error.items()]
|
||||
return {key: value for key, value in shown if value is not _DROPPED}
|
||||
|
||||
|
||||
def sse_error_object(data: str | bytes) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _ErrorPayload.model_validate_json(data).error_object()
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _ErrorPayload.model_validate(event).error_object() or None
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _original_failure(exception: Exception) -> Exception:
|
||||
current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal
|
||||
while isinstance(current, MidStreamFallbackError) and current.original_exception is not None:
|
||||
|
|
@ -95,6 +379,16 @@ def _status_error_code(status_code: int | None) -> str:
|
|||
return _CLIENT_ERROR_CODES.get(status_code, "invalid_request_error")
|
||||
|
||||
|
||||
def _status_of(code: str | int | None) -> int | None:
|
||||
if isinstance(code, int):
|
||||
return code
|
||||
if code is None or not code.isdecimal():
|
||||
return None
|
||||
with suppress(ValueError):
|
||||
return int(code)
|
||||
return None
|
||||
|
||||
|
||||
def _response_error_code(details: _FailureDetails) -> str:
|
||||
for value in (details.code, details.type):
|
||||
if value == "insufficient_quota":
|
||||
|
|
@ -106,6 +400,115 @@ def _response_error_code(details: _FailureDetails) -> str:
|
|||
return _status_error_code(details.status_code)
|
||||
|
||||
|
||||
_STATUS_PHRASES: Final = MappingProxyType({int(status): status.phrase for status in HTTPStatus})
|
||||
|
||||
|
||||
def _status_phrase(status_code: int | None) -> str:
|
||||
return _STATUS_PHRASES.get(status_code, "Unknown error") if status_code is not None else "Unknown error"
|
||||
|
||||
|
||||
def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]:
|
||||
details: Final = _FailureDetails.model_validate(error)
|
||||
status_code: Final = _status_of(details.code)
|
||||
return {
|
||||
"code": _response_error_code(details.model_copy(update={"status_code": status_code})),
|
||||
"message": details.message or _status_phrase(status_code),
|
||||
}
|
||||
|
||||
|
||||
def _error_event(
|
||||
fields: Mapping[str, object], error: Mapping[str, object], sequence_number: int
|
||||
) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": ResponsesAPIStreamEvents.ERROR.value,
|
||||
**fields,
|
||||
"param": error.get("param"),
|
||||
"sequence_number": sequence_number,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
|
||||
_TERMINAL_EVENT_TYPES: Final = frozenset(("response.completed", "response.failed", "response.incomplete", "error"))
|
||||
_MAX_SEQUENCE_NUMBER: Final = 2**63 - 1
|
||||
|
||||
|
||||
def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]:
|
||||
prefix: Final = f"{name}:"
|
||||
return tuple(line.removeprefix(prefix).removeprefix(" ") for line in lines if line.startswith(prefix))
|
||||
|
||||
|
||||
def _parsed_event(data: str) -> _FramedEvent:
|
||||
try:
|
||||
return _FramedEvent.model_validate_json(data)
|
||||
except ValidationError:
|
||||
return _FramedEvent()
|
||||
|
||||
|
||||
def _in_range_sequence(event: _FramedEvent) -> _FramedEvent:
|
||||
sequence_number: Final = event.sequence_number
|
||||
if sequence_number is None or 0 <= sequence_number < _MAX_SEQUENCE_NUMBER:
|
||||
return event
|
||||
return event.model_copy(update={"sequence_number": None})
|
||||
|
||||
|
||||
def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool:
|
||||
return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",)
|
||||
|
||||
|
||||
def _block_data(lines: Sequence[str]) -> str:
|
||||
return "\n".join(_field_values(lines, "data"))
|
||||
|
||||
|
||||
def _block_event(lines: Sequence[str]) -> _StreamEvent:
|
||||
parsed: Final = _in_range_sequence(_parsed_event(_block_data(lines)))
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines):
|
||||
return parsed
|
||||
return parsed.model_copy(update={"type": "error"})
|
||||
|
||||
|
||||
def _blocks(frames: str) -> Iterator[tuple[str, ...]]:
|
||||
return (tuple(lines) for filled, lines in groupby(SSE_LINE_BREAK.split(frames), key=bool) if filled)
|
||||
|
||||
|
||||
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
|
||||
return tuple(_block_event(lines) for lines in _blocks(frames))
|
||||
|
||||
|
||||
def _before_done(lines: Sequence[str]) -> bool:
|
||||
return _block_data(lines).strip() != "[DONE]"
|
||||
|
||||
|
||||
def _terminal_blocks(frame: str) -> tuple[tuple[tuple[str, ...], _StreamEvent], ...]:
|
||||
read: Final = ((lines, _block_event(lines)) for lines in takewhile(_before_done, _blocks(frame)))
|
||||
return tuple((lines, event) for lines, event in read if event.type in _TERMINAL_EVENT_TYPES)
|
||||
|
||||
|
||||
def frame_error_data(frame: str) -> str | None:
|
||||
terminal: Final = _terminal_blocks(frame)
|
||||
if not terminal:
|
||||
return None
|
||||
lines, event = terminal[-1]
|
||||
return _block_data(lines) if event.type == "error" else None
|
||||
|
||||
|
||||
def _observed_event(chunk: object) -> _StreamEvent | None:
|
||||
if not isinstance(chunk, (BaseModel, Mapping)):
|
||||
return None
|
||||
try:
|
||||
return _StreamEvent.model_validate(chunk)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _wire_event(frame: str | bytes) -> _StreamEvent | None:
|
||||
text: Final = frame.decode("utf-8", errors="replace") if isinstance(frame, bytes) else frame
|
||||
try:
|
||||
return _StreamEvent.model_validate_json(text.removeprefix("data: "))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
class ResponsesStreamErrorState:
|
||||
def __init__(self) -> None:
|
||||
self.response_id: str | None = None
|
||||
|
|
@ -114,14 +517,13 @@ class ResponsesStreamErrorState:
|
|||
self.sequence_number = -1
|
||||
self.terminal_emitted = False
|
||||
self._pending_event: _StreamEvent | None = None
|
||||
self._pending_on_wire = False
|
||||
|
||||
def observe_chunk(self, chunk: object) -> None:
|
||||
self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None
|
||||
self._pending_event = _observed_event(chunk)
|
||||
self._pending_on_wire = isinstance(chunk, Mapping)
|
||||
|
||||
def mark_emitted(self, frame: str | bytes) -> str | bytes:
|
||||
event: Final = self._pending_event
|
||||
if event is None:
|
||||
return frame
|
||||
def _record(self, event: _StreamEvent) -> None:
|
||||
if event.sequence_number is not None:
|
||||
self.sequence_number = max(self.sequence_number, event.sequence_number)
|
||||
if event.response is not None:
|
||||
|
|
@ -129,9 +531,34 @@ class ResponsesStreamErrorState:
|
|||
self.model = event.response.model or self.model
|
||||
if event.response.created_at is not None:
|
||||
self.created_at = event.response.created_at
|
||||
if event.type in ("response.completed", "response.failed", "response.incomplete"):
|
||||
if event.type in _TERMINAL_EVENT_TYPES:
|
||||
self.terminal_emitted = True
|
||||
return frame
|
||||
|
||||
def observe_frames(self, frames: str) -> str:
|
||||
for event in _frame_events(frames):
|
||||
self._record(event)
|
||||
return frames
|
||||
|
||||
def mark_emitted(self, frame: str | bytes) -> str | bytes:
|
||||
observed: Final = self._pending_event
|
||||
if observed is None:
|
||||
return frame
|
||||
event: Final = _wire_event(frame) if self._pending_on_wire else observed
|
||||
self._record(observed if event is None else event)
|
||||
if event is None or not event.type or "\n" in event.type or "\r" in event.type:
|
||||
return frame
|
||||
prefix: Final = f"event: {event.type}\n"
|
||||
return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame
|
||||
|
||||
def format_error(self, error: Mapping[str, object]) -> str:
|
||||
fields: Final = responses_error_fields(error)
|
||||
sequence_number: Final = self.sequence_number + 1
|
||||
payload: Final = json.dumps(_error_event(fields, _dumped_detail(error), sequence_number))
|
||||
return f"event: error\ndata: {payload}\n\n"
|
||||
|
||||
def format_error_message(self, frame: str) -> str:
|
||||
error: Final = sse_error_object(frame.removeprefix("data: "))
|
||||
return frame if error is None else self.format_error(error)
|
||||
|
||||
def format_failure(self, exception: Exception) -> str | None:
|
||||
if self.terminal_emitted:
|
||||
|
|
|
|||
|
|
@ -554,7 +554,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import (
|
|||
from litellm.proxy.common_utils.proxy_state import ProxyState
|
||||
from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
|
||||
from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_data
|
||||
from litellm.proxy.common_utils.scheduled_job_stagger import (
|
||||
apply_scheduled_job_stagger,
|
||||
attach_job_timing_logger,
|
||||
|
|
@ -10309,6 +10309,15 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes:
|
|||
return f"data: {chunk}\n\n"
|
||||
|
||||
|
||||
def _format_responses_sse_chunk(chunk: object) -> str | bytes:
|
||||
data: Final = responses_chunk_data(chunk)
|
||||
return _format_streaming_sse_chunk(chunk=chunk) if data is None else f"data: {data}\n\n"
|
||||
|
||||
|
||||
def _observed_frames(frames: str, error_state: ResponsesStreamErrorState | None) -> str:
|
||||
return frames if error_state is None else error_state.observe_frames(frames)
|
||||
|
||||
|
||||
_SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r")
|
||||
_OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
|
||||
_MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024
|
||||
|
|
@ -10548,7 +10557,9 @@ async def async_data_generator(
|
|||
stream_completed = False
|
||||
client_disconnected = False
|
||||
error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None
|
||||
ends_with_openai_done: Final = not request_data.get("_litellm_skip_openai_stream_done")
|
||||
ends_with_openai_done: Final = not responses_stream_errors and not request_data.get(
|
||||
"_litellm_skip_openai_stream_done"
|
||||
)
|
||||
needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap()
|
||||
stream_iterator: Final = _object_stream(
|
||||
proxy_logging_obj.async_post_call_streaming_iterator_hook(
|
||||
|
|
@ -10683,12 +10694,14 @@ async def async_data_generator(
|
|||
frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer)
|
||||
if frame is None:
|
||||
break
|
||||
yield frame
|
||||
yield _observed_frames(frame, error_state)
|
||||
if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS:
|
||||
raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter")
|
||||
raw_passthrough = True
|
||||
elif chunk.startswith(("data:", "event:", ":")):
|
||||
yield (chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n")
|
||||
yield _observed_frames(
|
||||
chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n", error_state
|
||||
)
|
||||
raw_passthrough = True
|
||||
elif isinstance(chunk, str) and is_raw_sse_stream:
|
||||
raw_sse_buffer += chunk
|
||||
|
|
@ -10696,7 +10709,7 @@ async def async_data_generator(
|
|||
frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer)
|
||||
if frame is None:
|
||||
break
|
||||
yield frame
|
||||
yield _observed_frames(frame, error_state)
|
||||
if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS:
|
||||
raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter")
|
||||
raw_passthrough = True
|
||||
|
|
@ -10707,7 +10720,7 @@ async def async_data_generator(
|
|||
if not raw_passthrough:
|
||||
try:
|
||||
if error_state is not None:
|
||||
yield error_state.mark_emitted(_format_streaming_sse_chunk(chunk=chunk))
|
||||
yield error_state.mark_emitted(_format_responses_sse_chunk(chunk=chunk))
|
||||
else:
|
||||
yield _format_streaming_sse_chunk(chunk=chunk)
|
||||
except Exception as e:
|
||||
|
|
@ -10730,11 +10743,14 @@ async def async_data_generator(
|
|||
ProxyLogging.fire_deferred_stream_logging(request_data)
|
||||
|
||||
if raw_sse_buffer:
|
||||
yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n")
|
||||
yield _observed_frames(
|
||||
raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n",
|
||||
error_state,
|
||||
)
|
||||
|
||||
if error_message is not None:
|
||||
yield error_message
|
||||
# OpenAI-compatible streams terminate with data: [DONE]; Google GenAI (?alt=sse) does not.
|
||||
yield error_message if error_state is None else error_state.format_error_message(error_message)
|
||||
# Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not.
|
||||
if ends_with_openai_done:
|
||||
yield _OPENAI_STREAM_DONE_FRAME
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ from litellm.proxy.auth.user_api_key_auth import (
|
|||
user_api_key_auth_websocket,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import (
|
||||
RESPONSES_SSE_FRAMING,
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
cost_headers,
|
||||
create_response,
|
||||
|
|
@ -483,6 +484,7 @@ async def responses_api(
|
|||
if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data))
|
||||
else native_data_generator
|
||||
),
|
||||
sse_framing=RESPONSES_SSE_FRAMING,
|
||||
model=None,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
|
|
@ -541,7 +543,6 @@ async def responses_api(
|
|||
async def _blocked_stream() -> AsyncGenerator[str, None]:
|
||||
for chunk in block_chunks:
|
||||
yield chunk.decode()
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return await create_response(generator=_blocked_stream(), media_type="text/event-stream", headers={})
|
||||
return build_blocked_response(e)
|
||||
|
|
@ -908,6 +909,7 @@ async def get_response(
|
|||
response_id,
|
||||
read_settles_billing,
|
||||
),
|
||||
sse_framing=RESPONSES_SSE_FRAMING,
|
||||
model=None,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
from fastapi import Request, Response
|
||||
|
|
@ -20,7 +21,13 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.responses_stream_errors import (
|
||||
event_error_object,
|
||||
frame_error_data,
|
||||
responses_error_fields,
|
||||
sse_error_object,
|
||||
)
|
||||
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
|
||||
from litellm.types.llms.openai import ResponsesAPIStatus
|
||||
|
||||
|
|
@ -69,6 +76,7 @@ class _StreamEvent(TypedDict, total=False):
|
|||
delta: ReadOnly[str]
|
||||
part: ReadOnly[object]
|
||||
response: ReadOnly[_TerminalResponse]
|
||||
error: ReadOnly[_JsonDict]
|
||||
|
||||
|
||||
class _StreamEventParser:
|
||||
|
|
@ -85,6 +93,22 @@ def _sse_frame_data(frame: str) -> str | None:
|
|||
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
|
||||
|
||||
|
||||
def _event_error(event: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match event.get("error"):
|
||||
case str() as message if message:
|
||||
return {**event, "message": message}
|
||||
case _:
|
||||
return event_error_object(event) or event
|
||||
|
||||
|
||||
def _first_chunk_error(response: object) -> Mapping[str, object] | None:
|
||||
if isinstance(response, _StreamsBody) or not isinstance(response, Response):
|
||||
return None
|
||||
if response.status_code < HTTPStatus.BAD_REQUEST:
|
||||
return None
|
||||
return responses_error_fields(sse_error_object(bytes(response.body)) or {"code": response.status_code})
|
||||
|
||||
|
||||
async def _never_receive() -> Message:
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
|
|
@ -165,6 +189,7 @@ async def background_streaming_task(
|
|||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
skip_pre_call_logic=True,
|
||||
sse_framing=RESPONSES_SSE_FRAMING,
|
||||
)
|
||||
|
||||
# Process streaming response following OpenAI events format
|
||||
|
|
@ -199,6 +224,7 @@ async def background_streaming_task(
|
|||
None # Will be set by response.completed/failed/incomplete/cancelled
|
||||
)
|
||||
terminal_error = None
|
||||
frame_failure: Mapping[str, object] | None = None # rebind-ok: set once by the first error frame
|
||||
_event_to_status: Final[Mapping[str, ResponsesAPIStatus]] = {
|
||||
"response.completed": "completed",
|
||||
"response.failed": "failed",
|
||||
|
|
@ -234,6 +260,13 @@ async def background_streaming_task(
|
|||
if isinstance(chunk, bytes):
|
||||
chunk = chunk.decode("utf-8")
|
||||
|
||||
if (
|
||||
isinstance(chunk, str)
|
||||
and frame_failure is None
|
||||
and (error_data := frame_error_data(chunk)) is not None
|
||||
):
|
||||
frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data)))
|
||||
|
||||
if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None:
|
||||
if chunk_data == "[DONE]":
|
||||
break
|
||||
|
|
@ -373,23 +406,25 @@ async def background_streaming_task(
|
|||
output_items[item_id] = item
|
||||
state_dirty = True
|
||||
|
||||
# Flush state to Redis if interval elapsed
|
||||
await flush_state_if_needed()
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
except (json.JSONDecodeError, AttributeError, TypeError) as e:
|
||||
verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e)
|
||||
else:
|
||||
await flush_state_if_needed()
|
||||
|
||||
# Final flush to ensure all accumulated state is saved
|
||||
await flush_state_if_needed(force=True)
|
||||
|
||||
# Use the terminal status from the stream, default to "completed"
|
||||
final_status: Final = terminal_status or "completed"
|
||||
early_error: Final = _first_chunk_error(response)
|
||||
stream_failure: Final = frame_failure if terminal_status is None else None
|
||||
failure: Final = stream_failure if early_error is None else early_error
|
||||
final_error: Final = terminal_error if failure is None else failure
|
||||
final_status: Final = (terminal_status or "completed") if failure is None else "failed"
|
||||
|
||||
await polling_handler.update_state(
|
||||
polling_id=polling_id,
|
||||
status=final_status,
|
||||
usage=usage_data,
|
||||
error=terminal_error,
|
||||
error=final_error,
|
||||
reasoning=reasoning_data,
|
||||
tool_choice=tool_choice_data,
|
||||
tools=tools_data,
|
||||
|
|
@ -411,7 +446,7 @@ async def background_streaming_task(
|
|||
"Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s",
|
||||
polling_id,
|
||||
final_status,
|
||||
terminal_error,
|
||||
final_error,
|
||||
incomplete_details_data,
|
||||
len(output_items),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class ResponsePollingHandler:
|
|||
polling_id: str,
|
||||
status: ResponsesAPIStatus | None = None,
|
||||
usage: dict | None = None,
|
||||
error: dict | None = None,
|
||||
error: Mapping[str, object] | None = None,
|
||||
incomplete_details: dict | None = None,
|
||||
reasoning: dict | None = None,
|
||||
tool_choice: object | None = None,
|
||||
|
|
|
|||
|
|
@ -1200,9 +1200,7 @@ def test_responses_pre_call_denial_streams_sse_with_typed_message_item(gateway:
|
|||
response.headers["content-type"],
|
||||
response.text,
|
||||
)
|
||||
lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: "))
|
||||
assert lines[-1] == "data: [DONE]", response.text
|
||||
events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1])
|
||||
events: Final = _blocked_stream_events(response.text)
|
||||
kinds: Final = tuple(event["type"] for event in events)
|
||||
assert tuple(kind for kind in kinds if kind != "response.output_text.delta") == (
|
||||
"response.created",
|
||||
|
|
@ -1289,9 +1287,13 @@ def _responses_output_denial_config(tmp_path: Path, identity: str, model: str) -
|
|||
|
||||
|
||||
def _blocked_stream_events(text: str) -> tuple[dict[str, object], ...]:
|
||||
lines: Final = tuple(line for line in text.split("\n") if line.startswith("data: "))
|
||||
assert lines[-1] == "data: [DONE]", text
|
||||
return tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1])
|
||||
assert "data: [DONE]" not in text, text
|
||||
lines: Final = tuple(line for line in text.split("\n") if line.startswith(("event: ", "data: ")))
|
||||
events: Final[tuple[dict[str, object], ...]] = tuple(
|
||||
json.loads(line.removeprefix("data: ")) for line in lines[1::2]
|
||||
)
|
||||
assert lines[::2] == tuple(f"event: {event['type']}" for event in events), text
|
||||
return events
|
||||
|
||||
|
||||
def _dead_api_base() -> str:
|
||||
|
|
@ -2798,7 +2800,7 @@ def test_client_disconnect_after_a_finished_responses_tool_call_scans_the_text_a
|
|||
with _disconnect_rig(gateway, tmp_path, shape="text_then_tool_call") as rig:
|
||||
try:
|
||||
received: Final = _stream_and_close(
|
||||
rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), "response.output_item.done"
|
||||
rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), '"type":"response.output_item.done"'
|
||||
)
|
||||
assert rig.token() in received, received
|
||||
scans: Final = eventually(
|
||||
|
|
|
|||
|
|
@ -261,7 +261,7 @@ async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once(
|
|||
assert len(served) == 10
|
||||
for item in served:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
assert item.text.rstrip().endswith("data: [DONE]"), item.text
|
||||
assert item.text.rstrip().endswith("data: [DONE]") == (item.call.endpoint == "chat"), item.text
|
||||
_assert_native_requests_for(wire.drain(), calls)
|
||||
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}
|
||||
|
||||
|
|
|
|||
|
|
@ -846,7 +846,12 @@ def test_responses_api_stream_on_a_mantle_claude_id_bridges_to_the_native_stream
|
|||
"POST", "/v1/responses", {"model": model, "input": _prompt(marker), "stream": True}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = _sse_objects(response.text)
|
||||
assert "data: [DONE]" not in response.text, response.text
|
||||
events: Final = tuple(
|
||||
_JSON_OBJECT.validate_json(line.removeprefix("data: "))
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("data: ")
|
||||
)
|
||||
kinds: Final = [event["type"] for event in events]
|
||||
assert (kinds[0], kinds[-1]) == ("response.created", "response.completed"), kinds
|
||||
assert "".join(
|
||||
|
|
|
|||
|
|
@ -482,8 +482,7 @@ def _responses_body(marker: str, *directives: str) -> dict[str, JsonValue]:
|
|||
|
||||
def _failed_response_error(response: httpx.Response) -> dict[str, JsonValue]:
|
||||
assert response.status_code == 200, response.text
|
||||
lines: Final = _data_lines(response.text)
|
||||
assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text
|
||||
assert "[DONE]" not in _data_lines(response.text), response.text
|
||||
events: Final = _payloads(response.text)
|
||||
kinds: Final = tuple(string_value(event["type"]) for event in events)
|
||||
assert kinds[0] == "response.created" and kinds[-1] == "response.failed", response.text
|
||||
|
|
|
|||
|
|
@ -474,9 +474,10 @@ def test_native_responses_stream_forwards_the_failed_response_on_both_legs(gatew
|
|||
assert response.status_code == 200, response.text
|
||||
frames: Final = data_frames(response.text)
|
||||
assert [frame["type"] for frame in frames] == ["response.created", "response.failed"], response.text
|
||||
assert [name for name, _ in sse_events(response.text)] == ["response.created", "response.failed"], response.text
|
||||
failed: Final = object_value(frames[-1]["response"])
|
||||
assert failed["status"] == "failed", failed
|
||||
assert object_value(failed["error"])["code"] == "rate_limit_exceeded", failed
|
||||
assert response.text.rstrip().endswith("data: [DONE]"), response.text
|
||||
assert "data: [DONE]" not in response.text, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
|
||||
|
|
|
|||
|
|
@ -256,8 +256,10 @@ def _responses_frames(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[http
|
|||
)
|
||||
_assert_one_attempt(served.wire, marker)
|
||||
assert served.response.status_code == 200, served.response.text
|
||||
assert served.response.text.rstrip().endswith("data: [DONE]"), served.response.text
|
||||
assert "data: [DONE]" not in served.response.text, served.response.text
|
||||
frames: Final = data_frames(served.response.text)
|
||||
names: Final = tuple(name for name, _ in sse_events(served.response.text))
|
||||
assert names == tuple(item["type"] for item in frames), served.response.text
|
||||
assert frames[-1]["type"] == "response.failed", served.response.text
|
||||
return served.response, object_value(object_value(frames[-1]["response"])["error"])
|
||||
|
||||
|
|
|
|||
|
|
@ -4461,7 +4461,14 @@ class TestBuildBlockSseChunks:
|
|||
def _payloads(self, chunks):
|
||||
import json
|
||||
|
||||
return [json.loads(chunk.decode().removeprefix("data: ").strip()) for chunk in chunks]
|
||||
payloads: Final = [
|
||||
json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: ")))
|
||||
for chunk in chunks
|
||||
]
|
||||
assert [chunk.decode().splitlines()[0] for chunk in chunks] == [
|
||||
f"event: {payload['type']}" for payload in payloads
|
||||
]
|
||||
return payloads
|
||||
|
||||
def test_standalone_block_emits_complete_synthetic_stream(self):
|
||||
handler = OpenAIResponsesHandler()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -9,7 +9,7 @@ import datetime
|
|||
import json
|
||||
import re
|
||||
import unittest
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from typing import Any, Final, Literal, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
|
|
@ -26,6 +26,7 @@ import litellm
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import EMPTY_MAPPING
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
|
@ -36,6 +37,7 @@ from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
|||
from litellm.types.router import RouterErrors
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
from tests._master_key import MASTER_KEY
|
||||
from tests._support.clock_loops import AutojumpClockLoop
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -154,8 +156,9 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
|
|||
assert result.status_code == 200, result.text
|
||||
assert message in result.text
|
||||
if path == "/v1/responses":
|
||||
assert frames[-1] == "data: [DONE]", result.text
|
||||
assert frames[-2].startswith("event: response.failed\n"), result.text
|
||||
assert all("data: [DONE]" not in frame for frame in frames), result.text
|
||||
assert frames[-1].startswith("event: response.failed\n"), result.text
|
||||
assert [frame.splitlines()[0] for frame in frames] == [f"event: {event['type']}" for event in events]
|
||||
if partial:
|
||||
assert [event["type"] for event in events] == [
|
||||
"response.created",
|
||||
|
|
@ -188,6 +191,173 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
|
|||
assert "error" in events[-1]
|
||||
|
||||
|
||||
class _FirstFrameHook(CustomLogger):
|
||||
def __init__(self, first: str | bytes | Exception) -> None:
|
||||
super().__init__()
|
||||
self.first: Final = first
|
||||
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self, user_api_key_dict: object, response: object, request_data: object
|
||||
) -> AsyncIterator[str | bytes]:
|
||||
if isinstance(self.first, Exception):
|
||||
raise self.first
|
||||
yield self.first
|
||||
|
||||
|
||||
_DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n'
|
||||
_NAMED_ERROR_FRAME: Final = (
|
||||
b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "denied", "code": "529"}}\n\n'
|
||||
)
|
||||
|
||||
|
||||
async def _post_through_first_frame_hook(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str, first: str | bytes | Exception
|
||||
) -> httpx.Response:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
model: Final = "gpt-6-astra"
|
||||
upstream_url: Final = "https://streaming.example/v1"
|
||||
is_chat: Final = path == "/v1/chat/completions"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first)])
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override)
|
||||
with respx.mock as transport:
|
||||
transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond(
|
||||
200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"}
|
||||
)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client:
|
||||
return await client.post(
|
||||
path,
|
||||
json={
|
||||
"model": model,
|
||||
"stream": True,
|
||||
**({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"path,first_frame,expected_status",
|
||||
[
|
||||
("/v1/responses", _DATA_ERROR_FRAME, 403),
|
||||
("/v1/chat/completions", _DATA_ERROR_FRAME, 403),
|
||||
("/v1/chat/completions", _NAMED_ERROR_FRAME, 200),
|
||||
("/cursor/chat/completions", _NAMED_ERROR_FRAME, 200),
|
||||
("/v1/responses", _NAMED_ERROR_FRAME, 529),
|
||||
("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200),
|
||||
("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200),
|
||||
("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200),
|
||||
],
|
||||
)
|
||||
async def test_only_a_lone_first_error_frame_answers_with_its_status(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int
|
||||
) -> None:
|
||||
result: Final = await _post_through_first_frame_hook(monkeypatch, path, first_frame)
|
||||
|
||||
assert result.status_code == expected_status, result.text
|
||||
if expected_status == 200:
|
||||
assert result.headers["content-type"].startswith("text/event-stream"), result.text
|
||||
else:
|
||||
assert result.json()["error"]["message"] == "denied"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ["/cursor/chat/completions", "/v1/chat/completions"])
|
||||
async def test_a_hook_failure_before_the_first_chunk_answers_in_chat_completions_shape(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
message: Final = "rate limited before the first chunk"
|
||||
result: Final = await _post_through_first_frame_hook(
|
||||
monkeypatch, path, HTTPException(status_code=429, detail=message)
|
||||
)
|
||||
frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame)
|
||||
|
||||
assert result.status_code == 429, result.text
|
||||
assert frames[-1] == "data: [DONE]", result.text
|
||||
assert message in json.loads(frames[0].removeprefix("data: "))["error"]["message"]
|
||||
assert len(frames) == 2, result.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"])
|
||||
def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None:
|
||||
async def request_and_check() -> None:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
model: Final = "gpt-6-astra"
|
||||
message: Final = "Upstream is rate limiting this key"
|
||||
upstream_url: Final = "https://streaming.example/v1"
|
||||
is_chat: Final = path == "/v1/chat/completions"
|
||||
|
||||
async def slow_rate_limit(_: httpx.Request) -> httpx.Response:
|
||||
await asyncio.sleep(0.3)
|
||||
return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}})
|
||||
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {
|
||||
"model": "openai/" + model,
|
||||
"api_base": upstream_url,
|
||||
"api_key": "fixture-key",
|
||||
"max_retries": 0,
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05)
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override)
|
||||
with respx.mock as transport:
|
||||
transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock(
|
||||
side_effect=slow_rate_limit
|
||||
)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client:
|
||||
result: Final = await client.post(
|
||||
path,
|
||||
json={
|
||||
"model": model,
|
||||
"stream": True,
|
||||
**({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}),
|
||||
},
|
||||
)
|
||||
frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame)
|
||||
|
||||
assert result.status_code == 200, result.text
|
||||
assert frames[0] == ": ping", result.text
|
||||
if is_chat:
|
||||
assert frames[-1] == "data: [DONE]", result.text
|
||||
assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"]
|
||||
else:
|
||||
assert "data: [DONE]" not in result.text
|
||||
assert frames[-1].startswith("event: error\ndata: "), result.text
|
||||
error: Final = json.loads(frames[-1].split("data: ", 1)[1])["error"]
|
||||
assert error["code"] == "429"
|
||||
assert message in error["message"]
|
||||
|
||||
loop: Final = AutojumpClockLoop()
|
||||
try:
|
||||
loop.run_until_complete(request_and_check())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
_COMPACTION_ITEM: Final = {
|
||||
"type": "compaction",
|
||||
"encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}),
|
||||
|
|
@ -1478,6 +1648,25 @@ async def test_a_streamed_read_cut_before_its_terminal_event_ends_in_a_responses
|
|||
assert "before any terminal event" in events[-1]["response"]["error"]["message"]
|
||||
|
||||
|
||||
def _resumed_wire() -> str:
|
||||
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in _resumed_events("resp_read"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_streamed_read_whose_first_event_is_a_named_error_answers_with_its_status(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(_NAMED_ERROR_FRAME)])
|
||||
result, _ = await _read_through_proxy(
|
||||
monkeypatch,
|
||||
{"stream": "true"},
|
||||
httpx.Response(200, content=_resumed_wire(), headers={"Content-Type": "text/event-stream"}),
|
||||
)
|
||||
|
||||
assert result.status_code == 529, result.text
|
||||
assert result.json()["error"]["message"] == "denied"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_plain_read_through_the_proxy_still_answers_json(monkeypatch: pytest.MonkeyPatch):
|
||||
result, upstream_requests = await _read_through_proxy(
|
||||
|
|
@ -5825,9 +6014,11 @@ class TestGuardrailBlockedResponsesShape:
|
|||
assert response.status_code == 200, response.text
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
frames = _sse_data_frames(response.text)
|
||||
assert frames[-1] == "[DONE]"
|
||||
events = [json.loads(frame) for frame in frames[:-1]]
|
||||
assert "[DONE]" not in frames
|
||||
events = [json.loads(frame) for frame in frames]
|
||||
types = [event["type"] for event in events]
|
||||
assert types[-1] == "response.completed"
|
||||
assert [line[7:] for line in response.text.splitlines() if line.startswith("event: ")] == types
|
||||
assert "response.created" in types
|
||||
completed = [event for event in events if event["type"] == "response.completed"]
|
||||
assert len(completed) == 1
|
||||
|
|
@ -5854,7 +6045,8 @@ class TestGuardrailBlockedResponsesShape:
|
|||
|
||||
assert response.status_code == 200, response.text
|
||||
frames = _sse_data_frames(response.text)
|
||||
completed = [json.loads(frame) for frame in frames[:-1] if json.loads(frame)["type"] == "response.completed"]
|
||||
assert "[DONE]" not in frames
|
||||
completed = [json.loads(frame) for frame in frames if json.loads(frame)["type"] == "response.completed"]
|
||||
usage = completed[0]["response"]["usage"]
|
||||
assert usage["input_tokens"] == 14
|
||||
assert usage["output_tokens"] == 20
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@ following the OpenAI Response API format.
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Final, Optional
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import Request
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
|
||||
|
|
@ -1524,6 +1525,272 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == error_payload
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_late_failure_after_keepalive_pings_sets_failed_status_and_error(self):
|
||||
from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
late_frames: Final = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited"))
|
||||
|
||||
async def _body_iterator():
|
||||
yield b": ping\n\n"
|
||||
for frame in late_frames:
|
||||
yield frame.encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_late_failure", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("data_type", "nested_error", "expected_error"),
|
||||
[
|
||||
(
|
||||
{"type": "error"},
|
||||
{"type": "guardrail_error", "code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
({"type": "error"}, "output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}),
|
||||
(
|
||||
{},
|
||||
{"code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
(
|
||||
{"type": "response.output_text.delta"},
|
||||
{"code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
],
|
||||
ids=["typed_object", "typed_string", "untyped_object", "delta_typed_object"],
|
||||
)
|
||||
async def test_a_nested_only_error_event_keeps_its_reason(
|
||||
self,
|
||||
data_type: Mapping[str, str],
|
||||
nested_error: Mapping[str, str] | str,
|
||||
expected_error: Mapping[str, str],
|
||||
):
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
error_event: Final = {**data_type, "sequence_number": 9, "error": nested_error}
|
||||
|
||||
async def _body_iterator():
|
||||
yield b'data: {"type": "response.in_progress"}\n\n'
|
||||
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_nested_error", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("chunks", "expected_status", "expected_error", "expected_usage"),
|
||||
[
|
||||
(
|
||||
(
|
||||
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n'
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
|
||||
),
|
||||
"failed",
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
None,
|
||||
),
|
||||
(
|
||||
(
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
(
|
||||
(
|
||||
'data:{"type": "response.output_text.delta", "delta": "hi"}\n\n'
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
(
|
||||
(
|
||||
'event: error\ndata: {"error": {"code": "' + "9" * 4400 + '", "message": "huge code"}}\n\n',
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
(
|
||||
(
|
||||
'data: {"type": "response.output_item.done", "output_index": 0, "item": "hook-text"}\n\n',
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
(
|
||||
(
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n'
|
||||
'event: response.completed\ndata: {"type": "response.completed", "response": {"status": "completed"}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
(
|
||||
(
|
||||
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n',
|
||||
'data: [DONE]\n\nevent: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"error_frame_after_a_delta",
|
||||
"error_frame_then_completed",
|
||||
"unspaced_delta_then_completed",
|
||||
"error_code_past_the_int_digit_limit_then_completed",
|
||||
"output_item_done_with_a_string_item_then_completed",
|
||||
"error_frame_then_completed_in_one_chunk",
|
||||
"error_frame_after_done_in_one_chunk",
|
||||
],
|
||||
)
|
||||
async def test_a_poll_stores_what_its_stream_ended_with(
|
||||
self,
|
||||
chunks: tuple[str, ...],
|
||||
expected_status: str,
|
||||
expected_error: Mapping[str, str] | None,
|
||||
expected_usage: Mapping[str, int] | None,
|
||||
):
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
async def _body_iterator():
|
||||
yield b'data: {"type": "response.in_progress"}\n\n'
|
||||
for chunk in chunks:
|
||||
yield chunk.encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_stream_end", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == expected_status
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
assert final_call.kwargs["usage"] == expected_usage
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "body", "expected_error"),
|
||||
[
|
||||
(
|
||||
404,
|
||||
json.dumps(
|
||||
{"error": {"message": "model not found", "type": "invalid_request_error", "param": None, "code": "404"}}
|
||||
),
|
||||
{"code": "not_found_error", "message": "model not found"},
|
||||
),
|
||||
(429, "rate limited", {"code": "rate_limit_exceeded", "message": "Too Many Requests"}),
|
||||
(
|
||||
403,
|
||||
json.dumps({"error": {"type": "guardrail_scan_error", "code": 403}}),
|
||||
{"code": "permission_error", "message": "Forbidden"},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_a_first_chunk_error_polls_failed_like_a_late_one(
|
||||
self, status_code: int, body: str, expected_error: Mapping[str, str]
|
||||
):
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
first_chunk_error: Final = Response(status_code=status_code, content=body)
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_first_chunk_error", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=first_chunk_error
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_non_streamed_success_response_still_polls_completed(self):
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_non_streamed_success", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=JSONResponse(status_code=200, content={"id": "resp_123", "status": "completed"})
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "completed"
|
||||
assert final_call.kwargs["error"] is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_incomplete_sets_incomplete_status_and_details(self):
|
||||
"""Test that a response.incomplete stream event results in incomplete status"""
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import copy
|
|||
import datetime
|
||||
import gzip
|
||||
import json
|
||||
from collections.abc import AsyncIterable, Mapping
|
||||
from collections.abc import AsyncIterable, Awaitable, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence
|
||||
from urllib.parse import unquote_plus
|
||||
|
|
@ -40,6 +40,7 @@ from litellm.constants import (
|
|||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.integrations.opentelemetry import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.common_request_processing import (
|
||||
|
|
@ -52,11 +53,15 @@ from litellm.proxy.common_request_processing import (
|
|||
_ClientDisconnectedBeforeFirstChunk,
|
||||
attach_guardrail_information,
|
||||
_extract_error_from_sse_chunk,
|
||||
FirstChunkSseFraming,
|
||||
_get_cost_breakdown_from_logging_obj,
|
||||
CostBreakdownHeaderValues,
|
||||
_has_attribute_error_in_chain,
|
||||
OPENAI_FIRST_CHUNK_FRAMING,
|
||||
RESPONSES_FIRST_CHUNK_FRAMING,
|
||||
include_guardrail_response_requested,
|
||||
is_azure_model_router_request,
|
||||
RESPONSES_LATE_SSE_FRAMING,
|
||||
open_sse_before_first_byte,
|
||||
resolve_litellm_call_id,
|
||||
ttft_keepalive_interval,
|
||||
|
|
@ -89,6 +94,7 @@ from litellm.router import Router
|
|||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.utils import ModelResponseListIterator
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
|
||||
from tests._support.clock_loops import AutojumpClockLoop
|
||||
|
||||
|
||||
def test_attach_guardrail_information_copies_recorded_entries_onto_model_response():
|
||||
|
|
@ -2257,6 +2263,14 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
)
|
||||
|
||||
|
||||
def _fail_stream_setup() -> str:
|
||||
raise RuntimeError("stream setup failed")
|
||||
|
||||
|
||||
async def _stream_failing_before_its_first_chunk() -> AsyncGenerator[str, None]:
|
||||
yield _fail_stream_setup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestCommonRequestProcessingHelpers:
|
||||
async def consume_stream(self, streaming_response: StreamingResponse) -> list:
|
||||
|
|
@ -2335,6 +2349,74 @@ class TestCommonRequestProcessingHelpers:
|
|||
assert body["error"]["code"] == 403
|
||||
assert body["error"]["message"] == "forbidden"
|
||||
|
||||
@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
|
||||
async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self, newline: str):
|
||||
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None}
|
||||
event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error})
|
||||
|
||||
async def responses_stream():
|
||||
yield f"event: error{newline}data: {event}{newline}{newline}"
|
||||
|
||||
response: Final = await create_response(
|
||||
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body) == {"error": error}
|
||||
|
||||
async def test_a_first_event_error_with_a_unicode_line_separator_in_its_message_answers_as_json(self):
|
||||
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked\u2028by policy", "param": None}
|
||||
|
||||
async def responses_stream():
|
||||
event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False)
|
||||
yield f"event: error\ndata: {event}\n\n"
|
||||
|
||||
response: Final = await create_response(
|
||||
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body) == {"error": error}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"framing,frame_starts",
|
||||
[
|
||||
(OPENAI_FIRST_CHUNK_FRAMING, ('data: {"error": ', "data: [DONE]")),
|
||||
(RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)),
|
||||
],
|
||||
)
|
||||
async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(
|
||||
self, framing: FirstChunkSseFraming, frame_starts: tuple[str, ...]
|
||||
) -> None:
|
||||
response: Final = await create_response(
|
||||
_stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing
|
||||
)
|
||||
frames: Final = await self.consume_stream(response)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert len(frames) == len(frame_starts)
|
||||
assert all(frame.startswith(start) for frame, start in zip(frames, frame_starts, strict=True))
|
||||
|
||||
async def test_a_responses_stream_whose_first_frame_names_a_success_code_streams_on(self):
|
||||
frames: Final = (
|
||||
'data: {"error": {"message": "note", "type": "guardrail_scan_error", "code": 200}}\n\n',
|
||||
'event: response.completed\ndata: {"type": "response.completed", "sequence_number": 1}\n\n',
|
||||
)
|
||||
|
||||
async def responses_stream():
|
||||
for frame in frames:
|
||||
yield frame
|
||||
|
||||
response: Final = await create_response(
|
||||
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
|
||||
)
|
||||
|
||||
assert isinstance(response, StreamingResponse)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert tuple(await self.consume_stream(response)) == frames
|
||||
|
||||
async def test_create_streaming_response_first_chunk_not_error(self):
|
||||
async def mock_generator():
|
||||
yield 'data: {"content": "first part"}\n\n'
|
||||
|
|
@ -10677,6 +10759,104 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
|
|||
assert collected[-1] == b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
async def _slow_rate_limit_failure() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
raise HTTPException(status_code=429, detail="rate limited")
|
||||
|
||||
|
||||
async def _slow_guardrail_block_body() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return JSONResponse(
|
||||
status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}
|
||||
)
|
||||
|
||||
|
||||
_POLICY_BLOCK_DETAIL: Final = MappingProxyType(
|
||||
{"error": {"message": "blocked by policy", "type": "guardrail_error"}, "guardrail": "content-filter"}
|
||||
)
|
||||
|
||||
|
||||
async def _slow_codeless_rate_limit_body() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return JSONResponse(status_code=429, content={"error": {"message": "slow down", "code": None}})
|
||||
|
||||
|
||||
async def _slow_dict_detail_failure() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"produce,expected_code,expected_error",
|
||||
[
|
||||
(_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}),
|
||||
(_slow_codeless_rate_limit_body, "rate_limit_exceeded", {"message": "slow down", "code": "429"}),
|
||||
(
|
||||
_slow_guardrail_block_body,
|
||||
"invalid_request_error",
|
||||
{"message": "blocked", "type": "guardrail_error", "code": "400"},
|
||||
),
|
||||
(
|
||||
_slow_dict_detail_failure,
|
||||
"invalid_request_error",
|
||||
{"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done(
|
||||
produce: Callable[[], Awaitable[Response]], expected_code: str, expected_error: Mapping[str, object]
|
||||
) -> None:
|
||||
async def collect_and_check() -> None:
|
||||
response: Final = await open_sse_before_first_byte(
|
||||
produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING
|
||||
)
|
||||
collected: Final = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert b"data: [DONE]\n\n" not in collected
|
||||
event_line, data_line = collected[-1].decode().strip().split("\n")
|
||||
event: Final = json.loads(data_line.removeprefix("data: "))
|
||||
assert event_line == "event: error"
|
||||
assert event["type"] == "error"
|
||||
assert (event["code"], event["message"]) == (expected_code, expected_error["message"])
|
||||
assert {key: event["error"].get(key) for key in expected_error} == expected_error
|
||||
|
||||
loop: Final = AutojumpClockLoop()
|
||||
try:
|
||||
loop.run_until_complete(collect_and_check())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
async def _slow_passthrough_block() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
raise ModifyResponseException(message="blocked by policy", model="gpt-4o", request_data={})
|
||||
|
||||
|
||||
def test_ttft_keepalive_ends_a_late_responses_passthrough_block_with_the_block_response() -> None:
|
||||
async def collect_and_check() -> None:
|
||||
response: Final = await open_sse_before_first_byte(
|
||||
_slow_passthrough_block(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING
|
||||
)
|
||||
collected: Final = await _drain(response)
|
||||
frames: Final = tuple(chunk.decode() for chunk in collected if chunk != TTFT_PING)
|
||||
names: Final = tuple(frame.split("\n", 1)[0] for frame in frames)
|
||||
completed: Final = json.loads(frames[-1].split("\n", 1)[1].removeprefix("data: "))
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert (names[0], names[-1]) == ("event: response.created", "event: response.completed")
|
||||
assert "event: error" not in names
|
||||
assert completed["response"]["output"][0]["content"][0]["text"] == "blocked by policy"
|
||||
|
||||
loop: Final = AutojumpClockLoop()
|
||||
try:
|
||||
loop.run_until_complete(collect_and_check())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
|
||||
"""Starlette never calls the produced response, so its own cleanup never runs
|
||||
|
|
|
|||
|
|
@ -763,7 +763,10 @@ def test_restamp_streaming_chunk_rewrites_nested_model_in_guardrail_block_frames
|
|||
)[0]
|
||||
for frame in frames
|
||||
]
|
||||
events = [json.loads(frame.decode().removeprefix("data: ")) for frame in client_frames]
|
||||
events = [
|
||||
json.loads(next(line[6:] for line in frame.decode().splitlines() if line.startswith("data: ")))
|
||||
for frame in client_frames
|
||||
]
|
||||
|
||||
assert all(frame.endswith(b"\n\n") for frame in client_frames)
|
||||
assert events[-1]["type"] == "response.completed"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue