mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 81dc6c1175 into 08b539e6a8
This commit is contained in:
commit
bfb57be62d
19 changed files with 918 additions and 64 deletions
|
|
@ -263,6 +263,7 @@ _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call"
|
|||
_OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE}
|
||||
_SCANNED_TOOL_SEARCH_NAME: Final = "tool_search"
|
||||
_TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str)
|
||||
_TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"function_call": "arguments", "custom_tool_call": "input"}
|
||||
)
|
||||
|
|
@ -1936,7 +1937,6 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
new output item under the same response id, and close with a
|
||||
``response.completed`` carrying only the replacement item.
|
||||
|
||||
The proxy's data generator appends ``data: [DONE]`` itself.
|
||||
"""
|
||||
events: Final = (
|
||||
self._block_continuation_events(exc, responses_so_far or ())
|
||||
|
|
@ -1944,6 +1944,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
else self._standalone_block_events(exc)
|
||||
)
|
||||
return tuple(
|
||||
f"event: {_STREAM_EVENT_TYPE_ADAPTER.validate_python(stream_item_field(event, 'type'))}\n"
|
||||
f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode()
|
||||
for event in events
|
||||
)
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ from litellm.constants import (
|
|||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
UNSAFE_PROXY_RESPONSE_HEADERS,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
|
|
@ -89,6 +89,7 @@ from litellm.litellm_core_utils.streaming_handler import (
|
|||
backfill_missing_cache_usage_fields,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler
|
||||
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_key_call_resolved_model,
|
||||
|
|
@ -115,6 +116,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_param,
|
||||
openai_error_type,
|
||||
)
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, sse_error_object
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING_BYTES,
|
||||
SSE_STREAM_START_TAIL,
|
||||
|
|
@ -936,11 +938,32 @@ async def _enforce_guardrail_added_tag_budgets(
|
|||
) from e
|
||||
|
||||
|
||||
async def _parse_event_data_for_error(event_line: str | bytes) -> int | None:
|
||||
SseDataPayload: TypeAlias = Callable[[str | bytes], str | None]
|
||||
|
||||
|
||||
def _sse_text(event_line: str | bytes) -> str:
|
||||
return event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
|
||||
|
||||
|
||||
def _sse_data_payload(event_line: str | bytes) -> str | None:
|
||||
text: Final = _sse_text(event_line)
|
||||
return text.removeprefix("data: ").strip() if text.startswith("data: ") else None
|
||||
|
||||
|
||||
def _responses_event_data_payload(event_line: str | bytes) -> str | None:
|
||||
match _sse_text(event_line).strip().split("\n"):
|
||||
case ["event: error", data_line] if data_line.startswith("data: "):
|
||||
return data_line.removeprefix("data: ").strip()
|
||||
case _:
|
||||
return _sse_data_payload(event_line)
|
||||
|
||||
|
||||
async def _parse_event_data_for_error(
|
||||
event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload
|
||||
) -> int | None:
|
||||
"""Parses an event line and returns an error code if present, else None."""
|
||||
event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
|
||||
if event_line.startswith("data: "):
|
||||
json_str: Final = event_line[len("data: ") :].strip()
|
||||
json_str: Final = data_payload(event_line)
|
||||
if json_str is not None:
|
||||
if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message
|
||||
return None
|
||||
try:
|
||||
|
|
@ -971,18 +994,17 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict:
|
||||
def _extract_error_from_sse_chunk(event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload) -> dict:
|
||||
"""
|
||||
Extract error dictionary from SSE format chunk.
|
||||
|
||||
Args:
|
||||
event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n"
|
||||
data_payload: Reads the JSON payload out of the event line
|
||||
|
||||
Returns:
|
||||
Error dictionary in OpenAI API format
|
||||
"""
|
||||
event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line
|
||||
|
||||
# Default error format
|
||||
default_error: Final = {
|
||||
"message": "Unknown error",
|
||||
|
|
@ -991,8 +1013,8 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict:
|
|||
"code": "500",
|
||||
}
|
||||
|
||||
if event_line.startswith("data: "):
|
||||
json_str: Final = event_line[len("data: ") :].strip()
|
||||
json_str: Final = data_payload(event_line)
|
||||
if json_str is not None:
|
||||
if not json_str or json_str == "[DONE]":
|
||||
return default_error
|
||||
|
||||
|
|
@ -1200,6 +1222,70 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
|
|||
return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _responses_error_frames(error_obj: Mapping[str, object]) -> tuple[str, ...]:
|
||||
return (ResponsesStreamErrorState().format_error(error_obj),)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LateSseFraming:
|
||||
"""How a response the keepalive already opened ends once its upstream fails late."""
|
||||
|
||||
failure: Callable[[Exception], tuple[str, ...]]
|
||||
body: Callable[[Response], tuple[bytes, ...]]
|
||||
|
||||
|
||||
def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]:
|
||||
_, error_obj = sse_error_payload(exc)
|
||||
return _sse_error_frames(error_obj)
|
||||
|
||||
|
||||
def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]:
|
||||
return b"data: " + (bytes(produced.body) or b"{}") + b"\n\n", b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]:
|
||||
if isinstance(exc, ModifyResponseException):
|
||||
return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc))
|
||||
return _responses_error_frames(sse_error_payload(exc)[1])
|
||||
|
||||
|
||||
def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]:
|
||||
body: Final = bytes(produced.body)
|
||||
error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode()}
|
||||
with_status: Final = {**error_obj, "code": error_obj.get("code") or str(produced.status_code)}
|
||||
return tuple(frame.encode() for frame in _responses_error_frames(with_status))
|
||||
|
||||
|
||||
OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames)
|
||||
RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming(
|
||||
failure=_responses_late_failure_frames, body=_responses_late_body_frames
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FirstChunkSseFraming:
|
||||
"""How a stream's first chunk is read for an error, and how the stream ends when reading it fails."""
|
||||
|
||||
data_payload: SseDataPayload
|
||||
error_frames: Callable[[Mapping[str, object]], tuple[str, ...]]
|
||||
|
||||
|
||||
OPENAI_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(data_payload=_sse_data_payload, error_frames=_sse_error_frames)
|
||||
RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(
|
||||
data_payload=_responses_event_data_payload, error_frames=_responses_error_frames
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SseFraming:
|
||||
first_chunk: FirstChunkSseFraming
|
||||
late: LateSseFraming
|
||||
|
||||
|
||||
OPENAI_SSE_FRAMING: Final = SseFraming(first_chunk=OPENAI_FIRST_CHUNK_FRAMING, late=OPENAI_LATE_SSE_FRAMING)
|
||||
RESPONSES_SSE_FRAMING: Final = SseFraming(first_chunk=RESPONSES_FIRST_CHUNK_FRAMING, late=RESPONSES_LATE_SSE_FRAMING)
|
||||
|
||||
|
||||
def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
"""`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384)."""
|
||||
return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS})
|
||||
|
|
@ -1228,6 +1314,7 @@ async def create_response(
|
|||
request: Request | None = None,
|
||||
refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None,
|
||||
general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS,
|
||||
framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING,
|
||||
) -> StreamingResponse | JSONResponse:
|
||||
"""
|
||||
Create streaming response, checking if the first chunk is an error.
|
||||
|
|
@ -1263,7 +1350,9 @@ async def create_response(
|
|||
|
||||
if first_chunk_value is not None:
|
||||
try:
|
||||
error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value)
|
||||
error_code_from_chunk: Final = await _parse_event_data_for_error(
|
||||
first_chunk_value, framing.data_payload
|
||||
)
|
||||
if error_code_from_chunk is not None:
|
||||
# First chunk is an error, stream hasn't really started yet
|
||||
# Should return standard JSON error response instead of SSE format
|
||||
|
|
@ -1275,7 +1364,9 @@ async def create_response(
|
|||
|
||||
# Parse error content
|
||||
error_dict: Final = with_call_id(
|
||||
JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)),
|
||||
JSON_OBJECT.validate_python(
|
||||
_extract_error_from_sse_chunk(first_chunk_value, framing.data_payload)
|
||||
),
|
||||
error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)),
|
||||
)
|
||||
|
||||
|
|
@ -1317,7 +1408,7 @@ async def create_response(
|
|||
error_status, error_obj = sse_error_payload(e)
|
||||
|
||||
async def error_gen_message() -> AsyncGenerator[str, None]:
|
||||
for frame in _sse_error_frames(error_obj):
|
||||
for frame in framing.error_frames(error_obj):
|
||||
yield frame
|
||||
|
||||
return StreamingResponse(
|
||||
|
|
@ -1400,13 +1491,13 @@ async def _aclose_late_response(produced: Response) -> None:
|
|||
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
|
||||
|
||||
|
||||
async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]:
|
||||
async def _relay_late_response(produced: Response, framing: LateSseFraming) -> AsyncGenerator[bytes, None]:
|
||||
"""Replay a Response that was built after a keepalive had already opened the wire."""
|
||||
if not isinstance(produced, StreamingResponse):
|
||||
# The status line is already on the wire, so a non-streaming body, an error
|
||||
# body included, can only reach the client as an SSE frame.
|
||||
yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n"
|
||||
yield b"data: [DONE]\n\n"
|
||||
for frame in framing.body(produced):
|
||||
yield frame
|
||||
return
|
||||
|
||||
try:
|
||||
|
|
@ -1448,6 +1539,7 @@ async def open_sse_before_first_byte(
|
|||
ping_interval_seconds: float | str | None,
|
||||
media_type: str = "text/event-stream",
|
||||
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
|
||||
late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING,
|
||||
) -> _LateResponseT | StreamingResponse:
|
||||
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
|
||||
|
||||
|
|
@ -1493,11 +1585,10 @@ async def open_sse_before_first_byte(
|
|||
# would never fire and the failure would go unaudited. The hook
|
||||
# also gets to sanitize what reaches the client, by returning or
|
||||
# raising a replacement, so its answer decides the frame.
|
||||
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
|
||||
for frame in _sse_error_frames(error_obj):
|
||||
for frame in late_framing.failure(await _sanitized_late_failure(exc, on_late_failure)):
|
||||
yield frame.encode()
|
||||
return
|
||||
async for chunk in _relay_late_response(produced):
|
||||
async for chunk in _relay_late_response(produced, late_framing):
|
||||
yield chunk
|
||||
finally:
|
||||
if not produce_task.done():
|
||||
|
|
@ -2913,6 +3004,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
is_streaming_request: bool | None = False,
|
||||
contents: list[object] | None = None,
|
||||
skip_pre_call_logic: bool = False,
|
||||
sse_framing: SseFraming = OPENAI_SSE_FRAMING,
|
||||
) -> object:
|
||||
"""Run the request, sending SSE keepalives while the upstream is still silent.
|
||||
|
||||
|
|
@ -2953,9 +3045,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
is_streaming_request=is_streaming_request,
|
||||
contents=contents,
|
||||
skip_pre_call_logic=skip_pre_call_logic,
|
||||
first_chunk_framing=sse_framing.first_chunk,
|
||||
),
|
||||
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
|
||||
on_late_failure=_audit_late_failure,
|
||||
late_framing=sse_framing.late,
|
||||
)
|
||||
|
||||
async def _process_llm_request(
|
||||
|
|
@ -2979,6 +3073,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
is_streaming_request: bool | None = False,
|
||||
contents: list[object] | None = None, # Add contents parameter
|
||||
skip_pre_call_logic: bool = False,
|
||||
first_chunk_framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING,
|
||||
) -> Any:
|
||||
"""
|
||||
Common request processing logic for both chat completions and responses API endpoints
|
||||
|
|
@ -3262,6 +3357,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request=request,
|
||||
refresh_headers=refresh_stream_headers,
|
||||
general_settings=general_settings,
|
||||
framing=first_chunk_framing,
|
||||
)
|
||||
|
||||
### CALL HOOKS ### - modify outgoing data
|
||||
|
|
@ -3501,8 +3597,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
Streaming ``/v1/responses`` short-circuits out of
|
||||
``base_process_llm_request`` before the non-streaming ownership
|
||||
tail runs. The OpenAI SDK closes the connection at ``data: [DONE]``
|
||||
and starlette cancels the body task on disconnect, so a write that
|
||||
tail runs. The OpenAI SDK can close the connection right after the terminal
|
||||
event and starlette cancels the body task on disconnect, so a write that
|
||||
waits for the generator to finish never lands. The iterator sets
|
||||
``completed_response`` before it hands over its terminal chunk, so
|
||||
the ``LiteLLM_ManagedObjectTable`` row is written the moment it
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import json
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from http import HTTPStatus
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
|
||||
|
||||
from litellm._logging import redact_internal_details_from_client_message
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -57,6 +58,29 @@ class _FailureDetails(LiteLLMBaseModel):
|
|||
return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields})
|
||||
|
||||
|
||||
class _ErrorPayload(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
error: Mapping[str, object] | str
|
||||
|
||||
def error_object(self) -> Mapping[str, object]:
|
||||
return {"message": self.error} if isinstance(self.error, str) else self.error
|
||||
|
||||
|
||||
def sse_error_object(data: str | bytes) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _ErrorPayload.model_validate_json(data).error_object()
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
try:
|
||||
return _ErrorPayload.model_validate(event).error_object() or None
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _original_failure(exception: Exception) -> Exception:
|
||||
current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal
|
||||
while isinstance(current, MidStreamFallbackError) and current.original_exception is not None:
|
||||
|
|
@ -95,6 +119,12 @@ def _status_error_code(status_code: int | None) -> str:
|
|||
return _CLIENT_ERROR_CODES.get(status_code, "invalid_request_error")
|
||||
|
||||
|
||||
def _status_of(code: str | int | None) -> int | None:
|
||||
if isinstance(code, int):
|
||||
return code
|
||||
return int(code) if code is not None and code.isdecimal() else None
|
||||
|
||||
|
||||
def _response_error_code(details: _FailureDetails) -> str:
|
||||
for value in (details.code, details.type):
|
||||
if value == "insufficient_quota":
|
||||
|
|
@ -106,6 +136,22 @@ def _response_error_code(details: _FailureDetails) -> str:
|
|||
return _status_error_code(details.status_code)
|
||||
|
||||
|
||||
_STATUS_PHRASES: Final = MappingProxyType({int(status): status.phrase for status in HTTPStatus})
|
||||
|
||||
|
||||
def _status_phrase(status_code: int | None) -> str:
|
||||
return _STATUS_PHRASES.get(status_code, "Unknown error") if status_code is not None else "Unknown error"
|
||||
|
||||
|
||||
def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]:
|
||||
details: Final = _FailureDetails.model_validate(error)
|
||||
status_code: Final = _status_of(details.code)
|
||||
return {
|
||||
"code": _response_error_code(details.model_copy(update={"status_code": status_code})),
|
||||
"message": details.message or _status_phrase(status_code),
|
||||
}
|
||||
|
||||
|
||||
class ResponsesStreamErrorState:
|
||||
def __init__(self) -> None:
|
||||
self.response_id: str | None = None
|
||||
|
|
@ -129,9 +175,29 @@ class ResponsesStreamErrorState:
|
|||
self.model = event.response.model or self.model
|
||||
if event.response.created_at is not None:
|
||||
self.created_at = event.response.created_at
|
||||
if event.type in ("response.completed", "response.failed", "response.incomplete"):
|
||||
if event.type in ("response.completed", "response.failed", "response.incomplete", "error"):
|
||||
self.terminal_emitted = True
|
||||
return frame
|
||||
if not event.type or "\n" in event.type or "\r" in event.type:
|
||||
return frame
|
||||
prefix: Final = f"event: {event.type}\n"
|
||||
return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame
|
||||
|
||||
def format_error(self, error: Mapping[str, object]) -> str:
|
||||
payload: Final = json.dumps(
|
||||
{
|
||||
"type": ResponsesAPIStreamEvents.ERROR.value,
|
||||
**responses_error_fields(error),
|
||||
"param": error.get("param"),
|
||||
"sequence_number": self.sequence_number + 1,
|
||||
"error": error,
|
||||
},
|
||||
default=str,
|
||||
)
|
||||
return f"event: error\ndata: {payload}\n\n"
|
||||
|
||||
def format_error_message(self, frame: str) -> str:
|
||||
error: Final = sse_error_object(frame.removeprefix("data: "))
|
||||
return frame if error is None else self.format_error(error)
|
||||
|
||||
def format_failure(self, exception: Exception) -> str | None:
|
||||
if self.terminal_emitted:
|
||||
|
|
|
|||
|
|
@ -10241,6 +10241,12 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes:
|
|||
return f"data: {chunk}\n\n"
|
||||
|
||||
|
||||
def _format_responses_sse_chunk(chunk: object) -> str | bytes:
|
||||
if isinstance(chunk, Mapping):
|
||||
return f"data: {json.dumps({**chunk}, default=str)}\n\n"
|
||||
return _format_streaming_sse_chunk(chunk=chunk)
|
||||
|
||||
|
||||
_SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r")
|
||||
_OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
|
||||
_MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024
|
||||
|
|
@ -10480,7 +10486,9 @@ async def async_data_generator(
|
|||
stream_completed = False
|
||||
client_disconnected = False
|
||||
error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None
|
||||
ends_with_openai_done: Final = not request_data.get("_litellm_skip_openai_stream_done")
|
||||
ends_with_openai_done: Final = not responses_stream_errors and not request_data.get(
|
||||
"_litellm_skip_openai_stream_done"
|
||||
)
|
||||
needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap()
|
||||
stream_iterator: Final = _object_stream(
|
||||
proxy_logging_obj.async_post_call_streaming_iterator_hook(
|
||||
|
|
@ -10639,7 +10647,7 @@ async def async_data_generator(
|
|||
if not raw_passthrough:
|
||||
try:
|
||||
if error_state is not None:
|
||||
yield error_state.mark_emitted(_format_streaming_sse_chunk(chunk=chunk))
|
||||
yield error_state.mark_emitted(_format_responses_sse_chunk(chunk=chunk))
|
||||
else:
|
||||
yield _format_streaming_sse_chunk(chunk=chunk)
|
||||
except Exception as e:
|
||||
|
|
@ -10665,8 +10673,8 @@ async def async_data_generator(
|
|||
yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n")
|
||||
|
||||
if error_message is not None:
|
||||
yield error_message
|
||||
# OpenAI-compatible streams terminate with data: [DONE]; Google GenAI (?alt=sse) does not.
|
||||
yield error_message if error_state is None else error_state.format_error_message(error_message)
|
||||
# Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not.
|
||||
if ends_with_openai_done:
|
||||
yield _OPENAI_STREAM_DONE_FRAME
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from litellm.proxy.auth.user_api_key_auth import (
|
|||
user_api_key_auth_websocket,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import (
|
||||
RESPONSES_SSE_FRAMING,
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
cost_headers,
|
||||
create_response,
|
||||
|
|
@ -474,6 +475,7 @@ async def responses_api(
|
|||
if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data))
|
||||
else native_data_generator
|
||||
),
|
||||
sse_framing=RESPONSES_SSE_FRAMING,
|
||||
model=None,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
|
|
@ -532,7 +534,6 @@ async def responses_api(
|
|||
async def _blocked_stream() -> AsyncGenerator[str, None]:
|
||||
for chunk in block_chunks:
|
||||
yield chunk.decode()
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return await create_response(generator=_blocked_stream(), media_type="text/event-stream", headers={})
|
||||
return build_blocked_response(e)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
from fastapi import Request, Response
|
||||
|
|
@ -20,7 +21,12 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines
|
||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.responses_stream_errors import (
|
||||
event_error_object,
|
||||
responses_error_fields,
|
||||
sse_error_object,
|
||||
)
|
||||
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
|
||||
from litellm.types.llms.openai import ResponsesAPIStatus
|
||||
|
||||
|
|
@ -69,6 +75,7 @@ class _StreamEvent(TypedDict, total=False):
|
|||
delta: ReadOnly[str]
|
||||
part: ReadOnly[object]
|
||||
response: ReadOnly[_TerminalResponse]
|
||||
error: ReadOnly[_JsonDict]
|
||||
|
||||
|
||||
class _StreamEventParser:
|
||||
|
|
@ -85,6 +92,22 @@ def _sse_frame_data(frame: str) -> str | None:
|
|||
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
|
||||
|
||||
|
||||
def _event_error(event: Mapping[str, object]) -> Mapping[str, object]:
|
||||
match event.get("error"):
|
||||
case str() as message if message:
|
||||
return {**event, "message": message}
|
||||
case _:
|
||||
return event_error_object(event) or event
|
||||
|
||||
|
||||
def _first_chunk_error(response: object) -> Mapping[str, object] | None:
|
||||
if isinstance(response, _StreamsBody) or not isinstance(response, Response):
|
||||
return None
|
||||
if response.status_code < HTTPStatus.BAD_REQUEST:
|
||||
return None
|
||||
return responses_error_fields(sse_error_object(bytes(response.body)) or {"code": response.status_code})
|
||||
|
||||
|
||||
async def _never_receive() -> Message:
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
|
|
@ -165,6 +188,7 @@ async def background_streaming_task(
|
|||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
skip_pre_call_logic=True,
|
||||
sse_framing=RESPONSES_SSE_FRAMING,
|
||||
)
|
||||
|
||||
# Process streaming response following OpenAI events format
|
||||
|
|
@ -326,6 +350,10 @@ async def background_streaming_task(
|
|||
status="in_progress",
|
||||
)
|
||||
|
||||
elif event_type == "error":
|
||||
terminal_status = "failed"
|
||||
terminal_error = responses_error_fields(_event_error(event))
|
||||
|
||||
elif event_type in (
|
||||
"response.completed",
|
||||
"response.failed",
|
||||
|
|
@ -382,14 +410,15 @@ async def background_streaming_task(
|
|||
# Final flush to ensure all accumulated state is saved
|
||||
await flush_state_if_needed(force=True)
|
||||
|
||||
# Use the terminal status from the stream, default to "completed"
|
||||
final_status: Final = terminal_status or "completed"
|
||||
early_error: Final = _first_chunk_error(response)
|
||||
final_error: Final = terminal_error if early_error is None else early_error
|
||||
final_status: Final = "failed" if early_error is not None else terminal_status or "completed"
|
||||
|
||||
await polling_handler.update_state(
|
||||
polling_id=polling_id,
|
||||
status=final_status,
|
||||
usage=usage_data,
|
||||
error=terminal_error,
|
||||
error=final_error,
|
||||
reasoning=reasoning_data,
|
||||
tool_choice=tool_choice_data,
|
||||
tools=tools_data,
|
||||
|
|
@ -411,7 +440,7 @@ async def background_streaming_task(
|
|||
"Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s",
|
||||
polling_id,
|
||||
final_status,
|
||||
terminal_error,
|
||||
final_error,
|
||||
incomplete_details_data,
|
||||
len(output_items),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class ResponsePollingHandler:
|
|||
polling_id: str,
|
||||
status: ResponsesAPIStatus | None = None,
|
||||
usage: dict | None = None,
|
||||
error: dict | None = None,
|
||||
error: Mapping[str, object] | None = None,
|
||||
incomplete_details: dict | None = None,
|
||||
reasoning: dict | None = None,
|
||||
tool_choice: object | None = None,
|
||||
|
|
|
|||
|
|
@ -1197,9 +1197,7 @@ def test_responses_pre_call_denial_streams_sse_with_typed_message_item(gateway:
|
|||
response.headers["content-type"],
|
||||
response.text,
|
||||
)
|
||||
lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: "))
|
||||
assert lines[-1] == "data: [DONE]", response.text
|
||||
events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1])
|
||||
events: Final = _blocked_stream_events(response.text)
|
||||
kinds: Final = tuple(event["type"] for event in events)
|
||||
assert tuple(kind for kind in kinds if kind != "response.output_text.delta") == (
|
||||
"response.created",
|
||||
|
|
@ -1286,9 +1284,13 @@ def _responses_output_denial_config(tmp_path: Path, identity: str, model: str) -
|
|||
|
||||
|
||||
def _blocked_stream_events(text: str) -> tuple[dict[str, object], ...]:
|
||||
lines: Final = tuple(line for line in text.split("\n") if line.startswith("data: "))
|
||||
assert lines[-1] == "data: [DONE]", text
|
||||
return tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1])
|
||||
assert "data: [DONE]" not in text, text
|
||||
lines: Final = tuple(line for line in text.split("\n") if line.startswith(("event: ", "data: ")))
|
||||
events: Final[tuple[dict[str, object], ...]] = tuple(
|
||||
json.loads(line.removeprefix("data: ")) for line in lines[1::2]
|
||||
)
|
||||
assert lines[::2] == tuple(f"event: {event['type']}" for event in events), text
|
||||
return events
|
||||
|
||||
|
||||
def _dead_api_base() -> str:
|
||||
|
|
@ -2795,7 +2797,7 @@ def test_client_disconnect_after_a_finished_responses_tool_call_scans_the_text_a
|
|||
with _disconnect_rig(gateway, tmp_path, shape="text_then_tool_call") as rig:
|
||||
try:
|
||||
received: Final = _stream_and_close(
|
||||
rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), "response.output_item.done"
|
||||
rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), '"type":"response.output_item.done"'
|
||||
)
|
||||
assert rig.token() in received, received
|
||||
scans: Final = eventually(
|
||||
|
|
|
|||
|
|
@ -261,7 +261,7 @@ async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once(
|
|||
assert len(served) == 10
|
||||
for item in served:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
assert item.text.rstrip().endswith("data: [DONE]"), item.text
|
||||
assert item.text.rstrip().endswith("data: [DONE]") == (item.call.endpoint == "chat"), item.text
|
||||
_assert_native_requests_for(wire.drain(), calls)
|
||||
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}
|
||||
|
||||
|
|
|
|||
|
|
@ -846,7 +846,12 @@ def test_responses_api_stream_on_a_mantle_claude_id_bridges_to_the_native_stream
|
|||
"POST", "/v1/responses", {"model": model, "input": _prompt(marker), "stream": True}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = _sse_objects(response.text)
|
||||
assert "data: [DONE]" not in response.text, response.text
|
||||
events: Final = tuple(
|
||||
_JSON_OBJECT.validate_json(line.removeprefix("data: "))
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("data: ")
|
||||
)
|
||||
kinds: Final = [event["type"] for event in events]
|
||||
assert (kinds[0], kinds[-1]) == ("response.created", "response.completed"), kinds
|
||||
assert "".join(
|
||||
|
|
|
|||
|
|
@ -482,8 +482,7 @@ def _responses_body(marker: str, *directives: str) -> dict[str, JsonValue]:
|
|||
|
||||
def _failed_response_error(response: httpx.Response) -> dict[str, JsonValue]:
|
||||
assert response.status_code == 200, response.text
|
||||
lines: Final = _data_lines(response.text)
|
||||
assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text
|
||||
assert "[DONE]" not in _data_lines(response.text), response.text
|
||||
events: Final = _payloads(response.text)
|
||||
kinds: Final = tuple(string_value(event["type"]) for event in events)
|
||||
assert kinds[0] == "response.created" and kinds[-1] == "response.failed", response.text
|
||||
|
|
|
|||
|
|
@ -474,9 +474,10 @@ def test_native_responses_stream_forwards_the_failed_response_on_both_legs(gatew
|
|||
assert response.status_code == 200, response.text
|
||||
frames: Final = data_frames(response.text)
|
||||
assert [frame["type"] for frame in frames] == ["response.created", "response.failed"], response.text
|
||||
assert [name for name, _ in sse_events(response.text)] == ["response.created", "response.failed"], response.text
|
||||
failed: Final = object_value(frames[-1]["response"])
|
||||
assert failed["status"] == "failed", failed
|
||||
assert object_value(failed["error"])["code"] == "rate_limit_exceeded", failed
|
||||
assert response.text.rstrip().endswith("data: [DONE]"), response.text
|
||||
assert "data: [DONE]" not in response.text, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure"
|
||||
|
|
|
|||
|
|
@ -256,8 +256,10 @@ def _responses_frames(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[http
|
|||
)
|
||||
_assert_one_attempt(served.wire, marker)
|
||||
assert served.response.status_code == 200, served.response.text
|
||||
assert served.response.text.rstrip().endswith("data: [DONE]"), served.response.text
|
||||
assert "data: [DONE]" not in served.response.text, served.response.text
|
||||
frames: Final = data_frames(served.response.text)
|
||||
names: Final = tuple(name for name, _ in sse_events(served.response.text))
|
||||
assert names == tuple(item["type"] for item in frames), served.response.text
|
||||
assert frames[-1]["type"] == "response.failed", served.response.text
|
||||
return served.response, object_value(object_value(frames[-1]["response"])["error"])
|
||||
|
||||
|
|
|
|||
|
|
@ -4423,7 +4423,14 @@ class TestBuildBlockSseChunks:
|
|||
def _payloads(self, chunks):
|
||||
import json
|
||||
|
||||
return [json.loads(chunk.decode().removeprefix("data: ").strip()) for chunk in chunks]
|
||||
payloads: Final = [
|
||||
json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: ")))
|
||||
for chunk in chunks
|
||||
]
|
||||
assert [chunk.decode().splitlines()[0] for chunk in chunks] == [
|
||||
f"event: {payload['type']}" for payload in payloads
|
||||
]
|
||||
return payloads
|
||||
|
||||
def test_standalone_block_emits_complete_synthetic_stream(self):
|
||||
handler = OpenAIResponsesHandler()
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import create_response
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState
|
||||
from litellm.proxy.proxy_server import (
|
||||
_apply_streaming_chunk_hooks,
|
||||
_fast_serialize_simple_model_response_stream,
|
||||
|
|
@ -50,9 +52,12 @@ from litellm.proxy.proxy_server import (
|
|||
select_data_generator,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ErrorEvent,
|
||||
OutputTextDeltaEvent,
|
||||
ResponseCompletedEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseFailedEvent,
|
||||
ResponseIncompleteEvent,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage
|
||||
|
|
@ -92,6 +97,161 @@ async def _async_iter_raises(exc: Exception):
|
|||
raise exc
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("terminal_type", ["response.completed", "response.failed", "response.incomplete", "error"])
|
||||
async def test_responses_stream_dispatches_named_events_without_a_chat_done_sentinel(terminal_type: str) -> None:
|
||||
response: Final = ResponsesAPIResponse(id="resp_framing", created_at=1, model="local-model", output=[])
|
||||
created: Final = ResponseCreatedEvent.model_validate(
|
||||
{"type": "response.created", "sequence_number": 0, "response": response}
|
||||
)
|
||||
delta: Final = OutputTextDeltaEvent.model_validate(
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 1,
|
||||
"item_id": "msg_framing",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "hello",
|
||||
}
|
||||
)
|
||||
terminal_models: Final = {
|
||||
"response.completed": ResponseCompletedEvent,
|
||||
"response.failed": ResponseFailedEvent,
|
||||
"response.incomplete": ResponseIncompleteEvent,
|
||||
"error": ErrorEvent,
|
||||
}
|
||||
terminal: Final = terminal_models[terminal_type].model_validate(
|
||||
{"type": terminal_type, "sequence_number": 2, "response": response}
|
||||
)
|
||||
|
||||
async def upstream() -> AsyncIterator[BaseModel]:
|
||||
for event in (created, delta, terminal):
|
||||
yield event
|
||||
|
||||
frames: Final = tuple(
|
||||
[
|
||||
frame.decode() if isinstance(frame, bytes) else frame
|
||||
async for frame in select_data_generator(
|
||||
response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True
|
||||
)
|
||||
]
|
||||
)
|
||||
assert len(frames) == 3
|
||||
for frame, event_type in zip(
|
||||
frames, ("response.created", "response.output_text.delta", terminal_type), strict=True
|
||||
):
|
||||
assert frame.startswith(f"event: {event_type}\n")
|
||||
payload: Final = json.loads(
|
||||
next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: "))
|
||||
)
|
||||
assert payload["type"] == event_type
|
||||
assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"as_chunk",
|
||||
[lambda event: event, lambda event: event.model_dump(), lambda event: MappingProxyType(event.model_dump())],
|
||||
ids=["model", "dict", "mapping_proxy"],
|
||||
)
|
||||
async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event(
|
||||
as_chunk: Callable[[ErrorEvent], BaseModel | Mapping[str, object]],
|
||||
) -> None:
|
||||
response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[])
|
||||
created: Final = ResponseCreatedEvent.model_validate(
|
||||
{"type": "response.created", "sequence_number": 0, "response": response}
|
||||
)
|
||||
error_event: Final = ErrorEvent.model_validate(
|
||||
{"type": "error", "sequence_number": 1, "code": "server_error", "message": "upstream failed", "param": None}
|
||||
)
|
||||
|
||||
async def upstream() -> AsyncIterator[BaseModel | Mapping[str, object]]:
|
||||
yield created
|
||||
yield as_chunk(error_event)
|
||||
raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model")
|
||||
|
||||
frames: Final = tuple(
|
||||
[
|
||||
frame.decode() if isinstance(frame, bytes) else frame
|
||||
async for frame in select_data_generator(
|
||||
response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert tuple(frame.split("\n", 1)[0] for frame in frames) == ("event: response.created", "event: error")
|
||||
assert json.loads(frames[-1].split("data: ", 1)[1])["message"] == "upstream failed"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("guardrail_error", "error_object"),
|
||||
[
|
||||
(
|
||||
{"message": "response withheld", "type": "guardrail_scan_error", "code": 500},
|
||||
{"message": "response withheld", "type": "guardrail_scan_error", "code": 500},
|
||||
),
|
||||
("response withheld", {"message": "response withheld"}),
|
||||
],
|
||||
)
|
||||
async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event(
|
||||
guardrail_error: Mapping[str, object] | str, error_object: Mapping[str, object]
|
||||
) -> None:
|
||||
response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[])
|
||||
created: Final = ResponseCreatedEvent.model_validate(
|
||||
{"type": "response.created", "sequence_number": 0, "response": response}
|
||||
)
|
||||
|
||||
async def upstream() -> AsyncIterator[object]:
|
||||
yield created
|
||||
yield f"data: {json.dumps({'error': guardrail_error})}\n\n"
|
||||
|
||||
frames: Final = tuple(
|
||||
[
|
||||
frame.decode() if isinstance(frame, bytes) else frame
|
||||
async for frame in select_data_generator(
|
||||
response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True
|
||||
)
|
||||
]
|
||||
)
|
||||
event_line, data_line = frames[-1].strip().split("\n")
|
||||
|
||||
assert "data: [DONE]" not in "".join(frames)
|
||||
assert event_line == "event: error"
|
||||
assert json.loads(data_line.removeprefix("data: ")) == {
|
||||
"type": "error",
|
||||
"code": "server_error",
|
||||
"message": "response withheld",
|
||||
"param": None,
|
||||
"sequence_number": 1,
|
||||
"error": error_object,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("binary", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("event_type", "event_line"),
|
||||
[
|
||||
(None, ""),
|
||||
("", ""),
|
||||
("response.delta\ninjected", ""),
|
||||
("response.delta\rinjected", ""),
|
||||
("response.delta\r\ninjected", ""),
|
||||
("response.vendor_delta", "event: response.vendor_delta\n"),
|
||||
],
|
||||
)
|
||||
def test_responses_event_names_preserve_payload_without_injecting_sse_lines(
|
||||
event_type: str | None, event_line: str, binary: bool
|
||||
) -> None:
|
||||
payload: Final = {"type": event_type, "sequence_number": 0}
|
||||
data_line: Final = f"data: {json.dumps(payload)}\n\n"
|
||||
frame: Final = data_line.encode("utf-8") if binary else data_line
|
||||
expected: Final = (event_line + data_line).encode("utf-8") if binary else event_line + data_line
|
||||
state: Final = ResponsesStreamErrorState()
|
||||
state.observe_chunk(payload)
|
||||
|
||||
assert state.mark_emitted(frame) == expected
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, chunks, hidden_params=None):
|
||||
self._chunks = chunks
|
||||
|
|
@ -1132,14 +1292,15 @@ async def test_responses_stream_failure_sends_the_failure_hook_replacement(monke
|
|||
]
|
||||
)
|
||||
failure: Final = ResponseFailedEvent.model_validate_json(
|
||||
next(line.removeprefix("data: ") for line in frames[-2].splitlines() if line.startswith("data: "))
|
||||
next(line.removeprefix("data: ") for line in frames[-1].splitlines() if line.startswith("data: "))
|
||||
)
|
||||
|
||||
assert _BLOCK_MESSAGE not in "".join(frames)
|
||||
assert failure.response.error is not None
|
||||
assert failure.response.error["code"] == "permission_error"
|
||||
assert "Request blocked by policy" in failure.response.error["message"]
|
||||
assert frames[-1] == "data: [DONE]\n\n"
|
||||
assert frames[-1].startswith("event: response.failed\n")
|
||||
assert "data: [DONE]\n\n" not in frames
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1335,8 +1496,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina
|
|||
for frame in event_frames
|
||||
)
|
||||
|
||||
assert decoded[-1] == "data: [DONE]\n\n"
|
||||
assert len(decoded) == len(event_frames) + 1
|
||||
assert len(decoded) == len(event_frames)
|
||||
assert "data: [DONE]\n\n" not in decoded
|
||||
assert payloads[0]["response"]["id"] == "resp_visible"
|
||||
assert payloads[1] == tool_delta.model_dump()
|
||||
assert len(payloads) == 3
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import dataclasses
|
|||
import datetime
|
||||
import json
|
||||
import unittest
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from typing import Any, Final, Literal, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
|
|
@ -23,12 +23,14 @@ from httpx import Response
|
|||
|
||||
import litellm
|
||||
from litellm.constants import EMPTY_MAPPING
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.router import RouterErrors
|
||||
from tests._master_key import MASTER_KEY
|
||||
from tests._support.clock_loops import AutojumpClockLoop
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -147,8 +149,9 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
|
|||
assert result.status_code == 200, result.text
|
||||
assert message in result.text
|
||||
if path == "/v1/responses":
|
||||
assert frames[-1] == "data: [DONE]", result.text
|
||||
assert frames[-2].startswith("event: response.failed\n"), result.text
|
||||
assert all("data: [DONE]" not in frame for frame in frames), result.text
|
||||
assert frames[-1].startswith("event: response.failed\n"), result.text
|
||||
assert [frame.splitlines()[0] for frame in frames] == [f"event: {event['type']}" for event in events]
|
||||
if partial:
|
||||
assert [event["type"] for event in events] == [
|
||||
"response.created",
|
||||
|
|
@ -181,6 +184,173 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
|
|||
assert "error" in events[-1]
|
||||
|
||||
|
||||
class _FirstFrameHook(CustomLogger):
|
||||
def __init__(self, first: str | bytes | Exception) -> None:
|
||||
super().__init__()
|
||||
self.first: Final = first
|
||||
|
||||
async def async_post_call_streaming_iterator_hook(
|
||||
self, user_api_key_dict: object, response: object, request_data: object
|
||||
) -> AsyncIterator[str | bytes]:
|
||||
if isinstance(self.first, Exception):
|
||||
raise self.first
|
||||
yield self.first
|
||||
|
||||
|
||||
_DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n'
|
||||
_NAMED_ERROR_FRAME: Final = (
|
||||
b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "denied", "code": "529"}}\n\n'
|
||||
)
|
||||
|
||||
|
||||
async def _post_through_first_frame_hook(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str, first: str | bytes | Exception
|
||||
) -> httpx.Response:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
model: Final = "gpt-6-astra"
|
||||
upstream_url: Final = "https://streaming.example/v1"
|
||||
is_chat: Final = path == "/v1/chat/completions"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first)])
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override)
|
||||
with respx.mock as transport:
|
||||
transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond(
|
||||
200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"}
|
||||
)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client:
|
||||
return await client.post(
|
||||
path,
|
||||
json={
|
||||
"model": model,
|
||||
"stream": True,
|
||||
**({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"path,first_frame,expected_status",
|
||||
[
|
||||
("/v1/responses", _DATA_ERROR_FRAME, 403),
|
||||
("/v1/chat/completions", _DATA_ERROR_FRAME, 403),
|
||||
("/v1/chat/completions", _NAMED_ERROR_FRAME, 200),
|
||||
("/cursor/chat/completions", _NAMED_ERROR_FRAME, 200),
|
||||
("/v1/responses", _NAMED_ERROR_FRAME, 529),
|
||||
("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200),
|
||||
("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200),
|
||||
("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200),
|
||||
],
|
||||
)
|
||||
async def test_only_a_lone_first_error_frame_answers_with_its_status(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int
|
||||
) -> None:
|
||||
result: Final = await _post_through_first_frame_hook(monkeypatch, path, first_frame)
|
||||
|
||||
assert result.status_code == expected_status, result.text
|
||||
if expected_status == 200:
|
||||
assert result.headers["content-type"].startswith("text/event-stream"), result.text
|
||||
else:
|
||||
assert result.json()["error"]["message"] == "denied"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ["/cursor/chat/completions", "/v1/chat/completions"])
|
||||
async def test_a_hook_failure_before_the_first_chunk_answers_in_chat_completions_shape(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
message: Final = "rate limited before the first chunk"
|
||||
result: Final = await _post_through_first_frame_hook(
|
||||
monkeypatch, path, HTTPException(status_code=429, detail=message)
|
||||
)
|
||||
frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame)
|
||||
|
||||
assert result.status_code == 429, result.text
|
||||
assert frames[-1] == "data: [DONE]", result.text
|
||||
assert message in json.loads(frames[0].removeprefix("data: "))["error"]["message"]
|
||||
assert len(frames) == 2, result.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"])
|
||||
def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None:
|
||||
async def request_and_check() -> None:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
model: Final = "gpt-6-astra"
|
||||
message: Final = "Upstream is rate limiting this key"
|
||||
upstream_url: Final = "https://streaming.example/v1"
|
||||
is_chat: Final = path == "/v1/chat/completions"
|
||||
|
||||
async def slow_rate_limit(_: httpx.Request) -> httpx.Response:
|
||||
await asyncio.sleep(0.3)
|
||||
return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}})
|
||||
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model,
|
||||
"litellm_params": {
|
||||
"model": "openai/" + model,
|
||||
"api_base": upstream_url,
|
||||
"api_key": "fixture-key",
|
||||
"max_retries": 0,
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05)
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override)
|
||||
with respx.mock as transport:
|
||||
transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock(
|
||||
side_effect=slow_rate_limit
|
||||
)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client:
|
||||
result: Final = await client.post(
|
||||
path,
|
||||
json={
|
||||
"model": model,
|
||||
"stream": True,
|
||||
**({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}),
|
||||
},
|
||||
)
|
||||
frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame)
|
||||
|
||||
assert result.status_code == 200, result.text
|
||||
assert frames[0] == ": ping", result.text
|
||||
if is_chat:
|
||||
assert frames[-1] == "data: [DONE]", result.text
|
||||
assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"]
|
||||
else:
|
||||
assert "data: [DONE]" not in result.text
|
||||
assert frames[-1].startswith("event: error\ndata: "), result.text
|
||||
error: Final = json.loads(frames[-1].split("data: ", 1)[1])["error"]
|
||||
assert error["code"] == "429"
|
||||
assert message in error["message"]
|
||||
|
||||
loop: Final = AutojumpClockLoop()
|
||||
try:
|
||||
loop.run_until_complete(request_and_check())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
_COMPACTION_ITEM: Final = {
|
||||
"type": "compaction",
|
||||
"encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}),
|
||||
|
|
@ -5191,9 +5361,11 @@ class TestGuardrailBlockedResponsesShape:
|
|||
assert response.status_code == 200, response.text
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
frames = _sse_data_frames(response.text)
|
||||
assert frames[-1] == "[DONE]"
|
||||
events = [json.loads(frame) for frame in frames[:-1]]
|
||||
assert "[DONE]" not in frames
|
||||
events = [json.loads(frame) for frame in frames]
|
||||
types = [event["type"] for event in events]
|
||||
assert types[-1] == "response.completed"
|
||||
assert [line[7:] for line in response.text.splitlines() if line.startswith("event: ")] == types
|
||||
assert "response.created" in types
|
||||
completed = [event for event in events if event["type"] == "response.completed"]
|
||||
assert len(completed) == 1
|
||||
|
|
@ -5220,7 +5392,8 @@ class TestGuardrailBlockedResponsesShape:
|
|||
|
||||
assert response.status_code == 200, response.text
|
||||
frames = _sse_data_frames(response.text)
|
||||
completed = [json.loads(frame) for frame in frames[:-1] if json.loads(frame)["type"] == "response.completed"]
|
||||
assert "[DONE]" not in frames
|
||||
completed = [json.loads(frame) for frame in frames if json.loads(frame)["type"] == "response.completed"]
|
||||
usage = completed[0]["response"]["usage"]
|
||||
assert usage["input_tokens"] == 14
|
||||
assert usage["output_tokens"] == 20
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@ following the OpenAI Response API format.
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Final, Optional
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import Request
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler
|
||||
|
|
@ -1524,6 +1525,145 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == error_payload
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_late_failure_after_keepalive_pings_sets_failed_status_and_error(self):
|
||||
from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
late_frames: Final = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited"))
|
||||
|
||||
async def _body_iterator():
|
||||
yield b": ping\n\n"
|
||||
for frame in late_frames:
|
||||
yield frame.encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_late_failure", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("nested_error", "expected_error"),
|
||||
[
|
||||
(
|
||||
{"type": "guardrail_error", "code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
("output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}),
|
||||
],
|
||||
)
|
||||
async def test_a_nested_only_error_event_keeps_its_reason(
|
||||
self, nested_error: Mapping[str, str] | str, expected_error: Mapping[str, str]
|
||||
):
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error}
|
||||
|
||||
async def _body_iterator():
|
||||
yield b'data: {"type": "response.in_progress"}\n\n'
|
||||
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_nested_error", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "body", "expected_error"),
|
||||
[
|
||||
(
|
||||
404,
|
||||
json.dumps(
|
||||
{"error": {"message": "model not found", "type": "invalid_request_error", "param": None, "code": "404"}}
|
||||
),
|
||||
{"code": "not_found_error", "message": "model not found"},
|
||||
),
|
||||
(429, "rate limited", {"code": "rate_limit_exceeded", "message": "Too Many Requests"}),
|
||||
(
|
||||
403,
|
||||
json.dumps({"error": {"type": "guardrail_scan_error", "code": 403}}),
|
||||
{"code": "permission_error", "message": "Forbidden"},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_a_first_chunk_error_polls_failed_like_a_late_one(
|
||||
self, status_code: int, body: str, expected_error: Mapping[str, str]
|
||||
):
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
first_chunk_error: Final = Response(status_code=status_code, content=body)
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_first_chunk_error", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=first_chunk_error
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_non_streamed_success_response_still_polls_completed(self):
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_non_streamed_success", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=JSONResponse(status_code=200, content={"id": "resp_123", "status": "completed"})
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == "completed"
|
||||
assert final_call.kwargs["error"] is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_incomplete_sets_incomplete_status_and_details(self):
|
||||
"""Test that a response.incomplete stream event results in incomplete status"""
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import contextvars
|
|||
import copy
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import AsyncIterable, Mapping
|
||||
from collections.abc import AsyncIterable, Awaitable, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence
|
||||
from urllib.parse import unquote_plus
|
||||
|
|
@ -37,6 +37,7 @@ from litellm.constants import (
|
|||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.integrations.opentelemetry import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.common_request_processing import (
|
||||
|
|
@ -49,11 +50,15 @@ from litellm.proxy.common_request_processing import (
|
|||
_ClientDisconnectedBeforeFirstChunk,
|
||||
attach_guardrail_information,
|
||||
_extract_error_from_sse_chunk,
|
||||
FirstChunkSseFraming,
|
||||
_get_cost_breakdown_from_logging_obj,
|
||||
CostBreakdownHeaderValues,
|
||||
_has_attribute_error_in_chain,
|
||||
OPENAI_FIRST_CHUNK_FRAMING,
|
||||
RESPONSES_FIRST_CHUNK_FRAMING,
|
||||
include_guardrail_response_requested,
|
||||
is_azure_model_router_request,
|
||||
RESPONSES_LATE_SSE_FRAMING,
|
||||
open_sse_before_first_byte,
|
||||
resolve_litellm_call_id,
|
||||
ttft_keepalive_interval,
|
||||
|
|
@ -84,6 +89,7 @@ from litellm.router import Router
|
|||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.utils import ModelResponseListIterator
|
||||
from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
|
||||
from tests._support.clock_loops import AutojumpClockLoop
|
||||
|
||||
|
||||
def test_attach_guardrail_information_copies_recorded_entries_onto_model_response():
|
||||
|
|
@ -2252,6 +2258,14 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
)
|
||||
|
||||
|
||||
def _fail_stream_setup() -> str:
|
||||
raise RuntimeError("stream setup failed")
|
||||
|
||||
|
||||
async def _stream_failing_before_its_first_chunk() -> AsyncGenerator[str, None]:
|
||||
yield _fail_stream_setup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestCommonRequestProcessingHelpers:
|
||||
async def consume_stream(self, streaming_response: StreamingResponse) -> list:
|
||||
|
|
@ -2330,6 +2344,54 @@ class TestCommonRequestProcessingHelpers:
|
|||
assert body["error"]["code"] == 403
|
||||
assert body["error"]["message"] == "forbidden"
|
||||
|
||||
async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self):
|
||||
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None}
|
||||
|
||||
async def responses_stream():
|
||||
yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n"
|
||||
|
||||
response: Final = await create_response(
|
||||
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body) == {"error": error}
|
||||
|
||||
async def test_a_first_event_error_with_a_unicode_line_separator_in_its_message_answers_as_json(self):
|
||||
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked\u2028by policy", "param": None}
|
||||
|
||||
async def responses_stream():
|
||||
event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False)
|
||||
yield f"event: error\ndata: {event}\n\n"
|
||||
|
||||
response: Final = await create_response(
|
||||
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body) == {"error": error}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"framing,frame_starts",
|
||||
[
|
||||
(OPENAI_FIRST_CHUNK_FRAMING, ('data: {"error": ', "data: [DONE]")),
|
||||
(RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)),
|
||||
],
|
||||
)
|
||||
async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(
|
||||
self, framing: FirstChunkSseFraming, frame_starts: tuple[str, ...]
|
||||
) -> None:
|
||||
response: Final = await create_response(
|
||||
_stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing
|
||||
)
|
||||
frames: Final = await self.consume_stream(response)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert len(frames) == len(frame_starts)
|
||||
assert all(frame.startswith(start) for frame, start in zip(frames, frame_starts, strict=True))
|
||||
|
||||
async def test_create_streaming_response_first_chunk_not_error(self):
|
||||
async def mock_generator():
|
||||
yield 'data: {"content": "first part"}\n\n'
|
||||
|
|
@ -10545,6 +10607,104 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
|
|||
assert collected[-1] == b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
async def _slow_rate_limit_failure() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
raise HTTPException(status_code=429, detail="rate limited")
|
||||
|
||||
|
||||
async def _slow_guardrail_block_body() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return JSONResponse(
|
||||
status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}
|
||||
)
|
||||
|
||||
|
||||
_POLICY_BLOCK_DETAIL: Final = MappingProxyType(
|
||||
{"error": {"message": "blocked by policy", "type": "guardrail_error"}, "guardrail": "content-filter"}
|
||||
)
|
||||
|
||||
|
||||
async def _slow_codeless_rate_limit_body() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
return JSONResponse(status_code=429, content={"error": {"message": "slow down", "code": None}})
|
||||
|
||||
|
||||
async def _slow_dict_detail_failure() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"produce,expected_code,expected_error",
|
||||
[
|
||||
(_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}),
|
||||
(_slow_codeless_rate_limit_body, "rate_limit_exceeded", {"message": "slow down", "code": "429"}),
|
||||
(
|
||||
_slow_guardrail_block_body,
|
||||
"invalid_request_error",
|
||||
{"message": "blocked", "type": "guardrail_error", "code": "400"},
|
||||
),
|
||||
(
|
||||
_slow_dict_detail_failure,
|
||||
"invalid_request_error",
|
||||
{"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done(
|
||||
produce: Callable[[], Awaitable[Response]], expected_code: str, expected_error: Mapping[str, object]
|
||||
) -> None:
|
||||
async def collect_and_check() -> None:
|
||||
response: Final = await open_sse_before_first_byte(
|
||||
produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING
|
||||
)
|
||||
collected: Final = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert b"data: [DONE]\n\n" not in collected
|
||||
event_line, data_line = collected[-1].decode().strip().split("\n")
|
||||
event: Final = json.loads(data_line.removeprefix("data: "))
|
||||
assert event_line == "event: error"
|
||||
assert event["type"] == "error"
|
||||
assert (event["code"], event["message"]) == (expected_code, expected_error["message"])
|
||||
assert {key: event["error"].get(key) for key in expected_error} == expected_error
|
||||
|
||||
loop: Final = AutojumpClockLoop()
|
||||
try:
|
||||
loop.run_until_complete(collect_and_check())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
async def _slow_passthrough_block() -> Response:
|
||||
await asyncio.sleep(0.2)
|
||||
raise ModifyResponseException(message="blocked by policy", model="gpt-4o", request_data={})
|
||||
|
||||
|
||||
def test_ttft_keepalive_ends_a_late_responses_passthrough_block_with_the_block_response() -> None:
|
||||
async def collect_and_check() -> None:
|
||||
response: Final = await open_sse_before_first_byte(
|
||||
_slow_passthrough_block(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING
|
||||
)
|
||||
collected: Final = await _drain(response)
|
||||
frames: Final = tuple(chunk.decode() for chunk in collected if chunk != TTFT_PING)
|
||||
names: Final = tuple(frame.split("\n", 1)[0] for frame in frames)
|
||||
completed: Final = json.loads(frames[-1].split("\n", 1)[1].removeprefix("data: "))
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert (names[0], names[-1]) == ("event: response.created", "event: response.completed")
|
||||
assert "event: error" not in names
|
||||
assert completed["response"]["output"][0]["content"][0]["text"] == "blocked by policy"
|
||||
|
||||
loop: Final = AutojumpClockLoop()
|
||||
try:
|
||||
loop.run_until_complete(collect_and_check())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
|
||||
"""Starlette never calls the produced response, so its own cleanup never runs
|
||||
|
|
|
|||
|
|
@ -763,7 +763,10 @@ def test_restamp_streaming_chunk_rewrites_nested_model_in_guardrail_block_frames
|
|||
)[0]
|
||||
for frame in frames
|
||||
]
|
||||
events = [json.loads(frame.decode().removeprefix("data: ")) for frame in client_frames]
|
||||
events = [
|
||||
json.loads(next(line[6:] for line in frame.decode().splitlines() if line.startswith("data: ")))
|
||||
for frame in client_frames
|
||||
]
|
||||
|
||||
assert all(frame.endswith(b"\n\n") for frame in client_frames)
|
||||
assert events[-1]["type"] == "response.completed"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue