mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): frame streamed Responses reads and send unreadable hook values as base did
Track pre-framed and raw SSE frames so a late failure never follows a terminal event, accept CRLF and CR line breaks in a first named error, frame GET /v1/responses/{id}?stream=true as Responses events, print a whole hook proxy chunk once and fall back to its text, ignore a non-integer status_code in a hook error frame, read a late-block dict subclass through items() when that view differs from its storage, and keep the order and inline form base's repr printed for nested mappings
This commit is contained in:
parent
1ef9830457
commit
8ac651b486
7 changed files with 354 additions and 48 deletions
|
|
@ -120,7 +120,11 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_type,
|
||||
relayed_provider_specific_fields,
|
||||
)
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, sse_error_object
|
||||
from litellm.proxy.common_utils.responses_stream_errors import (
|
||||
SSE_LINE_BREAK,
|
||||
ResponsesStreamErrorState,
|
||||
sse_error_object,
|
||||
)
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING_BYTES,
|
||||
SSE_STREAM_START_TAIL,
|
||||
|
|
@ -1019,7 +1023,7 @@ def _sse_data_payload(event_line: str | bytes) -> str | None:
|
|||
|
||||
|
||||
def _responses_event_data_payload(event_line: str | bytes) -> str | None:
|
||||
match _sse_text(event_line).strip().split("\n"):
|
||||
match SSE_LINE_BREAK.split(_sse_text(event_line).strip()):
|
||||
case ["event: error", data_line] if data_line.startswith("data: "):
|
||||
return data_line.removeprefix("data: ").strip()
|
||||
case _:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import json
|
||||
import re
|
||||
import time
|
||||
from collections import Counter, OrderedDict, defaultdict
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
from itertools import chain, groupby, zip_longest
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -41,6 +43,11 @@ class _FailureDetails(LiteLLMBaseModel):
|
|||
type: str | None = None
|
||||
status_code: int | None = None
|
||||
|
||||
@field_validator("status_code", mode="before")
|
||||
@classmethod
|
||||
def normalize_status_code(cls, value: object) -> int | None:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
||||
|
||||
@field_validator("message", mode="before")
|
||||
@classmethod
|
||||
def normalize_message(cls, value: object) -> str | None:
|
||||
|
|
@ -78,25 +85,35 @@ class _BaseForm(Enum):
|
|||
DUMPED = "dumped"
|
||||
|
||||
|
||||
def _prints_every_item(kind: type[object], form: _BaseForm) -> bool:
|
||||
repr_owner: Final = next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object)
|
||||
if form is _BaseForm.DUMPED:
|
||||
return kind.__str__ is object.__str__ and repr_owner in (dict, list, tuple, OrderedDict, defaultdict, Counter)
|
||||
return (
|
||||
kind.__str__ is object.__str__
|
||||
and kind.__format__ is object.__format__
|
||||
and (kind in (OrderedDict, Counter) or repr_owner in (dict, list, tuple, defaultdict))
|
||||
def _repr_owner(kind: type[object]) -> type[object]:
|
||||
return next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object)
|
||||
|
||||
|
||||
def _repr_prints_every_item(kind: type[object]) -> bool:
|
||||
return kind in (OrderedDict, Counter) or _repr_owner(kind) in (dict, list, tuple, defaultdict)
|
||||
|
||||
|
||||
def _format_prints_every_item(kind: type[object]) -> bool:
|
||||
return kind.__str__ is object.__str__ and kind.__format__ is object.__format__ and _repr_prints_every_item(kind)
|
||||
|
||||
|
||||
def _str_prints_every_item(kind: type[object]) -> bool:
|
||||
return kind.__str__ is object.__str__ and _repr_owner(kind) in (
|
||||
dict,
|
||||
list,
|
||||
tuple,
|
||||
OrderedDict,
|
||||
defaultdict,
|
||||
Counter,
|
||||
)
|
||||
|
||||
|
||||
def _is_shown_dict(value: object, form: _BaseForm) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object
|
||||
kind: Final = type(value)
|
||||
return isinstance(value, dict) and _prints_every_item(kind, form)
|
||||
def _is_dict(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_shown_sequence(value: object, form: _BaseForm) -> TypeIs[Sequence[object]]: # guard-ok: items as object
|
||||
kind: Final = type(value)
|
||||
return isinstance(value, (list, tuple)) and _prints_every_item(kind, form)
|
||||
def _is_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: items as object
|
||||
return isinstance(value, (list, tuple))
|
||||
|
||||
|
||||
def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed
|
||||
|
|
@ -142,22 +159,45 @@ def _json_key(key: object, form: _BaseForm) -> object:
|
|||
return key if key is None or type(key) in (str, int, float, bool) else _text(key, form)
|
||||
|
||||
|
||||
def _shown_sequence(value: object, form: _BaseForm) -> Iterator[object] | None:
|
||||
if not _is_shown_sequence(value, form):
|
||||
return None
|
||||
if form is _BaseForm.PRINTED and isinstance(value, list):
|
||||
def _stored_sequence(value: Sequence[object]) -> Iterator[object]:
|
||||
if isinstance(value, list):
|
||||
return list[object].__iter__(value)
|
||||
if form is _BaseForm.PRINTED and isinstance(value, tuple):
|
||||
if isinstance(value, tuple):
|
||||
return tuple[object, ...].__iter__(value)
|
||||
return iter(value)
|
||||
|
||||
|
||||
def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None:
|
||||
if not _is_shown_dict(value, form):
|
||||
def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, object]]:
|
||||
return dict[object, object].items(value) if isinstance(value, dict) else value.items()
|
||||
|
||||
|
||||
def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool:
|
||||
return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored))
|
||||
|
||||
|
||||
def _shown_sequence(value: object, form: _BaseForm) -> Iterable[object] | None:
|
||||
if not _is_sequence(value):
|
||||
return None
|
||||
if form is _BaseForm.PRINTED and isinstance(value, dict):
|
||||
return dict[object, object].items(value)
|
||||
return value.items()
|
||||
if form is _BaseForm.PRINTED:
|
||||
return _stored_sequence(value) if _repr_prints_every_item(type(value)) else None
|
||||
dumped: Final = tuple(value)
|
||||
return dumped if _dumps_shown(type(value), dumped, _stored_sequence(value)) else None
|
||||
|
||||
|
||||
def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None:
|
||||
if not _is_dict(value):
|
||||
return None
|
||||
if form is _BaseForm.DUMPED:
|
||||
dumped: Final = tuple(value.items())
|
||||
stored: Final = chain.from_iterable(_stored_items(value))
|
||||
return dumped if _dumps_shown(type(value), chain.from_iterable(dumped), stored) else None
|
||||
if not _repr_prints_every_item(type(value)):
|
||||
return None
|
||||
if isinstance(value, OrderedDict):
|
||||
return OrderedDict[object, object].items(value)
|
||||
if isinstance(value, Counter):
|
||||
return Counter[object].most_common(value)
|
||||
return _stored_items(value)
|
||||
|
||||
|
||||
def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None:
|
||||
|
|
@ -187,19 +227,36 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
|||
if expansion is None:
|
||||
return _text(value, form)
|
||||
items, inner = expansion
|
||||
pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items]
|
||||
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
|
||||
return _items_json(items, inner, inner_path)
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit
|
||||
return _text(value, form)
|
||||
|
||||
|
||||
def responses_chunk_json(chunk: object) -> str | None:
|
||||
if _is_proxy(chunk) and _printed_proxy_items(chunk, format(chunk, "")) is None:
|
||||
return None
|
||||
if not _is_shown_dict(chunk, _BaseForm.PRINTED) and not _is_proxy(chunk):
|
||||
def _items_json(
|
||||
items: Iterable[tuple[object, object]], form: _BaseForm, path: frozenset[int]
|
||||
) -> Mapping[object, object]:
|
||||
pairs: Final = [(_json_key(key, form), _json_safe(item, form, path)) for key, item in items]
|
||||
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
|
||||
|
||||
|
||||
def _proxy_chunk_data(proxy: Mapping[object, object]) -> str:
|
||||
printed: Final = format(proxy, "")
|
||||
try:
|
||||
items: Final = _printed_proxy_items(proxy, printed)
|
||||
return printed if items is None else json.dumps(_items_json(items, _BaseForm.PRINTED, frozenset((id(proxy),))))
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
|
||||
return printed
|
||||
|
||||
|
||||
def responses_chunk_data(chunk: object) -> str | None:
|
||||
if _is_proxy(chunk):
|
||||
return _proxy_chunk_data(chunk)
|
||||
if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)):
|
||||
return None
|
||||
shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset())
|
||||
return None if isinstance(shown, str) or shown is _DROPPED else json.dumps(shown)
|
||||
if isinstance(shown, str):
|
||||
return shown
|
||||
return None if shown is _DROPPED else json.dumps(shown)
|
||||
|
||||
|
||||
def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -305,6 +362,35 @@ def _error_event(
|
|||
}
|
||||
|
||||
|
||||
SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
|
||||
_TERMINAL_EVENT_TYPES: Final = frozenset(("response.completed", "response.failed", "response.incomplete", "error"))
|
||||
|
||||
|
||||
def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]:
|
||||
prefix: Final = f"{name}:"
|
||||
return tuple(line.removeprefix(prefix).removeprefix(" ") for line in lines if line.startswith(prefix))
|
||||
|
||||
|
||||
def _parsed_event(data: str) -> _StreamEvent:
|
||||
try:
|
||||
return _StreamEvent.model_validate_json(data)
|
||||
except ValidationError:
|
||||
return _StreamEvent()
|
||||
|
||||
|
||||
def _block_event(lines: Sequence[str]) -> _StreamEvent:
|
||||
named: Final = _field_values(lines, "event")
|
||||
parsed: Final = _parsed_event("\n".join(_field_values(lines, "data")))
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not named:
|
||||
return parsed
|
||||
return parsed.model_copy(update={"type": named[-1]})
|
||||
|
||||
|
||||
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
|
||||
blocks: Final = groupby(SSE_LINE_BREAK.split(frames), key=bool)
|
||||
return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled)
|
||||
|
||||
|
||||
class ResponsesStreamErrorState:
|
||||
def __init__(self) -> None:
|
||||
self.response_id: str | None = None
|
||||
|
|
@ -317,10 +403,7 @@ class ResponsesStreamErrorState:
|
|||
def observe_chunk(self, chunk: object) -> None:
|
||||
self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None
|
||||
|
||||
def mark_emitted(self, frame: str | bytes) -> str | bytes:
|
||||
event: Final = self._pending_event
|
||||
if event is None:
|
||||
return frame
|
||||
def _record(self, event: _StreamEvent) -> None:
|
||||
if event.sequence_number is not None:
|
||||
self.sequence_number = max(self.sequence_number, event.sequence_number)
|
||||
if event.response is not None:
|
||||
|
|
@ -328,8 +411,19 @@ class ResponsesStreamErrorState:
|
|||
self.model = event.response.model or self.model
|
||||
if event.response.created_at is not None:
|
||||
self.created_at = event.response.created_at
|
||||
if event.type in ("response.completed", "response.failed", "response.incomplete", "error"):
|
||||
if event.type in _TERMINAL_EVENT_TYPES:
|
||||
self.terminal_emitted = True
|
||||
|
||||
def observe_frames(self, frames: str) -> str:
|
||||
for event in _frame_events(frames):
|
||||
self._record(event)
|
||||
return frames
|
||||
|
||||
def mark_emitted(self, frame: str | bytes) -> str | bytes:
|
||||
event: Final = self._pending_event
|
||||
if event is None:
|
||||
return frame
|
||||
self._record(event)
|
||||
if not event.type or "\n" in event.type or "\r" in event.type:
|
||||
return frame
|
||||
prefix: Final = f"event: {event.type}\n"
|
||||
|
|
|
|||
|
|
@ -554,7 +554,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import (
|
|||
from litellm.proxy.common_utils.proxy_state import ProxyState
|
||||
from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids
|
||||
from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_json
|
||||
from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_data
|
||||
from litellm.proxy.common_utils.scheduled_job_stagger import (
|
||||
apply_scheduled_job_stagger,
|
||||
attach_job_timing_logger,
|
||||
|
|
@ -10267,10 +10267,14 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes:
|
|||
|
||||
|
||||
def _format_responses_sse_chunk(chunk: object) -> str | bytes:
|
||||
data: Final = responses_chunk_json(chunk)
|
||||
data: Final = responses_chunk_data(chunk)
|
||||
return _format_streaming_sse_chunk(chunk=chunk) if data is None else f"data: {data}\n\n"
|
||||
|
||||
|
||||
def _observed_frames(frames: str, error_state: ResponsesStreamErrorState | None) -> str:
|
||||
return frames if error_state is None else error_state.observe_frames(frames)
|
||||
|
||||
|
||||
_SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r")
|
||||
_OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
|
||||
_MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024
|
||||
|
|
@ -10647,12 +10651,14 @@ async def async_data_generator(
|
|||
frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer)
|
||||
if frame is None:
|
||||
break
|
||||
yield frame
|
||||
yield _observed_frames(frame, error_state)
|
||||
if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS:
|
||||
raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter")
|
||||
raw_passthrough = True
|
||||
elif chunk.startswith(("data:", "event:", ":")):
|
||||
yield (chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n")
|
||||
yield _observed_frames(
|
||||
chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n", error_state
|
||||
)
|
||||
raw_passthrough = True
|
||||
elif isinstance(chunk, str) and is_raw_sse_stream:
|
||||
raw_sse_buffer += chunk
|
||||
|
|
@ -10660,7 +10666,7 @@ async def async_data_generator(
|
|||
frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer)
|
||||
if frame is None:
|
||||
break
|
||||
yield frame
|
||||
yield _observed_frames(frame, error_state)
|
||||
if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS:
|
||||
raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter")
|
||||
raw_passthrough = True
|
||||
|
|
@ -10694,7 +10700,10 @@ async def async_data_generator(
|
|||
ProxyLogging.fire_deferred_stream_logging(request_data)
|
||||
|
||||
if raw_sse_buffer:
|
||||
yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n")
|
||||
yield _observed_frames(
|
||||
raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n",
|
||||
error_state,
|
||||
)
|
||||
|
||||
if error_message is not None:
|
||||
yield error_message if error_state is None else error_state.format_error_message(error_message)
|
||||
|
|
|
|||
|
|
@ -908,6 +908,7 @@ async def get_response(
|
|||
response_id,
|
||||
read_settles_billing,
|
||||
),
|
||||
sse_framing=RESPONSES_SSE_FRAMING,
|
||||
model=None,
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import asyncio
|
|||
import functools
|
||||
import json
|
||||
from collections import Counter, OrderedDict
|
||||
from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, Mapping
|
||||
from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
|
|
@ -387,10 +387,118 @@ def test_responses_sse_chunk_sends_a_whole_chunk_as_the_text_its_printer_gave(ch
|
|||
assert "hook-canary" not in frame
|
||||
|
||||
|
||||
class _RaisingRepr:
|
||||
def __repr__(self) -> str:
|
||||
raise RuntimeError("rig repr failure")
|
||||
|
||||
|
||||
class _MaskedStrHolder(dict[str, object]):
|
||||
def __str__(self) -> str:
|
||||
return "{'type': 'response.output_text.delta', 'member': 'MASKED'}"
|
||||
|
||||
|
||||
class _KeysRefusedDict(dict[str, str]):
|
||||
def keys(self) -> KeysView[str]: # pyright: ignore[reportIncompatibleMethodOverride] # rig refuses its keys
|
||||
raise RuntimeError("rig keys refused")
|
||||
|
||||
|
||||
class _ItemRefusedDict(dict[str, str]):
|
||||
def __getitem__(self, key: str) -> str:
|
||||
raise RuntimeError("rig item refused")
|
||||
|
||||
|
||||
class _PrintsOnceDict(dict[str, str]):
|
||||
printed: bool = False
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.printed:
|
||||
raise RuntimeError("rig prints once")
|
||||
self.printed = True
|
||||
return "{'type': 'response.output_text.delta', 'token': 'MASKED'}"
|
||||
|
||||
|
||||
_DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_underlying", "printed"),
|
||||
[
|
||||
(
|
||||
lambda: _MaskedStrHolder(type="response.output_text.delta", member=_RaisingRepr()),
|
||||
"{'type': 'response.output_text.delta', 'member': 'MASKED'}",
|
||||
),
|
||||
(lambda: _KeysRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)),
|
||||
(lambda: _ItemRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)),
|
||||
(
|
||||
lambda: _PrintsOnceDict(type="response.output_text.delta", token="hook-canary"),
|
||||
"{'type': 'response.output_text.delta', 'token': 'MASKED'}",
|
||||
),
|
||||
],
|
||||
ids=["member_repr_raises", "keys_raise", "item_lookup_raises", "str_works_once"],
|
||||
)
|
||||
def test_responses_sse_chunk_sends_a_whole_proxy_it_cannot_read_as_the_text_base_printed_once(
|
||||
make_underlying: Callable[[], Mapping[str, object]], printed: str
|
||||
) -> None:
|
||||
frame: Final = _format_responses_sse_chunk(chunk=MappingProxyType(make_underlying()))
|
||||
|
||||
assert frame == f"data: {printed}\n\n"
|
||||
|
||||
|
||||
class _OwnFormatHolder(dict[str, str]):
|
||||
def __format__(self, format_spec: str) -> str:
|
||||
return "MASKED"
|
||||
|
||||
|
||||
def _reordered() -> OrderedDict[str, int]:
|
||||
ordered: Final = OrderedDict(first=1, second=2)
|
||||
ordered.move_to_end("first")
|
||||
return ordered
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"nested",
|
||||
[
|
||||
_reordered(),
|
||||
Counter(rare=1, common=3),
|
||||
_OwnFormatHolder(token="shown"),
|
||||
_MaskedStrDict(token="shown"),
|
||||
],
|
||||
ids=["reordered_ordered_dict", "counter", "own_format", "own_str"],
|
||||
)
|
||||
def test_responses_sse_chunk_sends_a_nested_mapping_with_the_items_and_order_its_repr_printed(
|
||||
nested: Mapping[str, object],
|
||||
) -> None:
|
||||
frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested})
|
||||
|
||||
assert isinstance(frame, str)
|
||||
shown: Final = json.loads(frame.removeprefix("data: "))["nested"]
|
||||
assert repr(shown) in repr(nested)
|
||||
|
||||
|
||||
class _RigTuple(tuple[object, ...]):
|
||||
pass
|
||||
|
||||
|
||||
class _ItemsHideAliasDict(dict[str, str]):
|
||||
__repr__ = dict.__repr__
|
||||
|
||||
def items(self) -> ItemsView[str, str]: # pyright: ignore[reportIncompatibleMethodOverride] # rig hides an entry
|
||||
return {key: value for key, value in dict.items(self) if key != "h"}.items()
|
||||
|
||||
|
||||
class _ItemsHideRevealingDict(_ItemsHideAliasDict):
|
||||
def __repr__(self) -> str:
|
||||
return dict.__repr__(self)
|
||||
|
||||
|
||||
class _IterHideRevealingList(list[str]):
|
||||
def __repr__(self) -> str:
|
||||
return list.__repr__(self)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return (item for item in list.__iter__(self) if item != "hook-canary")
|
||||
|
||||
|
||||
class _RevealingReprModel(BaseModel, frozen=True):
|
||||
a: int
|
||||
_note: str = PrivateAttr(default="hook-canary")
|
||||
|
|
@ -406,8 +514,19 @@ class _RevealingReprModel(BaseModel, frozen=True):
|
|||
_RigTuple((_MaskedStrModel(token="hook-canary"),)),
|
||||
_HookOutputItem(id="msg_nested", type="message"),
|
||||
_RevealingReprModel(a=1),
|
||||
_ItemsHideAliasDict(k="shown", h="hook-canary"),
|
||||
_ItemsHideRevealingDict(k="shown", h="hook-canary"),
|
||||
_IterHideRevealingList(("shown", "hook-canary")),
|
||||
],
|
||||
ids=[
|
||||
"dict_subclass",
|
||||
"tuple_subclass",
|
||||
"model",
|
||||
"model_with_own_repr",
|
||||
"items_hide_with_dict_repr_alias",
|
||||
"items_hide_with_own_revealing_repr",
|
||||
"iter_hides_with_own_revealing_repr",
|
||||
],
|
||||
ids=["dict_subclass", "tuple_subclass", "model", "model_with_own_repr"],
|
||||
)
|
||||
def test_responses_error_event_sends_a_detail_the_way_base_json_dumps_did(detail: object) -> None:
|
||||
frame: Final = ResponsesStreamErrorState().format_error(
|
||||
|
|
@ -418,6 +537,18 @@ def test_responses_error_event_sends_a_detail_the_way_base_json_dumps_did(detail
|
|||
assert payload["error"]["provider_specific_fields"]["detail"] == json.loads(json.dumps(detail, default=str))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", ["abc", 1.5, True, None], ids=["string", "float", "bool", "null"])
|
||||
def test_responses_hook_error_frame_keeps_its_message_whatever_status_code_it_carries(status_code: object) -> None:
|
||||
hook_frame: Final = json.dumps(
|
||||
{"error": {"message": "rig status", "type": "guardrail_scan_error", "status_code": status_code}}
|
||||
)
|
||||
|
||||
frame: Final = ResponsesStreamErrorState().format_error_message(f"data: {hook_frame}\n\n")
|
||||
|
||||
assert frame.startswith("event: error\n")
|
||||
assert json.loads(frame.split("data: ", 1)[1])["message"] == "rig status"
|
||||
|
||||
|
||||
class _RigColor(Enum):
|
||||
RED = 1
|
||||
|
||||
|
|
@ -1975,6 +2106,52 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina
|
|||
assert "error" not in payloads[-1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("raw_sse_stream", [False, True])
|
||||
@pytest.mark.parametrize("newline", ["\n", "\r\n"])
|
||||
@pytest.mark.parametrize(
|
||||
"event_name,data,terminal",
|
||||
[
|
||||
(
|
||||
"response.output_text.delta",
|
||||
{"type": "response.output_text.delta", "sequence_number": 4, "delta": "hi"},
|
||||
False,
|
||||
),
|
||||
(
|
||||
"response.completed",
|
||||
{"type": "response.completed", "sequence_number": 4, "response": {"id": "resp_raw", "model": "gpt-6"}},
|
||||
True,
|
||||
),
|
||||
("error", {"error": {"message": "denied", "code": "529"}}, True),
|
||||
],
|
||||
)
|
||||
async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failure_only_without_a_terminal(
|
||||
raw_sse_stream: bool, newline: str, event_name: str, data: Mapping[str, object], terminal: bool
|
||||
) -> None:
|
||||
pre_framed: Final = f"event: {event_name}{newline}data: {json.dumps(data)}{newline}{newline}"
|
||||
|
||||
async def upstream() -> AsyncIterator[bytes]:
|
||||
yield pre_framed.encode()
|
||||
raise litellm.APIError(status_code=500, message="Stream closed", llm_provider="openai", model="gpt-6")
|
||||
|
||||
frames: Final = [
|
||||
frame.decode() if isinstance(frame, bytes) else frame
|
||||
async for frame in select_data_generator(
|
||||
response=upstream(),
|
||||
user_api_key_dict=_user_auth(),
|
||||
request_data={"_litellm_raw_sse_stream": True} if raw_sse_stream else {},
|
||||
responses_stream_errors=True,
|
||||
)
|
||||
]
|
||||
|
||||
assert frames[0] == pre_framed
|
||||
assert len(frames) == (1 if terminal else 2)
|
||||
if not terminal:
|
||||
failure: Final = ResponseFailedEvent.model_validate_json(frames[1].split("data: ", 1)[1])
|
||||
assert frames[1].startswith("event: response.failed\n")
|
||||
assert failure.sequence_number == 5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# select_data_generator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1647,6 +1647,25 @@ async def test_a_streamed_read_cut_before_its_terminal_event_ends_in_a_responses
|
|||
assert "before any terminal event" in events[-1]["response"]["error"]["message"]
|
||||
|
||||
|
||||
def _resumed_wire() -> str:
|
||||
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in _resumed_events("resp_read"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_streamed_read_whose_first_event_is_a_named_error_answers_with_its_status(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(_NAMED_ERROR_FRAME)])
|
||||
result, _ = await _read_through_proxy(
|
||||
monkeypatch,
|
||||
{"stream": "true"},
|
||||
httpx.Response(200, content=_resumed_wire(), headers={"Content-Type": "text/event-stream"}),
|
||||
)
|
||||
|
||||
assert result.status_code == 529, result.text
|
||||
assert result.json()["error"]["message"] == "denied"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_plain_read_through_the_proxy_still_answers_json(monkeypatch: pytest.MonkeyPatch):
|
||||
result, upstream_requests = await _read_through_proxy(
|
||||
|
|
|
|||
|
|
@ -2348,11 +2348,13 @@ class TestCommonRequestProcessingHelpers:
|
|||
assert body["error"]["code"] == 403
|
||||
assert body["error"]["message"] == "forbidden"
|
||||
|
||||
async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self):
|
||||
@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
|
||||
async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self, newline: str):
|
||||
error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None}
|
||||
event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error})
|
||||
|
||||
async def responses_stream():
|
||||
yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n"
|
||||
yield f"event: error{newline}data: {event}{newline}{newline}"
|
||||
|
||||
response: Final = await create_response(
|
||||
responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue