From 8ac651b4865b9feee8a7332917e3874ae33725ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:56:05 -0700 Subject: [PATCH] fix(proxy): frame streamed Responses reads and send unreadable hook values as base did Track pre-framed and raw SSE frames so a late failure never follows a terminal event, accept CRLF and CR line breaks in a first named error, frame GET /v1/responses/{id}?stream=true as Responses events, print a whole hook proxy chunk once and fall back to its text, ignore a non-integer status_code in a hook error frame, read a late-block dict subclass through items() when that view differs from its storage, and keep the order and inline form base's repr printed for nested mappings --- litellm/proxy/common_request_processing.py | 8 +- .../common_utils/responses_stream_errors.py | 166 ++++++++++++---- litellm/proxy/proxy_server.py | 21 +- .../proxy/response_api_endpoints/endpoints.py | 1 + .../proxy_server/test_streaming_helpers.py | 181 +++++++++++++++++- .../response_api_endpoints/test_endpoints.py | 19 ++ .../proxy/test_common_request_processing.py | 6 +- 7 files changed, 354 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 15c7325eaf8..55bba0749b5 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -120,7 +120,11 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_type, relayed_provider_specific_fields, ) -from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, sse_error_object +from litellm.proxy.common_utils.responses_stream_errors import ( + SSE_LINE_BREAK, + ResponsesStreamErrorState, + sse_error_object, +) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, SSE_STREAM_START_TAIL, @@ -1019,7 +1023,7 @@ def _sse_data_payload(event_line: str | bytes) -> str | None: def _responses_event_data_payload(event_line: str | bytes) -> str | None: - match _sse_text(event_line).strip().split("\n"): + match SSE_LINE_BREAK.split(_sse_text(event_line).strip()): case ["event: error", data_line] if data_line.startswith("data: "): return data_line.removeprefix("data: ").strip() case _: diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 951650543a7..aab49b5b031 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,9 +1,11 @@ import json +import re import time from collections import Counter, OrderedDict, defaultdict from collections.abc import Iterable, Iterator, Mapping, Sequence from enum import Enum from http import HTTPStatus +from itertools import chain, groupby, zip_longest from types import MappingProxyType from typing import Final @@ -41,6 +43,11 @@ class _FailureDetails(LiteLLMBaseModel): type: str | None = None status_code: int | None = None + @field_validator("status_code", mode="before") + @classmethod + def normalize_status_code(cls, value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + @field_validator("message", mode="before") @classmethod def normalize_message(cls, value: object) -> str | None: @@ -78,25 +85,35 @@ class _BaseForm(Enum): DUMPED = "dumped" -def _prints_every_item(kind: type[object], form: _BaseForm) -> bool: - repr_owner: Final = next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object) - if form is _BaseForm.DUMPED: - return kind.__str__ is object.__str__ and repr_owner in (dict, list, tuple, OrderedDict, defaultdict, Counter) - return ( - kind.__str__ is object.__str__ - and kind.__format__ is object.__format__ - and (kind in (OrderedDict, Counter) or repr_owner in (dict, list, tuple, defaultdict)) +def _repr_owner(kind: type[object]) -> type[object]: + return next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object) + + +def _repr_prints_every_item(kind: type[object]) -> bool: + return kind in (OrderedDict, Counter) or _repr_owner(kind) in (dict, list, tuple, defaultdict) + + +def _format_prints_every_item(kind: type[object]) -> bool: + return kind.__str__ is object.__str__ and kind.__format__ is object.__format__ and _repr_prints_every_item(kind) + + +def _str_prints_every_item(kind: type[object]) -> bool: + return kind.__str__ is object.__str__ and _repr_owner(kind) in ( + dict, + list, + tuple, + OrderedDict, + defaultdict, + Counter, ) -def _is_shown_dict(value: object, form: _BaseForm) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object - kind: Final = type(value) - return isinstance(value, dict) and _prints_every_item(kind, form) +def _is_dict(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object + return isinstance(value, dict) -def _is_shown_sequence(value: object, form: _BaseForm) -> TypeIs[Sequence[object]]: # guard-ok: items as object - kind: Final = type(value) - return isinstance(value, (list, tuple)) and _prints_every_item(kind, form) +def _is_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: items as object + return isinstance(value, (list, tuple)) def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed @@ -142,22 +159,45 @@ def _json_key(key: object, form: _BaseForm) -> object: return key if key is None or type(key) in (str, int, float, bool) else _text(key, form) -def _shown_sequence(value: object, form: _BaseForm) -> Iterator[object] | None: - if not _is_shown_sequence(value, form): - return None - if form is _BaseForm.PRINTED and isinstance(value, list): +def _stored_sequence(value: Sequence[object]) -> Iterator[object]: + if isinstance(value, list): return list[object].__iter__(value) - if form is _BaseForm.PRINTED and isinstance(value, tuple): + if isinstance(value, tuple): return tuple[object, ...].__iter__(value) return iter(value) -def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None: - if not _is_shown_dict(value, form): +def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, object]]: + return dict[object, object].items(value) if isinstance(value, dict) else value.items() + + +def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool: + return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored)) + + +def _shown_sequence(value: object, form: _BaseForm) -> Iterable[object] | None: + if not _is_sequence(value): return None - if form is _BaseForm.PRINTED and isinstance(value, dict): - return dict[object, object].items(value) - return value.items() + if form is _BaseForm.PRINTED: + return _stored_sequence(value) if _repr_prints_every_item(type(value)) else None + dumped: Final = tuple(value) + return dumped if _dumps_shown(type(value), dumped, _stored_sequence(value)) else None + + +def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None: + if not _is_dict(value): + return None + if form is _BaseForm.DUMPED: + dumped: Final = tuple(value.items()) + stored: Final = chain.from_iterable(_stored_items(value)) + return dumped if _dumps_shown(type(value), chain.from_iterable(dumped), stored) else None + if not _repr_prints_every_item(type(value)): + return None + if isinstance(value, OrderedDict): + return OrderedDict[object, object].items(value) + if isinstance(value, Counter): + return Counter[object].most_common(value) + return _stored_items(value) def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None: @@ -187,19 +227,36 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: if expansion is None: return _text(value, form) items, inner = expansion - pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items] - return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED} + return _items_json(items, inner, inner_path) except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit return _text(value, form) -def responses_chunk_json(chunk: object) -> str | None: - if _is_proxy(chunk) and _printed_proxy_items(chunk, format(chunk, "")) is None: - return None - if not _is_shown_dict(chunk, _BaseForm.PRINTED) and not _is_proxy(chunk): +def _items_json( + items: Iterable[tuple[object, object]], form: _BaseForm, path: frozenset[int] +) -> Mapping[object, object]: + pairs: Final = [(_json_key(key, form), _json_safe(item, form, path)) for key, item in items] + return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED} + + +def _proxy_chunk_data(proxy: Mapping[object, object]) -> str: + printed: Final = format(proxy, "") + try: + items: Final = _printed_proxy_items(proxy, printed) + return printed if items is None else json.dumps(_items_json(items, _BaseForm.PRINTED, frozenset((id(proxy),)))) + except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands + return printed + + +def responses_chunk_data(chunk: object) -> str | None: + if _is_proxy(chunk): + return _proxy_chunk_data(chunk) + if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)): return None shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset()) - return None if isinstance(shown, str) or shown is _DROPPED else json.dumps(shown) + if isinstance(shown, str): + return shown + return None if shown is _DROPPED else json.dumps(shown) def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]: @@ -305,6 +362,35 @@ def _error_event( } +SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n") +_TERMINAL_EVENT_TYPES: Final = frozenset(("response.completed", "response.failed", "response.incomplete", "error")) + + +def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]: + prefix: Final = f"{name}:" + return tuple(line.removeprefix(prefix).removeprefix(" ") for line in lines if line.startswith(prefix)) + + +def _parsed_event(data: str) -> _StreamEvent: + try: + return _StreamEvent.model_validate_json(data) + except ValidationError: + return _StreamEvent() + + +def _block_event(lines: Sequence[str]) -> _StreamEvent: + named: Final = _field_values(lines, "event") + parsed: Final = _parsed_event("\n".join(_field_values(lines, "data"))) + if parsed.type in _TERMINAL_EVENT_TYPES or not named: + return parsed + return parsed.model_copy(update={"type": named[-1]}) + + +def _frame_events(frames: str) -> tuple[_StreamEvent, ...]: + blocks: Final = groupby(SSE_LINE_BREAK.split(frames), key=bool) + return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled) + + class ResponsesStreamErrorState: def __init__(self) -> None: self.response_id: str | None = None @@ -317,10 +403,7 @@ class ResponsesStreamErrorState: def observe_chunk(self, chunk: object) -> None: self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None - def mark_emitted(self, frame: str | bytes) -> str | bytes: - event: Final = self._pending_event - if event is None: - return frame + def _record(self, event: _StreamEvent) -> None: if event.sequence_number is not None: self.sequence_number = max(self.sequence_number, event.sequence_number) if event.response is not None: @@ -328,8 +411,19 @@ class ResponsesStreamErrorState: self.model = event.response.model or self.model if event.response.created_at is not None: self.created_at = event.response.created_at - if event.type in ("response.completed", "response.failed", "response.incomplete", "error"): + if event.type in _TERMINAL_EVENT_TYPES: self.terminal_emitted = True + + def observe_frames(self, frames: str) -> str: + for event in _frame_events(frames): + self._record(event) + return frames + + def mark_emitted(self, frame: str | bytes) -> str | bytes: + event: Final = self._pending_event + if event is None: + return frame + self._record(event) if not event.type or "\n" in event.type or "\r" in event.type: return frame prefix: Final = f"event: {event.type}\n" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e50bb9d2891..5aee3ec3541 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -554,7 +554,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( from litellm.proxy.common_utils.proxy_state import ProxyState from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob -from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_json +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_data from litellm.proxy.common_utils.scheduled_job_stagger import ( apply_scheduled_job_stagger, attach_job_timing_logger, @@ -10267,10 +10267,14 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: def _format_responses_sse_chunk(chunk: object) -> str | bytes: - data: Final = responses_chunk_json(chunk) + data: Final = responses_chunk_data(chunk) return _format_streaming_sse_chunk(chunk=chunk) if data is None else f"data: {data}\n\n" +def _observed_frames(frames: str, error_state: ResponsesStreamErrorState | None) -> str: + return frames if error_state is None else error_state.observe_frames(frames) + + _SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r") _OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n" _MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024 @@ -10647,12 +10651,14 @@ async def async_data_generator( frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer) if frame is None: break - yield frame + yield _observed_frames(frame, error_state) if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS: raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter") raw_passthrough = True elif chunk.startswith(("data:", "event:", ":")): - yield (chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n") + yield _observed_frames( + chunk if chunk.endswith(_SSE_FRAME_DELIMITERS) else chunk + "\n\n", error_state + ) raw_passthrough = True elif isinstance(chunk, str) and is_raw_sse_stream: raw_sse_buffer += chunk @@ -10660,7 +10666,7 @@ async def async_data_generator( frame, raw_sse_buffer = _pop_complete_sse_frame(raw_sse_buffer) if frame is None: break - yield frame + yield _observed_frames(frame, error_state) if len(raw_sse_buffer) > _MAX_RAW_SSE_BUFFER_CHARS: raise ValueError("Raw SSE stream exceeded maximum buffered size without a frame delimiter") raw_passthrough = True @@ -10694,7 +10700,10 @@ async def async_data_generator( ProxyLogging.fire_deferred_stream_logging(request_data) if raw_sse_buffer: - yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n") + yield _observed_frames( + raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n", + error_state, + ) if error_message is not None: yield error_message if error_state is None else error_state.format_error_message(error_message) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index c2e84f585ca..edba3e7c056 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -908,6 +908,7 @@ async def get_response( response_id, read_settles_billing, ), + sse_framing=RESPONSES_SSE_FRAMING, model=None, user_model=user_model, user_temperature=user_temperature, diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 6deaac8983a..7a1ee706461 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -19,7 +19,7 @@ import asyncio import functools import json from collections import Counter, OrderedDict -from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping from datetime import datetime, timezone from enum import Enum from http import HTTPStatus @@ -387,10 +387,118 @@ def test_responses_sse_chunk_sends_a_whole_chunk_as_the_text_its_printer_gave(ch assert "hook-canary" not in frame +class _RaisingRepr: + def __repr__(self) -> str: + raise RuntimeError("rig repr failure") + + +class _MaskedStrHolder(dict[str, object]): + def __str__(self) -> str: + return "{'type': 'response.output_text.delta', 'member': 'MASKED'}" + + +class _KeysRefusedDict(dict[str, str]): + def keys(self) -> KeysView[str]: # pyright: ignore[reportIncompatibleMethodOverride] # rig refuses its keys + raise RuntimeError("rig keys refused") + + +class _ItemRefusedDict(dict[str, str]): + def __getitem__(self, key: str) -> str: + raise RuntimeError("rig item refused") + + +class _PrintsOnceDict(dict[str, str]): + printed: bool = False + + def __str__(self) -> str: + if self.printed: + raise RuntimeError("rig prints once") + self.printed = True + return "{'type': 'response.output_text.delta', 'token': 'MASKED'}" + + +_DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} + + +@pytest.mark.parametrize( + ("make_underlying", "printed"), + [ + ( + lambda: _MaskedStrHolder(type="response.output_text.delta", member=_RaisingRepr()), + "{'type': 'response.output_text.delta', 'member': 'MASKED'}", + ), + (lambda: _KeysRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)), + (lambda: _ItemRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)), + ( + lambda: _PrintsOnceDict(type="response.output_text.delta", token="hook-canary"), + "{'type': 'response.output_text.delta', 'token': 'MASKED'}", + ), + ], + ids=["member_repr_raises", "keys_raise", "item_lookup_raises", "str_works_once"], +) +def test_responses_sse_chunk_sends_a_whole_proxy_it_cannot_read_as_the_text_base_printed_once( + make_underlying: Callable[[], Mapping[str, object]], printed: str +) -> None: + frame: Final = _format_responses_sse_chunk(chunk=MappingProxyType(make_underlying())) + + assert frame == f"data: {printed}\n\n" + + +class _OwnFormatHolder(dict[str, str]): + def __format__(self, format_spec: str) -> str: + return "MASKED" + + +def _reordered() -> OrderedDict[str, int]: + ordered: Final = OrderedDict(first=1, second=2) + ordered.move_to_end("first") + return ordered + + +@pytest.mark.parametrize( + "nested", + [ + _reordered(), + Counter(rare=1, common=3), + _OwnFormatHolder(token="shown"), + _MaskedStrDict(token="shown"), + ], + ids=["reordered_ordered_dict", "counter", "own_format", "own_str"], +) +def test_responses_sse_chunk_sends_a_nested_mapping_with_the_items_and_order_its_repr_printed( + nested: Mapping[str, object], +) -> None: + frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested}) + + assert isinstance(frame, str) + shown: Final = json.loads(frame.removeprefix("data: "))["nested"] + assert repr(shown) in repr(nested) + + class _RigTuple(tuple[object, ...]): pass +class _ItemsHideAliasDict(dict[str, str]): + __repr__ = dict.__repr__ + + def items(self) -> ItemsView[str, str]: # pyright: ignore[reportIncompatibleMethodOverride] # rig hides an entry + return {key: value for key, value in dict.items(self) if key != "h"}.items() + + +class _ItemsHideRevealingDict(_ItemsHideAliasDict): + def __repr__(self) -> str: + return dict.__repr__(self) + + +class _IterHideRevealingList(list[str]): + def __repr__(self) -> str: + return list.__repr__(self) + + def __iter__(self) -> Iterator[str]: + return (item for item in list.__iter__(self) if item != "hook-canary") + + class _RevealingReprModel(BaseModel, frozen=True): a: int _note: str = PrivateAttr(default="hook-canary") @@ -406,8 +514,19 @@ class _RevealingReprModel(BaseModel, frozen=True): _RigTuple((_MaskedStrModel(token="hook-canary"),)), _HookOutputItem(id="msg_nested", type="message"), _RevealingReprModel(a=1), + _ItemsHideAliasDict(k="shown", h="hook-canary"), + _ItemsHideRevealingDict(k="shown", h="hook-canary"), + _IterHideRevealingList(("shown", "hook-canary")), + ], + ids=[ + "dict_subclass", + "tuple_subclass", + "model", + "model_with_own_repr", + "items_hide_with_dict_repr_alias", + "items_hide_with_own_revealing_repr", + "iter_hides_with_own_revealing_repr", ], - ids=["dict_subclass", "tuple_subclass", "model", "model_with_own_repr"], ) def test_responses_error_event_sends_a_detail_the_way_base_json_dumps_did(detail: object) -> None: frame: Final = ResponsesStreamErrorState().format_error( @@ -418,6 +537,18 @@ def test_responses_error_event_sends_a_detail_the_way_base_json_dumps_did(detail assert payload["error"]["provider_specific_fields"]["detail"] == json.loads(json.dumps(detail, default=str)) +@pytest.mark.parametrize("status_code", ["abc", 1.5, True, None], ids=["string", "float", "bool", "null"]) +def test_responses_hook_error_frame_keeps_its_message_whatever_status_code_it_carries(status_code: object) -> None: + hook_frame: Final = json.dumps( + {"error": {"message": "rig status", "type": "guardrail_scan_error", "status_code": status_code}} + ) + + frame: Final = ResponsesStreamErrorState().format_error_message(f"data: {hook_frame}\n\n") + + assert frame.startswith("event: error\n") + assert json.loads(frame.split("data: ", 1)[1])["message"] == "rig status" + + class _RigColor(Enum): RED = 1 @@ -1975,6 +2106,52 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina assert "error" not in payloads[-1] +@pytest.mark.asyncio +@pytest.mark.parametrize("raw_sse_stream", [False, True]) +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +@pytest.mark.parametrize( + "event_name,data,terminal", + [ + ( + "response.output_text.delta", + {"type": "response.output_text.delta", "sequence_number": 4, "delta": "hi"}, + False, + ), + ( + "response.completed", + {"type": "response.completed", "sequence_number": 4, "response": {"id": "resp_raw", "model": "gpt-6"}}, + True, + ), + ("error", {"error": {"message": "denied", "code": "529"}}, True), + ], +) +async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failure_only_without_a_terminal( + raw_sse_stream: bool, newline: str, event_name: str, data: Mapping[str, object], terminal: bool +) -> None: + pre_framed: Final = f"event: {event_name}{newline}data: {json.dumps(data)}{newline}{newline}" + + async def upstream() -> AsyncIterator[bytes]: + yield pre_framed.encode() + raise litellm.APIError(status_code=500, message="Stream closed", llm_provider="openai", model="gpt-6") + + frames: Final = [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), + user_api_key_dict=_user_auth(), + request_data={"_litellm_raw_sse_stream": True} if raw_sse_stream else {}, + responses_stream_errors=True, + ) + ] + + assert frames[0] == pre_framed + assert len(frames) == (1 if terminal else 2) + if not terminal: + failure: Final = ResponseFailedEvent.model_validate_json(frames[1].split("data: ", 1)[1]) + assert frames[1].startswith("event: response.failed\n") + assert failure.sequence_number == 5 + + # --------------------------------------------------------------------------- # select_data_generator # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index e32ab1840ad..303a284cc2a 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -1647,6 +1647,25 @@ async def test_a_streamed_read_cut_before_its_terminal_event_ends_in_a_responses assert "before any terminal event" in events[-1]["response"]["error"]["message"] +def _resumed_wire() -> str: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in _resumed_events("resp_read")) + + +@pytest.mark.asyncio +async def test_a_streamed_read_whose_first_event_is_a_named_error_answers_with_its_status( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(_NAMED_ERROR_FRAME)]) + result, _ = await _read_through_proxy( + monkeypatch, + {"stream": "true"}, + httpx.Response(200, content=_resumed_wire(), headers={"Content-Type": "text/event-stream"}), + ) + + assert result.status_code == 529, result.text + assert result.json()["error"]["message"] == "denied" + + @pytest.mark.asyncio async def test_a_plain_read_through_the_proxy_still_answers_json(monkeypatch: pytest.MonkeyPatch): result, upstream_requests = await _read_through_proxy( diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index ccffb82b1f6..f573e5349db 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -2348,11 +2348,13 @@ class TestCommonRequestProcessingHelpers: assert body["error"]["code"] == 403 assert body["error"]["message"] == "forbidden" - async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self): + @pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) + async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self, newline: str): error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None} + event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}) async def responses_stream(): - yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" + yield f"event: error{newline}data: {event}{newline}{newline}" response: Final = await create_response( responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING