diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index e5ea7b38049..c82cd1fd25d 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -4,9 +4,10 @@ import re import time from collections import Counter, OrderedDict, defaultdict from collections.abc import Iterable, Iterator, Mapping, Sequence +from contextlib import suppress from enum import Enum from http import HTTPStatus -from itertools import chain, groupby, zip_longest +from itertools import chain, groupby, islice, zip_longest from types import MappingProxyType from typing import Final @@ -133,6 +134,14 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma return isinstance(value, MappingProxyType) +def _is_defaultdict(value: object) -> TypeIs[defaultdict[object, object]]: # guard-ok: dict items as object + return isinstance(value, defaultdict) + + +def _proxied(proxy: Mapping[object, object]) -> object: + return gc.get_referents(proxy)[0] + + def _repr_prints_its_fields(model: BaseModel) -> bool: kind: Final = type(model) return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__ @@ -187,6 +196,16 @@ def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, obje return dict[object, object].items(value) if isinstance(value, dict) else value.items() +def _live_items(value: Mapping[object, object]) -> Iterator[tuple[object, object]]: + shown = 0 # rebind-ok: a repr that resizes its dict resumes after the items shown, the way dict's repr does + while True: + with suppress(RuntimeError): + for pair in islice(_stored_items(value), shown, None): + shown += 1 + yield pair + return + + def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool: return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored)) @@ -210,14 +229,14 @@ def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, objec if not _repr_prints_every_item(type(value)): return None if isinstance(value, OrderedDict): - return OrderedDict[object, object].items(value) + return tuple(OrderedDict[object, object].items(value)) if isinstance(value, Counter): return Counter[object].most_common(value) - return _stored_items(value) + return _live_items(value) def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None: - mapping: Final[object] = gc.get_referents(proxy)[0] + mapping: Final = _proxied(proxy) if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)): return None return _shown_items(mapping, _BaseForm.PRINTED) @@ -235,12 +254,42 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o return None +def _walked_ids(value: object) -> frozenset[int]: + return frozenset((id(value), id(_proxied(value)))) if _is_proxy(value) else frozenset((id(value),)) + + +def _container_marker(value: object) -> object: + if isinstance(value, list): + return "[...]" + if isinstance(value, tuple): + return "(...)" + if isinstance(value, OrderedDict): + return "..." + if _is_defaultdict(value): + factory: Final = _text(value.default_factory, _BaseForm.PRINTED) + return factory if factory is _DROPPED else f"{type(value).__name__}({factory}, {{...}})" + return "{...}" if _is_dict(value) and not isinstance(value, Counter) else None + + +def _cycle_marker(value: object) -> object: + if not _is_proxy(value): + return _container_marker(value) + inner: Final = _container_marker(_proxied(value)) + return inner if inner is None or inner is _DROPPED else f"mappingproxy({inner})" + + +def _cycle_text(value: object, form: _BaseForm) -> object: + marker: Final = _cycle_marker(value) + return _text(value, form) if marker is None else marker + + def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: if value is None or type(value) in (str, int, float, bool): return value - if id(value) in path: - return _text(value, form) - inner_path: Final = path | frozenset((id(value),)) + walked: Final = _walked_ids(value) + if not walked.isdisjoint(path): + return _cycle_text(value, form) + inner_path: Final = path | walked try: sequence: Final = _shown_sequence(value, form) if sequence is not None: @@ -264,7 +313,8 @@ def _chunk_json(shown: object) -> str | None: def responses_chunk_data(chunk: object) -> str | None: if _is_proxy(chunk): - return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) + expandable: Final = _proxy_items(chunk, _BaseForm.DUMPED) is not None + return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if expandable else None if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)): return None return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset())) @@ -401,20 +451,27 @@ def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool: return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",) +def _block_data(lines: Sequence[str]) -> str: + return "\n".join(_field_values(lines, "data")) + + def _block_event(lines: Sequence[str]) -> _StreamEvent: - parsed: Final = _in_range_sequence(_parsed_event("\n".join(_field_values(lines, "data")))) + parsed: Final = _in_range_sequence(_parsed_event(_block_data(lines))) if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines): return parsed return parsed.model_copy(update={"type": "error"}) +def _blocks(frames: str) -> Iterator[tuple[str, ...]]: + return (tuple(lines) for filled, lines in groupby(SSE_LINE_BREAK.split(frames), key=bool) if filled) + + def _frame_events(frames: str) -> tuple[_StreamEvent, ...]: - blocks: Final = groupby(SSE_LINE_BREAK.split(frames), key=bool) - return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled) + return tuple(_block_event(lines) for lines in _blocks(frames)) -def frame_event_type(frame: str) -> str | None: - return next((event.type for event in _frame_events(frame)), None) +def frame_error_data(frame: str) -> str | None: + return next((_block_data(lines) for lines in _blocks(frame) if _block_event(lines).type == "error"), None) class ResponsesStreamErrorState: diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 2253bfb4bc2..5c040f8353c 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -24,7 +24,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.responses_stream_errors import ( event_error_object, - frame_event_type, + frame_error_data, responses_error_fields, sse_error_object, ) @@ -224,6 +224,7 @@ async def background_streaming_task( None # Will be set by response.completed/failed/incomplete/cancelled ) terminal_error = None + frame_failure: Mapping[str, object] | None = None # rebind-ok: set once by the first error frame _event_to_status: Final[Mapping[str, ResponsesAPIStatus]] = { "response.completed": "completed", "response.failed": "failed", @@ -259,13 +260,20 @@ async def background_streaming_task( if isinstance(chunk, bytes): chunk = chunk.decode("utf-8") + if ( + isinstance(chunk, str) + and frame_failure is None + and (error_data := frame_error_data(chunk)) is not None + ): + frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data))) + if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None: if chunk_data == "[DONE]": break try: event: _StreamEvent = _StreamEventParser.parse(chunk_data) - event_type = frame_event_type(chunk) or event.get("type", "") + event_type = event.get("type", "") # Process different event types based on OpenAI streaming spec if event_type == "response.output_item.added": @@ -351,10 +359,6 @@ async def background_streaming_task( status="in_progress", ) - elif event_type == "error": - terminal_status = "failed" - terminal_error = responses_error_fields(_event_error(event)) - elif event_type in ( "response.completed", "response.failed", @@ -412,8 +416,10 @@ async def background_streaming_task( await flush_state_if_needed(force=True) early_error: Final = _first_chunk_error(response) - final_error: Final = terminal_error if early_error is None else early_error - final_status: Final = "failed" if early_error is not None else terminal_status or "completed" + stream_failure: Final = frame_failure if terminal_status is None else None + failure: Final = stream_failure if early_error is None else early_error + final_error: Final = terminal_error if failure is None else failure + final_status: Final = (terminal_status or "completed") if failure is None else "failed" await polling_handler.update_state( polling_id=polling_id, diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 35646d44b65..9c4f3e37d14 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -19,7 +19,7 @@ import asyncio import functools import itertools import json -from collections import Counter, OrderedDict +from collections import Counter, OrderedDict, defaultdict from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping from datetime import datetime, timezone from enum import Enum @@ -496,6 +496,114 @@ def test_responses_error_event_prints_each_proxy_item_once() -> None: assert json.loads(frame.split("data: ", 1)[1])["error"]["provider_specific_fields"] == {"v": "'***'"} +def _dict_holding_itself(value: str) -> dict[str, object]: + held: Final[dict[str, object]] = {"v": value} + held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself + return held + + +def _defaultdict_holding_itself(value: str) -> defaultdict[str, object]: + held: Final[defaultdict[str, object]] = defaultdict(list, v=value) + held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself + return held + + +def _ordered_dict_holding_itself(value: str) -> OrderedDict[str, object]: + held: Final[OrderedDict[str, object]] = OrderedDict(v=value) + held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself + return held + + +def _list_holding_itself(value: str) -> list[object]: + held: Final[list[object]] = [value] + held.append(held) # mutable-ok: a self-reference needs the list to exist before it can hold itself + return held + + +def _proxy_whose_mapping_holds_it(value: str) -> Mapping[str, object]: + held: Final[dict[str, object]] = {"type": "response.output_text.delta", "v": value} + held["me"] = MappingProxyType(held) # mutable-ok: a self-reference needs the dict to exist before it can hold itself + return MappingProxyType(held) + + +_SELF_HOLDING_PLACEMENTS: Final[tuple[tuple[Callable[[str], object], Mapping[str, object]], ...]] = ( + ( + lambda value: {"type": "response.output_text.delta", "meta": MappingProxyType(_dict_holding_itself(value))}, + {"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "{...}"}}, + ), + ( + lambda value: {"type": "response.output_text.delta", "meta": _dict_holding_itself(value)}, + {"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "{...}"}}, + ), + ( + lambda value: {"type": "response.output_text.delta", "meta": _defaultdict_holding_itself(value)}, + {"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "defaultdict(, {...})"}}, + ), + ( + lambda value: {"type": "response.output_text.delta", "meta": _ordered_dict_holding_itself(value)}, + {"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "..."}}, + ), + ( + lambda value: {"type": "response.output_text.delta", "meta": _list_holding_itself(value)}, + {"type": "response.output_text.delta", "meta": ["'***'", "[...]"]}, + ), + ( + _proxy_whose_mapping_holds_it, + {"type": "response.output_text.delta", "v": "'***'", "me": "mappingproxy({...})"}, + ), +) + + +@pytest.mark.parametrize( + ("place", "expected"), + _SELF_HOLDING_PLACEMENTS, + ids=["proxy_over_dict", "dict", "defaultdict", "ordered_dict", "list", "proxy_held_by_its_mapping"], +) +def test_responses_sse_chunk_prints_a_container_that_holds_itself_as_its_repr_marker( + place: Callable[[str], object], expected: Mapping[str, object] +) -> None: + frame: Final = _format_responses_sse_chunk(chunk=place(_AlternatingReprString("hook-canary"))) + + assert isinstance(frame, str) + assert "hook-canary" not in frame + assert json.loads(frame.removeprefix("data: ")) == expected + + +class _ParentGrower: + def __init__(self, parent: dict[str, object]) -> None: + self.parent: Final = parent + + def __repr__(self) -> str: + self.parent["added"] = 1 # mutable-ok: the hook value's repr grows the dict that holds it + return "_ParentGrower()" + + +def test_responses_sse_chunk_keeps_printing_a_dict_whose_item_repr_adds_a_key() -> None: + parent: Final[dict[str, object]] = {"v": _AlternatingReprString("hook-canary")} + parent["m"] = _ParentGrower(parent) # mutable-ok: the grower needs the dict it adds a key to + + frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "meta": parent}) + + assert isinstance(frame, str) + assert "hook-canary" not in frame + assert json.loads(frame.removeprefix("data: "))["meta"] == {"v": "'***'", "m": "_ParentGrower()", "added": 1} + + +class _StrFailsOnceDict(dict[str, str]): + failed: bool = False + + def __str__(self) -> str: + if self.failed: + return dict.__repr__(self) + self.failed = True + raise RuntimeError("rig str fails on its first call") + + +def test_responses_sse_chunk_raises_when_a_whole_proxy_fails_its_only_print() -> None: + with pytest.raises(RuntimeError, match="rig str fails on its first call"): + _format_responses_sse_chunk(chunk=MappingProxyType(_StrFailsOnceDict(_DELTA_STORAGE))) + + @pytest.mark.parametrize( ("make_underlying", "printed"), [ @@ -775,10 +883,10 @@ def _self_referencing_chunk() -> Mapping[str, object]: return chunk -def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_text() -> None: +def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_repr_marker() -> None: nested: Final = _self_referencing_chunk() - assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": repr(nested)} + assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": "{...}"} class _UnprintableValue: diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 9d631db6a00..d3d6008743a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1612,6 +1612,74 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["status"] == "failed" assert final_call.kwargs["error"] == expected_error + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("chunks", "expected_status", "expected_error", "expected_usage"), + [ + ( + ( + 'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n' + 'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n', + ), + "failed", + {"code": "invalid_request_error", "message": "output blocked by policy"}, + None, + ), + ( + ( + 'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n', + 'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n', + ), + "completed", + None, + {"input_tokens": 3}, + ), + ( + ( + 'data:{"type": "response.output_text.delta", "delta": "hi"}\n\n' + 'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n', + ), + "completed", + None, + {"input_tokens": 3}, + ), + ], + ids=["error_frame_after_a_delta", "error_frame_then_completed", "unspaced_delta_then_completed"], + ) + async def test_a_poll_stores_what_its_stream_ended_with( + self, + chunks: tuple[str, ...], + expected_status: str, + expected_error: Mapping[str, str] | None, + expected_usage: Mapping[str, int] | None, + ): + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + async def _body_iterator(): + yield b'data: {"type": "response.in_progress"}\n\n' + for chunk in chunks: + yield chunk.encode() + + mock_response: Final = Mock() + mock_response.body_iterator = _body_iterator() + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_stream_end", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == expected_status + assert final_call.kwargs["error"] == expected_error + assert final_call.kwargs["usage"] == expected_usage + @pytest.mark.asyncio @pytest.mark.parametrize( ("status_code", "body", "expected_error"),