diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index be935561e28..e5ea7b38049 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,3 +1,4 @@ +import gc import json import re import time @@ -215,7 +216,17 @@ def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, objec return _stored_items(value) +def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None: + mapping: Final[object] = gc.get_referents(proxy)[0] + if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)): + return None + return _shown_items(mapping, _BaseForm.PRINTED) + + def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None: + if _is_proxy(value): + proxy_items: Final = _proxy_items(value, form) + return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED) shown_items: Final = _shown_items(value, form) if shown_items is not None: return shown_items, form @@ -230,8 +241,6 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: if id(value) in path: return _text(value, form) inner_path: Final = path | frozenset((id(value),)) - if _is_proxy(value): - return _proxy_value(value, form, inner_path) try: sequence: Final = _shown_sequence(value, form) if sequence is not None: @@ -247,45 +256,18 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: return _text(value, form) -def _printed_proxy_value(proxy: Mapping[object, object], printed: str, path: frozenset[int]) -> object: - snapshot: Final = dict(proxy) - expanded: Final = _json_safe(snapshot, _BaseForm.PRINTED, path) - shown: Final = repr(snapshot) - return expanded if printed in (shown, f"mappingproxy({shown})") else printed - - -def _proxy_value(proxy: Mapping[object, object], form: _BaseForm, path: frozenset[int]) -> object: - printed: Final = _text(proxy, form) - if not isinstance(printed, str): - return printed - try: - return _printed_proxy_value(proxy, printed, path) - except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands - return printed - - -def _proxy_chunk_json(proxy: Mapping[object, object], printed: str) -> str: - shown: Final = _printed_proxy_value(proxy, printed, frozenset((id(proxy),))) - return json.dumps(shown) if isinstance(shown, dict) else printed - - -def _proxy_chunk_data(proxy: Mapping[object, object]) -> str: - printed: Final = format(proxy, "") - try: - return _proxy_chunk_json(proxy, printed) - except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands - return printed +def _chunk_json(shown: object) -> str | None: + if isinstance(shown, str): + return shown + return None if shown is _DROPPED else json.dumps(shown) def responses_chunk_data(chunk: object) -> str | None: if _is_proxy(chunk): - return _proxy_chunk_data(chunk) + return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)): return None - shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset()) - if isinstance(shown, str): - return shown - return None if shown is _DROPPED else json.dumps(shown) + return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset())) def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]: @@ -431,6 +413,10 @@ def _frame_events(frames: str) -> tuple[_StreamEvent, ...]: return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled) +def frame_event_type(frame: str) -> str | None: + return next((event.type for event in _frame_events(frame)), None) + + class ResponsesStreamErrorState: def __init__(self) -> None: self.response_id: str | None = None diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 99c988317bc..2253bfb4bc2 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -24,6 +24,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.responses_stream_errors import ( event_error_object, + frame_event_type, responses_error_fields, sse_error_object, ) @@ -264,7 +265,7 @@ async def background_streaming_task( try: event: _StreamEvent = _StreamEventParser.parse(chunk_data) - event_type = event.get("type", "") + event_type = frame_event_type(chunk) or event.get("type", "") # Process different event types based on OpenAI streaming spec if event_type == "response.output_item.added": diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 69789ce30d9..35646d44b65 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -361,12 +361,12 @@ class _OnceMaskedReprString(str): "wrap,printed", [ (lambda value: value, "'***'"), - (lambda value: MappingProxyType({"v": value}), "mappingproxy({'v': '***'})"), + (lambda value: MappingProxyType({"v": value}), {"v": "'***'"}), ], ids=["dict_value", "nested_mapping_proxy"], ) def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it( - wrap: Callable[[str], object], printed: str + wrap: Callable[[str], object], printed: object ) -> None: frame: Final = _format_responses_sse_chunk( chunk={"type": "response.output_text.delta", "v": wrap(_OnceMaskedReprString("hook-canary"))} @@ -449,6 +449,53 @@ class _PrintsOnceDict(dict[str, str]): _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} +class _AlternatingReprString(str): + def __init__(self, value: str) -> None: + super().__init__() + self.reprs: Final = itertools.count() + + def __repr__(self) -> str: + return "'***'" if next(self.reprs) % 2 == 0 else super().__repr__() + + +class _ReprOnceString(str): + def __init__(self, value: str) -> None: + super().__init__() + self.reprs: Final = itertools.count() + + def __repr__(self) -> str: + if next(self.reprs): + raise RuntimeError("rig repr works once") + return super().__repr__() + + +_PROXY_PLACEMENTS: Final[tuple[Callable[[str], object], ...]] = ( + lambda value: {"type": "response.output_text.delta", "meta": MappingProxyType({"v": value})}, + lambda value: MappingProxyType({"type": "response.output_text.delta", "v": value}), + lambda value: {"type": "response.output_text.delta", "meta": [MappingProxyType({"v": value})]}, +) + + +@pytest.mark.parametrize("place", _PROXY_PLACEMENTS, ids=["nested_proxy", "whole_proxy", "proxy_in_list"]) +def test_responses_sse_chunk_prints_each_proxy_item_once(place: Callable[[str], object]) -> None: + masked: Final = _format_responses_sse_chunk(chunk=place(_AlternatingReprString("hook-canary"))) + shown: Final = _format_responses_sse_chunk(chunk=place(_ReprOnceString("hook-value"))) + + assert isinstance(masked, str) + assert isinstance(shown, str) + assert "'***'" in masked + assert "hook-canary" not in masked + assert "hook-value" in shown + + +def test_responses_error_event_prints_each_proxy_item_once() -> None: + frame: Final = ResponsesStreamErrorState().format_error( + {"message": "blocked", "provider_specific_fields": MappingProxyType({"v": _AlternatingReprString("hook-canary")})} + ) + + assert json.loads(frame.split("data: ", 1)[1])["error"]["provider_specific_fields"] == {"v": "'***'"} + + @pytest.mark.parametrize( ("make_underlying", "printed"), [ @@ -456,8 +503,8 @@ _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} lambda: _MaskedStrHolder(type="response.output_text.delta", member=_RaisingRepr()), "{'type': 'response.output_text.delta', 'member': 'MASKED'}", ), - (lambda: _KeysRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)), - (lambda: _ItemRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)), + (lambda: _KeysRefusedDict(_DELTA_STORAGE), json.dumps(_DELTA_STORAGE)), + (lambda: _ItemRefusedDict(_DELTA_STORAGE), json.dumps(_DELTA_STORAGE)), ( lambda: _PrintsOnceDict(type="response.output_text.delta", token="hook-canary"), "{'type': 'response.output_text.delta', 'token': 'MASKED'}", @@ -465,7 +512,7 @@ _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} ], ids=["member_repr_raises", "keys_raise", "item_lookup_raises", "str_works_once"], ) -def test_responses_sse_chunk_sends_a_whole_proxy_it_cannot_read_as_the_text_base_printed_once( +def test_responses_sse_chunk_sends_a_whole_proxy_as_its_mapping_printed_once( make_underlying: Callable[[], Mapping[str, object]], printed: str ) -> None: frame: Final = _format_responses_sse_chunk(chunk=MappingProxyType(make_underlying())) @@ -773,8 +820,11 @@ def test_responses_error_event_keeps_a_detail_nested_past_the_recursion_limit() assert "'leaf': 1" in frame -def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json() -> None: - nested: Final = functools.reduce(lambda inner, _: {"x": inner}, range(700), {"leaf": 1}) +@pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) +def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json( + mapping: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> None: + nested: Final = functools.reduce(lambda inner, _: mapping({"x": inner}), range(700), mapping({"leaf": 1})) chunk_frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested}) error_frame: Final = ResponsesStreamErrorState().format_error( diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 1aa998db19f..9d631db6a00 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1558,23 +1558,38 @@ class TestBackgroundStreamingTerminalEvents: @pytest.mark.asyncio @pytest.mark.parametrize( - ("nested_error", "expected_error"), + ("data_type", "nested_error", "expected_error"), [ ( + {"type": "error"}, {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"}, {"code": "invalid_request_error", "message": "output blocked by policy"}, ), - ("output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}), + ({"type": "error"}, "output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}), + ( + {}, + {"code": "400", "message": "output blocked by policy"}, + {"code": "invalid_request_error", "message": "output blocked by policy"}, + ), + ( + {"type": "response.output_text.delta"}, + {"code": "400", "message": "output blocked by policy"}, + {"code": "invalid_request_error", "message": "output blocked by policy"}, + ), ], + ids=["typed_object", "typed_string", "untyped_object", "delta_typed_object"], ) async def test_a_nested_only_error_event_keeps_its_reason( - self, nested_error: Mapping[str, str] | str, expected_error: Mapping[str, str] + self, + data_type: Mapping[str, str], + nested_error: Mapping[str, str] | str, + expected_error: Mapping[str, str], ): from litellm.proxy.response_polling.background_streaming import ( background_streaming_task, ) - error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error} + error_event: Final = {**data_type, "sequence_number": 9, "error": nested_error} async def _body_iterator(): yield b'data: {"type": "response.in_progress"}\n\n'