From 7add5dce766c43706e212ad665ccf2c9bf9663d5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:32:41 -0700 Subject: [PATCH] fix(proxy): print each read-only mapping item once and poll framed error events as failed A MappingProxyType in a hook chunk or error detail now expands from the mapping it wraps, so each item is printed once, the way base printed it. A value whose repr changes between calls stays masked, a value whose repr works once is kept, and deep nesting keeps one stack frame per level The background reader takes each frame's type from the same terminal rule the stream uses, so an event: error frame with an error object polls failed even when its data names another type or none --- .../common_utils/responses_stream_errors.py | 56 ++++++---------- .../response_polling/background_streaming.py | 3 +- .../proxy_server/test_streaming_helpers.py | 64 +++++++++++++++++-- .../test_response_polling_handler.py | 23 +++++-- 4 files changed, 99 insertions(+), 47 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index be935561e28..e5ea7b38049 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,3 +1,4 @@ +import gc import json import re import time @@ -215,7 +216,17 @@ def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, objec return _stored_items(value) +def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None: + mapping: Final[object] = gc.get_referents(proxy)[0] + if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)): + return None + return _shown_items(mapping, _BaseForm.PRINTED) + + def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None: + if _is_proxy(value): + proxy_items: Final = _proxy_items(value, form) + return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED) shown_items: Final = _shown_items(value, form) if shown_items is not None: return shown_items, form @@ -230,8 +241,6 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: if id(value) in path: return _text(value, form) inner_path: Final = path | frozenset((id(value),)) - if _is_proxy(value): - return _proxy_value(value, form, inner_path) try: sequence: Final = _shown_sequence(value, form) if sequence is not None: @@ -247,45 +256,18 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: return _text(value, form) -def _printed_proxy_value(proxy: Mapping[object, object], printed: str, path: frozenset[int]) -> object: - snapshot: Final = dict(proxy) - expanded: Final = _json_safe(snapshot, _BaseForm.PRINTED, path) - shown: Final = repr(snapshot) - return expanded if printed in (shown, f"mappingproxy({shown})") else printed - - -def _proxy_value(proxy: Mapping[object, object], form: _BaseForm, path: frozenset[int]) -> object: - printed: Final = _text(proxy, form) - if not isinstance(printed, str): - return printed - try: - return _printed_proxy_value(proxy, printed, path) - except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands - return printed - - -def _proxy_chunk_json(proxy: Mapping[object, object], printed: str) -> str: - shown: Final = _printed_proxy_value(proxy, printed, frozenset((id(proxy),))) - return json.dumps(shown) if isinstance(shown, dict) else printed - - -def _proxy_chunk_data(proxy: Mapping[object, object]) -> str: - printed: Final = format(proxy, "") - try: - return _proxy_chunk_json(proxy, printed) - except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands - return printed +def _chunk_json(shown: object) -> str | None: + if isinstance(shown, str): + return shown + return None if shown is _DROPPED else json.dumps(shown) def responses_chunk_data(chunk: object) -> str | None: if _is_proxy(chunk): - return _proxy_chunk_data(chunk) + return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)): return None - shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset()) - if isinstance(shown, str): - return shown - return None if shown is _DROPPED else json.dumps(shown) + return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset())) def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]: @@ -431,6 +413,10 @@ def _frame_events(frames: str) -> tuple[_StreamEvent, ...]: return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled) +def frame_event_type(frame: str) -> str | None: + return next((event.type for event in _frame_events(frame)), None) + + class ResponsesStreamErrorState: def __init__(self) -> None: self.response_id: str | None = None diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 99c988317bc..2253bfb4bc2 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -24,6 +24,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.responses_stream_errors import ( event_error_object, + frame_event_type, responses_error_fields, sse_error_object, ) @@ -264,7 +265,7 @@ async def background_streaming_task( try: event: _StreamEvent = _StreamEventParser.parse(chunk_data) - event_type = event.get("type", "") + event_type = frame_event_type(chunk) or event.get("type", "") # Process different event types based on OpenAI streaming spec if event_type == "response.output_item.added": diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 69789ce30d9..35646d44b65 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -361,12 +361,12 @@ class _OnceMaskedReprString(str): "wrap,printed", [ (lambda value: value, "'***'"), - (lambda value: MappingProxyType({"v": value}), "mappingproxy({'v': '***'})"), + (lambda value: MappingProxyType({"v": value}), {"v": "'***'"}), ], ids=["dict_value", "nested_mapping_proxy"], ) def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it( - wrap: Callable[[str], object], printed: str + wrap: Callable[[str], object], printed: object ) -> None: frame: Final = _format_responses_sse_chunk( chunk={"type": "response.output_text.delta", "v": wrap(_OnceMaskedReprString("hook-canary"))} @@ -449,6 +449,53 @@ class _PrintsOnceDict(dict[str, str]): _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} +class _AlternatingReprString(str): + def __init__(self, value: str) -> None: + super().__init__() + self.reprs: Final = itertools.count() + + def __repr__(self) -> str: + return "'***'" if next(self.reprs) % 2 == 0 else super().__repr__() + + +class _ReprOnceString(str): + def __init__(self, value: str) -> None: + super().__init__() + self.reprs: Final = itertools.count() + + def __repr__(self) -> str: + if next(self.reprs): + raise RuntimeError("rig repr works once") + return super().__repr__() + + +_PROXY_PLACEMENTS: Final[tuple[Callable[[str], object], ...]] = ( + lambda value: {"type": "response.output_text.delta", "meta": MappingProxyType({"v": value})}, + lambda value: MappingProxyType({"type": "response.output_text.delta", "v": value}), + lambda value: {"type": "response.output_text.delta", "meta": [MappingProxyType({"v": value})]}, +) + + +@pytest.mark.parametrize("place", _PROXY_PLACEMENTS, ids=["nested_proxy", "whole_proxy", "proxy_in_list"]) +def test_responses_sse_chunk_prints_each_proxy_item_once(place: Callable[[str], object]) -> None: + masked: Final = _format_responses_sse_chunk(chunk=place(_AlternatingReprString("hook-canary"))) + shown: Final = _format_responses_sse_chunk(chunk=place(_ReprOnceString("hook-value"))) + + assert isinstance(masked, str) + assert isinstance(shown, str) + assert "'***'" in masked + assert "hook-canary" not in masked + assert "hook-value" in shown + + +def test_responses_error_event_prints_each_proxy_item_once() -> None: + frame: Final = ResponsesStreamErrorState().format_error( + {"message": "blocked", "provider_specific_fields": MappingProxyType({"v": _AlternatingReprString("hook-canary")})} + ) + + assert json.loads(frame.split("data: ", 1)[1])["error"]["provider_specific_fields"] == {"v": "'***'"} + + @pytest.mark.parametrize( ("make_underlying", "printed"), [ @@ -456,8 +503,8 @@ _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} lambda: _MaskedStrHolder(type="response.output_text.delta", member=_RaisingRepr()), "{'type': 'response.output_text.delta', 'member': 'MASKED'}", ), - (lambda: _KeysRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)), - (lambda: _ItemRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)), + (lambda: _KeysRefusedDict(_DELTA_STORAGE), json.dumps(_DELTA_STORAGE)), + (lambda: _ItemRefusedDict(_DELTA_STORAGE), json.dumps(_DELTA_STORAGE)), ( lambda: _PrintsOnceDict(type="response.output_text.delta", token="hook-canary"), "{'type': 'response.output_text.delta', 'token': 'MASKED'}", @@ -465,7 +512,7 @@ _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"} ], ids=["member_repr_raises", "keys_raise", "item_lookup_raises", "str_works_once"], ) -def test_responses_sse_chunk_sends_a_whole_proxy_it_cannot_read_as_the_text_base_printed_once( +def test_responses_sse_chunk_sends_a_whole_proxy_as_its_mapping_printed_once( make_underlying: Callable[[], Mapping[str, object]], printed: str ) -> None: frame: Final = _format_responses_sse_chunk(chunk=MappingProxyType(make_underlying())) @@ -773,8 +820,11 @@ def test_responses_error_event_keeps_a_detail_nested_past_the_recursion_limit() assert "'leaf': 1" in frame -def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json() -> None: - nested: Final = functools.reduce(lambda inner, _: {"x": inner}, range(700), {"leaf": 1}) +@pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) +def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json( + mapping: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> None: + nested: Final = functools.reduce(lambda inner, _: mapping({"x": inner}), range(700), mapping({"leaf": 1})) chunk_frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested}) error_frame: Final = ResponsesStreamErrorState().format_error( diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 1aa998db19f..9d631db6a00 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1558,23 +1558,38 @@ class TestBackgroundStreamingTerminalEvents: @pytest.mark.asyncio @pytest.mark.parametrize( - ("nested_error", "expected_error"), + ("data_type", "nested_error", "expected_error"), [ ( + {"type": "error"}, {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"}, {"code": "invalid_request_error", "message": "output blocked by policy"}, ), - ("output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}), + ({"type": "error"}, "output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}), + ( + {}, + {"code": "400", "message": "output blocked by policy"}, + {"code": "invalid_request_error", "message": "output blocked by policy"}, + ), + ( + {"type": "response.output_text.delta"}, + {"code": "400", "message": "output blocked by policy"}, + {"code": "invalid_request_error", "message": "output blocked by policy"}, + ), ], + ids=["typed_object", "typed_string", "untyped_object", "delta_typed_object"], ) async def test_a_nested_only_error_event_keeps_its_reason( - self, nested_error: Mapping[str, str] | str, expected_error: Mapping[str, str] + self, + data_type: Mapping[str, str], + nested_error: Mapping[str, str] | str, + expected_error: Mapping[str, str], ): from litellm.proxy.response_polling.background_streaming import ( background_streaming_task, ) - error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error} + error_event: Final = {**data_type, "sequence_number": 9, "error": nested_error} async def _body_iterator(): yield b'data: {"type": "response.in_progress"}\n\n'