From 77d35556bed4c20166969fe03ed816508ed4a56e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:12:50 -0700 Subject: [PATCH] fix(proxy): print hook proxies once and keep base's chunk tracking Read-only mappings in hook chunks now emit the text from their first print, hook chunks are read without their error field, only a non-empty error object makes a framed error event terminal, and framed sequence numbers stay below 2**63-1 while hook chunks keep base's unbounded tracking --- .../common_utils/responses_stream_errors.py | 63 +++++++++++----- .../proxy_server/test_streaming_helpers.py | 71 +++++++++++++++++-- 2 files changed, 112 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index e34cb7d0f5f..be935561e28 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -33,6 +33,9 @@ class _StreamEvent(LiteLLMBaseModel): type: str | None = None sequence_number: int | None = None response: _ResponseIdentity | None = None + + +class _FramedEvent(_StreamEvent): error: object = None @@ -129,12 +132,6 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma return isinstance(value, MappingProxyType) -def _printed_proxy_snapshot(proxy: Mapping[object, object], printed: str) -> Mapping[object, object] | None: - snapshot: Final = dict(proxy) - shown: Final = repr(snapshot) - return snapshot if printed in (shown, f"mappingproxy({shown})") else None - - def _repr_prints_its_fields(model: BaseModel) -> bool: kind: Final = type(model) return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__ @@ -222,9 +219,6 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o shown_items: Final = _shown_items(value, form) if shown_items is not None: return shown_items, form - if _is_proxy(value): - snapshot: Final = _printed_proxy_snapshot(value, repr(value) if form is _BaseForm.PRINTED else str(value)) - return None if snapshot is None else (snapshot.items(), _BaseForm.PRINTED) if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value): return value.__repr_args__(), _BaseForm.PRINTED return None @@ -236,6 +230,8 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: if id(value) in path: return _text(value, form) inner_path: Final = path | frozenset((id(value),)) + if _is_proxy(value): + return _proxy_value(value, form, inner_path) try: sequence: Final = _shown_sequence(value, form) if sequence is not None: @@ -251,12 +247,32 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: return _text(value, form) +def _printed_proxy_value(proxy: Mapping[object, object], printed: str, path: frozenset[int]) -> object: + snapshot: Final = dict(proxy) + expanded: Final = _json_safe(snapshot, _BaseForm.PRINTED, path) + shown: Final = repr(snapshot) + return expanded if printed in (shown, f"mappingproxy({shown})") else printed + + +def _proxy_value(proxy: Mapping[object, object], form: _BaseForm, path: frozenset[int]) -> object: + printed: Final = _text(proxy, form) + if not isinstance(printed, str): + return printed + try: + return _printed_proxy_value(proxy, printed, path) + except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands + return printed + + +def _proxy_chunk_json(proxy: Mapping[object, object], printed: str) -> str: + shown: Final = _printed_proxy_value(proxy, printed, frozenset((id(proxy),))) + return json.dumps(shown) if isinstance(shown, dict) else printed + + def _proxy_chunk_data(proxy: Mapping[object, object]) -> str: printed: Final = format(proxy, "") try: - snapshot: Final = _printed_proxy_snapshot(proxy, printed) - shown: Final = None if snapshot is None else _json_safe(snapshot, _BaseForm.PRINTED, frozenset((id(proxy),))) - return json.dumps(shown) if isinstance(shown, dict) else printed + return _proxy_chunk_json(proxy, printed) except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands return printed @@ -385,16 +401,27 @@ def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]: return tuple(line.removeprefix(prefix).removeprefix(" ") for line in lines if line.startswith(prefix)) -def _parsed_event(data: str) -> _StreamEvent: +def _parsed_event(data: str) -> _FramedEvent: try: - return _StreamEvent.model_validate_json(data) + return _FramedEvent.model_validate_json(data) except ValidationError: - return _StreamEvent() + return _FramedEvent() + + +def _in_range_sequence(event: _FramedEvent) -> _FramedEvent: + sequence_number: Final = event.sequence_number + if sequence_number is None or 0 <= sequence_number < _MAX_SEQUENCE_NUMBER: + return event + return event.model_copy(update={"sequence_number": None}) + + +def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool: + return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",) def _block_event(lines: Sequence[str]) -> _StreamEvent: - parsed: Final = _parsed_event("\n".join(_field_values(lines, "data"))) - if parsed.type in _TERMINAL_EVENT_TYPES or not parsed.error or _field_values(lines, "event")[-1:] != ("error",): + parsed: Final = _in_range_sequence(_parsed_event("\n".join(_field_values(lines, "data")))) + if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines): return parsed return parsed.model_copy(update={"type": "error"}) @@ -417,7 +444,7 @@ class ResponsesStreamErrorState: self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None def _record(self, event: _StreamEvent) -> None: - if event.sequence_number is not None and 0 <= event.sequence_number <= _MAX_SEQUENCE_NUMBER: + if event.sequence_number is not None: self.sequence_number = max(self.sequence_number, event.sequence_number) if event.response is not None: self.response_id = event.response.id or self.response_id diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 1828395aa2f..69789ce30d9 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -357,13 +357,23 @@ class _OnceMaskedReprString(str): return "'***'" if next(self.reprs) == 0 else super().__repr__() -def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it() -> None: +@pytest.mark.parametrize( + "wrap,printed", + [ + (lambda value: value, "'***'"), + (lambda value: MappingProxyType({"v": value}), "mappingproxy({'v': '***'})"), + ], + ids=["dict_value", "nested_mapping_proxy"], +) +def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it( + wrap: Callable[[str], object], printed: str +) -> None: frame: Final = _format_responses_sse_chunk( - chunk={"type": "response.output_text.delta", "v": _OnceMaskedReprString("hook-canary")} + chunk={"type": "response.output_text.delta", "v": wrap(_OnceMaskedReprString("hook-canary"))} ) assert isinstance(frame, str) - assert json.loads(frame.removeprefix("data: "))["v"] == "'***'" + assert json.loads(frame.removeprefix("data: "))["v"] == printed @pytest.mark.parametrize( @@ -2183,6 +2193,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina ("error", {"error": {"message": "denied", "code": "529"}}, True), ("response.completed", {"sequence_number": 4, "note": "typeless"}, False), ("error", {"sequence_number": 4, "note": "no error object"}, False), + ("error", {"sequence_number": 4, "error": "denied"}, False), + ("error", {"sequence_number": 4, "error": ["denied"]}, False), ], ) async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failure_only_without_a_terminal( @@ -2212,7 +2224,11 @@ async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failur assert failure.sequence_number == 5 -@pytest.mark.parametrize("sequence_number", ["9" * 4300, "-7", str(2**63)], ids=["4300_digits", "negative", "past_int64"]) +@pytest.mark.parametrize( + "sequence_number", + ["9" * 4300, "-7", str(2**63), str(2**63 - 1)], + ids=["4300_digits", "negative", "past_int64", "int64_max"], +) def test_responses_error_event_after_an_out_of_range_pre_framed_sequence_number_stays_parseable( sequence_number: str, ) -> None: @@ -2232,6 +2248,53 @@ def test_responses_error_event_after_an_out_of_range_pre_framed_sequence_number_ assert json.loads(failure_frame.split("data: ", 1)[1])["sequence_number"] == 5 +class _UnreadableErrorModel(BaseModel, frozen=True): + type: str = "response.output_text.delta" + sequence_number: int = 4 + + @property + def error(self) -> object: + raise RuntimeError("error is not readable") + + +class _UnreadableErrorMapping(Mapping[str, object]): + def __init__(self, data: Mapping[str, object]) -> None: + self.data: Final = data + + def __getitem__(self, key: str) -> object: + if key == "error": + raise RuntimeError("error is not readable") + return self.data[key] + + def __iter__(self) -> Iterator[str]: + return iter(self.data) + + def __len__(self) -> int: + return len(self.data) + + +@pytest.mark.parametrize( + "chunk,next_sequence_number", + [ + (_UnreadableErrorModel(), 5), + (_UnreadableErrorMapping({"type": "response.output_text.delta", "sequence_number": 4}), 5), + ({"type": "response.output_text.delta", "sequence_number": 2**63}, 2**63 + 1), + ], + ids=["unreadable_error_property", "unreadable_error_item", "past_int64"], +) +def test_responses_failure_after_a_hook_chunk_continues_its_sequence_number_like_base( + chunk: object, next_sequence_number: int +) -> None: + state: Final = ResponsesStreamErrorState() + state.observe_chunk(chunk) + state.mark_emitted("data: {}\n\n") + + failure_frame: Final = state.format_failure(RuntimeError("stream closed")) + + assert failure_frame is not None + assert json.loads(failure_frame.split("data: ", 1)[1])["sequence_number"] == next_sequence_number + + # --------------------------------------------------------------------------- # select_data_generator # ---------------------------------------------------------------------------