fix(proxy): print self-holding hook values once and poll how the stream ended

A container that holds itself now goes out as the marker its repr prints, like {...}, [...] or mappingproxy({...}), and the dict behind a read-only mapping joins the cycle check, so no item is printed a second time. A dict whose item repr adds a key keeps going after the items already shown, the way dict's repr does, instead of printing the whole dict again

A whole read-only mapping chunk that cannot go out as JSON is printed once by the formatter, so a print that fails fails the stream as it did on the base

The background poll reads the event type from the block it parses, as on the base, and keeps the first error frame the stream treats as terminal anywhere in a chunk. That frame fails the poll only when no response.completed, response.failed or response.incomplete follows it
This commit is contained in:
mateo-berri 2026-10-31 10:16:12 -07:00
parent 7add5dce76
commit 361c7bb569
4 changed files with 263 additions and 24 deletions

View file

@ -4,9 +4,10 @@ import re
import time
from collections import Counter, OrderedDict, defaultdict
from collections.abc import Iterable, Iterator, Mapping, Sequence
from contextlib import suppress
from enum import Enum
from http import HTTPStatus
from itertools import chain, groupby, zip_longest
from itertools import chain, groupby, islice, zip_longest
from types import MappingProxyType
from typing import Final
@ -133,6 +134,14 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma
return isinstance(value, MappingProxyType)
def _is_defaultdict(value: object) -> TypeIs[defaultdict[object, object]]: # guard-ok: dict items as object
return isinstance(value, defaultdict)
def _proxied(proxy: Mapping[object, object]) -> object:
return gc.get_referents(proxy)[0]
def _repr_prints_its_fields(model: BaseModel) -> bool:
kind: Final = type(model)
return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__
@ -187,6 +196,16 @@ def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, obje
return dict[object, object].items(value) if isinstance(value, dict) else value.items()
def _live_items(value: Mapping[object, object]) -> Iterator[tuple[object, object]]:
shown = 0 # rebind-ok: a repr that resizes its dict resumes after the items shown, the way dict's repr does
while True:
with suppress(RuntimeError):
for pair in islice(_stored_items(value), shown, None):
shown += 1
yield pair
return
def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool:
return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored))
@ -210,14 +229,14 @@ def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, objec
if not _repr_prints_every_item(type(value)):
return None
if isinstance(value, OrderedDict):
return OrderedDict[object, object].items(value)
return tuple(OrderedDict[object, object].items(value))
if isinstance(value, Counter):
return Counter[object].most_common(value)
return _stored_items(value)
return _live_items(value)
def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None:
mapping: Final[object] = gc.get_referents(proxy)[0]
mapping: Final = _proxied(proxy)
if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)):
return None
return _shown_items(mapping, _BaseForm.PRINTED)
@ -235,12 +254,42 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o
return None
def _walked_ids(value: object) -> frozenset[int]:
return frozenset((id(value), id(_proxied(value)))) if _is_proxy(value) else frozenset((id(value),))
def _container_marker(value: object) -> object:
if isinstance(value, list):
return "[...]"
if isinstance(value, tuple):
return "(...)"
if isinstance(value, OrderedDict):
return "..."
if _is_defaultdict(value):
factory: Final = _text(value.default_factory, _BaseForm.PRINTED)
return factory if factory is _DROPPED else f"{type(value).__name__}({factory}, {{...}})"
return "{...}" if _is_dict(value) and not isinstance(value, Counter) else None
def _cycle_marker(value: object) -> object:
if not _is_proxy(value):
return _container_marker(value)
inner: Final = _container_marker(_proxied(value))
return inner if inner is None or inner is _DROPPED else f"mappingproxy({inner})"
def _cycle_text(value: object, form: _BaseForm) -> object:
marker: Final = _cycle_marker(value)
return _text(value, form) if marker is None else marker
def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
if value is None or type(value) in (str, int, float, bool):
return value
if id(value) in path:
return _text(value, form)
inner_path: Final = path | frozenset((id(value),))
walked: Final = _walked_ids(value)
if not walked.isdisjoint(path):
return _cycle_text(value, form)
inner_path: Final = path | walked
try:
sequence: Final = _shown_sequence(value, form)
if sequence is not None:
@ -264,7 +313,8 @@ def _chunk_json(shown: object) -> str | None:
def responses_chunk_data(chunk: object) -> str | None:
if _is_proxy(chunk):
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset()))
expandable: Final = _proxy_items(chunk, _BaseForm.DUMPED) is not None
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if expandable else None
if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)):
return None
return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset()))
@ -401,20 +451,27 @@ def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool:
return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",)
def _block_data(lines: Sequence[str]) -> str:
return "\n".join(_field_values(lines, "data"))
def _block_event(lines: Sequence[str]) -> _StreamEvent:
parsed: Final = _in_range_sequence(_parsed_event("\n".join(_field_values(lines, "data"))))
parsed: Final = _in_range_sequence(_parsed_event(_block_data(lines)))
if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines):
return parsed
return parsed.model_copy(update={"type": "error"})
def _blocks(frames: str) -> Iterator[tuple[str, ...]]:
return (tuple(lines) for filled, lines in groupby(SSE_LINE_BREAK.split(frames), key=bool) if filled)
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
blocks: Final = groupby(SSE_LINE_BREAK.split(frames), key=bool)
return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled)
return tuple(_block_event(lines) for lines in _blocks(frames))
def frame_event_type(frame: str) -> str | None:
return next((event.type for event in _frame_events(frame)), None)
def frame_error_data(frame: str) -> str | None:
return next((_block_data(lines) for lines in _blocks(frame) if _block_event(lines).type == "error"), None)
class ResponsesStreamErrorState:

View file

@ -24,7 +24,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.responses_stream_errors import (
event_error_object,
frame_event_type,
frame_error_data,
responses_error_fields,
sse_error_object,
)
@ -224,6 +224,7 @@ async def background_streaming_task(
None # Will be set by response.completed/failed/incomplete/cancelled
)
terminal_error = None
frame_failure: Mapping[str, object] | None = None # rebind-ok: set once by the first error frame
_event_to_status: Final[Mapping[str, ResponsesAPIStatus]] = {
"response.completed": "completed",
"response.failed": "failed",
@ -259,13 +260,20 @@ async def background_streaming_task(
if isinstance(chunk, bytes):
chunk = chunk.decode("utf-8")
if (
isinstance(chunk, str)
and frame_failure is None
and (error_data := frame_error_data(chunk)) is not None
):
frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data)))
if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None:
if chunk_data == "[DONE]":
break
try:
event: _StreamEvent = _StreamEventParser.parse(chunk_data)
event_type = frame_event_type(chunk) or event.get("type", "")
event_type = event.get("type", "")
# Process different event types based on OpenAI streaming spec
if event_type == "response.output_item.added":
@ -351,10 +359,6 @@ async def background_streaming_task(
status="in_progress",
)
elif event_type == "error":
terminal_status = "failed"
terminal_error = responses_error_fields(_event_error(event))
elif event_type in (
"response.completed",
"response.failed",
@ -412,8 +416,10 @@ async def background_streaming_task(
await flush_state_if_needed(force=True)
early_error: Final = _first_chunk_error(response)
final_error: Final = terminal_error if early_error is None else early_error
final_status: Final = "failed" if early_error is not None else terminal_status or "completed"
stream_failure: Final = frame_failure if terminal_status is None else None
failure: Final = stream_failure if early_error is None else early_error
final_error: Final = terminal_error if failure is None else failure
final_status: Final = (terminal_status or "completed") if failure is None else "failed"
await polling_handler.update_state(
polling_id=polling_id,

View file

@ -19,7 +19,7 @@ import asyncio
import functools
import itertools
import json
from collections import Counter, OrderedDict
from collections import Counter, OrderedDict, defaultdict
from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping
from datetime import datetime, timezone
from enum import Enum
@ -496,6 +496,114 @@ def test_responses_error_event_prints_each_proxy_item_once() -> None:
assert json.loads(frame.split("data: ", 1)[1])["error"]["provider_specific_fields"] == {"v": "'***'"}
def _dict_holding_itself(value: str) -> dict[str, object]:
held: Final[dict[str, object]] = {"v": value}
held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself
return held
def _defaultdict_holding_itself(value: str) -> defaultdict[str, object]:
held: Final[defaultdict[str, object]] = defaultdict(list, v=value)
held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself
return held
def _ordered_dict_holding_itself(value: str) -> OrderedDict[str, object]:
held: Final[OrderedDict[str, object]] = OrderedDict(v=value)
held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself
return held
def _list_holding_itself(value: str) -> list[object]:
held: Final[list[object]] = [value]
held.append(held) # mutable-ok: a self-reference needs the list to exist before it can hold itself
return held
def _proxy_whose_mapping_holds_it(value: str) -> Mapping[str, object]:
held: Final[dict[str, object]] = {"type": "response.output_text.delta", "v": value}
held["me"] = MappingProxyType(held) # mutable-ok: a self-reference needs the dict to exist before it can hold itself
return MappingProxyType(held)
_SELF_HOLDING_PLACEMENTS: Final[tuple[tuple[Callable[[str], object], Mapping[str, object]], ...]] = (
(
lambda value: {"type": "response.output_text.delta", "meta": MappingProxyType(_dict_holding_itself(value))},
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "{...}"}},
),
(
lambda value: {"type": "response.output_text.delta", "meta": _dict_holding_itself(value)},
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "{...}"}},
),
(
lambda value: {"type": "response.output_text.delta", "meta": _defaultdict_holding_itself(value)},
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "defaultdict(<class 'list'>, {...})"}},
),
(
lambda value: {"type": "response.output_text.delta", "meta": _ordered_dict_holding_itself(value)},
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "..."}},
),
(
lambda value: {"type": "response.output_text.delta", "meta": _list_holding_itself(value)},
{"type": "response.output_text.delta", "meta": ["'***'", "[...]"]},
),
(
_proxy_whose_mapping_holds_it,
{"type": "response.output_text.delta", "v": "'***'", "me": "mappingproxy({...})"},
),
)
@pytest.mark.parametrize(
("place", "expected"),
_SELF_HOLDING_PLACEMENTS,
ids=["proxy_over_dict", "dict", "defaultdict", "ordered_dict", "list", "proxy_held_by_its_mapping"],
)
def test_responses_sse_chunk_prints_a_container_that_holds_itself_as_its_repr_marker(
place: Callable[[str], object], expected: Mapping[str, object]
) -> None:
frame: Final = _format_responses_sse_chunk(chunk=place(_AlternatingReprString("hook-canary")))
assert isinstance(frame, str)
assert "hook-canary" not in frame
assert json.loads(frame.removeprefix("data: ")) == expected
class _ParentGrower:
def __init__(self, parent: dict[str, object]) -> None:
self.parent: Final = parent
def __repr__(self) -> str:
self.parent["added"] = 1 # mutable-ok: the hook value's repr grows the dict that holds it
return "_ParentGrower()"
def test_responses_sse_chunk_keeps_printing_a_dict_whose_item_repr_adds_a_key() -> None:
parent: Final[dict[str, object]] = {"v": _AlternatingReprString("hook-canary")}
parent["m"] = _ParentGrower(parent) # mutable-ok: the grower needs the dict it adds a key to
frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "meta": parent})
assert isinstance(frame, str)
assert "hook-canary" not in frame
assert json.loads(frame.removeprefix("data: "))["meta"] == {"v": "'***'", "m": "_ParentGrower()", "added": 1}
class _StrFailsOnceDict(dict[str, str]):
failed: bool = False
def __str__(self) -> str:
if self.failed:
return dict.__repr__(self)
self.failed = True
raise RuntimeError("rig str fails on its first call")
def test_responses_sse_chunk_raises_when_a_whole_proxy_fails_its_only_print() -> None:
with pytest.raises(RuntimeError, match="rig str fails on its first call"):
_format_responses_sse_chunk(chunk=MappingProxyType(_StrFailsOnceDict(_DELTA_STORAGE)))
@pytest.mark.parametrize(
("make_underlying", "printed"),
[
@ -775,10 +883,10 @@ def _self_referencing_chunk() -> Mapping[str, object]:
return chunk
def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_text() -> None:
def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_repr_marker() -> None:
nested: Final = _self_referencing_chunk()
assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": repr(nested)}
assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": "{...}"}
class _UnprintableValue:

View file

@ -1612,6 +1612,74 @@ class TestBackgroundStreamingTerminalEvents:
assert final_call.kwargs["status"] == "failed"
assert final_call.kwargs["error"] == expected_error
@pytest.mark.asyncio
@pytest.mark.parametrize(
("chunks", "expected_status", "expected_error", "expected_usage"),
[
(
(
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n'
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
),
"failed",
{"code": "invalid_request_error", "message": "output blocked by policy"},
None,
),
(
(
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
(
(
'data:{"type": "response.output_text.delta", "delta": "hi"}\n\n'
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
],
ids=["error_frame_after_a_delta", "error_frame_then_completed", "unspaced_delta_then_completed"],
)
async def test_a_poll_stores_what_its_stream_ended_with(
self,
chunks: tuple[str, ...],
expected_status: str,
expected_error: Mapping[str, str] | None,
expected_usage: Mapping[str, int] | None,
):
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
async def _body_iterator():
yield b'data: {"type": "response.in_progress"}\n\n'
for chunk in chunks:
yield chunk.encode()
mock_response: Final = Mock()
mock_response.body_iterator = _body_iterator()
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_stream_end", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
final_call: Final = handler.update_state.call_args_list[-1]
assert final_call.kwargs["status"] == expected_status
assert final_call.kwargs["error"] == expected_error
assert final_call.kwargs["usage"] == expected_usage
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status_code", "body", "expected_error"),