mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): print self-holding hook values once and poll how the stream ended
A container that holds itself now goes out as the marker its repr prints, like {...}, [...] or mappingproxy({...}), and the dict behind a read-only mapping joins the cycle check, so no item is printed a second time. A dict whose item repr adds a key keeps going after the items already shown, the way dict's repr does, instead of printing the whole dict again
A whole read-only mapping chunk that cannot go out as JSON is printed once by the formatter, so a print that fails fails the stream as it did on the base
The background poll reads the event type from the block it parses, as on the base, and keeps the first error frame the stream treats as terminal anywhere in a chunk. That frame fails the poll only when no response.completed, response.failed or response.incomplete follows it
This commit is contained in:
parent
7add5dce76
commit
361c7bb569
4 changed files with 263 additions and 24 deletions
|
|
@ -4,9 +4,10 @@ import re
|
|||
import time
|
||||
from collections import Counter, OrderedDict, defaultdict
|
||||
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
from itertools import chain, groupby, zip_longest
|
||||
from itertools import chain, groupby, islice, zip_longest
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -133,6 +134,14 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma
|
|||
return isinstance(value, MappingProxyType)
|
||||
|
||||
|
||||
def _is_defaultdict(value: object) -> TypeIs[defaultdict[object, object]]: # guard-ok: dict items as object
|
||||
return isinstance(value, defaultdict)
|
||||
|
||||
|
||||
def _proxied(proxy: Mapping[object, object]) -> object:
|
||||
return gc.get_referents(proxy)[0]
|
||||
|
||||
|
||||
def _repr_prints_its_fields(model: BaseModel) -> bool:
|
||||
kind: Final = type(model)
|
||||
return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__
|
||||
|
|
@ -187,6 +196,16 @@ def _stored_items(value: Mapping[object, object]) -> Iterable[tuple[object, obje
|
|||
return dict[object, object].items(value) if isinstance(value, dict) else value.items()
|
||||
|
||||
|
||||
def _live_items(value: Mapping[object, object]) -> Iterator[tuple[object, object]]:
|
||||
shown = 0 # rebind-ok: a repr that resizes its dict resumes after the items shown, the way dict's repr does
|
||||
while True:
|
||||
with suppress(RuntimeError):
|
||||
for pair in islice(_stored_items(value), shown, None):
|
||||
shown += 1
|
||||
yield pair
|
||||
return
|
||||
|
||||
|
||||
def _dumps_shown(kind: type[object], shown: Iterable[object], stored: Iterable[object]) -> bool:
|
||||
return _str_prints_every_item(kind) or any(left is not right for left, right in zip_longest(shown, stored))
|
||||
|
||||
|
|
@ -210,14 +229,14 @@ def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, objec
|
|||
if not _repr_prints_every_item(type(value)):
|
||||
return None
|
||||
if isinstance(value, OrderedDict):
|
||||
return OrderedDict[object, object].items(value)
|
||||
return tuple(OrderedDict[object, object].items(value))
|
||||
if isinstance(value, Counter):
|
||||
return Counter[object].most_common(value)
|
||||
return _stored_items(value)
|
||||
return _live_items(value)
|
||||
|
||||
|
||||
def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None:
|
||||
mapping: Final[object] = gc.get_referents(proxy)[0]
|
||||
mapping: Final = _proxied(proxy)
|
||||
if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)):
|
||||
return None
|
||||
return _shown_items(mapping, _BaseForm.PRINTED)
|
||||
|
|
@ -235,12 +254,42 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o
|
|||
return None
|
||||
|
||||
|
||||
def _walked_ids(value: object) -> frozenset[int]:
|
||||
return frozenset((id(value), id(_proxied(value)))) if _is_proxy(value) else frozenset((id(value),))
|
||||
|
||||
|
||||
def _container_marker(value: object) -> object:
|
||||
if isinstance(value, list):
|
||||
return "[...]"
|
||||
if isinstance(value, tuple):
|
||||
return "(...)"
|
||||
if isinstance(value, OrderedDict):
|
||||
return "..."
|
||||
if _is_defaultdict(value):
|
||||
factory: Final = _text(value.default_factory, _BaseForm.PRINTED)
|
||||
return factory if factory is _DROPPED else f"{type(value).__name__}({factory}, {{...}})"
|
||||
return "{...}" if _is_dict(value) and not isinstance(value, Counter) else None
|
||||
|
||||
|
||||
def _cycle_marker(value: object) -> object:
|
||||
if not _is_proxy(value):
|
||||
return _container_marker(value)
|
||||
inner: Final = _container_marker(_proxied(value))
|
||||
return inner if inner is None or inner is _DROPPED else f"mappingproxy({inner})"
|
||||
|
||||
|
||||
def _cycle_text(value: object, form: _BaseForm) -> object:
|
||||
marker: Final = _cycle_marker(value)
|
||||
return _text(value, form) if marker is None else marker
|
||||
|
||||
|
||||
def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
||||
if value is None or type(value) in (str, int, float, bool):
|
||||
return value
|
||||
if id(value) in path:
|
||||
return _text(value, form)
|
||||
inner_path: Final = path | frozenset((id(value),))
|
||||
walked: Final = _walked_ids(value)
|
||||
if not walked.isdisjoint(path):
|
||||
return _cycle_text(value, form)
|
||||
inner_path: Final = path | walked
|
||||
try:
|
||||
sequence: Final = _shown_sequence(value, form)
|
||||
if sequence is not None:
|
||||
|
|
@ -264,7 +313,8 @@ def _chunk_json(shown: object) -> str | None:
|
|||
|
||||
def responses_chunk_data(chunk: object) -> str | None:
|
||||
if _is_proxy(chunk):
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset()))
|
||||
expandable: Final = _proxy_items(chunk, _BaseForm.DUMPED) is not None
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset())) if expandable else None
|
||||
if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)):
|
||||
return None
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset()))
|
||||
|
|
@ -401,20 +451,27 @@ def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool:
|
|||
return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",)
|
||||
|
||||
|
||||
def _block_data(lines: Sequence[str]) -> str:
|
||||
return "\n".join(_field_values(lines, "data"))
|
||||
|
||||
|
||||
def _block_event(lines: Sequence[str]) -> _StreamEvent:
|
||||
parsed: Final = _in_range_sequence(_parsed_event("\n".join(_field_values(lines, "data"))))
|
||||
parsed: Final = _in_range_sequence(_parsed_event(_block_data(lines)))
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines):
|
||||
return parsed
|
||||
return parsed.model_copy(update={"type": "error"})
|
||||
|
||||
|
||||
def _blocks(frames: str) -> Iterator[tuple[str, ...]]:
|
||||
return (tuple(lines) for filled, lines in groupby(SSE_LINE_BREAK.split(frames), key=bool) if filled)
|
||||
|
||||
|
||||
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
|
||||
blocks: Final = groupby(SSE_LINE_BREAK.split(frames), key=bool)
|
||||
return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled)
|
||||
return tuple(_block_event(lines) for lines in _blocks(frames))
|
||||
|
||||
|
||||
def frame_event_type(frame: str) -> str | None:
|
||||
return next((event.type for event in _frame_events(frame)), None)
|
||||
def frame_error_data(frame: str) -> str | None:
|
||||
return next((_block_data(lines) for lines in _blocks(frame) if _block_event(lines).type == "error"), None)
|
||||
|
||||
|
||||
class ResponsesStreamErrorState:
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
|||
from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.responses_stream_errors import (
|
||||
event_error_object,
|
||||
frame_event_type,
|
||||
frame_error_data,
|
||||
responses_error_fields,
|
||||
sse_error_object,
|
||||
)
|
||||
|
|
@ -224,6 +224,7 @@ async def background_streaming_task(
|
|||
None # Will be set by response.completed/failed/incomplete/cancelled
|
||||
)
|
||||
terminal_error = None
|
||||
frame_failure: Mapping[str, object] | None = None # rebind-ok: set once by the first error frame
|
||||
_event_to_status: Final[Mapping[str, ResponsesAPIStatus]] = {
|
||||
"response.completed": "completed",
|
||||
"response.failed": "failed",
|
||||
|
|
@ -259,13 +260,20 @@ async def background_streaming_task(
|
|||
if isinstance(chunk, bytes):
|
||||
chunk = chunk.decode("utf-8")
|
||||
|
||||
if (
|
||||
isinstance(chunk, str)
|
||||
and frame_failure is None
|
||||
and (error_data := frame_error_data(chunk)) is not None
|
||||
):
|
||||
frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data)))
|
||||
|
||||
if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None:
|
||||
if chunk_data == "[DONE]":
|
||||
break
|
||||
|
||||
try:
|
||||
event: _StreamEvent = _StreamEventParser.parse(chunk_data)
|
||||
event_type = frame_event_type(chunk) or event.get("type", "")
|
||||
event_type = event.get("type", "")
|
||||
|
||||
# Process different event types based on OpenAI streaming spec
|
||||
if event_type == "response.output_item.added":
|
||||
|
|
@ -351,10 +359,6 @@ async def background_streaming_task(
|
|||
status="in_progress",
|
||||
)
|
||||
|
||||
elif event_type == "error":
|
||||
terminal_status = "failed"
|
||||
terminal_error = responses_error_fields(_event_error(event))
|
||||
|
||||
elif event_type in (
|
||||
"response.completed",
|
||||
"response.failed",
|
||||
|
|
@ -412,8 +416,10 @@ async def background_streaming_task(
|
|||
await flush_state_if_needed(force=True)
|
||||
|
||||
early_error: Final = _first_chunk_error(response)
|
||||
final_error: Final = terminal_error if early_error is None else early_error
|
||||
final_status: Final = "failed" if early_error is not None else terminal_status or "completed"
|
||||
stream_failure: Final = frame_failure if terminal_status is None else None
|
||||
failure: Final = stream_failure if early_error is None else early_error
|
||||
final_error: Final = terminal_error if failure is None else failure
|
||||
final_status: Final = (terminal_status or "completed") if failure is None else "failed"
|
||||
|
||||
await polling_handler.update_state(
|
||||
polling_id=polling_id,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import asyncio
|
|||
import functools
|
||||
import itertools
|
||||
import json
|
||||
from collections import Counter, OrderedDict
|
||||
from collections import Counter, OrderedDict, defaultdict
|
||||
from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
|
|
@ -496,6 +496,114 @@ def test_responses_error_event_prints_each_proxy_item_once() -> None:
|
|||
assert json.loads(frame.split("data: ", 1)[1])["error"]["provider_specific_fields"] == {"v": "'***'"}
|
||||
|
||||
|
||||
def _dict_holding_itself(value: str) -> dict[str, object]:
|
||||
held: Final[dict[str, object]] = {"v": value}
|
||||
held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself
|
||||
return held
|
||||
|
||||
|
||||
def _defaultdict_holding_itself(value: str) -> defaultdict[str, object]:
|
||||
held: Final[defaultdict[str, object]] = defaultdict(list, v=value)
|
||||
held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself
|
||||
return held
|
||||
|
||||
|
||||
def _ordered_dict_holding_itself(value: str) -> OrderedDict[str, object]:
|
||||
held: Final[OrderedDict[str, object]] = OrderedDict(v=value)
|
||||
held["me"] = held # mutable-ok: a self-reference needs the dict to exist before it can hold itself
|
||||
return held
|
||||
|
||||
|
||||
def _list_holding_itself(value: str) -> list[object]:
|
||||
held: Final[list[object]] = [value]
|
||||
held.append(held) # mutable-ok: a self-reference needs the list to exist before it can hold itself
|
||||
return held
|
||||
|
||||
|
||||
def _proxy_whose_mapping_holds_it(value: str) -> Mapping[str, object]:
|
||||
held: Final[dict[str, object]] = {"type": "response.output_text.delta", "v": value}
|
||||
held["me"] = MappingProxyType(held) # mutable-ok: a self-reference needs the dict to exist before it can hold itself
|
||||
return MappingProxyType(held)
|
||||
|
||||
|
||||
_SELF_HOLDING_PLACEMENTS: Final[tuple[tuple[Callable[[str], object], Mapping[str, object]], ...]] = (
|
||||
(
|
||||
lambda value: {"type": "response.output_text.delta", "meta": MappingProxyType(_dict_holding_itself(value))},
|
||||
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "{...}"}},
|
||||
),
|
||||
(
|
||||
lambda value: {"type": "response.output_text.delta", "meta": _dict_holding_itself(value)},
|
||||
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "{...}"}},
|
||||
),
|
||||
(
|
||||
lambda value: {"type": "response.output_text.delta", "meta": _defaultdict_holding_itself(value)},
|
||||
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "defaultdict(<class 'list'>, {...})"}},
|
||||
),
|
||||
(
|
||||
lambda value: {"type": "response.output_text.delta", "meta": _ordered_dict_holding_itself(value)},
|
||||
{"type": "response.output_text.delta", "meta": {"v": "'***'", "me": "..."}},
|
||||
),
|
||||
(
|
||||
lambda value: {"type": "response.output_text.delta", "meta": _list_holding_itself(value)},
|
||||
{"type": "response.output_text.delta", "meta": ["'***'", "[...]"]},
|
||||
),
|
||||
(
|
||||
_proxy_whose_mapping_holds_it,
|
||||
{"type": "response.output_text.delta", "v": "'***'", "me": "mappingproxy({...})"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("place", "expected"),
|
||||
_SELF_HOLDING_PLACEMENTS,
|
||||
ids=["proxy_over_dict", "dict", "defaultdict", "ordered_dict", "list", "proxy_held_by_its_mapping"],
|
||||
)
|
||||
def test_responses_sse_chunk_prints_a_container_that_holds_itself_as_its_repr_marker(
|
||||
place: Callable[[str], object], expected: Mapping[str, object]
|
||||
) -> None:
|
||||
frame: Final = _format_responses_sse_chunk(chunk=place(_AlternatingReprString("hook-canary")))
|
||||
|
||||
assert isinstance(frame, str)
|
||||
assert "hook-canary" not in frame
|
||||
assert json.loads(frame.removeprefix("data: ")) == expected
|
||||
|
||||
|
||||
class _ParentGrower:
|
||||
def __init__(self, parent: dict[str, object]) -> None:
|
||||
self.parent: Final = parent
|
||||
|
||||
def __repr__(self) -> str:
|
||||
self.parent["added"] = 1 # mutable-ok: the hook value's repr grows the dict that holds it
|
||||
return "_ParentGrower()"
|
||||
|
||||
|
||||
def test_responses_sse_chunk_keeps_printing_a_dict_whose_item_repr_adds_a_key() -> None:
|
||||
parent: Final[dict[str, object]] = {"v": _AlternatingReprString("hook-canary")}
|
||||
parent["m"] = _ParentGrower(parent) # mutable-ok: the grower needs the dict it adds a key to
|
||||
|
||||
frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "meta": parent})
|
||||
|
||||
assert isinstance(frame, str)
|
||||
assert "hook-canary" not in frame
|
||||
assert json.loads(frame.removeprefix("data: "))["meta"] == {"v": "'***'", "m": "_ParentGrower()", "added": 1}
|
||||
|
||||
|
||||
class _StrFailsOnceDict(dict[str, str]):
|
||||
failed: bool = False
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.failed:
|
||||
return dict.__repr__(self)
|
||||
self.failed = True
|
||||
raise RuntimeError("rig str fails on its first call")
|
||||
|
||||
|
||||
def test_responses_sse_chunk_raises_when_a_whole_proxy_fails_its_only_print() -> None:
|
||||
with pytest.raises(RuntimeError, match="rig str fails on its first call"):
|
||||
_format_responses_sse_chunk(chunk=MappingProxyType(_StrFailsOnceDict(_DELTA_STORAGE)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_underlying", "printed"),
|
||||
[
|
||||
|
|
@ -775,10 +883,10 @@ def _self_referencing_chunk() -> Mapping[str, object]:
|
|||
return chunk
|
||||
|
||||
|
||||
def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_text() -> None:
|
||||
def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_repr_marker() -> None:
|
||||
nested: Final = _self_referencing_chunk()
|
||||
|
||||
assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": repr(nested)}
|
||||
assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": "{...}"}
|
||||
|
||||
|
||||
class _UnprintableValue:
|
||||
|
|
|
|||
|
|
@ -1612,6 +1612,74 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
assert final_call.kwargs["status"] == "failed"
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("chunks", "expected_status", "expected_error", "expected_usage"),
|
||||
[
|
||||
(
|
||||
(
|
||||
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n'
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
|
||||
),
|
||||
"failed",
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
None,
|
||||
),
|
||||
(
|
||||
(
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
(
|
||||
(
|
||||
'data:{"type": "response.output_text.delta", "delta": "hi"}\n\n'
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
],
|
||||
ids=["error_frame_after_a_delta", "error_frame_then_completed", "unspaced_delta_then_completed"],
|
||||
)
|
||||
async def test_a_poll_stores_what_its_stream_ended_with(
|
||||
self,
|
||||
chunks: tuple[str, ...],
|
||||
expected_status: str,
|
||||
expected_error: Mapping[str, str] | None,
|
||||
expected_usage: Mapping[str, int] | None,
|
||||
):
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
async def _body_iterator():
|
||||
yield b'data: {"type": "response.in_progress"}\n\n'
|
||||
for chunk in chunks:
|
||||
yield chunk.encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_stream_end", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
final_call: Final = handler.update_state.call_args_list[-1]
|
||||
assert final_call.kwargs["status"] == expected_status
|
||||
assert final_call.kwargs["error"] == expected_error
|
||||
assert final_call.kwargs["usage"] == expected_usage
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "body", "expected_error"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue