mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): print each read-only mapping item once and poll framed error events as failed
A MappingProxyType in a hook chunk or error detail now expands from the mapping it wraps, so each item is printed once, the way base printed it. A value whose repr changes between calls stays masked, a value whose repr works once is kept, and deep nesting keeps one stack frame per level The background reader takes each frame's type from the same terminal rule the stream uses, so an event: error frame with an error object polls failed even when its data names another type or none
This commit is contained in:
parent
77d35556be
commit
7add5dce76
4 changed files with 99 additions and 47 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import gc
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
|
@ -215,7 +216,17 @@ def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, objec
|
|||
return _stored_items(value)
|
||||
|
||||
|
||||
def _proxy_items(proxy: Mapping[object, object], form: _BaseForm) -> Iterable[tuple[object, object]] | None:
|
||||
mapping: Final[object] = gc.get_referents(proxy)[0]
|
||||
if form is _BaseForm.DUMPED and not _str_prints_every_item(type(mapping)):
|
||||
return None
|
||||
return _shown_items(mapping, _BaseForm.PRINTED)
|
||||
|
||||
|
||||
def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None:
|
||||
if _is_proxy(value):
|
||||
proxy_items: Final = _proxy_items(value, form)
|
||||
return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED)
|
||||
shown_items: Final = _shown_items(value, form)
|
||||
if shown_items is not None:
|
||||
return shown_items, form
|
||||
|
|
@ -230,8 +241,6 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
|||
if id(value) in path:
|
||||
return _text(value, form)
|
||||
inner_path: Final = path | frozenset((id(value),))
|
||||
if _is_proxy(value):
|
||||
return _proxy_value(value, form, inner_path)
|
||||
try:
|
||||
sequence: Final = _shown_sequence(value, form)
|
||||
if sequence is not None:
|
||||
|
|
@ -247,45 +256,18 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
|||
return _text(value, form)
|
||||
|
||||
|
||||
def _printed_proxy_value(proxy: Mapping[object, object], printed: str, path: frozenset[int]) -> object:
|
||||
snapshot: Final = dict(proxy)
|
||||
expanded: Final = _json_safe(snapshot, _BaseForm.PRINTED, path)
|
||||
shown: Final = repr(snapshot)
|
||||
return expanded if printed in (shown, f"mappingproxy({shown})") else printed
|
||||
|
||||
|
||||
def _proxy_value(proxy: Mapping[object, object], form: _BaseForm, path: frozenset[int]) -> object:
|
||||
printed: Final = _text(proxy, form)
|
||||
if not isinstance(printed, str):
|
||||
return printed
|
||||
try:
|
||||
return _printed_proxy_value(proxy, printed, path)
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
|
||||
return printed
|
||||
|
||||
|
||||
def _proxy_chunk_json(proxy: Mapping[object, object], printed: str) -> str:
|
||||
shown: Final = _printed_proxy_value(proxy, printed, frozenset((id(proxy),)))
|
||||
return json.dumps(shown) if isinstance(shown, dict) else printed
|
||||
|
||||
|
||||
def _proxy_chunk_data(proxy: Mapping[object, object]) -> str:
|
||||
printed: Final = format(proxy, "")
|
||||
try:
|
||||
return _proxy_chunk_json(proxy, printed)
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
|
||||
return printed
|
||||
def _chunk_json(shown: object) -> str | None:
|
||||
if isinstance(shown, str):
|
||||
return shown
|
||||
return None if shown is _DROPPED else json.dumps(shown)
|
||||
|
||||
|
||||
def responses_chunk_data(chunk: object) -> str | None:
|
||||
if _is_proxy(chunk):
|
||||
return _proxy_chunk_data(chunk)
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.DUMPED, frozenset()))
|
||||
if not _is_dict(chunk) or not _format_prints_every_item(type(chunk)):
|
||||
return None
|
||||
shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset())
|
||||
if isinstance(shown, str):
|
||||
return shown
|
||||
return None if shown is _DROPPED else json.dumps(shown)
|
||||
return _chunk_json(_json_safe(chunk, _BaseForm.PRINTED, frozenset()))
|
||||
|
||||
|
||||
def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -431,6 +413,10 @@ def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
|
|||
return tuple(_block_event(tuple(lines)) for filled, lines in blocks if filled)
|
||||
|
||||
|
||||
def frame_event_type(frame: str) -> str | None:
|
||||
return next((event.type for event in _frame_events(frame)), None)
|
||||
|
||||
|
||||
class ResponsesStreamErrorState:
|
||||
def __init__(self) -> None:
|
||||
self.response_id: str | None = None
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
|||
from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.responses_stream_errors import (
|
||||
event_error_object,
|
||||
frame_event_type,
|
||||
responses_error_fields,
|
||||
sse_error_object,
|
||||
)
|
||||
|
|
@ -264,7 +265,7 @@ async def background_streaming_task(
|
|||
|
||||
try:
|
||||
event: _StreamEvent = _StreamEventParser.parse(chunk_data)
|
||||
event_type = event.get("type", "")
|
||||
event_type = frame_event_type(chunk) or event.get("type", "")
|
||||
|
||||
# Process different event types based on OpenAI streaming spec
|
||||
if event_type == "response.output_item.added":
|
||||
|
|
|
|||
|
|
@ -361,12 +361,12 @@ class _OnceMaskedReprString(str):
|
|||
"wrap,printed",
|
||||
[
|
||||
(lambda value: value, "'***'"),
|
||||
(lambda value: MappingProxyType({"v": value}), "mappingproxy({'v': '***'})"),
|
||||
(lambda value: MappingProxyType({"v": value}), {"v": "'***'"}),
|
||||
],
|
||||
ids=["dict_value", "nested_mapping_proxy"],
|
||||
)
|
||||
def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it(
|
||||
wrap: Callable[[str], object], printed: str
|
||||
wrap: Callable[[str], object], printed: object
|
||||
) -> None:
|
||||
frame: Final = _format_responses_sse_chunk(
|
||||
chunk={"type": "response.output_text.delta", "v": wrap(_OnceMaskedReprString("hook-canary"))}
|
||||
|
|
@ -449,6 +449,53 @@ class _PrintsOnceDict(dict[str, str]):
|
|||
_DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"}
|
||||
|
||||
|
||||
class _AlternatingReprString(str):
|
||||
def __init__(self, value: str) -> None:
|
||||
super().__init__()
|
||||
self.reprs: Final = itertools.count()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "'***'" if next(self.reprs) % 2 == 0 else super().__repr__()
|
||||
|
||||
|
||||
class _ReprOnceString(str):
|
||||
def __init__(self, value: str) -> None:
|
||||
super().__init__()
|
||||
self.reprs: Final = itertools.count()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
if next(self.reprs):
|
||||
raise RuntimeError("rig repr works once")
|
||||
return super().__repr__()
|
||||
|
||||
|
||||
_PROXY_PLACEMENTS: Final[tuple[Callable[[str], object], ...]] = (
|
||||
lambda value: {"type": "response.output_text.delta", "meta": MappingProxyType({"v": value})},
|
||||
lambda value: MappingProxyType({"type": "response.output_text.delta", "v": value}),
|
||||
lambda value: {"type": "response.output_text.delta", "meta": [MappingProxyType({"v": value})]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("place", _PROXY_PLACEMENTS, ids=["nested_proxy", "whole_proxy", "proxy_in_list"])
|
||||
def test_responses_sse_chunk_prints_each_proxy_item_once(place: Callable[[str], object]) -> None:
|
||||
masked: Final = _format_responses_sse_chunk(chunk=place(_AlternatingReprString("hook-canary")))
|
||||
shown: Final = _format_responses_sse_chunk(chunk=place(_ReprOnceString("hook-value")))
|
||||
|
||||
assert isinstance(masked, str)
|
||||
assert isinstance(shown, str)
|
||||
assert "'***'" in masked
|
||||
assert "hook-canary" not in masked
|
||||
assert "hook-value" in shown
|
||||
|
||||
|
||||
def test_responses_error_event_prints_each_proxy_item_once() -> None:
|
||||
frame: Final = ResponsesStreamErrorState().format_error(
|
||||
{"message": "blocked", "provider_specific_fields": MappingProxyType({"v": _AlternatingReprString("hook-canary")})}
|
||||
)
|
||||
|
||||
assert json.loads(frame.split("data: ", 1)[1])["error"]["provider_specific_fields"] == {"v": "'***'"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_underlying", "printed"),
|
||||
[
|
||||
|
|
@ -456,8 +503,8 @@ _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"}
|
|||
lambda: _MaskedStrHolder(type="response.output_text.delta", member=_RaisingRepr()),
|
||||
"{'type': 'response.output_text.delta', 'member': 'MASKED'}",
|
||||
),
|
||||
(lambda: _KeysRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)),
|
||||
(lambda: _ItemRefusedDict(_DELTA_STORAGE), repr(_DELTA_STORAGE)),
|
||||
(lambda: _KeysRefusedDict(_DELTA_STORAGE), json.dumps(_DELTA_STORAGE)),
|
||||
(lambda: _ItemRefusedDict(_DELTA_STORAGE), json.dumps(_DELTA_STORAGE)),
|
||||
(
|
||||
lambda: _PrintsOnceDict(type="response.output_text.delta", token="hook-canary"),
|
||||
"{'type': 'response.output_text.delta', 'token': 'MASKED'}",
|
||||
|
|
@ -465,7 +512,7 @@ _DELTA_STORAGE: Final = {"type": "response.output_text.delta", "token": "shown"}
|
|||
],
|
||||
ids=["member_repr_raises", "keys_raise", "item_lookup_raises", "str_works_once"],
|
||||
)
|
||||
def test_responses_sse_chunk_sends_a_whole_proxy_it_cannot_read_as_the_text_base_printed_once(
|
||||
def test_responses_sse_chunk_sends_a_whole_proxy_as_its_mapping_printed_once(
|
||||
make_underlying: Callable[[], Mapping[str, object]], printed: str
|
||||
) -> None:
|
||||
frame: Final = _format_responses_sse_chunk(chunk=MappingProxyType(make_underlying()))
|
||||
|
|
@ -773,8 +820,11 @@ def test_responses_error_event_keeps_a_detail_nested_past_the_recursion_limit()
|
|||
assert "'leaf': 1" in frame
|
||||
|
||||
|
||||
def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json() -> None:
|
||||
nested: Final = functools.reduce(lambda inner, _: {"x": inner}, range(700), {"leaf": 1})
|
||||
@pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"])
|
||||
def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json(
|
||||
mapping: Callable[[Mapping[str, object]], Mapping[str, object]],
|
||||
) -> None:
|
||||
nested: Final = functools.reduce(lambda inner, _: mapping({"x": inner}), range(700), mapping({"leaf": 1}))
|
||||
|
||||
chunk_frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested})
|
||||
error_frame: Final = ResponsesStreamErrorState().format_error(
|
||||
|
|
|
|||
|
|
@ -1558,23 +1558,38 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("nested_error", "expected_error"),
|
||||
("data_type", "nested_error", "expected_error"),
|
||||
[
|
||||
(
|
||||
{"type": "error"},
|
||||
{"type": "guardrail_error", "code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
("output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}),
|
||||
({"type": "error"}, "output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}),
|
||||
(
|
||||
{},
|
||||
{"code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
(
|
||||
{"type": "response.output_text.delta"},
|
||||
{"code": "400", "message": "output blocked by policy"},
|
||||
{"code": "invalid_request_error", "message": "output blocked by policy"},
|
||||
),
|
||||
],
|
||||
ids=["typed_object", "typed_string", "untyped_object", "delta_typed_object"],
|
||||
)
|
||||
async def test_a_nested_only_error_event_keeps_its_reason(
|
||||
self, nested_error: Mapping[str, str] | str, expected_error: Mapping[str, str]
|
||||
self,
|
||||
data_type: Mapping[str, str],
|
||||
nested_error: Mapping[str, str] | str,
|
||||
expected_error: Mapping[str, str],
|
||||
):
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error}
|
||||
error_event: Final = {**data_type, "sequence_number": 9, "error": nested_error}
|
||||
|
||||
async def _body_iterator():
|
||||
yield b'data: {"type": "response.in_progress"}\n\n'
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue