mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): print hook proxies once and keep base's chunk tracking
Read-only mappings in hook chunks now emit the text from their first print, hook chunks are read without their error field, only a non-empty error object makes a framed error event terminal, and framed sequence numbers stay below 2**63-1 while hook chunks keep base's unbounded tracking
This commit is contained in:
parent
da876856fc
commit
77d35556be
2 changed files with 112 additions and 22 deletions
|
|
@ -33,6 +33,9 @@ class _StreamEvent(LiteLLMBaseModel):
|
|||
type: str | None = None
|
||||
sequence_number: int | None = None
|
||||
response: _ResponseIdentity | None = None
|
||||
|
||||
|
||||
class _FramedEvent(_StreamEvent):
|
||||
error: object = None
|
||||
|
||||
|
||||
|
|
@ -129,12 +132,6 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma
|
|||
return isinstance(value, MappingProxyType)
|
||||
|
||||
|
||||
def _printed_proxy_snapshot(proxy: Mapping[object, object], printed: str) -> Mapping[object, object] | None:
|
||||
snapshot: Final = dict(proxy)
|
||||
shown: Final = repr(snapshot)
|
||||
return snapshot if printed in (shown, f"mappingproxy({shown})") else None
|
||||
|
||||
|
||||
def _repr_prints_its_fields(model: BaseModel) -> bool:
|
||||
kind: Final = type(model)
|
||||
return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__
|
||||
|
|
@ -222,9 +219,6 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o
|
|||
shown_items: Final = _shown_items(value, form)
|
||||
if shown_items is not None:
|
||||
return shown_items, form
|
||||
if _is_proxy(value):
|
||||
snapshot: Final = _printed_proxy_snapshot(value, repr(value) if form is _BaseForm.PRINTED else str(value))
|
||||
return None if snapshot is None else (snapshot.items(), _BaseForm.PRINTED)
|
||||
if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value):
|
||||
return value.__repr_args__(), _BaseForm.PRINTED
|
||||
return None
|
||||
|
|
@ -236,6 +230,8 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
|||
if id(value) in path:
|
||||
return _text(value, form)
|
||||
inner_path: Final = path | frozenset((id(value),))
|
||||
if _is_proxy(value):
|
||||
return _proxy_value(value, form, inner_path)
|
||||
try:
|
||||
sequence: Final = _shown_sequence(value, form)
|
||||
if sequence is not None:
|
||||
|
|
@ -251,12 +247,32 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
|||
return _text(value, form)
|
||||
|
||||
|
||||
def _printed_proxy_value(proxy: Mapping[object, object], printed: str, path: frozenset[int]) -> object:
|
||||
snapshot: Final = dict(proxy)
|
||||
expanded: Final = _json_safe(snapshot, _BaseForm.PRINTED, path)
|
||||
shown: Final = repr(snapshot)
|
||||
return expanded if printed in (shown, f"mappingproxy({shown})") else printed
|
||||
|
||||
|
||||
def _proxy_value(proxy: Mapping[object, object], form: _BaseForm, path: frozenset[int]) -> object:
|
||||
printed: Final = _text(proxy, form)
|
||||
if not isinstance(printed, str):
|
||||
return printed
|
||||
try:
|
||||
return _printed_proxy_value(proxy, printed, path)
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
|
||||
return printed
|
||||
|
||||
|
||||
def _proxy_chunk_json(proxy: Mapping[object, object], printed: str) -> str:
|
||||
shown: Final = _printed_proxy_value(proxy, printed, frozenset((id(proxy),)))
|
||||
return json.dumps(shown) if isinstance(shown, dict) else printed
|
||||
|
||||
|
||||
def _proxy_chunk_data(proxy: Mapping[object, object]) -> str:
|
||||
printed: Final = format(proxy, "")
|
||||
try:
|
||||
snapshot: Final = _printed_proxy_snapshot(proxy, printed)
|
||||
shown: Final = None if snapshot is None else _json_safe(snapshot, _BaseForm.PRINTED, frozenset((id(proxy),)))
|
||||
return json.dumps(shown) if isinstance(shown, dict) else printed
|
||||
return _proxy_chunk_json(proxy, printed)
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
|
||||
return printed
|
||||
|
||||
|
|
@ -385,16 +401,27 @@ def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]:
|
|||
return tuple(line.removeprefix(prefix).removeprefix(" ") for line in lines if line.startswith(prefix))
|
||||
|
||||
|
||||
def _parsed_event(data: str) -> _StreamEvent:
|
||||
def _parsed_event(data: str) -> _FramedEvent:
|
||||
try:
|
||||
return _StreamEvent.model_validate_json(data)
|
||||
return _FramedEvent.model_validate_json(data)
|
||||
except ValidationError:
|
||||
return _StreamEvent()
|
||||
return _FramedEvent()
|
||||
|
||||
|
||||
def _in_range_sequence(event: _FramedEvent) -> _FramedEvent:
|
||||
sequence_number: Final = event.sequence_number
|
||||
if sequence_number is None or 0 <= sequence_number < _MAX_SEQUENCE_NUMBER:
|
||||
return event
|
||||
return event.model_copy(update={"sequence_number": None})
|
||||
|
||||
|
||||
def _names_an_error_object(event: _FramedEvent, lines: Sequence[str]) -> bool:
|
||||
return _is_dict(event.error) and bool(event.error) and _field_values(lines, "event")[-1:] == ("error",)
|
||||
|
||||
|
||||
def _block_event(lines: Sequence[str]) -> _StreamEvent:
|
||||
parsed: Final = _parsed_event("\n".join(_field_values(lines, "data")))
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not parsed.error or _field_values(lines, "event")[-1:] != ("error",):
|
||||
parsed: Final = _in_range_sequence(_parsed_event("\n".join(_field_values(lines, "data"))))
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not _names_an_error_object(parsed, lines):
|
||||
return parsed
|
||||
return parsed.model_copy(update={"type": "error"})
|
||||
|
||||
|
|
@ -417,7 +444,7 @@ class ResponsesStreamErrorState:
|
|||
self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None
|
||||
|
||||
def _record(self, event: _StreamEvent) -> None:
|
||||
if event.sequence_number is not None and 0 <= event.sequence_number <= _MAX_SEQUENCE_NUMBER:
|
||||
if event.sequence_number is not None:
|
||||
self.sequence_number = max(self.sequence_number, event.sequence_number)
|
||||
if event.response is not None:
|
||||
self.response_id = event.response.id or self.response_id
|
||||
|
|
|
|||
|
|
@ -357,13 +357,23 @@ class _OnceMaskedReprString(str):
|
|||
return "'***'" if next(self.reprs) == 0 else super().__repr__()
|
||||
|
||||
|
||||
def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"wrap,printed",
|
||||
[
|
||||
(lambda value: value, "'***'"),
|
||||
(lambda value: MappingProxyType({"v": value}), "mappingproxy({'v': '***'})"),
|
||||
],
|
||||
ids=["dict_value", "nested_mapping_proxy"],
|
||||
)
|
||||
def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it(
|
||||
wrap: Callable[[str], object], printed: str
|
||||
) -> None:
|
||||
frame: Final = _format_responses_sse_chunk(
|
||||
chunk={"type": "response.output_text.delta", "v": _OnceMaskedReprString("hook-canary")}
|
||||
chunk={"type": "response.output_text.delta", "v": wrap(_OnceMaskedReprString("hook-canary"))}
|
||||
)
|
||||
|
||||
assert isinstance(frame, str)
|
||||
assert json.loads(frame.removeprefix("data: "))["v"] == "'***'"
|
||||
assert json.loads(frame.removeprefix("data: "))["v"] == printed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -2183,6 +2193,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina
|
|||
("error", {"error": {"message": "denied", "code": "529"}}, True),
|
||||
("response.completed", {"sequence_number": 4, "note": "typeless"}, False),
|
||||
("error", {"sequence_number": 4, "note": "no error object"}, False),
|
||||
("error", {"sequence_number": 4, "error": "denied"}, False),
|
||||
("error", {"sequence_number": 4, "error": ["denied"]}, False),
|
||||
],
|
||||
)
|
||||
async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failure_only_without_a_terminal(
|
||||
|
|
@ -2212,7 +2224,11 @@ async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failur
|
|||
assert failure.sequence_number == 5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sequence_number", ["9" * 4300, "-7", str(2**63)], ids=["4300_digits", "negative", "past_int64"])
|
||||
@pytest.mark.parametrize(
|
||||
"sequence_number",
|
||||
["9" * 4300, "-7", str(2**63), str(2**63 - 1)],
|
||||
ids=["4300_digits", "negative", "past_int64", "int64_max"],
|
||||
)
|
||||
def test_responses_error_event_after_an_out_of_range_pre_framed_sequence_number_stays_parseable(
|
||||
sequence_number: str,
|
||||
) -> None:
|
||||
|
|
@ -2232,6 +2248,53 @@ def test_responses_error_event_after_an_out_of_range_pre_framed_sequence_number_
|
|||
assert json.loads(failure_frame.split("data: ", 1)[1])["sequence_number"] == 5
|
||||
|
||||
|
||||
class _UnreadableErrorModel(BaseModel, frozen=True):
|
||||
type: str = "response.output_text.delta"
|
||||
sequence_number: int = 4
|
||||
|
||||
@property
|
||||
def error(self) -> object:
|
||||
raise RuntimeError("error is not readable")
|
||||
|
||||
|
||||
class _UnreadableErrorMapping(Mapping[str, object]):
|
||||
def __init__(self, data: Mapping[str, object]) -> None:
|
||||
self.data: Final = data
|
||||
|
||||
def __getitem__(self, key: str) -> object:
|
||||
if key == "error":
|
||||
raise RuntimeError("error is not readable")
|
||||
return self.data[key]
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self.data)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"chunk,next_sequence_number",
|
||||
[
|
||||
(_UnreadableErrorModel(), 5),
|
||||
(_UnreadableErrorMapping({"type": "response.output_text.delta", "sequence_number": 4}), 5),
|
||||
({"type": "response.output_text.delta", "sequence_number": 2**63}, 2**63 + 1),
|
||||
],
|
||||
ids=["unreadable_error_property", "unreadable_error_item", "past_int64"],
|
||||
)
|
||||
def test_responses_failure_after_a_hook_chunk_continues_its_sequence_number_like_base(
|
||||
chunk: object, next_sequence_number: int
|
||||
) -> None:
|
||||
state: Final = ResponsesStreamErrorState()
|
||||
state.observe_chunk(chunk)
|
||||
state.mark_emitted("data: {}\n\n")
|
||||
|
||||
failure_frame: Final = state.format_failure(RuntimeError("stream closed"))
|
||||
|
||||
assert failure_frame is not None
|
||||
assert json.loads(failure_frame.split("data: ", 1)[1])["sequence_number"] == next_sequence_number
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# select_data_generator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue