mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): match base on Responses stream edge cases
Read a decimal-string or integral-float status_code as its integer again, so "429" and 429.0 still map to rate_limit_exceeded. Print each hook value's repr once and reuse it, so a repr that changes between calls cannot reach the client. Track only pre-framed sequence numbers from 0 to 2**63-1, so a huge one cannot break the failure event. Build nested JSON with one frame per level again, which restores the nesting depth sent as JSON. Count a pre-framed frame as terminal from its data type, and a named error frame only when its data carries an error, the way the OpenAI SDKs read them.
This commit is contained in:
parent
8ac651b486
commit
da876856fc
2 changed files with 119 additions and 26 deletions
|
|
@ -9,7 +9,7 @@ from itertools import chain, groupby, zip_longest
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
|
||||
from typing_extensions import TypeIs # noqa: TID251 # the guards narrow hook values to object items, isinstance cannot
|
||||
|
||||
from litellm._logging import redact_internal_details_from_client_message
|
||||
|
|
@ -33,6 +33,10 @@ class _StreamEvent(LiteLLMBaseModel):
|
|||
type: str | None = None
|
||||
sequence_number: int | None = None
|
||||
response: _ResponseIdentity | None = None
|
||||
error: object = None
|
||||
|
||||
|
||||
_LAX_INT: Final = TypeAdapter(int)
|
||||
|
||||
|
||||
class _FailureDetails(LiteLLMBaseModel):
|
||||
|
|
@ -46,7 +50,12 @@ class _FailureDetails(LiteLLMBaseModel):
|
|||
@field_validator("status_code", mode="before")
|
||||
@classmethod
|
||||
def normalize_status_code(cls, value: object) -> int | None:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return _LAX_INT.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
@field_validator("message", mode="before")
|
||||
@classmethod
|
||||
|
|
@ -120,10 +129,10 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma
|
|||
return isinstance(value, MappingProxyType)
|
||||
|
||||
|
||||
def _printed_proxy_items(proxy: Mapping[object, object], printed: str) -> Iterable[tuple[object, object]] | None:
|
||||
def _printed_proxy_snapshot(proxy: Mapping[object, object], printed: str) -> Mapping[object, object] | None:
|
||||
snapshot: Final = dict(proxy)
|
||||
shown: Final = repr(snapshot)
|
||||
return snapshot.items() if printed in (shown, f"mappingproxy({shown})") else None
|
||||
return snapshot if printed in (shown, f"mappingproxy({shown})") else None
|
||||
|
||||
|
||||
def _repr_prints_its_fields(model: BaseModel) -> bool:
|
||||
|
|
@ -139,18 +148,27 @@ def _plain_scalar(value: str | float) -> str | float:
|
|||
return float.__float__(value)
|
||||
|
||||
|
||||
def _repr_shows_its_value(value: str | float) -> bool:
|
||||
def _repr_shows_its_value(value: str | float, printed: str) -> bool:
|
||||
plain: Final = _plain_scalar(value)
|
||||
if isinstance(value, Enum):
|
||||
return repr(value) == f"<{type(value).__name__}.{value.name}: {plain!r}>"
|
||||
return repr(value) == repr(plain)
|
||||
return printed == f"<{type(value).__name__}.{value.name}: {plain!r}>"
|
||||
return printed == repr(plain)
|
||||
|
||||
|
||||
def _printed_text(value: object) -> object:
|
||||
printed: Final = repr(value)
|
||||
if isinstance(value, (str, int, float)) and _repr_shows_its_value(value, printed):
|
||||
return _plain_scalar(value)
|
||||
return printed
|
||||
|
||||
|
||||
def _dumped_text(value: object) -> object:
|
||||
return _plain_scalar(value) if isinstance(value, (str, int, float)) else str(value)
|
||||
|
||||
|
||||
def _text(value: object, form: _BaseForm) -> object:
|
||||
try:
|
||||
if isinstance(value, (str, int, float)) and (form is _BaseForm.DUMPED or _repr_shows_its_value(value)):
|
||||
return _plain_scalar(value)
|
||||
return repr(value) if form is _BaseForm.PRINTED else str(value)
|
||||
return _printed_text(value) if form is _BaseForm.PRINTED else _dumped_text(value)
|
||||
except Exception: # noqa: BLE001 # hook values run their own __repr__ and __str__, so a failure drops the value
|
||||
return _DROPPED
|
||||
|
||||
|
|
@ -205,8 +223,8 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o
|
|||
if shown_items is not None:
|
||||
return shown_items, form
|
||||
if _is_proxy(value):
|
||||
proxy_items: Final = _printed_proxy_items(value, repr(value) if form is _BaseForm.PRINTED else str(value))
|
||||
return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED)
|
||||
snapshot: Final = _printed_proxy_snapshot(value, repr(value) if form is _BaseForm.PRINTED else str(value))
|
||||
return None if snapshot is None else (snapshot.items(), _BaseForm.PRINTED)
|
||||
if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value):
|
||||
return value.__repr_args__(), _BaseForm.PRINTED
|
||||
return None
|
||||
|
|
@ -227,23 +245,18 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
|
|||
if expansion is None:
|
||||
return _text(value, form)
|
||||
items, inner = expansion
|
||||
return _items_json(items, inner, inner_path)
|
||||
pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items]
|
||||
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit
|
||||
return _text(value, form)
|
||||
|
||||
|
||||
def _items_json(
|
||||
items: Iterable[tuple[object, object]], form: _BaseForm, path: frozenset[int]
|
||||
) -> Mapping[object, object]:
|
||||
pairs: Final = [(_json_key(key, form), _json_safe(item, form, path)) for key, item in items]
|
||||
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
|
||||
|
||||
|
||||
def _proxy_chunk_data(proxy: Mapping[object, object]) -> str:
|
||||
printed: Final = format(proxy, "")
|
||||
try:
|
||||
items: Final = _printed_proxy_items(proxy, printed)
|
||||
return printed if items is None else json.dumps(_items_json(items, _BaseForm.PRINTED, frozenset((id(proxy),))))
|
||||
snapshot: Final = _printed_proxy_snapshot(proxy, printed)
|
||||
shown: Final = None if snapshot is None else _json_safe(snapshot, _BaseForm.PRINTED, frozenset((id(proxy),)))
|
||||
return json.dumps(shown) if isinstance(shown, dict) else printed
|
||||
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
|
||||
return printed
|
||||
|
||||
|
|
@ -364,6 +377,7 @@ def _error_event(
|
|||
|
||||
SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
|
||||
_TERMINAL_EVENT_TYPES: Final = frozenset(("response.completed", "response.failed", "response.incomplete", "error"))
|
||||
_MAX_SEQUENCE_NUMBER: Final = 2**63 - 1
|
||||
|
||||
|
||||
def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]:
|
||||
|
|
@ -379,11 +393,10 @@ def _parsed_event(data: str) -> _StreamEvent:
|
|||
|
||||
|
||||
def _block_event(lines: Sequence[str]) -> _StreamEvent:
|
||||
named: Final = _field_values(lines, "event")
|
||||
parsed: Final = _parsed_event("\n".join(_field_values(lines, "data")))
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not named:
|
||||
if parsed.type in _TERMINAL_EVENT_TYPES or not parsed.error or _field_values(lines, "event")[-1:] != ("error",):
|
||||
return parsed
|
||||
return parsed.model_copy(update={"type": named[-1]})
|
||||
return parsed.model_copy(update={"type": "error"})
|
||||
|
||||
|
||||
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
|
||||
|
|
@ -404,7 +417,7 @@ class ResponsesStreamErrorState:
|
|||
self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None
|
||||
|
||||
def _record(self, event: _StreamEvent) -> None:
|
||||
if event.sequence_number is not None:
|
||||
if event.sequence_number is not None and 0 <= event.sequence_number <= _MAX_SEQUENCE_NUMBER:
|
||||
self.sequence_number = max(self.sequence_number, event.sequence_number)
|
||||
if event.response is not None:
|
||||
self.response_id = event.response.id or self.response_id
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import functools
|
||||
import itertools
|
||||
import json
|
||||
from collections import Counter, OrderedDict
|
||||
from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping
|
||||
|
|
@ -347,6 +348,24 @@ def test_responses_sse_chunk_never_shows_what_a_nested_value_hides_from_its_repr
|
|||
assert "hook-canary" not in frame
|
||||
|
||||
|
||||
class _OnceMaskedReprString(str):
|
||||
def __init__(self, value: str) -> None:
|
||||
super().__init__()
|
||||
self.reprs: Final = itertools.count()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "'***'" if next(self.reprs) == 0 else super().__repr__()
|
||||
|
||||
|
||||
def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it() -> None:
|
||||
frame: Final = _format_responses_sse_chunk(
|
||||
chunk={"type": "response.output_text.delta", "v": _OnceMaskedReprString("hook-canary")}
|
||||
)
|
||||
|
||||
assert isinstance(frame, str)
|
||||
assert json.loads(frame.removeprefix("data: "))["v"] == "'***'"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"nested",
|
||||
[*_MASKED_IN_EVERY_FRAME, _MaskedStrModel(token="hook-canary")],
|
||||
|
|
@ -549,6 +568,32 @@ def test_responses_hook_error_frame_keeps_its_message_whatever_status_code_it_ca
|
|||
assert json.loads(frame.split("data: ", 1)[1])["message"] == "rig status"
|
||||
|
||||
|
||||
class _StatusCarryingError(Exception):
|
||||
def __init__(self, status_code: object) -> None:
|
||||
super().__init__("rig status")
|
||||
self.status_code: Final = status_code
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "code"),
|
||||
[
|
||||
(429, "rate_limit_exceeded"),
|
||||
("429", "rate_limit_exceeded"),
|
||||
(429.0, "rate_limit_exceeded"),
|
||||
("abc", "server_error"),
|
||||
(429.5, "server_error"),
|
||||
],
|
||||
ids=["int", "decimal_string", "integral_float", "word", "fractional_float"],
|
||||
)
|
||||
def test_responses_failure_event_reads_a_numeric_status_code_the_way_base_did(status_code: object, code: str) -> None:
|
||||
frame: Final = ResponsesStreamErrorState().format_failure(_StatusCarryingError(status_code))
|
||||
|
||||
assert frame is not None
|
||||
failure: Final = ResponseFailedEvent.model_validate_json(frame.split("data: ", 1)[1])
|
||||
assert failure.response.error is not None
|
||||
assert failure.response.error["code"] == code
|
||||
|
||||
|
||||
class _RigColor(Enum):
|
||||
RED = 1
|
||||
|
||||
|
|
@ -718,6 +763,19 @@ def test_responses_error_event_keeps_a_detail_nested_past_the_recursion_limit()
|
|||
assert "'leaf': 1" in frame
|
||||
|
||||
|
||||
def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json() -> None:
|
||||
nested: Final = functools.reduce(lambda inner, _: {"x": inner}, range(700), {"leaf": 1})
|
||||
|
||||
chunk_frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested})
|
||||
error_frame: Final = ResponsesStreamErrorState().format_error(
|
||||
{"message": "blocked", "provider_specific_fields": {"detail": nested}}
|
||||
)
|
||||
|
||||
assert isinstance(chunk_frame, str)
|
||||
assert json.loads(chunk_frame.removeprefix("data: "))["nested"] == nested
|
||||
assert json.loads(error_frame.split("data: ", 1)[1])["error"]["provider_specific_fields"]["detail"] == nested
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"chunk",
|
||||
[
|
||||
|
|
@ -2123,6 +2181,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina
|
|||
True,
|
||||
),
|
||||
("error", {"error": {"message": "denied", "code": "529"}}, True),
|
||||
("response.completed", {"sequence_number": 4, "note": "typeless"}, False),
|
||||
("error", {"sequence_number": 4, "note": "no error object"}, False),
|
||||
],
|
||||
)
|
||||
async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failure_only_without_a_terminal(
|
||||
|
|
@ -2152,6 +2212,26 @@ async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failur
|
|||
assert failure.sequence_number == 5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sequence_number", ["9" * 4300, "-7", str(2**63)], ids=["4300_digits", "negative", "past_int64"])
|
||||
def test_responses_error_event_after_an_out_of_range_pre_framed_sequence_number_stays_parseable(
|
||||
sequence_number: str,
|
||||
) -> None:
|
||||
state: Final = ResponsesStreamErrorState()
|
||||
state.observe_frames(
|
||||
'data: {"type": "response.output_text.delta", "sequence_number": 4}\n\n'
|
||||
f'data: {{"type": "response.output_text.delta", "sequence_number": {sequence_number}}}\n\n'
|
||||
)
|
||||
|
||||
error_frame: Final = state.format_error_message('data: {"error": {"message": "scan failed"}}\n\n')
|
||||
failure_frame: Final = state.format_failure(RuntimeError("stream closed"))
|
||||
|
||||
error_event: Final = json.loads(error_frame.split("data: ", 1)[1])
|
||||
assert failure_frame is not None
|
||||
assert error_event["message"] == "scan failed"
|
||||
assert error_event["sequence_number"] == 5
|
||||
assert json.loads(failure_frame.split("data: ", 1)[1])["sequence_number"] == 5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# select_data_generator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue