fix(proxy): match base on Responses stream edge cases

Read a decimal-string or integral-float status_code as its integer again, so
"429" and 429.0 still map to rate_limit_exceeded. Print each hook value's repr
once and reuse it, so a repr that changes between calls cannot reach the
client. Track only pre-framed sequence numbers from 0 to 2**63-1, so a huge
one cannot break the failure event. Build nested JSON with one frame per level
again, which restores the nesting depth sent as JSON. Count a pre-framed frame
as terminal from its data type, and a named error frame only when its data
carries an error, the way the OpenAI SDKs read them.
This commit is contained in:
mateo-berri 2026-10-31 02:56:19 -07:00
parent 8ac651b486
commit da876856fc
2 changed files with 119 additions and 26 deletions

View file

@ -9,7 +9,7 @@ from itertools import chain, groupby, zip_longest
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, ValidationError, field_validator
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
from typing_extensions import TypeIs # noqa: TID251 # the guards narrow hook values to object items, isinstance cannot
from litellm._logging import redact_internal_details_from_client_message
@ -33,6 +33,10 @@ class _StreamEvent(LiteLLMBaseModel):
type: str | None = None
sequence_number: int | None = None
response: _ResponseIdentity | None = None
error: object = None
_LAX_INT: Final = TypeAdapter(int)
class _FailureDetails(LiteLLMBaseModel):
@ -46,7 +50,12 @@ class _FailureDetails(LiteLLMBaseModel):
@field_validator("status_code", mode="before")
@classmethod
def normalize_status_code(cls, value: object) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) else None
if isinstance(value, bool):
return None
try:
return _LAX_INT.validate_python(value)
except ValidationError:
return None
@field_validator("message", mode="before")
@classmethod
@ -120,10 +129,10 @@ def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: ma
return isinstance(value, MappingProxyType)
def _printed_proxy_items(proxy: Mapping[object, object], printed: str) -> Iterable[tuple[object, object]] | None:
def _printed_proxy_snapshot(proxy: Mapping[object, object], printed: str) -> Mapping[object, object] | None:
snapshot: Final = dict(proxy)
shown: Final = repr(snapshot)
return snapshot.items() if printed in (shown, f"mappingproxy({shown})") else None
return snapshot if printed in (shown, f"mappingproxy({shown})") else None
def _repr_prints_its_fields(model: BaseModel) -> bool:
@ -139,18 +148,27 @@ def _plain_scalar(value: str | float) -> str | float:
return float.__float__(value)
def _repr_shows_its_value(value: str | float) -> bool:
def _repr_shows_its_value(value: str | float, printed: str) -> bool:
plain: Final = _plain_scalar(value)
if isinstance(value, Enum):
return repr(value) == f"<{type(value).__name__}.{value.name}: {plain!r}>"
return repr(value) == repr(plain)
return printed == f"<{type(value).__name__}.{value.name}: {plain!r}>"
return printed == repr(plain)
def _printed_text(value: object) -> object:
printed: Final = repr(value)
if isinstance(value, (str, int, float)) and _repr_shows_its_value(value, printed):
return _plain_scalar(value)
return printed
def _dumped_text(value: object) -> object:
return _plain_scalar(value) if isinstance(value, (str, int, float)) else str(value)
def _text(value: object, form: _BaseForm) -> object:
try:
if isinstance(value, (str, int, float)) and (form is _BaseForm.DUMPED or _repr_shows_its_value(value)):
return _plain_scalar(value)
return repr(value) if form is _BaseForm.PRINTED else str(value)
return _printed_text(value) if form is _BaseForm.PRINTED else _dumped_text(value)
except Exception: # noqa: BLE001 # hook values run their own __repr__ and __str__, so a failure drops the value
return _DROPPED
@ -205,8 +223,8 @@ def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, o
if shown_items is not None:
return shown_items, form
if _is_proxy(value):
proxy_items: Final = _printed_proxy_items(value, repr(value) if form is _BaseForm.PRINTED else str(value))
return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED)
snapshot: Final = _printed_proxy_snapshot(value, repr(value) if form is _BaseForm.PRINTED else str(value))
return None if snapshot is None else (snapshot.items(), _BaseForm.PRINTED)
if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value):
return value.__repr_args__(), _BaseForm.PRINTED
return None
@ -227,23 +245,18 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object:
if expansion is None:
return _text(value, form)
items, inner = expansion
return _items_json(items, inner, inner_path)
pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items]
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit
return _text(value, form)
def _items_json(
items: Iterable[tuple[object, object]], form: _BaseForm, path: frozenset[int]
) -> Mapping[object, object]:
pairs: Final = [(_json_key(key, form), _json_safe(item, form, path)) for key, item in items]
return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED}
def _proxy_chunk_data(proxy: Mapping[object, object]) -> str:
printed: Final = format(proxy, "")
try:
items: Final = _printed_proxy_items(proxy, printed)
return printed if items is None else json.dumps(_items_json(items, _BaseForm.PRINTED, frozenset((id(proxy),))))
snapshot: Final = _printed_proxy_snapshot(proxy, printed)
shown: Final = None if snapshot is None else _json_safe(snapshot, _BaseForm.PRINTED, frozenset((id(proxy),)))
return json.dumps(shown) if isinstance(shown, dict) else printed
except Exception: # noqa: BLE001 # hook values run their own printers, so the text base printed stands
return printed
@ -364,6 +377,7 @@ def _error_event(
SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
_TERMINAL_EVENT_TYPES: Final = frozenset(("response.completed", "response.failed", "response.incomplete", "error"))
_MAX_SEQUENCE_NUMBER: Final = 2**63 - 1
def _field_values(lines: Sequence[str], name: str) -> tuple[str, ...]:
@ -379,11 +393,10 @@ def _parsed_event(data: str) -> _StreamEvent:
def _block_event(lines: Sequence[str]) -> _StreamEvent:
named: Final = _field_values(lines, "event")
parsed: Final = _parsed_event("\n".join(_field_values(lines, "data")))
if parsed.type in _TERMINAL_EVENT_TYPES or not named:
if parsed.type in _TERMINAL_EVENT_TYPES or not parsed.error or _field_values(lines, "event")[-1:] != ("error",):
return parsed
return parsed.model_copy(update={"type": named[-1]})
return parsed.model_copy(update={"type": "error"})
def _frame_events(frames: str) -> tuple[_StreamEvent, ...]:
@ -404,7 +417,7 @@ class ResponsesStreamErrorState:
self._pending_event = _StreamEvent.model_validate(chunk) if isinstance(chunk, (BaseModel, Mapping)) else None
def _record(self, event: _StreamEvent) -> None:
if event.sequence_number is not None:
if event.sequence_number is not None and 0 <= event.sequence_number <= _MAX_SEQUENCE_NUMBER:
self.sequence_number = max(self.sequence_number, event.sequence_number)
if event.response is not None:
self.response_id = event.response.id or self.response_id

View file

@ -17,6 +17,7 @@ from __future__ import annotations
import asyncio
import functools
import itertools
import json
from collections import Counter, OrderedDict
from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, KeysView, Mapping
@ -347,6 +348,24 @@ def test_responses_sse_chunk_never_shows_what_a_nested_value_hides_from_its_repr
assert "hook-canary" not in frame
class _OnceMaskedReprString(str):
def __init__(self, value: str) -> None:
super().__init__()
self.reprs: Final = itertools.count()
def __repr__(self) -> str:
return "'***'" if next(self.reprs) == 0 else super().__repr__()
def test_responses_sse_chunk_prints_a_nested_string_once_the_way_base_printed_it() -> None:
frame: Final = _format_responses_sse_chunk(
chunk={"type": "response.output_text.delta", "v": _OnceMaskedReprString("hook-canary")}
)
assert isinstance(frame, str)
assert json.loads(frame.removeprefix("data: "))["v"] == "'***'"
@pytest.mark.parametrize(
"nested",
[*_MASKED_IN_EVERY_FRAME, _MaskedStrModel(token="hook-canary")],
@ -549,6 +568,32 @@ def test_responses_hook_error_frame_keeps_its_message_whatever_status_code_it_ca
assert json.loads(frame.split("data: ", 1)[1])["message"] == "rig status"
class _StatusCarryingError(Exception):
def __init__(self, status_code: object) -> None:
super().__init__("rig status")
self.status_code: Final = status_code
@pytest.mark.parametrize(
("status_code", "code"),
[
(429, "rate_limit_exceeded"),
("429", "rate_limit_exceeded"),
(429.0, "rate_limit_exceeded"),
("abc", "server_error"),
(429.5, "server_error"),
],
ids=["int", "decimal_string", "integral_float", "word", "fractional_float"],
)
def test_responses_failure_event_reads_a_numeric_status_code_the_way_base_did(status_code: object, code: str) -> None:
frame: Final = ResponsesStreamErrorState().format_failure(_StatusCarryingError(status_code))
assert frame is not None
failure: Final = ResponseFailedEvent.model_validate_json(frame.split("data: ", 1)[1])
assert failure.response.error is not None
assert failure.response.error["code"] == code
class _RigColor(Enum):
RED = 1
@ -718,6 +763,19 @@ def test_responses_error_event_keeps_a_detail_nested_past_the_recursion_limit()
assert "'leaf': 1" in frame
def test_responses_stream_frames_keep_a_value_nested_700_levels_deep_as_json() -> None:
nested: Final = functools.reduce(lambda inner, _: {"x": inner}, range(700), {"leaf": 1})
chunk_frame: Final = _format_responses_sse_chunk(chunk={"type": "response.output_text.delta", "nested": nested})
error_frame: Final = ResponsesStreamErrorState().format_error(
{"message": "blocked", "provider_specific_fields": {"detail": nested}}
)
assert isinstance(chunk_frame, str)
assert json.loads(chunk_frame.removeprefix("data: "))["nested"] == nested
assert json.loads(error_frame.split("data: ", 1)[1])["error"]["provider_specific_fields"]["detail"] == nested
@pytest.mark.parametrize(
"chunk",
[
@ -2123,6 +2181,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina
True,
),
("error", {"error": {"message": "denied", "code": "529"}}, True),
("response.completed", {"sequence_number": 4, "note": "typeless"}, False),
("error", {"sequence_number": 4, "note": "no error object"}, False),
],
)
async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failure_only_without_a_terminal(
@ -2152,6 +2212,26 @@ async def test_a_responses_stream_failing_after_a_pre_framed_event_adds_a_failur
assert failure.sequence_number == 5
@pytest.mark.parametrize("sequence_number", ["9" * 4300, "-7", str(2**63)], ids=["4300_digits", "negative", "past_int64"])
def test_responses_error_event_after_an_out_of_range_pre_framed_sequence_number_stays_parseable(
sequence_number: str,
) -> None:
state: Final = ResponsesStreamErrorState()
state.observe_frames(
'data: {"type": "response.output_text.delta", "sequence_number": 4}\n\n'
f'data: {{"type": "response.output_text.delta", "sequence_number": {sequence_number}}}\n\n'
)
error_frame: Final = state.format_error_message('data: {"error": {"message": "scan failed"}}\n\n')
failure_frame: Final = state.format_failure(RuntimeError("stream closed"))
error_event: Final = json.loads(error_frame.split("data: ", 1)[1])
assert failure_frame is not None
assert error_event["message"] == "scan failed"
assert error_event["sequence_number"] == 5
assert json.loads(failure_frame.split("data: ", 1)[1])["sequence_number"] == 5
# ---------------------------------------------------------------------------
# select_data_generator
# ---------------------------------------------------------------------------