From fc5902667427db6aa42e8e42478c9e60dff5acfb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:20:17 -0700 Subject: [PATCH] fix(proxy): drop only the nested chunk value JSON cannot encode and keep lone surrogates Main removed the StreamingChoices import, so the field drop helpers now take any BaseModel. A value JSON cannot encode is left out down to the nested key or list item that holds it, and a lone surrogate goes out as an ASCII JSON escape. --- litellm/proxy/proxy_server.py | 42 +++++++++---- .../proxy_server/test_streaming_helpers.py | 62 +++++++++++++++++-- 2 files changed, 87 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bf202d1233e..da1a48c4ab9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -62,7 +62,7 @@ import websockets.exceptions from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config from pydantic.fields import FieldInfo, PydanticUndefined from pydantic.main import IncEx -from pydantic_core import PydanticSerializationError, to_json +from pydantic_core import PydanticSerializationError, to_jsonable_python from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never from litellm._uuid import uuid @@ -10047,6 +10047,8 @@ def _event_with_restamped_nested_model(event: Mapping[str, object], target_model _SSE_DATA_FIELD: Final = b"data:" _SSE_NESTED_RESPONSE_MARKER: Final = b'"response"' _SSE_EVENT_OBJECT: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_JSON_ARRAY: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) def _restamp_nested_response_model_in_sse_frame(frame: bytes, target_model: str) -> bytes: @@ -10074,32 +10076,46 @@ def _restamp_nested_response_model_in_sse_line(line: bytes, target_model: str) - def _json_encodes(value: object) -> bool: try: - to_json(value) - except PydanticSerializationError: + to_jsonable_python(value) + except ValueError: return False return True -def _unencodable_fields(model: BaseModel, nested_field: str | None) -> dict[str, bool]: - return {key: True for key, value in model if key != nested_field and not _json_encodes(value)} +def _unencodable_part(value: object, ancestors: frozenset[int]) -> IncEx | bool: + if id(value) in ancestors: + return True + lineage: Final = ancestors | {id(value)} + if isinstance(value, BaseModel): + return _unencodable_keys(iter(value), lineage) or True + if isinstance(value, Mapping): + return _unencodable_keys(_JSON_OBJECT.validate_python(value).items(), lineage) or True + if isinstance(value, (list, tuple)): + return _unencodable_indexes(_JSON_ARRAY.validate_python(value), lineage) or True + return True -def _unencodable_choice_fields(choice: StreamingChoices) -> Mapping[str, IncEx | bool]: - return {**_unencodable_fields(choice, "delta"), "delta": _unencodable_fields(choice.delta, None)} +def _unencodable_keys(items: Iterable[tuple[str, object]], lineage: frozenset[int]) -> Mapping[str, IncEx | bool]: + return {key: _unencodable_part(item, lineage) for key, item in items if not _json_encodes(item)} -def _serialize_chat_chunk_without_unencodable_fields(chunk: ModelResponseStream) -> bytes: - choices: Final = {index: _unencodable_choice_fields(choice) for index, choice in enumerate(chunk.choices)} - exclude: Final = {**_unencodable_fields(chunk, "choices"), "choices": choices} - verbose_proxy_logger.warning("Dropping streamed chunk fields that JSON cannot encode: %s", exclude) - return chunk.model_dump_json(exclude_none=True, exclude_unset=True, exclude=exclude).encode() +def _unencodable_indexes(items: Sequence[object], lineage: frozenset[int]) -> Mapping[int, IncEx | bool]: + return {index: _unencodable_part(item, lineage) for index, item in enumerate(items) if not _json_encodes(item)} + + +def _serialize_chat_chunk_leniently(chunk: ModelResponseStream) -> bytes: + exclude: Final = _unencodable_keys(iter(chunk), frozenset({id(chunk)})) + if exclude: + verbose_proxy_logger.warning("Dropping streamed chunk fields that JSON cannot encode: %s", exclude) + jsonable: Final = chunk.model_dump(mode="json", exclude_none=True, exclude_unset=True, exclude=exclude) + return json.dumps(jsonable, ensure_ascii=True, separators=(",", ":")).encode() def _serialize_chat_chunk(chunk: ModelResponseStream) -> bytes: try: return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode() except PydanticSerializationError: - return _serialize_chat_chunk_without_unencodable_fields(chunk) + return _serialize_chat_chunk_leniently(chunk) def _serialize_streaming_chunk(chunk: BaseModel) -> bytes: diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 1f8be5a0572..4f4b19d1b91 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -55,7 +55,14 @@ from litellm.types.llms.openai import ( ResponseFailedEvent, ResponsesAPIResponse, ) -from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices +from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + Usage, +) from tests._support.clock_loops import AutojumpClockLoop from .conftest import normalize @@ -771,7 +778,7 @@ async def test_async_data_generator_sse_frame_keeps_provider_fields_on_a_text_ch def _circular_dict() -> dict[str, object]: - looped: Final[dict[str, object]] = {} # mutable-ok: a circular dict has to be inserted into itself + looped: Final[dict[str, object]] = {"keep": 1} # mutable-ok: a circular dict has to be inserted into itself looped["self"] = looped return looped @@ -782,8 +789,6 @@ def _circular_dict() -> dict[str, object]: [ pytest.param(object(), id="unknown-type"), pytest.param(hashlib.sha256(b"trace").digest(), id="invalid-utf8-bytes"), - pytest.param("\ud800", id="lone-surrogate"), - pytest.param(_circular_dict(), id="circular-dict"), ], ) async def test_async_data_generator_streams_a_text_chunk_a_hook_tagged_with_a_value_json_cannot_encode( @@ -814,6 +819,55 @@ async def test_async_data_generator_streams_a_text_chunk_a_hook_tagged_with_a_va assert all(name in caplog.text for name in ("trace_chunk", "trace_choice", "trace_delta")) +def test_serialize_streaming_chunk_keeps_lone_surrogates_a_provider_split_across_frames( + caplog: pytest.LogCaptureFixture, +) -> None: + tool_call: Final = ChatCompletionDeltaToolCall( + id="call_1", index=0, type="function", function=Function(name="get_weather", arguments='{"city": "Par\udc00is"}') + ) + chunk: Final = ModelResponseStream( + id="chatcmpl-split", + choices=[StreamingChoices(index=0, delta=Delta(content="Hi \ud83d", tool_calls=[tool_call]))], + created=0, + model="gpt-4", + object="chat.completion.chunk", + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + delta: Final = json.loads(_serialize_streaming_chunk(chunk))["choices"][0]["delta"] + + assert caplog.text == "" + assert delta["content"] == "Hi \ud83d" + assert delta["tool_calls"][0]["id"] == "call_1" + assert delta["tool_calls"][0]["function"] == {"name": "get_weather", "arguments": '{"city": "Par\udc00is"}'} + + +def test_serialize_streaming_chunk_drops_only_the_nested_value_json_cannot_encode() -> None: + digest: Final = hashlib.sha256(b"trace").digest() + chunk: Final = ModelResponseStream( + id="chatcmpl-nested", + choices=[ + StreamingChoices(index=0, delta=Delta(content="a")), + StreamingChoices( + index=1, delta=Delta(content="b", provider_specific_fields={"citations": ["ok"], "raw": digest}) + ), + ], + created=0, + model="gpt-4", + object="chat.completion.chunk", + usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5), + ) + setattr(chunk.usage, "raw", digest) + setattr(chunk, "trace_chunk", _circular_dict()) + + payload: Final = json.loads(_serialize_streaming_chunk(chunk)) + + assert payload["choices"][0]["delta"] == {"content": "a"} + assert payload["choices"][1]["delta"] == {"content": "b", "provider_specific_fields": {"citations": ["ok"]}} + assert payload["usage"] == {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5} + assert payload["trace_chunk"] == {"keep": 1} + + @pytest.mark.asyncio async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch): _patch_logging_flags(monkeypatch)