From de289861237df1d76a2b5d433abcd50a589a5593 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 05:01:39 -0800 Subject: [PATCH] fix(proxy): keep streaming when an unencodable value sits under non-string dict keys --- litellm/proxy/proxy_server.py | 22 +++++++++++++----- .../proxy_server/test_streaming_helpers.py | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index da1a48c4ab9..4651912594a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10047,7 +10047,7 @@ def _event_with_restamped_nested_model(event: Mapping[str, object], target_model _SSE_DATA_FIELD: Final = b"data:" _SSE_NESTED_RESPONSE_MARKER: Final = b'"response"' _SSE_EVENT_OBJECT: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -_JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_JSON_OBJECT: Final = TypeAdapter(dict[object, object]) _JSON_ARRAY: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) @@ -10088,10 +10088,20 @@ def _unencodable_part(value: object, ancestors: frozenset[int]) -> IncEx | bool: lineage: Final = ancestors | {id(value)} if isinstance(value, BaseModel): return _unencodable_keys(iter(value), lineage) or True - if isinstance(value, Mapping): - return _unencodable_keys(_JSON_OBJECT.validate_python(value).items(), lineage) or True + if isinstance(value, dict): + return _unencodable_entries(_JSON_OBJECT.validate_python(value), lineage) if isinstance(value, (list, tuple)): - return _unencodable_indexes(_JSON_ARRAY.validate_python(value), lineage) or True + return _unencodable_indexes(enumerate(_JSON_ARRAY.validate_python(value)), lineage) or True + return True + + +def _unencodable_entries(entries: Mapping[object, object], lineage: frozenset[int]) -> IncEx | bool: + by_name: Final = {key: item for key, item in entries.items() if isinstance(key, str)} + if len(by_name) == len(entries): + return _unencodable_keys(by_name.items(), lineage) or True + by_index: Final = {key: item for key, item in entries.items() if isinstance(key, int)} + if len(by_index) == len(entries): + return _unencodable_indexes(by_index.items(), lineage) or True return True @@ -10099,8 +10109,8 @@ def _unencodable_keys(items: Iterable[tuple[str, object]], lineage: frozenset[in return {key: _unencodable_part(item, lineage) for key, item in items if not _json_encodes(item)} -def _unencodable_indexes(items: Sequence[object], lineage: frozenset[int]) -> Mapping[int, IncEx | bool]: - return {index: _unencodable_part(item, lineage) for index, item in enumerate(items) if not _json_encodes(item)} +def _unencodable_indexes(items: Iterable[tuple[int, object]], lineage: frozenset[int]) -> Mapping[int, IncEx | bool]: + return {index: _unencodable_part(item, lineage) for index, item in items if not _json_encodes(item)} def _serialize_chat_chunk_leniently(chunk: ModelResponseStream) -> bytes: diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 4f4b19d1b91..104f4a8bb7a 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -19,6 +19,7 @@ import hashlib import json import logging from collections.abc import AsyncIterator, Callable +from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -868,6 +869,28 @@ def test_serialize_streaming_chunk_drops_only_the_nested_value_json_cannot_encod assert payload["trace_chunk"] == {"keep": 1} +def test_serialize_streaming_chunk_keeps_streaming_when_unencodable_values_sit_under_non_string_keys() -> None: + chunk: Final = ModelResponseStream( + id="chatcmpl-keys", + choices=[StreamingChoices(index=0, delta=Delta(content="still streaming"))], + created=0, + model="gpt-4", + object="chat.completion.chunk", + ) + setattr(chunk, "by_index", {0: "kept", 1: object()}) + setattr(chunk, "mixed_keys", {"name": "x", 2: object()}) + setattr(chunk, "bytes_keys", {b"raw": object()}) + setattr(chunk, "read_only_view", MappingProxyType({"name": object()})) + + payload: Final = json.loads(_serialize_streaming_chunk(chunk)) + + assert payload["choices"][0]["delta"] == {"content": "still streaming"} + assert payload["by_index"] == {"0": "kept"} + assert "mixed_keys" not in payload + assert "bytes_keys" not in payload + assert "read_only_view" not in payload + + @pytest.mark.asyncio async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch): _patch_logging_flags(monkeypatch)