mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): keep streaming when an unencodable value sits under non-string dict keys
This commit is contained in:
parent
fc59026674
commit
de28986123
2 changed files with 39 additions and 6 deletions
|
|
@ -10047,7 +10047,7 @@ def _event_with_restamped_nested_model(event: Mapping[str, object], target_model
|
|||
_SSE_DATA_FIELD: Final = b"data:"
|
||||
_SSE_NESTED_RESPONSE_MARKER: Final = b'"response"'
|
||||
_SSE_EVENT_OBJECT: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[object, object])
|
||||
_JSON_ARRAY: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
|
|
@ -10088,10 +10088,20 @@ def _unencodable_part(value: object, ancestors: frozenset[int]) -> IncEx | bool:
|
|||
lineage: Final = ancestors | {id(value)}
|
||||
if isinstance(value, BaseModel):
|
||||
return _unencodable_keys(iter(value), lineage) or True
|
||||
if isinstance(value, Mapping):
|
||||
return _unencodable_keys(_JSON_OBJECT.validate_python(value).items(), lineage) or True
|
||||
if isinstance(value, dict):
|
||||
return _unencodable_entries(_JSON_OBJECT.validate_python(value), lineage)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return _unencodable_indexes(_JSON_ARRAY.validate_python(value), lineage) or True
|
||||
return _unencodable_indexes(enumerate(_JSON_ARRAY.validate_python(value)), lineage) or True
|
||||
return True
|
||||
|
||||
|
||||
def _unencodable_entries(entries: Mapping[object, object], lineage: frozenset[int]) -> IncEx | bool:
|
||||
by_name: Final = {key: item for key, item in entries.items() if isinstance(key, str)}
|
||||
if len(by_name) == len(entries):
|
||||
return _unencodable_keys(by_name.items(), lineage) or True
|
||||
by_index: Final = {key: item for key, item in entries.items() if isinstance(key, int)}
|
||||
if len(by_index) == len(entries):
|
||||
return _unencodable_indexes(by_index.items(), lineage) or True
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -10099,8 +10109,8 @@ def _unencodable_keys(items: Iterable[tuple[str, object]], lineage: frozenset[in
|
|||
return {key: _unencodable_part(item, lineage) for key, item in items if not _json_encodes(item)}
|
||||
|
||||
|
||||
def _unencodable_indexes(items: Sequence[object], lineage: frozenset[int]) -> Mapping[int, IncEx | bool]:
|
||||
return {index: _unencodable_part(item, lineage) for index, item in enumerate(items) if not _json_encodes(item)}
|
||||
def _unencodable_indexes(items: Iterable[tuple[int, object]], lineage: frozenset[int]) -> Mapping[int, IncEx | bool]:
|
||||
return {index: _unencodable_part(item, lineage) for index, item in items if not _json_encodes(item)}
|
||||
|
||||
|
||||
def _serialize_chat_chunk_leniently(chunk: ModelResponseStream) -> bytes:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import hashlib
|
|||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -868,6 +869,28 @@ def test_serialize_streaming_chunk_drops_only_the_nested_value_json_cannot_encod
|
|||
assert payload["trace_chunk"] == {"keep": 1}
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_keeps_streaming_when_unencodable_values_sit_under_non_string_keys() -> None:
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-keys",
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content="still streaming"))],
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
setattr(chunk, "by_index", {0: "kept", 1: object()})
|
||||
setattr(chunk, "mixed_keys", {"name": "x", 2: object()})
|
||||
setattr(chunk, "bytes_keys", {b"raw": object()})
|
||||
setattr(chunk, "read_only_view", MappingProxyType({"name": object()}))
|
||||
|
||||
payload: Final = json.loads(_serialize_streaming_chunk(chunk))
|
||||
|
||||
assert payload["choices"][0]["delta"] == {"content": "still streaming"}
|
||||
assert payload["by_index"] == {"0": "kept"}
|
||||
assert "mixed_keys" not in payload
|
||||
assert "bytes_keys" not in payload
|
||||
assert "read_only_view" not in payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue