mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): drop only the nested chunk value JSON cannot encode and keep lone surrogates
Main removed the StreamingChoices import, so the field drop helpers now take any BaseModel. A value JSON cannot encode is left out down to the nested key or list item that holds it, and a lone surrogate goes out as an ASCII JSON escape.
This commit is contained in:
parent
4ffb78800d
commit
fc59026674
2 changed files with 87 additions and 17 deletions
|
|
@ -62,7 +62,7 @@ import websockets.exceptions
|
|||
from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config
|
||||
from pydantic.fields import FieldInfo, PydanticUndefined
|
||||
from pydantic.main import IncEx
|
||||
from pydantic_core import PydanticSerializationError, to_json
|
||||
from pydantic_core import PydanticSerializationError, to_jsonable_python
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -10047,6 +10047,8 @@ def _event_with_restamped_nested_model(event: Mapping[str, object], target_model
|
|||
_SSE_DATA_FIELD: Final = b"data:"
|
||||
_SSE_NESTED_RESPONSE_MARKER: Final = b'"response"'
|
||||
_SSE_EVENT_OBJECT: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])
|
||||
_JSON_ARRAY: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _restamp_nested_response_model_in_sse_frame(frame: bytes, target_model: str) -> bytes:
|
||||
|
|
@ -10074,32 +10076,46 @@ def _restamp_nested_response_model_in_sse_line(line: bytes, target_model: str) -
|
|||
|
||||
def _json_encodes(value: object) -> bool:
|
||||
try:
|
||||
to_json(value)
|
||||
except PydanticSerializationError:
|
||||
to_jsonable_python(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _unencodable_fields(model: BaseModel, nested_field: str | None) -> dict[str, bool]:
|
||||
return {key: True for key, value in model if key != nested_field and not _json_encodes(value)}
|
||||
def _unencodable_part(value: object, ancestors: frozenset[int]) -> IncEx | bool:
|
||||
if id(value) in ancestors:
|
||||
return True
|
||||
lineage: Final = ancestors | {id(value)}
|
||||
if isinstance(value, BaseModel):
|
||||
return _unencodable_keys(iter(value), lineage) or True
|
||||
if isinstance(value, Mapping):
|
||||
return _unencodable_keys(_JSON_OBJECT.validate_python(value).items(), lineage) or True
|
||||
if isinstance(value, (list, tuple)):
|
||||
return _unencodable_indexes(_JSON_ARRAY.validate_python(value), lineage) or True
|
||||
return True
|
||||
|
||||
|
||||
def _unencodable_choice_fields(choice: StreamingChoices) -> Mapping[str, IncEx | bool]:
|
||||
return {**_unencodable_fields(choice, "delta"), "delta": _unencodable_fields(choice.delta, None)}
|
||||
def _unencodable_keys(items: Iterable[tuple[str, object]], lineage: frozenset[int]) -> Mapping[str, IncEx | bool]:
|
||||
return {key: _unencodable_part(item, lineage) for key, item in items if not _json_encodes(item)}
|
||||
|
||||
|
||||
def _serialize_chat_chunk_without_unencodable_fields(chunk: ModelResponseStream) -> bytes:
|
||||
choices: Final = {index: _unencodable_choice_fields(choice) for index, choice in enumerate(chunk.choices)}
|
||||
exclude: Final = {**_unencodable_fields(chunk, "choices"), "choices": choices}
|
||||
verbose_proxy_logger.warning("Dropping streamed chunk fields that JSON cannot encode: %s", exclude)
|
||||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True, exclude=exclude).encode()
|
||||
def _unencodable_indexes(items: Sequence[object], lineage: frozenset[int]) -> Mapping[int, IncEx | bool]:
|
||||
return {index: _unencodable_part(item, lineage) for index, item in enumerate(items) if not _json_encodes(item)}
|
||||
|
||||
|
||||
def _serialize_chat_chunk_leniently(chunk: ModelResponseStream) -> bytes:
|
||||
exclude: Final = _unencodable_keys(iter(chunk), frozenset({id(chunk)}))
|
||||
if exclude:
|
||||
verbose_proxy_logger.warning("Dropping streamed chunk fields that JSON cannot encode: %s", exclude)
|
||||
jsonable: Final = chunk.model_dump(mode="json", exclude_none=True, exclude_unset=True, exclude=exclude)
|
||||
return json.dumps(jsonable, ensure_ascii=True, separators=(",", ":")).encode()
|
||||
|
||||
|
||||
def _serialize_chat_chunk(chunk: ModelResponseStream) -> bytes:
|
||||
try:
|
||||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode()
|
||||
except PydanticSerializationError:
|
||||
return _serialize_chat_chunk_without_unencodable_fields(chunk)
|
||||
return _serialize_chat_chunk_leniently(chunk)
|
||||
|
||||
|
||||
def _serialize_streaming_chunk(chunk: BaseModel) -> bytes:
|
||||
|
|
|
|||
|
|
@ -55,7 +55,14 @@ from litellm.types.llms.openai import (
|
|||
ResponseFailedEvent,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaToolCall,
|
||||
Delta,
|
||||
Function,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
Usage,
|
||||
)
|
||||
from tests._support.clock_loops import AutojumpClockLoop
|
||||
|
||||
from .conftest import normalize
|
||||
|
|
@ -771,7 +778,7 @@ async def test_async_data_generator_sse_frame_keeps_provider_fields_on_a_text_ch
|
|||
|
||||
|
||||
def _circular_dict() -> dict[str, object]:
|
||||
looped: Final[dict[str, object]] = {} # mutable-ok: a circular dict has to be inserted into itself
|
||||
looped: Final[dict[str, object]] = {"keep": 1} # mutable-ok: a circular dict has to be inserted into itself
|
||||
looped["self"] = looped
|
||||
return looped
|
||||
|
||||
|
|
@ -782,8 +789,6 @@ def _circular_dict() -> dict[str, object]:
|
|||
[
|
||||
pytest.param(object(), id="unknown-type"),
|
||||
pytest.param(hashlib.sha256(b"trace").digest(), id="invalid-utf8-bytes"),
|
||||
pytest.param("\ud800", id="lone-surrogate"),
|
||||
pytest.param(_circular_dict(), id="circular-dict"),
|
||||
],
|
||||
)
|
||||
async def test_async_data_generator_streams_a_text_chunk_a_hook_tagged_with_a_value_json_cannot_encode(
|
||||
|
|
@ -814,6 +819,55 @@ async def test_async_data_generator_streams_a_text_chunk_a_hook_tagged_with_a_va
|
|||
assert all(name in caplog.text for name in ("trace_chunk", "trace_choice", "trace_delta"))
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_keeps_lone_surrogates_a_provider_split_across_frames(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
tool_call: Final = ChatCompletionDeltaToolCall(
|
||||
id="call_1", index=0, type="function", function=Function(name="get_weather", arguments='{"city": "Par\udc00is"}')
|
||||
)
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-split",
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content="Hi \ud83d", tool_calls=[tool_call]))],
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
delta: Final = json.loads(_serialize_streaming_chunk(chunk))["choices"][0]["delta"]
|
||||
|
||||
assert caplog.text == ""
|
||||
assert delta["content"] == "Hi \ud83d"
|
||||
assert delta["tool_calls"][0]["id"] == "call_1"
|
||||
assert delta["tool_calls"][0]["function"] == {"name": "get_weather", "arguments": '{"city": "Par\udc00is"}'}
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_drops_only_the_nested_value_json_cannot_encode() -> None:
|
||||
digest: Final = hashlib.sha256(b"trace").digest()
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-nested",
|
||||
choices=[
|
||||
StreamingChoices(index=0, delta=Delta(content="a")),
|
||||
StreamingChoices(
|
||||
index=1, delta=Delta(content="b", provider_specific_fields={"citations": ["ok"], "raw": digest})
|
||||
),
|
||||
],
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion.chunk",
|
||||
usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
|
||||
)
|
||||
setattr(chunk.usage, "raw", digest)
|
||||
setattr(chunk, "trace_chunk", _circular_dict())
|
||||
|
||||
payload: Final = json.loads(_serialize_streaming_chunk(chunk))
|
||||
|
||||
assert payload["choices"][0]["delta"] == {"content": "a"}
|
||||
assert payload["choices"][1]["delta"] == {"content": "b", "provider_specific_fields": {"citations": ["ok"]}}
|
||||
assert payload["usage"] == {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
|
||||
assert payload["trace_chunk"] == {"keep": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue