diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b5f4a36f138..e885201272b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -61,6 +61,8 @@ import websockets import websockets.exceptions from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config from pydantic.fields import FieldInfo, PydanticUndefined +from pydantic.main import IncEx +from pydantic_core import PydanticSerializationError, to_json from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never from litellm._uuid import uuid @@ -10195,91 +10197,40 @@ def _restamp_nested_response_model_in_sse_line(line: bytes, target_model: str) - return _SSE_DATA_FIELD + b" " + serialized_event.encode() + line_ending -def _fast_serialize_simple_model_response_stream( - chunk: ModelResponseStream, -) -> bytes | None: - """ - Serialize the common OpenAI text streaming chunk without the full Pydantic - serializer. Fall back for richer chunks so tool calls, logprobs, usage, and - provider-specific fields keep the canonical model_dump_json behavior. - """ - if ( - getattr(chunk, "provider_specific_fields", None) is not None - or getattr(chunk, "system_fingerprint", None) is not None - or getattr(chunk, "usage", None) is not None - ): - return None - - choices: Final = getattr(chunk, "choices", None) - if not isinstance(choices, list) or len(choices) != 1: - return None - - choice: Final = choices[0] - if getattr(choice, "logprobs", None) is not None or getattr(choice, "enhancements", None) is not None: - return None - - delta: Final[object] = getattr(choice, "delta", None) - if delta is None: - return None - - unsupported_delta_fields: Final = ( - "function_call", - "tool_calls", - "audio", - "images", - "annotations", - "reasoning_content", - "thinking_blocks", - "provider_specific_fields", - "refusal", - ) - if any(getattr(delta, field, None) is not None for field in unsupported_delta_fields): - return None - - delta_dict: Final[dict] = {} - role: Final = getattr(delta, "role", None) - content: Final = getattr(delta, "content", None) - if role is not None: - delta_dict["role"] = role - if content is not None: - delta_dict["content"] = content - - choice_dict: Final = {"index": getattr(choice, "index", 0), "delta": delta_dict} - finish_reason: Final = getattr(choice, "finish_reason", None) - if finish_reason is not None: - choice_dict["finish_reason"] = finish_reason - - # Match the canonical ``model_dump_json(exclude_none=True)`` shape — if a - # field is None, omit it entirely rather than emitting ``"key": null``. - # Strict OpenAI-compatible clients reject ``null`` for optional fields like - # ``model``, so diverging here would surface as a client-side regression - # only on the fast path. Fall back to the slow path if a required-looking - # top-level identifier is missing. - model: Final = getattr(chunk, "model", None) - if model is None: - return None - - payload: Final[dict] = { - "id": getattr(chunk, "id", None), - "object": getattr(chunk, "object", None), - "created": getattr(chunk, "created", None), - "model": model, - "service_tier": getattr(chunk, "service_tier", None), - "choices": [choice_dict], - } - for top_level_key in ("id", "object", "created", "service_tier"): - if payload[top_level_key] is None: - payload.pop(top_level_key) - return orjson.dumps(payload) +def _json_encodes(value: object) -> bool: + try: + to_json(value) + except PydanticSerializationError: + return False + return True -def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: +def _unencodable_fields(model: BaseModel, nested_field: str | None) -> dict[str, bool]: + return {key: True for key, value in model if key != nested_field and not _json_encodes(value)} + + +def _unencodable_choice_fields(choice: StreamingChoices) -> Mapping[str, IncEx | bool]: + return {**_unencodable_fields(choice, "delta"), "delta": _unencodable_fields(choice.delta, None)} + + +def _serialize_chat_chunk_without_unencodable_fields(chunk: ModelResponseStream) -> bytes: + choices: Final = {index: _unencodable_choice_fields(choice) for index, choice in enumerate(chunk.choices)} + exclude: Final = {**_unencodable_fields(chunk, "choices"), "choices": choices} + verbose_proxy_logger.warning("Dropping streamed chunk fields that JSON cannot encode: %s", exclude) + return chunk.model_dump_json(exclude_none=True, exclude_unset=True, exclude=exclude).encode() + + +def _serialize_chat_chunk(chunk: ModelResponseStream) -> bytes: + try: + return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode() + except PydanticSerializationError: + return _serialize_chat_chunk_without_unencodable_fields(chunk) + + +def _serialize_streaming_chunk(chunk: BaseModel) -> bytes: if isinstance(chunk, ModelResponseStream): - serialized_chunk: Final = _fast_serialize_simple_model_response_stream(chunk) - if serialized_chunk is not None: - return serialized_chunk - - return chunk.model_dump_json(exclude_none=True, exclude_unset=True) + return _serialize_chat_chunk(chunk) + return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode() async def _apply_streaming_chunk_hooks( diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index c37475bd539..d5c27719b00 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -5,7 +5,6 @@ Pins covered: - ``async_assistants_data_generator`` - ``_get_client_requested_model_for_streaming`` - ``_restamp_streaming_chunk_model`` -- ``_fast_serialize_simple_model_response_stream`` - ``_serialize_streaming_chunk`` - ``_apply_streaming_chunk_hooks`` - ``_format_streaming_sse_chunk`` @@ -16,7 +15,9 @@ Pins covered: from __future__ import annotations import asyncio +import hashlib import json +import logging from collections.abc import AsyncIterator, Callable from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -36,7 +37,6 @@ from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, - _fast_serialize_simple_model_response_stream, _format_fallback_metadata_sse_event, _format_streaming_sse_chunk, _get_client_requested_model_for_streaming, @@ -55,7 +55,7 @@ from litellm.types.llms.openai import ( ResponseFailedEvent, ResponsesAPIResponse, ) -from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices from tests._support.clock_loops import AutojumpClockLoop from .conftest import normalize, route_entry_points_to_stubs @@ -81,6 +81,10 @@ def _simple_chunk(model: str = "gpt-4", content: str = "hi") -> ModelResponseStr ) +def _sse_text(frame: str | bytes) -> str: + return frame.decode() if isinstance(frame, bytes) else frame + + async def _async_iter(items): for it in items: yield it @@ -520,48 +524,13 @@ def test_format_fallback_metadata_sse_event(): assert isinstance(payload["created"], int) -# --------------------------------------------------------------------------- -# _fast_serialize_simple_model_response_stream -# --------------------------------------------------------------------------- - - -def test_fast_serialize_simple_model_response_stream_returns_bytes_payload(): - chunk = _simple_chunk() - result = _fast_serialize_simple_model_response_stream(chunk) - assert isinstance(result, bytes) - payload = json.loads(result) - assert normalize(payload) == { - "id": "", - "object": "chat.completion.chunk", - "created": "", - "model": "gpt-4", - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": "hi"}, - } - ], - } - - -def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_invalid(): - """Fast path bails (returns None) when ``usage`` is populated — the slow - path is required to preserve usage fields. Returning None here is the - "I cannot handle this" sentinel, not a hard error.""" - chunk = _simple_chunk() - chunk.usage = Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2) - assert _fast_serialize_simple_model_response_stream(chunk) is None - - # --------------------------------------------------------------------------- # _serialize_streaming_chunk # --------------------------------------------------------------------------- -def test_serialize_streaming_chunk_simple_uses_fast_path_bytes(): - result = _serialize_streaming_chunk(_simple_chunk()) - assert isinstance(result, bytes) - payload = json.loads(result) +def test_serialize_streaming_chunk_simple_payload(): + payload = json.loads(_serialize_streaming_chunk(_simple_chunk())) assert normalize(payload) == { "id": "", "object": "chat.completion.chunk", @@ -700,10 +669,7 @@ async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch): out.append(line) assert out[-1] == "data: [DONE]\n\n" - # First chunk is bytes (fast path) wrapped via _format_streaming_sse_chunk. - first = out[0] - assert isinstance(first, bytes) - payload = json.loads(first.removeprefix(b"data: ").removesuffix(b"\n\n")) + payload = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) assert normalize(payload) == { "id": "", "object": "chat.completion.chunk", @@ -718,6 +684,136 @@ async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch): } +_AZURE_CONTENT_FILTER_RESULTS: Final = { + "hate": {"filtered": False, "severity": "safe"}, + "protected_material_text": {"detected": False, "filtered": False}, +} + + +def _azure_chunk(choice_fields: dict[str, object], chunk_fields: dict[str, object]) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-azure", + choices=[StreamingChoices(index=0, delta=Delta(content="Red"), **choice_fields)], + created=0, + model="gpt-4", + object="chat.completion.chunk", + **chunk_fields, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("chunk", "read_field", "sent"), + [ + pytest.param( + _azure_chunk({"content_filter_results": _AZURE_CONTENT_FILTER_RESULTS}, {}), + lambda payload: payload["choices"][0]["content_filter_results"], + _AZURE_CONTENT_FILTER_RESULTS, + id="choice-content-filter-results", + ), + pytest.param( + _azure_chunk({}, {"obfuscation": "Ab3"}), + lambda payload: payload["obfuscation"], + "Ab3", + id="chunk-obfuscation", + ), + pytest.param( + ModelResponseStream( + id="chatcmpl-or", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content="Red", reasoning_details=[{"type": "reasoning.text", "text": "colors"}]), + ) + ], + created=0, + model="gpt-4", + object="chat.completion.chunk", + ), + lambda payload: payload["choices"][0]["delta"]["reasoning_details"], + [{"type": "reasoning.text", "text": "colors"}], + id="delta-reasoning-details", + ), + pytest.param( + ModelResponseStream( + id="chatcmpl-resp", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content="Red", reasoning_items=[{"type": "reasoning", "id": "rs_1", "summary": []}]), + ) + ], + created=0, + model="gpt-4", + object="chat.completion.chunk", + ), + lambda payload: payload["choices"][0]["delta"]["reasoning_items"], + [{"type": "reasoning", "id": "rs_1", "summary": []}], + id="delta-reasoning-items", + ), + ], +) +async def test_async_data_generator_sse_frame_keeps_provider_fields_on_a_text_chunk( + monkeypatch, chunk: ModelResponseStream, read_field: Callable[[dict[str, object]], object], sent: object +): + _patch_logging_flags(monkeypatch) + + out = [ + line + async for line in async_data_generator( + response=_async_iter([chunk]), + user_api_key_dict=_user_auth(), + request_data={"model": "gpt-4"}, + ) + ] + + assert read_field(json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n"))) == sent + + +def _circular_dict() -> dict[str, object]: + looped: Final[dict[str, object]] = {} # mutable-ok: a circular dict has to be inserted into itself + looped["self"] = looped + return looped + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "unencodable", + [ + pytest.param(object(), id="unknown-type"), + pytest.param(hashlib.sha256(b"trace").digest(), id="invalid-utf8-bytes"), + pytest.param("\ud800", id="lone-surrogate"), + pytest.param(_circular_dict(), id="circular-dict"), + ], +) +async def test_async_data_generator_streams_a_text_chunk_a_hook_tagged_with_a_value_json_cannot_encode( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, unencodable: object +) -> None: + _patch_logging_flags(monkeypatch) + chunk: Final = _azure_chunk({"content_filter_results": _AZURE_CONTENT_FILTER_RESULTS}, {"obfuscation": "Ab3"}) + setattr(chunk, "trace_chunk", unencodable) + setattr(chunk.choices[0], "trace_choice", unencodable) + setattr(chunk.choices[0].delta, "trace_delta", unencodable) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + out: Final = [ + line + async for line in async_data_generator( + response=_async_iter([chunk]), + user_api_key_dict=_user_auth(), + request_data={"model": "gpt-4"}, + ) + ] + + payload: Final = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) + assert payload["obfuscation"] == "Ab3" + assert payload["choices"][0]["content_filter_results"] == _AZURE_CONTENT_FILTER_RESULTS + assert payload["choices"][0]["delta"] == {"content": "Red"} + assert "trace_chunk" not in payload + assert "trace_choice" not in payload["choices"][0] + assert all(name in caplog.text for name in ("trace_chunk", "trace_choice", "trace_delta")) + + @pytest.mark.asyncio async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch): _patch_logging_flags(monkeypatch) @@ -739,9 +835,7 @@ async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch) ): out.append(line) - first = out[0] - assert isinstance(first, bytes) - payload = json.loads(first.removeprefix(b"data: ").removesuffix(b"\n\n")) + payload = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) assert payload["model"] == "fallback-model" @@ -764,9 +858,7 @@ async def test_async_data_generator_uses_chunk_fallback_metadata(monkeypatch): ): out.append(line) - first = out[0] - assert isinstance(first, bytes) - payload = json.loads(first.removeprefix(b"data: ").removesuffix(b"\n\n")) + payload = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) assert payload["model"] == "fallback-model" @@ -793,8 +885,8 @@ async def test_async_data_generator_switches_model_mid_stream_on_fallback(monkey ): out.append(line) - first_payload = json.loads(out[0].removeprefix(b"data: ").removesuffix(b"\n\n")) - second_payload = json.loads(out[1].removeprefix(b"data: ").removesuffix(b"\n\n")) + first_payload = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) + second_payload = json.loads(_sse_text(out[1]).removeprefix("data: ").removesuffix("\n\n")) assert first_payload["model"] == "primary-model" assert second_payload["model"] == "fallback-model" @@ -830,8 +922,7 @@ async def test_async_data_generator_emits_fallback_error_metadata_event(monkeypa ): out.append(line) - assert isinstance(out[0], bytes) - chunk_payload = json.loads(out[0].removeprefix(b"data: ").removesuffix(b"\n\n")) + chunk_payload = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) assert chunk_payload["model"] == "fallback-model" assert isinstance(out[1], str) assert out[1].startswith("data: ") @@ -879,8 +970,7 @@ async def test_async_data_generator_skips_fallback_error_event_without_opt_in( ): out.append(line) - assert isinstance(out[0], bytes) - payload = json.loads(out[0].removeprefix(b"data: ").removesuffix(b"\n\n")) + payload = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) assert payload["model"] == "fallback-model" @@ -2434,14 +2524,11 @@ async def test_queue_request_stream_is_untouched_while_keepalives_are_unconfigur assert chunks[-1] == b"data: [DONE]\n\n" -def test_fast_serialize_simple_model_response_stream_keeps_served_service_tier(): +def test_serialize_streaming_chunk_keeps_served_service_tier(): chunk = _simple_chunk() chunk.service_tier = "priority" - result = _fast_serialize_simple_model_response_stream(chunk) - - assert result is not None - assert json.loads(result)["service_tier"] == "priority" + assert json.loads(_serialize_streaming_chunk(chunk))["service_tier"] == "priority" @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_response_model_sanitization.py b/tests/unit/proxy/test_response_model_sanitization.py index f52736453c1..b3ad53221eb 100644 --- a/tests/unit/proxy/test_response_model_sanitization.py +++ b/tests/unit/proxy/test_response_model_sanitization.py @@ -98,42 +98,13 @@ def test_restamp_streaming_chunk_skips_matching_model(): assert model_mismatch_logged is False -def test_fast_serialize_simple_streaming_chunk_matches_model_dump_json(): +def test_serialize_streaming_chunk_omits_a_missing_model_instead_of_null(): from litellm.proxy.proxy_server import _serialize_streaming_chunk - chunk = _make_model_response_stream_chunk("client-model") - - assert json.loads(_serialize_streaming_chunk(chunk)) == json.loads( - chunk.model_dump_json(exclude_none=True, exclude_unset=True) - ) - - -def test_fast_serialize_returns_none_when_model_field_is_missing(): - """ - The fast path must mirror ``model_dump_json(exclude_none=True)``: when - ``chunk.model`` is ``None`` the slow path omits the field entirely. - Emitting ``"model": null`` would diverge and trip strict OpenAI- - compatible clients that reject ``null`` for optional string fields. - Falling back to ``None`` lets the canonical serializer handle the edge. - """ - from litellm.proxy.proxy_server import ( - _fast_serialize_simple_model_response_stream, - _serialize_streaming_chunk, - ) - chunk = _make_model_response_stream_chunk("client-model") chunk.model = None # type: ignore[assignment] - assert _fast_serialize_simple_model_response_stream(chunk) is None - - # Going through the public ``_serialize_streaming_chunk`` should still - # produce a serialized result via the slow-path fallback, and it must - # not contain ``"model": null``. - serialized = _serialize_streaming_chunk(chunk) - payload_str = serialized.decode("utf-8") if isinstance(serialized, bytes) else serialized - assert '"model": null' not in payload_str - assert '"model":null' not in payload_str - assert json.loads(payload_str) == json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True)) + assert "model" not in json.loads(_serialize_streaming_chunk(chunk)) def test_proxy_chat_completion_does_not_return_provider_prefixed_model(tmp_path, monkeypatch):