diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 593be06a16a..1c84e2c7f80 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10070,7 +10070,13 @@ def _restamp_nested_response_model_in_sse_line(line: bytes, target_model: str) - return _SSE_DATA_FIELD + b" " + serialized_event.encode() + line_ending +def _null_for_unencodable(_value: object) -> None: + return None + + def _serialize_streaming_chunk(chunk: BaseModel) -> bytes: + if isinstance(chunk, ModelResponseStream): + return chunk.model_dump_json(exclude_none=True, exclude_unset=True, fallback=_null_for_unencodable).encode() return chunk.model_dump_json(exclude_none=True, exclude_unset=True).encode() diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 72fc3924ce5..49ff66763cf 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -768,6 +768,27 @@ async def test_async_data_generator_sse_frame_keeps_provider_fields_on_a_text_ch assert read_field(json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n"))) == sent +@pytest.mark.asyncio +async def test_async_data_generator_streams_a_text_chunk_a_hook_tagged_with_a_value_json_cannot_encode(monkeypatch): + _patch_logging_flags(monkeypatch) + chunk: Final = _azure_chunk({}, {"obfuscation": "Ab3"}) + setattr(chunk, "trace_handle", object()) + + out: Final = [ + line + async for line in async_data_generator( + response=_async_iter([chunk]), + user_api_key_dict=_user_auth(), + request_data={"model": "gpt-4"}, + ) + ] + + payload: Final = json.loads(_sse_text(out[0]).removeprefix("data: ").removesuffix("\n\n")) + assert payload["choices"][0]["delta"]["content"] == "Red" + assert payload["obfuscation"] == "Ab3" + assert payload["trace_handle"] is None + + @pytest.mark.asyncio async def test_async_data_generator_uses_response_fallback_metadata(monkeypatch): _patch_logging_flags(monkeypatch)