From 1f3ddc54852f1ae02be22409b62f1963300b3793 Mon Sep 17 00:00:00 2001 From: MengMa Date: Sun, 27 Sep 2026 18:08:37 +0800 Subject: [PATCH 1/4] fix(proxy): serialize late SSE payloads that are dicts A keepalive can open the response before the body exists as bytes. Reading .body on a dict crashed the stream. --- litellm/proxy/common_request_processing.py | 25 ++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 15610da9aec..adeccef7e3f 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1164,12 +1164,33 @@ async def _aclose_late_response(produced: Response) -> None: verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc) -async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]: +def _late_response_body(produced: object) -> bytes: + """Bytes for one SSE data frame. + + A late payload may be a dict, raw bytes, or a Response whose ``body`` is either. + """ + payload: Final = ( + produced + if isinstance(produced, (bytes, bytearray, Mapping)) + else getattr(produced, "body", produced) + ) + if isinstance(payload, (bytes, bytearray)): + raw: Final = bytes(payload) + return raw or b"{}" + if isinstance(payload, str): + text: Final = payload.encode() + return text or b"{}" + if isinstance(payload, Mapping): + return orjson.dumps(payload) + return b"{}" + + +async def _relay_late_response(produced: object) -> AsyncGenerator[bytes, None]: """Replay a Response that was built after a keepalive had already opened the wire.""" if not isinstance(produced, StreamingResponse): # The status line is already on the wire, so a non-streaming body, an error # body included, can only reach the client as an SSE frame. - yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n" + yield b"data: " + _late_response_body(produced) + b"\n\n" yield b"data: [DONE]\n\n" return From 01a5fba937fa9d1ad3035b8cf7bda0a7f3a10ad2 Mon Sep 17 00:00:00 2001 From: MengMa Date: Sun, 27 Sep 2026 18:08:37 +0800 Subject: [PATCH 2/4] test(proxy): cover late SSE dict payloads Assert a dict, a Response body, and a dict body attribute all become SSE data frames. --- .../proxy/common_utils/test_sse_keepalive.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py index 228fd5bcae6..b754b31762d 100644 --- a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py +++ b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py @@ -3,7 +3,7 @@ from collections.abc import AsyncGenerator from typing import Final, cast import pytest -from fastapi.responses import StreamingResponse +from fastapi.responses import Response, StreamingResponse from litellm.proxy.common_request_processing import create_response from litellm.types.utils import ModelResponse @@ -408,3 +408,40 @@ def test_ttft_interval_resolves_through_the_deployments_it_could_land_on( deployments, global_interval, expected, why ): assert resolve_ttft_keepalive_interval(deployments, global_interval) == expected, why + + +@pytest.mark.asyncio +async def test_relay_late_response_serializes_a_dict_payload_as_sse(): + from litellm.proxy.common_request_processing import _relay_late_response + + frames: Final = [chunk async for chunk in _relay_late_response({"error": {"message": "late"}})] + + assert frames == [ + b'data: {"error":{"message":"late"}}\n\n', + b"data: [DONE]\n\n", + ] + + +@pytest.mark.asyncio +async def test_relay_late_response_keeps_a_response_body(): + from litellm.proxy.common_request_processing import _relay_late_response + + frames: Final = [chunk async for chunk in _relay_late_response(Response(content=b'{"ok":true}'))] + + assert frames == [ + b'data: {"ok":true}\n\n', + b"data: [DONE]\n\n", + ] + + +@pytest.mark.asyncio +async def test_relay_late_response_serializes_a_dict_body_attribute(): + from litellm.proxy.common_request_processing import _relay_late_response + + class _DictBody: + body = {"ok": True} + + frames: Final = [chunk async for chunk in _relay_late_response(_DictBody())] + + assert frames[0] == b'data: {"ok":true}\n\n' + assert frames[1] == b"data: [DONE]\n\n" From 0cc4d666247bd34980345943ba8d8aa21b4a3c4f Mon Sep 17 00:00:00 2001 From: MengMa Date: Sun, 27 Sep 2026 19:37:20 +0800 Subject: [PATCH 3/4] fix(proxy): serialize non-dict mappings in late SSE frames orjson rejects mappingproxy. Copy the mapping to a dict before writing the data frame, and let ruff format the helper. --- litellm/proxy/common_request_processing.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index adeccef7e3f..5f7bce971fe 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1167,12 +1167,10 @@ async def _aclose_late_response(produced: Response) -> None: def _late_response_body(produced: object) -> bytes: """Bytes for one SSE data frame. - A late payload may be a dict, raw bytes, or a Response whose ``body`` is either. + A late payload may be a mapping, raw bytes or text, or a Response whose ``body`` is one of those. """ payload: Final = ( - produced - if isinstance(produced, (bytes, bytearray, Mapping)) - else getattr(produced, "body", produced) + produced if isinstance(produced, (bytes, bytearray, Mapping)) else getattr(produced, "body", produced) ) if isinstance(payload, (bytes, bytearray)): raw: Final = bytes(payload) @@ -1181,7 +1179,8 @@ def _late_response_body(produced: object) -> bytes: text: Final = payload.encode() return text or b"{}" if isinstance(payload, Mapping): - return orjson.dumps(payload) + # orjson rejects mappingproxy and other non-dict mappings. + return orjson.dumps(dict(payload)) return b"{}" From f75a2b8651b0c3c9bae038566bc4a4e88775a5c5 Mon Sep 17 00:00:00 2001 From: MengMa Date: Sun, 27 Sep 2026 19:37:20 +0800 Subject: [PATCH 4/4] test(proxy): cover late SSE strings, mappings, and unknown payloads The string branch, the empty-object fallback, and a mappingproxy were untested. --- .../proxy/common_utils/test_sse_keepalive.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py index b754b31762d..ca447b00538 100644 --- a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py +++ b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py @@ -445,3 +445,35 @@ async def test_relay_late_response_serializes_a_dict_body_attribute(): assert frames[0] == b'data: {"ok":true}\n\n' assert frames[1] == b"data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_relay_late_response_encodes_a_string_payload(): + from litellm.proxy.common_request_processing import _relay_late_response + + frames: Final = [chunk async for chunk in _relay_late_response("late")] + + assert frames == [b"data: late\n\n", b"data: [DONE]\n\n"] + + +@pytest.mark.asyncio +async def test_relay_late_response_uses_an_empty_object_for_an_unknown_payload(): + from litellm.proxy.common_request_processing import _relay_late_response + + class _NoBody: + pass + + frames: Final = [chunk async for chunk in _relay_late_response(_NoBody())] + + assert frames == [b"data: {}\n\n", b"data: [DONE]\n\n"] + + +@pytest.mark.asyncio +async def test_relay_late_response_serializes_a_mapping_proxy(): + from types import MappingProxyType + + from litellm.proxy.common_request_processing import _relay_late_response + + frames: Final = [chunk async for chunk in _relay_late_response(MappingProxyType({"ok": True}))] + + assert frames == [b'data: {"ok":true}\n\n', b"data: [DONE]\n\n"]