From eb96d885ceff1b8daa6c04330e819e39653ba33b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:16:47 -0700 Subject: [PATCH] fix(proxy): end failed responses streams with [DONE] Emit data: [DONE] after event: response.failed, and after a late failure when a terminal event already went out, so OpenAI SDK clients see the same stream end as a completed response. Restore the lazy OpenAPI snapshot to its Python 3.12 rendering, which is what CI regenerates. --- litellm/proxy/_lazy_openapi_snapshot.json | 2 +- litellm/proxy/proxy_server.py | 6 ++++-- .../proxy/proxy_server/test_streaming_helpers.py | 2 ++ .../proxy/response_api_endpoints/test_endpoints.py | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 2aa2cf15ac1..8a8d08c6887 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -19394,7 +19394,7 @@ } } }, - "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n" + "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " }, "500": { "content": { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9cd233e5ad5..38a883a3b1b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8865,6 +8865,7 @@ def _format_streaming_sse_chunk(chunk: str | bytes) -> str | bytes: _SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r") +_OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n" _MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024 @@ -9272,8 +9273,7 @@ async def async_data_generator( yield error_message # OpenAI-compatible streams terminate with data: [DONE]; Google GenAI (?alt=sse) does not. if not request_data.get("_litellm_skip_openai_stream_done"): - done_message: Final = "[DONE]" - yield f"data: {done_message}\n\n" + yield _OPENAI_STREAM_DONE_FRAME except (asyncio.CancelledError, GeneratorExit): # Client disconnected mid-stream. CancelledError / GeneratorExit are # BaseException, so they bypass the success/failure logging callbacks @@ -9303,6 +9303,8 @@ async def async_data_generator( error_frame: Final = error_state.format_failure(e) if error_frame is not None: yield error_frame + if not request_data.get("_litellm_skip_openai_stream_done"): + yield _OPENAI_STREAM_DONE_FRAME return if isinstance(e, HTTPException): raise e diff --git a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py index 9adc2b80741..86dd356e5f5 100644 --- a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py @@ -995,6 +995,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina for frame in event_frames ) + assert decoded[-1] == "data: [DONE]\n\n" + assert len(decoded) == len(event_frames) + 1 assert payloads[0]["response"]["id"] == "resp_visible" assert payloads[1] == tool_delta.model_dump() assert len(payloads) == 3 diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 1fb3fef8fb3..63faef2366f 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -95,7 +95,8 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert result.status_code == 200, result.text assert message in result.text if path == "/v1/responses": - assert frames[-1].startswith("event: response.failed\n"), result.text + assert frames[-1] == "data: [DONE]", result.text + assert frames[-2].startswith("event: response.failed\n"), result.text if partial: assert [event["type"] for event in events] == [ "response.created", "response.output_item.added",