From 7ee563d9e809bdd9287cc7b0049758a06042279d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:56:16 -0700 Subject: [PATCH] fix(proxy): read every frame of a polled chunk and name model chunks from the sent JSON A background poll parsed only the first data frame of each chunk, so a terminal event behind an error or delta frame in the same chunk was lost, and the poll stored a status its own stream never ended with. It now reads every frame up to [DONE] A terminal event whose response is a string or null kept no status and polled completed. Its status now comes from the event type, and final output items that are not objects are skipped, so the rest of the final output is still stored Model chunks now take their event line from the JSON they send, as mapping chunks already do, so a serializer that drops type no longer sends event: response.completed over {} --- .../common_utils/responses_stream_errors.py | 4 +- .../response_polling/background_streaming.py | 32 +++++++---- .../proxy_server/test_streaming_helpers.py | 24 +++++++- .../test_response_polling_handler.py | 57 +++++++++++++++++++ 4 files changed, 101 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index e9caf82afff..ca948f65bd2 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -517,11 +517,9 @@ class ResponsesStreamErrorState: self.sequence_number = -1 self.terminal_emitted = False self._pending_event: _StreamEvent | None = None - self._pending_on_wire = False def observe_chunk(self, chunk: object) -> None: self._pending_event = _observed_event(chunk) - self._pending_on_wire = isinstance(chunk, Mapping) def _record(self, event: _StreamEvent) -> None: if event.sequence_number is not None: @@ -543,7 +541,7 @@ class ResponsesStreamErrorState: observed: Final = self._pending_event if observed is None: return frame - event: Final = _wire_event(frame) if self._pending_on_wire else observed + event: Final = _wire_event(frame) self._record(observed if event is None else event) if event is None or not event.type or "\n" in event.type or "\r" in event.type: return frame diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index ae8f4939c1b..9d211fe4e71 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -12,6 +12,7 @@ import asyncio import json from collections.abc import AsyncIterable, Callable, Mapping, Sequence from http import HTTPStatus +from itertools import takewhile from time import perf_counter from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable @@ -90,8 +91,17 @@ class _StreamsBody(Protocol): def body_iterator(self) -> AsyncIterable[str | bytes | memoryview]: ... -def _sse_frame_data(frame: str) -> str | None: - return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None) +def _payloads_before_done(frame: str) -> tuple[tuple[str, ...], bool]: + payloads: Final = tuple(line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")) + return tuple(takewhile(lambda payload: payload != "[DONE]", payloads)), "[DONE]" in payloads + + +def _terminal_response(event: _StreamEvent) -> _TerminalResponse: + match event.get("response"): + case Mapping() as response: + return response + case _: + return {} def _event_error(event: Mapping[str, object]) -> Mapping[str, object]: @@ -279,10 +289,8 @@ async def background_streaming_task( ): frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data))) - if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None: - if chunk_data == "[DONE]": - break - + payloads, stream_done = _payloads_before_done(chunk) if isinstance(chunk, str) else ((), False) + for chunk_data in payloads: try: event: _StreamEvent = _StreamEventParser.parse(chunk_data) event_type = event.get("type", "") @@ -394,7 +402,7 @@ async def background_streaming_task( # Terminal event - extract all ResponsesAPIResponse fields # https://platform.openai.com/docs/api-reference/responses-streaming finished_at = perf_counter() - response_data = event.get("response", {}) + response_data = _terminal_response(event) terminal_status = response_data.get( "status", _event_to_status.get(event_type, "completed"), @@ -428,15 +436,19 @@ async def background_streaming_task( if "output" in response_data: final_output = response_data.get("output", []) for item in final_output: - item_id = item.get("id") - if item_id: - output_items[item_id] = item + match item: + case {"id": item_id} if item_id: + output_items[item_id] = item + case _: + continue state_dirty = True except (json.JSONDecodeError, AttributeError, TypeError) as e: verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e) else: await flush_state_if_needed() + if stream_done: + break # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index c6628319110..60f92c4cd06 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -34,7 +34,7 @@ from fastapi import HTTPException, Response from fastapi.responses import JSONResponse, StreamingResponse from openai import APIError as OpenAIAPIError from openai.types.responses import ResponseOutputMessage, ResponseOutputText -from pydantic import BaseModel, Field, PrivateAttr +from pydantic import BaseModel, Field, PrivateAttr, model_serializer import litellm import litellm.proxy.proxy_server as ps @@ -204,8 +204,8 @@ class _MaskedEverywhereString(str): return "***" -async def _responses_frames(chunks: Iterable[Mapping[str, object]]) -> tuple[str, ...]: - async def upstream() -> AsyncIterator[Mapping[str, object]]: +async def _responses_frames(chunks: Iterable[object]) -> tuple[str, ...]: + async def upstream() -> AsyncIterator[object]: for chunk in chunks: yield chunk @@ -242,6 +242,24 @@ async def test_responses_stream_sends_a_hook_chunk_its_event_model_cannot_read_a assert frames == (f"data: {json.dumps(chunk)}\n\n",) +class _TypeDroppedOnTheWire(BaseModel, frozen=True): + type: str = "response.completed" + sequence_number: int = 3 + + @model_serializer + def _without_type(self) -> Mapping[str, int]: + return {"sequence_number": self.sequence_number} + + +@pytest.mark.asyncio +async def test_responses_stream_names_a_model_chunk_from_the_json_it_sends() -> None: + frames: Final = await _responses_frames((_TypeDroppedOnTheWire(),)) + + assert len(frames) == 1 + assert frames[0].startswith("data: ") + assert json.loads(frames[0].removeprefix("data: ")) == {"sequence_number": 3} + + class _HookOutputItem(BaseModel, frozen=True): id: str type: str diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 8d414c798f1..43184f95700 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1679,6 +1679,28 @@ class TestBackgroundStreamingTerminalEvents: None, None, ), + (('data: {"type": "response.failed", "response": "hook-text"}\n\n',), "failed", None, None), + (('data: {"type": "response.failed", "response": null}\n\n',), "failed", None, None), + ( + ( + 'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n' + 'event: response.incomplete\ndata: {"type": "response.incomplete", ' + '"response": {"status": "incomplete", "usage": {"input_tokens": 3}}}\n\n', + ), + "incomplete", + None, + {"input_tokens": 3}, + ), + ( + ( + 'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n', + 'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n' + 'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n', + ), + "completed", + None, + {"input_tokens": 3}, + ), ], ids=[ "error_frame_after_a_delta", @@ -1688,6 +1710,10 @@ class TestBackgroundStreamingTerminalEvents: "output_item_done_with_a_string_item_then_completed", "error_frame_then_completed_in_one_chunk", "error_frame_after_done_in_one_chunk", + "failed_event_whose_response_is_a_string", + "failed_event_whose_response_is_null", + "error_frame_and_incomplete_in_one_chunk", + "error_frame_then_completed_behind_a_delta_in_one_chunk", ], ) async def test_a_poll_stores_what_its_stream_ended_with( @@ -1724,6 +1750,37 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["error"] == expected_error assert final_call.kwargs["usage"] == expected_usage + @pytest.mark.asyncio + async def test_a_poll_stores_the_final_output_items_beside_one_that_is_not_an_object(self): + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + final_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": "final"}]} + draft_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": ""}]} + completed: Final = {"type": "response.completed", "response": {"status": "completed", "output": ["hook-text", final_item]}} + + async def _body_iterator(): + yield f"data: {json.dumps({'type': 'response.output_item.added', 'item': draft_item})}\n\n".encode() + yield b'data: {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "draft"}\n\n' + yield f"data: {json.dumps(completed)}\n\n".encode() + + mock_response: Final = Mock() + mock_response.body_iterator = _body_iterator() + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_final_output", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + stored_outputs: Final = [call.kwargs["output"] for call in handler.update_state.call_args_list if "output" in call.kwargs] + assert stored_outputs[-1] == [final_item] + @pytest.mark.asyncio @pytest.mark.parametrize( ("status_code", "body", "expected_error"),