fix(proxy): read every frame of a polled chunk and name model chunks from the sent JSON

A background poll parsed only the first data frame of each chunk, so a terminal event behind an error or delta frame in the same chunk was lost, and the poll stored a status its own stream never ended with. It now reads every frame up to [DONE]

A terminal event whose response is a string or null kept no status and polled completed. Its status now comes from the event type, and final output items that are not objects are skipped, so the rest of the final output is still stored

Model chunks now take their event line from the JSON they send, as mapping chunks already do, so a serializer that drops type no longer sends event: response.completed over {}
This commit is contained in:
mateo-berri 2026-10-31 16:56:16 -07:00
parent b62595b165
commit 7ee563d9e8
4 changed files with 101 additions and 16 deletions

View file

@ -517,11 +517,9 @@ class ResponsesStreamErrorState:
self.sequence_number = -1
self.terminal_emitted = False
self._pending_event: _StreamEvent | None = None
self._pending_on_wire = False
def observe_chunk(self, chunk: object) -> None:
self._pending_event = _observed_event(chunk)
self._pending_on_wire = isinstance(chunk, Mapping)
def _record(self, event: _StreamEvent) -> None:
if event.sequence_number is not None:
@ -543,7 +541,7 @@ class ResponsesStreamErrorState:
observed: Final = self._pending_event
if observed is None:
return frame
event: Final = _wire_event(frame) if self._pending_on_wire else observed
event: Final = _wire_event(frame)
self._record(observed if event is None else event)
if event is None or not event.type or "\n" in event.type or "\r" in event.type:
return frame

View file

@ -12,6 +12,7 @@ import asyncio
import json
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
from http import HTTPStatus
from itertools import takewhile
from time import perf_counter
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
@ -90,8 +91,17 @@ class _StreamsBody(Protocol):
def body_iterator(self) -> AsyncIterable[str | bytes | memoryview]: ...
def _sse_frame_data(frame: str) -> str | None:
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
def _payloads_before_done(frame: str) -> tuple[tuple[str, ...], bool]:
payloads: Final = tuple(line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: "))
return tuple(takewhile(lambda payload: payload != "[DONE]", payloads)), "[DONE]" in payloads
def _terminal_response(event: _StreamEvent) -> _TerminalResponse:
match event.get("response"):
case Mapping() as response:
return response
case _:
return {}
def _event_error(event: Mapping[str, object]) -> Mapping[str, object]:
@ -279,10 +289,8 @@ async def background_streaming_task(
):
frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data)))
if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None:
if chunk_data == "[DONE]":
break
payloads, stream_done = _payloads_before_done(chunk) if isinstance(chunk, str) else ((), False)
for chunk_data in payloads:
try:
event: _StreamEvent = _StreamEventParser.parse(chunk_data)
event_type = event.get("type", "")
@ -394,7 +402,7 @@ async def background_streaming_task(
# Terminal event - extract all ResponsesAPIResponse fields
# https://platform.openai.com/docs/api-reference/responses-streaming
finished_at = perf_counter()
response_data = event.get("response", {})
response_data = _terminal_response(event)
terminal_status = response_data.get(
"status",
_event_to_status.get(event_type, "completed"),
@ -428,15 +436,19 @@ async def background_streaming_task(
if "output" in response_data:
final_output = response_data.get("output", [])
for item in final_output:
item_id = item.get("id")
if item_id:
output_items[item_id] = item
match item:
case {"id": item_id} if item_id:
output_items[item_id] = item
case _:
continue
state_dirty = True
except (json.JSONDecodeError, AttributeError, TypeError) as e:
verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e)
else:
await flush_state_if_needed()
if stream_done:
break
# Final flush to ensure all accumulated state is saved
await flush_state_if_needed(force=True)

View file

@ -34,7 +34,7 @@ from fastapi import HTTPException, Response
from fastapi.responses import JSONResponse, StreamingResponse
from openai import APIError as OpenAIAPIError
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
from pydantic import BaseModel, Field, PrivateAttr
from pydantic import BaseModel, Field, PrivateAttr, model_serializer
import litellm
import litellm.proxy.proxy_server as ps
@ -204,8 +204,8 @@ class _MaskedEverywhereString(str):
return "***"
async def _responses_frames(chunks: Iterable[Mapping[str, object]]) -> tuple[str, ...]:
async def upstream() -> AsyncIterator[Mapping[str, object]]:
async def _responses_frames(chunks: Iterable[object]) -> tuple[str, ...]:
async def upstream() -> AsyncIterator[object]:
for chunk in chunks:
yield chunk
@ -242,6 +242,24 @@ async def test_responses_stream_sends_a_hook_chunk_its_event_model_cannot_read_a
assert frames == (f"data: {json.dumps(chunk)}\n\n",)
class _TypeDroppedOnTheWire(BaseModel, frozen=True):
type: str = "response.completed"
sequence_number: int = 3
@model_serializer
def _without_type(self) -> Mapping[str, int]:
return {"sequence_number": self.sequence_number}
@pytest.mark.asyncio
async def test_responses_stream_names_a_model_chunk_from_the_json_it_sends() -> None:
frames: Final = await _responses_frames((_TypeDroppedOnTheWire(),))
assert len(frames) == 1
assert frames[0].startswith("data: ")
assert json.loads(frames[0].removeprefix("data: ")) == {"sequence_number": 3}
class _HookOutputItem(BaseModel, frozen=True):
id: str
type: str

View file

@ -1679,6 +1679,28 @@ class TestBackgroundStreamingTerminalEvents:
None,
None,
),
(('data: {"type": "response.failed", "response": "hook-text"}\n\n',), "failed", None, None),
(('data: {"type": "response.failed", "response": null}\n\n',), "failed", None, None),
(
(
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n'
'event: response.incomplete\ndata: {"type": "response.incomplete", '
'"response": {"status": "incomplete", "usage": {"input_tokens": 3}}}\n\n',
),
"incomplete",
None,
{"input_tokens": 3},
),
(
(
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n'
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
),
"completed",
None,
{"input_tokens": 3},
),
],
ids=[
"error_frame_after_a_delta",
@ -1688,6 +1710,10 @@ class TestBackgroundStreamingTerminalEvents:
"output_item_done_with_a_string_item_then_completed",
"error_frame_then_completed_in_one_chunk",
"error_frame_after_done_in_one_chunk",
"failed_event_whose_response_is_a_string",
"failed_event_whose_response_is_null",
"error_frame_and_incomplete_in_one_chunk",
"error_frame_then_completed_behind_a_delta_in_one_chunk",
],
)
async def test_a_poll_stores_what_its_stream_ended_with(
@ -1724,6 +1750,37 @@ class TestBackgroundStreamingTerminalEvents:
assert final_call.kwargs["error"] == expected_error
assert final_call.kwargs["usage"] == expected_usage
@pytest.mark.asyncio
async def test_a_poll_stores_the_final_output_items_beside_one_that_is_not_an_object(self):
from litellm.proxy.response_polling.background_streaming import (
background_streaming_task,
)
final_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": "final"}]}
draft_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": ""}]}
completed: Final = {"type": "response.completed", "response": {"status": "completed", "output": ["hook-text", final_item]}}
async def _body_iterator():
yield f"data: {json.dumps({'type': 'response.output_item.added', 'item': draft_item})}\n\n".encode()
yield b'data: {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "draft"}\n\n'
yield f"data: {json.dumps(completed)}\n\n".encode()
mock_response: Final = Mock()
mock_response.body_iterator = _body_iterator()
handler: Final = AsyncMock(spec=ResponsePollingHandler)
kwargs: Final = _make_background_streaming_kwargs("poll_final_output", handler)
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
) as MockProcessor:
MockProcessor.return_value.base_process_llm_request = AsyncMock(
return_value=mock_response
)
await background_streaming_task(**kwargs)
stored_outputs: Final = [call.kwargs["output"] for call in handler.update_state.call_args_list if "output" in call.kwargs]
assert stored_outputs[-1] == [final_item]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("status_code", "body", "expected_error"),