mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(proxy): read every frame of a polled chunk and name model chunks from the sent JSON
A background poll parsed only the first data frame of each chunk, so a terminal event behind an error or delta frame in the same chunk was lost, and the poll stored a status its own stream never ended with. It now reads every frame up to [DONE]
A terminal event whose response is a string or null kept no status and polled completed. Its status now comes from the event type, and final output items that are not objects are skipped, so the rest of the final output is still stored
Model chunks now take their event line from the JSON they send, as mapping chunks already do, so a serializer that drops type no longer sends event: response.completed over {}
This commit is contained in:
parent
b62595b165
commit
7ee563d9e8
4 changed files with 101 additions and 16 deletions
|
|
@ -517,11 +517,9 @@ class ResponsesStreamErrorState:
|
|||
self.sequence_number = -1
|
||||
self.terminal_emitted = False
|
||||
self._pending_event: _StreamEvent | None = None
|
||||
self._pending_on_wire = False
|
||||
|
||||
def observe_chunk(self, chunk: object) -> None:
|
||||
self._pending_event = _observed_event(chunk)
|
||||
self._pending_on_wire = isinstance(chunk, Mapping)
|
||||
|
||||
def _record(self, event: _StreamEvent) -> None:
|
||||
if event.sequence_number is not None:
|
||||
|
|
@ -543,7 +541,7 @@ class ResponsesStreamErrorState:
|
|||
observed: Final = self._pending_event
|
||||
if observed is None:
|
||||
return frame
|
||||
event: Final = _wire_event(frame) if self._pending_on_wire else observed
|
||||
event: Final = _wire_event(frame)
|
||||
self._record(observed if event is None else event)
|
||||
if event is None or not event.type or "\n" in event.type or "\r" in event.type:
|
||||
return frame
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import AsyncIterable, Callable, Mapping, Sequence
|
||||
from http import HTTPStatus
|
||||
from itertools import takewhile
|
||||
from time import perf_counter
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
|
|
@ -90,8 +91,17 @@ class _StreamsBody(Protocol):
|
|||
def body_iterator(self) -> AsyncIterable[str | bytes | memoryview]: ...
|
||||
|
||||
|
||||
def _sse_frame_data(frame: str) -> str | None:
|
||||
return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None)
|
||||
def _payloads_before_done(frame: str) -> tuple[tuple[str, ...], bool]:
|
||||
payloads: Final = tuple(line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: "))
|
||||
return tuple(takewhile(lambda payload: payload != "[DONE]", payloads)), "[DONE]" in payloads
|
||||
|
||||
|
||||
def _terminal_response(event: _StreamEvent) -> _TerminalResponse:
|
||||
match event.get("response"):
|
||||
case Mapping() as response:
|
||||
return response
|
||||
case _:
|
||||
return {}
|
||||
|
||||
|
||||
def _event_error(event: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -279,10 +289,8 @@ async def background_streaming_task(
|
|||
):
|
||||
frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data)))
|
||||
|
||||
if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None:
|
||||
if chunk_data == "[DONE]":
|
||||
break
|
||||
|
||||
payloads, stream_done = _payloads_before_done(chunk) if isinstance(chunk, str) else ((), False)
|
||||
for chunk_data in payloads:
|
||||
try:
|
||||
event: _StreamEvent = _StreamEventParser.parse(chunk_data)
|
||||
event_type = event.get("type", "")
|
||||
|
|
@ -394,7 +402,7 @@ async def background_streaming_task(
|
|||
# Terminal event - extract all ResponsesAPIResponse fields
|
||||
# https://platform.openai.com/docs/api-reference/responses-streaming
|
||||
finished_at = perf_counter()
|
||||
response_data = event.get("response", {})
|
||||
response_data = _terminal_response(event)
|
||||
terminal_status = response_data.get(
|
||||
"status",
|
||||
_event_to_status.get(event_type, "completed"),
|
||||
|
|
@ -428,15 +436,19 @@ async def background_streaming_task(
|
|||
if "output" in response_data:
|
||||
final_output = response_data.get("output", [])
|
||||
for item in final_output:
|
||||
item_id = item.get("id")
|
||||
if item_id:
|
||||
output_items[item_id] = item
|
||||
match item:
|
||||
case {"id": item_id} if item_id:
|
||||
output_items[item_id] = item
|
||||
case _:
|
||||
continue
|
||||
state_dirty = True
|
||||
|
||||
except (json.JSONDecodeError, AttributeError, TypeError) as e:
|
||||
verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e)
|
||||
else:
|
||||
await flush_state_if_needed()
|
||||
if stream_done:
|
||||
break
|
||||
|
||||
# Final flush to ensure all accumulated state is saved
|
||||
await flush_state_if_needed(force=True)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ from fastapi import HTTPException, Response
|
|||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from openai import APIError as OpenAIAPIError
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
from pydantic import BaseModel, Field, PrivateAttr
|
||||
from pydantic import BaseModel, Field, PrivateAttr, model_serializer
|
||||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
|
@ -204,8 +204,8 @@ class _MaskedEverywhereString(str):
|
|||
return "***"
|
||||
|
||||
|
||||
async def _responses_frames(chunks: Iterable[Mapping[str, object]]) -> tuple[str, ...]:
|
||||
async def upstream() -> AsyncIterator[Mapping[str, object]]:
|
||||
async def _responses_frames(chunks: Iterable[object]) -> tuple[str, ...]:
|
||||
async def upstream() -> AsyncIterator[object]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
|
@ -242,6 +242,24 @@ async def test_responses_stream_sends_a_hook_chunk_its_event_model_cannot_read_a
|
|||
assert frames == (f"data: {json.dumps(chunk)}\n\n",)
|
||||
|
||||
|
||||
class _TypeDroppedOnTheWire(BaseModel, frozen=True):
|
||||
type: str = "response.completed"
|
||||
sequence_number: int = 3
|
||||
|
||||
@model_serializer
|
||||
def _without_type(self) -> Mapping[str, int]:
|
||||
return {"sequence_number": self.sequence_number}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_stream_names_a_model_chunk_from_the_json_it_sends() -> None:
|
||||
frames: Final = await _responses_frames((_TypeDroppedOnTheWire(),))
|
||||
|
||||
assert len(frames) == 1
|
||||
assert frames[0].startswith("data: ")
|
||||
assert json.loads(frames[0].removeprefix("data: ")) == {"sequence_number": 3}
|
||||
|
||||
|
||||
class _HookOutputItem(BaseModel, frozen=True):
|
||||
id: str
|
||||
type: str
|
||||
|
|
|
|||
|
|
@ -1679,6 +1679,28 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
None,
|
||||
None,
|
||||
),
|
||||
(('data: {"type": "response.failed", "response": "hook-text"}\n\n',), "failed", None, None),
|
||||
(('data: {"type": "response.failed", "response": null}\n\n',), "failed", None, None),
|
||||
(
|
||||
(
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n'
|
||||
'event: response.incomplete\ndata: {"type": "response.incomplete", '
|
||||
'"response": {"status": "incomplete", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"incomplete",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
(
|
||||
(
|
||||
'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n',
|
||||
'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n'
|
||||
'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n',
|
||||
),
|
||||
"completed",
|
||||
None,
|
||||
{"input_tokens": 3},
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"error_frame_after_a_delta",
|
||||
|
|
@ -1688,6 +1710,10 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
"output_item_done_with_a_string_item_then_completed",
|
||||
"error_frame_then_completed_in_one_chunk",
|
||||
"error_frame_after_done_in_one_chunk",
|
||||
"failed_event_whose_response_is_a_string",
|
||||
"failed_event_whose_response_is_null",
|
||||
"error_frame_and_incomplete_in_one_chunk",
|
||||
"error_frame_then_completed_behind_a_delta_in_one_chunk",
|
||||
],
|
||||
)
|
||||
async def test_a_poll_stores_what_its_stream_ended_with(
|
||||
|
|
@ -1724,6 +1750,37 @@ class TestBackgroundStreamingTerminalEvents:
|
|||
assert final_call.kwargs["error"] == expected_error
|
||||
assert final_call.kwargs["usage"] == expected_usage
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_poll_stores_the_final_output_items_beside_one_that_is_not_an_object(self):
|
||||
from litellm.proxy.response_polling.background_streaming import (
|
||||
background_streaming_task,
|
||||
)
|
||||
|
||||
final_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": "final"}]}
|
||||
draft_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": ""}]}
|
||||
completed: Final = {"type": "response.completed", "response": {"status": "completed", "output": ["hook-text", final_item]}}
|
||||
|
||||
async def _body_iterator():
|
||||
yield f"data: {json.dumps({'type': 'response.output_item.added', 'item': draft_item})}\n\n".encode()
|
||||
yield b'data: {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "draft"}\n\n'
|
||||
yield f"data: {json.dumps(completed)}\n\n".encode()
|
||||
|
||||
mock_response: Final = Mock()
|
||||
mock_response.body_iterator = _body_iterator()
|
||||
handler: Final = AsyncMock(spec=ResponsePollingHandler)
|
||||
kwargs: Final = _make_background_streaming_kwargs("poll_final_output", handler)
|
||||
|
||||
with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests
|
||||
"litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing"
|
||||
) as MockProcessor:
|
||||
MockProcessor.return_value.base_process_llm_request = AsyncMock(
|
||||
return_value=mock_response
|
||||
)
|
||||
await background_streaming_task(**kwargs)
|
||||
|
||||
stored_outputs: Final = [call.kwargs["output"] for call in handler.update_state.call_args_list if "output" in call.kwargs]
|
||||
assert stored_outputs[-1] == [final_item]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "body", "expected_error"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue