diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index e9caf82afff..ca948f65bd2 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -517,11 +517,9 @@ class ResponsesStreamErrorState: self.sequence_number = -1 self.terminal_emitted = False self._pending_event: _StreamEvent | None = None - self._pending_on_wire = False def observe_chunk(self, chunk: object) -> None: self._pending_event = _observed_event(chunk) - self._pending_on_wire = isinstance(chunk, Mapping) def _record(self, event: _StreamEvent) -> None: if event.sequence_number is not None: @@ -543,7 +541,7 @@ class ResponsesStreamErrorState: observed: Final = self._pending_event if observed is None: return frame - event: Final = _wire_event(frame) if self._pending_on_wire else observed + event: Final = _wire_event(frame) self._record(observed if event is None else event) if event is None or not event.type or "\n" in event.type or "\r" in event.type: return frame diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index ae8f4939c1b..9d211fe4e71 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -12,6 +12,7 @@ import asyncio import json from collections.abc import AsyncIterable, Callable, Mapping, Sequence from http import HTTPStatus +from itertools import takewhile from time import perf_counter from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable @@ -90,8 +91,17 @@ class _StreamsBody(Protocol): def body_iterator(self) -> AsyncIterable[str | bytes | memoryview]: ... -def _sse_frame_data(frame: str) -> str | None: - return next((line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")), None) +def _payloads_before_done(frame: str) -> tuple[tuple[str, ...], bool]: + payloads: Final = tuple(line[6:].strip() for line in split_sse_lines(frame) if line.startswith("data: ")) + return tuple(takewhile(lambda payload: payload != "[DONE]", payloads)), "[DONE]" in payloads + + +def _terminal_response(event: _StreamEvent) -> _TerminalResponse: + match event.get("response"): + case Mapping() as response: + return response + case _: + return {} def _event_error(event: Mapping[str, object]) -> Mapping[str, object]: @@ -279,10 +289,8 @@ async def background_streaming_task( ): frame_failure = responses_error_fields(_event_error(_StreamEventParser.parse(error_data))) - if isinstance(chunk, str) and (chunk_data := _sse_frame_data(chunk)) is not None: - if chunk_data == "[DONE]": - break - + payloads, stream_done = _payloads_before_done(chunk) if isinstance(chunk, str) else ((), False) + for chunk_data in payloads: try: event: _StreamEvent = _StreamEventParser.parse(chunk_data) event_type = event.get("type", "") @@ -394,7 +402,7 @@ async def background_streaming_task( # Terminal event - extract all ResponsesAPIResponse fields # https://platform.openai.com/docs/api-reference/responses-streaming finished_at = perf_counter() - response_data = event.get("response", {}) + response_data = _terminal_response(event) terminal_status = response_data.get( "status", _event_to_status.get(event_type, "completed"), @@ -428,15 +436,19 @@ async def background_streaming_task( if "output" in response_data: final_output = response_data.get("output", []) for item in final_output: - item_id = item.get("id") - if item_id: - output_items[item_id] = item + match item: + case {"id": item_id} if item_id: + output_items[item_id] = item + case _: + continue state_dirty = True except (json.JSONDecodeError, AttributeError, TypeError) as e: verbose_proxy_logger.warning("Failed to parse streaming chunk: %s", e) else: await flush_state_if_needed() + if stream_done: + break # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index c6628319110..60f92c4cd06 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -34,7 +34,7 @@ from fastapi import HTTPException, Response from fastapi.responses import JSONResponse, StreamingResponse from openai import APIError as OpenAIAPIError from openai.types.responses import ResponseOutputMessage, ResponseOutputText -from pydantic import BaseModel, Field, PrivateAttr +from pydantic import BaseModel, Field, PrivateAttr, model_serializer import litellm import litellm.proxy.proxy_server as ps @@ -204,8 +204,8 @@ class _MaskedEverywhereString(str): return "***" -async def _responses_frames(chunks: Iterable[Mapping[str, object]]) -> tuple[str, ...]: - async def upstream() -> AsyncIterator[Mapping[str, object]]: +async def _responses_frames(chunks: Iterable[object]) -> tuple[str, ...]: + async def upstream() -> AsyncIterator[object]: for chunk in chunks: yield chunk @@ -242,6 +242,24 @@ async def test_responses_stream_sends_a_hook_chunk_its_event_model_cannot_read_a assert frames == (f"data: {json.dumps(chunk)}\n\n",) +class _TypeDroppedOnTheWire(BaseModel, frozen=True): + type: str = "response.completed" + sequence_number: int = 3 + + @model_serializer + def _without_type(self) -> Mapping[str, int]: + return {"sequence_number": self.sequence_number} + + +@pytest.mark.asyncio +async def test_responses_stream_names_a_model_chunk_from_the_json_it_sends() -> None: + frames: Final = await _responses_frames((_TypeDroppedOnTheWire(),)) + + assert len(frames) == 1 + assert frames[0].startswith("data: ") + assert json.loads(frames[0].removeprefix("data: ")) == {"sequence_number": 3} + + class _HookOutputItem(BaseModel, frozen=True): id: str type: str diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 8d414c798f1..43184f95700 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1679,6 +1679,28 @@ class TestBackgroundStreamingTerminalEvents: None, None, ), + (('data: {"type": "response.failed", "response": "hook-text"}\n\n',), "failed", None, None), + (('data: {"type": "response.failed", "response": null}\n\n',), "failed", None, None), + ( + ( + 'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n' + 'event: response.incomplete\ndata: {"type": "response.incomplete", ' + '"response": {"status": "incomplete", "usage": {"input_tokens": 3}}}\n\n', + ), + "incomplete", + None, + {"input_tokens": 3}, + ), + ( + ( + 'event: error\ndata: {"error": {"code": "400", "message": "output blocked by policy"}}\n\n', + 'data: {"type": "response.output_text.delta", "delta": "hi"}\n\n' + 'data: {"type": "response.completed", "response": {"status": "completed", "usage": {"input_tokens": 3}}}\n\n', + ), + "completed", + None, + {"input_tokens": 3}, + ), ], ids=[ "error_frame_after_a_delta", @@ -1688,6 +1710,10 @@ class TestBackgroundStreamingTerminalEvents: "output_item_done_with_a_string_item_then_completed", "error_frame_then_completed_in_one_chunk", "error_frame_after_done_in_one_chunk", + "failed_event_whose_response_is_a_string", + "failed_event_whose_response_is_null", + "error_frame_and_incomplete_in_one_chunk", + "error_frame_then_completed_behind_a_delta_in_one_chunk", ], ) async def test_a_poll_stores_what_its_stream_ended_with( @@ -1724,6 +1750,37 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["error"] == expected_error assert final_call.kwargs["usage"] == expected_usage + @pytest.mark.asyncio + async def test_a_poll_stores_the_final_output_items_beside_one_that_is_not_an_object(self): + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + final_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": "final"}]} + draft_item: Final = {"id": "msg_1", "content": [{"type": "output_text", "text": ""}]} + completed: Final = {"type": "response.completed", "response": {"status": "completed", "output": ["hook-text", final_item]}} + + async def _body_iterator(): + yield f"data: {json.dumps({'type': 'response.output_item.added', 'item': draft_item})}\n\n".encode() + yield b'data: {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "draft"}\n\n' + yield f"data: {json.dumps(completed)}\n\n".encode() + + mock_response: Final = Mock() + mock_response.body_iterator = _body_iterator() + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_final_output", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + stored_outputs: Final = [call.kwargs["output"] for call in handler.update_state.call_args_list if "output" in call.kwargs] + assert stored_outputs[-1] == [final_item] + @pytest.mark.asyncio @pytest.mark.parametrize( ("status_code", "body", "expected_error"),