From 6b5ed50633348efa3e7c9b416e2a99d5a959a00a Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 23 Oct 2026 07:42:05 +0800 Subject: [PATCH 01/14] fix(proxy): preserve Responses SSE event framing --- .../guardrail_translation/handler.py | 2 +- .../common_utils/responses_stream_errors.py | 5 +- litellm/proxy/proxy_server.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 1 - ...test_openai_responses_guardrail_handler.py | 9 ++- .../proxy_server/test_streaming_helpers.py | 62 +++++++++++++++++-- .../response_api_endpoints/test_endpoints.py | 14 +++-- .../proxy/test_response_model_sanitization.py | 5 +- 8 files changed, 85 insertions(+), 15 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 3ea126c5a2a..d08806b19cb 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1789,7 +1789,6 @@ class OpenAIResponsesHandler(BaseTranslation): new output item under the same response id, and close with a ``response.completed`` carrying only the replacement item. - The proxy's data generator appends ``data: [DONE]`` itself. """ events: Final = ( self._block_continuation_events(exc, responses_so_far or ()) @@ -1797,6 +1796,7 @@ class OpenAIResponsesHandler(BaseTranslation): else self._standalone_block_events(exc) ) return tuple( + f"event: {event.type.value if isinstance(event.type, ResponsesAPIStreamEvents) else event.type}\n" f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode() for event in events ) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 3409b4b6657..6e563b2b257 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -131,7 +131,10 @@ class ResponsesStreamErrorState: self.created_at = event.response.created_at if event.type in ("response.completed", "response.failed", "response.incomplete"): self.terminal_emitted = True - return frame + if not event.type or "\n" in event.type or "\r" in event.type: + return frame + prefix: Final = f"event: {event.type}\n" + return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_failure(self, exception: Exception) -> str | None: if self.terminal_emitted: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f632317d9a8..6aff6601ac4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10390,7 +10390,7 @@ async def async_data_generator( stream_completed = False client_disconnected = False error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None - ends_with_openai_done: Final = not request_data.get("_litellm_skip_openai_stream_done") + ends_with_openai_done: Final = not responses_stream_errors and not request_data.get("_litellm_skip_openai_stream_done") needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap() stream_iterator: Final = _object_stream( proxy_logging_obj.async_post_call_streaming_iterator_hook( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 312c5124ca5..8a13ce876ea 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -493,7 +493,6 @@ async def responses_api( async def _blocked_stream() -> AsyncGenerator[str, None]: for chunk in block_chunks: yield chunk.decode() - yield "data: [DONE]\n\n" return await create_response(generator=_blocked_stream(), media_type="text/event-stream", headers={}) return build_blocked_response(e) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 2f10fd6fd0d..23dbf3906d6 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3766,7 +3766,14 @@ class TestBuildBlockSseChunks: def _payloads(self, chunks): import json - return [json.loads(chunk.decode().removeprefix("data: ").strip()) for chunk in chunks] + payloads = [ + json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: "))) + for chunk in chunks + ] + assert [chunk.decode().splitlines()[0] for chunk in chunks] == [ + f"event: {payload['type']}" for payload in payloads + ] + return payloads def test_standalone_block_emits_complete_synthetic_stream(self): handler = OpenAIResponsesHandler() diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 9a3516d20d6..015269801b8 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -50,9 +50,12 @@ from litellm.proxy.proxy_server import ( select_data_generator, ) from litellm.types.llms.openai import ( + ErrorEvent, + OutputTextDeltaEvent, ResponseCompletedEvent, ResponseCreatedEvent, ResponseFailedEvent, + ResponseIncompleteEvent, ResponsesAPIResponse, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage @@ -92,6 +95,56 @@ async def _async_iter_raises(exc: Exception): raise exc +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal_type", ["response.completed", "response.failed", "response.incomplete", "error"]) +async def test_responses_stream_dispatches_named_events_without_a_chat_done_sentinel(terminal_type: str) -> None: + response: Final = ResponsesAPIResponse(id="resp_framing", created_at=1, model="local-model", output=[]) + created: Final = ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ) + delta: Final = OutputTextDeltaEvent.model_validate( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_framing", + "output_index": 0, + "content_index": 0, + "delta": "hello", + } + ) + terminal_models: Final = { + "response.completed": ResponseCompletedEvent, + "response.failed": ResponseFailedEvent, + "response.incomplete": ResponseIncompleteEvent, + "error": ErrorEvent, + } + terminal: Final = terminal_models[terminal_type].model_validate( + {"type": terminal_type, "sequence_number": 2, "response": response} + ) + + async def upstream() -> AsyncIterator[BaseModel]: + for event in (created, delta, terminal): + yield event + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True + ) + ] + ) + assert len(frames) == 3 + for frame, event_type in zip( + frames, ("response.created", "response.output_text.delta", terminal_type), strict=True + ): + assert frame.startswith(f"event: {event_type}\n") + payload: Final = json.loads( + next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: ")) + ) + assert payload["type"] == event_type + assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" + class _FakeStream: def __init__(self, chunks, hidden_params=None): self._chunks = chunks @@ -1132,14 +1185,15 @@ async def test_responses_stream_failure_sends_the_failure_hook_replacement(monke ] ) failure: Final = ResponseFailedEvent.model_validate_json( - next(line.removeprefix("data: ") for line in frames[-2].splitlines() if line.startswith("data: ")) + next(line.removeprefix("data: ") for line in frames[-1].splitlines() if line.startswith("data: ")) ) assert _BLOCK_MESSAGE not in "".join(frames) assert failure.response.error is not None assert failure.response.error["code"] == "permission_error" assert "Request blocked by policy" in failure.response.error["message"] - assert frames[-1] == "data: [DONE]\n\n" + assert frames[-1].startswith("event: response.failed\n") + assert "data: [DONE]\n\n" not in frames @pytest.mark.asyncio @@ -1335,8 +1389,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina for frame in event_frames ) - assert decoded[-1] == "data: [DONE]\n\n" - assert len(decoded) == len(event_frames) + 1 + assert len(decoded) == len(event_frames) + assert "data: [DONE]\n\n" not in decoded assert payloads[0]["response"]["id"] == "resp_visible" assert payloads[1] == tool_delta.model_dump() assert len(payloads) == 3 diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 985aee061bc..0bf14f59b1a 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -142,8 +142,9 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert result.status_code == 200, result.text assert message in result.text if path == "/v1/responses": - assert frames[-1] == "data: [DONE]", result.text - assert frames[-2].startswith("event: response.failed\n"), result.text + assert all("data: [DONE]" not in frame for frame in frames), result.text + assert frames[-1].startswith("event: response.failed\n"), result.text + assert [frame.splitlines()[0] for frame in frames] == [f"event: {event['type']}" for event in events] if partial: assert [event["type"] for event in events] == [ "response.created", @@ -4862,9 +4863,11 @@ class TestGuardrailBlockedResponsesShape: assert response.status_code == 200, response.text assert response.headers["content-type"].startswith("text/event-stream") frames = _sse_data_frames(response.text) - assert frames[-1] == "[DONE]" - events = [json.loads(frame) for frame in frames[:-1]] + assert "[DONE]" not in frames + events = [json.loads(frame) for frame in frames] types = [event["type"] for event in events] + assert types[-1] == "response.completed" + assert [line[7:] for line in response.text.splitlines() if line.startswith("event: ")] == types assert "response.created" in types completed = [event for event in events if event["type"] == "response.completed"] assert len(completed) == 1 @@ -4891,7 +4894,8 @@ class TestGuardrailBlockedResponsesShape: assert response.status_code == 200, response.text frames = _sse_data_frames(response.text) - completed = [json.loads(frame) for frame in frames[:-1] if json.loads(frame)["type"] == "response.completed"] + assert "[DONE]" not in frames + completed = [json.loads(frame) for frame in frames if json.loads(frame)["type"] == "response.completed"] usage = completed[0]["response"]["usage"] assert usage["input_tokens"] == 14 assert usage["output_tokens"] == 20 diff --git a/tests/unit/proxy/test_response_model_sanitization.py b/tests/unit/proxy/test_response_model_sanitization.py index f52736453c1..6f3c5d8430b 100644 --- a/tests/unit/proxy/test_response_model_sanitization.py +++ b/tests/unit/proxy/test_response_model_sanitization.py @@ -763,7 +763,10 @@ def test_restamp_streaming_chunk_rewrites_nested_model_in_guardrail_block_frames )[0] for frame in frames ] - events = [json.loads(frame.decode().removeprefix("data: ")) for frame in client_frames] + events = [ + json.loads(next(line[6:] for line in frame.decode().splitlines() if line.startswith("data: "))) + for frame in client_frames + ] assert all(frame.endswith(b"\n\n") for frame in client_frames) assert events[-1]["type"] == "response.completed" From dfe2b4fd87f3b774330c7832822aa503a2b4ece6 Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 23 Oct 2026 07:57:01 +0800 Subject: [PATCH 02/14] style(proxy): format Responses stream terminator condition --- litellm/proxy/proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6aff6601ac4..987c8739c04 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10390,7 +10390,9 @@ async def async_data_generator( stream_completed = False client_disconnected = False error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None - ends_with_openai_done: Final = not responses_stream_errors and not request_data.get("_litellm_skip_openai_stream_done") + ends_with_openai_done: Final = not responses_stream_errors and not request_data.get( + "_litellm_skip_openai_stream_done" + ) needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap() stream_iterator: Final = _object_stream( proxy_logging_obj.async_post_call_streaming_iterator_hook( From 4fb052e786b66248329dea5747927aa5f1730165 Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 23 Oct 2026 08:26:33 +0800 Subject: [PATCH 03/14] fix(proxy): validate Responses guardrail event names --- .../guardrail_translation/handler.py | 3 ++- .../proxy_server/test_streaming_helpers.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index d08806b19cb..2e77fdd62f0 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -251,6 +251,7 @@ _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call" _OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE} _SCANNED_TOOL_SEARCH_NAME: Final = "tool_search" _TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str) _TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call": "arguments", "custom_tool_call": "input"} ) @@ -1796,7 +1797,7 @@ class OpenAIResponsesHandler(BaseTranslation): else self._standalone_block_events(exc) ) return tuple( - f"event: {event.type.value if isinstance(event.type, ResponsesAPIStreamEvents) else event.type}\n" + f"event: {_STREAM_EVENT_TYPE_ADAPTER.validate_python(stream_item_field(event, 'type'))}\n" f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode() for event in events ) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 015269801b8..835411608ed 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -34,6 +34,7 @@ from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, _fast_serialize_simple_model_response_stream, @@ -145,6 +146,31 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent assert payload["type"] == event_type assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" +@pytest.mark.parametrize("binary", [False, True]) +@pytest.mark.parametrize( + ("event_type", "event_line"), + [ + (None, ""), + ("", ""), + ("response.delta\ninjected", ""), + ("response.delta\rinjected", ""), + ("response.delta\r\ninjected", ""), + ("response.vendor_delta", "event: response.vendor_delta\n"), + ], +) +def test_responses_event_names_preserve_payload_without_injecting_sse_lines( + event_type: str | None, event_line: str, binary: bool +) -> None: + payload: Final = {"type": event_type, "sequence_number": 0} + data_line: Final = f"data: {json.dumps(payload)}\n\n" + frame: Final = data_line.encode("utf-8") if binary else data_line + expected: Final = (event_line + data_line).encode("utf-8") if binary else event_line + data_line + state: Final = ResponsesStreamErrorState() + state.observe_chunk(payload) + + assert state.mark_emitted(frame) == expected + + class _FakeStream: def __init__(self, chunks, hidden_params=None): self._chunks = chunks From 7586485b4e274dd90a0ccfbf45886aeab392f9d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:03:13 -0700 Subject: [PATCH 04/14] fix(proxy): keep Responses first-event errors and keepalive failures in Responses framing --- litellm/proxy/common_request_processing.py | 68 +++++++++++++++---- .../common_utils/responses_stream_errors.py | 18 ++++- litellm/proxy/proxy_server.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 2 + .../observability/test_guardrail_effects.py | 16 +++-- .../test_bedrock_mantle_claude_chat_chaos.py | 2 +- .../test_bedrock_mantle_claude_chat_wire.py | 7 +- ...test_chat_stream_guardrail_error_events.py | 3 +- .../test_responses_bridge_stream_errors.py | 3 +- .../test_responses_top_level_error_events.py | 4 +- ...test_openai_responses_guardrail_handler.py | 2 +- .../response_api_endpoints/test_endpoints.py | 61 +++++++++++++++++ .../proxy/test_common_request_processing.py | 50 ++++++++++++++ 13 files changed, 207 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a36a7f2dc..abc99a3be23 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4,6 +4,7 @@ import json import logging import math from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from functools import lru_cache from itertools import chain @@ -114,6 +115,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, openai_error_type, ) +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, SSE_STREAM_START_TAIL, @@ -907,11 +909,18 @@ async def _enforce_guardrail_added_tag_budgets( ) from e +def _sse_data_payload(event_line: str | bytes) -> str | None: + text: Final = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line + return next( + (line.removeprefix("data: ").strip() for line in text.splitlines() if line.startswith("data: ")), + None, + ) + + async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: """Parses an event line and returns an error code if present, else None.""" - event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line - if event_line.startswith("data: "): - json_str: Final = event_line[len("data: ") :].strip() + json_str: Final = _sse_data_payload(event_line) + if json_str is not None: if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message return None try: @@ -952,8 +961,6 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: Returns: Error dictionary in OpenAI API format """ - event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line - # Default error format default_error: Final = { "message": "Unknown error", @@ -962,8 +969,8 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: "code": "500", } - if event_line.startswith("data: "): - json_str: Final = event_line[len("data: ") :].strip() + json_str: Final = _sse_data_payload(event_line) + if json_str is not None: if not json_str or json_str == "[DONE]": return default_error @@ -1171,6 +1178,39 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]: return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n" +@dataclass(frozen=True, slots=True) +class LateSseFraming: + """How a response the keepalive already opened ends once its upstream fails late.""" + + failure: Callable[[Exception], tuple[str, ...]] + body: Callable[[Response], tuple[bytes, ...]] + + +def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]: + _, error_obj = sse_error_payload(exc) + return _sse_error_frames(error_obj) + + +def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: + return b"data: " + (bytes(produced.body) or b"{}") + b"\n\n", b"data: [DONE]\n\n" + + +def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: + frame: Final = ResponsesStreamErrorState().format_failure(exc) + return () if frame is None else (frame,) + + +def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: + frame: Final = ResponsesStreamErrorState().format_error_body(bytes(produced.body), produced.status_code) + return () if frame is None else (frame.encode(),) + + +OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames) +RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming( + failure=_responses_late_failure_frames, body=_responses_late_body_frames +) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -1371,13 +1411,13 @@ async def _aclose_late_response(produced: Response) -> None: verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc) -async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]: +async def _relay_late_response(produced: Response, framing: LateSseFraming) -> AsyncGenerator[bytes, None]: """Replay a Response that was built after a keepalive had already opened the wire.""" if not isinstance(produced, StreamingResponse): # The status line is already on the wire, so a non-streaming body, an error # body included, can only reach the client as an SSE frame. - yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n" - yield b"data: [DONE]\n\n" + for frame in framing.body(produced): + yield frame return try: @@ -1419,6 +1459,7 @@ async def open_sse_before_first_byte( ping_interval_seconds: float | str | None, media_type: str = "text/event-stream", on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None, + late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, ) -> _LateResponseT | StreamingResponse: """Write SSE keepalive comments while the upstream LLM call is still in flight. @@ -1464,11 +1505,10 @@ async def open_sse_before_first_byte( # would never fire and the failure would go unaudited. The hook # also gets to sanitize what reaches the client, by returning or # raising a replacement, so its answer decides the frame. - _, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure)) - for frame in _sse_error_frames(error_obj): + for frame in late_framing.failure(await _sanitized_late_failure(exc, on_late_failure)): yield frame.encode() return - async for chunk in _relay_late_response(produced): + async for chunk in _relay_late_response(produced, late_framing): yield chunk finally: if not produce_task.done(): @@ -2812,6 +2852,7 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request: bool | None = False, contents: list[object] | None = None, skip_pre_call_logic: bool = False, + late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, ) -> object: """Run the request, sending SSE keepalives while the upstream is still silent. @@ -2855,6 +2896,7 @@ class ProxyBaseLLMRequestProcessing: ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, + late_framing=late_framing, ) async def _process_llm_request( diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 6e563b2b257..312bf472594 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -57,6 +57,12 @@ class _FailureDetails(LiteLLMBaseModel): return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields}) +class _ErrorBody(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + error: _FailureDetails = _FailureDetails() + + def _original_failure(exception: Exception) -> Exception: current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(current, MidStreamFallbackError) and current.original_exception is not None: @@ -137,10 +143,16 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_failure(self, exception: Exception) -> str | None: + original: Final = _original_failure(exception) + return self._format_failed(_failure_details(original), str(original)) + + def format_error_body(self, body: bytes, status_code: int) -> str | None: + error: Final = _ErrorBody.model_validate_json(body or b"{}").error + return self._format_failed(error.model_copy(update={"status_code": status_code}), body.decode()) + + def _format_failed(self, details: _FailureDetails, fallback_message: str) -> str | None: if self.terminal_emitted: return None - original: Final = _original_failure(exception) - details: Final = _failure_details(original) response: Final = ResponsesAPIResponse.model_validate( MappingProxyType( { @@ -153,7 +165,7 @@ class ResponsesStreamErrorState: "error": MappingProxyType( { "code": _response_error_code(details), - "message": redact_internal_details_from_client_message(details.message or str(original)), + "message": redact_internal_details_from_client_message(details.message or fallback_message), } ), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a5a287fb42e..15832d81b7f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10620,7 +10620,7 @@ async def async_data_generator( if error_message is not None: yield error_message - # OpenAI-compatible streams terminate with data: [DONE]; Google GenAI (?alt=sse) does not. + # Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not. if ends_with_openai_done: yield _OPENAI_STREAM_DONE_FRAME except (asyncio.CancelledError, GeneratorExit): diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index aa5fe472c66..1e16c16f33a 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -52,6 +52,7 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.common_request_processing import ( + RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing, cost_headers, create_response, @@ -435,6 +436,7 @@ async def responses_api( if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data)) else native_data_generator ), + late_framing=RESPONSES_LATE_SSE_FRAMING, model=None, user_model=user_model, user_temperature=user_temperature, diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 8f83287696c..c1ed810a428 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1197,9 +1197,7 @@ def test_responses_pre_call_denial_streams_sse_with_typed_message_item(gateway: response.headers["content-type"], response.text, ) - lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: ")) - assert lines[-1] == "data: [DONE]", response.text - events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1]) + events: Final = _blocked_stream_events(response.text) kinds: Final = tuple(event["type"] for event in events) assert tuple(kind for kind in kinds if kind != "response.output_text.delta") == ( "response.created", @@ -1286,9 +1284,13 @@ def _responses_output_denial_config(tmp_path: Path, identity: str, model: str) - def _blocked_stream_events(text: str) -> tuple[dict[str, object], ...]: - lines: Final = tuple(line for line in text.split("\n") if line.startswith("data: ")) - assert lines[-1] == "data: [DONE]", text - return tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1]) + assert "data: [DONE]" not in text, text + lines: Final = tuple(line for line in text.split("\n") if line.startswith(("event: ", "data: "))) + events: Final[tuple[dict[str, object], ...]] = tuple( + json.loads(line.removeprefix("data: ")) for line in lines[1::2] + ) + assert lines[::2] == tuple(f"event: {event['type']}" for event in events), text + return events def _dead_api_base() -> str: @@ -2786,7 +2788,7 @@ def test_client_disconnect_after_a_finished_responses_tool_call_scans_the_text_a with _disconnect_rig(gateway, tmp_path, shape="text_then_tool_call") as rig: try: received: Final = _stream_and_close( - rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), "response.output_item.done" + rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), '"type":"response.output_item.done"' ) assert rig.token() in received, received scans: Final = eventually( diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py index bcbf6d0f6f3..2325b705790 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py @@ -261,7 +261,7 @@ async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once( assert len(served) == 10 for item in served: _assert_answered_with_its_own_marker(item) - assert item.text.rstrip().endswith("data: [DONE]"), item.text + assert item.text.rstrip().endswith("data: [DONE]") == (item.call.endpoint == "chat"), item.text _assert_native_requests_for(wire.drain(), calls) assert _spend_status_by_call(served) == {item.call_id: "success" for item in served} diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index d4f9533fcdc..9c73fcc3b6e 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -846,7 +846,12 @@ def test_responses_api_stream_on_a_mantle_claude_id_bridges_to_the_native_stream "POST", "/v1/responses", {"model": model, "input": _prompt(marker), "stream": True} ) assert response.status_code == 200, response.text - events: Final = _sse_objects(response.text) + assert "data: [DONE]" not in response.text, response.text + events: Final = tuple( + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") + ) kinds: Final = [event["type"] for event in events] assert (kinds[0], kinds[-1]) == ("response.created", "response.completed"), kinds assert "".join( diff --git a/tests/integration/streaming/test_chat_stream_guardrail_error_events.py b/tests/integration/streaming/test_chat_stream_guardrail_error_events.py index a8ebe510b8e..b2390de7737 100644 --- a/tests/integration/streaming/test_chat_stream_guardrail_error_events.py +++ b/tests/integration/streaming/test_chat_stream_guardrail_error_events.py @@ -482,8 +482,7 @@ def _responses_body(marker: str, *directives: str) -> dict[str, JsonValue]: def _failed_response_error(response: httpx.Response) -> dict[str, JsonValue]: assert response.status_code == 200, response.text - lines: Final = _data_lines(response.text) - assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text + assert "[DONE]" not in _data_lines(response.text), response.text events: Final = _payloads(response.text) kinds: Final = tuple(string_value(event["type"]) for event in events) assert kinds[0] == "response.created" and kinds[-1] == "response.failed", response.text diff --git a/tests/integration/streaming/test_responses_bridge_stream_errors.py b/tests/integration/streaming/test_responses_bridge_stream_errors.py index 8a5655be0f8..6b0e35f9c56 100644 --- a/tests/integration/streaming/test_responses_bridge_stream_errors.py +++ b/tests/integration/streaming/test_responses_bridge_stream_errors.py @@ -474,9 +474,10 @@ def test_native_responses_stream_forwards_the_failed_response_on_both_legs(gatew assert response.status_code == 200, response.text frames: Final = data_frames(response.text) assert [frame["type"] for frame in frames] == ["response.created", "response.failed"], response.text + assert [name for name, _ in sse_events(response.text)] == ["response.created", "response.failed"], response.text failed: Final = object_value(frames[-1]["response"]) assert failed["status"] == "failed", failed assert object_value(failed["error"])["code"] == "rate_limit_exceeded", failed - assert response.text.rstrip().endswith("data: [DONE]"), response.text + assert "data: [DONE]" not in response.text, response.text assert len(wire.drain()) == 1 assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" diff --git a/tests/integration/streaming/test_responses_top_level_error_events.py b/tests/integration/streaming/test_responses_top_level_error_events.py index 63bdadceebc..6d361492dd6 100644 --- a/tests/integration/streaming/test_responses_top_level_error_events.py +++ b/tests/integration/streaming/test_responses_top_level_error_events.py @@ -256,8 +256,10 @@ def _responses_frames(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[http ) _assert_one_attempt(served.wire, marker) assert served.response.status_code == 200, served.response.text - assert served.response.text.rstrip().endswith("data: [DONE]"), served.response.text + assert "data: [DONE]" not in served.response.text, served.response.text frames: Final = data_frames(served.response.text) + names: Final = tuple(name for name, _ in sse_events(served.response.text)) + assert names == tuple(item["type"] for item in frames), served.response.text assert frames[-1]["type"] == "response.failed", served.response.text return served.response, object_value(object_value(frames[-1]["response"])["error"]) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 0b7f32575c5..8aa887bfd78 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3783,7 +3783,7 @@ class TestBuildBlockSseChunks: def _payloads(self, chunks): import json - payloads = [ + payloads: Final = [ json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: "))) for chunk in chunks ] diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 0bf14f59b1a..5945352c61e 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -177,6 +177,67 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) +async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str): + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-6-astra" + message: Final = "Upstream is rate limiting this key" + upstream_url: Final = "https://streaming.example/v1" + is_chat: Final = path == "/v1/chat/completions" + + async def slow_rate_limit(_: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.3) + return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "openai/" + model, + "api_base": upstream_url, + "api_key": "fixture-key", + "max_retries": 0, + }, + } + ], + num_retries=0, + ) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( + side_effect=slow_rate_limit + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + path, + json={ + "model": model, + "stream": True, + **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), + }, + ) + frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) + + assert result.status_code == 200, result.text + assert frames[0] == ": ping", result.text + if is_chat: + assert frames[-1] == "data: [DONE]", result.text + assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] + else: + assert "data: [DONE]" not in result.text + assert frames[-1].startswith("event: response.failed\ndata: "), result.text + failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] + assert failed_error["code"] == "rate_limit_exceeded" + assert message in failed_error["message"] + + _COMPACTION_ITEM: Final = { "type": "compaction", "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 52816dc50a8..310b526365f 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -50,6 +50,7 @@ from litellm.proxy.common_request_processing import ( _has_attribute_error_in_chain, include_guardrail_response_requested, is_azure_model_router_request, + RESPONSES_LATE_SSE_FRAMING, open_sse_before_first_byte, resolve_litellm_call_id, ttft_keepalive_interval, @@ -2161,6 +2162,18 @@ class TestCommonRequestProcessingHelpers: assert body["error"]["code"] == 403 assert body["error"]["message"] == "forbidden" + async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self): + error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None} + + async def responses_stream(): + yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" + + response: Final = await create_response(responses_stream(), "text/event-stream", {}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(response.body) == {"error": error} + async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): yield 'data: {"content": "first part"}\n\n' @@ -10378,6 +10391,43 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" +async def _slow_rate_limit_failure(): + await asyncio.sleep(0.2) + raise HTTPException(status_code=429, detail="rate limited") + + +async def _slow_guardrail_block_body(): + await asyncio.sleep(0.2) + return JSONResponse(status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "produce,expected_code,expected_message", + [ + (_slow_rate_limit_failure, "rate_limit_exceeded", "rate limited"), + (_slow_guardrail_block_body, "invalid_request_error", "blocked"), + ], +) +async def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( + produce, expected_code, expected_message +): + response: Final = await open_sse_before_first_byte( + produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + ) + collected: Final = await _drain(response) + + assert collected[0] == TTFT_PING + assert b"data: [DONE]\n\n" not in collected + event_line, data_line = collected[-1].decode().strip().split("\n") + failed: Final = json.loads(data_line.removeprefix("data: ")) + assert event_line == "event: response.failed" + assert failed["type"] == "response.failed" + assert failed["response"]["status"] == "failed" + assert failed["response"]["error"]["code"] == expected_code + assert expected_message in failed["response"]["error"]["message"] + + @pytest.mark.asyncio async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed(): """Starlette never calls the produced response, so its own cleanup never runs From 9f09276e73ae22aef0dd724ea7f77bc5ffd8c554 Mon Sep 17 00:00:00 2001 From: HootRock Date: Wed, 28 Oct 2026 20:35:05 +0800 Subject: [PATCH 05/14] test(proxy): use controlled clocks for Responses keepalive failures --- .../response_api_endpoints/test_endpoints.py | 112 ++++++++++-------- .../proxy/test_common_request_processing.py | 52 ++++---- 2 files changed, 91 insertions(+), 73 deletions(-) diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 5945352c61e..370b9586464 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -24,6 +24,7 @@ from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.router import RouterErrors from tests._master_key import MASTER_KEY +from tests._support.clock_loops import AutojumpClockLoop @pytest.mark.asyncio @@ -177,65 +178,72 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] -@pytest.mark.asyncio @pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) -async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str): - import litellm.proxy.proxy_server as ps - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None: + async def request_and_check() -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - model: Final = "gpt-6-astra" - message: Final = "Upstream is rate limiting this key" - upstream_url: Final = "https://streaming.example/v1" - is_chat: Final = path == "/v1/chat/completions" + model: Final = "gpt-6-astra" + message: Final = "Upstream is rate limiting this key" + upstream_url: Final = "https://streaming.example/v1" + is_chat: Final = path == "/v1/chat/completions" - async def slow_rate_limit(_: httpx.Request) -> httpx.Response: - await asyncio.sleep(0.3) - return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) + async def slow_rate_limit(_: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.3) + return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) - router: Final = litellm.Router( - model_list=[ - { - "model_name": model, - "litellm_params": { - "model": "openai/" + model, - "api_base": upstream_url, - "api_key": "fixture-key", - "max_retries": 0, - }, - } - ], - num_retries=0, - ) - monkeypatch.setattr(ps, "llm_router", router) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) - monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) - with respx.mock as transport: - transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( - side_effect=slow_rate_limit + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "openai/" + model, + "api_base": upstream_url, + "api_key": "fixture-key", + "max_retries": 0, + }, + } + ], + num_retries=0, ) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: - result: Final = await client.post( - path, - json={ - "model": model, - "stream": True, - **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), - }, + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( + side_effect=slow_rate_limit ) - frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + path, + json={ + "model": model, + "stream": True, + **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), + }, + ) + frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) - assert result.status_code == 200, result.text - assert frames[0] == ": ping", result.text - if is_chat: - assert frames[-1] == "data: [DONE]", result.text - assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] - else: - assert "data: [DONE]" not in result.text - assert frames[-1].startswith("event: response.failed\ndata: "), result.text - failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] - assert failed_error["code"] == "rate_limit_exceeded" - assert message in failed_error["message"] + assert result.status_code == 200, result.text + assert frames[0] == ": ping", result.text + if is_chat: + assert frames[-1] == "data: [DONE]", result.text + assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] + else: + assert "data: [DONE]" not in result.text + assert frames[-1].startswith("event: response.failed\ndata: "), result.text + failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] + assert failed_error["code"] == "rate_limit_exceeded" + assert message in failed_error["message"] + + loop: Final = AutojumpClockLoop() + try: + loop.run_until_complete(request_and_check()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() _COMPACTION_ITEM: Final = { diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 310b526365f..a888fcc431d 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -3,7 +3,7 @@ import contextvars import copy import datetime import json -from collections.abc import AsyncIterable, Mapping +from collections.abc import AsyncIterable, Awaitable, Mapping from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence from urllib.parse import unquote_plus @@ -81,6 +81,7 @@ from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment +from tests._support.clock_loops import AutojumpClockLoop def test_attach_guardrail_information_copies_recorded_entries_onto_model_response(): @@ -10391,17 +10392,18 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" -async def _slow_rate_limit_failure(): +async def _slow_rate_limit_failure() -> Response: await asyncio.sleep(0.2) raise HTTPException(status_code=429, detail="rate limited") -async def _slow_guardrail_block_body(): +async def _slow_guardrail_block_body() -> Response: await asyncio.sleep(0.2) - return JSONResponse(status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}) + return JSONResponse( + status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}} + ) -@pytest.mark.asyncio @pytest.mark.parametrize( "produce,expected_code,expected_message", [ @@ -10409,23 +10411,31 @@ async def _slow_guardrail_block_body(): (_slow_guardrail_block_body, "invalid_request_error", "blocked"), ], ) -async def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( - produce, expected_code, expected_message -): - response: Final = await open_sse_before_first_byte( - produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING - ) - collected: Final = await _drain(response) +def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( + produce: Callable[[], Awaitable[Response]], expected_code: str, expected_message: str +) -> None: + async def collect_and_check() -> None: + response: Final = await open_sse_before_first_byte( + produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + ) + collected: Final = await _drain(response) - assert collected[0] == TTFT_PING - assert b"data: [DONE]\n\n" not in collected - event_line, data_line = collected[-1].decode().strip().split("\n") - failed: Final = json.loads(data_line.removeprefix("data: ")) - assert event_line == "event: response.failed" - assert failed["type"] == "response.failed" - assert failed["response"]["status"] == "failed" - assert failed["response"]["error"]["code"] == expected_code - assert expected_message in failed["response"]["error"]["message"] + assert collected[0] == TTFT_PING + assert b"data: [DONE]\n\n" not in collected + event_line, data_line = collected[-1].decode().strip().split("\n") + failed: Final = json.loads(data_line.removeprefix("data: ")) + assert event_line == "event: response.failed" + assert failed["type"] == "response.failed" + assert failed["response"]["status"] == "failed" + assert failed["response"]["error"]["code"] == expected_code + assert expected_message in failed["response"]["error"]["message"] + + loop: Final = AutojumpClockLoop() + try: + loop.run_until_complete(collect_and_check()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() @pytest.mark.asyncio From e954b0f49a16b2a13da2f8b294aa4917aedc300f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:57:59 -0700 Subject: [PATCH 06/14] fix(proxy): end late Responses failures with an error event the SDK raises on Errors that arrive after keepalive pings, and guardrail error strings yielded mid-stream, now end /v1/responses with a named `error` event carrying the same error object chat sends, so the OpenAI SDK still raises APIError with a clean message and provider_specific_fields. Background polling passes the same framing and records an `error` event as a failed response. The first data line of a frame is found by splitting on newlines only, so a U+2028 in an error message no longer hides a first-event error. --- litellm/proxy/common_request_processing.py | 17 ++--- .../common_utils/responses_stream_errors.py | 41 +++++++---- litellm/proxy/proxy_server.py | 2 +- .../response_polling/background_streaming.py | 8 ++- .../proxy_server/test_streaming_helpers.py | 31 +++++++++ .../response_api_endpoints/test_endpoints.py | 29 +++++--- .../test_response_polling_handler.py | 34 +++++++++- .../proxy/test_common_request_processing.py | 68 +++++++++++++------ 8 files changed, 180 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index abc99a3be23..6cdab2daa82 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -115,7 +115,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, openai_error_type, ) -from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, sse_error_object from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, SSE_STREAM_START_TAIL, @@ -912,7 +912,7 @@ async def _enforce_guardrail_added_tag_budgets( def _sse_data_payload(event_line: str | bytes) -> str | None: text: Final = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line return next( - (line.removeprefix("data: ").strip() for line in text.splitlines() if line.startswith("data: ")), + (line.removeprefix("data: ").strip() for line in text.split("\n") if line.startswith("data: ")), None, ) @@ -1196,13 +1196,14 @@ def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: - frame: Final = ResponsesStreamErrorState().format_failure(exc) - return () if frame is None else (frame,) + _, error_obj = sse_error_payload(exc) + return (ResponsesStreamErrorState().format_error(error_obj),) def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: - frame: Final = ResponsesStreamErrorState().format_error_body(bytes(produced.body), produced.status_code) - return () if frame is None else (frame.encode(),) + body: Final = bytes(produced.body) + error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode(), "code": str(produced.status_code)} + return (ResponsesStreamErrorState().format_error(error_obj).encode(),) OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames) @@ -3423,8 +3424,8 @@ class ProxyBaseLLMRequestProcessing: Streaming ``/v1/responses`` short-circuits out of ``base_process_llm_request`` before the non-streaming ownership - tail runs. The OpenAI SDK closes the connection at ``data: [DONE]`` - and starlette cancels the body task on disconnect, so a write that + tail runs. The OpenAI SDK can close the connection right after the terminal + event and starlette cancels the body task on disconnect, so a write that waits for the generator to finish never lands. The iterator sets ``completed_response`` before it hands over its terminal chunk, so the ``LiteLLM_ManagedObjectTable`` row is written the moment it diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 312bf472594..132155d9492 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,10 +1,11 @@ +import json import time from collections.abc import Mapping from http import HTTPStatus from types import MappingProxyType from typing import Final -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid @@ -57,10 +58,17 @@ class _FailureDetails(LiteLLMBaseModel): return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields}) -class _ErrorBody(LiteLLMBaseModel): +class _ErrorPayload(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - error: _FailureDetails = _FailureDetails() + error: Mapping[str, object] + + +def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: + try: + return _ErrorPayload.model_validate_json(data).error + except ValidationError: + return None def _original_failure(exception: Exception) -> Exception: @@ -142,17 +150,26 @@ class ResponsesStreamErrorState: prefix: Final = f"event: {event.type}\n" return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame + def format_error(self, error: Mapping[str, object]) -> str: + payload: Final = json.dumps( + { + "type": ResponsesAPIStreamEvents.ERROR.value, + "sequence_number": self.sequence_number + 1, + "error": error, + }, + default=str, + ) + return f"event: error\ndata: {payload}\n\n" + + def format_error_message(self, frame: str) -> str: + error: Final = sse_error_object(frame.removeprefix("data: ")) + return frame if error is None else self.format_error(error) + def format_failure(self, exception: Exception) -> str | None: - original: Final = _original_failure(exception) - return self._format_failed(_failure_details(original), str(original)) - - def format_error_body(self, body: bytes, status_code: int) -> str | None: - error: Final = _ErrorBody.model_validate_json(body or b"{}").error - return self._format_failed(error.model_copy(update={"status_code": status_code}), body.decode()) - - def _format_failed(self, details: _FailureDetails, fallback_message: str) -> str | None: if self.terminal_emitted: return None + original: Final = _original_failure(exception) + details: Final = _failure_details(original) response: Final = ResponsesAPIResponse.model_validate( MappingProxyType( { @@ -165,7 +182,7 @@ class ResponsesStreamErrorState: "error": MappingProxyType( { "code": _response_error_code(details), - "message": redact_internal_details_from_client_message(details.message or fallback_message), + "message": redact_internal_details_from_client_message(details.message or str(original)), } ), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 15832d81b7f..f6cfc88300c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10619,7 +10619,7 @@ async def async_data_generator( yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n") if error_message is not None: - yield error_message + yield error_message if error_state is None else error_state.format_error_message(error_message) # Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not. if ends_with_openai_done: yield _OPENAI_STREAM_DONE_FRAME diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 5f6a7432afb..975d700688e 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -19,7 +19,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth -from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus @@ -68,6 +68,7 @@ class _StreamEvent(TypedDict, total=False): delta: ReadOnly[str] part: ReadOnly[object] response: ReadOnly[_TerminalResponse] + error: ReadOnly[_JsonDict] class _StreamEventParser: @@ -164,6 +165,7 @@ async def background_streaming_task( user_api_base=user_api_base, version=version, skip_pre_call_logic=True, + late_framing=RESPONSES_LATE_SSE_FRAMING, ) # Process streaming response following OpenAI events format @@ -325,6 +327,10 @@ async def background_streaming_task( status="in_progress", ) + elif event_type == "error": + terminal_status = "failed" + terminal_error = event.get("error") + elif event_type in ( "response.completed", "response.failed", diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 835411608ed..b01487af5d1 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -146,6 +146,37 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent assert payload["type"] == event_type assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" +@pytest.mark.asyncio +async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event() -> None: + response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[]) + created: Final = ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ) + guardrail_error: Final = {"message": "response withheld", "type": "guardrail_scan_error", "code": 500} + + async def upstream() -> AsyncIterator[object]: + yield created + yield f"data: {json.dumps({'error': guardrail_error})}\n\n" + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True + ) + ] + ) + event_line, data_line = frames[-1].strip().split("\n") + + assert "data: [DONE]" not in "".join(frames) + assert event_line == "event: error" + assert json.loads(data_line.removeprefix("data: ")) == { + "type": "error", + "sequence_number": 1, + "error": guardrail_error, + } + + @pytest.mark.parametrize("binary", [False, True]) @pytest.mark.parametrize( ("event_type", "event_line"), diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 5945352c61e..d31ddaf0adf 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -17,6 +17,7 @@ import respx from fastapi import HTTPException from fastapi.testclient import TestClient from httpx import Response +from starlette.types import Message, Receive, Scope, Send import litellm from litellm.proxy._types import ProxyException @@ -188,10 +189,20 @@ async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeyp upstream_url: Final = "https://streaming.example/v1" is_chat: Final = path == "/v1/chat/completions" - async def slow_rate_limit(_: httpx.Request) -> httpx.Response: - await asyncio.sleep(0.3) + first_ping_sent: Final = asyncio.Event() + + async def rate_limit_after_first_ping(_: httpx.Request) -> httpx.Response: + await first_ping_sent.wait() return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) + async def app_marking_first_ping(scope: Scope, receive: Receive, send: Send) -> None: + async def send_and_mark(asgi_message: Message) -> None: + await send(asgi_message) + if asgi_message["type"] == "http.response.body" and asgi_message.get("body", b"").startswith(b": ping"): + first_ping_sent.set() + + await app(scope, receive, send_and_mark) + router: Final = litellm.Router( model_list=[ { @@ -212,9 +223,11 @@ async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeyp monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) with respx.mock as transport: transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( - side_effect=slow_rate_limit + side_effect=rate_limit_after_first_ping ) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app_marking_first_ping), base_url="http://testserver" + ) as client: result: Final = await client.post( path, json={ @@ -232,10 +245,10 @@ async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeyp assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] else: assert "data: [DONE]" not in result.text - assert frames[-1].startswith("event: response.failed\ndata: "), result.text - failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] - assert failed_error["code"] == "rate_limit_exceeded" - assert message in failed_error["message"] + assert frames[-1].startswith("event: error\ndata: "), result.text + error: Final = json.loads(frames[-1].split("data: ", 1)[1])["error"] + assert error["code"] == "429" + assert message in error["message"] _COMPACTION_ITEM: Final = { diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 63e38ce533a..8affc8a6e19 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -21,7 +21,7 @@ from typing import Any, Dict, Optional from unittest.mock import AsyncMock, Mock, patch import pytest -from fastapi import Request +from fastapi import HTTPException, Request from pydantic import ValidationError from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler @@ -1522,6 +1522,38 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["status"] == "failed" assert final_call.kwargs["error"] == error_payload + @pytest.mark.asyncio + async def test_a_late_failure_after_keepalive_pings_sets_failed_status_and_error(self): + from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + late_frames = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited")) + + async def _body_iterator(): + yield b": ping\n\n" + for frame in late_frames: + yield frame.encode() + + mock_response = Mock() + mock_response.body_iterator = _body_iterator() + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_late_failure", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"]["message"] == "rate limited" + assert final_call.kwargs["error"]["code"] == "429" + @pytest.mark.asyncio async def test_response_incomplete_sets_incomplete_status_and_details(self): """Test that a response.incomplete stream event results in incomplete status""" diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 310b526365f..ecc3ff860d8 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -5,7 +5,7 @@ import datetime import json from collections.abc import AsyncIterable, Mapping from types import MappingProxyType, SimpleNamespace -from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence +from typing import AsyncGenerator, Awaitable, Callable, Final, Iterator, Literal, Optional, Sequence from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch @@ -2174,6 +2174,19 @@ class TestCommonRequestProcessingHelpers: assert response.status_code == 400 assert json.loads(response.body) == {"error": error} + async def test_a_first_event_error_with_a_unicode_line_separator_in_its_message_answers_as_json(self): + error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked\u2028by policy", "param": None} + + async def responses_stream(): + event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False) + yield f"event: error\ndata: {event}\n\n" + + response: Final = await create_response(responses_stream(), "text/event-stream", {}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(response.body) == {"error": error} + async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): yield 'data: {"content": "first part"}\n\n' @@ -10391,41 +10404,58 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" -async def _slow_rate_limit_failure(): - await asyncio.sleep(0.2) +async def _rate_limit_failure_once_released(released: asyncio.Event) -> JSONResponse: + await released.wait() raise HTTPException(status_code=429, detail="rate limited") -async def _slow_guardrail_block_body(): - await asyncio.sleep(0.2) +async def _guardrail_block_body_once_released(released: asyncio.Event) -> JSONResponse: + await released.wait() return JSONResponse(status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}) +_POLICY_BLOCK_DETAIL: Final = MappingProxyType( + {"error": {"message": "blocked by policy", "type": "guardrail_error"}, "guardrail": "content-filter"} +) + + +async def _dict_detail_failure_once_released(released: asyncio.Event) -> JSONResponse: + await released.wait() + raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL)) + + @pytest.mark.asyncio @pytest.mark.parametrize( - "produce,expected_code,expected_message", + "produce,expected_error", [ - (_slow_rate_limit_failure, "rate_limit_exceeded", "rate limited"), - (_slow_guardrail_block_body, "invalid_request_error", "blocked"), + (_rate_limit_failure_once_released, {"message": "rate limited", "code": "429"}), + (_guardrail_block_body_once_released, {"message": "blocked", "type": "guardrail_error", "code": "400"}), + ( + _dict_detail_failure_once_released, + {"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)}, + ), ], ) -async def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( - produce, expected_code, expected_message -): +async def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done( + produce: Callable[[asyncio.Event], Awaitable[JSONResponse]], expected_error: Mapping[str, object] +) -> None: + released: Final = asyncio.Event() response: Final = await open_sse_before_first_byte( - produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + produce(released), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING ) - collected: Final = await _drain(response) + assert isinstance(response, StreamingResponse) + frames: Final = aiter(response.body_iterator) + first_frame: Final = await anext(frames) + released.set() + collected: Final = [first_frame, *[frame async for frame in frames]] assert collected[0] == TTFT_PING assert b"data: [DONE]\n\n" not in collected event_line, data_line = collected[-1].decode().strip().split("\n") - failed: Final = json.loads(data_line.removeprefix("data: ")) - assert event_line == "event: response.failed" - assert failed["type"] == "response.failed" - assert failed["response"]["status"] == "failed" - assert failed["response"]["error"]["code"] == expected_code - assert expected_message in failed["response"]["error"]["message"] + event: Final = json.loads(data_line.removeprefix("data: ")) + assert event_line == "event: error" + assert event["type"] == "error" + assert {key: event["error"].get(key) for key in expected_error} == expected_error @pytest.mark.asyncio From b2f3614faf87b357d2a2d896157d6412a6a430d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:15:41 -0700 Subject: [PATCH 07/14] fix(proxy): match OpenAI's Responses error event and stream late passthrough blocks as block responses --- litellm/proxy/common_request_processing.py | 5 ++- .../common_utils/responses_stream_errors.py | 10 +++++ .../response_polling/background_streaming.py | 2 +- .../proxy_server/test_streaming_helpers.py | 3 ++ .../test_response_polling_handler.py | 15 +++---- .../proxy/test_common_request_processing.py | 43 +++++++++++++++++-- 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6cdab2daa82..10a5c6d61ee 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -51,7 +51,7 @@ from litellm.constants import ( STREAM_SSE_KEEPALIVE_PING_BYTES, UNSAFE_PROXY_RESPONSE_HEADERS, ) -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.bug_report import ( allowlisted, @@ -89,6 +89,7 @@ from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields +from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, @@ -1196,6 +1197,8 @@ def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: + if isinstance(exc, ModifyResponseException): + return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc)) _, error_obj = sse_error_payload(exc) return (ResponsesStreamErrorState().format_error(error_obj),) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 132155d9492..05ec7603a3d 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -109,6 +109,12 @@ def _status_error_code(status_code: int | None) -> str: return _CLIENT_ERROR_CODES.get(status_code, "invalid_request_error") +def _status_of(code: str | int | None) -> int | None: + if isinstance(code, int): + return code + return int(code) if code is not None and code.isdecimal() else None + + def _response_error_code(details: _FailureDetails) -> str: for value in (details.code, details.type): if value == "insufficient_quota": @@ -151,9 +157,13 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_error(self, error: Mapping[str, object]) -> str: + details: Final = _FailureDetails.model_validate(error) payload: Final = json.dumps( { "type": ResponsesAPIStreamEvents.ERROR.value, + "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), + "message": details.message, + "param": error.get("param"), "sequence_number": self.sequence_number + 1, "error": error, }, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 975d700688e..7411e204d90 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -329,7 +329,7 @@ async def background_streaming_task( elif event_type == "error": terminal_status = "failed" - terminal_error = event.get("error") + terminal_error = {"code": event.get("code"), "message": event.get("message")} elif event_type in ( "response.completed", diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index b01487af5d1..fb8ff519fbc 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -172,6 +172,9 @@ async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_ assert event_line == "event: error" assert json.loads(data_line.removeprefix("data: ")) == { "type": "error", + "code": "server_error", + "message": "response withheld", + "param": None, "sequence_number": 1, "error": guardrail_error, } diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 8affc8a6e19..f398d5acb1c 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -17,7 +17,7 @@ following the OpenAI Response API format. import asyncio import json from datetime import datetime, timezone -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, Mock, patch import pytest @@ -1529,17 +1529,17 @@ class TestBackgroundStreamingTerminalEvents: background_streaming_task, ) - late_frames = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited")) + late_frames: Final = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited")) async def _body_iterator(): yield b": ping\n\n" for frame in late_frames: yield frame.encode() - mock_response = Mock() + mock_response: Final = Mock() mock_response.body_iterator = _body_iterator() - handler = AsyncMock(spec=ResponsePollingHandler) - kwargs = _make_background_streaming_kwargs("poll_late_failure", handler) + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_late_failure", handler) with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" @@ -1549,10 +1549,9 @@ class TestBackgroundStreamingTerminalEvents: ) await background_streaming_task(**kwargs) - final_call = handler.update_state.call_args_list[-1] + final_call: Final = handler.update_state.call_args_list[-1] assert final_call.kwargs["status"] == "failed" - assert final_call.kwargs["error"]["message"] == "rate limited" - assert final_call.kwargs["error"]["code"] == "429" + assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"} @pytest.mark.asyncio async def test_response_incomplete_sets_incomplete_status_and_details(self): diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 944432f91b1..63324c11930 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -34,6 +34,7 @@ from litellm.constants import ( STREAM_SSE_KEEPALIVE_PING_BYTES, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -10428,18 +10429,23 @@ async def _slow_dict_detail_failure() -> Response: @pytest.mark.parametrize( - "produce,expected_error", + "produce,expected_code,expected_error", [ - (_slow_rate_limit_failure, {"message": "rate limited", "code": "429"}), - (_slow_guardrail_block_body, {"message": "blocked", "type": "guardrail_error", "code": "400"}), + (_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}), + ( + _slow_guardrail_block_body, + "invalid_request_error", + {"message": "blocked", "type": "guardrail_error", "code": "400"}, + ), ( _slow_dict_detail_failure, + "invalid_request_error", {"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)}, ), ], ) def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done( - produce: Callable[[], Awaitable[Response]], expected_error: Mapping[str, object] + produce: Callable[[], Awaitable[Response]], expected_code: str, expected_error: Mapping[str, object] ) -> None: async def collect_and_check() -> None: response: Final = await open_sse_before_first_byte( @@ -10453,6 +10459,7 @@ def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no event: Final = json.loads(data_line.removeprefix("data: ")) assert event_line == "event: error" assert event["type"] == "error" + assert (event["code"], event["message"]) == (expected_code, expected_error["message"]) assert {key: event["error"].get(key) for key in expected_error} == expected_error loop: Final = AutojumpClockLoop() @@ -10463,6 +10470,34 @@ def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no loop.close() +async def _slow_passthrough_block() -> Response: + await asyncio.sleep(0.2) + raise ModifyResponseException(message="blocked by policy", model="gpt-4o", request_data={}) + + +def test_ttft_keepalive_ends_a_late_responses_passthrough_block_with_the_block_response() -> None: + async def collect_and_check() -> None: + response: Final = await open_sse_before_first_byte( + _slow_passthrough_block(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + ) + collected: Final = await _drain(response) + frames: Final = tuple(chunk.decode() for chunk in collected if chunk != TTFT_PING) + names: Final = tuple(frame.split("\n", 1)[0] for frame in frames) + completed: Final = json.loads(frames[-1].split("\n", 1)[1].removeprefix("data: ")) + + assert collected[0] == TTFT_PING + assert (names[0], names[-1]) == ("event: response.created", "event: response.completed") + assert "event: error" not in names + assert completed["response"]["output"][0]["content"][0]["text"] == "blocked by policy" + + loop: Final = AutojumpClockLoop() + try: + loop.run_until_complete(collect_and_check()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + @pytest.mark.asyncio async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed(): """Starlette never calls the produced response, so its own cleanup never runs From f343c2421a3554a43720420b3b7d104a7103986d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:01:53 -0700 Subject: [PATCH 08/14] fix(proxy): keep the reason when a background Responses request fails A background poll of a request a post-call guardrail blocked saved a null code and message, because the guardrail's error event nests them. A request whose first chunk was an error polled as completed with no error. Both now poll failed with the same code and message a late failure gets. --- .../common_utils/responses_stream_errors.py | 12 ++- .../response_polling/background_streaming.py | 21 +++-- .../proxy/response_polling/polling_handler.py | 2 +- .../test_response_polling_handler.py | 90 +++++++++++++++++++ 4 files changed, 116 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 05ec7603a3d..c730a94dc62 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -126,6 +126,14 @@ def _response_error_code(details: _FailureDetails) -> str: return _status_error_code(details.status_code) +def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]: + details: Final = _FailureDetails.model_validate(error) + return { + "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), + "message": details.message, + } + + class ResponsesStreamErrorState: def __init__(self) -> None: self.response_id: str | None = None @@ -157,12 +165,10 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_error(self, error: Mapping[str, object]) -> str: - details: Final = _FailureDetails.model_validate(error) payload: Final = json.dumps( { "type": ResponsesAPIStreamEvents.ERROR.value, - "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), - "message": details.message, + **responses_error_fields(error), "param": error.get("param"), "sequence_number": self.sequence_number + 1, "error": error, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 7411e204d90..ecf9bea4ab7 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming import asyncio import json from collections.abc import AsyncIterable, Callable, Mapping, Sequence +from http import HTTPStatus from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable from fastapi import Request, Response @@ -20,6 +21,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.responses_stream_errors import responses_error_fields, sse_error_object from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus @@ -85,6 +87,14 @@ def _sse_frame_data(frame: str) -> str | None: return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None) +def _first_chunk_error(response: object) -> Mapping[str, object] | None: + if isinstance(response, _StreamsBody) or not isinstance(response, Response): + return None + if response.status_code < HTTPStatus.BAD_REQUEST: + return None + return responses_error_fields(sse_error_object(bytes(response.body)) or {"code": response.status_code}) + + async def _never_receive() -> Message: await asyncio.Event().wait() raise AssertionError("unreachable") @@ -329,7 +339,7 @@ async def background_streaming_task( elif event_type == "error": terminal_status = "failed" - terminal_error = {"code": event.get("code"), "message": event.get("message")} + terminal_error = responses_error_fields(event.get("error") or event) elif event_type in ( "response.completed", @@ -387,14 +397,15 @@ async def background_streaming_task( # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) - # Use the terminal status from the stream, default to "completed" - final_status: Final = terminal_status or "completed" + early_error: Final = _first_chunk_error(response) + final_error: Final = terminal_error if early_error is None else early_error + final_status: Final = "failed" if early_error is not None else terminal_status or "completed" await polling_handler.update_state( polling_id=polling_id, status=final_status, usage=usage_data, - error=terminal_error, + error=final_error, reasoning=reasoning_data, tool_choice=tool_choice_data, tools=tools_data, @@ -416,7 +427,7 @@ async def background_streaming_task( "Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s", polling_id, final_status, - terminal_error, + final_error, incomplete_details_data, len(output_items), ) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index e50587b1a0e..158b1f0f63b 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -98,7 +98,7 @@ class ResponsePollingHandler: polling_id: str, status: ResponsesAPIStatus | None = None, usage: dict | None = None, - error: dict | None = None, + error: Mapping[str, object] | None = None, incomplete_details: dict | None = None, reasoning: dict | None = None, tool_choice: object | None = None, diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index f398d5acb1c..6cb9376c0e5 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1553,6 +1553,96 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["status"] == "failed" assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"} + @pytest.mark.asyncio + async def test_a_nested_only_error_event_keeps_its_reason(self): + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + nested_error: Final = {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"} + error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error} + + async def _body_iterator(): + yield b'data: {"type": "response.in_progress"}\n\n' + yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode() + + mock_response: Final = Mock() + mock_response.body_iterator = _body_iterator() + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_nested_error", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"] == {"code": "invalid_request_error", "message": "output blocked by policy"} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("status_code", "body", "expected_error"), + [ + ( + 404, + json.dumps( + {"error": {"message": "model not found", "type": "invalid_request_error", "param": None, "code": "404"}} + ), + {"code": "not_found_error", "message": "model not found"}, + ), + (429, "Too Many Requests", {"code": "rate_limit_exceeded", "message": None}), + ], + ) + async def test_a_first_chunk_error_polls_failed_like_a_late_one(self, status_code, body, expected_error): + from fastapi import Response + + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + first_chunk_error: Final = Response(status_code=status_code, content=body) + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_first_chunk_error", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=first_chunk_error + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"] == expected_error + + @pytest.mark.asyncio + async def test_a_non_streamed_success_response_still_polls_completed(self): + from fastapi.responses import JSONResponse + + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_non_streamed_success", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=JSONResponse(status_code=200, content={"id": "resp_123", "status": "completed"}) + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + assert final_call.kwargs["error"] is None + @pytest.mark.asyncio async def test_response_incomplete_sets_incomplete_status_and_details(self): """Test that a response.incomplete stream event results in incomplete status""" From e6290016c1e7b17e91404bf1167767d3c3dbc0d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:00:03 -0700 Subject: [PATCH 09/14] test(proxy): type the first-chunk poll test parameters --- .../proxy/response_polling/test_response_polling_handler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 6cb9376c0e5..830e3f2585c 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -16,6 +16,7 @@ following the OpenAI Response API format. import asyncio import json +from collections.abc import Mapping from datetime import datetime, timezone from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, Mock, patch @@ -1597,7 +1598,9 @@ class TestBackgroundStreamingTerminalEvents: (429, "Too Many Requests", {"code": "rate_limit_exceeded", "message": None}), ], ) - async def test_a_first_chunk_error_polls_failed_like_a_late_one(self, status_code, body, expected_error): + async def test_a_first_chunk_error_polls_failed_like_a_late_one( + self, status_code: int, body: str, expected_error: Mapping[str, str | None] + ): from fastapi import Response from litellm.proxy.response_polling.background_streaming import ( From 4bfca9cfd484e41aa9950e07bb8c68434136f712 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:33:43 -0700 Subject: [PATCH 10/14] fix(proxy): keep first-chunk error detection to lone error frames and handle string poll errors --- litellm/proxy/common_request_processing.py | 43 ++++++++--- .../common_utils/responses_stream_errors.py | 19 ++++- .../response_polling/background_streaming.py | 16 +++- .../response_api_endpoints/test_endpoints.py | 76 ++++++++++++++++++- .../test_response_polling_handler.py | 26 +++++-- .../proxy/test_common_request_processing.py | 9 ++- 6 files changed, 166 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 10a5c6d61ee..bf321d5cd20 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -910,17 +910,31 @@ async def _enforce_guardrail_added_tag_budgets( ) from e +SseDataPayload: TypeAlias = Callable[[str | bytes], str | None] + + +def _sse_text(event_line: str | bytes) -> str: + return event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line + + def _sse_data_payload(event_line: str | bytes) -> str | None: - text: Final = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line - return next( - (line.removeprefix("data: ").strip() for line in text.split("\n") if line.startswith("data: ")), - None, - ) + text: Final = _sse_text(event_line) + return text.removeprefix("data: ").strip() if text.startswith("data: ") else None -async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: +def _responses_event_data_payload(event_line: str | bytes) -> str | None: + match _sse_text(event_line).strip().split("\n"): + case ["event: error", data_line] if data_line.startswith("data: "): + return data_line.removeprefix("data: ").strip() + case _: + return _sse_data_payload(event_line) + + +async def _parse_event_data_for_error( + event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload +) -> int | None: """Parses an event line and returns an error code if present, else None.""" - json_str: Final = _sse_data_payload(event_line) + json_str: Final = data_payload(event_line) if json_str is not None: if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message return None @@ -952,12 +966,13 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: return None -def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: +def _extract_error_from_sse_chunk(event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload) -> dict: """ Extract error dictionary from SSE format chunk. Args: event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n" + data_payload: Reads the JSON payload out of the event line Returns: Error dictionary in OpenAI API format @@ -970,7 +985,7 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: "code": "500", } - json_str: Final = _sse_data_payload(event_line) + json_str: Final = data_payload(event_line) if json_str is not None: if not json_str or json_str == "[DONE]": return default_error @@ -1243,6 +1258,7 @@ async def create_response( request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, + first_chunk_payload: SseDataPayload = _sse_data_payload, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -1278,7 +1294,7 @@ async def create_response( if first_chunk_value is not None: try: - error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value) + error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value, first_chunk_payload) if error_code_from_chunk is not None: # First chunk is an error, stream hasn't really started yet # Should return standard JSON error response instead of SSE format @@ -1290,7 +1306,9 @@ async def create_response( # Parse error content error_dict: Final = with_call_id( - JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)), + JSON_OBJECT.validate_python( + _extract_error_from_sse_chunk(first_chunk_value, first_chunk_payload) + ), error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), ) @@ -3217,6 +3235,9 @@ class ProxyBaseLLMRequestProcessing: request=request, refresh_headers=refresh_stream_headers, general_settings=general_settings, + first_chunk_payload=( + _responses_event_data_payload if route_type == "aresponses" else _sse_data_payload + ), ) ### CALL HOOKS ### - modify outgoing data diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index c730a94dc62..560365a4cdb 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -71,6 +71,13 @@ def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: return None +def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None: + try: + return _ErrorPayload.model_validate(event).error or None + except ValidationError: + return None + + def _original_failure(exception: Exception) -> Exception: current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(current, MidStreamFallbackError) and current.original_exception is not None: @@ -126,11 +133,19 @@ def _response_error_code(details: _FailureDetails) -> str: return _status_error_code(details.status_code) +_STATUS_PHRASES: Final = MappingProxyType({int(status): status.phrase for status in HTTPStatus}) + + +def _status_phrase(status_code: int | None) -> str: + return _STATUS_PHRASES.get(status_code, "Unknown error") if status_code is not None else "Unknown error" + + def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]: details: Final = _FailureDetails.model_validate(error) + status_code: Final = _status_of(details.code) return { - "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), - "message": details.message, + "code": _response_error_code(details.model_copy(update={"status_code": status_code})), + "message": details.message or _status_phrase(status_code), } diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index ecf9bea4ab7..0ddedd52fb6 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -21,7 +21,11 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing -from litellm.proxy.common_utils.responses_stream_errors import responses_error_fields, sse_error_object +from litellm.proxy.common_utils.responses_stream_errors import ( + event_error_object, + responses_error_fields, + sse_error_object, +) from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus @@ -87,6 +91,14 @@ def _sse_frame_data(frame: str) -> str | None: return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None) +def _event_error(event: Mapping[str, object]) -> Mapping[str, object]: + match event.get("error"): + case str() as message if message: + return {**event, "message": message} + case _: + return event_error_object(event) or event + + def _first_chunk_error(response: object) -> Mapping[str, object] | None: if isinstance(response, _StreamsBody) or not isinstance(response, Response): return None @@ -339,7 +351,7 @@ async def background_streaming_task( elif event_type == "error": terminal_status = "failed" - terminal_error = responses_error_fields(event.get("error") or event) + terminal_error = responses_error_fields(_event_error(event)) elif event_type in ( "response.completed", diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 4a04d33cc42..a510799cf37 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,7 +7,7 @@ import contextlib import dataclasses import json import unittest -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -19,6 +19,7 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse @@ -178,6 +179,79 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] +class _FirstFrameHook(CustomLogger): + def __init__(self, frame: str | bytes) -> None: + super().__init__() + self.frame: Final = frame + + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict: object, response: object, request_data: object + ) -> AsyncIterator[str | bytes]: + yield self.frame + + +_DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n' +_NAMED_ERROR_FRAME: Final = ( + b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "denied", "code": "529"}}\n\n' +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,first_frame,expected_status", + [ + ("/v1/responses", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _NAMED_ERROR_FRAME, 200), + ("/v1/responses", _NAMED_ERROR_FRAME, 529), + ("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200), + ("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ], +) +async def test_only_a_lone_first_error_frame_answers_with_its_status( + monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-6-astra" + upstream_url: Final = "https://streaming.example/v1" + is_chat: Final = path == "/v1/chat/completions" + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"}, + } + ], + num_retries=0, + ) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first_frame)]) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond( + 200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"} + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + path, + json={ + "model": model, + "stream": True, + **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), + }, + ) + + assert result.status_code == expected_status, result.text + if expected_status == 200: + assert result.headers["content-type"].startswith("text/event-stream"), result.text + else: + assert result.json()["error"]["message"] == "denied" + + @pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None: async def request_and_check() -> None: diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 830e3f2585c..d97cd73e6ed 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1555,12 +1555,23 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"} @pytest.mark.asyncio - async def test_a_nested_only_error_event_keeps_its_reason(self): + @pytest.mark.parametrize( + ("nested_error", "expected_error"), + [ + ( + {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"}, + {"code": "invalid_request_error", "message": "output blocked by policy"}, + ), + ("output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}), + ], + ) + async def test_a_nested_only_error_event_keeps_its_reason( + self, nested_error: Mapping[str, str] | str, expected_error: Mapping[str, str] + ): from litellm.proxy.response_polling.background_streaming import ( background_streaming_task, ) - nested_error: Final = {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"} error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error} async def _body_iterator(): @@ -1582,7 +1593,7 @@ class TestBackgroundStreamingTerminalEvents: final_call: Final = handler.update_state.call_args_list[-1] assert final_call.kwargs["status"] == "failed" - assert final_call.kwargs["error"] == {"code": "invalid_request_error", "message": "output blocked by policy"} + assert final_call.kwargs["error"] == expected_error @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1595,11 +1606,16 @@ class TestBackgroundStreamingTerminalEvents: ), {"code": "not_found_error", "message": "model not found"}, ), - (429, "Too Many Requests", {"code": "rate_limit_exceeded", "message": None}), + (429, "rate limited", {"code": "rate_limit_exceeded", "message": "Too Many Requests"}), + ( + 403, + json.dumps({"error": {"type": "guardrail_scan_error", "code": 403}}), + {"code": "permission_error", "message": "Forbidden"}, + ), ], ) async def test_a_first_chunk_error_polls_failed_like_a_late_one( - self, status_code: int, body: str, expected_error: Mapping[str, str | None] + self, status_code: int, body: str, expected_error: Mapping[str, str] ): from fastapi import Response diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 63324c11930..a2a7deb3931 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -49,6 +49,7 @@ from litellm.proxy.common_request_processing import ( _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, + _responses_event_data_payload, include_guardrail_response_requested, is_azure_model_router_request, RESPONSES_LATE_SSE_FRAMING, @@ -2170,7 +2171,9 @@ class TestCommonRequestProcessingHelpers: async def responses_stream(): yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" - response: Final = await create_response(responses_stream(), "text/event-stream", {}) + response: Final = await create_response( + responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + ) assert isinstance(response, JSONResponse) assert response.status_code == 400 @@ -2183,7 +2186,9 @@ class TestCommonRequestProcessingHelpers: event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False) yield f"event: error\ndata: {event}\n\n" - response: Final = await create_response(responses_stream(), "text/event-stream", {}) + response: Final = await create_response( + responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + ) assert isinstance(response, JSONResponse) assert response.status_code == 400 From 86360077c637cd915f4bee9e375ddc7f233218c8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:57:45 -0700 Subject: [PATCH 11/14] fix(proxy): keep the HTTP status on late Responses errors and end Responses streams on error events A late error body without a code now takes the response's status, an upstream error event counts as the stream's terminal event, and a Responses stream that fails before its first chunk ends with an error event instead of data: [DONE]. --- litellm/proxy/common_request_processing.py | 40 ++++++++++++++----- .../common_utils/responses_stream_errors.py | 2 +- .../proxy_server/test_streaming_helpers.py | 27 +++++++++++++ .../proxy/test_common_request_processing.py | 38 ++++++++++++++++-- 4 files changed, 93 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 350f4d6cb9c..27164ba0c83 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1196,6 +1196,10 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]: return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n" +def _responses_error_frames(error_obj: Mapping[str, object]) -> tuple[str, ...]: + return (ResponsesStreamErrorState().format_error(error_obj),) + + @dataclass(frozen=True, slots=True) class LateSseFraming: """How a response the keepalive already opened ends once its upstream fails late.""" @@ -1216,14 +1220,14 @@ def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: if isinstance(exc, ModifyResponseException): return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc)) - _, error_obj = sse_error_payload(exc) - return (ResponsesStreamErrorState().format_error(error_obj),) + return _responses_error_frames(sse_error_payload(exc)[1]) def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: body: Final = bytes(produced.body) - error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode(), "code": str(produced.status_code)} - return (ResponsesStreamErrorState().format_error(error_obj).encode(),) + error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode()} + with_status: Final = {**error_obj, "code": error_obj.get("code") or str(produced.status_code)} + return tuple(frame.encode() for frame in _responses_error_frames(with_status)) OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames) @@ -1232,6 +1236,20 @@ RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming( ) +@dataclass(frozen=True, slots=True) +class FirstChunkSseFraming: + """How a stream's first chunk is read for an error, and how the stream ends when reading it fails.""" + + data_payload: SseDataPayload + error_frames: Callable[[Mapping[str, object]], tuple[str, ...]] + + +OPENAI_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(data_payload=_sse_data_payload, error_frames=_sse_error_frames) +RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming( + data_payload=_responses_event_data_payload, error_frames=_responses_error_frames +) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -1260,7 +1278,7 @@ async def create_response( request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, - first_chunk_payload: SseDataPayload = _sse_data_payload, + framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -1296,7 +1314,9 @@ async def create_response( if first_chunk_value is not None: try: - error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value, first_chunk_payload) + error_code_from_chunk: Final = await _parse_event_data_for_error( + first_chunk_value, framing.data_payload + ) if error_code_from_chunk is not None: # First chunk is an error, stream hasn't really started yet # Should return standard JSON error response instead of SSE format @@ -1309,7 +1329,7 @@ async def create_response( # Parse error content error_dict: Final = with_call_id( JSON_OBJECT.validate_python( - _extract_error_from_sse_chunk(first_chunk_value, first_chunk_payload) + _extract_error_from_sse_chunk(first_chunk_value, framing.data_payload) ), error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), ) @@ -1352,7 +1372,7 @@ async def create_response( error_status, error_obj = sse_error_payload(e) async def error_gen_message() -> AsyncGenerator[str, None]: - for frame in _sse_error_frames(error_obj): + for frame in framing.error_frames(error_obj): yield frame return StreamingResponse( @@ -3286,8 +3306,8 @@ class ProxyBaseLLMRequestProcessing: request=request, refresh_headers=refresh_stream_headers, general_settings=general_settings, - first_chunk_payload=( - _responses_event_data_payload if route_type == "aresponses" else _sse_data_payload + framing=( + RESPONSES_FIRST_CHUNK_FRAMING if route_type == "aresponses" else OPENAI_FIRST_CHUNK_FRAMING ), ) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 560365a4cdb..4a806c32284 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -172,7 +172,7 @@ class ResponsesStreamErrorState: self.model = event.response.model or self.model if event.response.created_at is not None: self.created_at = event.response.created_at - if event.type in ("response.completed", "response.failed", "response.incomplete"): + if event.type in ("response.completed", "response.failed", "response.incomplete", "error"): self.terminal_emitted = True if not event.type or "\n" in event.type or "\r" in event.type: return frame diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index fb8ff519fbc..93eb8fa226b 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -146,6 +146,33 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent assert payload["type"] == event_type assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" + +@pytest.mark.asyncio +async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event() -> None: + response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[]) + created: Final = ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ) + error_event: Final = ErrorEvent.model_validate( + {"type": "error", "sequence_number": 1, "code": "server_error", "message": "upstream failed", "param": None} + ) + + async def upstream() -> AsyncIterator[BaseModel]: + yield created + yield error_event + raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model") + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True + ) + ] + ) + + assert tuple(frame.split("\n", 1)[0] for frame in frames) == ("event: response.created", "event: error") + @pytest.mark.asyncio async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event() -> None: response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[]) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 1b1c7ce1ccb..3f8ff1c9b14 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -50,7 +50,8 @@ from litellm.proxy.common_request_processing import ( _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, - _responses_event_data_payload, + OPENAI_FIRST_CHUNK_FRAMING, + RESPONSES_FIRST_CHUNK_FRAMING, include_guardrail_response_requested, is_azure_model_router_request, RESPONSES_LATE_SSE_FRAMING, @@ -2250,6 +2251,14 @@ class TestProxyBaseLLMRequestProcessing: ) +def _fail_stream_setup() -> str: + raise RuntimeError("stream setup failed") + + +async def _stream_failing_before_its_first_chunk() -> AsyncGenerator[str, None]: + yield _fail_stream_setup() + + @pytest.mark.asyncio class TestCommonRequestProcessingHelpers: async def consume_stream(self, streaming_response: StreamingResponse) -> list: @@ -2335,7 +2344,7 @@ class TestCommonRequestProcessingHelpers: yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" response: Final = await create_response( - responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING ) assert isinstance(response, JSONResponse) @@ -2350,13 +2359,30 @@ class TestCommonRequestProcessingHelpers: yield f"event: error\ndata: {event}\n\n" response: Final = await create_response( - responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING ) assert isinstance(response, JSONResponse) assert response.status_code == 400 assert json.loads(response.body) == {"error": error} + @pytest.mark.parametrize( + "framing,frame_starts", + [ + (OPENAI_FIRST_CHUNK_FRAMING, ('data: {"error": ', "data: [DONE]")), + (RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)), + ], + ) + async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(self, framing, frame_starts): + response: Final = await create_response( + _stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing + ) + frames: Final = await self.consume_stream(response) + + assert response.status_code == 500 + assert len(frames) == len(frame_starts) + assert all(frame.startswith(start) for frame, start in zip(frames, frame_starts, strict=True)) + async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): yield 'data: {"content": "first part"}\n\n' @@ -10591,6 +10617,11 @@ _POLICY_BLOCK_DETAIL: Final = MappingProxyType( ) +async def _slow_codeless_rate_limit_body() -> Response: + await asyncio.sleep(0.2) + return JSONResponse(status_code=429, content={"error": {"message": "slow down", "code": None}}) + + async def _slow_dict_detail_failure() -> Response: await asyncio.sleep(0.2) raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL)) @@ -10600,6 +10631,7 @@ async def _slow_dict_detail_failure() -> Response: "produce,expected_code,expected_error", [ (_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}), + (_slow_codeless_rate_limit_body, "rate_limit_exceeded", {"message": "slow down", "code": "429"}), ( _slow_guardrail_block_body, "invalid_request_error", From 5003ef1f31e68fbadc95974655f79d18a24118cf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:20:31 -0700 Subject: [PATCH 12/14] test(proxy): type the first-chunk failure test's parameters --- tests/unit/proxy/test_common_request_processing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 3f8ff1c9b14..7e0eedc388d 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -47,6 +47,7 @@ from litellm.proxy.common_request_processing import ( _ClientDisconnectedBeforeFirstChunk, attach_guardrail_information, _extract_error_from_sse_chunk, + FirstChunkSseFraming, _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, @@ -2373,7 +2374,9 @@ class TestCommonRequestProcessingHelpers: (RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)), ], ) - async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(self, framing, frame_starts): + async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape( + self, framing: FirstChunkSseFraming, frame_starts: tuple[str, ...] + ) -> None: response: Final = await create_response( _stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing ) From ca873a1327928e8cc1812d19ae892e10c7399a50 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:26:01 -0700 Subject: [PATCH 13/14] fix(proxy): keep cursor chat framing and JSON-encode Responses hook errors --- litellm/proxy/common_request_processing.py | 20 ++++-- .../common_utils/responses_stream_errors.py | 9 ++- litellm/proxy/proxy_server.py | 8 ++- .../proxy/response_api_endpoints/endpoints.py | 4 +- .../response_polling/background_streaming.py | 4 +- .../proxy_server/test_streaming_helpers.py | 27 ++++++-- .../response_api_endpoints/test_endpoints.py | 68 +++++++++++++------ 7 files changed, 99 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 27164ba0c83..a920b42b9d0 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1250,6 +1250,16 @@ RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming( ) +@dataclass(frozen=True, slots=True) +class SseFraming: + first_chunk: FirstChunkSseFraming + late: LateSseFraming + + +OPENAI_SSE_FRAMING: Final = SseFraming(first_chunk=OPENAI_FIRST_CHUNK_FRAMING, late=OPENAI_LATE_SSE_FRAMING) +RESPONSES_SSE_FRAMING: Final = SseFraming(first_chunk=RESPONSES_FIRST_CHUNK_FRAMING, late=RESPONSES_LATE_SSE_FRAMING) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -2945,7 +2955,7 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request: bool | None = False, contents: list[object] | None = None, skip_pre_call_logic: bool = False, - late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, + sse_framing: SseFraming = OPENAI_SSE_FRAMING, ) -> object: """Run the request, sending SSE keepalives while the upstream is still silent. @@ -2986,10 +2996,11 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request=is_streaming_request, contents=contents, skip_pre_call_logic=skip_pre_call_logic, + first_chunk_framing=sse_framing.first_chunk, ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, - late_framing=late_framing, + late_framing=sse_framing.late, ) async def _process_llm_request( @@ -3013,6 +3024,7 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request: bool | None = False, contents: list[object] | None = None, # Add contents parameter skip_pre_call_logic: bool = False, + first_chunk_framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING, ) -> Any: """ Common request processing logic for both chat completions and responses API endpoints @@ -3306,9 +3318,7 @@ class ProxyBaseLLMRequestProcessing: request=request, refresh_headers=refresh_stream_headers, general_settings=general_settings, - framing=( - RESPONSES_FIRST_CHUNK_FRAMING if route_type == "aresponses" else OPENAI_FIRST_CHUNK_FRAMING - ), + framing=first_chunk_framing, ) ### CALL HOOKS ### - modify outgoing data diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 4a806c32284..0024c210fab 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -61,19 +61,22 @@ class _FailureDetails(LiteLLMBaseModel): class _ErrorPayload(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - error: Mapping[str, object] + error: Mapping[str, object] | str + + def error_object(self) -> Mapping[str, object]: + return {"message": self.error} if isinstance(self.error, str) else self.error def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: try: - return _ErrorPayload.model_validate_json(data).error + return _ErrorPayload.model_validate_json(data).error_object() except ValidationError: return None def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None: try: - return _ErrorPayload.model_validate(event).error or None + return _ErrorPayload.model_validate(event).error_object() or None except ValidationError: return None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0377ed809ad..a6648e47de6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10202,6 +10202,12 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: return f"data: {chunk}\n\n" +def _format_responses_sse_chunk(chunk: object) -> str | bytes: + if isinstance(chunk, Mapping): + return f"data: {json.dumps(chunk, default=str)}\n\n" + return _format_streaming_sse_chunk(chunk=chunk) + + _SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r") _OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n" _MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024 @@ -10602,7 +10608,7 @@ async def async_data_generator( if not raw_passthrough: try: if error_state is not None: - yield error_state.mark_emitted(_format_streaming_sse_chunk(chunk=chunk)) + yield error_state.mark_emitted(_format_responses_sse_chunk(chunk=chunk)) else: yield _format_streaming_sse_chunk(chunk=chunk) except Exception as e: diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 1fd487ea076..948a32d4449 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -52,7 +52,7 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.common_request_processing import ( - RESPONSES_LATE_SSE_FRAMING, + RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing, cost_headers, create_response, @@ -438,7 +438,7 @@ async def responses_api( if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data)) else native_data_generator ), - late_framing=RESPONSES_LATE_SSE_FRAMING, + sse_framing=RESPONSES_SSE_FRAMING, model=None, user_model=user_model, user_temperature=user_temperature, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index cd91bfa204f..99c988317bc 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -21,7 +21,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth -from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing +from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.responses_stream_errors import ( event_error_object, responses_error_fields, @@ -188,7 +188,7 @@ async def background_streaming_task( user_api_base=user_api_base, version=version, skip_pre_call_logic=True, - late_framing=RESPONSES_LATE_SSE_FRAMING, + sse_framing=RESPONSES_SSE_FRAMING, ) # Process streaming response following OpenAI events format diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 93eb8fa226b..b3a6e7b1fbf 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio import json -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable, Mapping from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -148,7 +148,8 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent @pytest.mark.asyncio -async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event() -> None: +@pytest.mark.parametrize("as_mapping", [False, True]) +async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event(as_mapping: bool) -> None: response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[]) created: Final = ResponseCreatedEvent.model_validate( {"type": "response.created", "sequence_number": 0, "response": response} @@ -157,9 +158,9 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e {"type": "error", "sequence_number": 1, "code": "server_error", "message": "upstream failed", "param": None} ) - async def upstream() -> AsyncIterator[BaseModel]: + async def upstream() -> AsyncIterator[BaseModel | Mapping[str, object]]: yield created - yield error_event + yield error_event.model_dump() if as_mapping else error_event raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model") frames: Final = tuple( @@ -172,14 +173,26 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e ) assert tuple(frame.split("\n", 1)[0] for frame in frames) == ("event: response.created", "event: error") + assert json.loads(frames[-1].split("data: ", 1)[1])["message"] == "upstream failed" @pytest.mark.asyncio -async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event() -> None: +@pytest.mark.parametrize( + ("guardrail_error", "error_object"), + [ + ( + {"message": "response withheld", "type": "guardrail_scan_error", "code": 500}, + {"message": "response withheld", "type": "guardrail_scan_error", "code": 500}, + ), + ("response withheld", {"message": "response withheld"}), + ], +) +async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event( + guardrail_error: Mapping[str, object] | str, error_object: Mapping[str, object] +) -> None: response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[]) created: Final = ResponseCreatedEvent.model_validate( {"type": "response.created", "sequence_number": 0, "response": response} ) - guardrail_error: Final = {"message": "response withheld", "type": "guardrail_scan_error", "code": 500} async def upstream() -> AsyncIterator[object]: yield created @@ -203,7 +216,7 @@ async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_ "message": "response withheld", "param": None, "sequence_number": 1, - "error": guardrail_error, + "error": error_object, } diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index abf5933bd35..d7d6321dd2b 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -181,14 +181,16 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( class _FirstFrameHook(CustomLogger): - def __init__(self, frame: str | bytes) -> None: + def __init__(self, first: str | bytes | Exception) -> None: super().__init__() - self.frame: Final = frame + self.first: Final = first async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: object, response: object, request_data: object ) -> AsyncIterator[str | bytes]: - yield self.frame + if isinstance(self.first, Exception): + raise self.first + yield self.first _DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n' @@ -197,22 +199,9 @@ _NAMED_ERROR_FRAME: Final = ( ) -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path,first_frame,expected_status", - [ - ("/v1/responses", _DATA_ERROR_FRAME, 403), - ("/v1/chat/completions", _DATA_ERROR_FRAME, 403), - ("/v1/chat/completions", _NAMED_ERROR_FRAME, 200), - ("/v1/responses", _NAMED_ERROR_FRAME, 529), - ("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200), - ("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), - ("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), - ], -) -async def test_only_a_lone_first_error_frame_answers_with_its_status( - monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int -) -> None: +async def _post_through_first_frame_hook( + monkeypatch: pytest.MonkeyPatch, path: str, first: str | bytes | Exception +) -> httpx.Response: import litellm.proxy.proxy_server as ps from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -230,14 +219,14 @@ async def test_only_a_lone_first_error_frame_answers_with_its_status( ) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first_frame)]) + monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first)]) monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) with respx.mock as transport: transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond( 200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"} ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: - result: Final = await client.post( + return await client.post( path, json={ "model": model, @@ -246,6 +235,26 @@ async def test_only_a_lone_first_error_frame_answers_with_its_status( }, ) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,first_frame,expected_status", + [ + ("/v1/responses", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _NAMED_ERROR_FRAME, 200), + ("/cursor/chat/completions", _NAMED_ERROR_FRAME, 200), + ("/v1/responses", _NAMED_ERROR_FRAME, 529), + ("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200), + ("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ], +) +async def test_only_a_lone_first_error_frame_answers_with_its_status( + monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int +) -> None: + result: Final = await _post_through_first_frame_hook(monkeypatch, path, first_frame) + assert result.status_code == expected_status, result.text if expected_status == 200: assert result.headers["content-type"].startswith("text/event-stream"), result.text @@ -253,6 +262,23 @@ async def test_only_a_lone_first_error_frame_answers_with_its_status( assert result.json()["error"]["message"] == "denied" +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/cursor/chat/completions", "/v1/chat/completions"]) +async def test_a_hook_failure_before_the_first_chunk_answers_in_chat_completions_shape( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + message: Final = "rate limited before the first chunk" + result: Final = await _post_through_first_frame_hook( + monkeypatch, path, HTTPException(status_code=429, detail=message) + ) + frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) + + assert result.status_code == 429, result.text + assert frames[-1] == "data: [DONE]", result.text + assert message in json.loads(frames[0].removeprefix("data: "))["error"]["message"] + assert len(frames) == 2, result.text + + @pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None: async def request_and_check() -> None: From 81dc6c1175fabc0acb7cbdd50c8f3db88bfef9b8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:04:26 -0700 Subject: [PATCH 14/14] fix(proxy): encode non-dict Mapping Responses chunks as JSON objects --- litellm/proxy/proxy_server.py | 2 +- .../proxy/proxy_server/test_streaming_helpers.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a6648e47de6..aef2bca785c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10204,7 +10204,7 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: def _format_responses_sse_chunk(chunk: object) -> str | bytes: if isinstance(chunk, Mapping): - return f"data: {json.dumps(chunk, default=str)}\n\n" + return f"data: {json.dumps({**chunk}, default=str)}\n\n" return _format_streaming_sse_chunk(chunk=chunk) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index b3a6e7b1fbf..b75a2494588 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio import json from collections.abc import AsyncIterator, Callable, Mapping +from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -148,8 +149,14 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent @pytest.mark.asyncio -@pytest.mark.parametrize("as_mapping", [False, True]) -async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event(as_mapping: bool) -> None: +@pytest.mark.parametrize( + "as_chunk", + [lambda event: event, lambda event: event.model_dump(), lambda event: MappingProxyType(event.model_dump())], + ids=["model", "dict", "mapping_proxy"], +) +async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event( + as_chunk: Callable[[ErrorEvent], BaseModel | Mapping[str, object]], +) -> None: response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[]) created: Final = ResponseCreatedEvent.model_validate( {"type": "response.created", "sequence_number": 0, "response": response} @@ -160,7 +167,7 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e async def upstream() -> AsyncIterator[BaseModel | Mapping[str, object]]: yield created - yield error_event.model_dump() if as_mapping else error_event + yield as_chunk(error_event) raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model") frames: Final = tuple(