From 6b5ed50633348efa3e7c9b416e2a99d5a959a00a Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 23 Oct 2026 07:42:05 +0800 Subject: [PATCH 01/23] fix(proxy): preserve Responses SSE event framing --- .../guardrail_translation/handler.py | 2 +- .../common_utils/responses_stream_errors.py | 5 +- litellm/proxy/proxy_server.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 1 - ...test_openai_responses_guardrail_handler.py | 9 ++- .../proxy_server/test_streaming_helpers.py | 62 +++++++++++++++++-- .../response_api_endpoints/test_endpoints.py | 14 +++-- .../proxy/test_response_model_sanitization.py | 5 +- 8 files changed, 85 insertions(+), 15 deletions(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 3ea126c5a2a..d08806b19cb 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -1789,7 +1789,6 @@ class OpenAIResponsesHandler(BaseTranslation): new output item under the same response id, and close with a ``response.completed`` carrying only the replacement item. - The proxy's data generator appends ``data: [DONE]`` itself. """ events: Final = ( self._block_continuation_events(exc, responses_so_far or ()) @@ -1797,6 +1796,7 @@ class OpenAIResponsesHandler(BaseTranslation): else self._standalone_block_events(exc) ) return tuple( + f"event: {event.type.value if isinstance(event.type, ResponsesAPIStreamEvents) else event.type}\n" f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode() for event in events ) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 3409b4b6657..6e563b2b257 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -131,7 +131,10 @@ class ResponsesStreamErrorState: self.created_at = event.response.created_at if event.type in ("response.completed", "response.failed", "response.incomplete"): self.terminal_emitted = True - return frame + if not event.type or "\n" in event.type or "\r" in event.type: + return frame + prefix: Final = f"event: {event.type}\n" + return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_failure(self, exception: Exception) -> str | None: if self.terminal_emitted: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f632317d9a8..6aff6601ac4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10390,7 +10390,7 @@ async def async_data_generator( stream_completed = False client_disconnected = False error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None - ends_with_openai_done: Final = not request_data.get("_litellm_skip_openai_stream_done") + ends_with_openai_done: Final = not responses_stream_errors and not request_data.get("_litellm_skip_openai_stream_done") needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap() stream_iterator: Final = _object_stream( proxy_logging_obj.async_post_call_streaming_iterator_hook( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 312c5124ca5..8a13ce876ea 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -493,7 +493,6 @@ async def responses_api( async def _blocked_stream() -> AsyncGenerator[str, None]: for chunk in block_chunks: yield chunk.decode() - yield "data: [DONE]\n\n" return await create_response(generator=_blocked_stream(), media_type="text/event-stream", headers={}) return build_blocked_response(e) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 2f10fd6fd0d..23dbf3906d6 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3766,7 +3766,14 @@ class TestBuildBlockSseChunks: def _payloads(self, chunks): import json - return [json.loads(chunk.decode().removeprefix("data: ").strip()) for chunk in chunks] + payloads = [ + json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: "))) + for chunk in chunks + ] + assert [chunk.decode().splitlines()[0] for chunk in chunks] == [ + f"event: {payload['type']}" for payload in payloads + ] + return payloads def test_standalone_block_emits_complete_synthetic_stream(self): handler = OpenAIResponsesHandler() diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 9a3516d20d6..015269801b8 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -50,9 +50,12 @@ from litellm.proxy.proxy_server import ( select_data_generator, ) from litellm.types.llms.openai import ( + ErrorEvent, + OutputTextDeltaEvent, ResponseCompletedEvent, ResponseCreatedEvent, ResponseFailedEvent, + ResponseIncompleteEvent, ResponsesAPIResponse, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage @@ -92,6 +95,56 @@ async def _async_iter_raises(exc: Exception): raise exc +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal_type", ["response.completed", "response.failed", "response.incomplete", "error"]) +async def test_responses_stream_dispatches_named_events_without_a_chat_done_sentinel(terminal_type: str) -> None: + response: Final = ResponsesAPIResponse(id="resp_framing", created_at=1, model="local-model", output=[]) + created: Final = ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ) + delta: Final = OutputTextDeltaEvent.model_validate( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_framing", + "output_index": 0, + "content_index": 0, + "delta": "hello", + } + ) + terminal_models: Final = { + "response.completed": ResponseCompletedEvent, + "response.failed": ResponseFailedEvent, + "response.incomplete": ResponseIncompleteEvent, + "error": ErrorEvent, + } + terminal: Final = terminal_models[terminal_type].model_validate( + {"type": terminal_type, "sequence_number": 2, "response": response} + ) + + async def upstream() -> AsyncIterator[BaseModel]: + for event in (created, delta, terminal): + yield event + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True + ) + ] + ) + assert len(frames) == 3 + for frame, event_type in zip( + frames, ("response.created", "response.output_text.delta", terminal_type), strict=True + ): + assert frame.startswith(f"event: {event_type}\n") + payload: Final = json.loads( + next(line.removeprefix("data: ") for line in frame.splitlines() if line.startswith("data: ")) + ) + assert payload["type"] == event_type + assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" + class _FakeStream: def __init__(self, chunks, hidden_params=None): self._chunks = chunks @@ -1132,14 +1185,15 @@ async def test_responses_stream_failure_sends_the_failure_hook_replacement(monke ] ) failure: Final = ResponseFailedEvent.model_validate_json( - next(line.removeprefix("data: ") for line in frames[-2].splitlines() if line.startswith("data: ")) + next(line.removeprefix("data: ") for line in frames[-1].splitlines() if line.startswith("data: ")) ) assert _BLOCK_MESSAGE not in "".join(frames) assert failure.response.error is not None assert failure.response.error["code"] == "permission_error" assert "Request blocked by policy" in failure.response.error["message"] - assert frames[-1] == "data: [DONE]\n\n" + assert frames[-1].startswith("event: response.failed\n") + assert "data: [DONE]\n\n" not in frames @pytest.mark.asyncio @@ -1335,8 +1389,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina for frame in event_frames ) - assert decoded[-1] == "data: [DONE]\n\n" - assert len(decoded) == len(event_frames) + 1 + assert len(decoded) == len(event_frames) + assert "data: [DONE]\n\n" not in decoded assert payloads[0]["response"]["id"] == "resp_visible" assert payloads[1] == tool_delta.model_dump() assert len(payloads) == 3 diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 985aee061bc..0bf14f59b1a 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -142,8 +142,9 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert result.status_code == 200, result.text assert message in result.text if path == "/v1/responses": - assert frames[-1] == "data: [DONE]", result.text - assert frames[-2].startswith("event: response.failed\n"), result.text + assert all("data: [DONE]" not in frame for frame in frames), result.text + assert frames[-1].startswith("event: response.failed\n"), result.text + assert [frame.splitlines()[0] for frame in frames] == [f"event: {event['type']}" for event in events] if partial: assert [event["type"] for event in events] == [ "response.created", @@ -4862,9 +4863,11 @@ class TestGuardrailBlockedResponsesShape: assert response.status_code == 200, response.text assert response.headers["content-type"].startswith("text/event-stream") frames = _sse_data_frames(response.text) - assert frames[-1] == "[DONE]" - events = [json.loads(frame) for frame in frames[:-1]] + assert "[DONE]" not in frames + events = [json.loads(frame) for frame in frames] types = [event["type"] for event in events] + assert types[-1] == "response.completed" + assert [line[7:] for line in response.text.splitlines() if line.startswith("event: ")] == types assert "response.created" in types completed = [event for event in events if event["type"] == "response.completed"] assert len(completed) == 1 @@ -4891,7 +4894,8 @@ class TestGuardrailBlockedResponsesShape: assert response.status_code == 200, response.text frames = _sse_data_frames(response.text) - completed = [json.loads(frame) for frame in frames[:-1] if json.loads(frame)["type"] == "response.completed"] + assert "[DONE]" not in frames + completed = [json.loads(frame) for frame in frames if json.loads(frame)["type"] == "response.completed"] usage = completed[0]["response"]["usage"] assert usage["input_tokens"] == 14 assert usage["output_tokens"] == 20 diff --git a/tests/unit/proxy/test_response_model_sanitization.py b/tests/unit/proxy/test_response_model_sanitization.py index f52736453c1..6f3c5d8430b 100644 --- a/tests/unit/proxy/test_response_model_sanitization.py +++ b/tests/unit/proxy/test_response_model_sanitization.py @@ -763,7 +763,10 @@ def test_restamp_streaming_chunk_rewrites_nested_model_in_guardrail_block_frames )[0] for frame in frames ] - events = [json.loads(frame.decode().removeprefix("data: ")) for frame in client_frames] + events = [ + json.loads(next(line[6:] for line in frame.decode().splitlines() if line.startswith("data: "))) + for frame in client_frames + ] assert all(frame.endswith(b"\n\n") for frame in client_frames) assert events[-1]["type"] == "response.completed" From dfe2b4fd87f3b774330c7832822aa503a2b4ece6 Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 23 Oct 2026 07:57:01 +0800 Subject: [PATCH 02/23] style(proxy): format Responses stream terminator condition --- litellm/proxy/proxy_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6aff6601ac4..987c8739c04 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10390,7 +10390,9 @@ async def async_data_generator( stream_completed = False client_disconnected = False error_state: Final = ResponsesStreamErrorState() if responses_stream_errors else None - ends_with_openai_done: Final = not responses_stream_errors and not request_data.get("_litellm_skip_openai_stream_done") + ends_with_openai_done: Final = not responses_stream_errors and not request_data.get( + "_litellm_skip_openai_stream_done" + ) needs_iterator_wrap: Final = proxy_logging_obj.needs_iterator_wrap() stream_iterator: Final = _object_stream( proxy_logging_obj.async_post_call_streaming_iterator_hook( From 4fb052e786b66248329dea5747927aa5f1730165 Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 23 Oct 2026 08:26:33 +0800 Subject: [PATCH 03/23] fix(proxy): validate Responses guardrail event names --- .../guardrail_translation/handler.py | 3 ++- .../proxy_server/test_streaming_helpers.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index d08806b19cb..2e77fdd62f0 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -251,6 +251,7 @@ _TOOL_SEARCH_CALL_TYPE: Final = "tool_search_call" _OUTPUT_TOOL_CALL_ITEM_TYPES: Final = _TOOL_CALL_ITEM_TYPES | {_TOOL_SEARCH_CALL_TYPE} _SCANNED_TOOL_SEARCH_NAME: Final = "tool_search" _TOOL_SEARCH_ARGUMENTS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_STREAM_EVENT_TYPE_ADAPTER: Final = TypeAdapter(str) _TOOL_CALL_PAYLOAD_FIELDS: Final[Mapping[str, str]] = MappingProxyType( {"function_call": "arguments", "custom_tool_call": "input"} ) @@ -1796,7 +1797,7 @@ class OpenAIResponsesHandler(BaseTranslation): else self._standalone_block_events(exc) ) return tuple( - f"event: {event.type.value if isinstance(event.type, ResponsesAPIStreamEvents) else event.type}\n" + f"event: {_STREAM_EVENT_TYPE_ADAPTER.validate_python(stream_item_field(event, 'type'))}\n" f"data: {event.model_dump_json(exclude_none=True, exclude_unset=True, serialize_as_any=True)}\n\n".encode() for event in events ) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 015269801b8..835411608ed 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -34,6 +34,7 @@ from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, _fast_serialize_simple_model_response_stream, @@ -145,6 +146,31 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent assert payload["type"] == event_type assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" +@pytest.mark.parametrize("binary", [False, True]) +@pytest.mark.parametrize( + ("event_type", "event_line"), + [ + (None, ""), + ("", ""), + ("response.delta\ninjected", ""), + ("response.delta\rinjected", ""), + ("response.delta\r\ninjected", ""), + ("response.vendor_delta", "event: response.vendor_delta\n"), + ], +) +def test_responses_event_names_preserve_payload_without_injecting_sse_lines( + event_type: str | None, event_line: str, binary: bool +) -> None: + payload: Final = {"type": event_type, "sequence_number": 0} + data_line: Final = f"data: {json.dumps(payload)}\n\n" + frame: Final = data_line.encode("utf-8") if binary else data_line + expected: Final = (event_line + data_line).encode("utf-8") if binary else event_line + data_line + state: Final = ResponsesStreamErrorState() + state.observe_chunk(payload) + + assert state.mark_emitted(frame) == expected + + class _FakeStream: def __init__(self, chunks, hidden_params=None): self._chunks = chunks From 7586485b4e274dd90a0ccfbf45886aeab392f9d0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:03:13 -0700 Subject: [PATCH 04/23] fix(proxy): keep Responses first-event errors and keepalive failures in Responses framing --- litellm/proxy/common_request_processing.py | 68 +++++++++++++++---- .../common_utils/responses_stream_errors.py | 18 ++++- litellm/proxy/proxy_server.py | 2 +- .../proxy/response_api_endpoints/endpoints.py | 2 + .../observability/test_guardrail_effects.py | 16 +++-- .../test_bedrock_mantle_claude_chat_chaos.py | 2 +- .../test_bedrock_mantle_claude_chat_wire.py | 7 +- ...test_chat_stream_guardrail_error_events.py | 3 +- .../test_responses_bridge_stream_errors.py | 3 +- .../test_responses_top_level_error_events.py | 4 +- ...test_openai_responses_guardrail_handler.py | 2 +- .../response_api_endpoints/test_endpoints.py | 61 +++++++++++++++++ .../proxy/test_common_request_processing.py | 50 ++++++++++++++ 13 files changed, 207 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a36a7f2dc..abc99a3be23 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4,6 +4,7 @@ import json import logging import math from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from functools import lru_cache from itertools import chain @@ -114,6 +115,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, openai_error_type, ) +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, SSE_STREAM_START_TAIL, @@ -907,11 +909,18 @@ async def _enforce_guardrail_added_tag_budgets( ) from e +def _sse_data_payload(event_line: str | bytes) -> str | None: + text: Final = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line + return next( + (line.removeprefix("data: ").strip() for line in text.splitlines() if line.startswith("data: ")), + None, + ) + + async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: """Parses an event line and returns an error code if present, else None.""" - event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line - if event_line.startswith("data: "): - json_str: Final = event_line[len("data: ") :].strip() + json_str: Final = _sse_data_payload(event_line) + if json_str is not None: if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message return None try: @@ -952,8 +961,6 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: Returns: Error dictionary in OpenAI API format """ - event_line = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line - # Default error format default_error: Final = { "message": "Unknown error", @@ -962,8 +969,8 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: "code": "500", } - if event_line.startswith("data: "): - json_str: Final = event_line[len("data: ") :].strip() + json_str: Final = _sse_data_payload(event_line) + if json_str is not None: if not json_str or json_str == "[DONE]": return default_error @@ -1171,6 +1178,39 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]: return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n" +@dataclass(frozen=True, slots=True) +class LateSseFraming: + """How a response the keepalive already opened ends once its upstream fails late.""" + + failure: Callable[[Exception], tuple[str, ...]] + body: Callable[[Response], tuple[bytes, ...]] + + +def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]: + _, error_obj = sse_error_payload(exc) + return _sse_error_frames(error_obj) + + +def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: + return b"data: " + (bytes(produced.body) or b"{}") + b"\n\n", b"data: [DONE]\n\n" + + +def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: + frame: Final = ResponsesStreamErrorState().format_failure(exc) + return () if frame is None else (frame,) + + +def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: + frame: Final = ResponsesStreamErrorState().format_error_body(bytes(produced.body), produced.status_code) + return () if frame is None else (frame.encode(),) + + +OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames) +RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming( + failure=_responses_late_failure_frames, body=_responses_late_body_frames +) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -1371,13 +1411,13 @@ async def _aclose_late_response(produced: Response) -> None: verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc) -async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]: +async def _relay_late_response(produced: Response, framing: LateSseFraming) -> AsyncGenerator[bytes, None]: """Replay a Response that was built after a keepalive had already opened the wire.""" if not isinstance(produced, StreamingResponse): # The status line is already on the wire, so a non-streaming body, an error # body included, can only reach the client as an SSE frame. - yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n" - yield b"data: [DONE]\n\n" + for frame in framing.body(produced): + yield frame return try: @@ -1419,6 +1459,7 @@ async def open_sse_before_first_byte( ping_interval_seconds: float | str | None, media_type: str = "text/event-stream", on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None, + late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, ) -> _LateResponseT | StreamingResponse: """Write SSE keepalive comments while the upstream LLM call is still in flight. @@ -1464,11 +1505,10 @@ async def open_sse_before_first_byte( # would never fire and the failure would go unaudited. The hook # also gets to sanitize what reaches the client, by returning or # raising a replacement, so its answer decides the frame. - _, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure)) - for frame in _sse_error_frames(error_obj): + for frame in late_framing.failure(await _sanitized_late_failure(exc, on_late_failure)): yield frame.encode() return - async for chunk in _relay_late_response(produced): + async for chunk in _relay_late_response(produced, late_framing): yield chunk finally: if not produce_task.done(): @@ -2812,6 +2852,7 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request: bool | None = False, contents: list[object] | None = None, skip_pre_call_logic: bool = False, + late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, ) -> object: """Run the request, sending SSE keepalives while the upstream is still silent. @@ -2855,6 +2896,7 @@ class ProxyBaseLLMRequestProcessing: ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, + late_framing=late_framing, ) async def _process_llm_request( diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 6e563b2b257..312bf472594 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -57,6 +57,12 @@ class _FailureDetails(LiteLLMBaseModel): return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields}) +class _ErrorBody(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + error: _FailureDetails = _FailureDetails() + + def _original_failure(exception: Exception) -> Exception: current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(current, MidStreamFallbackError) and current.original_exception is not None: @@ -137,10 +143,16 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_failure(self, exception: Exception) -> str | None: + original: Final = _original_failure(exception) + return self._format_failed(_failure_details(original), str(original)) + + def format_error_body(self, body: bytes, status_code: int) -> str | None: + error: Final = _ErrorBody.model_validate_json(body or b"{}").error + return self._format_failed(error.model_copy(update={"status_code": status_code}), body.decode()) + + def _format_failed(self, details: _FailureDetails, fallback_message: str) -> str | None: if self.terminal_emitted: return None - original: Final = _original_failure(exception) - details: Final = _failure_details(original) response: Final = ResponsesAPIResponse.model_validate( MappingProxyType( { @@ -153,7 +165,7 @@ class ResponsesStreamErrorState: "error": MappingProxyType( { "code": _response_error_code(details), - "message": redact_internal_details_from_client_message(details.message or str(original)), + "message": redact_internal_details_from_client_message(details.message or fallback_message), } ), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a5a287fb42e..15832d81b7f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10620,7 +10620,7 @@ async def async_data_generator( if error_message is not None: yield error_message - # OpenAI-compatible streams terminate with data: [DONE]; Google GenAI (?alt=sse) does not. + # Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not. if ends_with_openai_done: yield _OPENAI_STREAM_DONE_FRAME except (asyncio.CancelledError, GeneratorExit): diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index aa5fe472c66..1e16c16f33a 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -52,6 +52,7 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.common_request_processing import ( + RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing, cost_headers, create_response, @@ -435,6 +436,7 @@ async def responses_api( if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data)) else native_data_generator ), + late_framing=RESPONSES_LATE_SSE_FRAMING, model=None, user_model=user_model, user_temperature=user_temperature, diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 8f83287696c..c1ed810a428 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1197,9 +1197,7 @@ def test_responses_pre_call_denial_streams_sse_with_typed_message_item(gateway: response.headers["content-type"], response.text, ) - lines: Final = tuple(line for line in response.text.split("\n") if line.startswith("data: ")) - assert lines[-1] == "data: [DONE]", response.text - events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1]) + events: Final = _blocked_stream_events(response.text) kinds: Final = tuple(event["type"] for event in events) assert tuple(kind for kind in kinds if kind != "response.output_text.delta") == ( "response.created", @@ -1286,9 +1284,13 @@ def _responses_output_denial_config(tmp_path: Path, identity: str, model: str) - def _blocked_stream_events(text: str) -> tuple[dict[str, object], ...]: - lines: Final = tuple(line for line in text.split("\n") if line.startswith("data: ")) - assert lines[-1] == "data: [DONE]", text - return tuple(json.loads(line.removeprefix("data: ")) for line in lines[:-1]) + assert "data: [DONE]" not in text, text + lines: Final = tuple(line for line in text.split("\n") if line.startswith(("event: ", "data: "))) + events: Final[tuple[dict[str, object], ...]] = tuple( + json.loads(line.removeprefix("data: ")) for line in lines[1::2] + ) + assert lines[::2] == tuple(f"event: {event['type']}" for event in events), text + return events def _dead_api_base() -> str: @@ -2786,7 +2788,7 @@ def test_client_disconnect_after_a_finished_responses_tool_call_scans_the_text_a with _disconnect_rig(gateway, tmp_path, shape="text_then_tool_call") as rig: try: received: Final = _stream_and_close( - rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), "response.output_item.done" + rig, "/v1/responses", _tool_call_request(rig, "/v1/responses"), '"type":"response.output_item.done"' ) assert rig.token() in received, received scans: Final = eventually( diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py index bcbf6d0f6f3..2325b705790 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_chaos.py @@ -261,7 +261,7 @@ async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once( assert len(served) == 10 for item in served: _assert_answered_with_its_own_marker(item) - assert item.text.rstrip().endswith("data: [DONE]"), item.text + assert item.text.rstrip().endswith("data: [DONE]") == (item.call.endpoint == "chat"), item.text _assert_native_requests_for(wire.drain(), calls) assert _spend_status_by_call(served) == {item.call_id: "success" for item in served} diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index d4f9533fcdc..9c73fcc3b6e 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -846,7 +846,12 @@ def test_responses_api_stream_on_a_mantle_claude_id_bridges_to_the_native_stream "POST", "/v1/responses", {"model": model, "input": _prompt(marker), "stream": True} ) assert response.status_code == 200, response.text - events: Final = _sse_objects(response.text) + assert "data: [DONE]" not in response.text, response.text + events: Final = tuple( + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") + ) kinds: Final = [event["type"] for event in events] assert (kinds[0], kinds[-1]) == ("response.created", "response.completed"), kinds assert "".join( diff --git a/tests/integration/streaming/test_chat_stream_guardrail_error_events.py b/tests/integration/streaming/test_chat_stream_guardrail_error_events.py index a8ebe510b8e..b2390de7737 100644 --- a/tests/integration/streaming/test_chat_stream_guardrail_error_events.py +++ b/tests/integration/streaming/test_chat_stream_guardrail_error_events.py @@ -482,8 +482,7 @@ def _responses_body(marker: str, *directives: str) -> dict[str, JsonValue]: def _failed_response_error(response: httpx.Response) -> dict[str, JsonValue]: assert response.status_code == 200, response.text - lines: Final = _data_lines(response.text) - assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text + assert "[DONE]" not in _data_lines(response.text), response.text events: Final = _payloads(response.text) kinds: Final = tuple(string_value(event["type"]) for event in events) assert kinds[0] == "response.created" and kinds[-1] == "response.failed", response.text diff --git a/tests/integration/streaming/test_responses_bridge_stream_errors.py b/tests/integration/streaming/test_responses_bridge_stream_errors.py index 8a5655be0f8..6b0e35f9c56 100644 --- a/tests/integration/streaming/test_responses_bridge_stream_errors.py +++ b/tests/integration/streaming/test_responses_bridge_stream_errors.py @@ -474,9 +474,10 @@ def test_native_responses_stream_forwards_the_failed_response_on_both_legs(gatew assert response.status_code == 200, response.text frames: Final = data_frames(response.text) assert [frame["type"] for frame in frames] == ["response.created", "response.failed"], response.text + assert [name for name, _ in sse_events(response.text)] == ["response.created", "response.failed"], response.text failed: Final = object_value(frames[-1]["response"]) assert failed["status"] == "failed", failed assert object_value(failed["error"])["code"] == "rate_limit_exceeded", failed - assert response.text.rstrip().endswith("data: [DONE]"), response.text + assert "data: [DONE]" not in response.text, response.text assert len(wire.drain()) == 1 assert spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" diff --git a/tests/integration/streaming/test_responses_top_level_error_events.py b/tests/integration/streaming/test_responses_top_level_error_events.py index 63bdadceebc..6d361492dd6 100644 --- a/tests/integration/streaming/test_responses_top_level_error_events.py +++ b/tests/integration/streaming/test_responses_top_level_error_events.py @@ -256,8 +256,10 @@ def _responses_frames(gateway: Gateway, stream: tuple[bytes, ...]) -> tuple[http ) _assert_one_attempt(served.wire, marker) assert served.response.status_code == 200, served.response.text - assert served.response.text.rstrip().endswith("data: [DONE]"), served.response.text + assert "data: [DONE]" not in served.response.text, served.response.text frames: Final = data_frames(served.response.text) + names: Final = tuple(name for name, _ in sse_events(served.response.text)) + assert names == tuple(item["type"] for item in frames), served.response.text assert frames[-1]["type"] == "response.failed", served.response.text return served.response, object_value(object_value(frames[-1]["response"])["error"]) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 0b7f32575c5..8aa887bfd78 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3783,7 +3783,7 @@ class TestBuildBlockSseChunks: def _payloads(self, chunks): import json - payloads = [ + payloads: Final = [ json.loads(next(line[6:] for line in chunk.decode().splitlines() if line.startswith("data: "))) for chunk in chunks ] diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 0bf14f59b1a..5945352c61e 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -177,6 +177,67 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) +async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str): + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-6-astra" + message: Final = "Upstream is rate limiting this key" + upstream_url: Final = "https://streaming.example/v1" + is_chat: Final = path == "/v1/chat/completions" + + async def slow_rate_limit(_: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.3) + return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "openai/" + model, + "api_base": upstream_url, + "api_key": "fixture-key", + "max_retries": 0, + }, + } + ], + num_retries=0, + ) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( + side_effect=slow_rate_limit + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + path, + json={ + "model": model, + "stream": True, + **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), + }, + ) + frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) + + assert result.status_code == 200, result.text + assert frames[0] == ": ping", result.text + if is_chat: + assert frames[-1] == "data: [DONE]", result.text + assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] + else: + assert "data: [DONE]" not in result.text + assert frames[-1].startswith("event: response.failed\ndata: "), result.text + failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] + assert failed_error["code"] == "rate_limit_exceeded" + assert message in failed_error["message"] + + _COMPACTION_ITEM: Final = { "type": "compaction", "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 52816dc50a8..310b526365f 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -50,6 +50,7 @@ from litellm.proxy.common_request_processing import ( _has_attribute_error_in_chain, include_guardrail_response_requested, is_azure_model_router_request, + RESPONSES_LATE_SSE_FRAMING, open_sse_before_first_byte, resolve_litellm_call_id, ttft_keepalive_interval, @@ -2161,6 +2162,18 @@ class TestCommonRequestProcessingHelpers: assert body["error"]["code"] == 403 assert body["error"]["message"] == "forbidden" + async def test_a_responses_stream_whose_first_event_is_a_named_error_answers_as_json(self): + error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked", "param": None} + + async def responses_stream(): + yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" + + response: Final = await create_response(responses_stream(), "text/event-stream", {}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(response.body) == {"error": error} + async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): yield 'data: {"content": "first part"}\n\n' @@ -10378,6 +10391,43 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" +async def _slow_rate_limit_failure(): + await asyncio.sleep(0.2) + raise HTTPException(status_code=429, detail="rate limited") + + +async def _slow_guardrail_block_body(): + await asyncio.sleep(0.2) + return JSONResponse(status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "produce,expected_code,expected_message", + [ + (_slow_rate_limit_failure, "rate_limit_exceeded", "rate limited"), + (_slow_guardrail_block_body, "invalid_request_error", "blocked"), + ], +) +async def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( + produce, expected_code, expected_message +): + response: Final = await open_sse_before_first_byte( + produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + ) + collected: Final = await _drain(response) + + assert collected[0] == TTFT_PING + assert b"data: [DONE]\n\n" not in collected + event_line, data_line = collected[-1].decode().strip().split("\n") + failed: Final = json.loads(data_line.removeprefix("data: ")) + assert event_line == "event: response.failed" + assert failed["type"] == "response.failed" + assert failed["response"]["status"] == "failed" + assert failed["response"]["error"]["code"] == expected_code + assert expected_message in failed["response"]["error"]["message"] + + @pytest.mark.asyncio async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed(): """Starlette never calls the produced response, so its own cleanup never runs From 9f09276e73ae22aef0dd724ea7f77bc5ffd8c554 Mon Sep 17 00:00:00 2001 From: HootRock Date: Wed, 28 Oct 2026 20:35:05 +0800 Subject: [PATCH 05/23] test(proxy): use controlled clocks for Responses keepalive failures --- .../response_api_endpoints/test_endpoints.py | 112 ++++++++++-------- .../proxy/test_common_request_processing.py | 52 ++++---- 2 files changed, 91 insertions(+), 73 deletions(-) diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 5945352c61e..370b9586464 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -24,6 +24,7 @@ from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.router import RouterErrors from tests._master_key import MASTER_KEY +from tests._support.clock_loops import AutojumpClockLoop @pytest.mark.asyncio @@ -177,65 +178,72 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] -@pytest.mark.asyncio @pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) -async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str): - import litellm.proxy.proxy_server as ps - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None: + async def request_and_check() -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - model: Final = "gpt-6-astra" - message: Final = "Upstream is rate limiting this key" - upstream_url: Final = "https://streaming.example/v1" - is_chat: Final = path == "/v1/chat/completions" + model: Final = "gpt-6-astra" + message: Final = "Upstream is rate limiting this key" + upstream_url: Final = "https://streaming.example/v1" + is_chat: Final = path == "/v1/chat/completions" - async def slow_rate_limit(_: httpx.Request) -> httpx.Response: - await asyncio.sleep(0.3) - return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) + async def slow_rate_limit(_: httpx.Request) -> httpx.Response: + await asyncio.sleep(0.3) + return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) - router: Final = litellm.Router( - model_list=[ - { - "model_name": model, - "litellm_params": { - "model": "openai/" + model, - "api_base": upstream_url, - "api_key": "fixture-key", - "max_retries": 0, - }, - } - ], - num_retries=0, - ) - monkeypatch.setattr(ps, "llm_router", router) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) - monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) - with respx.mock as transport: - transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( - side_effect=slow_rate_limit + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "openai/" + model, + "api_base": upstream_url, + "api_key": "fixture-key", + "max_retries": 0, + }, + } + ], + num_retries=0, ) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: - result: Final = await client.post( - path, - json={ - "model": model, - "stream": True, - **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), - }, + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", 0.05) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( + side_effect=slow_rate_limit ) - frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + path, + json={ + "model": model, + "stream": True, + **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), + }, + ) + frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) - assert result.status_code == 200, result.text - assert frames[0] == ": ping", result.text - if is_chat: - assert frames[-1] == "data: [DONE]", result.text - assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] - else: - assert "data: [DONE]" not in result.text - assert frames[-1].startswith("event: response.failed\ndata: "), result.text - failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] - assert failed_error["code"] == "rate_limit_exceeded" - assert message in failed_error["message"] + assert result.status_code == 200, result.text + assert frames[0] == ": ping", result.text + if is_chat: + assert frames[-1] == "data: [DONE]", result.text + assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] + else: + assert "data: [DONE]" not in result.text + assert frames[-1].startswith("event: response.failed\ndata: "), result.text + failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] + assert failed_error["code"] == "rate_limit_exceeded" + assert message in failed_error["message"] + + loop: Final = AutojumpClockLoop() + try: + loop.run_until_complete(request_and_check()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() _COMPACTION_ITEM: Final = { diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 310b526365f..a888fcc431d 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -3,7 +3,7 @@ import contextvars import copy import datetime import json -from collections.abc import AsyncIterable, Mapping +from collections.abc import AsyncIterable, Awaitable, Mapping from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence from urllib.parse import unquote_plus @@ -81,6 +81,7 @@ from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment +from tests._support.clock_loops import AutojumpClockLoop def test_attach_guardrail_information_copies_recorded_entries_onto_model_response(): @@ -10391,17 +10392,18 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" -async def _slow_rate_limit_failure(): +async def _slow_rate_limit_failure() -> Response: await asyncio.sleep(0.2) raise HTTPException(status_code=429, detail="rate limited") -async def _slow_guardrail_block_body(): +async def _slow_guardrail_block_body() -> Response: await asyncio.sleep(0.2) - return JSONResponse(status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}) + return JSONResponse( + status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}} + ) -@pytest.mark.asyncio @pytest.mark.parametrize( "produce,expected_code,expected_message", [ @@ -10409,23 +10411,31 @@ async def _slow_guardrail_block_body(): (_slow_guardrail_block_body, "invalid_request_error", "blocked"), ], ) -async def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( - produce, expected_code, expected_message -): - response: Final = await open_sse_before_first_byte( - produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING - ) - collected: Final = await _drain(response) +def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( + produce: Callable[[], Awaitable[Response]], expected_code: str, expected_message: str +) -> None: + async def collect_and_check() -> None: + response: Final = await open_sse_before_first_byte( + produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + ) + collected: Final = await _drain(response) - assert collected[0] == TTFT_PING - assert b"data: [DONE]\n\n" not in collected - event_line, data_line = collected[-1].decode().strip().split("\n") - failed: Final = json.loads(data_line.removeprefix("data: ")) - assert event_line == "event: response.failed" - assert failed["type"] == "response.failed" - assert failed["response"]["status"] == "failed" - assert failed["response"]["error"]["code"] == expected_code - assert expected_message in failed["response"]["error"]["message"] + assert collected[0] == TTFT_PING + assert b"data: [DONE]\n\n" not in collected + event_line, data_line = collected[-1].decode().strip().split("\n") + failed: Final = json.loads(data_line.removeprefix("data: ")) + assert event_line == "event: response.failed" + assert failed["type"] == "response.failed" + assert failed["response"]["status"] == "failed" + assert failed["response"]["error"]["code"] == expected_code + assert expected_message in failed["response"]["error"]["message"] + + loop: Final = AutojumpClockLoop() + try: + loop.run_until_complete(collect_and_check()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() @pytest.mark.asyncio From e954b0f49a16b2a13da2f8b294aa4917aedc300f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:57:59 -0700 Subject: [PATCH 06/23] fix(proxy): end late Responses failures with an error event the SDK raises on Errors that arrive after keepalive pings, and guardrail error strings yielded mid-stream, now end /v1/responses with a named `error` event carrying the same error object chat sends, so the OpenAI SDK still raises APIError with a clean message and provider_specific_fields. Background polling passes the same framing and records an `error` event as a failed response. The first data line of a frame is found by splitting on newlines only, so a U+2028 in an error message no longer hides a first-event error. --- litellm/proxy/common_request_processing.py | 17 ++--- .../common_utils/responses_stream_errors.py | 41 +++++++---- litellm/proxy/proxy_server.py | 2 +- .../response_polling/background_streaming.py | 8 ++- .../proxy_server/test_streaming_helpers.py | 31 +++++++++ .../response_api_endpoints/test_endpoints.py | 29 +++++--- .../test_response_polling_handler.py | 34 +++++++++- .../proxy/test_common_request_processing.py | 68 +++++++++++++------ 8 files changed, 180 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index abc99a3be23..6cdab2daa82 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -115,7 +115,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, openai_error_type, ) -from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, sse_error_object from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, SSE_STREAM_START_TAIL, @@ -912,7 +912,7 @@ async def _enforce_guardrail_added_tag_budgets( def _sse_data_payload(event_line: str | bytes) -> str | None: text: Final = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line return next( - (line.removeprefix("data: ").strip() for line in text.splitlines() if line.startswith("data: ")), + (line.removeprefix("data: ").strip() for line in text.split("\n") if line.startswith("data: ")), None, ) @@ -1196,13 +1196,14 @@ def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: - frame: Final = ResponsesStreamErrorState().format_failure(exc) - return () if frame is None else (frame,) + _, error_obj = sse_error_payload(exc) + return (ResponsesStreamErrorState().format_error(error_obj),) def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: - frame: Final = ResponsesStreamErrorState().format_error_body(bytes(produced.body), produced.status_code) - return () if frame is None else (frame.encode(),) + body: Final = bytes(produced.body) + error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode(), "code": str(produced.status_code)} + return (ResponsesStreamErrorState().format_error(error_obj).encode(),) OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames) @@ -3423,8 +3424,8 @@ class ProxyBaseLLMRequestProcessing: Streaming ``/v1/responses`` short-circuits out of ``base_process_llm_request`` before the non-streaming ownership - tail runs. The OpenAI SDK closes the connection at ``data: [DONE]`` - and starlette cancels the body task on disconnect, so a write that + tail runs. The OpenAI SDK can close the connection right after the terminal + event and starlette cancels the body task on disconnect, so a write that waits for the generator to finish never lands. The iterator sets ``completed_response`` before it hands over its terminal chunk, so the ``LiteLLM_ManagedObjectTable`` row is written the moment it diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 312bf472594..132155d9492 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,10 +1,11 @@ +import json import time from collections.abc import Mapping from http import HTTPStatus from types import MappingProxyType from typing import Final -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, ValidationError, field_validator from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid @@ -57,10 +58,17 @@ class _FailureDetails(LiteLLMBaseModel): return cls.model_validate({field: getattr(exception, field, None) for field in cls.model_fields}) -class _ErrorBody(LiteLLMBaseModel): +class _ErrorPayload(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - error: _FailureDetails = _FailureDetails() + error: Mapping[str, object] + + +def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: + try: + return _ErrorPayload.model_validate_json(data).error + except ValidationError: + return None def _original_failure(exception: Exception) -> Exception: @@ -142,17 +150,26 @@ class ResponsesStreamErrorState: prefix: Final = f"event: {event.type}\n" return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame + def format_error(self, error: Mapping[str, object]) -> str: + payload: Final = json.dumps( + { + "type": ResponsesAPIStreamEvents.ERROR.value, + "sequence_number": self.sequence_number + 1, + "error": error, + }, + default=str, + ) + return f"event: error\ndata: {payload}\n\n" + + def format_error_message(self, frame: str) -> str: + error: Final = sse_error_object(frame.removeprefix("data: ")) + return frame if error is None else self.format_error(error) + def format_failure(self, exception: Exception) -> str | None: - original: Final = _original_failure(exception) - return self._format_failed(_failure_details(original), str(original)) - - def format_error_body(self, body: bytes, status_code: int) -> str | None: - error: Final = _ErrorBody.model_validate_json(body or b"{}").error - return self._format_failed(error.model_copy(update={"status_code": status_code}), body.decode()) - - def _format_failed(self, details: _FailureDetails, fallback_message: str) -> str | None: if self.terminal_emitted: return None + original: Final = _original_failure(exception) + details: Final = _failure_details(original) response: Final = ResponsesAPIResponse.model_validate( MappingProxyType( { @@ -165,7 +182,7 @@ class ResponsesStreamErrorState: "error": MappingProxyType( { "code": _response_error_code(details), - "message": redact_internal_details_from_client_message(details.message or fallback_message), + "message": redact_internal_details_from_client_message(details.message or str(original)), } ), } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 15832d81b7f..f6cfc88300c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10619,7 +10619,7 @@ async def async_data_generator( yield (raw_sse_buffer if raw_sse_buffer.endswith(_SSE_FRAME_DELIMITERS) else raw_sse_buffer + "\n\n") if error_message is not None: - yield error_message + yield error_message if error_state is None else error_state.format_error_message(error_message) # Chat Completions streams terminate with data: [DONE]; Responses and Google GenAI (?alt=sse) streams do not. if ends_with_openai_done: yield _OPENAI_STREAM_DONE_FRAME diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 5f6a7432afb..975d700688e 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -19,7 +19,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth -from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus @@ -68,6 +68,7 @@ class _StreamEvent(TypedDict, total=False): delta: ReadOnly[str] part: ReadOnly[object] response: ReadOnly[_TerminalResponse] + error: ReadOnly[_JsonDict] class _StreamEventParser: @@ -164,6 +165,7 @@ async def background_streaming_task( user_api_base=user_api_base, version=version, skip_pre_call_logic=True, + late_framing=RESPONSES_LATE_SSE_FRAMING, ) # Process streaming response following OpenAI events format @@ -325,6 +327,10 @@ async def background_streaming_task( status="in_progress", ) + elif event_type == "error": + terminal_status = "failed" + terminal_error = event.get("error") + elif event_type in ( "response.completed", "response.failed", diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 835411608ed..b01487af5d1 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -146,6 +146,37 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent assert payload["type"] == event_type assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" +@pytest.mark.asyncio +async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event() -> None: + response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[]) + created: Final = ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ) + guardrail_error: Final = {"message": "response withheld", "type": "guardrail_scan_error", "code": 500} + + async def upstream() -> AsyncIterator[object]: + yield created + yield f"data: {json.dumps({'error': guardrail_error})}\n\n" + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True + ) + ] + ) + event_line, data_line = frames[-1].strip().split("\n") + + assert "data: [DONE]" not in "".join(frames) + assert event_line == "event: error" + assert json.loads(data_line.removeprefix("data: ")) == { + "type": "error", + "sequence_number": 1, + "error": guardrail_error, + } + + @pytest.mark.parametrize("binary", [False, True]) @pytest.mark.parametrize( ("event_type", "event_line"), diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 5945352c61e..d31ddaf0adf 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -17,6 +17,7 @@ import respx from fastapi import HTTPException from fastapi.testclient import TestClient from httpx import Response +from starlette.types import Message, Receive, Scope, Send import litellm from litellm.proxy._types import ProxyException @@ -188,10 +189,20 @@ async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeyp upstream_url: Final = "https://streaming.example/v1" is_chat: Final = path == "/v1/chat/completions" - async def slow_rate_limit(_: httpx.Request) -> httpx.Response: - await asyncio.sleep(0.3) + first_ping_sent: Final = asyncio.Event() + + async def rate_limit_after_first_ping(_: httpx.Request) -> httpx.Response: + await first_ping_sent.wait() return httpx.Response(429, json={"error": {"message": message, "type": "rate_limit_error", "code": None}}) + async def app_marking_first_ping(scope: Scope, receive: Receive, send: Send) -> None: + async def send_and_mark(asgi_message: Message) -> None: + await send(asgi_message) + if asgi_message["type"] == "http.response.body" and asgi_message.get("body", b"").startswith(b": ping"): + first_ping_sent.set() + + await app(scope, receive, send_and_mark) + router: Final = litellm.Router( model_list=[ { @@ -212,9 +223,11 @@ async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeyp monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) with respx.mock as transport: transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).mock( - side_effect=slow_rate_limit + side_effect=rate_limit_after_first_ping ) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app_marking_first_ping), base_url="http://testserver" + ) as client: result: Final = await client.post( path, json={ @@ -232,10 +245,10 @@ async def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeyp assert message in json.loads(frames[-2].removeprefix("data: "))["error"]["message"] else: assert "data: [DONE]" not in result.text - assert frames[-1].startswith("event: response.failed\ndata: "), result.text - failed_error: Final = json.loads(frames[-1].split("data: ", 1)[1])["response"]["error"] - assert failed_error["code"] == "rate_limit_exceeded" - assert message in failed_error["message"] + assert frames[-1].startswith("event: error\ndata: "), result.text + error: Final = json.loads(frames[-1].split("data: ", 1)[1])["error"] + assert error["code"] == "429" + assert message in error["message"] _COMPACTION_ITEM: Final = { diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 63e38ce533a..8affc8a6e19 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -21,7 +21,7 @@ from typing import Any, Dict, Optional from unittest.mock import AsyncMock, Mock, patch import pytest -from fastapi import Request +from fastapi import HTTPException, Request from pydantic import ValidationError from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler @@ -1522,6 +1522,38 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["status"] == "failed" assert final_call.kwargs["error"] == error_payload + @pytest.mark.asyncio + async def test_a_late_failure_after_keepalive_pings_sets_failed_status_and_error(self): + from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + late_frames = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited")) + + async def _body_iterator(): + yield b": ping\n\n" + for frame in late_frames: + yield frame.encode() + + mock_response = Mock() + mock_response.body_iterator = _body_iterator() + handler = AsyncMock(spec=ResponsePollingHandler) + kwargs = _make_background_streaming_kwargs("poll_late_failure", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"]["message"] == "rate limited" + assert final_call.kwargs["error"]["code"] == "429" + @pytest.mark.asyncio async def test_response_incomplete_sets_incomplete_status_and_details(self): """Test that a response.incomplete stream event results in incomplete status""" diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 310b526365f..ecc3ff860d8 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -5,7 +5,7 @@ import datetime import json from collections.abc import AsyncIterable, Mapping from types import MappingProxyType, SimpleNamespace -from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence +from typing import AsyncGenerator, Awaitable, Callable, Final, Iterator, Literal, Optional, Sequence from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch @@ -2174,6 +2174,19 @@ class TestCommonRequestProcessingHelpers: assert response.status_code == 400 assert json.loads(response.body) == {"error": error} + async def test_a_first_event_error_with_a_unicode_line_separator_in_its_message_answers_as_json(self): + error: Final = {"type": "guardrail_error", "code": "400", "message": "blocked\u2028by policy", "param": None} + + async def responses_stream(): + event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False) + yield f"event: error\ndata: {event}\n\n" + + response: Final = await create_response(responses_stream(), "text/event-stream", {}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 400 + assert json.loads(response.body) == {"error": error} + async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): yield 'data: {"content": "first part"}\n\n' @@ -10391,41 +10404,58 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" -async def _slow_rate_limit_failure(): - await asyncio.sleep(0.2) +async def _rate_limit_failure_once_released(released: asyncio.Event) -> JSONResponse: + await released.wait() raise HTTPException(status_code=429, detail="rate limited") -async def _slow_guardrail_block_body(): - await asyncio.sleep(0.2) +async def _guardrail_block_body_once_released(released: asyncio.Event) -> JSONResponse: + await released.wait() return JSONResponse(status_code=400, content={"error": {"message": "blocked", "type": "guardrail_error", "code": "400"}}) +_POLICY_BLOCK_DETAIL: Final = MappingProxyType( + {"error": {"message": "blocked by policy", "type": "guardrail_error"}, "guardrail": "content-filter"} +) + + +async def _dict_detail_failure_once_released(released: asyncio.Event) -> JSONResponse: + await released.wait() + raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL)) + + @pytest.mark.asyncio @pytest.mark.parametrize( - "produce,expected_code,expected_message", + "produce,expected_error", [ - (_slow_rate_limit_failure, "rate_limit_exceeded", "rate limited"), - (_slow_guardrail_block_body, "invalid_request_error", "blocked"), + (_rate_limit_failure_once_released, {"message": "rate limited", "code": "429"}), + (_guardrail_block_body_once_released, {"message": "blocked", "type": "guardrail_error", "code": "400"}), + ( + _dict_detail_failure_once_released, + {"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)}, + ), ], ) -async def test_ttft_keepalive_ends_a_late_responses_failure_with_response_failed_and_no_done( - produce, expected_code, expected_message -): +async def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done( + produce: Callable[[asyncio.Event], Awaitable[JSONResponse]], expected_error: Mapping[str, object] +) -> None: + released: Final = asyncio.Event() response: Final = await open_sse_before_first_byte( - produce(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + produce(released), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING ) - collected: Final = await _drain(response) + assert isinstance(response, StreamingResponse) + frames: Final = aiter(response.body_iterator) + first_frame: Final = await anext(frames) + released.set() + collected: Final = [first_frame, *[frame async for frame in frames]] assert collected[0] == TTFT_PING assert b"data: [DONE]\n\n" not in collected event_line, data_line = collected[-1].decode().strip().split("\n") - failed: Final = json.loads(data_line.removeprefix("data: ")) - assert event_line == "event: response.failed" - assert failed["type"] == "response.failed" - assert failed["response"]["status"] == "failed" - assert failed["response"]["error"]["code"] == expected_code - assert expected_message in failed["response"]["error"]["message"] + event: Final = json.loads(data_line.removeprefix("data: ")) + assert event_line == "event: error" + assert event["type"] == "error" + assert {key: event["error"].get(key) for key in expected_error} == expected_error @pytest.mark.asyncio From b2f3614faf87b357d2a2d896157d6412a6a430d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:15:41 -0700 Subject: [PATCH 07/23] fix(proxy): match OpenAI's Responses error event and stream late passthrough blocks as block responses --- litellm/proxy/common_request_processing.py | 5 ++- .../common_utils/responses_stream_errors.py | 10 +++++ .../response_polling/background_streaming.py | 2 +- .../proxy_server/test_streaming_helpers.py | 3 ++ .../test_response_polling_handler.py | 15 +++---- .../proxy/test_common_request_processing.py | 43 +++++++++++++++++-- 6 files changed, 64 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6cdab2daa82..10a5c6d61ee 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -51,7 +51,7 @@ from litellm.constants import ( STREAM_SSE_KEEPALIVE_PING_BYTES, UNSAFE_PROXY_RESPONSE_HEADERS, ) -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.bug_report import ( allowlisted, @@ -89,6 +89,7 @@ from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields +from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, @@ -1196,6 +1197,8 @@ def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: + if isinstance(exc, ModifyResponseException): + return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc)) _, error_obj = sse_error_payload(exc) return (ResponsesStreamErrorState().format_error(error_obj),) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 132155d9492..05ec7603a3d 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -109,6 +109,12 @@ def _status_error_code(status_code: int | None) -> str: return _CLIENT_ERROR_CODES.get(status_code, "invalid_request_error") +def _status_of(code: str | int | None) -> int | None: + if isinstance(code, int): + return code + return int(code) if code is not None and code.isdecimal() else None + + def _response_error_code(details: _FailureDetails) -> str: for value in (details.code, details.type): if value == "insufficient_quota": @@ -151,9 +157,13 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_error(self, error: Mapping[str, object]) -> str: + details: Final = _FailureDetails.model_validate(error) payload: Final = json.dumps( { "type": ResponsesAPIStreamEvents.ERROR.value, + "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), + "message": details.message, + "param": error.get("param"), "sequence_number": self.sequence_number + 1, "error": error, }, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 975d700688e..7411e204d90 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -329,7 +329,7 @@ async def background_streaming_task( elif event_type == "error": terminal_status = "failed" - terminal_error = event.get("error") + terminal_error = {"code": event.get("code"), "message": event.get("message")} elif event_type in ( "response.completed", diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index b01487af5d1..fb8ff519fbc 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -172,6 +172,9 @@ async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_ assert event_line == "event: error" assert json.loads(data_line.removeprefix("data: ")) == { "type": "error", + "code": "server_error", + "message": "response withheld", + "param": None, "sequence_number": 1, "error": guardrail_error, } diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 8affc8a6e19..f398d5acb1c 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -17,7 +17,7 @@ following the OpenAI Response API format. import asyncio import json from datetime import datetime, timezone -from typing import Any, Dict, Optional +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, Mock, patch import pytest @@ -1529,17 +1529,17 @@ class TestBackgroundStreamingTerminalEvents: background_streaming_task, ) - late_frames = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited")) + late_frames: Final = RESPONSES_LATE_SSE_FRAMING.failure(HTTPException(status_code=429, detail="rate limited")) async def _body_iterator(): yield b": ping\n\n" for frame in late_frames: yield frame.encode() - mock_response = Mock() + mock_response: Final = Mock() mock_response.body_iterator = _body_iterator() - handler = AsyncMock(spec=ResponsePollingHandler) - kwargs = _make_background_streaming_kwargs("poll_late_failure", handler) + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_late_failure", handler) with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" @@ -1549,10 +1549,9 @@ class TestBackgroundStreamingTerminalEvents: ) await background_streaming_task(**kwargs) - final_call = handler.update_state.call_args_list[-1] + final_call: Final = handler.update_state.call_args_list[-1] assert final_call.kwargs["status"] == "failed" - assert final_call.kwargs["error"]["message"] == "rate limited" - assert final_call.kwargs["error"]["code"] == "429" + assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"} @pytest.mark.asyncio async def test_response_incomplete_sets_incomplete_status_and_details(self): diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 944432f91b1..63324c11930 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -34,6 +34,7 @@ from litellm.constants import ( STREAM_SSE_KEEPALIVE_PING_BYTES, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -10428,18 +10429,23 @@ async def _slow_dict_detail_failure() -> Response: @pytest.mark.parametrize( - "produce,expected_error", + "produce,expected_code,expected_error", [ - (_slow_rate_limit_failure, {"message": "rate limited", "code": "429"}), - (_slow_guardrail_block_body, {"message": "blocked", "type": "guardrail_error", "code": "400"}), + (_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}), + ( + _slow_guardrail_block_body, + "invalid_request_error", + {"message": "blocked", "type": "guardrail_error", "code": "400"}, + ), ( _slow_dict_detail_failure, + "invalid_request_error", {"message": "blocked by policy", "code": "400", "provider_specific_fields": dict(_POLICY_BLOCK_DETAIL)}, ), ], ) def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no_done( - produce: Callable[[], Awaitable[Response]], expected_error: Mapping[str, object] + produce: Callable[[], Awaitable[Response]], expected_code: str, expected_error: Mapping[str, object] ) -> None: async def collect_and_check() -> None: response: Final = await open_sse_before_first_byte( @@ -10453,6 +10459,7 @@ def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no event: Final = json.loads(data_line.removeprefix("data: ")) assert event_line == "event: error" assert event["type"] == "error" + assert (event["code"], event["message"]) == (expected_code, expected_error["message"]) assert {key: event["error"].get(key) for key in expected_error} == expected_error loop: Final = AutojumpClockLoop() @@ -10463,6 +10470,34 @@ def test_ttft_keepalive_ends_a_late_responses_failure_with_an_error_event_and_no loop.close() +async def _slow_passthrough_block() -> Response: + await asyncio.sleep(0.2) + raise ModifyResponseException(message="blocked by policy", model="gpt-4o", request_data={}) + + +def test_ttft_keepalive_ends_a_late_responses_passthrough_block_with_the_block_response() -> None: + async def collect_and_check() -> None: + response: Final = await open_sse_before_first_byte( + _slow_passthrough_block(), ping_interval_seconds=0.05, late_framing=RESPONSES_LATE_SSE_FRAMING + ) + collected: Final = await _drain(response) + frames: Final = tuple(chunk.decode() for chunk in collected if chunk != TTFT_PING) + names: Final = tuple(frame.split("\n", 1)[0] for frame in frames) + completed: Final = json.loads(frames[-1].split("\n", 1)[1].removeprefix("data: ")) + + assert collected[0] == TTFT_PING + assert (names[0], names[-1]) == ("event: response.created", "event: response.completed") + assert "event: error" not in names + assert completed["response"]["output"][0]["content"][0]["text"] == "blocked by policy" + + loop: Final = AutojumpClockLoop() + try: + loop.run_until_complete(collect_and_check()) + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + @pytest.mark.asyncio async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed(): """Starlette never calls the produced response, so its own cleanup never runs From f343c2421a3554a43720420b3b7d104a7103986d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:01:53 -0700 Subject: [PATCH 08/23] fix(proxy): keep the reason when a background Responses request fails A background poll of a request a post-call guardrail blocked saved a null code and message, because the guardrail's error event nests them. A request whose first chunk was an error polled as completed with no error. Both now poll failed with the same code and message a late failure gets. --- .../common_utils/responses_stream_errors.py | 12 ++- .../response_polling/background_streaming.py | 21 +++-- .../proxy/response_polling/polling_handler.py | 2 +- .../test_response_polling_handler.py | 90 +++++++++++++++++++ 4 files changed, 116 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 05ec7603a3d..c730a94dc62 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -126,6 +126,14 @@ def _response_error_code(details: _FailureDetails) -> str: return _status_error_code(details.status_code) +def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]: + details: Final = _FailureDetails.model_validate(error) + return { + "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), + "message": details.message, + } + + class ResponsesStreamErrorState: def __init__(self) -> None: self.response_id: str | None = None @@ -157,12 +165,10 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_error(self, error: Mapping[str, object]) -> str: - details: Final = _FailureDetails.model_validate(error) payload: Final = json.dumps( { "type": ResponsesAPIStreamEvents.ERROR.value, - "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), - "message": details.message, + **responses_error_fields(error), "param": error.get("param"), "sequence_number": self.sequence_number + 1, "error": error, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index 7411e204d90..ecf9bea4ab7 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -11,6 +11,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming import asyncio import json from collections.abc import AsyncIterable, Callable, Mapping, Sequence +from http import HTTPStatus from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, runtime_checkable from fastapi import Request, Response @@ -20,6 +21,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.responses_stream_errors import responses_error_fields, sse_error_object from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus @@ -85,6 +87,14 @@ def _sse_frame_data(frame: str) -> str | None: return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None) +def _first_chunk_error(response: object) -> Mapping[str, object] | None: + if isinstance(response, _StreamsBody) or not isinstance(response, Response): + return None + if response.status_code < HTTPStatus.BAD_REQUEST: + return None + return responses_error_fields(sse_error_object(bytes(response.body)) or {"code": response.status_code}) + + async def _never_receive() -> Message: await asyncio.Event().wait() raise AssertionError("unreachable") @@ -329,7 +339,7 @@ async def background_streaming_task( elif event_type == "error": terminal_status = "failed" - terminal_error = {"code": event.get("code"), "message": event.get("message")} + terminal_error = responses_error_fields(event.get("error") or event) elif event_type in ( "response.completed", @@ -387,14 +397,15 @@ async def background_streaming_task( # Final flush to ensure all accumulated state is saved await flush_state_if_needed(force=True) - # Use the terminal status from the stream, default to "completed" - final_status: Final = terminal_status or "completed" + early_error: Final = _first_chunk_error(response) + final_error: Final = terminal_error if early_error is None else early_error + final_status: Final = "failed" if early_error is not None else terminal_status or "completed" await polling_handler.update_state( polling_id=polling_id, status=final_status, usage=usage_data, - error=terminal_error, + error=final_error, reasoning=reasoning_data, tool_choice=tool_choice_data, tools=tools_data, @@ -416,7 +427,7 @@ async def background_streaming_task( "Finished background streaming for %s, status=%s, error=%s, incomplete_details=%s, output_items=%s", polling_id, final_status, - terminal_error, + final_error, incomplete_details_data, len(output_items), ) diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index e50587b1a0e..158b1f0f63b 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -98,7 +98,7 @@ class ResponsePollingHandler: polling_id: str, status: ResponsesAPIStatus | None = None, usage: dict | None = None, - error: dict | None = None, + error: Mapping[str, object] | None = None, incomplete_details: dict | None = None, reasoning: dict | None = None, tool_choice: object | None = None, diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index f398d5acb1c..6cb9376c0e5 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1553,6 +1553,96 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["status"] == "failed" assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"} + @pytest.mark.asyncio + async def test_a_nested_only_error_event_keeps_its_reason(self): + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + nested_error: Final = {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"} + error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error} + + async def _body_iterator(): + yield b'data: {"type": "response.in_progress"}\n\n' + yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode() + + mock_response: Final = Mock() + mock_response.body_iterator = _body_iterator() + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_nested_error", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=mock_response + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"] == {"code": "invalid_request_error", "message": "output blocked by policy"} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("status_code", "body", "expected_error"), + [ + ( + 404, + json.dumps( + {"error": {"message": "model not found", "type": "invalid_request_error", "param": None, "code": "404"}} + ), + {"code": "not_found_error", "message": "model not found"}, + ), + (429, "Too Many Requests", {"code": "rate_limit_exceeded", "message": None}), + ], + ) + async def test_a_first_chunk_error_polls_failed_like_a_late_one(self, status_code, body, expected_error): + from fastapi import Response + + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + first_chunk_error: Final = Response(status_code=status_code, content=body) + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_first_chunk_error", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=first_chunk_error + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "failed" + assert final_call.kwargs["error"] == expected_error + + @pytest.mark.asyncio + async def test_a_non_streamed_success_response_still_polls_completed(self): + from fastapi.responses import JSONResponse + + from litellm.proxy.response_polling.background_streaming import ( + background_streaming_task, + ) + + handler: Final = AsyncMock(spec=ResponsePollingHandler) + kwargs: Final = _make_background_streaming_kwargs("poll_non_streamed_success", handler) + + with patch( # test-quality-ok: the processor is built inside the task, same idiom as the sibling tests + "litellm.proxy.response_polling.background_streaming.ProxyBaseLLMRequestProcessing" + ) as MockProcessor: + MockProcessor.return_value.base_process_llm_request = AsyncMock( + return_value=JSONResponse(status_code=200, content={"id": "resp_123", "status": "completed"}) + ) + await background_streaming_task(**kwargs) + + final_call: Final = handler.update_state.call_args_list[-1] + assert final_call.kwargs["status"] == "completed" + assert final_call.kwargs["error"] is None + @pytest.mark.asyncio async def test_response_incomplete_sets_incomplete_status_and_details(self): """Test that a response.incomplete stream event results in incomplete status""" From e6290016c1e7b17e91404bf1167767d3c3dbc0d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:00:03 -0700 Subject: [PATCH 09/23] test(proxy): type the first-chunk poll test parameters --- .../proxy/response_polling/test_response_polling_handler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 6cb9376c0e5..830e3f2585c 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -16,6 +16,7 @@ following the OpenAI Response API format. import asyncio import json +from collections.abc import Mapping from datetime import datetime, timezone from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, Mock, patch @@ -1597,7 +1598,9 @@ class TestBackgroundStreamingTerminalEvents: (429, "Too Many Requests", {"code": "rate_limit_exceeded", "message": None}), ], ) - async def test_a_first_chunk_error_polls_failed_like_a_late_one(self, status_code, body, expected_error): + async def test_a_first_chunk_error_polls_failed_like_a_late_one( + self, status_code: int, body: str, expected_error: Mapping[str, str | None] + ): from fastapi import Response from litellm.proxy.response_polling.background_streaming import ( From 4bfca9cfd484e41aa9950e07bb8c68434136f712 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:33:43 -0700 Subject: [PATCH 10/23] fix(proxy): keep first-chunk error detection to lone error frames and handle string poll errors --- litellm/proxy/common_request_processing.py | 43 ++++++++--- .../common_utils/responses_stream_errors.py | 19 ++++- .../response_polling/background_streaming.py | 16 +++- .../response_api_endpoints/test_endpoints.py | 76 ++++++++++++++++++- .../test_response_polling_handler.py | 26 +++++-- .../proxy/test_common_request_processing.py | 9 ++- 6 files changed, 166 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 10a5c6d61ee..bf321d5cd20 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -910,17 +910,31 @@ async def _enforce_guardrail_added_tag_budgets( ) from e +SseDataPayload: TypeAlias = Callable[[str | bytes], str | None] + + +def _sse_text(event_line: str | bytes) -> str: + return event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line + + def _sse_data_payload(event_line: str | bytes) -> str | None: - text: Final = event_line.decode("utf-8") if isinstance(event_line, bytes) else event_line - return next( - (line.removeprefix("data: ").strip() for line in text.split("\n") if line.startswith("data: ")), - None, - ) + text: Final = _sse_text(event_line) + return text.removeprefix("data: ").strip() if text.startswith("data: ") else None -async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: +def _responses_event_data_payload(event_line: str | bytes) -> str | None: + match _sse_text(event_line).strip().split("\n"): + case ["event: error", data_line] if data_line.startswith("data: "): + return data_line.removeprefix("data: ").strip() + case _: + return _sse_data_payload(event_line) + + +async def _parse_event_data_for_error( + event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload +) -> int | None: """Parses an event line and returns an error code if present, else None.""" - json_str: Final = _sse_data_payload(event_line) + json_str: Final = data_payload(event_line) if json_str is not None: if not json_str or json_str == "[DONE]": # handle empty data or [DONE] message return None @@ -952,12 +966,13 @@ async def _parse_event_data_for_error(event_line: str | bytes) -> int | None: return None -def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: +def _extract_error_from_sse_chunk(event_line: str | bytes, data_payload: SseDataPayload = _sse_data_payload) -> dict: """ Extract error dictionary from SSE format chunk. Args: event_line: SSE format event line, e.g. "data: {"error": {...}}\n\n" + data_payload: Reads the JSON payload out of the event line Returns: Error dictionary in OpenAI API format @@ -970,7 +985,7 @@ def _extract_error_from_sse_chunk(event_line: str | bytes) -> dict: "code": "500", } - json_str: Final = _sse_data_payload(event_line) + json_str: Final = data_payload(event_line) if json_str is not None: if not json_str or json_str == "[DONE]": return default_error @@ -1243,6 +1258,7 @@ async def create_response( request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, + first_chunk_payload: SseDataPayload = _sse_data_payload, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -1278,7 +1294,7 @@ async def create_response( if first_chunk_value is not None: try: - error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value) + error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value, first_chunk_payload) if error_code_from_chunk is not None: # First chunk is an error, stream hasn't really started yet # Should return standard JSON error response instead of SSE format @@ -1290,7 +1306,9 @@ async def create_response( # Parse error content error_dict: Final = with_call_id( - JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)), + JSON_OBJECT.validate_python( + _extract_error_from_sse_chunk(first_chunk_value, first_chunk_payload) + ), error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), ) @@ -3217,6 +3235,9 @@ class ProxyBaseLLMRequestProcessing: request=request, refresh_headers=refresh_stream_headers, general_settings=general_settings, + first_chunk_payload=( + _responses_event_data_payload if route_type == "aresponses" else _sse_data_payload + ), ) ### CALL HOOKS ### - modify outgoing data diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index c730a94dc62..560365a4cdb 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -71,6 +71,13 @@ def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: return None +def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None: + try: + return _ErrorPayload.model_validate(event).error or None + except ValidationError: + return None + + def _original_failure(exception: Exception) -> Exception: current = exception # rebind-ok: the recursion gate requires iterative wrapper traversal while isinstance(current, MidStreamFallbackError) and current.original_exception is not None: @@ -126,11 +133,19 @@ def _response_error_code(details: _FailureDetails) -> str: return _status_error_code(details.status_code) +_STATUS_PHRASES: Final = MappingProxyType({int(status): status.phrase for status in HTTPStatus}) + + +def _status_phrase(status_code: int | None) -> str: + return _STATUS_PHRASES.get(status_code, "Unknown error") if status_code is not None else "Unknown error" + + def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]: details: Final = _FailureDetails.model_validate(error) + status_code: Final = _status_of(details.code) return { - "code": _response_error_code(details.model_copy(update={"status_code": _status_of(details.code)})), - "message": details.message, + "code": _response_error_code(details.model_copy(update={"status_code": status_code})), + "message": details.message or _status_phrase(status_code), } diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index ecf9bea4ab7..0ddedd52fb6 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -21,7 +21,11 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing -from litellm.proxy.common_utils.responses_stream_errors import responses_error_fields, sse_error_object +from litellm.proxy.common_utils.responses_stream_errors import ( + event_error_object, + responses_error_fields, + sse_error_object, +) from litellm.proxy.response_polling.polling_handler import ResponsePollingHandler from litellm.types.llms.openai import ResponsesAPIStatus @@ -87,6 +91,14 @@ def _sse_frame_data(frame: str) -> str | None: return next((line[6:].strip() for line in frame.splitlines() if line.startswith("data: ")), None) +def _event_error(event: Mapping[str, object]) -> Mapping[str, object]: + match event.get("error"): + case str() as message if message: + return {**event, "message": message} + case _: + return event_error_object(event) or event + + def _first_chunk_error(response: object) -> Mapping[str, object] | None: if isinstance(response, _StreamsBody) or not isinstance(response, Response): return None @@ -339,7 +351,7 @@ async def background_streaming_task( elif event_type == "error": terminal_status = "failed" - terminal_error = responses_error_fields(event.get("error") or event) + terminal_error = responses_error_fields(_event_error(event)) elif event_type in ( "response.completed", diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 4a04d33cc42..a510799cf37 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,7 +7,7 @@ import contextlib import dataclasses import json import unittest -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping from typing import Any, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, call, patch @@ -19,6 +19,7 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import app from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse @@ -178,6 +179,79 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] +class _FirstFrameHook(CustomLogger): + def __init__(self, frame: str | bytes) -> None: + super().__init__() + self.frame: Final = frame + + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict: object, response: object, request_data: object + ) -> AsyncIterator[str | bytes]: + yield self.frame + + +_DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n' +_NAMED_ERROR_FRAME: Final = ( + b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "denied", "code": "529"}}\n\n' +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,first_frame,expected_status", + [ + ("/v1/responses", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _NAMED_ERROR_FRAME, 200), + ("/v1/responses", _NAMED_ERROR_FRAME, 529), + ("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200), + ("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ], +) +async def test_only_a_lone_first_error_frame_answers_with_its_status( + monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-6-astra" + upstream_url: Final = "https://streaming.example/v1" + is_chat: Final = path == "/v1/chat/completions" + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"}, + } + ], + num_retries=0, + ) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first_frame)]) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond( + 200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"} + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + path, + json={ + "model": model, + "stream": True, + **({"messages": [{"role": "user", "content": "hello"}]} if is_chat else {"input": "hello"}), + }, + ) + + assert result.status_code == expected_status, result.text + if expected_status == 200: + assert result.headers["content-type"].startswith("text/event-stream"), result.text + else: + assert result.json()["error"]["message"] == "denied" + + @pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None: async def request_and_check() -> None: diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 830e3f2585c..d97cd73e6ed 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1555,12 +1555,23 @@ class TestBackgroundStreamingTerminalEvents: assert final_call.kwargs["error"] == {"code": "rate_limit_exceeded", "message": "rate limited"} @pytest.mark.asyncio - async def test_a_nested_only_error_event_keeps_its_reason(self): + @pytest.mark.parametrize( + ("nested_error", "expected_error"), + [ + ( + {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"}, + {"code": "invalid_request_error", "message": "output blocked by policy"}, + ), + ("output blocked by policy", {"code": "server_error", "message": "output blocked by policy"}), + ], + ) + async def test_a_nested_only_error_event_keeps_its_reason( + self, nested_error: Mapping[str, str] | str, expected_error: Mapping[str, str] + ): from litellm.proxy.response_polling.background_streaming import ( background_streaming_task, ) - nested_error: Final = {"type": "guardrail_error", "code": "400", "message": "output blocked by policy"} error_event: Final = {"type": "error", "sequence_number": 9, "error": nested_error} async def _body_iterator(): @@ -1582,7 +1593,7 @@ class TestBackgroundStreamingTerminalEvents: final_call: Final = handler.update_state.call_args_list[-1] assert final_call.kwargs["status"] == "failed" - assert final_call.kwargs["error"] == {"code": "invalid_request_error", "message": "output blocked by policy"} + assert final_call.kwargs["error"] == expected_error @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1595,11 +1606,16 @@ class TestBackgroundStreamingTerminalEvents: ), {"code": "not_found_error", "message": "model not found"}, ), - (429, "Too Many Requests", {"code": "rate_limit_exceeded", "message": None}), + (429, "rate limited", {"code": "rate_limit_exceeded", "message": "Too Many Requests"}), + ( + 403, + json.dumps({"error": {"type": "guardrail_scan_error", "code": 403}}), + {"code": "permission_error", "message": "Forbidden"}, + ), ], ) async def test_a_first_chunk_error_polls_failed_like_a_late_one( - self, status_code: int, body: str, expected_error: Mapping[str, str | None] + self, status_code: int, body: str, expected_error: Mapping[str, str] ): from fastapi import Response diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 63324c11930..a2a7deb3931 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -49,6 +49,7 @@ from litellm.proxy.common_request_processing import ( _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, + _responses_event_data_payload, include_guardrail_response_requested, is_azure_model_router_request, RESPONSES_LATE_SSE_FRAMING, @@ -2170,7 +2171,9 @@ class TestCommonRequestProcessingHelpers: async def responses_stream(): yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" - response: Final = await create_response(responses_stream(), "text/event-stream", {}) + response: Final = await create_response( + responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + ) assert isinstance(response, JSONResponse) assert response.status_code == 400 @@ -2183,7 +2186,9 @@ class TestCommonRequestProcessingHelpers: event: Final = json.dumps({"type": "error", "sequence_number": 0, "error": error}, ensure_ascii=False) yield f"event: error\ndata: {event}\n\n" - response: Final = await create_response(responses_stream(), "text/event-stream", {}) + response: Final = await create_response( + responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + ) assert isinstance(response, JSONResponse) assert response.status_code == 400 From 86360077c637cd915f4bee9e375ddc7f233218c8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:57:45 -0700 Subject: [PATCH 11/23] fix(proxy): keep the HTTP status on late Responses errors and end Responses streams on error events A late error body without a code now takes the response's status, an upstream error event counts as the stream's terminal event, and a Responses stream that fails before its first chunk ends with an error event instead of data: [DONE]. --- litellm/proxy/common_request_processing.py | 40 ++++++++++++++----- .../common_utils/responses_stream_errors.py | 2 +- .../proxy_server/test_streaming_helpers.py | 27 +++++++++++++ .../proxy/test_common_request_processing.py | 38 ++++++++++++++++-- 4 files changed, 93 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 350f4d6cb9c..27164ba0c83 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1196,6 +1196,10 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]: return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n" +def _responses_error_frames(error_obj: Mapping[str, object]) -> tuple[str, ...]: + return (ResponsesStreamErrorState().format_error(error_obj),) + + @dataclass(frozen=True, slots=True) class LateSseFraming: """How a response the keepalive already opened ends once its upstream fails late.""" @@ -1216,14 +1220,14 @@ def _openai_late_body_frames(produced: Response) -> tuple[bytes, ...]: def _responses_late_failure_frames(exc: Exception) -> tuple[str, ...]: if isinstance(exc, ModifyResponseException): return tuple(chunk.decode() for chunk in OpenAIResponsesHandler().build_block_sse_chunks(exc)) - _, error_obj = sse_error_payload(exc) - return (ResponsesStreamErrorState().format_error(error_obj),) + return _responses_error_frames(sse_error_payload(exc)[1]) def _responses_late_body_frames(produced: Response) -> tuple[bytes, ...]: body: Final = bytes(produced.body) - error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode(), "code": str(produced.status_code)} - return (ResponsesStreamErrorState().format_error(error_obj).encode(),) + error_obj: Final = sse_error_object(body or b"{}") or {"message": body.decode()} + with_status: Final = {**error_obj, "code": error_obj.get("code") or str(produced.status_code)} + return tuple(frame.encode() for frame in _responses_error_frames(with_status)) OPENAI_LATE_SSE_FRAMING: Final = LateSseFraming(failure=_openai_late_failure_frames, body=_openai_late_body_frames) @@ -1232,6 +1236,20 @@ RESPONSES_LATE_SSE_FRAMING: Final = LateSseFraming( ) +@dataclass(frozen=True, slots=True) +class FirstChunkSseFraming: + """How a stream's first chunk is read for an error, and how the stream ends when reading it fails.""" + + data_payload: SseDataPayload + error_frames: Callable[[Mapping[str, object]], tuple[str, ...]] + + +OPENAI_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming(data_payload=_sse_data_payload, error_frames=_sse_error_frames) +RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming( + data_payload=_responses_event_data_payload, error_frames=_responses_error_frames +) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -1260,7 +1278,7 @@ async def create_response( request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, - first_chunk_payload: SseDataPayload = _sse_data_payload, + framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -1296,7 +1314,9 @@ async def create_response( if first_chunk_value is not None: try: - error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value, first_chunk_payload) + error_code_from_chunk: Final = await _parse_event_data_for_error( + first_chunk_value, framing.data_payload + ) if error_code_from_chunk is not None: # First chunk is an error, stream hasn't really started yet # Should return standard JSON error response instead of SSE format @@ -1309,7 +1329,7 @@ async def create_response( # Parse error content error_dict: Final = with_call_id( JSON_OBJECT.validate_python( - _extract_error_from_sse_chunk(first_chunk_value, first_chunk_payload) + _extract_error_from_sse_chunk(first_chunk_value, framing.data_payload) ), error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), ) @@ -1352,7 +1372,7 @@ async def create_response( error_status, error_obj = sse_error_payload(e) async def error_gen_message() -> AsyncGenerator[str, None]: - for frame in _sse_error_frames(error_obj): + for frame in framing.error_frames(error_obj): yield frame return StreamingResponse( @@ -3286,8 +3306,8 @@ class ProxyBaseLLMRequestProcessing: request=request, refresh_headers=refresh_stream_headers, general_settings=general_settings, - first_chunk_payload=( - _responses_event_data_payload if route_type == "aresponses" else _sse_data_payload + framing=( + RESPONSES_FIRST_CHUNK_FRAMING if route_type == "aresponses" else OPENAI_FIRST_CHUNK_FRAMING ), ) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 560365a4cdb..4a806c32284 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -172,7 +172,7 @@ class ResponsesStreamErrorState: self.model = event.response.model or self.model if event.response.created_at is not None: self.created_at = event.response.created_at - if event.type in ("response.completed", "response.failed", "response.incomplete"): + if event.type in ("response.completed", "response.failed", "response.incomplete", "error"): self.terminal_emitted = True if not event.type or "\n" in event.type or "\r" in event.type: return frame diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index fb8ff519fbc..93eb8fa226b 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -146,6 +146,33 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent assert payload["type"] == event_type assert json.loads(frames[1].split("data: ", 1)[1])["delta"] == "hello" + +@pytest.mark.asyncio +async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event() -> None: + response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[]) + created: Final = ResponseCreatedEvent.model_validate( + {"type": "response.created", "sequence_number": 0, "response": response} + ) + error_event: Final = ErrorEvent.model_validate( + {"type": "error", "sequence_number": 1, "code": "server_error", "message": "upstream failed", "param": None} + ) + + async def upstream() -> AsyncIterator[BaseModel]: + yield created + yield error_event + raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model") + + frames: Final = tuple( + [ + frame.decode() if isinstance(frame, bytes) else frame + async for frame in select_data_generator( + response=upstream(), user_api_key_dict=_user_auth(), request_data={}, responses_stream_errors=True + ) + ] + ) + + assert tuple(frame.split("\n", 1)[0] for frame in frames) == ("event: response.created", "event: error") + @pytest.mark.asyncio async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event() -> None: response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[]) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 1b1c7ce1ccb..3f8ff1c9b14 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -50,7 +50,8 @@ from litellm.proxy.common_request_processing import ( _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, - _responses_event_data_payload, + OPENAI_FIRST_CHUNK_FRAMING, + RESPONSES_FIRST_CHUNK_FRAMING, include_guardrail_response_requested, is_azure_model_router_request, RESPONSES_LATE_SSE_FRAMING, @@ -2250,6 +2251,14 @@ class TestProxyBaseLLMRequestProcessing: ) +def _fail_stream_setup() -> str: + raise RuntimeError("stream setup failed") + + +async def _stream_failing_before_its_first_chunk() -> AsyncGenerator[str, None]: + yield _fail_stream_setup() + + @pytest.mark.asyncio class TestCommonRequestProcessingHelpers: async def consume_stream(self, streaming_response: StreamingResponse) -> list: @@ -2335,7 +2344,7 @@ class TestCommonRequestProcessingHelpers: yield f"event: error\ndata: {json.dumps({'type': 'error', 'sequence_number': 0, 'error': error})}\n\n" response: Final = await create_response( - responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING ) assert isinstance(response, JSONResponse) @@ -2350,13 +2359,30 @@ class TestCommonRequestProcessingHelpers: yield f"event: error\ndata: {event}\n\n" response: Final = await create_response( - responses_stream(), "text/event-stream", {}, first_chunk_payload=_responses_event_data_payload + responses_stream(), "text/event-stream", {}, framing=RESPONSES_FIRST_CHUNK_FRAMING ) assert isinstance(response, JSONResponse) assert response.status_code == 400 assert json.loads(response.body) == {"error": error} + @pytest.mark.parametrize( + "framing,frame_starts", + [ + (OPENAI_FIRST_CHUNK_FRAMING, ('data: {"error": ', "data: [DONE]")), + (RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)), + ], + ) + async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(self, framing, frame_starts): + response: Final = await create_response( + _stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing + ) + frames: Final = await self.consume_stream(response) + + assert response.status_code == 500 + assert len(frames) == len(frame_starts) + assert all(frame.startswith(start) for frame, start in zip(frames, frame_starts, strict=True)) + async def test_create_streaming_response_first_chunk_not_error(self): async def mock_generator(): yield 'data: {"content": "first part"}\n\n' @@ -10591,6 +10617,11 @@ _POLICY_BLOCK_DETAIL: Final = MappingProxyType( ) +async def _slow_codeless_rate_limit_body() -> Response: + await asyncio.sleep(0.2) + return JSONResponse(status_code=429, content={"error": {"message": "slow down", "code": None}}) + + async def _slow_dict_detail_failure() -> Response: await asyncio.sleep(0.2) raise HTTPException(status_code=400, detail=dict(_POLICY_BLOCK_DETAIL)) @@ -10600,6 +10631,7 @@ async def _slow_dict_detail_failure() -> Response: "produce,expected_code,expected_error", [ (_slow_rate_limit_failure, "rate_limit_exceeded", {"message": "rate limited", "code": "429"}), + (_slow_codeless_rate_limit_body, "rate_limit_exceeded", {"message": "slow down", "code": "429"}), ( _slow_guardrail_block_body, "invalid_request_error", From 5003ef1f31e68fbadc95974655f79d18a24118cf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:20:31 -0700 Subject: [PATCH 12/23] test(proxy): type the first-chunk failure test's parameters --- tests/unit/proxy/test_common_request_processing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 3f8ff1c9b14..7e0eedc388d 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -47,6 +47,7 @@ from litellm.proxy.common_request_processing import ( _ClientDisconnectedBeforeFirstChunk, attach_guardrail_information, _extract_error_from_sse_chunk, + FirstChunkSseFraming, _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, @@ -2373,7 +2374,9 @@ class TestCommonRequestProcessingHelpers: (RESPONSES_FIRST_CHUNK_FRAMING, ('event: error\ndata: {"type": "error", "code": "server_error"',)), ], ) - async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape(self, framing, frame_starts): + async def test_a_stream_that_fails_before_its_first_chunk_ends_in_its_own_api_shape( + self, framing: FirstChunkSseFraming, frame_starts: tuple[str, ...] + ) -> None: response: Final = await create_response( _stream_failing_before_its_first_chunk(), "text/event-stream", {}, framing=framing ) From ca873a1327928e8cc1812d19ae892e10c7399a50 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:26:01 -0700 Subject: [PATCH 13/23] fix(proxy): keep cursor chat framing and JSON-encode Responses hook errors --- litellm/proxy/common_request_processing.py | 20 ++++-- .../common_utils/responses_stream_errors.py | 9 ++- litellm/proxy/proxy_server.py | 8 ++- .../proxy/response_api_endpoints/endpoints.py | 4 +- .../response_polling/background_streaming.py | 4 +- .../proxy_server/test_streaming_helpers.py | 27 ++++++-- .../response_api_endpoints/test_endpoints.py | 68 +++++++++++++------ 7 files changed, 99 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 27164ba0c83..a920b42b9d0 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1250,6 +1250,16 @@ RESPONSES_FIRST_CHUNK_FRAMING: Final = FirstChunkSseFraming( ) +@dataclass(frozen=True, slots=True) +class SseFraming: + first_chunk: FirstChunkSseFraming + late: LateSseFraming + + +OPENAI_SSE_FRAMING: Final = SseFraming(first_chunk=OPENAI_FIRST_CHUNK_FRAMING, late=OPENAI_LATE_SSE_FRAMING) +RESPONSES_SSE_FRAMING: Final = SseFraming(first_chunk=RESPONSES_FIRST_CHUNK_FRAMING, late=RESPONSES_LATE_SSE_FRAMING) + + def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]: """`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384).""" return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS}) @@ -2945,7 +2955,7 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request: bool | None = False, contents: list[object] | None = None, skip_pre_call_logic: bool = False, - late_framing: LateSseFraming = OPENAI_LATE_SSE_FRAMING, + sse_framing: SseFraming = OPENAI_SSE_FRAMING, ) -> object: """Run the request, sending SSE keepalives while the upstream is still silent. @@ -2986,10 +2996,11 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request=is_streaming_request, contents=contents, skip_pre_call_logic=skip_pre_call_logic, + first_chunk_framing=sse_framing.first_chunk, ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, - late_framing=late_framing, + late_framing=sse_framing.late, ) async def _process_llm_request( @@ -3013,6 +3024,7 @@ class ProxyBaseLLMRequestProcessing: is_streaming_request: bool | None = False, contents: list[object] | None = None, # Add contents parameter skip_pre_call_logic: bool = False, + first_chunk_framing: FirstChunkSseFraming = OPENAI_FIRST_CHUNK_FRAMING, ) -> Any: """ Common request processing logic for both chat completions and responses API endpoints @@ -3306,9 +3318,7 @@ class ProxyBaseLLMRequestProcessing: request=request, refresh_headers=refresh_stream_headers, general_settings=general_settings, - framing=( - RESPONSES_FIRST_CHUNK_FRAMING if route_type == "aresponses" else OPENAI_FIRST_CHUNK_FRAMING - ), + framing=first_chunk_framing, ) ### CALL HOOKS ### - modify outgoing data diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 4a806c32284..0024c210fab 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -61,19 +61,22 @@ class _FailureDetails(LiteLLMBaseModel): class _ErrorPayload(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - error: Mapping[str, object] + error: Mapping[str, object] | str + + def error_object(self) -> Mapping[str, object]: + return {"message": self.error} if isinstance(self.error, str) else self.error def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: try: - return _ErrorPayload.model_validate_json(data).error + return _ErrorPayload.model_validate_json(data).error_object() except ValidationError: return None def event_error_object(event: Mapping[str, object]) -> Mapping[str, object] | None: try: - return _ErrorPayload.model_validate(event).error or None + return _ErrorPayload.model_validate(event).error_object() or None except ValidationError: return None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0377ed809ad..a6648e47de6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10202,6 +10202,12 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: return f"data: {chunk}\n\n" +def _format_responses_sse_chunk(chunk: object) -> str | bytes: + if isinstance(chunk, Mapping): + return f"data: {json.dumps(chunk, default=str)}\n\n" + return _format_streaming_sse_chunk(chunk=chunk) + + _SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r") _OPENAI_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n" _MAX_RAW_SSE_BUFFER_CHARS: Final = 8 * 1024 * 1024 @@ -10602,7 +10608,7 @@ async def async_data_generator( if not raw_passthrough: try: if error_state is not None: - yield error_state.mark_emitted(_format_streaming_sse_chunk(chunk=chunk)) + yield error_state.mark_emitted(_format_responses_sse_chunk(chunk=chunk)) else: yield _format_streaming_sse_chunk(chunk=chunk) except Exception as e: diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 1fd487ea076..948a32d4449 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -52,7 +52,7 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.common_request_processing import ( - RESPONSES_LATE_SSE_FRAMING, + RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing, cost_headers, create_response, @@ -438,7 +438,7 @@ async def responses_api( if requests_background(_PROXY_SERVER_REQUEST_ADAPTER.validate_python(data)) else native_data_generator ), - late_framing=RESPONSES_LATE_SSE_FRAMING, + sse_framing=RESPONSES_SSE_FRAMING, model=None, user_model=user_model, user_temperature=user_temperature, diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index cd91bfa204f..99c988317bc 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -21,7 +21,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.llms.custom_httpx.sse_lines import split_sse_lines from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth -from litellm.proxy.common_request_processing import RESPONSES_LATE_SSE_FRAMING, ProxyBaseLLMRequestProcessing +from litellm.proxy.common_request_processing import RESPONSES_SSE_FRAMING, ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.responses_stream_errors import ( event_error_object, responses_error_fields, @@ -188,7 +188,7 @@ async def background_streaming_task( user_api_base=user_api_base, version=version, skip_pre_call_logic=True, - late_framing=RESPONSES_LATE_SSE_FRAMING, + sse_framing=RESPONSES_SSE_FRAMING, ) # Process streaming response following OpenAI events format diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 93eb8fa226b..b3a6e7b1fbf 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -17,7 +17,7 @@ from __future__ import annotations import asyncio import json -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable, Mapping from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -148,7 +148,8 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent @pytest.mark.asyncio -async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event() -> None: +@pytest.mark.parametrize("as_mapping", [False, True]) +async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event(as_mapping: bool) -> None: response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[]) created: Final = ResponseCreatedEvent.model_validate( {"type": "response.created", "sequence_number": 0, "response": response} @@ -157,9 +158,9 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e {"type": "error", "sequence_number": 1, "code": "server_error", "message": "upstream failed", "param": None} ) - async def upstream() -> AsyncIterator[BaseModel]: + async def upstream() -> AsyncIterator[BaseModel | Mapping[str, object]]: yield created - yield error_event + yield error_event.model_dump() if as_mapping else error_event raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model") frames: Final = tuple( @@ -172,14 +173,26 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e ) assert tuple(frame.split("\n", 1)[0] for frame in frames) == ("event: response.created", "event: error") + assert json.loads(frames[-1].split("data: ", 1)[1])["message"] == "upstream failed" @pytest.mark.asyncio -async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event() -> None: +@pytest.mark.parametrize( + ("guardrail_error", "error_object"), + [ + ( + {"message": "response withheld", "type": "guardrail_scan_error", "code": 500}, + {"message": "response withheld", "type": "guardrail_scan_error", "code": 500}, + ), + ("response withheld", {"message": "response withheld"}), + ], +) +async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_error_event( + guardrail_error: Mapping[str, object] | str, error_object: Mapping[str, object] +) -> None: response: Final = ResponsesAPIResponse(id="resp_hook_error", created_at=1, model="local-model", output=[]) created: Final = ResponseCreatedEvent.model_validate( {"type": "response.created", "sequence_number": 0, "response": response} ) - guardrail_error: Final = {"message": "response withheld", "type": "guardrail_scan_error", "code": 500} async def upstream() -> AsyncIterator[object]: yield created @@ -203,7 +216,7 @@ async def test_a_guardrail_error_string_mid_responses_stream_ends_it_as_a_named_ "message": "response withheld", "param": None, "sequence_number": 1, - "error": guardrail_error, + "error": error_object, } diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index abf5933bd35..d7d6321dd2b 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -181,14 +181,16 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( class _FirstFrameHook(CustomLogger): - def __init__(self, frame: str | bytes) -> None: + def __init__(self, first: str | bytes | Exception) -> None: super().__init__() - self.frame: Final = frame + self.first: Final = first async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: object, response: object, request_data: object ) -> AsyncIterator[str | bytes]: - yield self.frame + if isinstance(self.first, Exception): + raise self.first + yield self.first _DATA_ERROR_FRAME: Final = 'data: {"error": {"message": "denied", "type": "guardrail_scan_error", "code": 403}}\n\n' @@ -197,22 +199,9 @@ _NAMED_ERROR_FRAME: Final = ( ) -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path,first_frame,expected_status", - [ - ("/v1/responses", _DATA_ERROR_FRAME, 403), - ("/v1/chat/completions", _DATA_ERROR_FRAME, 403), - ("/v1/chat/completions", _NAMED_ERROR_FRAME, 200), - ("/v1/responses", _NAMED_ERROR_FRAME, 529), - ("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200), - ("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), - ("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), - ], -) -async def test_only_a_lone_first_error_frame_answers_with_its_status( - monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int -) -> None: +async def _post_through_first_frame_hook( + monkeypatch: pytest.MonkeyPatch, path: str, first: str | bytes | Exception +) -> httpx.Response: import litellm.proxy.proxy_server as ps from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -230,14 +219,14 @@ async def test_only_a_lone_first_error_frame_answers_with_its_status( ) monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first_frame)]) + monkeypatch.setattr(litellm, "callbacks", [_FirstFrameHook(first)]) monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) with respx.mock as transport: transport.post(upstream_url + ("/chat/completions" if is_chat else "/responses")).respond( 200, content="data: [DONE]\n\n", headers={"Content-Type": "text/event-stream"} ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: - result: Final = await client.post( + return await client.post( path, json={ "model": model, @@ -246,6 +235,26 @@ async def test_only_a_lone_first_error_frame_answers_with_its_status( }, ) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,first_frame,expected_status", + [ + ("/v1/responses", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _DATA_ERROR_FRAME, 403), + ("/v1/chat/completions", _NAMED_ERROR_FRAME, 200), + ("/cursor/chat/completions", _NAMED_ERROR_FRAME, 200), + ("/v1/responses", _NAMED_ERROR_FRAME, 529), + ("/v1/responses", _NAMED_ERROR_FRAME.replace(b"event: error", b"event: response.created"), 200), + ("/v1/chat/completions", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ("/v1/responses", _DATA_ERROR_FRAME + "data: [DONE]\n\n", 200), + ], +) +async def test_only_a_lone_first_error_frame_answers_with_its_status( + monkeypatch: pytest.MonkeyPatch, path: str, first_frame: str | bytes, expected_status: int +) -> None: + result: Final = await _post_through_first_frame_hook(monkeypatch, path, first_frame) + assert result.status_code == expected_status, result.text if expected_status == 200: assert result.headers["content-type"].startswith("text/event-stream"), result.text @@ -253,6 +262,23 @@ async def test_only_a_lone_first_error_frame_answers_with_its_status( assert result.json()["error"]["message"] == "denied" +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/cursor/chat/completions", "/v1/chat/completions"]) +async def test_a_hook_failure_before_the_first_chunk_answers_in_chat_completions_shape( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + message: Final = "rate limited before the first chunk" + result: Final = await _post_through_first_frame_hook( + monkeypatch, path, HTTPException(status_code=429, detail=message) + ) + frames: Final = tuple(frame for frame in result.text.split("\n\n") if frame) + + assert result.status_code == 429, result.text + assert frames[-1] == "data: [DONE]", result.text + assert message in json.loads(frames[0].removeprefix("data: "))["error"]["message"] + assert len(frames) == 2, result.text + + @pytest.mark.parametrize("path", ["/v1/responses", "/v1/chat/completions"]) def test_a_failure_after_keepalive_pings_keeps_the_client_protocol(monkeypatch: pytest.MonkeyPatch, path: str) -> None: async def request_and_check() -> None: From 81dc6c1175fabc0acb7cbdd50c8f3db88bfef9b8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:04:26 -0700 Subject: [PATCH 14/23] fix(proxy): encode non-dict Mapping Responses chunks as JSON objects --- litellm/proxy/proxy_server.py | 2 +- .../proxy/proxy_server/test_streaming_helpers.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a6648e47de6..aef2bca785c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10204,7 +10204,7 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: def _format_responses_sse_chunk(chunk: object) -> str | bytes: if isinstance(chunk, Mapping): - return f"data: {json.dumps(chunk, default=str)}\n\n" + return f"data: {json.dumps({**chunk}, default=str)}\n\n" return _format_streaming_sse_chunk(chunk=chunk) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index b3a6e7b1fbf..b75a2494588 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio import json from collections.abc import AsyncIterator, Callable, Mapping +from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -148,8 +149,14 @@ async def test_responses_stream_dispatches_named_events_without_a_chat_done_sent @pytest.mark.asyncio -@pytest.mark.parametrize("as_mapping", [False, True]) -async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event(as_mapping: bool) -> None: +@pytest.mark.parametrize( + "as_chunk", + [lambda event: event, lambda event: event.model_dump(), lambda event: MappingProxyType(event.model_dump())], + ids=["model", "dict", "mapping_proxy"], +) +async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_event( + as_chunk: Callable[[ErrorEvent], BaseModel | Mapping[str, object]], +) -> None: response: Final = ResponsesAPIResponse(id="resp_error_then_raise", created_at=1, model="local-model", output=[]) created: Final = ResponseCreatedEvent.model_validate( {"type": "response.created", "sequence_number": 0, "response": response} @@ -160,7 +167,7 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e async def upstream() -> AsyncIterator[BaseModel | Mapping[str, object]]: yield created - yield error_event.model_dump() if as_mapping else error_event + yield as_chunk(error_event) raise litellm.APIError(status_code=500, message="Stream close failed", llm_provider="openai", model="local-model") frames: Final = tuple( From b8bbb4039e43296b005fad4ebf90f20eca00978f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:07:04 -0700 Subject: [PATCH 15/23] fix(proxy): encode mappings and models nested in Responses stream chunks as JSON objects --- .../common_utils/responses_stream_errors.py | 17 ++++++++- litellm/proxy/proxy_server.py | 4 +- .../proxy_server/test_streaming_helpers.py | 37 +++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 0024c210fab..ac4854f4014 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -6,6 +6,7 @@ from types import MappingProxyType from typing import Final from pydantic import BaseModel, ConfigDict, ValidationError, field_validator +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid @@ -67,6 +68,20 @@ class _ErrorPayload(LiteLLMBaseModel): return {"message": self.error} if isinstance(self.error, str) else self.error +def _is_mapping( + value: object, +) -> TypeIs[Mapping[object, object]]: # guard-ok: isinstance decides it, keys and values stay object + return isinstance(value, Mapping) + + +def jsonable_stream_value(value: object) -> object: + if _is_mapping(value): + return {**value} + if isinstance(value, BaseModel): + return value.model_dump() + return str(value) + + def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: try: return _ErrorPayload.model_validate_json(data).error_object() @@ -191,7 +206,7 @@ class ResponsesStreamErrorState: "sequence_number": self.sequence_number + 1, "error": error, }, - default=str, + default=jsonable_stream_value, ) return f"event: error\ndata: {payload}\n\n" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index aef2bca785c..2ddb7a9241b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -539,7 +539,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( ) from litellm.proxy.common_utils.proxy_state import ProxyState from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob -from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, jsonable_stream_value from litellm.proxy.common_utils.scheduled_job_stagger import ( apply_scheduled_job_stagger, attach_job_timing_logger, @@ -10204,7 +10204,7 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: def _format_responses_sse_chunk(chunk: object) -> str | bytes: if isinstance(chunk, Mapping): - return f"data: {json.dumps({**chunk}, default=str)}\n\n" + return f"data: {json.dumps(chunk, default=jsonable_stream_value)}\n\n" return _format_streaming_sse_chunk(chunk=chunk) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index b75a2494588..9899e7d2e64 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -40,6 +40,7 @@ from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, _fast_serialize_simple_model_response_stream, _format_fallback_metadata_sse_event, + _format_responses_sse_chunk, _format_streaming_sse_chunk, _get_client_requested_model_for_streaming, _get_streaming_fallback_metadata, @@ -182,6 +183,42 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e assert tuple(frame.split("\n", 1)[0] for frame in frames) == ("event: response.created", "event: error") assert json.loads(frames[-1].split("data: ", 1)[1])["message"] == "upstream failed" + +class _HookOutputItem(BaseModel): + id: str + type: str + + +@pytest.mark.parametrize( + "item", + [ + MappingProxyType({"id": "msg_nested", "type": "message"}), + _HookOutputItem(id="msg_nested", type="message"), + ], + ids=["mapping_proxy", "model"], +) +def test_responses_sse_chunk_encodes_values_nested_in_a_hook_chunk_as_json_objects( + item: Mapping[str, object] | BaseModel, +) -> None: + frame: Final = _format_responses_sse_chunk( + chunk=MappingProxyType({"type": "response.output_item.added", "output_index": 0, "item": item}) + ) + + assert isinstance(frame, str) + assert json.loads(frame.removeprefix("data: "))["item"] == {"id": "msg_nested", "type": "message"} + + +def test_responses_error_event_encodes_nested_error_fields_as_json_objects() -> None: + frame: Final = ResponsesStreamErrorState().format_error( + { + "message": "overloaded", + "provider_specific_fields": MappingProxyType({"upstream": MappingProxyType({"type": "overloaded_error"})}), + } + ) + + payload: Final = json.loads(frame.split("data: ", 1)[1]) + assert payload["error"]["provider_specific_fields"] == {"upstream": {"type": "overloaded_error"}} + @pytest.mark.asyncio @pytest.mark.parametrize( ("guardrail_error", "error_object"), From 2452a8cae8acb1164a3a316fec378489371e46ee Mon Sep 17 00:00:00 2001 From: HootRock Date: Fri, 30 Oct 2026 18:27:58 +0800 Subject: [PATCH 16/23] test(proxy): freeze the hook-output fixture --- tests/unit/proxy/proxy_server/test_streaming_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 9899e7d2e64..a188326c685 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -184,7 +184,7 @@ async def test_responses_stream_sends_no_failure_event_after_an_upstream_error_e assert json.loads(frames[-1].split("data: ", 1)[1])["message"] == "upstream failed" -class _HookOutputItem(BaseModel): +class _HookOutputItem(BaseModel, frozen=True): id: str type: str From 34156235687cc2d6a9bf11782844c889886b6f48 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:05:44 -0700 Subject: [PATCH 17/23] fix(proxy): keep masked mappings and odd keys safe in Responses stream frames Only read-only dict views and pydantic models nested in a hook chunk or error event expand into JSON objects. Other mappings, such as httpx.Headers, keep their own string form so masked values stay masked. Keys JSON cannot take become strings, and a chunk JSON still cannot encode goes out as the plain data frame it was before. --- .../common_utils/responses_stream_errors.py | 31 ++++++++++-- litellm/proxy/proxy_server.py | 7 ++- .../proxy_server/test_streaming_helpers.py | 47 +++++++++++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index ac4854f4014..137b1221d7d 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -74,14 +74,37 @@ def _is_mapping( return isinstance(value, Mapping) -def jsonable_stream_value(value: object) -> object: - if _is_mapping(value): - return {**value} +def _is_mapping_proxy( + value: object, +) -> TypeIs[MappingProxyType[object, object]]: # guard-ok: isinstance decides it, keys and values stay object + return isinstance(value, MappingProxyType) + + +def _json_key(key: object) -> str | int | float | bool | None: + return key if key is None or isinstance(key, str | int | float | bool) else str(key) + + +def _with_json_keys(value: Mapping[object, object]) -> Mapping[str | int | float | bool | None, object]: + return {_json_key(key): item for key, item in value.items()} + + +def _jsonable_stream_value(value: object) -> object: + if _is_mapping_proxy(value): + return _with_json_keys(value) if isinstance(value, BaseModel): return value.model_dump() return str(value) +def responses_chunk_json(chunk: object) -> str | None: + if not _is_mapping(chunk): + return None + try: + return json.dumps(_with_json_keys(chunk), default=_jsonable_stream_value) + except (TypeError, ValueError): + return None + + def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: try: return _ErrorPayload.model_validate_json(data).error_object() @@ -206,7 +229,7 @@ class ResponsesStreamErrorState: "sequence_number": self.sequence_number + 1, "error": error, }, - default=jsonable_stream_value, + default=_jsonable_stream_value, ) return f"event: error\ndata: {payload}\n\n" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2ddb7a9241b..7acdd582658 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -539,7 +539,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( ) from litellm.proxy.common_utils.proxy_state import ProxyState from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob -from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, jsonable_stream_value +from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState, responses_chunk_json from litellm.proxy.common_utils.scheduled_job_stagger import ( apply_scheduled_job_stagger, attach_job_timing_logger, @@ -10203,9 +10203,8 @@ def _format_streaming_sse_chunk(chunk: object) -> str | bytes: def _format_responses_sse_chunk(chunk: object) -> str | bytes: - if isinstance(chunk, Mapping): - return f"data: {json.dumps(chunk, default=jsonable_stream_value)}\n\n" - return _format_streaming_sse_chunk(chunk=chunk) + data: Final = responses_chunk_json(chunk) + return _format_streaming_sse_chunk(chunk=chunk) if data is None else f"data: {data}\n\n" _SSE_FRAME_DELIMITERS: Final = ("\r\n\r\n", "\n\n", "\r\r") diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index a188326c685..05bde512608 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -219,6 +219,53 @@ def test_responses_error_event_encodes_nested_error_fields_as_json_objects() -> payload: Final = json.loads(frame.split("data: ", 1)[1]) assert payload["error"]["provider_specific_fields"] == {"upstream": {"type": "overloaded_error"}} + +_UPSTREAM_HEADERS: Final = httpx.Headers({"authorization": "Bearer hook-canary", "x-request-id": "req_1"}) + + +def _responses_frame_payload(nested: object) -> Mapping[str, object]: + chunk_frame: Final = _format_responses_sse_chunk( + chunk=MappingProxyType({"type": "response.output_text.delta", "nested": nested}) + ) + error_frame: Final = ResponsesStreamErrorState().format_error({"message": "blocked", "nested": nested}) + assert isinstance(chunk_frame, str) + chunk_payload: Final = json.loads(chunk_frame.removeprefix("data: ")) + error_payload: Final = json.loads(error_frame.split("data: ", 1)[1]) + assert chunk_payload["nested"] == error_payload["error"]["nested"] + return chunk_payload + + +def test_responses_stream_frames_send_nested_header_mappings_in_their_masked_string_form() -> None: + assert _responses_frame_payload(_UPSTREAM_HEADERS)["nested"] == str(_UPSTREAM_HEADERS) + + +def test_responses_stream_frames_turn_mapping_keys_json_cannot_take_into_strings() -> None: + assert _responses_frame_payload(MappingProxyType({("a", "b"): 1}))["nested"] == {str(("a", "b")): 1} + + +def test_responses_sse_chunk_turns_top_level_keys_json_cannot_take_into_strings() -> None: + frame: Final = _format_responses_sse_chunk( + chunk=MappingProxyType({"type": "response.output_text.delta", ("a", "b"): "x"}) + ) + + assert isinstance(frame, str) + assert json.loads(frame.removeprefix("data: ")) == {"type": "response.output_text.delta", str(("a", "b")): "x"} + + +def _self_referencing_chunk() -> Mapping[str, object]: + chunk: Final[dict[str, object]] = {"type": "response.output_text.delta"} + chunk["self"] = chunk # mutable-ok: a self-reference needs the dict to exist before it can hold itself + return chunk + + +@pytest.mark.parametrize( + "chunk", + [{"type": "response.output_text.delta", "meta": {("a", "b"): 1}}, _self_referencing_chunk()], + ids=["nested_dict_tuple_key", "self_reference"], +) +def test_responses_sse_chunk_json_cannot_encode_streams_as_a_plain_data_frame(chunk: Mapping[str, object]) -> None: + assert _format_responses_sse_chunk(chunk=chunk) == _format_streaming_sse_chunk(chunk=chunk) + @pytest.mark.asyncio @pytest.mark.parametrize( ("guardrail_error", "error_object"), From cc3d71987855d65097183c69b7c629caa07defed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:02:53 -0700 Subject: [PATCH 18/23] fix(proxy): never fail a Responses stream frame on a value JSON cannot encode Only dicts and read-only dict views expand into JSON, at the top level too, so a header mapping yielded as a whole chunk keeps its masked string form. Any failure while encoding a chunk falls back to the plain data frame the base sent, and an error event that cannot encode still goes out with its code and message, so a late block never drops the connection. --- .../common_utils/responses_stream_errors.py | 37 ++++++++++++------- .../proxy_server/test_streaming_helpers.py | 32 +++++++++++++++- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 137b1221d7d..7bec25bc941 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -68,10 +68,10 @@ class _ErrorPayload(LiteLLMBaseModel): return {"message": self.error} if isinstance(self.error, str) else self.error -def _is_mapping( +def _is_dict_like( value: object, -) -> TypeIs[Mapping[object, object]]: # guard-ok: isinstance decides it, keys and values stay object - return isinstance(value, Mapping) +) -> TypeIs[dict[object, object] | MappingProxyType[object, object]]: # guard-ok: isinstance decides it + return isinstance(value, dict | MappingProxyType) def _is_mapping_proxy( @@ -96,15 +96,17 @@ def _jsonable_stream_value(value: object) -> object: return str(value) -def responses_chunk_json(chunk: object) -> str | None: - if not _is_mapping(chunk): - return None +def _stream_json(value: Mapping[object, object]) -> str | None: try: - return json.dumps(_with_json_keys(chunk), default=_jsonable_stream_value) - except (TypeError, ValueError): + return json.dumps(_with_json_keys(value), default=_jsonable_stream_value) + except Exception: # noqa: BLE001 # hook values run their own __str__ and serializers, so any failure must fall back return None +def responses_chunk_json(chunk: object) -> str | None: + return _stream_json(chunk) if _is_dict_like(chunk) else None + + def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: try: return _ErrorPayload.model_validate_json(data).error_object() @@ -221,15 +223,24 @@ class ResponsesStreamErrorState: return prefix.encode("utf-8") + frame if isinstance(frame, bytes) else prefix + frame def format_error(self, error: Mapping[str, object]) -> str: - payload: Final = json.dumps( + fields: Final = responses_error_fields(error) + sequence_number: Final = self.sequence_number + 1 + payload: Final = _stream_json( { "type": ResponsesAPIStreamEvents.ERROR.value, - **responses_error_fields(error), + **fields, "param": error.get("param"), - "sequence_number": self.sequence_number + 1, + "sequence_number": sequence_number, "error": error, - }, - default=_jsonable_stream_value, + } + ) or json.dumps( + { + "type": ResponsesAPIStreamEvents.ERROR.value, + **fields, + "param": None, + "sequence_number": sequence_number, + "error": fields, + } ) return f"event: error\ndata: {payload}\n\n" diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 05bde512608..ab0a2f3b6ba 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -258,14 +258,42 @@ def _self_referencing_chunk() -> Mapping[str, object]: return chunk +class _UnprintableValue: + def __str__(self) -> str: + raise RuntimeError("no string form") + + @pytest.mark.parametrize( "chunk", - [{"type": "response.output_text.delta", "meta": {("a", "b"): 1}}, _self_referencing_chunk()], - ids=["nested_dict_tuple_key", "self_reference"], + [ + {"type": "response.output_text.delta", "meta": {("a", "b"): 1}}, + _self_referencing_chunk(), + {"type": "response.output_text.delta", "meta": _UnprintableValue()}, + _UPSTREAM_HEADERS, + ], + ids=["nested_dict_tuple_key", "self_reference", "unprintable_value", "header_mapping_chunk"], ) def test_responses_sse_chunk_json_cannot_encode_streams_as_a_plain_data_frame(chunk: Mapping[str, object]) -> None: assert _format_responses_sse_chunk(chunk=chunk) == _format_streaming_sse_chunk(chunk=chunk) + +@pytest.mark.parametrize( + "nested", + [ + MappingProxyType({"bad": {("a", "b"): 1}}), + _self_referencing_chunk(), + _UnprintableValue(), + ], + ids=["proxy_over_tuple_key", "self_reference", "unprintable_value"], +) +def test_responses_error_event_json_cannot_encode_still_ends_the_stream_with_its_message(nested: object) -> None: + frame: Final = ResponsesStreamErrorState().format_error({"message": "blocked", "code": 400, "nested": nested}) + + assert frame.startswith("event: error\ndata: ") + payload: Final = json.loads(frame.split("data: ", 1)[1]) + assert (payload["type"], payload["message"], payload["error"]["message"]) == ("error", "blocked", "blocked") + assert payload["error"]["code"] == payload["code"] + @pytest.mark.asyncio @pytest.mark.parametrize( ("guardrail_error", "error_object"), From 91334483b54a0581e95838afe34fa13e70bdd7ae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:15:36 -0700 Subject: [PATCH 19/23] fix(proxy): keep what a nested value's repr hides out of Responses stream frames --- .../common_utils/responses_stream_errors.py | 81 ++++++++++--------- .../proxy_server/test_streaming_helpers.py | 69 +++++++++++----- 2 files changed, 91 insertions(+), 59 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 7bec25bc941..88ef6c25422 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,12 +1,12 @@ import json import time -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from http import HTTPStatus from types import MappingProxyType from typing import Final from pydantic import BaseModel, ConfigDict, ValidationError, field_validator -from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 +from typing_extensions import TypeIs # noqa: TID251 # exact-type checks keep subclasses out, isinstance cannot from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid @@ -68,43 +68,48 @@ class _ErrorPayload(LiteLLMBaseModel): return {"message": self.error} if isinstance(self.error, str) else self.error -def _is_dict_like( - value: object, -) -> TypeIs[dict[object, object] | MappingProxyType[object, object]]: # guard-ok: isinstance decides it - return isinstance(value, dict | MappingProxyType) +def _is_plain_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: exact type check + return type(value) in (dict, MappingProxyType) -def _is_mapping_proxy( - value: object, -) -> TypeIs[MappingProxyType[object, object]]: # guard-ok: isinstance decides it, keys and values stay object - return isinstance(value, MappingProxyType) +def _is_plain_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: exact type check + return type(value) in (list, tuple) def _json_key(key: object) -> str | int | float | bool | None: - return key if key is None or isinstance(key, str | int | float | bool) else str(key) + return key if key is None or isinstance(key, str | int | float | bool) else repr(key) -def _with_json_keys(value: Mapping[object, object]) -> Mapping[str | int | float | bool | None, object]: - return {_json_key(key): item for key, item in value.items()} +def _json_safe_items(items: Iterable[tuple[object, object]]) -> Mapping[str | int | float | bool | None, object]: + return {_json_key(key): _json_safe(item) for key, item in items} -def _jsonable_stream_value(value: object) -> object: - if _is_mapping_proxy(value): - return _with_json_keys(value) - if isinstance(value, BaseModel): - return value.model_dump() - return str(value) +def _json_safe(value: object) -> object: + if value is None or isinstance(value, str | int | float | bool): + return value + if _is_plain_mapping(value): + return _json_safe_items(value.items()) + if _is_plain_sequence(value): + return [_json_safe(item) for item in value] + if isinstance(value, BaseModel) and type(value).__repr__ is BaseModel.__repr__: + shown: Final[Iterable[tuple[str | None, object]]] = value.__repr_args__() + return _json_safe_items(shown) + return repr(value) -def _stream_json(value: Mapping[object, object]) -> str | None: +def _stream_json(value: object) -> str | None: try: - return json.dumps(_with_json_keys(value), default=_jsonable_stream_value) - except Exception: # noqa: BLE001 # hook values run their own __str__ and serializers, so any failure must fall back + return json.dumps(_json_safe(value)) + except Exception: # noqa: BLE001 # hook values run their own __repr__, so any failure must fall back return None +def _encodable_items(error: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in error.items() if _stream_json(value) is not None} + + def responses_chunk_json(chunk: object) -> str | None: - return _stream_json(chunk) if _is_dict_like(chunk) else None + return _stream_json(chunk) if _is_plain_mapping(chunk) else None def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: @@ -192,6 +197,18 @@ def responses_error_fields(error: Mapping[str, object]) -> Mapping[str, object]: } +def _error_event( + fields: Mapping[str, object], error: Mapping[str, object], sequence_number: int +) -> Mapping[str, object]: + return { + "type": ResponsesAPIStreamEvents.ERROR.value, + **fields, + "param": error.get("param"), + "sequence_number": sequence_number, + "error": error, + } + + class ResponsesStreamErrorState: def __init__(self) -> None: self.response_id: str | None = None @@ -225,22 +242,8 @@ class ResponsesStreamErrorState: def format_error(self, error: Mapping[str, object]) -> str: fields: Final = responses_error_fields(error) sequence_number: Final = self.sequence_number + 1 - payload: Final = _stream_json( - { - "type": ResponsesAPIStreamEvents.ERROR.value, - **fields, - "param": error.get("param"), - "sequence_number": sequence_number, - "error": error, - } - ) or json.dumps( - { - "type": ResponsesAPIStreamEvents.ERROR.value, - **fields, - "param": None, - "sequence_number": sequence_number, - "error": fields, - } + payload: Final = _stream_json(_error_event(fields, _encodable_items(error), sequence_number)) or json.dumps( + _error_event(fields, fields, sequence_number) ) return f"event: error\ndata: {payload}\n\n" diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index ab0a2f3b6ba..4dfb3daec52 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -27,7 +27,7 @@ import pytest from fastapi import HTTPException, Response from fastapi.responses import JSONResponse, StreamingResponse from openai import APIError as OpenAIAPIError -from pydantic import BaseModel +from pydantic import BaseModel, Field import litellm import litellm.proxy.proxy_server as ps @@ -61,6 +61,7 @@ from litellm.types.llms.openai import ( ResponseIncompleteEvent, ResponsesAPIResponse, ) +from litellm.types.proxy.litellm_pre_call_utils import RedactedDict from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage from tests._support.clock_loops import AutojumpClockLoop @@ -235,12 +236,43 @@ def _responses_frame_payload(nested: object) -> Mapping[str, object]: return chunk_payload -def test_responses_stream_frames_send_nested_header_mappings_in_their_masked_string_form() -> None: - assert _responses_frame_payload(_UPSTREAM_HEADERS)["nested"] == str(_UPSTREAM_HEADERS) +class _HiddenFieldModel(BaseModel, frozen=True): + user: str + password: str = Field(repr=False) -def test_responses_stream_frames_turn_mapping_keys_json_cannot_take_into_strings() -> None: - assert _responses_frame_payload(MappingProxyType({("a", "b"): 1}))["nested"] == {str(("a", "b")): 1} +class _MaskedReprModel(BaseModel, frozen=True): + token: str + + def __repr__(self) -> str: + return "_MaskedReprModel(token=[secure])" + + +_REDACTED_HEADERS: Final = RedactedDict({"authorization": "Bearer hook-canary"}) + + +@pytest.mark.parametrize( + ("nested", "expected"), + [ + (_UPSTREAM_HEADERS, repr(_UPSTREAM_HEADERS)), + (_REDACTED_HEADERS, repr(_REDACTED_HEADERS)), + (MappingProxyType({"headers": _REDACTED_HEADERS}), {"headers": repr(_REDACTED_HEADERS)}), + (_HiddenFieldModel(user="rig", password="hook-canary"), {"user": "rig"}), + (_MaskedReprModel(token="hook-canary"), repr(_MaskedReprModel(token="hook-canary"))), + ], + ids=["header_mapping", "redacted_dict", "proxy_over_redacted_dict", "repr_false_field", "masked_repr_model"], +) +def test_responses_stream_frames_never_show_what_a_nested_value_hides_from_its_repr( + nested: object, expected: object +) -> None: + assert _responses_frame_payload(nested)["nested"] == expected + + +@pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) +def test_responses_stream_frames_turn_mapping_keys_json_cannot_take_into_strings( + mapping: Callable[[Mapping[object, object]], Mapping[object, object]], +) -> None: + assert _responses_frame_payload(mapping({("a", "b"): 1}))["nested"] == {repr(("a", "b")): 1} def test_responses_sse_chunk_turns_top_level_keys_json_cannot_take_into_strings() -> None: @@ -249,7 +281,7 @@ def test_responses_sse_chunk_turns_top_level_keys_json_cannot_take_into_strings( ) assert isinstance(frame, str) - assert json.loads(frame.removeprefix("data: ")) == {"type": "response.output_text.delta", str(("a", "b")): "x"} + assert json.loads(frame.removeprefix("data: ")) == {"type": "response.output_text.delta", repr(("a", "b")): "x"} def _self_referencing_chunk() -> Mapping[str, object]: @@ -259,19 +291,18 @@ def _self_referencing_chunk() -> Mapping[str, object]: class _UnprintableValue: - def __str__(self) -> str: + def __repr__(self) -> str: raise RuntimeError("no string form") @pytest.mark.parametrize( "chunk", [ - {"type": "response.output_text.delta", "meta": {("a", "b"): 1}}, _self_referencing_chunk(), - {"type": "response.output_text.delta", "meta": _UnprintableValue()}, _UPSTREAM_HEADERS, + RedactedDict({"type": "response.output_text.delta", "authorization": "Bearer hook-canary"}), ], - ids=["nested_dict_tuple_key", "self_reference", "unprintable_value", "header_mapping_chunk"], + ids=["self_reference", "header_mapping_chunk", "redacted_dict_chunk"], ) def test_responses_sse_chunk_json_cannot_encode_streams_as_a_plain_data_frame(chunk: Mapping[str, object]) -> None: assert _format_responses_sse_chunk(chunk=chunk) == _format_streaming_sse_chunk(chunk=chunk) @@ -279,20 +310,18 @@ def test_responses_sse_chunk_json_cannot_encode_streams_as_a_plain_data_frame(ch @pytest.mark.parametrize( "nested", - [ - MappingProxyType({"bad": {("a", "b"): 1}}), - _self_referencing_chunk(), - _UnprintableValue(), - ], - ids=["proxy_over_tuple_key", "self_reference", "unprintable_value"], + [_self_referencing_chunk(), _UnprintableValue()], + ids=["self_reference", "unprintable_value"], ) -def test_responses_error_event_json_cannot_encode_still_ends_the_stream_with_its_message(nested: object) -> None: - frame: Final = ResponsesStreamErrorState().format_error({"message": "blocked", "code": 400, "nested": nested}) +def test_responses_error_event_drops_only_the_error_detail_json_cannot_encode(nested: object) -> None: + frame: Final = ResponsesStreamErrorState().format_error( + {"message": "blocked", "code": "400", "type": "guardrail_error", "nested": nested} + ) assert frame.startswith("event: error\ndata: ") payload: Final = json.loads(frame.split("data: ", 1)[1]) - assert (payload["type"], payload["message"], payload["error"]["message"]) == ("error", "blocked", "blocked") - assert payload["error"]["code"] == payload["code"] + assert (payload["type"], payload["message"]) == ("error", "blocked") + assert payload["error"] == {"message": "blocked", "code": "400", "type": "guardrail_error"} @pytest.mark.asyncio @pytest.mark.parametrize( From 4180859208d00e16090ea1cc54a4552d5763f04e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:20:16 -0700 Subject: [PATCH 20/23] fix(proxy): print values in Responses stream frames the way each base path did --- .../common_utils/responses_stream_errors.py | 109 +++++++--- .../proxy_server/test_streaming_helpers.py | 188 +++++++++++++++--- 2 files changed, 238 insertions(+), 59 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 88ef6c25422..0f9df6b6985 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,6 +1,7 @@ import json import time from collections.abc import Iterable, Mapping, Sequence +from enum import Enum from http import HTTPStatus from types import MappingProxyType from typing import Final @@ -68,48 +69,104 @@ class _ErrorPayload(LiteLLMBaseModel): return {"message": self.error} if isinstance(self.error, str) else self.error -def _is_plain_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: exact type check - return type(value) in (dict, MappingProxyType) +_DROPPED: Final = object() + + +class _BaseForm(Enum): + PRINTED = "printed" + DUMPED = "dumped" + + +def _is_plain_dict(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: exact type check + return type(value) is dict def _is_plain_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: exact type check return type(value) in (list, tuple) -def _json_key(key: object) -> str | int | float | bool | None: - return key if key is None or isinstance(key, str | int | float | bool) else repr(key) +def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed + return isinstance(value, MappingProxyType) -def _json_safe_items(items: Iterable[tuple[object, object]]) -> Mapping[str | int | float | bool | None, object]: - return {_json_key(key): _json_safe(item) for key, item in items} +def _prints_as_dict(proxy: Mapping[object, object]) -> bool: + return repr(proxy) == f"mappingproxy({dict(proxy)!r})" -def _json_safe(value: object) -> object: - if value is None or isinstance(value, str | int | float | bool): - return value - if _is_plain_mapping(value): - return _json_safe_items(value.items()) - if _is_plain_sequence(value): - return [_json_safe(item) for item in value] - if isinstance(value, BaseModel) and type(value).__repr__ is BaseModel.__repr__: - shown: Final[Iterable[tuple[str | None, object]]] = value.__repr_args__() - return _json_safe_items(shown) - return repr(value) +def _prints_its_fields(model: BaseModel) -> bool: + kind: Final = type(model) + return ( + kind.__repr__ is BaseModel.__repr__ + and kind.__str__ is BaseModel.__str__ + and kind.__repr_str__ is BaseModel.__repr_str__ + ) -def _stream_json(value: object) -> str | None: +def _str_skips_its_repr(model: BaseModel) -> bool: + kind: Final = type(model) + return kind.__str__ is BaseModel.__str__ and kind.__repr__ is not BaseModel.__repr__ + + +def _text(value: object, form: _BaseForm) -> object: try: - return json.dumps(_json_safe(value)) - except Exception: # noqa: BLE001 # hook values run their own __repr__, so any failure must fall back - return None + if form is _BaseForm.PRINTED: + return repr(value) + if isinstance(value, int): + return int(value) + if isinstance(value, float): + return float(value) + if isinstance(value, BaseModel) and _str_skips_its_repr(value): + return repr(value) + return str(value) + except Exception: # noqa: BLE001 # hook values run their own __repr__ and __str__, so a failure drops the value + return _DROPPED -def _encodable_items(error: Mapping[str, object]) -> Mapping[str, object]: - return {key: value for key, value in error.items() if _stream_json(value) is not None} +def _json_key(key: object, form: _BaseForm) -> object: + return key if key is None or type(key) in (str, int, float, bool) else _text(key, form) + + +def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None: + if _is_plain_dict(value): + return value.items(), form + if _is_proxy(value) and _prints_as_dict(value): + return value.items(), _BaseForm.PRINTED + if isinstance(value, BaseModel) and _prints_its_fields(value): + return value.__repr_args__(), _BaseForm.PRINTED + return None + + +def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: + if value is None or type(value) in (str, int, float, bool): + return value + if id(value) in path: + return _text(value, form) + inner_path: Final = path | frozenset((id(value),)) + try: + if _is_plain_sequence(value): + shown_items: Final = [_json_safe(item, form, inner_path) for item in value] + return [item for item in shown_items if item is not _DROPPED] + expansion: Final = _expansion(value, form) + if expansion is None: + return _text(value, form) + items, inner = expansion + pairs: Final = [(_json_key(key, inner), _json_safe(item, inner, inner_path)) for key, item in items] + return {key: item for key, item in pairs if key is not _DROPPED and item is not _DROPPED} + except Exception: # noqa: BLE001 # hook values run their own printers and can nest past the recursion limit + return _text(value, form) def responses_chunk_json(chunk: object) -> str | None: - return _stream_json(chunk) if _is_plain_mapping(chunk) else None + if not _is_plain_dict(chunk) and not _is_proxy(chunk): + return None + shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset()) + return None if isinstance(shown, str) or shown is _DROPPED else json.dumps(shown) + + +def _dumped_detail(error: Mapping[str, object]) -> Mapping[str, object]: + path: Final = frozenset((id(error),)) + shown: Final = [(key, _json_safe(value, _BaseForm.DUMPED, path)) for key, value in error.items()] + return {key: value for key, value in shown if value is not _DROPPED} def sse_error_object(data: str | bytes) -> Mapping[str, object] | None: @@ -242,9 +299,7 @@ class ResponsesStreamErrorState: def format_error(self, error: Mapping[str, object]) -> str: fields: Final = responses_error_fields(error) sequence_number: Final = self.sequence_number + 1 - payload: Final = _stream_json(_error_event(fields, _encodable_items(error), sequence_number)) or json.dumps( - _error_event(fields, fields, sequence_number) - ) + payload: Final = json.dumps(_error_event(fields, _dumped_detail(error), sequence_number)) return f"event: error\ndata: {payload}\n\n" def format_error_message(self, frame: str) -> str: diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 4dfb3daec52..e055d98761a 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -16,8 +16,12 @@ Pins covered: from __future__ import annotations import asyncio +import functools import json -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Callable, Iterable, Mapping +from datetime import datetime, timezone +from enum import Enum +from http import HTTPStatus from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock @@ -248,24 +252,118 @@ class _MaskedReprModel(BaseModel, frozen=True): return "_MaskedReprModel(token=[secure])" +class _MaskedReprStrModel(BaseModel, frozen=True): + token: str + + def __repr_str__(self, join_str: str) -> str: + return "token=[secure]" + + +class _MaskedReprArgsModel(BaseModel, frozen=True): + token: str + + def __repr_args__(self) -> Iterable[tuple[str | None, object]]: + return (("token", "[secure]"),) + + +class _MaskedStrModel(BaseModel, frozen=True): + token: str + + def __str__(self) -> str: + return "_MaskedStrModel(token=[secure])" + + +class _MaskedReprString(str): + def __repr__(self) -> str: + return "_MaskedReprString([secure])" + + _REDACTED_HEADERS: Final = RedactedDict({"authorization": "Bearer hook-canary"}) +_MASKED_IN_EVERY_FRAME: Final = ( + _UPSTREAM_HEADERS, + _REDACTED_HEADERS, + MappingProxyType({"headers": _REDACTED_HEADERS}), + MappingProxyType(_REDACTED_HEADERS), + _HiddenFieldModel(user="rig", password="hook-canary"), + _MaskedReprModel(token="hook-canary"), + _MaskedReprStrModel(token="hook-canary"), + _MaskedReprArgsModel(token="hook-canary"), +) +_MASKED_IN_EVERY_FRAME_IDS: Final = ( + "header_mapping", + "redacted_dict", + "proxy_over_dict_holding_redacted_dict", + "proxy_wrapping_redacted_dict", + "repr_false_field", + "masked_repr_model", + "masked_repr_str_model", + "masked_repr_args_model", +) + @pytest.mark.parametrize( - ("nested", "expected"), - [ - (_UPSTREAM_HEADERS, repr(_UPSTREAM_HEADERS)), - (_REDACTED_HEADERS, repr(_REDACTED_HEADERS)), - (MappingProxyType({"headers": _REDACTED_HEADERS}), {"headers": repr(_REDACTED_HEADERS)}), - (_HiddenFieldModel(user="rig", password="hook-canary"), {"user": "rig"}), - (_MaskedReprModel(token="hook-canary"), repr(_MaskedReprModel(token="hook-canary"))), - ], - ids=["header_mapping", "redacted_dict", "proxy_over_redacted_dict", "repr_false_field", "masked_repr_model"], + "nested", + [*_MASKED_IN_EVERY_FRAME, _MaskedReprString("hook-canary")], + ids=[*_MASKED_IN_EVERY_FRAME_IDS, "masked_repr_string"], ) -def test_responses_stream_frames_never_show_what_a_nested_value_hides_from_its_repr( - nested: object, expected: object +def test_responses_sse_chunk_never_shows_what_a_nested_value_hides_from_its_repr(nested: object) -> None: + frame: Final = _format_responses_sse_chunk( + chunk=MappingProxyType({"type": "response.output_text.delta", "nested": nested}) + ) + + assert isinstance(frame, str) + assert json.loads(frame.removeprefix("data: "))["type"] == "response.output_text.delta" + assert "hook-canary" not in frame + + +@pytest.mark.parametrize( + "nested", + [*_MASKED_IN_EVERY_FRAME, _MaskedStrModel(token="hook-canary")], + ids=[*_MASKED_IN_EVERY_FRAME_IDS, "masked_str_model"], +) +def test_responses_error_event_never_shows_what_a_nested_value_hides_from_its_string_form(nested: object) -> None: + frame: Final = ResponsesStreamErrorState().format_error( + {"message": "blocked", "provider_specific_fields": {"guardrail": "rig-guard", "detail": nested}} + ) + + payload: Final = json.loads(frame.split("data: ", 1)[1]) + assert payload["error"]["provider_specific_fields"]["guardrail"] == "rig-guard" + assert "hook-canary" not in frame + + +class _RigColor(Enum): + RED = 1 + + +class _Score(float): + pass + + +_WHEN: Final = datetime(2026, 10, 30, 12, 0, tzinfo=timezone.utc) + + +@pytest.mark.parametrize( + ("nested", "printed", "dumped"), + [ + (_WHEN, repr(_WHEN), str(_WHEN)), + (_RigColor.RED, repr(_RigColor.RED), str(_RigColor.RED)), + (HTTPStatus.BAD_REQUEST, repr(HTTPStatus.BAD_REQUEST), 400), + (_Score(0.5), repr(_Score(0.5)), 0.5), + ], + ids=["datetime", "enum", "int_enum", "float_subclass"], +) +def test_responses_stream_frames_print_values_json_cannot_take_as_base_printed_them( + nested: object, printed: str, dumped: object ) -> None: - assert _responses_frame_payload(nested)["nested"] == expected + chunk_frame: Final = _format_responses_sse_chunk( + chunk=MappingProxyType({"type": "response.output_text.delta", "nested": nested}) + ) + error_frame: Final = ResponsesStreamErrorState().format_error({"message": "blocked", "nested": nested}) + + assert isinstance(chunk_frame, str) + assert json.loads(chunk_frame.removeprefix("data: "))["nested"] == printed + assert json.loads(error_frame.split("data: ", 1)[1])["error"]["nested"] == dumped @pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) @@ -290,39 +388,65 @@ def _self_referencing_chunk() -> Mapping[str, object]: return chunk +def test_responses_stream_frames_print_a_value_that_holds_itself_as_its_text() -> None: + nested: Final = _self_referencing_chunk() + + assert _responses_frame_payload(nested)["nested"] == {"type": "response.output_text.delta", "self": repr(nested)} + + class _UnprintableValue: def __repr__(self) -> str: raise RuntimeError("no string form") +def test_responses_stream_frames_leave_out_only_the_nested_value_with_no_string_form() -> None: + chunk_frame: Final = _format_responses_sse_chunk( + chunk={"type": "response.output_text.delta", "meta": {"guardrail": "rig-guard", "bad": _UnprintableValue()}} + ) + error_frame: Final = ResponsesStreamErrorState().format_error( + { + "message": "blocked", + "code": "400", + "type": "guardrail_error", + "provider_specific_fields": {"guardrail": "rig-guard", "bad": _UnprintableValue()}, + } + ) + + assert isinstance(chunk_frame, str) + assert json.loads(chunk_frame.removeprefix("data: "))["meta"] == {"guardrail": "rig-guard"} + assert json.loads(error_frame.split("data: ", 1)[1])["error"] == { + "message": "blocked", + "code": "400", + "type": "guardrail_error", + "provider_specific_fields": {"guardrail": "rig-guard"}, + } + + +def test_responses_error_event_keeps_a_detail_nested_past_the_recursion_limit() -> None: + nested: Final = functools.reduce(lambda inner, _: {"x": inner}, range(5000), {"leaf": 1}) + frame: Final = ResponsesStreamErrorState().format_error( + {"message": "blocked", "provider_specific_fields": {"guardrail": "rig-guard", "detail": nested}} + ) + + payload: Final = json.loads(frame.split("data: ", 1)[1]) + assert payload["error"]["provider_specific_fields"]["guardrail"] == "rig-guard" + assert "'leaf': 1" in frame + + @pytest.mark.parametrize( "chunk", [ - _self_referencing_chunk(), _UPSTREAM_HEADERS, RedactedDict({"type": "response.output_text.delta", "authorization": "Bearer hook-canary"}), + MappingProxyType(RedactedDict({"type": "response.output_text.delta", "authorization": "Bearer hook-canary"})), ], - ids=["self_reference", "header_mapping_chunk", "redacted_dict_chunk"], + ids=["header_mapping_chunk", "redacted_dict_chunk", "proxy_wrapping_redacted_dict_chunk"], ) -def test_responses_sse_chunk_json_cannot_encode_streams_as_a_plain_data_frame(chunk: Mapping[str, object]) -> None: +def test_responses_sse_chunk_that_is_not_a_plain_mapping_streams_as_the_plain_data_frame( + chunk: Mapping[str, object], +) -> None: assert _format_responses_sse_chunk(chunk=chunk) == _format_streaming_sse_chunk(chunk=chunk) - -@pytest.mark.parametrize( - "nested", - [_self_referencing_chunk(), _UnprintableValue()], - ids=["self_reference", "unprintable_value"], -) -def test_responses_error_event_drops_only_the_error_detail_json_cannot_encode(nested: object) -> None: - frame: Final = ResponsesStreamErrorState().format_error( - {"message": "blocked", "code": "400", "type": "guardrail_error", "nested": nested} - ) - - assert frame.startswith("event: error\ndata: ") - payload: Final = json.loads(frame.split("data: ", 1)[1]) - assert (payload["type"], payload["message"]) == ("error", "blocked") - assert payload["error"] == {"message": "blocked", "code": "400", "type": "guardrail_error"} - @pytest.mark.asyncio @pytest.mark.parametrize( ("guardrail_error", "error_object"), From 6b851e937167a06590d2e13f0cea3d5c8d8c9e4d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:09:38 -0700 Subject: [PATCH 21/23] fix(proxy): send string and number subclasses whose repr shows their value as that value in Responses stream frames --- .../common_utils/responses_stream_errors.py | 21 +++++++-- .../proxy_server/test_streaming_helpers.py | 46 +++++++++++++++++-- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 0f9df6b6985..8916c0be8a9 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -107,14 +107,27 @@ def _str_skips_its_repr(model: BaseModel) -> bool: return kind.__str__ is BaseModel.__str__ and kind.__repr__ is not BaseModel.__repr__ +def _plain_scalar(value: str | float) -> str | float: + if isinstance(value, str): + return str.__str__(value) + if isinstance(value, int): + return int.__int__(value) + return float.__float__(value) + + +def _repr_shows_its_value(value: str | float) -> bool: + plain: Final = _plain_scalar(value) + if isinstance(value, Enum): + return repr(value) == f"<{type(value).__name__}.{value.name}: {plain!r}>" + return repr(value) == repr(plain) + + def _text(value: object, form: _BaseForm) -> object: try: + if isinstance(value, (str, int, float)) and (form is _BaseForm.DUMPED or _repr_shows_its_value(value)): + return _plain_scalar(value) if form is _BaseForm.PRINTED: return repr(value) - if isinstance(value, int): - return int(value) - if isinstance(value, float): - return float(value) if isinstance(value, BaseModel) and _str_skips_its_repr(value): return repr(value) return str(value) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index e055d98761a..a2045995093 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -64,6 +64,7 @@ from litellm.types.llms.openai import ( ResponseFailedEvent, ResponseIncompleteEvent, ResponsesAPIResponse, + ResponsesAPIStreamEvents, ) from litellm.types.proxy.litellm_pre_call_utils import RedactedDict from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage @@ -340,6 +341,11 @@ class _Score(float): pass +class _MaskedCount(int): + def __repr__(self) -> str: + return "_MaskedCount([secure])" + + _WHEN: Final = datetime(2026, 10, 30, 12, 0, tzinfo=timezone.utc) @@ -348,10 +354,9 @@ _WHEN: Final = datetime(2026, 10, 30, 12, 0, tzinfo=timezone.utc) [ (_WHEN, repr(_WHEN), str(_WHEN)), (_RigColor.RED, repr(_RigColor.RED), str(_RigColor.RED)), - (HTTPStatus.BAD_REQUEST, repr(HTTPStatus.BAD_REQUEST), 400), - (_Score(0.5), repr(_Score(0.5)), 0.5), + (_MaskedCount(7), repr(_MaskedCount(7)), 7), ], - ids=["datetime", "enum", "int_enum", "float_subclass"], + ids=["datetime", "enum", "number_subclass_with_its_own_repr"], ) def test_responses_stream_frames_print_values_json_cannot_take_as_base_printed_them( nested: object, printed: str, dumped: object @@ -366,6 +371,41 @@ def test_responses_stream_frames_print_values_json_cannot_take_as_base_printed_t assert json.loads(error_frame.split("data: ", 1)[1])["error"]["nested"] == dumped +class _RigCode(str, Enum): + BLOCKED = "blocked_by_rig" + + +@pytest.mark.parametrize( + ("nested", "plain"), + [(_RigCode.BLOCKED, "blocked_by_rig"), (HTTPStatus.BAD_REQUEST, 400), (_Score(0.5), 0.5)], + ids=["str_enum", "int_enum", "float_subclass"], +) +def test_responses_stream_frames_send_string_and_number_subclasses_as_their_plain_value( + nested: object, plain: object +) -> None: + assert _responses_frame_payload(nested)["nested"] == plain + + +_DUMPED_DELTA: Final = OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="hi", + sequence_number=3, +) + + +@pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) +def test_responses_sse_chunk_a_hook_dumped_from_an_event_streams_as_that_event( + mapping: Callable[[Mapping[str, object]], Mapping[str, object]], +) -> None: + frame: Final = _format_responses_sse_chunk(chunk=mapping(_DUMPED_DELTA.model_dump())) + + assert isinstance(frame, str) + assert json.loads(frame.removeprefix("data: ")) == json.loads(_DUMPED_DELTA.model_dump_json()) + + @pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) def test_responses_stream_frames_turn_mapping_keys_json_cannot_take_into_strings( mapping: Callable[[Mapping[object, object]], Mapping[object, object]], From f68ec184301d7a0486476693c77cecc6f72ca437 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:00:27 -0700 Subject: [PATCH 22/23] fix(proxy): print container subclasses and SDK models in Responses stream frames the way base did --- .../common_utils/responses_stream_errors.py | 34 ++++++------ .../proxy_server/test_streaming_helpers.py | 53 +++++++++++++++++++ 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 8916c0be8a9..f4acb2858fb 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,5 +1,6 @@ import json import time +from collections import Counter, OrderedDict, defaultdict from collections.abc import Iterable, Mapping, Sequence from enum import Enum from http import HTTPStatus @@ -7,7 +8,7 @@ from types import MappingProxyType from typing import Final from pydantic import BaseModel, ConfigDict, ValidationError, field_validator -from typing_extensions import TypeIs # noqa: TID251 # exact-type checks keep subclasses out, isinstance cannot +from typing_extensions import TypeIs # noqa: TID251 # the guards narrow hook values to object items, isinstance cannot from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid @@ -77,12 +78,19 @@ class _BaseForm(Enum): DUMPED = "dumped" -def _is_plain_dict(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: exact type check - return type(value) is dict +def _prints_every_item(kind: type[object]) -> bool: + repr_owner: Final = next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object) + return kind.__str__ is object.__str__ and repr_owner in (dict, list, tuple, OrderedDict, defaultdict, Counter) -def _is_plain_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: exact type check - return type(value) in (list, tuple) +def _is_shown_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: narrows dicts to object items + kind: Final = type(value) + return isinstance(value, dict) and _prints_every_item(kind) + + +def _is_shown_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: narrows lists and tuples to object items + kind: Final = type(value) + return isinstance(value, (list, tuple)) and _prints_every_item(kind) def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed @@ -93,13 +101,9 @@ def _prints_as_dict(proxy: Mapping[object, object]) -> bool: return repr(proxy) == f"mappingproxy({dict(proxy)!r})" -def _prints_its_fields(model: BaseModel) -> bool: +def _repr_prints_its_fields(model: BaseModel) -> bool: kind: Final = type(model) - return ( - kind.__repr__ is BaseModel.__repr__ - and kind.__str__ is BaseModel.__str__ - and kind.__repr_str__ is BaseModel.__repr_str__ - ) + return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__ def _str_skips_its_repr(model: BaseModel) -> bool: @@ -140,11 +144,11 @@ def _json_key(key: object, form: _BaseForm) -> object: def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None: - if _is_plain_dict(value): + if _is_shown_mapping(value): return value.items(), form if _is_proxy(value) and _prints_as_dict(value): return value.items(), _BaseForm.PRINTED - if isinstance(value, BaseModel) and _prints_its_fields(value): + if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value): return value.__repr_args__(), _BaseForm.PRINTED return None @@ -156,7 +160,7 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: return _text(value, form) inner_path: Final = path | frozenset((id(value),)) try: - if _is_plain_sequence(value): + if _is_shown_sequence(value): shown_items: Final = [_json_safe(item, form, inner_path) for item in value] return [item for item in shown_items if item is not _DROPPED] expansion: Final = _expansion(value, form) @@ -170,7 +174,7 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: def responses_chunk_json(chunk: object) -> str | None: - if not _is_plain_dict(chunk) and not _is_proxy(chunk): + if not _is_shown_mapping(chunk) and not _is_proxy(chunk): return None shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset()) return None if isinstance(shown, str) or shown is _DROPPED else json.dumps(shown) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index a2045995093..1b900d78336 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio import functools import json +from collections import OrderedDict from collections.abc import AsyncIterator, Callable, Iterable, Mapping from datetime import datetime, timezone from enum import Enum @@ -31,6 +32,7 @@ import pytest from fastapi import HTTPException, Response from fastapi.responses import JSONResponse, StreamingResponse from openai import APIError as OpenAIAPIError +from openai.types.responses import ResponseOutputMessage, ResponseOutputText from pydantic import BaseModel, Field import litellm @@ -333,6 +335,28 @@ def test_responses_error_event_never_shows_what_a_nested_value_hides_from_its_st assert "hook-canary" not in frame +class _RigTuple(tuple[object, ...]): + pass + + +@pytest.mark.parametrize( + "detail", + [ + OrderedDict(member=_MaskedStrModel(token="hook-canary")), + _RigTuple((_MaskedStrModel(token="hook-canary"),)), + _HookOutputItem(id="msg_nested", type="message"), + ], + ids=["dict_subclass", "tuple_subclass", "model"], +) +def test_responses_error_event_sends_a_detail_the_way_base_json_dumps_did(detail: object) -> None: + frame: Final = ResponsesStreamErrorState().format_error( + {"message": "blocked", "provider_specific_fields": {"detail": detail}} + ) + + payload: Final = json.loads(frame.split("data: ", 1)[1]) + assert payload["error"]["provider_specific_fields"]["detail"] == json.loads(json.dumps(detail, default=str)) + + class _RigColor(Enum): RED = 1 @@ -406,6 +430,35 @@ def test_responses_sse_chunk_a_hook_dumped_from_an_event_streams_as_that_event( assert json.loads(frame.removeprefix("data: ")) == json.loads(_DUMPED_DELTA.model_dump_json()) +_COMPLETED_WITH_SDK_OUTPUT: Final = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + sequence_number=4, + response=ResponsesAPIResponse( + id="resp_1", + object="response", + created_at=0, + model="gpt-x", + status="completed", + output=[ + ResponseOutputMessage( + id="msg_1", + type="message", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="hi", annotations=[])], + ) + ], + ), +) + + +def test_responses_sse_chunk_a_hook_copied_from_an_event_streams_as_that_event() -> None: + frame: Final = _format_responses_sse_chunk(chunk=dict(_COMPLETED_WITH_SDK_OUTPUT)) + + assert isinstance(frame, str) + assert json.loads(frame.removeprefix("data: ")) == json.loads(_COMPLETED_WITH_SDK_OUTPUT.model_dump_json()) + + @pytest.mark.parametrize("mapping", [dict, MappingProxyType], ids=["dict", "mapping_proxy"]) def test_responses_stream_frames_turn_mapping_keys_json_cannot_take_into_strings( mapping: Callable[[Mapping[object, object]], Mapping[object, object]], From 79f719aece45c5357a63a6c2e84fea0e61ad5970 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:32:09 -0700 Subject: [PATCH 23/23] fix(proxy): read hook container storage the way repr does and send error-event models as str() --- .../common_utils/responses_stream_errors.py | 74 +++++++++++------- .../proxy_server/test_streaming_helpers.py | 77 +++++++++++++++++-- 2 files changed, 117 insertions(+), 34 deletions(-) diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index f4acb2858fb..951650543a7 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -1,7 +1,7 @@ import json import time from collections import Counter, OrderedDict, defaultdict -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from enum import Enum from http import HTTPStatus from types import MappingProxyType @@ -78,27 +78,35 @@ class _BaseForm(Enum): DUMPED = "dumped" -def _prints_every_item(kind: type[object]) -> bool: +def _prints_every_item(kind: type[object], form: _BaseForm) -> bool: repr_owner: Final = next((owner for owner in kind.__mro__ if "__repr__" in owner.__dict__), object) - return kind.__str__ is object.__str__ and repr_owner in (dict, list, tuple, OrderedDict, defaultdict, Counter) + if form is _BaseForm.DUMPED: + return kind.__str__ is object.__str__ and repr_owner in (dict, list, tuple, OrderedDict, defaultdict, Counter) + return ( + kind.__str__ is object.__str__ + and kind.__format__ is object.__format__ + and (kind in (OrderedDict, Counter) or repr_owner in (dict, list, tuple, defaultdict)) + ) -def _is_shown_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: narrows dicts to object items +def _is_shown_dict(value: object, form: _BaseForm) -> TypeIs[Mapping[object, object]]: # guard-ok: dict items as object kind: Final = type(value) - return isinstance(value, dict) and _prints_every_item(kind) + return isinstance(value, dict) and _prints_every_item(kind, form) -def _is_shown_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: narrows lists and tuples to object items +def _is_shown_sequence(value: object, form: _BaseForm) -> TypeIs[Sequence[object]]: # guard-ok: items as object kind: Final = type(value) - return isinstance(value, (list, tuple)) and _prints_every_item(kind) + return isinstance(value, (list, tuple)) and _prints_every_item(kind, form) def _is_proxy(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: mappingproxy cannot be subclassed return isinstance(value, MappingProxyType) -def _prints_as_dict(proxy: Mapping[object, object]) -> bool: - return repr(proxy) == f"mappingproxy({dict(proxy)!r})" +def _printed_proxy_items(proxy: Mapping[object, object], printed: str) -> Iterable[tuple[object, object]] | None: + snapshot: Final = dict(proxy) + shown: Final = repr(snapshot) + return snapshot.items() if printed in (shown, f"mappingproxy({shown})") else None def _repr_prints_its_fields(model: BaseModel) -> bool: @@ -106,11 +114,6 @@ def _repr_prints_its_fields(model: BaseModel) -> bool: return kind.__repr__ is BaseModel.__repr__ and kind.__repr_str__ is BaseModel.__repr_str__ -def _str_skips_its_repr(model: BaseModel) -> bool: - kind: Final = type(model) - return kind.__str__ is BaseModel.__str__ and kind.__repr__ is not BaseModel.__repr__ - - def _plain_scalar(value: str | float) -> str | float: if isinstance(value, str): return str.__str__(value) @@ -130,11 +133,7 @@ def _text(value: object, form: _BaseForm) -> object: try: if isinstance(value, (str, int, float)) and (form is _BaseForm.DUMPED or _repr_shows_its_value(value)): return _plain_scalar(value) - if form is _BaseForm.PRINTED: - return repr(value) - if isinstance(value, BaseModel) and _str_skips_its_repr(value): - return repr(value) - return str(value) + return repr(value) if form is _BaseForm.PRINTED else str(value) except Exception: # noqa: BLE001 # hook values run their own __repr__ and __str__, so a failure drops the value return _DROPPED @@ -143,11 +142,31 @@ def _json_key(key: object, form: _BaseForm) -> object: return key if key is None or type(key) in (str, int, float, bool) else _text(key, form) +def _shown_sequence(value: object, form: _BaseForm) -> Iterator[object] | None: + if not _is_shown_sequence(value, form): + return None + if form is _BaseForm.PRINTED and isinstance(value, list): + return list[object].__iter__(value) + if form is _BaseForm.PRINTED and isinstance(value, tuple): + return tuple[object, ...].__iter__(value) + return iter(value) + + +def _shown_items(value: object, form: _BaseForm) -> Iterable[tuple[object, object]] | None: + if not _is_shown_dict(value, form): + return None + if form is _BaseForm.PRINTED and isinstance(value, dict): + return dict[object, object].items(value) + return value.items() + + def _expansion(value: object, form: _BaseForm) -> tuple[Iterable[tuple[object, object]], _BaseForm] | None: - if _is_shown_mapping(value): - return value.items(), form - if _is_proxy(value) and _prints_as_dict(value): - return value.items(), _BaseForm.PRINTED + shown_items: Final = _shown_items(value, form) + if shown_items is not None: + return shown_items, form + if _is_proxy(value): + proxy_items: Final = _printed_proxy_items(value, repr(value) if form is _BaseForm.PRINTED else str(value)) + return None if proxy_items is None else (proxy_items, _BaseForm.PRINTED) if form is _BaseForm.PRINTED and isinstance(value, BaseModel) and _repr_prints_its_fields(value): return value.__repr_args__(), _BaseForm.PRINTED return None @@ -160,8 +179,9 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: return _text(value, form) inner_path: Final = path | frozenset((id(value),)) try: - if _is_shown_sequence(value): - shown_items: Final = [_json_safe(item, form, inner_path) for item in value] + sequence: Final = _shown_sequence(value, form) + if sequence is not None: + shown_items: Final = [_json_safe(item, form, inner_path) for item in sequence] return [item for item in shown_items if item is not _DROPPED] expansion: Final = _expansion(value, form) if expansion is None: @@ -174,7 +194,9 @@ def _json_safe(value: object, form: _BaseForm, path: frozenset[int]) -> object: def responses_chunk_json(chunk: object) -> str | None: - if not _is_shown_mapping(chunk) and not _is_proxy(chunk): + if _is_proxy(chunk) and _printed_proxy_items(chunk, format(chunk, "")) is None: + return None + if not _is_shown_dict(chunk, _BaseForm.PRINTED) and not _is_proxy(chunk): return None shown: Final = _json_safe(chunk, _BaseForm.PRINTED, frozenset()) return None if isinstance(shown, str) or shown is _DROPPED else json.dumps(shown) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 1b900d78336..2e3bd57d43d 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -18,8 +18,8 @@ from __future__ import annotations import asyncio import functools import json -from collections import OrderedDict -from collections.abc import AsyncIterator, Callable, Iterable, Mapping +from collections import Counter, OrderedDict +from collections.abc import AsyncIterator, Callable, ItemsView, Iterable, Iterator, Mapping from datetime import datetime, timezone from enum import Enum from http import HTTPStatus @@ -33,7 +33,7 @@ from fastapi import HTTPException, Response from fastapi.responses import JSONResponse, StreamingResponse from openai import APIError as OpenAIAPIError from openai.types.responses import ResponseOutputMessage, ResponseOutputText -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, PrivateAttr import litellm import litellm.proxy.proxy_server as ps @@ -289,7 +289,6 @@ _MASKED_IN_EVERY_FRAME: Final = ( MappingProxyType({"headers": _REDACTED_HEADERS}), MappingProxyType(_REDACTED_HEADERS), _HiddenFieldModel(user="rig", password="hook-canary"), - _MaskedReprModel(token="hook-canary"), _MaskedReprStrModel(token="hook-canary"), _MaskedReprArgsModel(token="hook-canary"), ) @@ -299,16 +298,44 @@ _MASKED_IN_EVERY_FRAME_IDS: Final = ( "proxy_over_dict_holding_redacted_dict", "proxy_wrapping_redacted_dict", "repr_false_field", - "masked_repr_model", "masked_repr_str_model", "masked_repr_args_model", ) +class _UnmaskingDict(dict[str, str]): + def items(self) -> ItemsView[str, str]: # pyright: ignore[reportIncompatibleMethodOverride] # rig hides storage + return {key: "hook-canary" for key in dict.keys(self)}.items() + + +class _UnmaskingList(list[str]): + def __iter__(self) -> Iterator[str]: + return iter(("hook-canary",)) + + +class _MostCommonMaskedCounter(Counter[str]): + def most_common(self, n: int | None = None) -> list[tuple[str, int]]: + return [(key, count) for key, count in super().most_common(n) if key != "hook-canary"] + + @pytest.mark.parametrize( "nested", - [*_MASKED_IN_EVERY_FRAME, _MaskedReprString("hook-canary")], - ids=[*_MASKED_IN_EVERY_FRAME_IDS, "masked_repr_string"], + [ + *_MASKED_IN_EVERY_FRAME, + _MaskedReprModel(token="hook-canary"), + _MaskedReprString("hook-canary"), + _UnmaskingDict(token="enc:MASKED"), + _UnmaskingList(("enc:MASKED",)), + _MostCommonMaskedCounter(("shown", "hook-canary")), + ], + ids=[ + *_MASKED_IN_EVERY_FRAME_IDS, + "masked_repr_model", + "masked_repr_string", + "dict_items_differ_from_storage", + "list_iter_differs_from_storage", + "counter_most_common_masks", + ], ) def test_responses_sse_chunk_never_shows_what_a_nested_value_hides_from_its_repr(nested: object) -> None: frame: Final = _format_responses_sse_chunk( @@ -335,18 +362,52 @@ def test_responses_error_event_never_shows_what_a_nested_value_hides_from_its_st assert "hook-canary" not in frame +class _OwnFormatDict(dict[str, str]): + def __format__(self, format_spec: str) -> str: + return json.dumps({"type": "response.output_text.delta", "token": "MASKED"}) + + +class _MaskedStrDict(dict[str, str]): + def __str__(self) -> str: + return "{'type': 'response.output_text.delta', 'token': 'MASKED'}" + + +@pytest.mark.parametrize( + "chunk", + [ + _OwnFormatDict(type="response.output_text.delta", token="hook-canary"), + MappingProxyType(_MaskedStrDict(type="response.output_text.delta", token="hook-canary")), + ], + ids=["own_format", "proxy_over_masked_str"], +) +def test_responses_sse_chunk_sends_a_whole_chunk_as_the_text_its_printer_gave(chunk: object) -> None: + frame: Final = _format_responses_sse_chunk(chunk=chunk) + + assert isinstance(frame, str) + assert "hook-canary" not in frame + + class _RigTuple(tuple[object, ...]): pass +class _RevealingReprModel(BaseModel, frozen=True): + a: int + _note: str = PrivateAttr(default="hook-canary") + + def __repr__(self) -> str: + return f"_RevealingReprModel(note={self._note})" + + @pytest.mark.parametrize( "detail", [ OrderedDict(member=_MaskedStrModel(token="hook-canary")), _RigTuple((_MaskedStrModel(token="hook-canary"),)), _HookOutputItem(id="msg_nested", type="message"), + _RevealingReprModel(a=1), ], - ids=["dict_subclass", "tuple_subclass", "model"], + ids=["dict_subclass", "tuple_subclass", "model", "model_with_own_repr"], ) def test_responses_error_event_sends_a_detail_the_way_base_json_dumps_did(detail: object) -> None: frame: Final = ResponsesStreamErrorState().format_error(