From 9d5edddb0aa6fb6a9df83347bf075c2b4bf15d42 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:07:17 -0700 Subject: [PATCH] fix(proxy): keep /v1/messages keepalive pings out of half-sent SSE events and send late failures as Anthropic error events --- litellm/proxy/common_request_processing.py | 39 ++++++-- litellm/proxy/common_utils/sse_keepalive.py | 64 ++++-------- .../proxy/common_utils/test_sse_keepalive.py | 36 +++++++ .../proxy/test_common_request_processing.py | 97 +++++++++++++++++++ 4 files changed, 186 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9bb451495c8..5b2d02e4025 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -156,6 +156,7 @@ KNOWN_PROXY_ROUTES: Final = frozenset( route for member in LiteLLMRoutes for route in member.value if route.startswith("/") ) +SseErrorDialect: TypeAlias = Literal["openai", "anthropic"] ProxyRouteType: TypeAlias = Literal[ "acompletion", "aembedding", @@ -1374,13 +1375,38 @@ async def _aclose_late_response(produced: Response) -> None: verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc) -async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]: +def _late_failure_frames(exc: Exception, error_dialect: SseErrorDialect) -> tuple[str, ...]: + match error_dialect: + case "openai": + return _sse_error_frames(sse_error_payload(exc)[1]) + case "anthropic": + proxy_exc: Final = proxy_exception_from_route_error(exc) + return ( + anthropic_error_sse_frame( + status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR), + raw_message=proxy_exc.message, + ), + ) + + +def _late_body_frames(produced: Response, error_dialect: SseErrorDialect) -> tuple[bytes, ...]: + body: Final = bytes(produced.body) or b"{}" + if error_dialect == "anthropic" and produced.status_code >= status.HTTP_400_BAD_REQUEST: + return ( + anthropic_error_sse_frame( + status_code=produced.status_code, raw_message=body.decode(errors="replace") + ).encode(), + ) + return b"data: " + body + b"\n\n", b"data: [DONE]\n\n" + + +async def _relay_late_response(produced: Response, error_dialect: SseErrorDialect) -> AsyncGenerator[bytes, None]: """Replay a Response that was built after a keepalive had already opened the wire.""" if not isinstance(produced, StreamingResponse): # The status line is already on the wire, so a non-streaming body, an error # body included, can only reach the client as an SSE frame. - yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n" - yield b"data: [DONE]\n\n" + for frame in _late_body_frames(produced, error_dialect): + yield frame return try: @@ -1422,6 +1448,7 @@ async def open_sse_before_first_byte( ping_interval_seconds: float | str | None, media_type: str = "text/event-stream", on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None, + error_dialect: SseErrorDialect = "openai", ) -> _LateResponseT | StreamingResponse: """Write SSE keepalive comments while the upstream LLM call is still in flight. @@ -1467,11 +1494,10 @@ async def open_sse_before_first_byte( # would never fire and the failure would go unaudited. The hook # also gets to sanitize what reaches the client, by returning or # raising a replacement, so its answer decides the frame. - _, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure)) - for frame in _sse_error_frames(error_obj): + for frame in _late_failure_frames(await _sanitized_late_failure(exc, on_late_failure), error_dialect): yield frame.encode() return - async for chunk in _relay_late_response(produced): + async for chunk in _relay_late_response(produced, error_dialect): yield chunk finally: if not produce_task.done(): @@ -2907,6 +2933,7 @@ class ProxyBaseLLMRequestProcessing: ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, + error_dialect="anthropic" if route_type == "anthropic_messages" else "openai", ) async def _process_llm_request( diff --git a/litellm/proxy/common_utils/sse_keepalive.py b/litellm/proxy/common_utils/sse_keepalive.py index d9685971f52..be9c24532b4 100644 --- a/litellm/proxy/common_utils/sse_keepalive.py +++ b/litellm/proxy/common_utils/sse_keepalive.py @@ -2,7 +2,7 @@ import asyncio import contextlib import math from collections.abc import AsyncGenerator, Iterable, Mapping -from typing import Final +from typing import Final, TypeVar import anyio @@ -17,6 +17,7 @@ _SSE_FRAME_DELIMITERS: Final = (b"\r\n\r\n", b"\n\n", b"\r\r") _SSE_DELIMITER_LOOKBACK: Final = max(len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS) SSE_STREAM_START_TAIL: Final = b"\n\n" _SSE_MEDIA_TYPE: Final = "text/event-stream" +_ChunkT: Final = TypeVar("_ChunkT", str, bytes) def coerce_keepalive_interval(ping_interval_seconds: float | str | None) -> float | None: @@ -61,21 +62,32 @@ def wrap_sse_stream_with_keepalive_pings( async def _keepalive_ping_stream( - stream: AsyncGenerator[str, None], + stream: AsyncGenerator[_ChunkT, None], ping_interval_seconds: float, - ping_chunk: str, -) -> AsyncGenerator[str, None]: + ping_chunk: _ChunkT, +) -> AsyncGenerator[_ChunkT, None]: pending = asyncio.ensure_future(stream.__anext__()) + # The tail of the bytes relayed so far, long enough to hold any delimiter. + # Seeded as a delimiter because a stream starts at a frame boundary, and kept + # across chunks because a delimiter can be split between two transport reads, + # which testing only the latest chunk would miss for the rest of the stream. + recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes try: while True: - await asyncio.wait({pending}, timeout=ping_interval_seconds) + await asyncio.wait((pending,), timeout=ping_interval_seconds) if not pending.done(): - yield ping_chunk + # A relayed chunk can be a raw transport read rather than a whole + # SSE frame, so an upstream that stalls halfway through a frame + # must not have a ping spliced into it. + if recent_tail.endswith(_SSE_FRAME_DELIMITERS): + yield ping_chunk continue try: - yield pending.result() + chunk: _ChunkT = pending.result() except StopAsyncIteration: return + recent_tail = advance_sse_tail(recent_tail, chunk) + yield chunk pending = asyncio.ensure_future(stream.__anext__()) finally: pending.cancel() @@ -116,43 +128,7 @@ def wrap_passthrough_sse_bytes_with_keepalive_pings( interval: Final = coerce_keepalive_interval(ping_interval_seconds) if interval is None or not is_sse_content_type(upstream_headers.get("content-type")): return stream - return _keepalive_ping_byte_stream(stream=stream, ping_interval_seconds=interval) - - -async def _keepalive_ping_byte_stream( - stream: AsyncGenerator[bytes, None], - ping_interval_seconds: float, -) -> AsyncGenerator[bytes, None]: - pending = asyncio.ensure_future(stream.__anext__()) - # The tail of the bytes relayed so far, long enough to hold any delimiter. - # Seeded as a delimiter because a stream starts at a frame boundary, and kept - # across chunks because a delimiter can be split between two transport reads, - # which testing only the latest chunk would miss for the rest of the stream. - recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes - try: - while True: - await asyncio.wait((pending,), timeout=ping_interval_seconds) - if not pending.done(): - # The relayed chunks are raw transport reads, not whole SSE - # frames, so an upstream that stalls halfway through a frame - # must not have a comment spliced into it. - if recent_tail.endswith(_SSE_FRAME_DELIMITERS): - yield SSE_COMMENT_PING_BYTES - continue - try: - chunk: bytes = pending.result() - except StopAsyncIteration: - return - if chunk: - recent_tail = (recent_tail + chunk)[-_SSE_DELIMITER_LOOKBACK:] - yield chunk - pending = asyncio.ensure_future(stream.__anext__()) - finally: - pending.cancel() - with anyio.CancelScope(shield=True): - with contextlib.suppress(BaseException): - await pending - await stream.aclose() + return _keepalive_ping_stream(stream=stream, ping_interval_seconds=interval, ping_chunk=SSE_COMMENT_PING_BYTES) def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes: diff --git a/tests/unit/proxy/common_utils/test_sse_keepalive.py b/tests/unit/proxy/common_utils/test_sse_keepalive.py index 228fd5bcae6..7608d5dc438 100644 --- a/tests/unit/proxy/common_utils/test_sse_keepalive.py +++ b/tests/unit/proxy/common_utils/test_sse_keepalive.py @@ -73,6 +73,42 @@ async def test_ping_emitted_while_waiting_for_first_chunk(): assert collected[-1] == MESSAGE_START_CHUNK +@pytest.mark.asyncio +async def test_ping_is_never_spliced_into_a_half_delivered_frame(): + async def stalls_mid_frame() -> AsyncGenerator[str, None]: + yield 'data: {"type":"content_block_delta","index":0,"delta":{' + await asyncio.sleep(0.3) + yield '"type":"text_delta","text":"hi"}}\n\n' + + wrapped: Final = wrap_sse_stream_with_keepalive_pings(stream=stalls_mid_frame(), ping_interval_seconds=0.05) + collected: Final = [chunk async for chunk in wrapped] + + assert collected == [ + 'data: {"type":"content_block_delta","index":0,"delta":{', + '"type":"text_delta","text":"hi"}}\n\n', + ] + + +@pytest.mark.asyncio +async def test_ping_resumes_once_the_stalled_frame_completes(): + async def stalls_mid_frame_then_at_boundary() -> AsyncGenerator[str, None]: + yield "data: {" + await asyncio.sleep(0.2) + yield "}\n" + yield "\n" + await asyncio.sleep(0.2) + yield TEXT_DELTA_CHUNK + + wrapped: Final = wrap_sse_stream_with_keepalive_pings( + stream=stalls_mid_frame_then_at_boundary(), ping_interval_seconds=0.05 + ) + collected: Final = [chunk async for chunk in wrapped] + + ping_index: Final = collected.index(ANTHROPIC_PING_SSE_CHUNK) + assert collected[:ping_index] == ["data: {", "}\n", "\n"] + assert collected[-1] == TEXT_DELTA_CHUNK + + @pytest.mark.asyncio async def test_no_pings_when_chunks_arrive_faster_than_interval(): async def fast_stream() -> AsyncGenerator[str, None]: diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fdcbcbfcd05..fa4fefeb191 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10541,6 +10541,74 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" +def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError: + def serve(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire) + + client: Final = anthropic.Anthropic( + api_key="sk-test", + base_url="http://proxy.test", + http_client=httpx.Client(transport=httpx.MockTransport(serve)), + max_retries=0, + ) + with pytest.raises(anthropic.APIStatusError) as raised: + for _ in client.messages.create( + model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True + ): + pass + return raised.value + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "late_failure, expected_message", + [ + (HTTPException(status_code=429, detail="rate limited"), "rate limited"), + ( + litellm.RateLimitError(message="No deployments available", llm_provider="anthropic", model="m"), + "No deployments available", + ), + ], + ids=["http-exception", "router-exception"], +) +async def test_ttft_keepalive_raises_a_late_failure_in_the_anthropic_sdk(late_failure, expected_message): + """An Anthropic client picks stream events by their `event:` name, so a bare + `data:` error frame is skipped and the stream just ends with no message.""" + + async def slow_failure(): + await asyncio.sleep(0.2) + raise late_failure + + response = await open_sse_before_first_byte( + slow_failure(), ping_interval_seconds=0.05, error_dialect="anthropic" + ) + collected = await _drain(response) + + assert collected[0] == TTFT_PING + body = _anthropic_sdk_stream_error(b"".join(collected)).body + assert isinstance(body, dict) + assert body["type"] == "error" + assert body["error"]["type"] == "rate_limit_error" + assert expected_message in body["error"]["message"] + + +@pytest.mark.asyncio +async def test_ttft_keepalive_raises_a_late_error_body_in_the_anthropic_sdk(): + """On /v1/messages a stream that fails before its first byte comes back as an + Anthropic JSON error, which has to reach the client as an `event: error`.""" + envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} + + async def slow_json(): + await asyncio.sleep(0.2) + return JSONResponse(status_code=529, content=envelope) + + response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect="anthropic") + collected = await _drain(response) + + assert collected[0] == TTFT_PING + assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope + + @pytest.mark.asyncio async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed(): """Starlette never calls the produced response, so its own cleanup never runs @@ -10821,6 +10889,35 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke assert collected[-1] == b"data: [DONE]\n\n" +@pytest.mark.asyncio +async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_anthropic_sdk(): + async def slow_failure(self, **kwargs): + await asyncio.sleep(0.25) + raise HTTPException(status_code=503, detail="upstream exploded") + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + processor = ProxyBaseLLMRequestProcessing(data={"model": "claude-sonnet-4-5", "stream": True}) + + with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05): + with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_failure): + response = await processor.base_process_llm_request( + request=MagicMock(spec=Request), + fastapi_response=Response(), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + route_type="anthropic_messages", + proxy_logging_obj=proxy_logging_obj, + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + ) + collected = await _drain(response) + + assert collected[0] == TTFT_PING + body = _anthropic_sdk_stream_error(b"".join(collected)).body + assert isinstance(body, dict) + assert body["error"]["message"] == "upstream exploded" + + @pytest.mark.asyncio @pytest.mark.parametrize( "deployment_keepalive, expect_ping",