fix(proxy): keep /v1/messages keepalive pings out of half-sent SSE events and send late failures as Anthropic error events

This commit is contained in:
mateo-berri 2026-10-28 20:07:17 -07:00
parent 57c1df8c33
commit 9d5edddb0a
4 changed files with 186 additions and 50 deletions

View file

@ -156,6 +156,7 @@ KNOWN_PROXY_ROUTES: Final = frozenset(
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
)
SseErrorDialect: TypeAlias = Literal["openai", "anthropic"]
ProxyRouteType: TypeAlias = Literal[
"acompletion",
"aembedding",
@ -1374,13 +1375,38 @@ async def _aclose_late_response(produced: Response) -> None:
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]:
def _late_failure_frames(exc: Exception, error_dialect: SseErrorDialect) -> tuple[str, ...]:
match error_dialect:
case "openai":
return _sse_error_frames(sse_error_payload(exc)[1])
case "anthropic":
proxy_exc: Final = proxy_exception_from_route_error(exc)
return (
anthropic_error_sse_frame(
status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR),
raw_message=proxy_exc.message,
),
)
def _late_body_frames(produced: Response, error_dialect: SseErrorDialect) -> tuple[bytes, ...]:
body: Final = bytes(produced.body) or b"{}"
if error_dialect == "anthropic" and produced.status_code >= status.HTTP_400_BAD_REQUEST:
return (
anthropic_error_sse_frame(
status_code=produced.status_code, raw_message=body.decode(errors="replace")
).encode(),
)
return b"data: " + body + b"\n\n", b"data: [DONE]\n\n"
async def _relay_late_response(produced: Response, error_dialect: SseErrorDialect) -> AsyncGenerator[bytes, None]:
"""Replay a Response that was built after a keepalive had already opened the wire."""
if not isinstance(produced, StreamingResponse):
# The status line is already on the wire, so a non-streaming body, an error
# body included, can only reach the client as an SSE frame.
yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n"
yield b"data: [DONE]\n\n"
for frame in _late_body_frames(produced, error_dialect):
yield frame
return
try:
@ -1422,6 +1448,7 @@ async def open_sse_before_first_byte(
ping_interval_seconds: float | str | None,
media_type: str = "text/event-stream",
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
error_dialect: SseErrorDialect = "openai",
) -> _LateResponseT | StreamingResponse:
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
@ -1467,11 +1494,10 @@ async def open_sse_before_first_byte(
# would never fire and the failure would go unaudited. The hook
# also gets to sanitize what reaches the client, by returning or
# raising a replacement, so its answer decides the frame.
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
for frame in _sse_error_frames(error_obj):
for frame in _late_failure_frames(await _sanitized_late_failure(exc, on_late_failure), error_dialect):
yield frame.encode()
return
async for chunk in _relay_late_response(produced):
async for chunk in _relay_late_response(produced, error_dialect):
yield chunk
finally:
if not produce_task.done():
@ -2907,6 +2933,7 @@ class ProxyBaseLLMRequestProcessing:
),
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
on_late_failure=_audit_late_failure,
error_dialect="anthropic" if route_type == "anthropic_messages" else "openai",
)
async def _process_llm_request(

View file

@ -2,7 +2,7 @@ import asyncio
import contextlib
import math
from collections.abc import AsyncGenerator, Iterable, Mapping
from typing import Final
from typing import Final, TypeVar
import anyio
@ -17,6 +17,7 @@ _SSE_FRAME_DELIMITERS: Final = (b"\r\n\r\n", b"\n\n", b"\r\r")
_SSE_DELIMITER_LOOKBACK: Final = max(len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS)
SSE_STREAM_START_TAIL: Final = b"\n\n"
_SSE_MEDIA_TYPE: Final = "text/event-stream"
_ChunkT: Final = TypeVar("_ChunkT", str, bytes)
def coerce_keepalive_interval(ping_interval_seconds: float | str | None) -> float | None:
@ -61,21 +62,32 @@ def wrap_sse_stream_with_keepalive_pings(
async def _keepalive_ping_stream(
stream: AsyncGenerator[str, None],
stream: AsyncGenerator[_ChunkT, None],
ping_interval_seconds: float,
ping_chunk: str,
) -> AsyncGenerator[str, None]:
ping_chunk: _ChunkT,
) -> AsyncGenerator[_ChunkT, None]:
pending = asyncio.ensure_future(stream.__anext__())
# The tail of the bytes relayed so far, long enough to hold any delimiter.
# Seeded as a delimiter because a stream starts at a frame boundary, and kept
# across chunks because a delimiter can be split between two transport reads,
# which testing only the latest chunk would miss for the rest of the stream.
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes
try:
while True:
await asyncio.wait({pending}, timeout=ping_interval_seconds)
await asyncio.wait((pending,), timeout=ping_interval_seconds)
if not pending.done():
yield ping_chunk
# A relayed chunk can be a raw transport read rather than a whole
# SSE frame, so an upstream that stalls halfway through a frame
# must not have a ping spliced into it.
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
yield ping_chunk
continue
try:
yield pending.result()
chunk: _ChunkT = pending.result()
except StopAsyncIteration:
return
recent_tail = advance_sse_tail(recent_tail, chunk)
yield chunk
pending = asyncio.ensure_future(stream.__anext__())
finally:
pending.cancel()
@ -116,43 +128,7 @@ def wrap_passthrough_sse_bytes_with_keepalive_pings(
interval: Final = coerce_keepalive_interval(ping_interval_seconds)
if interval is None or not is_sse_content_type(upstream_headers.get("content-type")):
return stream
return _keepalive_ping_byte_stream(stream=stream, ping_interval_seconds=interval)
async def _keepalive_ping_byte_stream(
stream: AsyncGenerator[bytes, None],
ping_interval_seconds: float,
) -> AsyncGenerator[bytes, None]:
pending = asyncio.ensure_future(stream.__anext__())
# The tail of the bytes relayed so far, long enough to hold any delimiter.
# Seeded as a delimiter because a stream starts at a frame boundary, and kept
# across chunks because a delimiter can be split between two transport reads,
# which testing only the latest chunk would miss for the rest of the stream.
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes
try:
while True:
await asyncio.wait((pending,), timeout=ping_interval_seconds)
if not pending.done():
# The relayed chunks are raw transport reads, not whole SSE
# frames, so an upstream that stalls halfway through a frame
# must not have a comment spliced into it.
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
yield SSE_COMMENT_PING_BYTES
continue
try:
chunk: bytes = pending.result()
except StopAsyncIteration:
return
if chunk:
recent_tail = (recent_tail + chunk)[-_SSE_DELIMITER_LOOKBACK:]
yield chunk
pending = asyncio.ensure_future(stream.__anext__())
finally:
pending.cancel()
with anyio.CancelScope(shield=True):
with contextlib.suppress(BaseException):
await pending
await stream.aclose()
return _keepalive_ping_stream(stream=stream, ping_interval_seconds=interval, ping_chunk=SSE_COMMENT_PING_BYTES)
def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes:

View file

@ -73,6 +73,42 @@ async def test_ping_emitted_while_waiting_for_first_chunk():
assert collected[-1] == MESSAGE_START_CHUNK
@pytest.mark.asyncio
async def test_ping_is_never_spliced_into_a_half_delivered_frame():
async def stalls_mid_frame() -> AsyncGenerator[str, None]:
yield 'data: {"type":"content_block_delta","index":0,"delta":{'
await asyncio.sleep(0.3)
yield '"type":"text_delta","text":"hi"}}\n\n'
wrapped: Final = wrap_sse_stream_with_keepalive_pings(stream=stalls_mid_frame(), ping_interval_seconds=0.05)
collected: Final = [chunk async for chunk in wrapped]
assert collected == [
'data: {"type":"content_block_delta","index":0,"delta":{',
'"type":"text_delta","text":"hi"}}\n\n',
]
@pytest.mark.asyncio
async def test_ping_resumes_once_the_stalled_frame_completes():
async def stalls_mid_frame_then_at_boundary() -> AsyncGenerator[str, None]:
yield "data: {"
await asyncio.sleep(0.2)
yield "}\n"
yield "\n"
await asyncio.sleep(0.2)
yield TEXT_DELTA_CHUNK
wrapped: Final = wrap_sse_stream_with_keepalive_pings(
stream=stalls_mid_frame_then_at_boundary(), ping_interval_seconds=0.05
)
collected: Final = [chunk async for chunk in wrapped]
ping_index: Final = collected.index(ANTHROPIC_PING_SSE_CHUNK)
assert collected[:ping_index] == ["data: {", "}\n", "\n"]
assert collected[-1] == TEXT_DELTA_CHUNK
@pytest.mark.asyncio
async def test_no_pings_when_chunks_arrive_faster_than_interval():
async def fast_stream() -> AsyncGenerator[str, None]:

View file

@ -10541,6 +10541,74 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
assert collected[-1] == b"data: [DONE]\n\n"
def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError:
def serve(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire)
client: Final = anthropic.Anthropic(
api_key="sk-test",
base_url="http://proxy.test",
http_client=httpx.Client(transport=httpx.MockTransport(serve)),
max_retries=0,
)
with pytest.raises(anthropic.APIStatusError) as raised:
for _ in client.messages.create(
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True
):
pass
return raised.value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"late_failure, expected_message",
[
(HTTPException(status_code=429, detail="rate limited"), "rate limited"),
(
litellm.RateLimitError(message="No deployments available", llm_provider="anthropic", model="m"),
"No deployments available",
),
],
ids=["http-exception", "router-exception"],
)
async def test_ttft_keepalive_raises_a_late_failure_in_the_anthropic_sdk(late_failure, expected_message):
"""An Anthropic client picks stream events by their `event:` name, so a bare
`data:` error frame is skipped and the stream just ends with no message."""
async def slow_failure():
await asyncio.sleep(0.2)
raise late_failure
response = await open_sse_before_first_byte(
slow_failure(), ping_interval_seconds=0.05, error_dialect="anthropic"
)
collected = await _drain(response)
assert collected[0] == TTFT_PING
body = _anthropic_sdk_stream_error(b"".join(collected)).body
assert isinstance(body, dict)
assert body["type"] == "error"
assert body["error"]["type"] == "rate_limit_error"
assert expected_message in body["error"]["message"]
@pytest.mark.asyncio
async def test_ttft_keepalive_raises_a_late_error_body_in_the_anthropic_sdk():
"""On /v1/messages a stream that fails before its first byte comes back as an
Anthropic JSON error, which has to reach the client as an `event: error`."""
envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
async def slow_json():
await asyncio.sleep(0.2)
return JSONResponse(status_code=529, content=envelope)
response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect="anthropic")
collected = await _drain(response)
assert collected[0] == TTFT_PING
assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope
@pytest.mark.asyncio
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
"""Starlette never calls the produced response, so its own cleanup never runs
@ -10821,6 +10889,35 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke
assert collected[-1] == b"data: [DONE]\n\n"
@pytest.mark.asyncio
async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_anthropic_sdk():
async def slow_failure(self, **kwargs):
await asyncio.sleep(0.25)
raise HTTPException(status_code=503, detail="upstream exploded")
proxy_logging_obj = MagicMock(spec=ProxyLogging)
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
processor = ProxyBaseLLMRequestProcessing(data={"model": "claude-sonnet-4-5", "stream": True})
with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05):
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_failure):
response = await processor.base_process_llm_request(
request=MagicMock(spec=Request),
fastapi_response=Response(),
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
route_type="anthropic_messages",
proxy_logging_obj=proxy_logging_obj,
general_settings={},
proxy_config=MagicMock(spec=ProxyConfig),
)
collected = await _drain(response)
assert collected[0] == TTFT_PING
body = _anthropic_sdk_stream_error(b"".join(collected)).body
assert isinstance(body, dict)
assert body["error"]["message"] == "upstream exploded"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"deployment_keepalive, expect_ping",