mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): keep /v1/messages keepalive pings out of half-sent SSE events and send late failures as Anthropic error events
This commit is contained in:
parent
57c1df8c33
commit
9d5edddb0a
4 changed files with 186 additions and 50 deletions
|
|
@ -156,6 +156,7 @@ KNOWN_PROXY_ROUTES: Final = frozenset(
|
|||
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
|
||||
)
|
||||
|
||||
SseErrorDialect: TypeAlias = Literal["openai", "anthropic"]
|
||||
ProxyRouteType: TypeAlias = Literal[
|
||||
"acompletion",
|
||||
"aembedding",
|
||||
|
|
@ -1374,13 +1375,38 @@ async def _aclose_late_response(produced: Response) -> None:
|
|||
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
|
||||
|
||||
|
||||
async def _relay_late_response(produced: Response) -> AsyncGenerator[bytes, None]:
|
||||
def _late_failure_frames(exc: Exception, error_dialect: SseErrorDialect) -> tuple[str, ...]:
|
||||
match error_dialect:
|
||||
case "openai":
|
||||
return _sse_error_frames(sse_error_payload(exc)[1])
|
||||
case "anthropic":
|
||||
proxy_exc: Final = proxy_exception_from_route_error(exc)
|
||||
return (
|
||||
anthropic_error_sse_frame(
|
||||
status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
raw_message=proxy_exc.message,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _late_body_frames(produced: Response, error_dialect: SseErrorDialect) -> tuple[bytes, ...]:
|
||||
body: Final = bytes(produced.body) or b"{}"
|
||||
if error_dialect == "anthropic" and produced.status_code >= status.HTTP_400_BAD_REQUEST:
|
||||
return (
|
||||
anthropic_error_sse_frame(
|
||||
status_code=produced.status_code, raw_message=body.decode(errors="replace")
|
||||
).encode(),
|
||||
)
|
||||
return b"data: " + body + b"\n\n", b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
async def _relay_late_response(produced: Response, error_dialect: SseErrorDialect) -> AsyncGenerator[bytes, None]:
|
||||
"""Replay a Response that was built after a keepalive had already opened the wire."""
|
||||
if not isinstance(produced, StreamingResponse):
|
||||
# The status line is already on the wire, so a non-streaming body, an error
|
||||
# body included, can only reach the client as an SSE frame.
|
||||
yield b"data: " + (bytes(produced.body) or b"{}") + b"\n\n"
|
||||
yield b"data: [DONE]\n\n"
|
||||
for frame in _late_body_frames(produced, error_dialect):
|
||||
yield frame
|
||||
return
|
||||
|
||||
try:
|
||||
|
|
@ -1422,6 +1448,7 @@ async def open_sse_before_first_byte(
|
|||
ping_interval_seconds: float | str | None,
|
||||
media_type: str = "text/event-stream",
|
||||
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
|
||||
error_dialect: SseErrorDialect = "openai",
|
||||
) -> _LateResponseT | StreamingResponse:
|
||||
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
|
||||
|
||||
|
|
@ -1467,11 +1494,10 @@ async def open_sse_before_first_byte(
|
|||
# would never fire and the failure would go unaudited. The hook
|
||||
# also gets to sanitize what reaches the client, by returning or
|
||||
# raising a replacement, so its answer decides the frame.
|
||||
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
|
||||
for frame in _sse_error_frames(error_obj):
|
||||
for frame in _late_failure_frames(await _sanitized_late_failure(exc, on_late_failure), error_dialect):
|
||||
yield frame.encode()
|
||||
return
|
||||
async for chunk in _relay_late_response(produced):
|
||||
async for chunk in _relay_late_response(produced, error_dialect):
|
||||
yield chunk
|
||||
finally:
|
||||
if not produce_task.done():
|
||||
|
|
@ -2907,6 +2933,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
),
|
||||
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
|
||||
on_late_failure=_audit_late_failure,
|
||||
error_dialect="anthropic" if route_type == "anthropic_messages" else "openai",
|
||||
)
|
||||
|
||||
async def _process_llm_request(
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import asyncio
|
|||
import contextlib
|
||||
import math
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping
|
||||
from typing import Final
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import anyio
|
||||
|
||||
|
|
@ -17,6 +17,7 @@ _SSE_FRAME_DELIMITERS: Final = (b"\r\n\r\n", b"\n\n", b"\r\r")
|
|||
_SSE_DELIMITER_LOOKBACK: Final = max(len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS)
|
||||
SSE_STREAM_START_TAIL: Final = b"\n\n"
|
||||
_SSE_MEDIA_TYPE: Final = "text/event-stream"
|
||||
_ChunkT: Final = TypeVar("_ChunkT", str, bytes)
|
||||
|
||||
|
||||
def coerce_keepalive_interval(ping_interval_seconds: float | str | None) -> float | None:
|
||||
|
|
@ -61,21 +62,32 @@ def wrap_sse_stream_with_keepalive_pings(
|
|||
|
||||
|
||||
async def _keepalive_ping_stream(
|
||||
stream: AsyncGenerator[str, None],
|
||||
stream: AsyncGenerator[_ChunkT, None],
|
||||
ping_interval_seconds: float,
|
||||
ping_chunk: str,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
ping_chunk: _ChunkT,
|
||||
) -> AsyncGenerator[_ChunkT, None]:
|
||||
pending = asyncio.ensure_future(stream.__anext__())
|
||||
# The tail of the bytes relayed so far, long enough to hold any delimiter.
|
||||
# Seeded as a delimiter because a stream starts at a frame boundary, and kept
|
||||
# across chunks because a delimiter can be split between two transport reads,
|
||||
# which testing only the latest chunk would miss for the rest of the stream.
|
||||
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes
|
||||
try:
|
||||
while True:
|
||||
await asyncio.wait({pending}, timeout=ping_interval_seconds)
|
||||
await asyncio.wait((pending,), timeout=ping_interval_seconds)
|
||||
if not pending.done():
|
||||
yield ping_chunk
|
||||
# A relayed chunk can be a raw transport read rather than a whole
|
||||
# SSE frame, so an upstream that stalls halfway through a frame
|
||||
# must not have a ping spliced into it.
|
||||
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
|
||||
yield ping_chunk
|
||||
continue
|
||||
try:
|
||||
yield pending.result()
|
||||
chunk: _ChunkT = pending.result()
|
||||
except StopAsyncIteration:
|
||||
return
|
||||
recent_tail = advance_sse_tail(recent_tail, chunk)
|
||||
yield chunk
|
||||
pending = asyncio.ensure_future(stream.__anext__())
|
||||
finally:
|
||||
pending.cancel()
|
||||
|
|
@ -116,43 +128,7 @@ def wrap_passthrough_sse_bytes_with_keepalive_pings(
|
|||
interval: Final = coerce_keepalive_interval(ping_interval_seconds)
|
||||
if interval is None or not is_sse_content_type(upstream_headers.get("content-type")):
|
||||
return stream
|
||||
return _keepalive_ping_byte_stream(stream=stream, ping_interval_seconds=interval)
|
||||
|
||||
|
||||
async def _keepalive_ping_byte_stream(
|
||||
stream: AsyncGenerator[bytes, None],
|
||||
ping_interval_seconds: float,
|
||||
) -> AsyncGenerator[bytes, None]:
|
||||
pending = asyncio.ensure_future(stream.__anext__())
|
||||
# The tail of the bytes relayed so far, long enough to hold any delimiter.
|
||||
# Seeded as a delimiter because a stream starts at a frame boundary, and kept
|
||||
# across chunks because a delimiter can be split between two transport reads,
|
||||
# which testing only the latest chunk would miss for the rest of the stream.
|
||||
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes
|
||||
try:
|
||||
while True:
|
||||
await asyncio.wait((pending,), timeout=ping_interval_seconds)
|
||||
if not pending.done():
|
||||
# The relayed chunks are raw transport reads, not whole SSE
|
||||
# frames, so an upstream that stalls halfway through a frame
|
||||
# must not have a comment spliced into it.
|
||||
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
|
||||
yield SSE_COMMENT_PING_BYTES
|
||||
continue
|
||||
try:
|
||||
chunk: bytes = pending.result()
|
||||
except StopAsyncIteration:
|
||||
return
|
||||
if chunk:
|
||||
recent_tail = (recent_tail + chunk)[-_SSE_DELIMITER_LOOKBACK:]
|
||||
yield chunk
|
||||
pending = asyncio.ensure_future(stream.__anext__())
|
||||
finally:
|
||||
pending.cancel()
|
||||
with anyio.CancelScope(shield=True):
|
||||
with contextlib.suppress(BaseException):
|
||||
await pending
|
||||
await stream.aclose()
|
||||
return _keepalive_ping_stream(stream=stream, ping_interval_seconds=interval, ping_chunk=SSE_COMMENT_PING_BYTES)
|
||||
|
||||
|
||||
def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes:
|
||||
|
|
|
|||
|
|
@ -73,6 +73,42 @@ async def test_ping_emitted_while_waiting_for_first_chunk():
|
|||
assert collected[-1] == MESSAGE_START_CHUNK
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ping_is_never_spliced_into_a_half_delivered_frame():
|
||||
async def stalls_mid_frame() -> AsyncGenerator[str, None]:
|
||||
yield 'data: {"type":"content_block_delta","index":0,"delta":{'
|
||||
await asyncio.sleep(0.3)
|
||||
yield '"type":"text_delta","text":"hi"}}\n\n'
|
||||
|
||||
wrapped: Final = wrap_sse_stream_with_keepalive_pings(stream=stalls_mid_frame(), ping_interval_seconds=0.05)
|
||||
collected: Final = [chunk async for chunk in wrapped]
|
||||
|
||||
assert collected == [
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{',
|
||||
'"type":"text_delta","text":"hi"}}\n\n',
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ping_resumes_once_the_stalled_frame_completes():
|
||||
async def stalls_mid_frame_then_at_boundary() -> AsyncGenerator[str, None]:
|
||||
yield "data: {"
|
||||
await asyncio.sleep(0.2)
|
||||
yield "}\n"
|
||||
yield "\n"
|
||||
await asyncio.sleep(0.2)
|
||||
yield TEXT_DELTA_CHUNK
|
||||
|
||||
wrapped: Final = wrap_sse_stream_with_keepalive_pings(
|
||||
stream=stalls_mid_frame_then_at_boundary(), ping_interval_seconds=0.05
|
||||
)
|
||||
collected: Final = [chunk async for chunk in wrapped]
|
||||
|
||||
ping_index: Final = collected.index(ANTHROPIC_PING_SSE_CHUNK)
|
||||
assert collected[:ping_index] == ["data: {", "}\n", "\n"]
|
||||
assert collected[-1] == TEXT_DELTA_CHUNK
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_pings_when_chunks_arrive_faster_than_interval():
|
||||
async def fast_stream() -> AsyncGenerator[str, None]:
|
||||
|
|
|
|||
|
|
@ -10541,6 +10541,74 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
|
|||
assert collected[-1] == b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError:
|
||||
def serve(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire)
|
||||
|
||||
client: Final = anthropic.Anthropic(
|
||||
api_key="sk-test",
|
||||
base_url="http://proxy.test",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(serve)),
|
||||
max_retries=0,
|
||||
)
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
for _ in client.messages.create(
|
||||
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True
|
||||
):
|
||||
pass
|
||||
return raised.value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"late_failure, expected_message",
|
||||
[
|
||||
(HTTPException(status_code=429, detail="rate limited"), "rate limited"),
|
||||
(
|
||||
litellm.RateLimitError(message="No deployments available", llm_provider="anthropic", model="m"),
|
||||
"No deployments available",
|
||||
),
|
||||
],
|
||||
ids=["http-exception", "router-exception"],
|
||||
)
|
||||
async def test_ttft_keepalive_raises_a_late_failure_in_the_anthropic_sdk(late_failure, expected_message):
|
||||
"""An Anthropic client picks stream events by their `event:` name, so a bare
|
||||
`data:` error frame is skipped and the stream just ends with no message."""
|
||||
|
||||
async def slow_failure():
|
||||
await asyncio.sleep(0.2)
|
||||
raise late_failure
|
||||
|
||||
response = await open_sse_before_first_byte(
|
||||
slow_failure(), ping_interval_seconds=0.05, error_dialect="anthropic"
|
||||
)
|
||||
collected = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
body = _anthropic_sdk_stream_error(b"".join(collected)).body
|
||||
assert isinstance(body, dict)
|
||||
assert body["type"] == "error"
|
||||
assert body["error"]["type"] == "rate_limit_error"
|
||||
assert expected_message in body["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_raises_a_late_error_body_in_the_anthropic_sdk():
|
||||
"""On /v1/messages a stream that fails before its first byte comes back as an
|
||||
Anthropic JSON error, which has to reach the client as an `event: error`."""
|
||||
envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
|
||||
|
||||
async def slow_json():
|
||||
await asyncio.sleep(0.2)
|
||||
return JSONResponse(status_code=529, content=envelope)
|
||||
|
||||
response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect="anthropic")
|
||||
collected = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
|
||||
"""Starlette never calls the produced response, so its own cleanup never runs
|
||||
|
|
@ -10821,6 +10889,35 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke
|
|||
assert collected[-1] == b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_anthropic_sdk():
|
||||
async def slow_failure(self, **kwargs):
|
||||
await asyncio.sleep(0.25)
|
||||
raise HTTPException(status_code=503, detail="upstream exploded")
|
||||
|
||||
proxy_logging_obj = MagicMock(spec=ProxyLogging)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
processor = ProxyBaseLLMRequestProcessing(data={"model": "claude-sonnet-4-5", "stream": True})
|
||||
|
||||
with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05):
|
||||
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_failure):
|
||||
response = await processor.base_process_llm_request(
|
||||
request=MagicMock(spec=Request),
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
|
||||
route_type="anthropic_messages",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings={},
|
||||
proxy_config=MagicMock(spec=ProxyConfig),
|
||||
)
|
||||
collected = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
body = _anthropic_sdk_stream_error(b"".join(collected)).body
|
||||
assert isinstance(body, dict)
|
||||
assert body["error"]["message"] == "upstream exploded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_keepalive, expect_ping",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue