From e065a2575bf31f8b11aa642d7bd6b3adb8edd0ed Mon Sep 17 00:00:00 2001 From: Refael Iliaguyev Date: Sat, 26 Sep 2026 01:36:35 +0300 Subject: [PATCH] fix(proxy): send a real error event when a /v1/messages stream fails (#41826) * fix(proxy): send a real error event when a /v1/messages stream fails When a stream failed halfway through, the proxy wrote the error as a plain `data: {"error": ...}` line with no `event:` in front of it. Anthropic clients pick stream events by that name, so they skip the line and the request looks like it simply stopped with nothing in it. Write the failure as an `event: error` frame with Anthropic's own payload, and take the error type from the status code * fix(proxy): use the shared Anthropic error mapping for the stream error frame The first pass added a third copy of the status to error-type table, and it disagreed with the documented one: 529 came out as `api_error` rather than `overloaded_error`, and 413 as `invalid_request_error` rather than `request_too_large`, which hides the two failures a client can actually act on. Drop that copy and put the frame builder next to the table litellm already keeps in anthropic_interface/exceptions. The bridged adapter path was building the same frame inline, so it uses the shared one now too * fix(proxy): seal a torn SSE frame before the /v1/messages error event An upstream that drops mid-frame leaves the client inside an open event, so the error frame that follows is glued onto the torn data line and the Anthropic SDK raises a JSON decode error instead of an APIStatusError. Close the open frame with a ping event the SDK skips before writing the error event, and add the e2e stream-cut edge with Bedrock, Anthropic boundary, and Anthropic mid-frame legs. * fix(proxy): keep the SSE tail unchanged on a chunk that is not text A serializer that hands a dict or model object through as-is has no bytes the frame tail can learn from, so advance_sse_tail leaves it alone instead of slicing it. * fix(proxy): answer a /v1/messages stream that fails before its first byte as a JSON error carrying its status * test: move the Anthropic error frame tests into tests/unit * test(e2e): cut the upstream stream only after content has been relayed * test(e2e): carry split SSE lines across chunks and always tear a data line mid-frame * test(e2e): find the next data line across a chunk boundary before tearing it --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../exceptions/__init__.py | 4 + .../exceptions/exception_mapping_utils.py | 36 ++- .../adapters/streaming_iterator.py | 8 +- litellm/proxy/common_request_processing.py | 39 ++- litellm/proxy/common_utils/sse_keepalive.py | 26 +- .../coverage_registry/llm_conversational.yaml | 2 + tests/e2e/coverage_registry/schema.py | 1 + .../e2e/llm_translation/test_messages_e2e.py | 260 ++++++++++++++++- tests/e2e/models.py | 10 + tests/e2e/provider_edge.py | 152 +++++++++- tests/e2e/test_provider_edge.py | 32 +++ .../proxy/common_utils/test_sse_keepalive.py | 9 + .../proxy/test_common_request_processing.py | 263 ++++++++++++++++++ .../test_exception_mapping_utils.py | 48 +++- 14 files changed, 869 insertions(+), 21 deletions(-) diff --git a/litellm/anthropic_interface/exceptions/__init__.py b/litellm/anthropic_interface/exceptions/__init__.py index 7f2de0e60dc..7c3cea0a28a 100644 --- a/litellm/anthropic_interface/exceptions/__init__.py +++ b/litellm/anthropic_interface/exceptions/__init__.py @@ -2,7 +2,9 @@ from .exception_mapping_utils import ( ANTHROPIC_ERROR_TYPE_MAP, + AnthropicErrorSseFrame, AnthropicExceptionMapping, + anthropic_error_sse_frame, ) from .exceptions import ( AnthropicErrorDetail, @@ -14,6 +16,8 @@ __all__ = [ "ANTHROPIC_ERROR_TYPE_MAP", "AnthropicErrorDetail", "AnthropicErrorResponse", + "AnthropicErrorSseFrame", "AnthropicErrorType", "AnthropicExceptionMapping", + "anthropic_error_sse_frame", ] diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index d9c9925275b..eb3ec8aaee2 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -4,11 +4,12 @@ Utilities for mapping exceptions to Anthropic error format. Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthropic response format. """ +import json from typing import Final from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from .exceptions import AnthropicErrorResponse, AnthropicErrorType +from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType # HTTP status code -> Anthropic error type # Source: https://docs.anthropic.com/en/api/errors @@ -166,3 +167,36 @@ class AnthropicExceptionMapping: message=message, request_id=request_id, ) + + +class AnthropicErrorSseFrame(str): + """One `event: error` frame, for a stream that fails once the response headers are out. + + Anthropic clients pick stream events by the `event:` name, so a frame carrying only a `data:` + line is skipped and the failure never reaches the caller. The frame remembers the status and + body it was built from, so a stream that fails before its first byte can still answer as a + JSON error with that exact status instead of a 200 that only says `api_error` + """ + + status_code: int + error_response: AnthropicErrorResponse + + def __new__(cls, status_code: int, error_response: AnthropicErrorResponse) -> "AnthropicErrorSseFrame": + frame: Final = super().__new__(cls, f"event: error\ndata: {json.dumps(error_response)}\n\n") + frame.status_code = status_code + frame.error_response = error_response + return frame + + def json_body(self, call_id: str | None) -> AnthropicErrorResponse: + if call_id is None: + return self.error_response + detail: Final[AnthropicErrorDetail] = {**self.error_response["error"], "litellm_call_id": call_id} + body: Final[AnthropicErrorResponse] = {**self.error_response, "error": detail} + return body + + +def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame: + return AnthropicErrorSseFrame( + status_code, + AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message), + ) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 20753afee5c..24d5b7f366e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -68,15 +68,11 @@ def _error_status_and_message(exc: Exception) -> tuple[int, str]: def _mid_stream_error_sse_event(exc: Exception) -> bytes: from litellm.anthropic_interface.exceptions.exception_mapping_utils import ( - AnthropicExceptionMapping, + anthropic_error_sse_frame, ) status_code, message = _error_status_and_message(exc) - error_response = AnthropicExceptionMapping.transform_to_anthropic_error( - status_code=status_code, - raw_message=message, - ) - return f"event: error\ndata: {json.dumps(error_response)}\n\n".encode() + return anthropic_error_sse_frame(status_code=status_code, raw_message=message).encode() def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 4f9b6b3a96f..64b0c6c1967 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -32,6 +32,7 @@ from starlette.types import Receive, Scope, Send import litellm from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger from litellm._uuid import uuid +from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame from litellm.constants import ( DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE, DEFAULT_MAX_RECURSE_DEPTH, @@ -102,8 +103,11 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, + SSE_STREAM_START_TAIL, + advance_sse_tail, coerce_keepalive_interval, resolve_ttft_keepalive_interval, + seal_open_sse_frame, wrap_sse_stream_with_keepalive_pings, ) from litellm.proxy.dd_span_tagger import DDSpanTagger @@ -999,6 +1003,17 @@ async def create_response( first_chunk_value = await _buffer_first_chunk_honoring_disconnect(generator, request) resolved_headers: Final = await _resolve_stream_headers(headers, refresh_headers) + if isinstance(first_chunk_value, AnthropicErrorSseFrame): + with contextlib.suppress(Exception): + await generator.aclose() + return JSONResponse( + status_code=first_chunk_value.status_code, + content=first_chunk_value.json_body( + error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)) + ), + headers=resolved_headers, + ) + if first_chunk_value is not None: try: error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value) @@ -3852,6 +3867,7 @@ class ProxyBaseLLMRequestProcessing: serialize_error: StreamErrorSerializer, request: Request | None = None, flush_tail: Callable[[], bytes] | None = None, + seal_open_frame: Callable[[bytes], str] | None = None, ) -> AsyncGenerator[str, None]: """ Shared streaming data generator: runs proxy iterator hook, per-chunk hook, @@ -3861,6 +3877,12 @@ class ProxyBaseLLMRequestProcessing: ``flush_tail`` runs once after the upstream iterator completes cleanly and its non-empty result is yielded, so a serializer that buffers bytes across chunks can emit anything still held at end of stream. + + ``seal_open_frame`` is given the tail of what has been yielded when the + error frame goes out, and what it returns is written first. A passthrough + relays raw upstream bytes, so an upstream that hangs up mid-frame leaves the + client inside an open frame, where an error frame would be swallowed or + misparsed instead of raised. """ verbose_proxy_logger.debug("inside generator") # Resolve per-stream (not per-chunk) whether the heavy per-chunk path @@ -3877,6 +3899,7 @@ class ProxyBaseLLMRequestProcessing: stream_completed = False client_disconnected = False delivered_chunk = False + recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the yielded bytes try: str_so_far = "" async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook( @@ -3922,7 +3945,9 @@ class ProxyBaseLLMRequestProcessing: # False and refunds. A keepalive ping carries no provider output, # so it must not suppress that refund. delivered_chunk = delivered_chunk or chunk != STREAM_SSE_KEEPALIVE_PING_BYTES - yield serialize_chunk(chunk) + serialized = serialize_chunk(chunk) + recent_tail = advance_sse_tail(recent_tail, serialized) + yield serialized held_tail: Final = flush_tail() if flush_tail is not None else b"" if held_tail: yield serialize_chunk(held_tail) @@ -3970,7 +3995,9 @@ class ProxyBaseLLMRequestProcessing: code=stream_error_status, ) stream_completed = True - yield serialize_error(proxy_exception) + error_frame: Final = serialize_error(proxy_exception) + seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail) + yield seal + error_frame if seal else error_frame finally: await ProxyBaseLLMRequestProcessing._finalize_streaming_generator_cleanup( request=request, @@ -3992,7 +4019,7 @@ class ProxyBaseLLMRequestProcessing: restamp_model: str | None = None, ) -> AsyncGenerator[str, None]: """ - Anthropic /messages and Google /generateContent streaming data generator require SSE events. + Anthropic /messages streaming data generator, which requires SSE events. Returns the underlying ``async_streaming_data_generator`` configured with SSE serializers directly (rather than re-wrapping it in another @@ -4010,11 +4037,13 @@ class ProxyBaseLLMRequestProcessing: request_data=request_data, proxy_logging_obj=proxy_logging_obj, serialize_chunk=ProxyBaseLLMRequestProcessing._sse_chunk_serializer(restamper), - serialize_error=lambda proxy_exc: ( - f"{STREAM_SSE_DATA_PREFIX}{json.dumps({'error': proxy_exc.to_dict()})}\n\n" + serialize_error=lambda proxy_exc: anthropic_error_sse_frame( + status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR), + raw_message=proxy_exc.message, ), request=request, flush_tail=None if restamper is None else restamper.flush, + seal_open_frame=seal_open_sse_frame, ) @overload diff --git a/litellm/proxy/common_utils/sse_keepalive.py b/litellm/proxy/common_utils/sse_keepalive.py index cf98a7e9224..d9685971f52 100644 --- a/litellm/proxy/common_utils/sse_keepalive.py +++ b/litellm/proxy/common_utils/sse_keepalive.py @@ -15,7 +15,7 @@ SSE_COMMENT_PING_BYTES: Final = SSE_COMMENT_PING.encode() # terminates a line with CRLF, LF or CR, so a blank line is any of these three. _SSE_FRAME_DELIMITERS: Final = (b"\r\n\r\n", b"\n\n", b"\r\r") _SSE_DELIMITER_LOOKBACK: Final = max(len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS) -_STREAM_START_TAIL: Final = b"\n\n" +SSE_STREAM_START_TAIL: Final = b"\n\n" _SSE_MEDIA_TYPE: Final = "text/event-stream" @@ -128,7 +128,7 @@ async def _keepalive_ping_byte_stream( # Seeded as a delimiter because a stream starts at a frame boundary, and kept # across chunks because a delimiter can be split between two transport reads, # which testing only the latest chunk would miss for the rest of the stream. - recent_tail = _STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes + recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes try: while True: await asyncio.wait((pending,), timeout=ping_interval_seconds) @@ -155,6 +155,28 @@ async def _keepalive_ping_byte_stream( await stream.aclose() +def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes: + written: Final = _sse_tail_bytes(chunk) + if not written: + return recent_tail + return (recent_tail + written)[-_SSE_DELIMITER_LOOKBACK:] + + +def _sse_tail_bytes(chunk: object) -> bytes: + if isinstance(chunk, bytes): + return chunk[-_SSE_DELIMITER_LOOKBACK:] + if isinstance(chunk, str): + return chunk[-_SSE_DELIMITER_LOOKBACK:].encode() + return b"" + + +def seal_open_sse_frame(recent_tail: bytes) -> str: + if recent_tail.endswith(_SSE_FRAME_DELIMITERS): + return "" + line_break: Final = "" if recent_tail.endswith((b"\n", b"\r")) else "\n" + return f"{line_break}{ANTHROPIC_PING_SSE_CHUNK}" + + def resolve_ttft_keepalive_interval( deployments: Iterable[Mapping[str, object]], global_interval: float | str | None, diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 20c87dbbd74..9cfe6e33ed6 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -103,6 +103,8 @@ - {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"} - {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"} - {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"} +- {id: llm.messages.anthropic.upstream_stream_failure.stream.error_event, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: upstream_stream_failure, streaming: stream, assertions: [error_event], source: "customer report", rationale: "An upstream that hangs up mid-stream must reach Anthropic clients as an event: error frame, not an OpenAI-shaped data-only error they silently drop"} +- {id: llm.messages.anthropic.upstream_stream_failure.stream.error_status, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: upstream_stream_failure, streaming: stream, assertions: [error_status], source: "customer report", rationale: "An upstream that hangs up before its first byte must answer as a JSON error carrying its status, so Anthropic clients raise the status-specific error and retry on it instead of reading a 200 stream that only carries an error event"} - {id: llm.messages.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI models served on the Anthropic Messages contract"} - {id: llm.messages.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages streams the Anthropic event grammar"} - {id: llm.messages.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages: cost header and spend row agree"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 5fd19212ab7..fec1934059c 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -86,6 +86,7 @@ LlmCapability = Literal[ "tool_search", "tool_search_history", "tool_use", + "upstream_stream_failure", "vision", "web_search", "web_search_server_tool", diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index d048d1343eb..871fd2f9aef 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -11,8 +11,11 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations import time +from collections.abc import Callable +from types import MappingProxyType from typing import Final +import anthropic import pytest from anthropic import Anthropic from anthropic.types import ( @@ -30,12 +33,21 @@ from anthropic.types import ( ToolParam, ToolUseBlock, ) -from e2e_config import STREAM_MIN_LEAD_SECONDS, provider_edge_base, provider_paces_stream, unique_marker +from e2e_config import ( + PROVIDER_EDGE_ADVERTISE_HOST, + PROVIDER_EDGE_BIND_HOST, + STREAM_MIN_LEAD_SECONDS, + provider_edge_base, + provider_paces_stream, + unique_marker, +) from e2e_http import assert_client_error from lifecycle import ResourceManager -from models import ChatMessage, LiteLLMParamsBody, SpendLogRow +from models import AnthropicErrorEvent, AnthropicMessagesBody, ChatMessage, LiteLLMParamsBody, SpendLogRow +from provider_edge import EDGE_MOUNTS, LiveEdge, RunningEdge, StreamCut, start_provider_edge +from provider_edge_bedrock import bedrock_signer from proxy_client import ProxyClient -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header pytestmark = [pytest.mark.e2e, pytest.mark.replayable] @@ -385,3 +397,245 @@ class TestOpenAIMessagesToolContinuation: ) assert _text(continuation).strip() == receipt, "continuation did not consume the correlated tool result" assert all(not isinstance(block, ToolUseBlock) for block in continuation.content) + + +BEDROCK_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +BEDROCK_EDGE_REGION: Final = "us-east-1" +_STREAM_FAILURE_PROMPT: Final = "Count from 1 to 100, one number per line." +_FRAME_PAYLOAD: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_AT_FRAME_BOUNDARY: Final = StreamCut(after_content=True) +_MID_FRAME: Final = StreamCut(after_content=True, mid_chunk=True) +_BEFORE_FIRST_BYTE: Final = StreamCut(after_content=False) + +type _CutRegistration = Callable[[ProxyClient, ResourceManager, StreamCut], tuple[str, str]] + + +def _cut_edge(backend: LiveEdge, mount: str) -> RunningEdge: + return start_provider_edge( + backend, + mounts=MappingProxyType({mount: EDGE_MOUNTS[mount]}), + bind_host=PROVIDER_EDGE_BIND_HOST, + advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, + ) + + +def _register_cut_bedrock(proxy: ProxyClient, resources: ResourceManager, cut: StreamCut) -> tuple[str, str]: + mount: Final = f"bedrock/{BEDROCK_EDGE_REGION}" + edge: Final = _cut_edge(LiveEdge(cut=cut, sign=bedrock_signer(BEDROCK_EDGE_REGION)), mount) + resources.defer(edge.shutdown) + return _register( + proxy, + resources, + LiteLLMParamsBody( + model=BEDROCK_BACKEND, + api_base=edge.edge.api_base(mount), + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name=BEDROCK_EDGE_REGION, + ), + prefix="e2e-messages-cut", + ) + + +def _register_cut_anthropic(proxy: ProxyClient, resources: ResourceManager, cut: StreamCut) -> tuple[str, str]: + edge: Final = _cut_edge(LiveEdge(cut=cut), "anthropic") + resources.defer(edge.shutdown) + return _register( + proxy, + resources, + LiteLLMParamsBody( + model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY", api_base=edge.edge.api_base("anthropic") + ), + prefix="e2e-messages-cut", + ) + + +_DROPPED_UPSTREAMS: Final[tuple[tuple[str, _CutRegistration, StreamCut], ...]] = ( + ("bedrock_at_a_frame_boundary", _register_cut_bedrock, _AT_FRAME_BOUNDARY), + ("anthropic_at_a_frame_boundary", _register_cut_anthropic, _AT_FRAME_BOUNDARY), + ("anthropic_mid_frame", _register_cut_anthropic, _MID_FRAME), +) +_DROPPED_BEFORE_FIRST_BYTE: Final[tuple[tuple[str, _CutRegistration, StreamCut], ...]] = ( + ("bedrock_before_the_first_byte", _register_cut_bedrock, _BEFORE_FIRST_BYTE), + ("anthropic_before_the_first_byte", _register_cut_anthropic, _BEFORE_FIRST_BYTE), +) + + +def _payload(frame: str) -> JsonValue | None: + try: + return _FRAME_PAYLOAD.validate_json(frame) + except ValidationError: + return None + + +def _bare_error_frame(frame: str) -> bool: + payload: Final = _payload(frame) + return isinstance(payload, dict) and "error" in payload and payload.get("type") != "error" + + +@pytest.mark.provider_edge_host +@pytest.mark.provider_live +class TestMessagesUpstreamStreamFailure: + @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_event") + @pytest.mark.parametrize( + ("register", "cut"), [case[1:] for case in _DROPPED_UPSTREAMS], ids=[case[0] for case in _DROPPED_UPSTREAMS] + ) + def test_interrupted_upstream_stream_raises_in_the_anthropic_sdk( + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + register: _CutRegistration, + cut: StreamCut, + ) -> None: + model, key = register(proxy, resources, cut) + client: Final = sdk.anthropic(key) + + stream: Final = client.messages.create( + model=model, + max_tokens=300, + stream=True, + messages=[_user_turn(_STREAM_FAILURE_PROMPT)], + extra_body=NO_PROXY_CACHE, + ) + first: Final = next(stream) + assert first.type == "message_start", ( + f"the stream produced a first event that is not message_start, so this run proves a " + f"startup failure, not an interrupted stream: {first!r}" + ) + with pytest.raises(anthropic.APIStatusError) as raised: + for _ in stream: + pass + try: + AnthropicErrorEvent.model_validate(raised.value.body) + except ValidationError: + pytest.fail( + f"the SDK raised on the interrupted stream but without the Anthropic error envelope a " + f"client reads the failure from: body={raised.value.body!r} message={raised.value}" + ) + + @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_event") + @pytest.mark.parametrize( + ("register", "cut"), [case[1:] for case in _DROPPED_UPSTREAMS], ids=[case[0] for case in _DROPPED_UPSTREAMS] + ) + def test_interrupted_upstream_stream_is_an_anthropic_error_event( + self, proxy: ProxyClient, resources: ResourceManager, register: _CutRegistration, cut: StreamCut + ) -> None: + model, key = register(proxy, resources, cut) + + outcome: Final = proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + max_tokens=300, + stream=True, + messages=[ChatMessage(role="user", content=_STREAM_FAILURE_PROMPT)], + ), + ) + frames: Final = outcome.stream_events + assert outcome.is_streaming, ( + f"/v1/messages did not answer with an SSE stream: status={outcome.status_code} body={outcome.body}" + ) + assert frames, ( + f"the proxy sent no SSE data frames although the upstream hung up; stream_error={outcome.stream_error!r}" + ) + assert outcome.stream_error == "event: error", ( + f"the interrupted stream was not announced by an 'event: error' line Anthropic clients read; " + f"stream_error={outcome.stream_error!r} frames={frames}" + ) + try: + AnthropicErrorEvent.model_validate_json(frames[-1]) + except ValidationError: + pytest.fail( + f'the last SSE frame was not an Anthropic {{"type": "error", "error": ...}} envelope; frames={frames}' + ) + torn: Final = tuple(index for index, frame in enumerate(frames) if _payload(frame) is None) + expected_torn: Final = 1 if cut.mid_chunk else 0 + assert len(torn) == expected_torn, ( + f"expected {expected_torn} data line(s) that are not JSON, since the edge tears one only when it " + f"cuts mid-frame, but the proxy relayed {[frames[index] for index in torn]}; all frames={frames}" + ) + for index in torn: + assert _payload(frames[index + 1]) == {"type": "ping"}, ( + f"the frame the upstream tore was not closed as a ping event before the error, so an " + f"Anthropic client parses the error inside it: after {frames[index]!r} came " + f"{frames[index + 1]!r}; all frames={frames}" + ) + bare: Final = tuple(frame for frame in frames if _bare_error_frame(frame)) + assert not bare, ( + f"the proxy emitted error frames without the Anthropic envelope, which Anthropic clients drop: " + f"{bare}; all frames={frames}" + ) + + @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_status") + @pytest.mark.parametrize( + ("register", "cut"), + [case[1:] for case in _DROPPED_BEFORE_FIRST_BYTE], + ids=[case[0] for case in _DROPPED_BEFORE_FIRST_BYTE], + ) + def test_upstream_that_hangs_up_before_the_first_byte_raises_with_its_status_in_the_anthropic_sdk( + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + register: _CutRegistration, + cut: StreamCut, + ) -> None: + model, key = register(proxy, resources, cut) + client: Final = sdk.anthropic(key) + + with pytest.raises(anthropic.APIStatusError) as raised: + client.messages.create( + model=model, + max_tokens=300, + stream=True, + messages=[_user_turn(_STREAM_FAILURE_PROMPT)], + extra_body=NO_PROXY_CACHE, + ) + assert 500 <= raised.value.status_code < 600, ( + f"an upstream that hung up before sending anything must answer with a server error status the SDK " + f"retries on, not {raised.value.status_code}: {raised.value}" + ) + try: + AnthropicErrorEvent.model_validate(raised.value.body) + except ValidationError: + pytest.fail( + f"the SDK raised with the right status but without the Anthropic error envelope a client reads " + f"the failure from: body={raised.value.body!r} message={raised.value}" + ) + + @pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_status") + @pytest.mark.parametrize( + ("register", "cut"), + [case[1:] for case in _DROPPED_BEFORE_FIRST_BYTE], + ids=[case[0] for case in _DROPPED_BEFORE_FIRST_BYTE], + ) + def test_upstream_that_hangs_up_before_the_first_byte_is_a_json_error_with_its_status( + self, proxy: ProxyClient, resources: ResourceManager, register: _CutRegistration, cut: StreamCut + ) -> None: + model, key = register(proxy, resources, cut) + + outcome: Final = proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + max_tokens=300, + stream=True, + messages=[ChatMessage(role="user", content=_STREAM_FAILURE_PROMPT)], + ), + ) + assert not outcome.is_streaming, ( + f"nothing had been streamed when the upstream hung up, yet /v1/messages opened a 200 SSE stream " + f"instead of answering with the failure's status: stream_error={outcome.stream_error!r} " + f"frames={outcome.stream_events}" + ) + assert 500 <= outcome.status_code < 600, ( + f"/v1/messages answered {outcome.status_code} for an upstream that hung up before its first byte; " + f"body={outcome.body}" + ) + try: + AnthropicErrorEvent.model_validate_json(outcome.body) + except ValidationError: + pytest.fail( + f'the error body is not an Anthropic {{"type": "error", "error": ...}} envelope; body={outcome.body}' + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index c96f4b0bef1..84399fd6155 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -598,6 +598,16 @@ class CountTokensResponse(BaseModel): input_tokens: int +class AnthropicErrorBody(BaseModel): + type: str + message: str + + +class AnthropicErrorEvent(BaseModel): + type: Literal["error"] + error: AnthropicErrorBody + + # ---------- mcp servers ---------- diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index fc10dde2a77..3680375b6af 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -45,6 +45,7 @@ import hashlib import os import re import threading +import time from collections import deque from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import closing, contextmanager @@ -56,6 +57,7 @@ from types import MappingProxyType from typing import Final, Literal, assert_never from urllib.parse import parse_qsl, urlsplit +from botocore.eventstream import EventStreamBuffer from e2e_http import ( NetworkError, StreamChunk, @@ -96,16 +98,18 @@ from fixture_mode import ( ) from fixture_profile import IneligibleRequest, MatchProfile, match_profile, strict_identity from provider_cache import ( + JSON_VALUE, SIGNATURE_HEADERS, CacheEdge, MountPolicy, RequestSigner, + invoke_chunk_value, is_bedrock, scoped_edge_base, split_test_segment, ) from provider_cache_routing import LIVE_PROVIDER_REQUIRED -from pydantic import JsonValue, TypeAdapter +from pydantic import JsonValue, TypeAdapter, ValidationError BEDROCK_REGIONS: Final[tuple[str, ...]] = ("us-east-1",) @@ -537,10 +541,33 @@ class ReplayEdge: source: ReplaySource +@dataclass(frozen=True, slots=True) +class StreamCut: + """Where a live edge hangs up on a streamed upstream body: before its first byte, or with + ``after_content`` set, right after the first transfer chunk carrying assistant output (a + ``content_block_delta``). That frame is what commits the proxy's mid-stream fallback + wrapper to the client: it holds the lifecycle frames before it back and drops them when + the transport fails first, so a cut after a fixed number of chunks landed on either side + of that commit depending on how the provider batched its frames. With ``mid_chunk`` set + the hang-up comes part way through the next ``data:`` line the provider sends after that, + so the client is left inside an SSE frame the way a dropped transport leaves it. + + Whatever was relayed sits on the wire for ``_CUT_SETTLE_SECONDS`` before the hang-up, so + the client has read it by then instead of receiving the data and the close in one burst, + where its reader can surface the close before what it buffered.""" + + after_content: bool + mid_chunk: bool = False + + +_CUT_SETTLE_SECONDS: Final = 1.0 + + @dataclass(frozen=True, slots=True) class LiveEdge: observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None sign: RequestSigner | None = None + cut: StreamCut | None = None type EdgeBackend = RecordEdge | ReplayEdge | LiveEdge | CacheEdge @@ -786,11 +813,128 @@ def _handle_record( assert_never(head) +def _data_line_start(data: bytes) -> int: + if data.startswith(b"data:"): + return 0 + at_line_start: Final = data.find(b"\ndata:") + return -1 if at_line_start < 0 else at_line_start + 1 + + +def _torn_prefix(data: bytes) -> bytes: + start: Final = _data_line_start(data) + line_end: Final = data.find(b"\n", start) + end: Final = len(data) if line_end < 0 else line_end + return data[: start + (end - start) // 2] + + +class _DataLineTearer: + __slots__ = ("_unfinished_line",) + + _unfinished_line: bytes + + def __init__(self) -> None: + self._unfinished_line = b"" + + def observe(self, data: bytes) -> None: + self._unfinished_line = (self._unfinished_line + data).rsplit(b"\n", 1)[-1] + + def tear(self, data: bytes) -> bytes | None: + buffered: Final = self._unfinished_line + data + if _data_line_start(buffered) < 0: + self.observe(data) + return None + return _torn_prefix(buffered)[len(self._unfinished_line):] + + +def _is_content_delta(value: JsonValue | None) -> bool: + return isinstance(value, dict) and value.get("type") == "content_block_delta" + + +def _sse_data_carries_content(line: bytes) -> bool: + if not line.startswith(b"data:"): + return False + try: + return _is_content_delta(JSON_VALUE.validate_json(line[len(b"data:"):].strip())) + except ValidationError: + return False + + +class _AnthropicContentDetector: + __slots__ = ("_unfinished_line",) + + _unfinished_line: bytes + + def __init__(self) -> None: + self._unfinished_line = b"" + + def __call__(self, data: bytes) -> bool: + lines: Final = (self._unfinished_line + data).split(b"\n") + self._unfinished_line = lines[-1] + return any(_sse_data_carries_content(line.rstrip(b"\r")) for line in lines[:-1]) + + +def _invoke_frame_carries_content(payload: bytes) -> bool: + try: + return _is_content_delta(invoke_chunk_value(JSON_VALUE.validate_json(payload))) + except ValidationError: + return False + + +def _bedrock_content_detector() -> Callable[[bytes], bool]: + """Bedrock's invoke stream wraps each Anthropic event in an eventstream frame that a + transfer chunk can split, so the frames are reassembled across chunks before being read.""" + frames: Final = EventStreamBuffer() + + def carries_content(data: bytes) -> bool: + frames.add_data(data) + return any(_invoke_frame_carries_content(frame.payload) for frame in frames) + + return carries_content + + +def _content_detector(mount: str) -> Callable[[bytes], bool]: + return _bedrock_content_detector() if is_bedrock(mount) else _AnthropicContentDetector() + + +def _cut_steps( + steps: Generator[StreamStep, None, None], cut: StreamCut, carries_content: Callable[[bytes], bool] +) -> Generator[StreamStep, None, None]: + with closing(steps) as source: + tearer: Final = _DataLineTearer() + if cut.after_content: + for step in source: + yield step + if isinstance(step, StreamTruncation): + return + tearer.observe(step.data) + if carries_content(step.data): + break + else: + return + if cut.mid_chunk: + for step in source: + if isinstance(step, StreamTruncation): + yield step + return + if (torn := tearer.tear(step.data)) is None: + yield step + continue + if torn: + yield StreamChunk(data=torn) + break + else: + return + if cut.after_content or cut.mid_chunk: + time.sleep(_CUT_SETTLE_SECONDS) + yield StreamTruncation(reason=f"edge cut the upstream stream: {cut!r}") + + def _handle_live( method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float, cache: CacheEdge | None = None, mount: str = "", test_key: str | None = None, observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None, sign: RequestSigner | None = None, + cut: StreamCut | None = None, ) -> EdgeOutcome: forwarded: Final = { name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS @@ -805,6 +949,8 @@ def _handle_live( match head: case NetworkError(message=message): return _recorded_outcome(_network_error_response(message)) + case StreamHead() if cut is not None: + return EdgeStream(head.status_code, _filtered_response_headers(head.headers), _cut_steps(head.steps, cut, _content_detector(mount))) case StreamHead() if _is_streamed(head.headers): return EdgeStream(head.status_code, _filtered_response_headers(head.headers), head.steps) case StreamHead(): @@ -875,10 +1021,10 @@ def handle_edge_request( method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout, backend, mount, test_key, ) - case LiveEdge(observe_request=observe_request, sign=sign): + case LiveEdge(observe_request=observe_request, sign=sign, cut=cut): return _handle_live( method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout, - observe_request=observe_request, sign=sign, + mount=mount, observe_request=observe_request, sign=sign, cut=cut, ) case RecordEdge(): return _handle_record( diff --git a/tests/e2e/test_provider_edge.py b/tests/e2e/test_provider_edge.py index d776c338ef7..978c3671a77 100644 --- a/tests/e2e/test_provider_edge.py +++ b/tests/e2e/test_provider_edge.py @@ -54,11 +54,13 @@ from provider_edge import ( EdgeBackend, EdgeReply, EdgeStream, + LiveEdge, ProviderEdge, ProviderRequestObservation, RecordEdge, ReplayEdge, ReplaySource, + StreamCut, edge_request, handle_edge_request, observed_provider_edge, @@ -1000,6 +1002,36 @@ def stream_chunks(response: RecordedStreamedResponse) -> list[bytes]: return [base64.b64decode(chunk) for chunk in response.chunks_b64] +SECOND_DATA_LINE: Final = b'data: {"type":"content_block_delta","delta":{"text":" two"}}' +SPLIT_MARKER_CHUNKS: tuple[bytes, ...] = ( + b'data: {"type":"content_block_delta","delta":{"text":"one"}}\n\nda', + b"ta" + SECOND_DATA_LINE[4:] + b"\n\nda", + b'ta: {"type":"message_delta","usage":{"output_tokens":7}}\n\nda', + b"ta: [DONE]\n\n", +) + + +class TestStreamCut: + def test_a_mid_frame_cut_tears_a_data_line_whose_marker_is_split_across_chunks(self) -> None: + """Every ``data:`` marker after the first content delta straddles a transfer + chunk boundary, so a tearer that inspects each chunk on its own never finds + one and lets the stream finish cleanly instead of cutting it.""" + backend: Final = LiveEdge(cut=StreamCut(after_content=True, mid_chunk=True)) + with chunked_provider(chunks=SPLIT_MARKER_CHUNKS) as provider: + with running_edge(backend, {"openai": provider_url(provider)}) as edge: + head, chunks, ending = raw_stream_post(edge.port, STREAM_PATH, STREAM_BODY) + + assert head.startswith("HTTP/1.1 200 OK") + assert ending == "truncated" + relayed: Final = b"".join(chunks) + whole: Final = b"".join(SPLIT_MARKER_CHUNKS) + assert whole.startswith(relayed) and relayed != whole + assert relayed.startswith(SPLIT_MARKER_CHUNKS[0]) + torn_line: Final = relayed.rsplit(b"\n", 1)[-1] + assert torn_line and SECOND_DATA_LINE.startswith(torn_line) and torn_line != SECOND_DATA_LINE + assert b"[DONE]" not in relayed + + class TestStreamingFidelity: """LIT-5742: a streamed response records and replays as the chunk sequence the provider actually sent, not as one coalesced body. The unit of fidelity is the diff --git a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py index 69b92f5e4d7..228fd5bcae6 100644 --- a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py +++ b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py @@ -6,10 +6,13 @@ import pytest from fastapi.responses import StreamingResponse from litellm.proxy.common_request_processing import create_response +from litellm.types.utils import ModelResponse from litellm.proxy.common_utils.sse_keepalive import ( ANTHROPIC_PING_SSE_CHUNK, SSE_COMMENT_PING_BYTES, + advance_sse_tail, resolve_ttft_keepalive_interval, + seal_open_sse_frame, split_complete_sse_frames, wrap_passthrough_sse_bytes_with_keepalive_pings, wrap_sse_stream_with_keepalive_pings, @@ -32,6 +35,12 @@ def test_split_complete_sse_frames_holds_bytes_with_no_complete_frame(): assert split_complete_sse_frames(b"data: unterminated") == (b"", b"data: unterminated") +@pytest.mark.parametrize("chunk", [{"content": "hi"}, ModelResponse()]) +def test_advance_sse_tail_ignores_a_chunk_that_is_not_sse_text(chunk: object): + assert advance_sse_tail(b"\n\n", chunk) == b"\n\n" + assert seal_open_sse_frame(advance_sse_tail(b"data: {", chunk)) == "\n" + ANTHROPIC_PING_SSE_CHUNK + + @pytest.mark.asyncio async def test_pings_fill_mid_stream_silence_and_preserve_chunk_order(): async def gappy_stream() -> AsyncGenerator[str, None]: diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index bdf003085ef..c17f41a8b8f 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -7,6 +7,7 @@ from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch +import anthropic import httpx import pytest from fastapi import HTTPException, Request, Response, status @@ -14,6 +15,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame from litellm.litellm_core_utils.bug_report import ( DISABLE_ENV_VAR, ISSUE_URL_BASE, @@ -55,6 +57,7 @@ from litellm.proxy.common_request_processing import ( sse_error_payload, ) from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header +from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import ProxyErrorTypes, ProxyException @@ -2543,6 +2546,63 @@ class TestCommonRequestProcessingHelpers: assert response.headers["x-litellm-call-id"] == "call-8302" assert json.loads(response.body) == {"error": {"code": 403, "message": "forbidden"}} + async def test_a_stream_that_fails_before_its_first_byte_answers_as_an_anthropic_json_error(self): + """A /v1/messages stream whose first chunk is already the error frame has nothing + streamed yet, so the failure answers as JSON with the status the upstream gave, + the shape Anthropic clients raise their status-specific errors on""" + + async def stream(): + yield anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable") + yield ANTHROPIC_PING_SSE_CHUNK + + generator: Final = stream() + response = await create_response(generator, "text/event-stream", {"x-litellm-call-id": "call-8609"}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 503 + assert response.headers["content-type"] == "application/json" + assert response.headers["x-litellm-call-id"] == "call-8609" + assert json.loads(response.body) == { + "type": "error", + "error": {"type": "api_error", "message": "upstream unavailable"}, + } + assert generator.ag_frame is None + + async def test_a_stream_that_fails_before_its_first_byte_names_the_call_when_opted_in(self): + async def stream(): + yield anthropic_error_sse_frame(status_code=429, raw_message="slow down") + + response = await create_response( + stream(), + "text/event-stream", + {"x-litellm-call-id": "call-8609"}, + general_settings={"include_call_id_in_error_body": True}, + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 429 + assert json.loads(response.body) == { + "type": "error", + "error": {"type": "rate_limit_error", "message": "slow down", "litellm_call_id": "call-8609"}, + } + + async def test_an_error_event_after_a_keepalive_ping_still_streams(self): + """Once a keepalive ping went out the headers are committed, so the error frame + streams as an event instead of turning into a JSON answer""" + + async def stream(): + yield ANTHROPIC_PING_SSE_CHUNK + yield anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable") + + response = await create_response(stream(), "text/event-stream", {}) + + assert isinstance(response, StreamingResponse) + assert response.status_code == 200 + assert "".join(await self.consume_stream(response)) == ( + ANTHROPIC_PING_SSE_CHUNK + + 'event: error\ndata: {"type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}}\n\n' + ) + async def test_create_streaming_response_disables_proxy_buffering(self): """Regression for #28384: every StreamingResponse create_response returns must carry the headers that stop nginx/ingress/Envoy from buffering the @@ -9901,6 +9961,209 @@ class TestErrorLogCarriesCallId: assert call_id in record.getMessage() +class TestAnthropicMessagesStreamErrorFrame: + """A ``/v1/messages`` stream that fails after the headers are out has to say so with an + ``event: error`` frame. Anthropic clients pick events by name, so a bare ``data:`` line is + skipped and the request looks like it ended with nothing in it""" + + @staticmethod + def _sse_generator_failing_with(failure: Exception) -> AsyncGenerator[str, None]: + class FailingUpstream: + def __aiter__(self) -> "FailingUpstream": + return self + + async def __anext__(self) -> object: + raise failure + + ProxyLogging._callback_capabilities_cache.clear() + return ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=FailingUpstream(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + request_data={"model": "claude-sonnet-4-5"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + ) + + @pytest.mark.parametrize( + "status_code, expected_error_type", + [ + (429, "rate_limit_error"), + (529, "overloaded_error"), + (413, "request_too_large"), + (500, "api_error"), + (502, "api_error"), + (400, "invalid_request_error"), + ], + ) + async def test_mid_stream_failure_arrives_as_an_anthropic_error_event( + self, status_code: int, expected_error_type: str + ) -> None: + class UpstreamFailure(Exception): + def __init__(self) -> None: + super().__init__("upstream stopped sending") + self.status_code: Final = status_code + + frames: Final = [frame async for frame in self._sse_generator_failing_with(UpstreamFailure())] + + assert len(frames) == 1 + event_line, data_line, first_blank, second_blank = frames[0].split("\n") + assert isinstance(frames[0], AnthropicErrorSseFrame) + assert frames[0].status_code == status_code + assert event_line == "event: error" + assert (first_blank, second_blank) == ("", "") + payload: Final = json.loads(data_line.removeprefix("data: ")) + assert payload["type"] == "error" + assert payload["error"]["type"] == expected_error_type + assert "upstream stopped sending" in payload["error"]["message"] + + _CONTENT_DELTA_FRAME: Final = ( + b"event: content_block_delta\n" + b'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1\\n2\\n3"}}\n\n' + ) + _TORN_DATA_LINE: Final = ( + b"event: content_block_delta\n" + b'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4' + ) + _PING: Final = ANTHROPIC_PING_SSE_CHUNK.encode() + + @staticmethod + def _upstream_failure(status_code: int) -> Exception: + class UpstreamFailure(Exception): + def __init__(self) -> None: + super().__init__("upstream stopped sending") + self.status_code: Final = status_code + + return UpstreamFailure() + + @staticmethod + def _sse_generator_cut_after(relayed: Sequence[bytes], failure: Exception) -> AsyncGenerator[str, None]: + class CutUpstream: + def __init__(self) -> None: + self._remaining: Final = iter(relayed) + + def __aiter__(self) -> "CutUpstream": + return self + + async def __anext__(self) -> object: + chunk: Final = next(self._remaining, None) + if chunk is None: + raise failure + return chunk + + ProxyLogging._callback_capabilities_cache.clear() + return ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=CutUpstream(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + request_data={"model": "claude-sonnet-4-5"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + ) + + @staticmethod + def _as_bytes(chunk: object) -> bytes: + if isinstance(chunk, bytes): + return chunk + assert isinstance(chunk, str) + return chunk.encode() + + async def _wire_bytes(self, relayed: Sequence[bytes]) -> bytes: + stream: Final = self._sse_generator_cut_after(relayed, self._upstream_failure(500)) + return b"".join([self._as_bytes(chunk) async for chunk in stream]) + + @staticmethod + def _error_frame_after(wire: bytes, relayed: bytes) -> bytes: + assert wire.startswith(relayed), f"the wire did not open with {relayed!r}: {wire!r}" + return wire.removeprefix(relayed) + + @staticmethod + def _assert_error_frame(frame: bytes) -> None: + event_line, data_line, first_blank, second_blank = frame.split(b"\n") + assert event_line == b"event: error" + assert (first_blank, second_blank) == (b"", b"") + payload: Final = json.loads(data_line.removeprefix(b"data: ")) + assert payload["type"] == "error" + assert "upstream stopped sending" in payload["error"]["message"] + + @pytest.mark.parametrize( + "torn, seal", + [ + (_TORN_DATA_LINE, b"\n" + _PING), + (b"event: content_bl", b"\n" + _PING), + (b"event: content_block_delta\n", _PING), + (b'event: content_block_delta\r\ndata: {"type":"content_block_delta"}\r\n', _PING), + ], + ids=["mid_data_line", "mid_event_line", "after_a_complete_line", "after_a_crlf_line"], + ) + async def test_a_frame_the_upstream_tore_is_closed_as_a_ping_before_the_error_event( + self, torn: bytes, seal: bytes + ) -> None: + wire: Final = await self._wire_bytes((self._CONTENT_DELTA_FRAME, torn)) + + self._assert_error_frame(self._error_frame_after(wire, self._CONTENT_DELTA_FRAME + torn + seal)) + + async def test_a_cut_at_a_frame_boundary_gets_the_error_event_alone(self) -> None: + wire: Final = await self._wire_bytes((self._CONTENT_DELTA_FRAME,)) + + self._assert_error_frame(self._error_frame_after(wire, self._CONTENT_DELTA_FRAME)) + + async def test_a_torn_frame_still_raises_the_error_in_the_anthropic_sdk(self) -> None: + wire: Final = await self._wire_bytes((self._CONTENT_DELTA_FRAME, self._TORN_DATA_LINE)) + + def serve(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire) + + client: Final = anthropic.Anthropic( + api_key="sk-test", + base_url="http://proxy.test", + http_client=httpx.Client(transport=httpx.MockTransport(serve)), + max_retries=0, + ) + with pytest.raises(anthropic.APIStatusError) as raised: + for _ in client.messages.create( + model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "count"}], stream=True + ): + pass + body: Final = raised.value.body + assert isinstance(body, dict) + assert body["type"] == "error" + assert "upstream stopped sending" in body["error"]["message"] + + async def test_a_failure_before_the_first_byte_answers_with_its_status_as_json(self) -> None: + response: Final = await create_response( + self._sse_generator_failing_with(self._upstream_failure(502)), "text/event-stream", {} + ) + + assert isinstance(response, JSONResponse) + assert response.status_code == 502 + body: Final = json.loads(response.body) + assert body["type"] == "error" + assert body["error"]["type"] == "api_error" + assert "upstream stopped sending" in body["error"]["message"] + + async def test_a_failure_before_the_first_byte_raises_with_its_status_in_the_anthropic_sdk(self) -> None: + response: Final = await create_response( + self._sse_generator_failing_with(self._upstream_failure(502)), "text/event-stream", {} + ) + assert isinstance(response, JSONResponse) + + def serve(request: httpx.Request) -> httpx.Response: + return httpx.Response(response.status_code, headers=dict(response.headers), content=response.body) + + client: Final = anthropic.Anthropic( + api_key="sk-test", + base_url="http://proxy.test", + http_client=httpx.Client(transport=httpx.MockTransport(serve)), + max_retries=0, + ) + with pytest.raises(anthropic.APIStatusError) as raised: + client.messages.create( + model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "count"}], stream=True + ) + assert raised.value.status_code == 502 + body: Final = raised.value.body + assert isinstance(body, dict) + assert body["type"] == "error" + assert "upstream stopped sending" in body["error"]["message"] + + class TestStreamingContainerOwnershipRecordedBeforeDone: """Regression for LIT-8612: the OpenAI SDK closes the connection at ``data: [DONE]`` and starlette cancels the body task, so an ownership row diff --git a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py index ef092b65f28..0d8e7674e7d 100644 --- a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -3,8 +3,15 @@ Tests for AnthropicExceptionMapping class in litellm/anthropic_interface/excepti """ import json +from typing import Final -from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping +import pytest + +from litellm.anthropic_interface.exceptions import ( + AnthropicErrorSseFrame, + AnthropicExceptionMapping, + anthropic_error_sse_frame, +) class TestCreateErrorResponse: @@ -206,3 +213,42 @@ class TestTransformToAnthropicError: ) assert result["type"] == "error" assert result["error"]["message"] == '["error1", "error2"]' + + +class TestAnthropicErrorSseFrame: + @pytest.mark.parametrize( + ("status_code", "expected_error_type"), + [(429, "rate_limit_error"), (503, "api_error"), (400, "invalid_request_error")], + ) + def test_the_frame_is_one_error_event_carrying_the_anthropic_envelope( + self, status_code: int, expected_error_type: str + ) -> None: + frame: Final = anthropic_error_sse_frame(status_code=status_code, raw_message="upstream unavailable") + + event_line, data_line, first_blank, second_blank = frame.split("\n") + assert event_line == "event: error" + assert (first_blank, second_blank) == ("", "") + assert json.loads(data_line.removeprefix("data: ")) == { + "type": "error", + "error": {"type": expected_error_type, "message": "upstream unavailable"}, + } + + def test_the_frame_remembers_the_status_and_body_it_was_built_from(self) -> None: + frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable") + + assert isinstance(frame, AnthropicErrorSseFrame) + assert frame.status_code == 503 + data_line: Final = frame.split("\n")[1] + assert data_line == f"data: {json.dumps(frame.json_body(call_id=None))}" + + def test_the_json_body_names_the_call_only_when_asked(self) -> None: + frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable") + + assert frame.json_body(call_id="call-1") == { + "type": "error", + "error": {"type": "api_error", "message": "upstream unavailable", "litellm_call_id": "call-1"}, + } + assert frame.json_body(call_id=None) == { + "type": "error", + "error": {"type": "api_error", "message": "upstream unavailable"}, + }