mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
fix(proxy): send a real error event when a /v1/messages stream fails (#41826)
* fix(proxy): send a real error event when a /v1/messages stream fails
When a stream failed halfway through, the proxy wrote the error as a plain
`data: {"error": ...}` line with no `event:` in front of it. Anthropic clients
pick stream events by that name, so they skip the line and the request looks
like it simply stopped with nothing in it.
Write the failure as an `event: error` frame with Anthropic's own payload, and
take the error type from the status code
* fix(proxy): use the shared Anthropic error mapping for the stream error frame
The first pass added a third copy of the status to error-type table, and it
disagreed with the documented one: 529 came out as `api_error` rather than
`overloaded_error`, and 413 as `invalid_request_error` rather than
`request_too_large`, which hides the two failures a client can actually act on.
Drop that copy and put the frame builder next to the table litellm already
keeps in anthropic_interface/exceptions. The bridged adapter path was building
the same frame inline, so it uses the shared one now too
* fix(proxy): seal a torn SSE frame before the /v1/messages error event
An upstream that drops mid-frame leaves the client inside an open event,
so the error frame that follows is glued onto the torn data line and the
Anthropic SDK raises a JSON decode error instead of an APIStatusError.
Close the open frame with a ping event the SDK skips before writing the
error event, and add the e2e stream-cut edge with Bedrock, Anthropic
boundary, and Anthropic mid-frame legs.
* fix(proxy): keep the SSE tail unchanged on a chunk that is not text
A serializer that hands a dict or model object through as-is has no bytes
the frame tail can learn from, so advance_sse_tail leaves it alone instead
of slicing it.
* fix(proxy): answer a /v1/messages stream that fails before its first byte as a JSON error carrying its status
* test: move the Anthropic error frame tests into tests/unit
* test(e2e): cut the upstream stream only after content has been relayed
* test(e2e): carry split SSE lines across chunks and always tear a data line mid-frame
* test(e2e): find the next data line across a chunk boundary before tearing it
---------
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Refael Iliaguyev
GitHub
parent
8ef85a45ce
commit
e065a2575b
14 changed files with 869 additions and 21 deletions
|
|
@ -2,7 +2,9 @@
|
|||
|
||||
from .exception_mapping_utils import (
|
||||
ANTHROPIC_ERROR_TYPE_MAP,
|
||||
AnthropicErrorSseFrame,
|
||||
AnthropicExceptionMapping,
|
||||
anthropic_error_sse_frame,
|
||||
)
|
||||
from .exceptions import (
|
||||
AnthropicErrorDetail,
|
||||
|
|
@ -14,6 +16,8 @@ __all__ = [
|
|||
"ANTHROPIC_ERROR_TYPE_MAP",
|
||||
"AnthropicErrorDetail",
|
||||
"AnthropicErrorResponse",
|
||||
"AnthropicErrorSseFrame",
|
||||
"AnthropicErrorType",
|
||||
"AnthropicExceptionMapping",
|
||||
"anthropic_error_sse_frame",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ Utilities for mapping exceptions to Anthropic error format.
|
|||
Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthropic response format.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
||||
from .exceptions import AnthropicErrorResponse, AnthropicErrorType
|
||||
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
|
||||
|
||||
# HTTP status code -> Anthropic error type
|
||||
# Source: https://docs.anthropic.com/en/api/errors
|
||||
|
|
@ -166,3 +167,36 @@ class AnthropicExceptionMapping:
|
|||
message=message,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
|
||||
class AnthropicErrorSseFrame(str):
|
||||
"""One `event: error` frame, for a stream that fails once the response headers are out.
|
||||
|
||||
Anthropic clients pick stream events by the `event:` name, so a frame carrying only a `data:`
|
||||
line is skipped and the failure never reaches the caller. The frame remembers the status and
|
||||
body it was built from, so a stream that fails before its first byte can still answer as a
|
||||
JSON error with that exact status instead of a 200 that only says `api_error`
|
||||
"""
|
||||
|
||||
status_code: int
|
||||
error_response: AnthropicErrorResponse
|
||||
|
||||
def __new__(cls, status_code: int, error_response: AnthropicErrorResponse) -> "AnthropicErrorSseFrame":
|
||||
frame: Final = super().__new__(cls, f"event: error\ndata: {json.dumps(error_response)}\n\n")
|
||||
frame.status_code = status_code
|
||||
frame.error_response = error_response
|
||||
return frame
|
||||
|
||||
def json_body(self, call_id: str | None) -> AnthropicErrorResponse:
|
||||
if call_id is None:
|
||||
return self.error_response
|
||||
detail: Final[AnthropicErrorDetail] = {**self.error_response["error"], "litellm_call_id": call_id}
|
||||
body: Final[AnthropicErrorResponse] = {**self.error_response, "error": detail}
|
||||
return body
|
||||
|
||||
|
||||
def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame:
|
||||
return AnthropicErrorSseFrame(
|
||||
status_code,
|
||||
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -68,15 +68,11 @@ def _error_status_and_message(exc: Exception) -> tuple[int, str]:
|
|||
|
||||
def _mid_stream_error_sse_event(exc: Exception) -> bytes:
|
||||
from litellm.anthropic_interface.exceptions.exception_mapping_utils import (
|
||||
AnthropicExceptionMapping,
|
||||
anthropic_error_sse_frame,
|
||||
)
|
||||
|
||||
status_code, message = _error_status_and_message(exc)
|
||||
error_response = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
status_code=status_code,
|
||||
raw_message=message,
|
||||
)
|
||||
return f"event: error\ndata: {json.dumps(error_response)}\n\n".encode()
|
||||
return anthropic_error_sse_frame(status_code=status_code, raw_message=message).encode()
|
||||
|
||||
|
||||
def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from starlette.types import Receive, Scope, Send
|
|||
import litellm
|
||||
from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
|
||||
from litellm.constants import (
|
||||
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
|
||||
DEFAULT_MAX_RECURSE_DEPTH,
|
||||
|
|
@ -102,8 +103,11 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
)
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING_BYTES,
|
||||
SSE_STREAM_START_TAIL,
|
||||
advance_sse_tail,
|
||||
coerce_keepalive_interval,
|
||||
resolve_ttft_keepalive_interval,
|
||||
seal_open_sse_frame,
|
||||
wrap_sse_stream_with_keepalive_pings,
|
||||
)
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
|
|
@ -999,6 +1003,17 @@ async def create_response(
|
|||
first_chunk_value = await _buffer_first_chunk_honoring_disconnect(generator, request)
|
||||
resolved_headers: Final = await _resolve_stream_headers(headers, refresh_headers)
|
||||
|
||||
if isinstance(first_chunk_value, AnthropicErrorSseFrame):
|
||||
with contextlib.suppress(Exception):
|
||||
await generator.aclose()
|
||||
return JSONResponse(
|
||||
status_code=first_chunk_value.status_code,
|
||||
content=first_chunk_value.json_body(
|
||||
error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER))
|
||||
),
|
||||
headers=resolved_headers,
|
||||
)
|
||||
|
||||
if first_chunk_value is not None:
|
||||
try:
|
||||
error_code_from_chunk: Final = await _parse_event_data_for_error(first_chunk_value)
|
||||
|
|
@ -3852,6 +3867,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
serialize_error: StreamErrorSerializer,
|
||||
request: Request | None = None,
|
||||
flush_tail: Callable[[], bytes] | None = None,
|
||||
seal_open_frame: Callable[[bytes], str] | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Shared streaming data generator: runs proxy iterator hook, per-chunk hook,
|
||||
|
|
@ -3861,6 +3877,12 @@ class ProxyBaseLLMRequestProcessing:
|
|||
``flush_tail`` runs once after the upstream iterator completes cleanly and
|
||||
its non-empty result is yielded, so a serializer that buffers bytes across
|
||||
chunks can emit anything still held at end of stream.
|
||||
|
||||
``seal_open_frame`` is given the tail of what has been yielded when the
|
||||
error frame goes out, and what it returns is written first. A passthrough
|
||||
relays raw upstream bytes, so an upstream that hangs up mid-frame leaves the
|
||||
client inside an open frame, where an error frame would be swallowed or
|
||||
misparsed instead of raised.
|
||||
"""
|
||||
verbose_proxy_logger.debug("inside generator")
|
||||
# Resolve per-stream (not per-chunk) whether the heavy per-chunk path
|
||||
|
|
@ -3877,6 +3899,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
stream_completed = False
|
||||
client_disconnected = False
|
||||
delivered_chunk = False
|
||||
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the yielded bytes
|
||||
try:
|
||||
str_so_far = ""
|
||||
async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook(
|
||||
|
|
@ -3922,7 +3945,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
# False and refunds. A keepalive ping carries no provider output,
|
||||
# so it must not suppress that refund.
|
||||
delivered_chunk = delivered_chunk or chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
yield serialize_chunk(chunk)
|
||||
serialized = serialize_chunk(chunk)
|
||||
recent_tail = advance_sse_tail(recent_tail, serialized)
|
||||
yield serialized
|
||||
held_tail: Final = flush_tail() if flush_tail is not None else b""
|
||||
if held_tail:
|
||||
yield serialize_chunk(held_tail)
|
||||
|
|
@ -3970,7 +3995,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
code=stream_error_status,
|
||||
)
|
||||
stream_completed = True
|
||||
yield serialize_error(proxy_exception)
|
||||
error_frame: Final = serialize_error(proxy_exception)
|
||||
seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail)
|
||||
yield seal + error_frame if seal else error_frame
|
||||
finally:
|
||||
await ProxyBaseLLMRequestProcessing._finalize_streaming_generator_cleanup(
|
||||
request=request,
|
||||
|
|
@ -3992,7 +4019,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
restamp_model: str | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Anthropic /messages and Google /generateContent streaming data generator require SSE events.
|
||||
Anthropic /messages streaming data generator, which requires SSE events.
|
||||
|
||||
Returns the underlying ``async_streaming_data_generator`` configured with
|
||||
SSE serializers directly (rather than re-wrapping it in another
|
||||
|
|
@ -4010,11 +4037,13 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request_data=request_data,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
serialize_chunk=ProxyBaseLLMRequestProcessing._sse_chunk_serializer(restamper),
|
||||
serialize_error=lambda proxy_exc: (
|
||||
f"{STREAM_SSE_DATA_PREFIX}{json.dumps({'error': proxy_exc.to_dict()})}\n\n"
|
||||
serialize_error=lambda proxy_exc: anthropic_error_sse_frame(
|
||||
status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
raw_message=proxy_exc.message,
|
||||
),
|
||||
request=request,
|
||||
flush_tail=None if restamper is None else restamper.flush,
|
||||
seal_open_frame=seal_open_sse_frame,
|
||||
)
|
||||
|
||||
@overload
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ SSE_COMMENT_PING_BYTES: Final = SSE_COMMENT_PING.encode()
|
|||
# terminates a line with CRLF, LF or CR, so a blank line is any of these three.
|
||||
_SSE_FRAME_DELIMITERS: Final = (b"\r\n\r\n", b"\n\n", b"\r\r")
|
||||
_SSE_DELIMITER_LOOKBACK: Final = max(len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS)
|
||||
_STREAM_START_TAIL: Final = b"\n\n"
|
||||
SSE_STREAM_START_TAIL: Final = b"\n\n"
|
||||
_SSE_MEDIA_TYPE: Final = "text/event-stream"
|
||||
|
||||
|
||||
|
|
@ -128,7 +128,7 @@ async def _keepalive_ping_byte_stream(
|
|||
# Seeded as a delimiter because a stream starts at a frame boundary, and kept
|
||||
# across chunks because a delimiter can be split between two transport reads,
|
||||
# which testing only the latest chunk would miss for the rest of the stream.
|
||||
recent_tail = _STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes
|
||||
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the relayed bytes
|
||||
try:
|
||||
while True:
|
||||
await asyncio.wait((pending,), timeout=ping_interval_seconds)
|
||||
|
|
@ -155,6 +155,28 @@ async def _keepalive_ping_byte_stream(
|
|||
await stream.aclose()
|
||||
|
||||
|
||||
def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes:
|
||||
written: Final = _sse_tail_bytes(chunk)
|
||||
if not written:
|
||||
return recent_tail
|
||||
return (recent_tail + written)[-_SSE_DELIMITER_LOOKBACK:]
|
||||
|
||||
|
||||
def _sse_tail_bytes(chunk: object) -> bytes:
|
||||
if isinstance(chunk, bytes):
|
||||
return chunk[-_SSE_DELIMITER_LOOKBACK:]
|
||||
if isinstance(chunk, str):
|
||||
return chunk[-_SSE_DELIMITER_LOOKBACK:].encode()
|
||||
return b""
|
||||
|
||||
|
||||
def seal_open_sse_frame(recent_tail: bytes) -> str:
|
||||
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
|
||||
return ""
|
||||
line_break: Final = "" if recent_tail.endswith((b"\n", b"\r")) else "\n"
|
||||
return f"{line_break}{ANTHROPIC_PING_SSE_CHUNK}"
|
||||
|
||||
|
||||
def resolve_ttft_keepalive_interval(
|
||||
deployments: Iterable[Mapping[str, object]],
|
||||
global_interval: float | str | None,
|
||||
|
|
|
|||
|
|
@ -103,6 +103,8 @@
|
|||
- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"}
|
||||
- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"}
|
||||
- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"}
|
||||
- {id: llm.messages.anthropic.upstream_stream_failure.stream.error_event, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: upstream_stream_failure, streaming: stream, assertions: [error_event], source: "customer report", rationale: "An upstream that hangs up mid-stream must reach Anthropic clients as an event: error frame, not an OpenAI-shaped data-only error they silently drop"}
|
||||
- {id: llm.messages.anthropic.upstream_stream_failure.stream.error_status, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: upstream_stream_failure, streaming: stream, assertions: [error_status], source: "customer report", rationale: "An upstream that hangs up before its first byte must answer as a JSON error carrying its status, so Anthropic clients raise the status-specific error and retry on it instead of reading a 200 stream that only carries an error event"}
|
||||
- {id: llm.messages.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI models served on the Anthropic Messages contract"}
|
||||
- {id: llm.messages.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages streams the Anthropic event grammar"}
|
||||
- {id: llm.messages.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages: cost header and spend row agree"}
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ LlmCapability = Literal[
|
|||
"tool_search",
|
||||
"tool_search_history",
|
||||
"tool_use",
|
||||
"upstream_stream_failure",
|
||||
"vision",
|
||||
"web_search",
|
||||
"web_search_server_tool",
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@ litellm-regression-tests/tests/test_inference_endpoints.py.
|
|||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import pytest
|
||||
from anthropic import Anthropic
|
||||
from anthropic.types import (
|
||||
|
|
@ -30,12 +33,21 @@ from anthropic.types import (
|
|||
ToolParam,
|
||||
ToolUseBlock,
|
||||
)
|
||||
from e2e_config import STREAM_MIN_LEAD_SECONDS, provider_edge_base, provider_paces_stream, unique_marker
|
||||
from e2e_config import (
|
||||
PROVIDER_EDGE_ADVERTISE_HOST,
|
||||
PROVIDER_EDGE_BIND_HOST,
|
||||
STREAM_MIN_LEAD_SECONDS,
|
||||
provider_edge_base,
|
||||
provider_paces_stream,
|
||||
unique_marker,
|
||||
)
|
||||
from e2e_http import assert_client_error
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatMessage, LiteLLMParamsBody, SpendLogRow
|
||||
from models import AnthropicErrorEvent, AnthropicMessagesBody, ChatMessage, LiteLLMParamsBody, SpendLogRow
|
||||
from provider_edge import EDGE_MOUNTS, LiveEdge, RunningEdge, StreamCut, start_provider_edge
|
||||
from provider_edge_bedrock import bedrock_signer
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header
|
||||
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.replayable]
|
||||
|
|
@ -385,3 +397,245 @@ class TestOpenAIMessagesToolContinuation:
|
|||
)
|
||||
assert _text(continuation).strip() == receipt, "continuation did not consume the correlated tool result"
|
||||
assert all(not isinstance(block, ToolUseBlock) for block in continuation.content)
|
||||
|
||||
|
||||
BEDROCK_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
BEDROCK_EDGE_REGION: Final = "us-east-1"
|
||||
_STREAM_FAILURE_PROMPT: Final = "Count from 1 to 100, one number per line."
|
||||
_FRAME_PAYLOAD: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
_AT_FRAME_BOUNDARY: Final = StreamCut(after_content=True)
|
||||
_MID_FRAME: Final = StreamCut(after_content=True, mid_chunk=True)
|
||||
_BEFORE_FIRST_BYTE: Final = StreamCut(after_content=False)
|
||||
|
||||
type _CutRegistration = Callable[[ProxyClient, ResourceManager, StreamCut], tuple[str, str]]
|
||||
|
||||
|
||||
def _cut_edge(backend: LiveEdge, mount: str) -> RunningEdge:
|
||||
return start_provider_edge(
|
||||
backend,
|
||||
mounts=MappingProxyType({mount: EDGE_MOUNTS[mount]}),
|
||||
bind_host=PROVIDER_EDGE_BIND_HOST,
|
||||
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
|
||||
)
|
||||
|
||||
|
||||
def _register_cut_bedrock(proxy: ProxyClient, resources: ResourceManager, cut: StreamCut) -> tuple[str, str]:
|
||||
mount: Final = f"bedrock/{BEDROCK_EDGE_REGION}"
|
||||
edge: Final = _cut_edge(LiveEdge(cut=cut, sign=bedrock_signer(BEDROCK_EDGE_REGION)), mount)
|
||||
resources.defer(edge.shutdown)
|
||||
return _register(
|
||||
proxy,
|
||||
resources,
|
||||
LiteLLMParamsBody(
|
||||
model=BEDROCK_BACKEND,
|
||||
api_base=edge.edge.api_base(mount),
|
||||
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
||||
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
aws_region_name=BEDROCK_EDGE_REGION,
|
||||
),
|
||||
prefix="e2e-messages-cut",
|
||||
)
|
||||
|
||||
|
||||
def _register_cut_anthropic(proxy: ProxyClient, resources: ResourceManager, cut: StreamCut) -> tuple[str, str]:
|
||||
edge: Final = _cut_edge(LiveEdge(cut=cut), "anthropic")
|
||||
resources.defer(edge.shutdown)
|
||||
return _register(
|
||||
proxy,
|
||||
resources,
|
||||
LiteLLMParamsBody(
|
||||
model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY", api_base=edge.edge.api_base("anthropic")
|
||||
),
|
||||
prefix="e2e-messages-cut",
|
||||
)
|
||||
|
||||
|
||||
_DROPPED_UPSTREAMS: Final[tuple[tuple[str, _CutRegistration, StreamCut], ...]] = (
|
||||
("bedrock_at_a_frame_boundary", _register_cut_bedrock, _AT_FRAME_BOUNDARY),
|
||||
("anthropic_at_a_frame_boundary", _register_cut_anthropic, _AT_FRAME_BOUNDARY),
|
||||
("anthropic_mid_frame", _register_cut_anthropic, _MID_FRAME),
|
||||
)
|
||||
_DROPPED_BEFORE_FIRST_BYTE: Final[tuple[tuple[str, _CutRegistration, StreamCut], ...]] = (
|
||||
("bedrock_before_the_first_byte", _register_cut_bedrock, _BEFORE_FIRST_BYTE),
|
||||
("anthropic_before_the_first_byte", _register_cut_anthropic, _BEFORE_FIRST_BYTE),
|
||||
)
|
||||
|
||||
|
||||
def _payload(frame: str) -> JsonValue | None:
|
||||
try:
|
||||
return _FRAME_PAYLOAD.validate_json(frame)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _bare_error_frame(frame: str) -> bool:
|
||||
payload: Final = _payload(frame)
|
||||
return isinstance(payload, dict) and "error" in payload and payload.get("type") != "error"
|
||||
|
||||
|
||||
@pytest.mark.provider_edge_host
|
||||
@pytest.mark.provider_live
|
||||
class TestMessagesUpstreamStreamFailure:
|
||||
@pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_event")
|
||||
@pytest.mark.parametrize(
|
||||
("register", "cut"), [case[1:] for case in _DROPPED_UPSTREAMS], ids=[case[0] for case in _DROPPED_UPSTREAMS]
|
||||
)
|
||||
def test_interrupted_upstream_stream_raises_in_the_anthropic_sdk(
|
||||
self,
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
sdk: SdkClients,
|
||||
register: _CutRegistration,
|
||||
cut: StreamCut,
|
||||
) -> None:
|
||||
model, key = register(proxy, resources, cut)
|
||||
client: Final = sdk.anthropic(key)
|
||||
|
||||
stream: Final = client.messages.create(
|
||||
model=model,
|
||||
max_tokens=300,
|
||||
stream=True,
|
||||
messages=[_user_turn(_STREAM_FAILURE_PROMPT)],
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
first: Final = next(stream)
|
||||
assert first.type == "message_start", (
|
||||
f"the stream produced a first event that is not message_start, so this run proves a "
|
||||
f"startup failure, not an interrupted stream: {first!r}"
|
||||
)
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
for _ in stream:
|
||||
pass
|
||||
try:
|
||||
AnthropicErrorEvent.model_validate(raised.value.body)
|
||||
except ValidationError:
|
||||
pytest.fail(
|
||||
f"the SDK raised on the interrupted stream but without the Anthropic error envelope a "
|
||||
f"client reads the failure from: body={raised.value.body!r} message={raised.value}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_event")
|
||||
@pytest.mark.parametrize(
|
||||
("register", "cut"), [case[1:] for case in _DROPPED_UPSTREAMS], ids=[case[0] for case in _DROPPED_UPSTREAMS]
|
||||
)
|
||||
def test_interrupted_upstream_stream_is_an_anthropic_error_event(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, register: _CutRegistration, cut: StreamCut
|
||||
) -> None:
|
||||
model, key = register(proxy, resources, cut)
|
||||
|
||||
outcome: Final = proxy.messages_stream(
|
||||
key,
|
||||
AnthropicMessagesBody(
|
||||
model=model,
|
||||
max_tokens=300,
|
||||
stream=True,
|
||||
messages=[ChatMessage(role="user", content=_STREAM_FAILURE_PROMPT)],
|
||||
),
|
||||
)
|
||||
frames: Final = outcome.stream_events
|
||||
assert outcome.is_streaming, (
|
||||
f"/v1/messages did not answer with an SSE stream: status={outcome.status_code} body={outcome.body}"
|
||||
)
|
||||
assert frames, (
|
||||
f"the proxy sent no SSE data frames although the upstream hung up; stream_error={outcome.stream_error!r}"
|
||||
)
|
||||
assert outcome.stream_error == "event: error", (
|
||||
f"the interrupted stream was not announced by an 'event: error' line Anthropic clients read; "
|
||||
f"stream_error={outcome.stream_error!r} frames={frames}"
|
||||
)
|
||||
try:
|
||||
AnthropicErrorEvent.model_validate_json(frames[-1])
|
||||
except ValidationError:
|
||||
pytest.fail(
|
||||
f'the last SSE frame was not an Anthropic {{"type": "error", "error": ...}} envelope; frames={frames}'
|
||||
)
|
||||
torn: Final = tuple(index for index, frame in enumerate(frames) if _payload(frame) is None)
|
||||
expected_torn: Final = 1 if cut.mid_chunk else 0
|
||||
assert len(torn) == expected_torn, (
|
||||
f"expected {expected_torn} data line(s) that are not JSON, since the edge tears one only when it "
|
||||
f"cuts mid-frame, but the proxy relayed {[frames[index] for index in torn]}; all frames={frames}"
|
||||
)
|
||||
for index in torn:
|
||||
assert _payload(frames[index + 1]) == {"type": "ping"}, (
|
||||
f"the frame the upstream tore was not closed as a ping event before the error, so an "
|
||||
f"Anthropic client parses the error inside it: after {frames[index]!r} came "
|
||||
f"{frames[index + 1]!r}; all frames={frames}"
|
||||
)
|
||||
bare: Final = tuple(frame for frame in frames if _bare_error_frame(frame))
|
||||
assert not bare, (
|
||||
f"the proxy emitted error frames without the Anthropic envelope, which Anthropic clients drop: "
|
||||
f"{bare}; all frames={frames}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_status")
|
||||
@pytest.mark.parametrize(
|
||||
("register", "cut"),
|
||||
[case[1:] for case in _DROPPED_BEFORE_FIRST_BYTE],
|
||||
ids=[case[0] for case in _DROPPED_BEFORE_FIRST_BYTE],
|
||||
)
|
||||
def test_upstream_that_hangs_up_before_the_first_byte_raises_with_its_status_in_the_anthropic_sdk(
|
||||
self,
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
sdk: SdkClients,
|
||||
register: _CutRegistration,
|
||||
cut: StreamCut,
|
||||
) -> None:
|
||||
model, key = register(proxy, resources, cut)
|
||||
client: Final = sdk.anthropic(key)
|
||||
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
client.messages.create(
|
||||
model=model,
|
||||
max_tokens=300,
|
||||
stream=True,
|
||||
messages=[_user_turn(_STREAM_FAILURE_PROMPT)],
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert 500 <= raised.value.status_code < 600, (
|
||||
f"an upstream that hung up before sending anything must answer with a server error status the SDK "
|
||||
f"retries on, not {raised.value.status_code}: {raised.value}"
|
||||
)
|
||||
try:
|
||||
AnthropicErrorEvent.model_validate(raised.value.body)
|
||||
except ValidationError:
|
||||
pytest.fail(
|
||||
f"the SDK raised with the right status but without the Anthropic error envelope a client reads "
|
||||
f"the failure from: body={raised.value.body!r} message={raised.value}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.messages.anthropic.upstream_stream_failure.stream.error_status")
|
||||
@pytest.mark.parametrize(
|
||||
("register", "cut"),
|
||||
[case[1:] for case in _DROPPED_BEFORE_FIRST_BYTE],
|
||||
ids=[case[0] for case in _DROPPED_BEFORE_FIRST_BYTE],
|
||||
)
|
||||
def test_upstream_that_hangs_up_before_the_first_byte_is_a_json_error_with_its_status(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, register: _CutRegistration, cut: StreamCut
|
||||
) -> None:
|
||||
model, key = register(proxy, resources, cut)
|
||||
|
||||
outcome: Final = proxy.messages_stream(
|
||||
key,
|
||||
AnthropicMessagesBody(
|
||||
model=model,
|
||||
max_tokens=300,
|
||||
stream=True,
|
||||
messages=[ChatMessage(role="user", content=_STREAM_FAILURE_PROMPT)],
|
||||
),
|
||||
)
|
||||
assert not outcome.is_streaming, (
|
||||
f"nothing had been streamed when the upstream hung up, yet /v1/messages opened a 200 SSE stream "
|
||||
f"instead of answering with the failure's status: stream_error={outcome.stream_error!r} "
|
||||
f"frames={outcome.stream_events}"
|
||||
)
|
||||
assert 500 <= outcome.status_code < 600, (
|
||||
f"/v1/messages answered {outcome.status_code} for an upstream that hung up before its first byte; "
|
||||
f"body={outcome.body}"
|
||||
)
|
||||
try:
|
||||
AnthropicErrorEvent.model_validate_json(outcome.body)
|
||||
except ValidationError:
|
||||
pytest.fail(
|
||||
f'the error body is not an Anthropic {{"type": "error", "error": ...}} envelope; body={outcome.body}'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -598,6 +598,16 @@ class CountTokensResponse(BaseModel):
|
|||
input_tokens: int
|
||||
|
||||
|
||||
class AnthropicErrorBody(BaseModel):
|
||||
type: str
|
||||
message: str
|
||||
|
||||
|
||||
class AnthropicErrorEvent(BaseModel):
|
||||
type: Literal["error"]
|
||||
error: AnthropicErrorBody
|
||||
|
||||
|
||||
# ---------- mcp servers ----------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ import hashlib
|
|||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable, Generator, Mapping, Sequence
|
||||
from contextlib import closing, contextmanager
|
||||
|
|
@ -56,6 +57,7 @@ from types import MappingProxyType
|
|||
from typing import Final, Literal, assert_never
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
from botocore.eventstream import EventStreamBuffer
|
||||
from e2e_http import (
|
||||
NetworkError,
|
||||
StreamChunk,
|
||||
|
|
@ -96,16 +98,18 @@ from fixture_mode import (
|
|||
)
|
||||
from fixture_profile import IneligibleRequest, MatchProfile, match_profile, strict_identity
|
||||
from provider_cache import (
|
||||
JSON_VALUE,
|
||||
SIGNATURE_HEADERS,
|
||||
CacheEdge,
|
||||
MountPolicy,
|
||||
RequestSigner,
|
||||
invoke_chunk_value,
|
||||
is_bedrock,
|
||||
scoped_edge_base,
|
||||
split_test_segment,
|
||||
)
|
||||
from provider_cache_routing import LIVE_PROVIDER_REQUIRED
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
BEDROCK_REGIONS: Final[tuple[str, ...]] = ("us-east-1",)
|
||||
|
||||
|
|
@ -537,10 +541,33 @@ class ReplayEdge:
|
|||
source: ReplaySource
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StreamCut:
|
||||
"""Where a live edge hangs up on a streamed upstream body: before its first byte, or with
|
||||
``after_content`` set, right after the first transfer chunk carrying assistant output (a
|
||||
``content_block_delta``). That frame is what commits the proxy's mid-stream fallback
|
||||
wrapper to the client: it holds the lifecycle frames before it back and drops them when
|
||||
the transport fails first, so a cut after a fixed number of chunks landed on either side
|
||||
of that commit depending on how the provider batched its frames. With ``mid_chunk`` set
|
||||
the hang-up comes part way through the next ``data:`` line the provider sends after that,
|
||||
so the client is left inside an SSE frame the way a dropped transport leaves it.
|
||||
|
||||
Whatever was relayed sits on the wire for ``_CUT_SETTLE_SECONDS`` before the hang-up, so
|
||||
the client has read it by then instead of receiving the data and the close in one burst,
|
||||
where its reader can surface the close before what it buffered."""
|
||||
|
||||
after_content: bool
|
||||
mid_chunk: bool = False
|
||||
|
||||
|
||||
_CUT_SETTLE_SECONDS: Final = 1.0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LiveEdge:
|
||||
observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None
|
||||
sign: RequestSigner | None = None
|
||||
cut: StreamCut | None = None
|
||||
|
||||
|
||||
type EdgeBackend = RecordEdge | ReplayEdge | LiveEdge | CacheEdge
|
||||
|
|
@ -786,11 +813,128 @@ def _handle_record(
|
|||
assert_never(head)
|
||||
|
||||
|
||||
def _data_line_start(data: bytes) -> int:
|
||||
if data.startswith(b"data:"):
|
||||
return 0
|
||||
at_line_start: Final = data.find(b"\ndata:")
|
||||
return -1 if at_line_start < 0 else at_line_start + 1
|
||||
|
||||
|
||||
def _torn_prefix(data: bytes) -> bytes:
|
||||
start: Final = _data_line_start(data)
|
||||
line_end: Final = data.find(b"\n", start)
|
||||
end: Final = len(data) if line_end < 0 else line_end
|
||||
return data[: start + (end - start) // 2]
|
||||
|
||||
|
||||
class _DataLineTearer:
|
||||
__slots__ = ("_unfinished_line",)
|
||||
|
||||
_unfinished_line: bytes
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._unfinished_line = b""
|
||||
|
||||
def observe(self, data: bytes) -> None:
|
||||
self._unfinished_line = (self._unfinished_line + data).rsplit(b"\n", 1)[-1]
|
||||
|
||||
def tear(self, data: bytes) -> bytes | None:
|
||||
buffered: Final = self._unfinished_line + data
|
||||
if _data_line_start(buffered) < 0:
|
||||
self.observe(data)
|
||||
return None
|
||||
return _torn_prefix(buffered)[len(self._unfinished_line):]
|
||||
|
||||
|
||||
def _is_content_delta(value: JsonValue | None) -> bool:
|
||||
return isinstance(value, dict) and value.get("type") == "content_block_delta"
|
||||
|
||||
|
||||
def _sse_data_carries_content(line: bytes) -> bool:
|
||||
if not line.startswith(b"data:"):
|
||||
return False
|
||||
try:
|
||||
return _is_content_delta(JSON_VALUE.validate_json(line[len(b"data:"):].strip()))
|
||||
except ValidationError:
|
||||
return False
|
||||
|
||||
|
||||
class _AnthropicContentDetector:
|
||||
__slots__ = ("_unfinished_line",)
|
||||
|
||||
_unfinished_line: bytes
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._unfinished_line = b""
|
||||
|
||||
def __call__(self, data: bytes) -> bool:
|
||||
lines: Final = (self._unfinished_line + data).split(b"\n")
|
||||
self._unfinished_line = lines[-1]
|
||||
return any(_sse_data_carries_content(line.rstrip(b"\r")) for line in lines[:-1])
|
||||
|
||||
|
||||
def _invoke_frame_carries_content(payload: bytes) -> bool:
|
||||
try:
|
||||
return _is_content_delta(invoke_chunk_value(JSON_VALUE.validate_json(payload)))
|
||||
except ValidationError:
|
||||
return False
|
||||
|
||||
|
||||
def _bedrock_content_detector() -> Callable[[bytes], bool]:
|
||||
"""Bedrock's invoke stream wraps each Anthropic event in an eventstream frame that a
|
||||
transfer chunk can split, so the frames are reassembled across chunks before being read."""
|
||||
frames: Final = EventStreamBuffer()
|
||||
|
||||
def carries_content(data: bytes) -> bool:
|
||||
frames.add_data(data)
|
||||
return any(_invoke_frame_carries_content(frame.payload) for frame in frames)
|
||||
|
||||
return carries_content
|
||||
|
||||
|
||||
def _content_detector(mount: str) -> Callable[[bytes], bool]:
|
||||
return _bedrock_content_detector() if is_bedrock(mount) else _AnthropicContentDetector()
|
||||
|
||||
|
||||
def _cut_steps(
|
||||
steps: Generator[StreamStep, None, None], cut: StreamCut, carries_content: Callable[[bytes], bool]
|
||||
) -> Generator[StreamStep, None, None]:
|
||||
with closing(steps) as source:
|
||||
tearer: Final = _DataLineTearer()
|
||||
if cut.after_content:
|
||||
for step in source:
|
||||
yield step
|
||||
if isinstance(step, StreamTruncation):
|
||||
return
|
||||
tearer.observe(step.data)
|
||||
if carries_content(step.data):
|
||||
break
|
||||
else:
|
||||
return
|
||||
if cut.mid_chunk:
|
||||
for step in source:
|
||||
if isinstance(step, StreamTruncation):
|
||||
yield step
|
||||
return
|
||||
if (torn := tearer.tear(step.data)) is None:
|
||||
yield step
|
||||
continue
|
||||
if torn:
|
||||
yield StreamChunk(data=torn)
|
||||
break
|
||||
else:
|
||||
return
|
||||
if cut.after_content or cut.mid_chunk:
|
||||
time.sleep(_CUT_SETTLE_SECONDS)
|
||||
yield StreamTruncation(reason=f"edge cut the upstream stream: {cut!r}")
|
||||
|
||||
|
||||
def _handle_live(
|
||||
method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float,
|
||||
cache: CacheEdge | None = None, mount: str = "", test_key: str | None = None,
|
||||
observe_request: Callable[[str, Mapping[str, str], bytes | None], None] | None = None,
|
||||
sign: RequestSigner | None = None,
|
||||
cut: StreamCut | None = None,
|
||||
) -> EdgeOutcome:
|
||||
forwarded: Final = {
|
||||
name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS
|
||||
|
|
@ -805,6 +949,8 @@ def _handle_live(
|
|||
match head:
|
||||
case NetworkError(message=message):
|
||||
return _recorded_outcome(_network_error_response(message))
|
||||
case StreamHead() if cut is not None:
|
||||
return EdgeStream(head.status_code, _filtered_response_headers(head.headers), _cut_steps(head.steps, cut, _content_detector(mount)))
|
||||
case StreamHead() if _is_streamed(head.headers):
|
||||
return EdgeStream(head.status_code, _filtered_response_headers(head.headers), head.steps)
|
||||
case StreamHead():
|
||||
|
|
@ -875,10 +1021,10 @@ def handle_edge_request(
|
|||
method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout,
|
||||
backend, mount, test_key,
|
||||
)
|
||||
case LiveEdge(observe_request=observe_request, sign=sign):
|
||||
case LiveEdge(observe_request=observe_request, sign=sign, cut=cut):
|
||||
return _handle_live(
|
||||
method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout,
|
||||
observe_request=observe_request, sign=sign,
|
||||
mount=mount, observe_request=observe_request, sign=sign, cut=cut,
|
||||
)
|
||||
case RecordEdge():
|
||||
return _handle_record(
|
||||
|
|
|
|||
|
|
@ -54,11 +54,13 @@ from provider_edge import (
|
|||
EdgeBackend,
|
||||
EdgeReply,
|
||||
EdgeStream,
|
||||
LiveEdge,
|
||||
ProviderEdge,
|
||||
ProviderRequestObservation,
|
||||
RecordEdge,
|
||||
ReplayEdge,
|
||||
ReplaySource,
|
||||
StreamCut,
|
||||
edge_request,
|
||||
handle_edge_request,
|
||||
observed_provider_edge,
|
||||
|
|
@ -1000,6 +1002,36 @@ def stream_chunks(response: RecordedStreamedResponse) -> list[bytes]:
|
|||
return [base64.b64decode(chunk) for chunk in response.chunks_b64]
|
||||
|
||||
|
||||
SECOND_DATA_LINE: Final = b'data: {"type":"content_block_delta","delta":{"text":" two"}}'
|
||||
SPLIT_MARKER_CHUNKS: tuple[bytes, ...] = (
|
||||
b'data: {"type":"content_block_delta","delta":{"text":"one"}}\n\nda',
|
||||
b"ta" + SECOND_DATA_LINE[4:] + b"\n\nda",
|
||||
b'ta: {"type":"message_delta","usage":{"output_tokens":7}}\n\nda',
|
||||
b"ta: [DONE]\n\n",
|
||||
)
|
||||
|
||||
|
||||
class TestStreamCut:
|
||||
def test_a_mid_frame_cut_tears_a_data_line_whose_marker_is_split_across_chunks(self) -> None:
|
||||
"""Every ``data:`` marker after the first content delta straddles a transfer
|
||||
chunk boundary, so a tearer that inspects each chunk on its own never finds
|
||||
one and lets the stream finish cleanly instead of cutting it."""
|
||||
backend: Final = LiveEdge(cut=StreamCut(after_content=True, mid_chunk=True))
|
||||
with chunked_provider(chunks=SPLIT_MARKER_CHUNKS) as provider:
|
||||
with running_edge(backend, {"openai": provider_url(provider)}) as edge:
|
||||
head, chunks, ending = raw_stream_post(edge.port, STREAM_PATH, STREAM_BODY)
|
||||
|
||||
assert head.startswith("HTTP/1.1 200 OK")
|
||||
assert ending == "truncated"
|
||||
relayed: Final = b"".join(chunks)
|
||||
whole: Final = b"".join(SPLIT_MARKER_CHUNKS)
|
||||
assert whole.startswith(relayed) and relayed != whole
|
||||
assert relayed.startswith(SPLIT_MARKER_CHUNKS[0])
|
||||
torn_line: Final = relayed.rsplit(b"\n", 1)[-1]
|
||||
assert torn_line and SECOND_DATA_LINE.startswith(torn_line) and torn_line != SECOND_DATA_LINE
|
||||
assert b"[DONE]" not in relayed
|
||||
|
||||
|
||||
class TestStreamingFidelity:
|
||||
"""LIT-5742: a streamed response records and replays as the chunk sequence the
|
||||
provider actually sent, not as one coalesced body. The unit of fidelity is the
|
||||
|
|
|
|||
|
|
@ -6,10 +6,13 @@ import pytest
|
|||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from litellm.proxy.common_request_processing import create_response
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
ANTHROPIC_PING_SSE_CHUNK,
|
||||
SSE_COMMENT_PING_BYTES,
|
||||
advance_sse_tail,
|
||||
resolve_ttft_keepalive_interval,
|
||||
seal_open_sse_frame,
|
||||
split_complete_sse_frames,
|
||||
wrap_passthrough_sse_bytes_with_keepalive_pings,
|
||||
wrap_sse_stream_with_keepalive_pings,
|
||||
|
|
@ -32,6 +35,12 @@ def test_split_complete_sse_frames_holds_bytes_with_no_complete_frame():
|
|||
assert split_complete_sse_frames(b"data: unterminated") == (b"", b"data: unterminated")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk", [{"content": "hi"}, ModelResponse()])
|
||||
def test_advance_sse_tail_ignores_a_chunk_that_is_not_sse_text(chunk: object):
|
||||
assert advance_sse_tail(b"\n\n", chunk) == b"\n\n"
|
||||
assert seal_open_sse_frame(advance_sse_tail(b"data: {", chunk)) == "\n" + ANTHROPIC_PING_SSE_CHUNK
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pings_fill_mid_stream_silence_and_preserve_chunk_order():
|
||||
async def gappy_stream() -> AsyncGenerator[str, None]:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional,
|
|||
from urllib.parse import unquote_plus
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import HTTPException, Request, Response, status
|
||||
|
|
@ -14,6 +15,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
DISABLE_ENV_VAR,
|
||||
ISSUE_URL_BASE,
|
||||
|
|
@ -55,6 +57,7 @@ from litellm.proxy.common_request_processing import (
|
|||
sse_error_payload,
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
|
||||
from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
|
|
@ -2543,6 +2546,63 @@ class TestCommonRequestProcessingHelpers:
|
|||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert json.loads(response.body) == {"error": {"code": 403, "message": "forbidden"}}
|
||||
|
||||
async def test_a_stream_that_fails_before_its_first_byte_answers_as_an_anthropic_json_error(self):
|
||||
"""A /v1/messages stream whose first chunk is already the error frame has nothing
|
||||
streamed yet, so the failure answers as JSON with the status the upstream gave,
|
||||
the shape Anthropic clients raise their status-specific errors on"""
|
||||
|
||||
async def stream():
|
||||
yield anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable")
|
||||
yield ANTHROPIC_PING_SSE_CHUNK
|
||||
|
||||
generator: Final = stream()
|
||||
response = await create_response(generator, "text/event-stream", {"x-litellm-call-id": "call-8609"})
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 503
|
||||
assert response.headers["content-type"] == "application/json"
|
||||
assert response.headers["x-litellm-call-id"] == "call-8609"
|
||||
assert json.loads(response.body) == {
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": "upstream unavailable"},
|
||||
}
|
||||
assert generator.ag_frame is None
|
||||
|
||||
async def test_a_stream_that_fails_before_its_first_byte_names_the_call_when_opted_in(self):
|
||||
async def stream():
|
||||
yield anthropic_error_sse_frame(status_code=429, raw_message="slow down")
|
||||
|
||||
response = await create_response(
|
||||
stream(),
|
||||
"text/event-stream",
|
||||
{"x-litellm-call-id": "call-8609"},
|
||||
general_settings={"include_call_id_in_error_body": True},
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 429
|
||||
assert json.loads(response.body) == {
|
||||
"type": "error",
|
||||
"error": {"type": "rate_limit_error", "message": "slow down", "litellm_call_id": "call-8609"},
|
||||
}
|
||||
|
||||
async def test_an_error_event_after_a_keepalive_ping_still_streams(self):
|
||||
"""Once a keepalive ping went out the headers are committed, so the error frame
|
||||
streams as an event instead of turning into a JSON answer"""
|
||||
|
||||
async def stream():
|
||||
yield ANTHROPIC_PING_SSE_CHUNK
|
||||
yield anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable")
|
||||
|
||||
response = await create_response(stream(), "text/event-stream", {})
|
||||
|
||||
assert isinstance(response, StreamingResponse)
|
||||
assert response.status_code == 200
|
||||
assert "".join(await self.consume_stream(response)) == (
|
||||
ANTHROPIC_PING_SSE_CHUNK
|
||||
+ 'event: error\ndata: {"type": "error", "error": {"type": "api_error", "message": "upstream unavailable"}}\n\n'
|
||||
)
|
||||
|
||||
async def test_create_streaming_response_disables_proxy_buffering(self):
|
||||
"""Regression for #28384: every StreamingResponse create_response returns
|
||||
must carry the headers that stop nginx/ingress/Envoy from buffering the
|
||||
|
|
@ -9901,6 +9961,209 @@ class TestErrorLogCarriesCallId:
|
|||
assert call_id in record.getMessage()
|
||||
|
||||
|
||||
class TestAnthropicMessagesStreamErrorFrame:
|
||||
"""A ``/v1/messages`` stream that fails after the headers are out has to say so with an
|
||||
``event: error`` frame. Anthropic clients pick events by name, so a bare ``data:`` line is
|
||||
skipped and the request looks like it ended with nothing in it"""
|
||||
|
||||
@staticmethod
|
||||
def _sse_generator_failing_with(failure: Exception) -> AsyncGenerator[str, None]:
|
||||
class FailingUpstream:
|
||||
def __aiter__(self) -> "FailingUpstream":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> object:
|
||||
raise failure
|
||||
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
return ProxyBaseLLMRequestProcessing.async_sse_data_generator(
|
||||
response=FailingUpstream(),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
request_data={"model": "claude-sonnet-4-5"},
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status_code, expected_error_type",
|
||||
[
|
||||
(429, "rate_limit_error"),
|
||||
(529, "overloaded_error"),
|
||||
(413, "request_too_large"),
|
||||
(500, "api_error"),
|
||||
(502, "api_error"),
|
||||
(400, "invalid_request_error"),
|
||||
],
|
||||
)
|
||||
async def test_mid_stream_failure_arrives_as_an_anthropic_error_event(
|
||||
self, status_code: int, expected_error_type: str
|
||||
) -> None:
|
||||
class UpstreamFailure(Exception):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("upstream stopped sending")
|
||||
self.status_code: Final = status_code
|
||||
|
||||
frames: Final = [frame async for frame in self._sse_generator_failing_with(UpstreamFailure())]
|
||||
|
||||
assert len(frames) == 1
|
||||
event_line, data_line, first_blank, second_blank = frames[0].split("\n")
|
||||
assert isinstance(frames[0], AnthropicErrorSseFrame)
|
||||
assert frames[0].status_code == status_code
|
||||
assert event_line == "event: error"
|
||||
assert (first_blank, second_blank) == ("", "")
|
||||
payload: Final = json.loads(data_line.removeprefix("data: "))
|
||||
assert payload["type"] == "error"
|
||||
assert payload["error"]["type"] == expected_error_type
|
||||
assert "upstream stopped sending" in payload["error"]["message"]
|
||||
|
||||
_CONTENT_DELTA_FRAME: Final = (
|
||||
b"event: content_block_delta\n"
|
||||
b'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1\\n2\\n3"}}\n\n'
|
||||
)
|
||||
_TORN_DATA_LINE: Final = (
|
||||
b"event: content_block_delta\n"
|
||||
b'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"4'
|
||||
)
|
||||
_PING: Final = ANTHROPIC_PING_SSE_CHUNK.encode()
|
||||
|
||||
@staticmethod
|
||||
def _upstream_failure(status_code: int) -> Exception:
|
||||
class UpstreamFailure(Exception):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("upstream stopped sending")
|
||||
self.status_code: Final = status_code
|
||||
|
||||
return UpstreamFailure()
|
||||
|
||||
@staticmethod
|
||||
def _sse_generator_cut_after(relayed: Sequence[bytes], failure: Exception) -> AsyncGenerator[str, None]:
|
||||
class CutUpstream:
|
||||
def __init__(self) -> None:
|
||||
self._remaining: Final = iter(relayed)
|
||||
|
||||
def __aiter__(self) -> "CutUpstream":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> object:
|
||||
chunk: Final = next(self._remaining, None)
|
||||
if chunk is None:
|
||||
raise failure
|
||||
return chunk
|
||||
|
||||
ProxyLogging._callback_capabilities_cache.clear()
|
||||
return ProxyBaseLLMRequestProcessing.async_sse_data_generator(
|
||||
response=CutUpstream(),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
request_data={"model": "claude-sonnet-4-5"},
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_bytes(chunk: object) -> bytes:
|
||||
if isinstance(chunk, bytes):
|
||||
return chunk
|
||||
assert isinstance(chunk, str)
|
||||
return chunk.encode()
|
||||
|
||||
async def _wire_bytes(self, relayed: Sequence[bytes]) -> bytes:
|
||||
stream: Final = self._sse_generator_cut_after(relayed, self._upstream_failure(500))
|
||||
return b"".join([self._as_bytes(chunk) async for chunk in stream])
|
||||
|
||||
@staticmethod
|
||||
def _error_frame_after(wire: bytes, relayed: bytes) -> bytes:
|
||||
assert wire.startswith(relayed), f"the wire did not open with {relayed!r}: {wire!r}"
|
||||
return wire.removeprefix(relayed)
|
||||
|
||||
@staticmethod
|
||||
def _assert_error_frame(frame: bytes) -> None:
|
||||
event_line, data_line, first_blank, second_blank = frame.split(b"\n")
|
||||
assert event_line == b"event: error"
|
||||
assert (first_blank, second_blank) == (b"", b"")
|
||||
payload: Final = json.loads(data_line.removeprefix(b"data: "))
|
||||
assert payload["type"] == "error"
|
||||
assert "upstream stopped sending" in payload["error"]["message"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"torn, seal",
|
||||
[
|
||||
(_TORN_DATA_LINE, b"\n" + _PING),
|
||||
(b"event: content_bl", b"\n" + _PING),
|
||||
(b"event: content_block_delta\n", _PING),
|
||||
(b'event: content_block_delta\r\ndata: {"type":"content_block_delta"}\r\n', _PING),
|
||||
],
|
||||
ids=["mid_data_line", "mid_event_line", "after_a_complete_line", "after_a_crlf_line"],
|
||||
)
|
||||
async def test_a_frame_the_upstream_tore_is_closed_as_a_ping_before_the_error_event(
|
||||
self, torn: bytes, seal: bytes
|
||||
) -> None:
|
||||
wire: Final = await self._wire_bytes((self._CONTENT_DELTA_FRAME, torn))
|
||||
|
||||
self._assert_error_frame(self._error_frame_after(wire, self._CONTENT_DELTA_FRAME + torn + seal))
|
||||
|
||||
async def test_a_cut_at_a_frame_boundary_gets_the_error_event_alone(self) -> None:
|
||||
wire: Final = await self._wire_bytes((self._CONTENT_DELTA_FRAME,))
|
||||
|
||||
self._assert_error_frame(self._error_frame_after(wire, self._CONTENT_DELTA_FRAME))
|
||||
|
||||
async def test_a_torn_frame_still_raises_the_error_in_the_anthropic_sdk(self) -> None:
|
||||
wire: Final = await self._wire_bytes((self._CONTENT_DELTA_FRAME, self._TORN_DATA_LINE))
|
||||
|
||||
def serve(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire)
|
||||
|
||||
client: Final = anthropic.Anthropic(
|
||||
api_key="sk-test",
|
||||
base_url="http://proxy.test",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(serve)),
|
||||
max_retries=0,
|
||||
)
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
for _ in client.messages.create(
|
||||
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "count"}], stream=True
|
||||
):
|
||||
pass
|
||||
body: Final = raised.value.body
|
||||
assert isinstance(body, dict)
|
||||
assert body["type"] == "error"
|
||||
assert "upstream stopped sending" in body["error"]["message"]
|
||||
|
||||
async def test_a_failure_before_the_first_byte_answers_with_its_status_as_json(self) -> None:
|
||||
response: Final = await create_response(
|
||||
self._sse_generator_failing_with(self._upstream_failure(502)), "text/event-stream", {}
|
||||
)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 502
|
||||
body: Final = json.loads(response.body)
|
||||
assert body["type"] == "error"
|
||||
assert body["error"]["type"] == "api_error"
|
||||
assert "upstream stopped sending" in body["error"]["message"]
|
||||
|
||||
async def test_a_failure_before_the_first_byte_raises_with_its_status_in_the_anthropic_sdk(self) -> None:
|
||||
response: Final = await create_response(
|
||||
self._sse_generator_failing_with(self._upstream_failure(502)), "text/event-stream", {}
|
||||
)
|
||||
assert isinstance(response, JSONResponse)
|
||||
|
||||
def serve(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(response.status_code, headers=dict(response.headers), content=response.body)
|
||||
|
||||
client: Final = anthropic.Anthropic(
|
||||
api_key="sk-test",
|
||||
base_url="http://proxy.test",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(serve)),
|
||||
max_retries=0,
|
||||
)
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
client.messages.create(
|
||||
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "count"}], stream=True
|
||||
)
|
||||
assert raised.value.status_code == 502
|
||||
body: Final = raised.value.body
|
||||
assert isinstance(body, dict)
|
||||
assert body["type"] == "error"
|
||||
assert "upstream stopped sending" in body["error"]["message"]
|
||||
|
||||
|
||||
class TestStreamingContainerOwnershipRecordedBeforeDone:
|
||||
"""Regression for LIT-8612: the OpenAI SDK closes the connection at
|
||||
``data: [DONE]`` and starlette cancels the body task, so an ownership row
|
||||
|
|
|
|||
|
|
@ -3,8 +3,15 @@ Tests for AnthropicExceptionMapping class in litellm/anthropic_interface/excepti
|
|||
"""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping
|
||||
import pytest
|
||||
|
||||
from litellm.anthropic_interface.exceptions import (
|
||||
AnthropicErrorSseFrame,
|
||||
AnthropicExceptionMapping,
|
||||
anthropic_error_sse_frame,
|
||||
)
|
||||
|
||||
|
||||
class TestCreateErrorResponse:
|
||||
|
|
@ -206,3 +213,42 @@ class TestTransformToAnthropicError:
|
|||
)
|
||||
assert result["type"] == "error"
|
||||
assert result["error"]["message"] == '["error1", "error2"]'
|
||||
|
||||
|
||||
class TestAnthropicErrorSseFrame:
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "expected_error_type"),
|
||||
[(429, "rate_limit_error"), (503, "api_error"), (400, "invalid_request_error")],
|
||||
)
|
||||
def test_the_frame_is_one_error_event_carrying_the_anthropic_envelope(
|
||||
self, status_code: int, expected_error_type: str
|
||||
) -> None:
|
||||
frame: Final = anthropic_error_sse_frame(status_code=status_code, raw_message="upstream unavailable")
|
||||
|
||||
event_line, data_line, first_blank, second_blank = frame.split("\n")
|
||||
assert event_line == "event: error"
|
||||
assert (first_blank, second_blank) == ("", "")
|
||||
assert json.loads(data_line.removeprefix("data: ")) == {
|
||||
"type": "error",
|
||||
"error": {"type": expected_error_type, "message": "upstream unavailable"},
|
||||
}
|
||||
|
||||
def test_the_frame_remembers_the_status_and_body_it_was_built_from(self) -> None:
|
||||
frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable")
|
||||
|
||||
assert isinstance(frame, AnthropicErrorSseFrame)
|
||||
assert frame.status_code == 503
|
||||
data_line: Final = frame.split("\n")[1]
|
||||
assert data_line == f"data: {json.dumps(frame.json_body(call_id=None))}"
|
||||
|
||||
def test_the_json_body_names_the_call_only_when_asked(self) -> None:
|
||||
frame: Final = anthropic_error_sse_frame(status_code=503, raw_message="upstream unavailable")
|
||||
|
||||
assert frame.json_body(call_id="call-1") == {
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": "upstream unavailable", "litellm_call_id": "call-1"},
|
||||
}
|
||||
assert frame.json_body(call_id=None) == {
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": "upstream unavailable"},
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue