diff --git a/litellm/anthropic_interface/exceptions/__init__.py b/litellm/anthropic_interface/exceptions/__init__.py index 7c3cea0a28a..39568b69cdc 100644 --- a/litellm/anthropic_interface/exceptions/__init__.py +++ b/litellm/anthropic_interface/exceptions/__init__.py @@ -4,6 +4,7 @@ from .exception_mapping_utils import ( ANTHROPIC_ERROR_TYPE_MAP, AnthropicErrorSseFrame, AnthropicExceptionMapping, + anthropic_error_response, anthropic_error_sse_frame, ) from .exceptions import ( @@ -19,5 +20,6 @@ __all__ = [ "AnthropicErrorSseFrame", "AnthropicErrorType", "AnthropicExceptionMapping", + "anthropic_error_response", "anthropic_error_sse_frame", ] diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index d8198e41239..cf82a2fb8a1 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -5,6 +5,7 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop """ import json +from collections.abc import Mapping from typing import Final from pydantic import TypeAdapter @@ -191,15 +192,46 @@ class AnthropicErrorSseFrame(str): return frame def json_body(self, call_id: str | None) -> AnthropicErrorResponse: - if call_id is None: - return self.error_response - detail: Final[AnthropicErrorDetail] = {**self.error_response["error"], "litellm_call_id": call_id} - body: Final[AnthropicErrorResponse] = {**self.error_response, "error": detail} - return body + return _with_error_detail(self.error_response, _with_call_id(self.error_response["error"], call_id)) + + +def _with_provider_specific_fields( + detail: AnthropicErrorDetail, provider_specific_fields: Mapping[str, object] | None +) -> AnthropicErrorDetail: + if not provider_specific_fields: + return detail + with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": provider_specific_fields} + return with_fields + + +def _with_call_id(detail: AnthropicErrorDetail, call_id: str | None) -> AnthropicErrorDetail: + if call_id is None: + return detail + with_call_id: Final[AnthropicErrorDetail] = {**detail, "litellm_call_id": call_id} + return with_call_id + + +def _with_error_detail(envelope: AnthropicErrorResponse, detail: AnthropicErrorDetail) -> AnthropicErrorResponse: + if detail is envelope["error"]: + return envelope + body: Final[AnthropicErrorResponse] = {**envelope, "error": detail} + return body + + +def anthropic_error_response( + status_code: int, + raw_message: str, + request_id: str | None = None, + provider_specific_fields: Mapping[str, object] | None = None, + call_id: str | None = None, +) -> AnthropicErrorResponse: + envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=status_code, raw_message=raw_message, request_id=request_id + ) + return _with_error_detail( + envelope, _with_call_id(_with_provider_specific_fields(envelope["error"], provider_specific_fields), call_id) + ) def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame: - return AnthropicErrorSseFrame( - status_code, - AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message), - ) + return AnthropicErrorSseFrame(status_code, anthropic_error_response(status_code, raw_message)) diff --git a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py index 25d4d7b3ce1..45917f177e8 100644 --- a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES +from litellm.proxy.common_utils.sse_keepalive import SSE_STREAM_START_TAIL, advance_sse_tail, ends_sse_frame if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -213,7 +214,8 @@ class AgenticAnthropicStreamingIterator: self._follow_up_iterator: AsyncIterator | None = None self._drain_task: asyncio.Task | None = None self._hook_task: asyncio.Task | None = None - self._follow_up_chunk_task: asyncio.Task | None = None + self._follow_up_chunk_task: asyncio.Task[bytes | None] | None = None + self._follow_up_tail = SSE_STREAM_START_TAIL self._replay_index = 0 self._error_emitted = False @@ -300,12 +302,14 @@ class AgenticAnthropicStreamingIterator: async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes: if self._follow_up_chunk_task is None: self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator)) - if not await self._completed_within_ping_interval(self._follow_up_chunk_task): - return STREAM_SSE_KEEPALIVE_PING_BYTES + while not await self._completed_within_ping_interval(self._follow_up_chunk_task): + if ends_sse_frame(self._follow_up_tail): + return STREAM_SSE_KEEPALIVE_PING_BYTES chunk: Final = self._follow_up_chunk_task.result() self._follow_up_chunk_task = None if chunk is None: raise StopAsyncIteration + self._follow_up_tail = advance_sse_tail(self._follow_up_tail, chunk) return chunk def _buffer_holds_server_fulfilled_tool_use(self) -> bool: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ce6cd93a22..09398bbb7fa 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3827,7 +3827,7 @@ class ProxyException(Exception): code: int | str | None = None, # maps to status code headers: dict[str, str] | None = None, openai_code: str | None = None, # maps to 'code' in openai - provider_specific_fields: dict | None = None, + provider_specific_fields: Mapping[str, object] | None = None, ): self.message = str(message) super().__init__(self.message) @@ -3854,7 +3854,7 @@ class ProxyException(Exception): def to_dict(self) -> dict: """Converts the ProxyException instance to a dictionary.""" - error_dict: Final[dict[str, str | dict | None]] = { + error_dict: Final[dict[str, str | Mapping[str, object] | None]] = { "message": self.message, "type": self.type, "param": self.param, diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1d88e133cf9..5fe7acf6872 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -13,22 +13,19 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.anthropic_interface.exceptions import ( - AnthropicErrorDetail, - AnthropicErrorResponse, AnthropicExceptionMapping, + anthropic_error_response, ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) -from litellm.llms.base_llm.guardrail_translation.utils import ( - blocked_response_usage as _blocked_response_usage, -) from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, + blocked_anthropic_message, create_response, log_llm_api_exception, proxy_exception_from_http_exception, @@ -59,25 +56,6 @@ from litellm.router import Router router: Final = APIRouter() -def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: - if not exc.provider_specific_fields: - return detail - with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields} - return with_fields - - -def _anthropic_error_detail( - exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None -) -> AnthropicErrorDetail: - if call_id is None: - return _with_provider_specific_fields(exc, detail) - with_call_id: Final[AnthropicErrorDetail] = { - **_with_provider_specific_fields(exc, detail), - "litellm_call_id": call_id, - } - return with_call_id - - def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does @@ -86,16 +64,13 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500 _close_dangling_otel_server_span(request, status_code, exc=exc) - envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error( + content: Final = anthropic_error_response( status_code=status_code, raw_message=exc.message, request_id=request.headers.get("x-request-id"), + provider_specific_fields=exc.provider_specific_fields, + call_id=error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)), ) - body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)) - content: Final[AnthropicErrorResponse] = { - **envelope, - "error": _anthropic_error_detail(exc, envelope["error"], body_call_id), - } return JSONResponse(status_code=status_code, content=content, headers=exc.headers) @@ -229,24 +204,7 @@ async def anthropic_response( request_data=_data, ) - # Create Anthropic-formatted response with violation message - import uuid - - from litellm.types.utils import AnthropicMessagesResponse - - # Report the blocked LLM response's real token usage (carried on the - # exception) instead of discarding it; zero for pre-call blocks. - _usage: Final = _blocked_response_usage(e.original_response) - - _anthropic_response: Final = AnthropicMessagesResponse( - id=f"msg_{uuid.uuid4()}", - type="message", - role="assistant", - content=[{"type": "text", "text": e.message}], - model=e.model, - stop_reason="end_turn", - usage=_usage, - ) + _anthropic_response: Final = blocked_anthropic_message(e) if data.get("stream", None) is not None and data["stream"] is True: # For streaming, use the standard SSE data generator diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 5b2d02e4025..489de137278 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -36,7 +36,11 @@ from typing_extensions import Never import litellm from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger from litellm._uuid import uuid -from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame +from litellm.anthropic_interface.exceptions import ( + AnthropicErrorSseFrame, + anthropic_error_response, + anthropic_error_sse_frame, +) from litellm.constants import ( DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE, DEFAULT_MAX_RECURSE_DEPTH, @@ -51,6 +55,7 @@ from litellm.constants import ( STREAM_SSE_KEEPALIVE_PING_BYTES, UNSAFE_PROXY_RESPONSE_HEADERS, ) +from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.otel.runtime import phase_event from litellm.litellm_core_utils.bug_report import ( @@ -156,7 +161,20 @@ KNOWN_PROXY_ROUTES: Final = frozenset( route for member in LiteLLMRoutes for route in member.value if route.startswith("/") ) -SseErrorDialect: TypeAlias = Literal["openai", "anthropic"] + +@dataclass(frozen=True, slots=True) +class OpenAISseErrors: + pass + + +@dataclass(frozen=True, slots=True) +class AnthropicSseErrors: + request_id: str | None + call_id: Callable[[], str | None] + + +SseErrorDialect: TypeAlias = OpenAISseErrors | AnthropicSseErrors +_OPENAI_SSE_ERRORS: Final = OpenAISseErrors() ProxyRouteType: TypeAlias = Literal[ "acompletion", "aembedding", @@ -251,6 +269,8 @@ ProxyRouteType: TypeAlias = Literal[ "adelete_run", ] from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator +from litellm.llms.base_llm.guardrail_translation.utils import blocked_response_usage # Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format) StreamChunkSerializer = Callable[[object], str] @@ -279,6 +299,7 @@ from litellm.proxy.litellm_pre_call_utils import ( from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.utils import ( + AnthropicMessagesResponse, ModelResponse, ModelResponseStream, StandardLoggingPayloadErrorInformation, @@ -752,6 +773,9 @@ def resolve_counted_tool_fields( return resolved +_PROVIDER_FIELDS: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException: raw_detail: Final = attribute_of(exc, "detail", str(exc)) message, structured_fields = serialize_http_exception_detail(raw_detail) @@ -763,7 +787,7 @@ def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, s type=openai_error_type(exc, error_status), param=openai_error_param(exc), code=error_status, - provider_specific_fields=merged_fields, + provider_specific_fields=_PROVIDER_FIELDS.validate_python(merged_fields), headers=headers, ) @@ -785,7 +809,9 @@ def proxy_exception_from_route_error(exc: Exception) -> ProxyException: param=openai_error_param(exc), code=error_status, openai_code=carried_code if isinstance(carried_code, str) else None, - provider_specific_fields=provider_fields if isinstance(provider_fields, dict) else None, + provider_specific_fields=_PROVIDER_FIELDS.validate_python( + provider_fields if isinstance(provider_fields, dict) else None + ), ) @@ -998,6 +1024,24 @@ async def _close_upstream_and_log_its_spend(upstream: object) -> bool: return wait_for_spend_flush is not None and await wait_for_spend_flush() is True +class _FirstChunkFailedResponse(StreamingResponse): + def __init__( + self, + content: AsyncGenerator[str, None], + *, + error: Exception, + media_type: str, + headers: Mapping[str, str], + status_code: int, + ) -> None: + super().__init__(content, status_code=status_code, headers=headers, media_type=media_type) + self._error = error + + @property + def error(self) -> Exception: + return self._error + + class _UpstreamClosingStreamingResponse(StreamingResponse): """StreamingResponse that always closes its body iterator and the wrapped upstream generator. @@ -1295,8 +1339,9 @@ async def create_response( for frame in _sse_error_frames(error_obj): yield frame - return StreamingResponse( + return _FirstChunkFailedResponse( error_gen_message(), + error=e, media_type=media_type, headers=_sse_stream_headers(await _resolve_stream_headers(headers, refresh_headers)), status_code=error_status, @@ -1375,29 +1420,92 @@ async def _aclose_late_response(produced: Response) -> None: verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc) +def blocked_anthropic_message(exc: ModifyResponseException) -> AnthropicMessagesResponse: + return AnthropicMessagesResponse( + id=f"msg_{uuid.uuid4()}", + type="message", + role="assistant", + content=[{"type": "text", "text": exc.message}], + model=exc.model, + stop_reason="end_turn", + usage=blocked_response_usage(exc.original_response), + ) + + +def _anthropic_error_frame( + error_dialect: AnthropicSseErrors, + status_code: int, + raw_message: str, + provider_specific_fields: Mapping[str, object] | None = None, +) -> AnthropicErrorSseFrame: + return AnthropicErrorSseFrame( + status_code, + anthropic_error_response( + status_code=status_code, + raw_message=raw_message, + request_id=error_dialect.request_id, + provider_specific_fields=provider_specific_fields, + call_id=error_dialect.call_id(), + ), + ) + + def _late_failure_frames(exc: Exception, error_dialect: SseErrorDialect) -> tuple[str, ...]: match error_dialect: - case "openai": + case OpenAISseErrors(): return _sse_error_frames(sse_error_payload(exc)[1]) - case "anthropic": + case AnthropicSseErrors() if isinstance(exc, ModifyResponseException): + return tuple( + chunk.decode() for chunk in FakeAnthropicMessagesStreamIterator(blocked_anthropic_message(exc)).chunks + ) + case AnthropicSseErrors(): proxy_exc: Final = proxy_exception_from_route_error(exc) return ( - anthropic_error_sse_frame( + _anthropic_error_frame( + error_dialect, status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR), raw_message=proxy_exc.message, + provider_specific_fields=proxy_exc.provider_specific_fields, ), ) def _late_body_frames(produced: Response, error_dialect: SseErrorDialect) -> tuple[bytes, ...]: body: Final = bytes(produced.body) or b"{}" - if error_dialect == "anthropic" and produced.status_code >= status.HTTP_400_BAD_REQUEST: - return ( - anthropic_error_sse_frame( - status_code=produced.status_code, raw_message=body.decode(errors="replace") - ).encode(), - ) - return b"data: " + body + b"\n\n", b"data: [DONE]\n\n" + match error_dialect: + case AnthropicSseErrors() if produced.status_code >= status.HTTP_400_BAD_REQUEST: + return ( + _anthropic_error_frame( + error_dialect, status_code=produced.status_code, raw_message=body.decode(errors="replace") + ).encode(), + ) + case _: + return b"data: " + body + b"\n\n", b"data: [DONE]\n\n" + + +def _chunk_bytes(chunk: str | bytes | memoryview) -> bytes: + return chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk) + + +async def _late_stream_error_frames(produced: StreamingResponse, error_dialect: AnthropicSseErrors) -> tuple[str, ...]: + if isinstance(produced, _FirstChunkFailedResponse): + return _late_failure_frames(produced.error, error_dialect) + body: Final = b"".join([_chunk_bytes(chunk) async for chunk in produced.body_iterator]) + return ( + _anthropic_error_frame( + error_dialect, status_code=produced.status_code, raw_message=body.decode(errors="replace") + ), + ) + + +async def _late_stream_chunks(produced: StreamingResponse, error_dialect: SseErrorDialect) -> AsyncIterator[bytes]: + match error_dialect: + case AnthropicSseErrors() if produced.status_code >= status.HTTP_400_BAD_REQUEST: + for frame in await _late_stream_error_frames(produced, error_dialect): + yield frame.encode() + case _: + async for chunk in produced.body_iterator: + yield _chunk_bytes(chunk) async def _relay_late_response(produced: Response, error_dialect: SseErrorDialect) -> AsyncGenerator[bytes, None]: @@ -1410,8 +1518,8 @@ async def _relay_late_response(produced: Response, error_dialect: SseErrorDialec return try: - async for chunk in produced.body_iterator: - yield chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk) + async for chunk in _late_stream_chunks(produced, error_dialect): + yield chunk finally: # Starlette never called this response, so the cleanup its __call__ would # have run has to happen here or the upstream LLM connection leaks. @@ -1448,7 +1556,7 @@ async def open_sse_before_first_byte( ping_interval_seconds: float | str | None, media_type: str = "text/event-stream", on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None, - error_dialect: SseErrorDialect = "openai", + error_dialect: SseErrorDialect = _OPENAI_SSE_ERRORS, ) -> _LateResponseT | StreamingResponse: """Write SSE keepalive comments while the upstream LLM call is still in flight. @@ -2933,7 +3041,14 @@ class ProxyBaseLLMRequestProcessing: ), ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router), on_late_failure=_audit_late_failure, - error_dialect="anthropic" if route_type == "anthropic_messages" else "openai", + error_dialect=( + AnthropicSseErrors( + request_id=request.headers.get("x-request-id"), + call_id=lambda: error_body_call_id(general_settings, self.litellm_call_id), + ) + if route_type == "anthropic_messages" + else _OPENAI_SSE_ERRORS + ), ) async def _process_llm_request( diff --git a/litellm/proxy/common_utils/sse_keepalive.py b/litellm/proxy/common_utils/sse_keepalive.py index be9c24532b4..9c2e1097617 100644 --- a/litellm/proxy/common_utils/sse_keepalive.py +++ b/litellm/proxy/common_utils/sse_keepalive.py @@ -79,7 +79,7 @@ async def _keepalive_ping_stream( # A relayed chunk can be a raw transport read rather than a whole # SSE frame, so an upstream that stalls halfway through a frame # must not have a ping spliced into it. - if recent_tail.endswith(_SSE_FRAME_DELIMITERS): + if ends_sse_frame(recent_tail): yield ping_chunk continue try: @@ -131,6 +131,10 @@ def wrap_passthrough_sse_bytes_with_keepalive_pings( return _keepalive_ping_stream(stream=stream, ping_interval_seconds=interval, ping_chunk=SSE_COMMENT_PING_BYTES) +def ends_sse_frame(recent_tail: bytes) -> bool: + return recent_tail.endswith(_SSE_FRAME_DELIMITERS) + + def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes: written: Final = _sse_tail_bytes(chunk) if not written: @@ -147,7 +151,7 @@ def _sse_tail_bytes(chunk: object) -> bytes: def seal_open_sse_frame(recent_tail: bytes) -> str: - if recent_tail.endswith(_SSE_FRAME_DELIMITERS): + if ends_sse_frame(recent_tail): return "" line_break: Final = "" if recent_tail.endswith((b"\n", b"\r")) else "\n" return f"{line_break}{ANTHROPIC_PING_SSE_CHUNK}" diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e017eee3d27..a90b7e3d6af 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -92,6 +92,8 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( + AnthropicSseErrors, + OpenAISseErrors, ProxyBaseLLMRequestProcessing, log_llm_api_exception, open_sse_before_first_byte, @@ -2338,6 +2340,11 @@ def create_pass_through_route( return await open_sse_before_first_byte( _relay(), ping_interval_seconds=(litellm.sse_keepalive_ping_interval_seconds if is_stream else None), + error_dialect=( + AnthropicSseErrors(request_id=request.headers.get("x-request-id"), call_id=lambda: None) + if custom_llm_provider == "anthropic" + else OpenAISseErrors() + ), ) setattr(endpoint_func, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, True) diff --git a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py index 90a6ae3a574..b549a9e67da 100644 --- a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py @@ -980,7 +980,10 @@ class TestAgenticStreamingIteratorHoldBack: async def test_should_emit_pings_while_the_follow_up_stream_is_slow(self): """The corrected answer can be slow to generate, so the follow-up stream gets keepalives too.""" chunks = _build_tool_use_stream() - phase2_chunks = [b"follow-up-chunk-1", b"follow-up-chunk-2"] + phase2_chunks = [ + _sse_event("message_start", {"type": "message_start", "message": {"id": "follow-up"}}), + _sse_event("message_stop", {"type": "message_stop"}), + ] mock_handler = MagicMock() mock_handler._call_agentic_completion_hooks = AsyncMock( @@ -1001,6 +1004,28 @@ class TestAgenticStreamingIteratorHoldBack: assert collected[first_follow_up_index + 1] == STREAM_SSE_KEEPALIVE_PING_BYTES assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks + @pytest.mark.asyncio + async def test_should_not_ping_inside_a_follow_up_event_that_stalls(self): + """A ping written into a half-sent event corrupts that event for the client.""" + event = _sse_event("content_block_delta", {"type": "content_block_delta", "delta": {"text": "hello"}}) + halves = [event[:20], event[20:]] + + mock_handler = MagicMock() + mock_handler._call_agentic_completion_hooks = AsyncMock( + return_value=MockSlowAsyncStream(halves, delay_seconds=0.06) + ) + + iterator = _build_hold_back_iterator( + MockAsyncStream(_build_tool_use_stream()), + mock_handler, + ping_interval_seconds=0.02, + ) + + collected = [chunk async for chunk in iterator] + + first_half_index = collected.index(halves[0]) + assert collected[first_half_index + 1] == halves[1] + @pytest.mark.asyncio async def test_should_propagate_follow_up_stream_error(self): """A failing follow-up stream surfaces its error instead of hanging on pings forever.""" diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 037b0a6c29a..41af53b6871 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -74,7 +74,9 @@ class TestBlockedResponseUsage: """Blocked responses report the blocked LLM response's real usage.""" def test_uses_original_response_usage(self): - from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage + from litellm.llms.base_llm.guardrail_translation.utils import ( + blocked_response_usage as _blocked_response_usage, + ) # original_response is the AnthropicMessagesResponse the LLM produced # before the guardrail blocked it; its usage is real. @@ -85,7 +87,9 @@ class TestBlockedResponseUsage: } def test_zero_usage_when_no_original_response(self): - from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage + from litellm.llms.base_llm.guardrail_translation.utils import ( + blocked_response_usage as _blocked_response_usage, + ) # Pre-call blocks never invoked the LLM -> nothing consumed. assert _blocked_response_usage(None) == { diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b0504af1b5e..987a275e310 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -8031,6 +8031,76 @@ async def test_pass_through_route_pings_while_the_upstream_call_is_still_running assert collected[-1] in (MESSAGE_START_SSE_FRAME, MESSAGE_START_SSE_FRAME.decode()) +@pytest.mark.asyncio +@pytest.mark.parametrize("custom_llm_provider, expect_error_event", [("anthropic", True), (None, False)]) +async def test_pass_through_route_sends_a_late_upstream_rejection_in_the_providers_error_shape( + custom_llm_provider, expect_error_event +): + """Once a ping is out the upstream's 401 status never reaches the client, and an + Anthropic client only sees an error that arrives as an `event: error`.""" + from fastapi import Response + from fastapi.responses import StreamingResponse + + module = "litellm.proxy.pass_through_endpoints.pass_through_endpoints" + envelope = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}} + payload = json.dumps(envelope).encode() + + async def _rejection(): + yield payload + + async def slow_rejection(**kwargs): + await asyncio.sleep(0.25) + return StreamingResponse(_rejection(), status_code=401, media_type="application/json") + + with ExitStack() as stack: + stack.enter_context( + patch( + f"{module}.InitPassThroughEndpointHelpers.is_registered_pass_through_route", + return_value=True, + ) + ) + stack.enter_context( + patch( + f"{module}.InitPassThroughEndpointHelpers.get_registered_pass_through_route", + return_value=None, + ) + ) + stack.enter_context(patch(f"{module}.pass_through_request", slow_rejection)) + stack.enter_context(patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05)) + + endpoint_func = create_pass_through_route( + endpoint="/anthropic/v1/messages", + target="https://api.anthropic.com/v1/messages", + custom_headers={}, + custom_llm_provider=custom_llm_provider, + is_streaming_request=True, + ) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = httpx.URL("http://test-proxy.com/anthropic/v1/messages") + mock_request.scope = {} + mock_request.body = AsyncMock(return_value=b'{"model": "claude-3", "stream": true}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + mock_request.state = SimpleNamespace() + + response = await endpoint_func( + request=mock_request, + fastapi_response=Response(), + user_api_key_dict=MagicMock(), + ) + wire = b"".join([chunk async for chunk in response.body_iterator]) + + error_events = [ + json.loads(frame.split(b"data: ", 1)[1]) + for frame in wire.split(b"\n\n") + if frame.startswith(b"event: error\n") + ] + assert error_events == ([envelope] if expect_error_event else []) + assert wire.endswith(payload) is not expect_error_event + + def test_passthrough_carries_the_per_model_budgets(): """ Native passthrough builds its logging metadata from diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fa4fefeb191..d91875cefa9 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -18,6 +18,7 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame +from litellm.exceptions import ModifyResponseException from litellm.caching.caching import DualCache from litellm.litellm_core_utils.bug_report import ( DISABLE_ENV_VAR, @@ -37,6 +38,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.common_request_processing import ( + AnthropicSseErrors, ProxyBaseLLMRequestProcessing, ProxyConfig, _await_llm_call_cancelling_on_disconnect, @@ -10541,16 +10543,23 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame() assert collected[-1] == b"data: [DONE]\n\n" -def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError: +_ANTHROPIC_ERRORS: Final = AnthropicSseErrors(request_id=None, call_id=lambda: None) + + +def _anthropic_sdk_over(wire: bytes) -> anthropic.Anthropic: def serve(request: httpx.Request) -> httpx.Response: return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire) - client: Final = anthropic.Anthropic( + return anthropic.Anthropic( api_key="sk-test", base_url="http://proxy.test", http_client=httpx.Client(transport=httpx.MockTransport(serve)), max_retries=0, ) + + +def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError: + client: Final = _anthropic_sdk_over(wire) with pytest.raises(anthropic.APIStatusError) as raised: for _ in client.messages.create( model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True @@ -10580,7 +10589,7 @@ async def test_ttft_keepalive_raises_a_late_failure_in_the_anthropic_sdk(late_fa raise late_failure response = await open_sse_before_first_byte( - slow_failure(), ping_interval_seconds=0.05, error_dialect="anthropic" + slow_failure(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS ) collected = await _drain(response) @@ -10602,13 +10611,111 @@ async def test_ttft_keepalive_raises_a_late_error_body_in_the_anthropic_sdk(): await asyncio.sleep(0.2) return JSONResponse(status_code=529, content=envelope) - response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect="anthropic") + response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS) collected = await _drain(response) assert collected[0] == TTFT_PING assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope +@pytest.mark.asyncio +async def test_ttft_keepalive_raises_a_late_upstream_error_stream_in_the_anthropic_sdk(): + """A pass-through relays the upstream's rejection as a stream under its own + status, which the keepalive already replaced with a 200.""" + envelope = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}} + payload = json.dumps(envelope).encode() + + async def slow_rejection(): + await asyncio.sleep(0.2) + return StreamingResponse(iter([payload[:10], payload[10:]]), status_code=401, media_type="application/json") + + response = await open_sse_before_first_byte( + slow_rejection(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS + ) + collected = await _drain(response) + + assert collected[0] == TTFT_PING + assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope + + +@pytest.mark.asyncio +async def test_ttft_keepalive_raises_a_first_chunk_failure_in_the_anthropic_sdk(): + """A guardrail that rejects the first streamed chunk makes create_response answer + with an error stream, whose status the keepalive already replaced with a 200.""" + + async def rejected_stream(): + for chunk in (): + yield chunk + raise HTTPException(status_code=400, detail="blocked by guardrail") + + async def slow_rejected_stream(): + await asyncio.sleep(0.2) + return await create_response(rejected_stream(), media_type="text/event-stream", headers={}) + + response = await open_sse_before_first_byte( + slow_rejected_stream(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS + ) + collected = await _drain(response) + + assert collected[0] == TTFT_PING + body = _anthropic_sdk_stream_error(b"".join(collected)).body + assert isinstance(body, dict) + assert body["error"] == {"type": "invalid_request_error", "message": "blocked by guardrail"} + + +@pytest.mark.asyncio +async def test_ttft_keepalive_late_anthropic_error_keeps_the_provider_fields(): + failure = ProxyException( + message="quota exhausted", + type="rate_limit_error", + param=None, + code=429, + provider_specific_fields={"retry_after_seconds": 7}, + ) + + async def slow_failure(): + await asyncio.sleep(0.2) + raise failure + + response = await open_sse_before_first_byte( + slow_failure(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS + ) + body = _anthropic_sdk_stream_error(b"".join(await _drain(response))).body + + assert isinstance(body, dict) + assert body["error"]["provider_specific_fields"] == {"retry_after_seconds": 7} + + +@pytest.mark.asyncio +async def test_ttft_keepalive_streams_a_late_guardrail_block_as_the_assistant_message(): + """/v1/messages answers a guardrail block with a 200 message carrying the + violation text, so the keepalive path has to stream that message too.""" + block = ModifyResponseException( + message="Blocked by policy", + model="claude-sonnet-4-5", + request_data={}, + original_response={"usage": {"input_tokens": 31, "output_tokens": 9}}, + ) + + async def slow_block(): + await asyncio.sleep(0.2) + raise block + + response = await open_sse_before_first_byte( + slow_block(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS + ) + wire = b"".join(await _drain(response)) + + with _anthropic_sdk_over(wire).messages.stream( + model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}] + ) as stream: + message = stream.get_final_message() + + assert [block.text for block in message.content if block.type == "text"] == ["Blocked by policy"] + assert message.stop_reason == "end_turn" + assert message.usage.output_tokens == 9 + + @pytest.mark.asyncio async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed(): """Starlette never calls the produced response, so its own cleanup never runs @@ -10897,17 +11004,19 @@ async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_an proxy_logging_obj = MagicMock(spec=ProxyLogging) proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) - processor = ProxyBaseLLMRequestProcessing(data={"model": "claude-sonnet-4-5", "stream": True}) + processor = ProxyBaseLLMRequestProcessing( + data={"model": "claude-sonnet-4-5", "stream": True, "litellm_call_id": "call-1"} + ) with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05): with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_failure): response = await processor.base_process_llm_request( - request=MagicMock(spec=Request), + request=MagicMock(spec=Request, headers={"x-request-id": "req_1"}), fastapi_response=Response(), user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), route_type="anthropic_messages", proxy_logging_obj=proxy_logging_obj, - general_settings={}, + general_settings={"include_call_id_in_error_body": True}, proxy_config=MagicMock(spec=ProxyConfig), ) collected = await _drain(response) @@ -10916,6 +11025,8 @@ async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_an body = _anthropic_sdk_stream_error(b"".join(collected)).body assert isinstance(body, dict) assert body["error"]["message"] == "upstream exploded" + assert body["error"]["litellm_call_id"] == "call-1" + assert body["request_id"] == "req_1" @pytest.mark.asyncio