fix(proxy): stream late guardrail blocks and pass-through failures in the Anthropic shape, and keep agentic follow-up pings between events

This commit is contained in:
mateo-berri 2026-10-29 02:04:35 -07:00
parent 9d5edddb0a
commit 205139ea6d
12 changed files with 425 additions and 93 deletions

View file

@ -4,6 +4,7 @@ from .exception_mapping_utils import (
ANTHROPIC_ERROR_TYPE_MAP,
AnthropicErrorSseFrame,
AnthropicExceptionMapping,
anthropic_error_response,
anthropic_error_sse_frame,
)
from .exceptions import (
@ -19,5 +20,6 @@ __all__ = [
"AnthropicErrorSseFrame",
"AnthropicErrorType",
"AnthropicExceptionMapping",
"anthropic_error_response",
"anthropic_error_sse_frame",
]

View file

@ -5,6 +5,7 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
"""
import json
from collections.abc import Mapping
from typing import Final
from pydantic import TypeAdapter
@ -191,15 +192,46 @@ class AnthropicErrorSseFrame(str):
return frame
def json_body(self, call_id: str | None) -> AnthropicErrorResponse:
if call_id is None:
return self.error_response
detail: Final[AnthropicErrorDetail] = {**self.error_response["error"], "litellm_call_id": call_id}
body: Final[AnthropicErrorResponse] = {**self.error_response, "error": detail}
return body
return _with_error_detail(self.error_response, _with_call_id(self.error_response["error"], call_id))
def _with_provider_specific_fields(
detail: AnthropicErrorDetail, provider_specific_fields: Mapping[str, object] | None
) -> AnthropicErrorDetail:
if not provider_specific_fields:
return detail
with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": provider_specific_fields}
return with_fields
def _with_call_id(detail: AnthropicErrorDetail, call_id: str | None) -> AnthropicErrorDetail:
if call_id is None:
return detail
with_call_id: Final[AnthropicErrorDetail] = {**detail, "litellm_call_id": call_id}
return with_call_id
def _with_error_detail(envelope: AnthropicErrorResponse, detail: AnthropicErrorDetail) -> AnthropicErrorResponse:
if detail is envelope["error"]:
return envelope
body: Final[AnthropicErrorResponse] = {**envelope, "error": detail}
return body
def anthropic_error_response(
status_code: int,
raw_message: str,
request_id: str | None = None,
provider_specific_fields: Mapping[str, object] | None = None,
call_id: str | None = None,
) -> AnthropicErrorResponse:
envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
status_code=status_code, raw_message=raw_message, request_id=request_id
)
return _with_error_detail(
envelope, _with_call_id(_with_provider_specific_fields(envelope["error"], provider_specific_fields), call_id)
)
def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame:
return AnthropicErrorSseFrame(
status_code,
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message),
)
return AnthropicErrorSseFrame(status_code, anthropic_error_response(status_code, raw_message))

View file

@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
from litellm.proxy.common_utils.sse_keepalive import SSE_STREAM_START_TAIL, advance_sse_tail, ends_sse_frame
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -213,7 +214,8 @@ class AgenticAnthropicStreamingIterator:
self._follow_up_iterator: AsyncIterator | None = None
self._drain_task: asyncio.Task | None = None
self._hook_task: asyncio.Task | None = None
self._follow_up_chunk_task: asyncio.Task | None = None
self._follow_up_chunk_task: asyncio.Task[bytes | None] | None = None
self._follow_up_tail = SSE_STREAM_START_TAIL
self._replay_index = 0
self._error_emitted = False
@ -300,12 +302,14 @@ class AgenticAnthropicStreamingIterator:
async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes:
if self._follow_up_chunk_task is None:
self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
return STREAM_SSE_KEEPALIVE_PING_BYTES
while not await self._completed_within_ping_interval(self._follow_up_chunk_task):
if ends_sse_frame(self._follow_up_tail):
return STREAM_SSE_KEEPALIVE_PING_BYTES
chunk: Final = self._follow_up_chunk_task.result()
self._follow_up_chunk_task = None
if chunk is None:
raise StopAsyncIteration
self._follow_up_tail = advance_sse_tail(self._follow_up_tail, chunk)
return chunk
def _buffer_holds_server_fulfilled_tool_use(self) -> bool:

View file

@ -3827,7 +3827,7 @@ class ProxyException(Exception):
code: int | str | None = None, # maps to status code
headers: dict[str, str] | None = None,
openai_code: str | None = None, # maps to 'code' in openai
provider_specific_fields: dict | None = None,
provider_specific_fields: Mapping[str, object] | None = None,
):
self.message = str(message)
super().__init__(self.message)
@ -3854,7 +3854,7 @@ class ProxyException(Exception):
def to_dict(self) -> dict:
"""Converts the ProxyException instance to a dictionary."""
error_dict: Final[dict[str, str | dict | None]] = {
error_dict: Final[dict[str, str | Mapping[str, object] | None]] = {
"message": self.message,
"type": self.type,
"param": self.param,

View file

@ -13,22 +13,19 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.anthropic_interface.exceptions import (
AnthropicErrorDetail,
AnthropicErrorResponse,
AnthropicExceptionMapping,
anthropic_error_response,
)
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage as _blocked_response_usage,
)
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
blocked_anthropic_message,
create_response,
log_llm_api_exception,
proxy_exception_from_http_exception,
@ -59,25 +56,6 @@ from litellm.router import Router
router: Final = APIRouter()
def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail:
if not exc.provider_specific_fields:
return detail
with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields}
return with_fields
def _anthropic_error_detail(
exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None
) -> AnthropicErrorDetail:
if call_id is None:
return _with_provider_specific_fields(exc, detail)
with_call_id: Final[AnthropicErrorDetail] = {
**_with_provider_specific_fields(exc, detail),
"litellm_call_id": call_id,
}
return with_call_id
def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
from litellm.proxy.proxy_server import (
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
@ -86,16 +64,13 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO
status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500
_close_dangling_otel_server_span(request, status_code, exc=exc)
envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
content: Final = anthropic_error_response(
status_code=status_code,
raw_message=exc.message,
request_id=request.headers.get("x-request-id"),
provider_specific_fields=exc.provider_specific_fields,
call_id=error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)),
)
body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER))
content: Final[AnthropicErrorResponse] = {
**envelope,
"error": _anthropic_error_detail(exc, envelope["error"], body_call_id),
}
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
@ -229,24 +204,7 @@ async def anthropic_response(
request_data=_data,
)
# Create Anthropic-formatted response with violation message
import uuid
from litellm.types.utils import AnthropicMessagesResponse
# Report the blocked LLM response's real token usage (carried on the
# exception) instead of discarding it; zero for pre-call blocks.
_usage: Final = _blocked_response_usage(e.original_response)
_anthropic_response: Final = AnthropicMessagesResponse(
id=f"msg_{uuid.uuid4()}",
type="message",
role="assistant",
content=[{"type": "text", "text": e.message}],
model=e.model,
stop_reason="end_turn",
usage=_usage,
)
_anthropic_response: Final = blocked_anthropic_message(e)
if data.get("stream", None) is not None and data["stream"] is True:
# For streaming, use the standard SSE data generator

View file

@ -36,7 +36,11 @@ from typing_extensions import Never
import litellm
from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger
from litellm._uuid import uuid
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
from litellm.anthropic_interface.exceptions import (
AnthropicErrorSseFrame,
anthropic_error_response,
anthropic_error_sse_frame,
)
from litellm.constants import (
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
DEFAULT_MAX_RECURSE_DEPTH,
@ -51,6 +55,7 @@ from litellm.constants import (
STREAM_SSE_KEEPALIVE_PING_BYTES,
UNSAFE_PROXY_RESPONSE_HEADERS,
)
from litellm.exceptions import ModifyResponseException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.otel.runtime import phase_event
from litellm.litellm_core_utils.bug_report import (
@ -156,7 +161,20 @@ KNOWN_PROXY_ROUTES: Final = frozenset(
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
)
SseErrorDialect: TypeAlias = Literal["openai", "anthropic"]
@dataclass(frozen=True, slots=True)
class OpenAISseErrors:
pass
@dataclass(frozen=True, slots=True)
class AnthropicSseErrors:
request_id: str | None
call_id: Callable[[], str | None]
SseErrorDialect: TypeAlias = OpenAISseErrors | AnthropicSseErrors
_OPENAI_SSE_ERRORS: Final = OpenAISseErrors()
ProxyRouteType: TypeAlias = Literal[
"acompletion",
"aembedding",
@ -251,6 +269,8 @@ ProxyRouteType: TypeAlias = Literal[
"adelete_run",
]
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator
from litellm.llms.base_llm.guardrail_translation.utils import blocked_response_usage
# Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format)
StreamChunkSerializer = Callable[[object], str]
@ -279,6 +299,7 @@ from litellm.proxy.litellm_pre_call_utils import (
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.utils import (
AnthropicMessagesResponse,
ModelResponse,
ModelResponseStream,
StandardLoggingPayloadErrorInformation,
@ -752,6 +773,9 @@ def resolve_counted_tool_fields(
return resolved
_PROVIDER_FIELDS: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
raw_detail: Final = attribute_of(exc, "detail", str(exc))
message, structured_fields = serialize_http_exception_detail(raw_detail)
@ -763,7 +787,7 @@ def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, s
type=openai_error_type(exc, error_status),
param=openai_error_param(exc),
code=error_status,
provider_specific_fields=merged_fields,
provider_specific_fields=_PROVIDER_FIELDS.validate_python(merged_fields),
headers=headers,
)
@ -785,7 +809,9 @@ def proxy_exception_from_route_error(exc: Exception) -> ProxyException:
param=openai_error_param(exc),
code=error_status,
openai_code=carried_code if isinstance(carried_code, str) else None,
provider_specific_fields=provider_fields if isinstance(provider_fields, dict) else None,
provider_specific_fields=_PROVIDER_FIELDS.validate_python(
provider_fields if isinstance(provider_fields, dict) else None
),
)
@ -998,6 +1024,24 @@ async def _close_upstream_and_log_its_spend(upstream: object) -> bool:
return wait_for_spend_flush is not None and await wait_for_spend_flush() is True
class _FirstChunkFailedResponse(StreamingResponse):
def __init__(
self,
content: AsyncGenerator[str, None],
*,
error: Exception,
media_type: str,
headers: Mapping[str, str],
status_code: int,
) -> None:
super().__init__(content, status_code=status_code, headers=headers, media_type=media_type)
self._error = error
@property
def error(self) -> Exception:
return self._error
class _UpstreamClosingStreamingResponse(StreamingResponse):
"""StreamingResponse that always closes its body iterator and the wrapped
upstream generator.
@ -1295,8 +1339,9 @@ async def create_response(
for frame in _sse_error_frames(error_obj):
yield frame
return StreamingResponse(
return _FirstChunkFailedResponse(
error_gen_message(),
error=e,
media_type=media_type,
headers=_sse_stream_headers(await _resolve_stream_headers(headers, refresh_headers)),
status_code=error_status,
@ -1375,29 +1420,92 @@ async def _aclose_late_response(produced: Response) -> None:
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
def blocked_anthropic_message(exc: ModifyResponseException) -> AnthropicMessagesResponse:
return AnthropicMessagesResponse(
id=f"msg_{uuid.uuid4()}",
type="message",
role="assistant",
content=[{"type": "text", "text": exc.message}],
model=exc.model,
stop_reason="end_turn",
usage=blocked_response_usage(exc.original_response),
)
def _anthropic_error_frame(
error_dialect: AnthropicSseErrors,
status_code: int,
raw_message: str,
provider_specific_fields: Mapping[str, object] | None = None,
) -> AnthropicErrorSseFrame:
return AnthropicErrorSseFrame(
status_code,
anthropic_error_response(
status_code=status_code,
raw_message=raw_message,
request_id=error_dialect.request_id,
provider_specific_fields=provider_specific_fields,
call_id=error_dialect.call_id(),
),
)
def _late_failure_frames(exc: Exception, error_dialect: SseErrorDialect) -> tuple[str, ...]:
match error_dialect:
case "openai":
case OpenAISseErrors():
return _sse_error_frames(sse_error_payload(exc)[1])
case "anthropic":
case AnthropicSseErrors() if isinstance(exc, ModifyResponseException):
return tuple(
chunk.decode() for chunk in FakeAnthropicMessagesStreamIterator(blocked_anthropic_message(exc)).chunks
)
case AnthropicSseErrors():
proxy_exc: Final = proxy_exception_from_route_error(exc)
return (
anthropic_error_sse_frame(
_anthropic_error_frame(
error_dialect,
status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR),
raw_message=proxy_exc.message,
provider_specific_fields=proxy_exc.provider_specific_fields,
),
)
def _late_body_frames(produced: Response, error_dialect: SseErrorDialect) -> tuple[bytes, ...]:
body: Final = bytes(produced.body) or b"{}"
if error_dialect == "anthropic" and produced.status_code >= status.HTTP_400_BAD_REQUEST:
return (
anthropic_error_sse_frame(
status_code=produced.status_code, raw_message=body.decode(errors="replace")
).encode(),
)
return b"data: " + body + b"\n\n", b"data: [DONE]\n\n"
match error_dialect:
case AnthropicSseErrors() if produced.status_code >= status.HTTP_400_BAD_REQUEST:
return (
_anthropic_error_frame(
error_dialect, status_code=produced.status_code, raw_message=body.decode(errors="replace")
).encode(),
)
case _:
return b"data: " + body + b"\n\n", b"data: [DONE]\n\n"
def _chunk_bytes(chunk: str | bytes | memoryview) -> bytes:
return chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk)
async def _late_stream_error_frames(produced: StreamingResponse, error_dialect: AnthropicSseErrors) -> tuple[str, ...]:
if isinstance(produced, _FirstChunkFailedResponse):
return _late_failure_frames(produced.error, error_dialect)
body: Final = b"".join([_chunk_bytes(chunk) async for chunk in produced.body_iterator])
return (
_anthropic_error_frame(
error_dialect, status_code=produced.status_code, raw_message=body.decode(errors="replace")
),
)
async def _late_stream_chunks(produced: StreamingResponse, error_dialect: SseErrorDialect) -> AsyncIterator[bytes]:
match error_dialect:
case AnthropicSseErrors() if produced.status_code >= status.HTTP_400_BAD_REQUEST:
for frame in await _late_stream_error_frames(produced, error_dialect):
yield frame.encode()
case _:
async for chunk in produced.body_iterator:
yield _chunk_bytes(chunk)
async def _relay_late_response(produced: Response, error_dialect: SseErrorDialect) -> AsyncGenerator[bytes, None]:
@ -1410,8 +1518,8 @@ async def _relay_late_response(produced: Response, error_dialect: SseErrorDialec
return
try:
async for chunk in produced.body_iterator:
yield chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk)
async for chunk in _late_stream_chunks(produced, error_dialect):
yield chunk
finally:
# Starlette never called this response, so the cleanup its __call__ would
# have run has to happen here or the upstream LLM connection leaks.
@ -1448,7 +1556,7 @@ async def open_sse_before_first_byte(
ping_interval_seconds: float | str | None,
media_type: str = "text/event-stream",
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
error_dialect: SseErrorDialect = "openai",
error_dialect: SseErrorDialect = _OPENAI_SSE_ERRORS,
) -> _LateResponseT | StreamingResponse:
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
@ -2933,7 +3041,14 @@ class ProxyBaseLLMRequestProcessing:
),
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
on_late_failure=_audit_late_failure,
error_dialect="anthropic" if route_type == "anthropic_messages" else "openai",
error_dialect=(
AnthropicSseErrors(
request_id=request.headers.get("x-request-id"),
call_id=lambda: error_body_call_id(general_settings, self.litellm_call_id),
)
if route_type == "anthropic_messages"
else _OPENAI_SSE_ERRORS
),
)
async def _process_llm_request(

View file

@ -79,7 +79,7 @@ async def _keepalive_ping_stream(
# A relayed chunk can be a raw transport read rather than a whole
# SSE frame, so an upstream that stalls halfway through a frame
# must not have a ping spliced into it.
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
if ends_sse_frame(recent_tail):
yield ping_chunk
continue
try:
@ -131,6 +131,10 @@ def wrap_passthrough_sse_bytes_with_keepalive_pings(
return _keepalive_ping_stream(stream=stream, ping_interval_seconds=interval, ping_chunk=SSE_COMMENT_PING_BYTES)
def ends_sse_frame(recent_tail: bytes) -> bool:
return recent_tail.endswith(_SSE_FRAME_DELIMITERS)
def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes:
written: Final = _sse_tail_bytes(chunk)
if not written:
@ -147,7 +151,7 @@ def _sse_tail_bytes(chunk: object) -> bytes:
def seal_open_sse_frame(recent_tail: bytes) -> str:
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
if ends_sse_frame(recent_tail):
return ""
line_break: Final = "" if recent_tail.endswith((b"\n", b"\r")) else "\n"
return f"{line_break}{ANTHROPIC_PING_SSE_CHUNK}"

View file

@ -92,6 +92,8 @@ from litellm.proxy.auth.auth_utils import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
AnthropicSseErrors,
OpenAISseErrors,
ProxyBaseLLMRequestProcessing,
log_llm_api_exception,
open_sse_before_first_byte,
@ -2338,6 +2340,11 @@ def create_pass_through_route(
return await open_sse_before_first_byte(
_relay(),
ping_interval_seconds=(litellm.sse_keepalive_ping_interval_seconds if is_stream else None),
error_dialect=(
AnthropicSseErrors(request_id=request.headers.get("x-request-id"), call_id=lambda: None)
if custom_llm_provider == "anthropic"
else OpenAISseErrors()
),
)
setattr(endpoint_func, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, True)

View file

@ -980,7 +980,10 @@ class TestAgenticStreamingIteratorHoldBack:
async def test_should_emit_pings_while_the_follow_up_stream_is_slow(self):
"""The corrected answer can be slow to generate, so the follow-up stream gets keepalives too."""
chunks = _build_tool_use_stream()
phase2_chunks = [b"follow-up-chunk-1", b"follow-up-chunk-2"]
phase2_chunks = [
_sse_event("message_start", {"type": "message_start", "message": {"id": "follow-up"}}),
_sse_event("message_stop", {"type": "message_stop"}),
]
mock_handler = MagicMock()
mock_handler._call_agentic_completion_hooks = AsyncMock(
@ -1001,6 +1004,28 @@ class TestAgenticStreamingIteratorHoldBack:
assert collected[first_follow_up_index + 1] == STREAM_SSE_KEEPALIVE_PING_BYTES
assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks
@pytest.mark.asyncio
async def test_should_not_ping_inside_a_follow_up_event_that_stalls(self):
"""A ping written into a half-sent event corrupts that event for the client."""
event = _sse_event("content_block_delta", {"type": "content_block_delta", "delta": {"text": "hello"}})
halves = [event[:20], event[20:]]
mock_handler = MagicMock()
mock_handler._call_agentic_completion_hooks = AsyncMock(
return_value=MockSlowAsyncStream(halves, delay_seconds=0.06)
)
iterator = _build_hold_back_iterator(
MockAsyncStream(_build_tool_use_stream()),
mock_handler,
ping_interval_seconds=0.02,
)
collected = [chunk async for chunk in iterator]
first_half_index = collected.index(halves[0])
assert collected[first_half_index + 1] == halves[1]
@pytest.mark.asyncio
async def test_should_propagate_follow_up_stream_error(self):
"""A failing follow-up stream surfaces its error instead of hanging on pings forever."""

View file

@ -74,7 +74,9 @@ class TestBlockedResponseUsage:
"""Blocked responses report the blocked LLM response's real usage."""
def test_uses_original_response_usage(self):
from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage as _blocked_response_usage,
)
# original_response is the AnthropicMessagesResponse the LLM produced
# before the guardrail blocked it; its usage is real.
@ -85,7 +87,9 @@ class TestBlockedResponseUsage:
}
def test_zero_usage_when_no_original_response(self):
from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_response_usage as _blocked_response_usage,
)
# Pre-call blocks never invoked the LLM -> nothing consumed.
assert _blocked_response_usage(None) == {

View file

@ -8031,6 +8031,76 @@ async def test_pass_through_route_pings_while_the_upstream_call_is_still_running
assert collected[-1] in (MESSAGE_START_SSE_FRAME, MESSAGE_START_SSE_FRAME.decode())
@pytest.mark.asyncio
@pytest.mark.parametrize("custom_llm_provider, expect_error_event", [("anthropic", True), (None, False)])
async def test_pass_through_route_sends_a_late_upstream_rejection_in_the_providers_error_shape(
custom_llm_provider, expect_error_event
):
"""Once a ping is out the upstream's 401 status never reaches the client, and an
Anthropic client only sees an error that arrives as an `event: error`."""
from fastapi import Response
from fastapi.responses import StreamingResponse
module = "litellm.proxy.pass_through_endpoints.pass_through_endpoints"
envelope = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}}
payload = json.dumps(envelope).encode()
async def _rejection():
yield payload
async def slow_rejection(**kwargs):
await asyncio.sleep(0.25)
return StreamingResponse(_rejection(), status_code=401, media_type="application/json")
with ExitStack() as stack:
stack.enter_context(
patch(
f"{module}.InitPassThroughEndpointHelpers.is_registered_pass_through_route",
return_value=True,
)
)
stack.enter_context(
patch(
f"{module}.InitPassThroughEndpointHelpers.get_registered_pass_through_route",
return_value=None,
)
)
stack.enter_context(patch(f"{module}.pass_through_request", slow_rejection))
stack.enter_context(patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05))
endpoint_func = create_pass_through_route(
endpoint="/anthropic/v1/messages",
target="https://api.anthropic.com/v1/messages",
custom_headers={},
custom_llm_provider=custom_llm_provider,
is_streaming_request=True,
)
mock_request = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.url = httpx.URL("http://test-proxy.com/anthropic/v1/messages")
mock_request.scope = {}
mock_request.body = AsyncMock(return_value=b'{"model": "claude-3", "stream": true}')
mock_request.headers = Headers({"content-type": "application/json"})
mock_request.query_params = QueryParams({})
mock_request.state = SimpleNamespace()
response = await endpoint_func(
request=mock_request,
fastapi_response=Response(),
user_api_key_dict=MagicMock(),
)
wire = b"".join([chunk async for chunk in response.body_iterator])
error_events = [
json.loads(frame.split(b"data: ", 1)[1])
for frame in wire.split(b"\n\n")
if frame.startswith(b"event: error\n")
]
assert error_events == ([envelope] if expect_error_event else [])
assert wire.endswith(payload) is not expect_error_event
def test_passthrough_carries_the_per_model_budgets():
"""
Native passthrough builds its logging metadata from

View file

@ -18,6 +18,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
import litellm
from litellm._uuid import uuid
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
from litellm.exceptions import ModifyResponseException
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.bug_report import (
DISABLE_ENV_VAR,
@ -37,6 +38,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.opentelemetry import UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.common_request_processing import (
AnthropicSseErrors,
ProxyBaseLLMRequestProcessing,
ProxyConfig,
_await_llm_call_cancelling_on_disconnect,
@ -10541,16 +10543,23 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
assert collected[-1] == b"data: [DONE]\n\n"
def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError:
_ANTHROPIC_ERRORS: Final = AnthropicSseErrors(request_id=None, call_id=lambda: None)
def _anthropic_sdk_over(wire: bytes) -> anthropic.Anthropic:
def serve(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire)
client: Final = anthropic.Anthropic(
return anthropic.Anthropic(
api_key="sk-test",
base_url="http://proxy.test",
http_client=httpx.Client(transport=httpx.MockTransport(serve)),
max_retries=0,
)
def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError:
client: Final = _anthropic_sdk_over(wire)
with pytest.raises(anthropic.APIStatusError) as raised:
for _ in client.messages.create(
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True
@ -10580,7 +10589,7 @@ async def test_ttft_keepalive_raises_a_late_failure_in_the_anthropic_sdk(late_fa
raise late_failure
response = await open_sse_before_first_byte(
slow_failure(), ping_interval_seconds=0.05, error_dialect="anthropic"
slow_failure(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
)
collected = await _drain(response)
@ -10602,13 +10611,111 @@ async def test_ttft_keepalive_raises_a_late_error_body_in_the_anthropic_sdk():
await asyncio.sleep(0.2)
return JSONResponse(status_code=529, content=envelope)
response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect="anthropic")
response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS)
collected = await _drain(response)
assert collected[0] == TTFT_PING
assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope
@pytest.mark.asyncio
async def test_ttft_keepalive_raises_a_late_upstream_error_stream_in_the_anthropic_sdk():
"""A pass-through relays the upstream's rejection as a stream under its own
status, which the keepalive already replaced with a 200."""
envelope = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}}
payload = json.dumps(envelope).encode()
async def slow_rejection():
await asyncio.sleep(0.2)
return StreamingResponse(iter([payload[:10], payload[10:]]), status_code=401, media_type="application/json")
response = await open_sse_before_first_byte(
slow_rejection(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
)
collected = await _drain(response)
assert collected[0] == TTFT_PING
assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope
@pytest.mark.asyncio
async def test_ttft_keepalive_raises_a_first_chunk_failure_in_the_anthropic_sdk():
"""A guardrail that rejects the first streamed chunk makes create_response answer
with an error stream, whose status the keepalive already replaced with a 200."""
async def rejected_stream():
for chunk in ():
yield chunk
raise HTTPException(status_code=400, detail="blocked by guardrail")
async def slow_rejected_stream():
await asyncio.sleep(0.2)
return await create_response(rejected_stream(), media_type="text/event-stream", headers={})
response = await open_sse_before_first_byte(
slow_rejected_stream(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
)
collected = await _drain(response)
assert collected[0] == TTFT_PING
body = _anthropic_sdk_stream_error(b"".join(collected)).body
assert isinstance(body, dict)
assert body["error"] == {"type": "invalid_request_error", "message": "blocked by guardrail"}
@pytest.mark.asyncio
async def test_ttft_keepalive_late_anthropic_error_keeps_the_provider_fields():
failure = ProxyException(
message="quota exhausted",
type="rate_limit_error",
param=None,
code=429,
provider_specific_fields={"retry_after_seconds": 7},
)
async def slow_failure():
await asyncio.sleep(0.2)
raise failure
response = await open_sse_before_first_byte(
slow_failure(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
)
body = _anthropic_sdk_stream_error(b"".join(await _drain(response))).body
assert isinstance(body, dict)
assert body["error"]["provider_specific_fields"] == {"retry_after_seconds": 7}
@pytest.mark.asyncio
async def test_ttft_keepalive_streams_a_late_guardrail_block_as_the_assistant_message():
"""/v1/messages answers a guardrail block with a 200 message carrying the
violation text, so the keepalive path has to stream that message too."""
block = ModifyResponseException(
message="Blocked by policy",
model="claude-sonnet-4-5",
request_data={},
original_response={"usage": {"input_tokens": 31, "output_tokens": 9}},
)
async def slow_block():
await asyncio.sleep(0.2)
raise block
response = await open_sse_before_first_byte(
slow_block(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
)
wire = b"".join(await _drain(response))
with _anthropic_sdk_over(wire).messages.stream(
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}]
) as stream:
message = stream.get_final_message()
assert [block.text for block in message.content if block.type == "text"] == ["Blocked by policy"]
assert message.stop_reason == "end_turn"
assert message.usage.output_tokens == 9
@pytest.mark.asyncio
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
"""Starlette never calls the produced response, so its own cleanup never runs
@ -10897,17 +11004,19 @@ async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_an
proxy_logging_obj = MagicMock(spec=ProxyLogging)
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
processor = ProxyBaseLLMRequestProcessing(data={"model": "claude-sonnet-4-5", "stream": True})
processor = ProxyBaseLLMRequestProcessing(
data={"model": "claude-sonnet-4-5", "stream": True, "litellm_call_id": "call-1"}
)
with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05):
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_failure):
response = await processor.base_process_llm_request(
request=MagicMock(spec=Request),
request=MagicMock(spec=Request, headers={"x-request-id": "req_1"}),
fastapi_response=Response(),
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
route_type="anthropic_messages",
proxy_logging_obj=proxy_logging_obj,
general_settings={},
general_settings={"include_call_id_in_error_body": True},
proxy_config=MagicMock(spec=ProxyConfig),
)
collected = await _drain(response)
@ -10916,6 +11025,8 @@ async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_an
body = _anthropic_sdk_stream_error(b"".join(collected)).body
assert isinstance(body, dict)
assert body["error"]["message"] == "upstream exploded"
assert body["error"]["litellm_call_id"] == "call-1"
assert body["request_id"] == "req_1"
@pytest.mark.asyncio