mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): stream late guardrail blocks and pass-through failures in the Anthropic shape, and keep agentic follow-up pings between events
This commit is contained in:
parent
9d5edddb0a
commit
205139ea6d
12 changed files with 425 additions and 93 deletions
|
|
@ -4,6 +4,7 @@ from .exception_mapping_utils import (
|
|||
ANTHROPIC_ERROR_TYPE_MAP,
|
||||
AnthropicErrorSseFrame,
|
||||
AnthropicExceptionMapping,
|
||||
anthropic_error_response,
|
||||
anthropic_error_sse_frame,
|
||||
)
|
||||
from .exceptions import (
|
||||
|
|
@ -19,5 +20,6 @@ __all__ = [
|
|||
"AnthropicErrorSseFrame",
|
||||
"AnthropicErrorType",
|
||||
"AnthropicExceptionMapping",
|
||||
"anthropic_error_response",
|
||||
"anthropic_error_sse_frame",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -191,15 +192,46 @@ class AnthropicErrorSseFrame(str):
|
|||
return frame
|
||||
|
||||
def json_body(self, call_id: str | None) -> AnthropicErrorResponse:
|
||||
if call_id is None:
|
||||
return self.error_response
|
||||
detail: Final[AnthropicErrorDetail] = {**self.error_response["error"], "litellm_call_id": call_id}
|
||||
body: Final[AnthropicErrorResponse] = {**self.error_response, "error": detail}
|
||||
return body
|
||||
return _with_error_detail(self.error_response, _with_call_id(self.error_response["error"], call_id))
|
||||
|
||||
|
||||
def _with_provider_specific_fields(
|
||||
detail: AnthropicErrorDetail, provider_specific_fields: Mapping[str, object] | None
|
||||
) -> AnthropicErrorDetail:
|
||||
if not provider_specific_fields:
|
||||
return detail
|
||||
with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": provider_specific_fields}
|
||||
return with_fields
|
||||
|
||||
|
||||
def _with_call_id(detail: AnthropicErrorDetail, call_id: str | None) -> AnthropicErrorDetail:
|
||||
if call_id is None:
|
||||
return detail
|
||||
with_call_id: Final[AnthropicErrorDetail] = {**detail, "litellm_call_id": call_id}
|
||||
return with_call_id
|
||||
|
||||
|
||||
def _with_error_detail(envelope: AnthropicErrorResponse, detail: AnthropicErrorDetail) -> AnthropicErrorResponse:
|
||||
if detail is envelope["error"]:
|
||||
return envelope
|
||||
body: Final[AnthropicErrorResponse] = {**envelope, "error": detail}
|
||||
return body
|
||||
|
||||
|
||||
def anthropic_error_response(
|
||||
status_code: int,
|
||||
raw_message: str,
|
||||
request_id: str | None = None,
|
||||
provider_specific_fields: Mapping[str, object] | None = None,
|
||||
call_id: str | None = None,
|
||||
) -> AnthropicErrorResponse:
|
||||
envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
status_code=status_code, raw_message=raw_message, request_id=request_id
|
||||
)
|
||||
return _with_error_detail(
|
||||
envelope, _with_call_id(_with_provider_specific_fields(envelope["error"], provider_specific_fields), call_id)
|
||||
)
|
||||
|
||||
|
||||
def anthropic_error_sse_frame(status_code: int, raw_message: str) -> AnthropicErrorSseFrame:
|
||||
return AnthropicErrorSseFrame(
|
||||
status_code,
|
||||
AnthropicExceptionMapping.transform_to_anthropic_error(status_code=status_code, raw_message=raw_message),
|
||||
)
|
||||
return AnthropicErrorSseFrame(status_code, anthropic_error_response(status_code, raw_message))
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
from litellm.proxy.common_utils.sse_keepalive import SSE_STREAM_START_TAIL, advance_sse_tail, ends_sse_frame
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -213,7 +214,8 @@ class AgenticAnthropicStreamingIterator:
|
|||
self._follow_up_iterator: AsyncIterator | None = None
|
||||
self._drain_task: asyncio.Task | None = None
|
||||
self._hook_task: asyncio.Task | None = None
|
||||
self._follow_up_chunk_task: asyncio.Task | None = None
|
||||
self._follow_up_chunk_task: asyncio.Task[bytes | None] | None = None
|
||||
self._follow_up_tail = SSE_STREAM_START_TAIL
|
||||
self._replay_index = 0
|
||||
self._error_emitted = False
|
||||
|
||||
|
|
@ -300,12 +302,14 @@ class AgenticAnthropicStreamingIterator:
|
|||
async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes:
|
||||
if self._follow_up_chunk_task is None:
|
||||
self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
|
||||
if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
|
||||
return STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
while not await self._completed_within_ping_interval(self._follow_up_chunk_task):
|
||||
if ends_sse_frame(self._follow_up_tail):
|
||||
return STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
chunk: Final = self._follow_up_chunk_task.result()
|
||||
self._follow_up_chunk_task = None
|
||||
if chunk is None:
|
||||
raise StopAsyncIteration
|
||||
self._follow_up_tail = advance_sse_tail(self._follow_up_tail, chunk)
|
||||
return chunk
|
||||
|
||||
def _buffer_holds_server_fulfilled_tool_use(self) -> bool:
|
||||
|
|
|
|||
|
|
@ -3827,7 +3827,7 @@ class ProxyException(Exception):
|
|||
code: int | str | None = None, # maps to status code
|
||||
headers: dict[str, str] | None = None,
|
||||
openai_code: str | None = None, # maps to 'code' in openai
|
||||
provider_specific_fields: dict | None = None,
|
||||
provider_specific_fields: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.message = str(message)
|
||||
super().__init__(self.message)
|
||||
|
|
@ -3854,7 +3854,7 @@ class ProxyException(Exception):
|
|||
|
||||
def to_dict(self) -> dict:
|
||||
"""Converts the ProxyException instance to a dictionary."""
|
||||
error_dict: Final[dict[str, str | dict | None]] = {
|
||||
error_dict: Final[dict[str, str | Mapping[str, object] | None]] = {
|
||||
"message": self.message,
|
||||
"type": self.type,
|
||||
"param": self.param,
|
||||
|
|
|
|||
|
|
@ -13,22 +13,19 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.anthropic_interface.exceptions import (
|
||||
AnthropicErrorDetail,
|
||||
AnthropicErrorResponse,
|
||||
AnthropicExceptionMapping,
|
||||
anthropic_error_response,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
)
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
blocked_response_usage as _blocked_response_usage,
|
||||
)
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
blocked_anthropic_message,
|
||||
create_response,
|
||||
log_llm_api_exception,
|
||||
proxy_exception_from_http_exception,
|
||||
|
|
@ -59,25 +56,6 @@ from litellm.router import Router
|
|||
router: Final = APIRouter()
|
||||
|
||||
|
||||
def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail:
|
||||
if not exc.provider_specific_fields:
|
||||
return detail
|
||||
with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields}
|
||||
return with_fields
|
||||
|
||||
|
||||
def _anthropic_error_detail(
|
||||
exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None
|
||||
) -> AnthropicErrorDetail:
|
||||
if call_id is None:
|
||||
return _with_provider_specific_fields(exc, detail)
|
||||
with_call_id: Final[AnthropicErrorDetail] = {
|
||||
**_with_provider_specific_fields(exc, detail),
|
||||
"litellm_call_id": call_id,
|
||||
}
|
||||
return with_call_id
|
||||
|
||||
|
||||
def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
|
||||
from litellm.proxy.proxy_server import (
|
||||
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
|
||||
|
|
@ -86,16 +64,13 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO
|
|||
|
||||
status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500
|
||||
_close_dangling_otel_server_span(request, status_code, exc=exc)
|
||||
envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
content: Final = anthropic_error_response(
|
||||
status_code=status_code,
|
||||
raw_message=exc.message,
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
provider_specific_fields=exc.provider_specific_fields,
|
||||
call_id=error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)),
|
||||
)
|
||||
body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER))
|
||||
content: Final[AnthropicErrorResponse] = {
|
||||
**envelope,
|
||||
"error": _anthropic_error_detail(exc, envelope["error"], body_call_id),
|
||||
}
|
||||
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
|
||||
|
||||
|
||||
|
|
@ -229,24 +204,7 @@ async def anthropic_response(
|
|||
request_data=_data,
|
||||
)
|
||||
|
||||
# Create Anthropic-formatted response with violation message
|
||||
import uuid
|
||||
|
||||
from litellm.types.utils import AnthropicMessagesResponse
|
||||
|
||||
# Report the blocked LLM response's real token usage (carried on the
|
||||
# exception) instead of discarding it; zero for pre-call blocks.
|
||||
_usage: Final = _blocked_response_usage(e.original_response)
|
||||
|
||||
_anthropic_response: Final = AnthropicMessagesResponse(
|
||||
id=f"msg_{uuid.uuid4()}",
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[{"type": "text", "text": e.message}],
|
||||
model=e.model,
|
||||
stop_reason="end_turn",
|
||||
usage=_usage,
|
||||
)
|
||||
_anthropic_response: Final = blocked_anthropic_message(e)
|
||||
|
||||
if data.get("stream", None) is not None and data["stream"] is True:
|
||||
# For streaming, use the standard SSE data generator
|
||||
|
|
|
|||
|
|
@ -36,7 +36,11 @@ from typing_extensions import Never
|
|||
import litellm
|
||||
from litellm._logging import redact_internal_details_from_client_message, verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
|
||||
from litellm.anthropic_interface.exceptions import (
|
||||
AnthropicErrorSseFrame,
|
||||
anthropic_error_response,
|
||||
anthropic_error_sse_frame,
|
||||
)
|
||||
from litellm.constants import (
|
||||
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
|
||||
DEFAULT_MAX_RECURSE_DEPTH,
|
||||
|
|
@ -51,6 +55,7 @@ from litellm.constants import (
|
|||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
UNSAFE_PROXY_RESPONSE_HEADERS,
|
||||
)
|
||||
from litellm.exceptions import ModifyResponseException
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
|
|
@ -156,7 +161,20 @@ KNOWN_PROXY_ROUTES: Final = frozenset(
|
|||
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
|
||||
)
|
||||
|
||||
SseErrorDialect: TypeAlias = Literal["openai", "anthropic"]
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OpenAISseErrors:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AnthropicSseErrors:
|
||||
request_id: str | None
|
||||
call_id: Callable[[], str | None]
|
||||
|
||||
|
||||
SseErrorDialect: TypeAlias = OpenAISseErrors | AnthropicSseErrors
|
||||
_OPENAI_SSE_ERRORS: Final = OpenAISseErrors()
|
||||
ProxyRouteType: TypeAlias = Literal[
|
||||
"acompletion",
|
||||
"aembedding",
|
||||
|
|
@ -251,6 +269,8 @@ ProxyRouteType: TypeAlias = Literal[
|
|||
"adelete_run",
|
||||
]
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import blocked_response_usage
|
||||
|
||||
# Type alias for streaming chunk serializer (chunk after hooks + cost injection -> wire format)
|
||||
StreamChunkSerializer = Callable[[object], str]
|
||||
|
|
@ -279,6 +299,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.utils import (
|
||||
AnthropicMessagesResponse,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
|
|
@ -752,6 +773,9 @@ def resolve_counted_tool_fields(
|
|||
return resolved
|
||||
|
||||
|
||||
_PROVIDER_FIELDS: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
||||
|
||||
def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
|
||||
raw_detail: Final = attribute_of(exc, "detail", str(exc))
|
||||
message, structured_fields = serialize_http_exception_detail(raw_detail)
|
||||
|
|
@ -763,7 +787,7 @@ def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, s
|
|||
type=openai_error_type(exc, error_status),
|
||||
param=openai_error_param(exc),
|
||||
code=error_status,
|
||||
provider_specific_fields=merged_fields,
|
||||
provider_specific_fields=_PROVIDER_FIELDS.validate_python(merged_fields),
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
|
@ -785,7 +809,9 @@ def proxy_exception_from_route_error(exc: Exception) -> ProxyException:
|
|||
param=openai_error_param(exc),
|
||||
code=error_status,
|
||||
openai_code=carried_code if isinstance(carried_code, str) else None,
|
||||
provider_specific_fields=provider_fields if isinstance(provider_fields, dict) else None,
|
||||
provider_specific_fields=_PROVIDER_FIELDS.validate_python(
|
||||
provider_fields if isinstance(provider_fields, dict) else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -998,6 +1024,24 @@ async def _close_upstream_and_log_its_spend(upstream: object) -> bool:
|
|||
return wait_for_spend_flush is not None and await wait_for_spend_flush() is True
|
||||
|
||||
|
||||
class _FirstChunkFailedResponse(StreamingResponse):
|
||||
def __init__(
|
||||
self,
|
||||
content: AsyncGenerator[str, None],
|
||||
*,
|
||||
error: Exception,
|
||||
media_type: str,
|
||||
headers: Mapping[str, str],
|
||||
status_code: int,
|
||||
) -> None:
|
||||
super().__init__(content, status_code=status_code, headers=headers, media_type=media_type)
|
||||
self._error = error
|
||||
|
||||
@property
|
||||
def error(self) -> Exception:
|
||||
return self._error
|
||||
|
||||
|
||||
class _UpstreamClosingStreamingResponse(StreamingResponse):
|
||||
"""StreamingResponse that always closes its body iterator and the wrapped
|
||||
upstream generator.
|
||||
|
|
@ -1295,8 +1339,9 @@ async def create_response(
|
|||
for frame in _sse_error_frames(error_obj):
|
||||
yield frame
|
||||
|
||||
return StreamingResponse(
|
||||
return _FirstChunkFailedResponse(
|
||||
error_gen_message(),
|
||||
error=e,
|
||||
media_type=media_type,
|
||||
headers=_sse_stream_headers(await _resolve_stream_headers(headers, refresh_headers)),
|
||||
status_code=error_status,
|
||||
|
|
@ -1375,29 +1420,92 @@ async def _aclose_late_response(produced: Response) -> None:
|
|||
verbose_proxy_logger.debug("error closing relayed streaming generator: %s", exc)
|
||||
|
||||
|
||||
def blocked_anthropic_message(exc: ModifyResponseException) -> AnthropicMessagesResponse:
|
||||
return AnthropicMessagesResponse(
|
||||
id=f"msg_{uuid.uuid4()}",
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[{"type": "text", "text": exc.message}],
|
||||
model=exc.model,
|
||||
stop_reason="end_turn",
|
||||
usage=blocked_response_usage(exc.original_response),
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_error_frame(
|
||||
error_dialect: AnthropicSseErrors,
|
||||
status_code: int,
|
||||
raw_message: str,
|
||||
provider_specific_fields: Mapping[str, object] | None = None,
|
||||
) -> AnthropicErrorSseFrame:
|
||||
return AnthropicErrorSseFrame(
|
||||
status_code,
|
||||
anthropic_error_response(
|
||||
status_code=status_code,
|
||||
raw_message=raw_message,
|
||||
request_id=error_dialect.request_id,
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
call_id=error_dialect.call_id(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _late_failure_frames(exc: Exception, error_dialect: SseErrorDialect) -> tuple[str, ...]:
|
||||
match error_dialect:
|
||||
case "openai":
|
||||
case OpenAISseErrors():
|
||||
return _sse_error_frames(sse_error_payload(exc)[1])
|
||||
case "anthropic":
|
||||
case AnthropicSseErrors() if isinstance(exc, ModifyResponseException):
|
||||
return tuple(
|
||||
chunk.decode() for chunk in FakeAnthropicMessagesStreamIterator(blocked_anthropic_message(exc)).chunks
|
||||
)
|
||||
case AnthropicSseErrors():
|
||||
proxy_exc: Final = proxy_exception_from_route_error(exc)
|
||||
return (
|
||||
anthropic_error_sse_frame(
|
||||
_anthropic_error_frame(
|
||||
error_dialect,
|
||||
status_code=error_status_code(proxy_exc, status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
raw_message=proxy_exc.message,
|
||||
provider_specific_fields=proxy_exc.provider_specific_fields,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _late_body_frames(produced: Response, error_dialect: SseErrorDialect) -> tuple[bytes, ...]:
|
||||
body: Final = bytes(produced.body) or b"{}"
|
||||
if error_dialect == "anthropic" and produced.status_code >= status.HTTP_400_BAD_REQUEST:
|
||||
return (
|
||||
anthropic_error_sse_frame(
|
||||
status_code=produced.status_code, raw_message=body.decode(errors="replace")
|
||||
).encode(),
|
||||
)
|
||||
return b"data: " + body + b"\n\n", b"data: [DONE]\n\n"
|
||||
match error_dialect:
|
||||
case AnthropicSseErrors() if produced.status_code >= status.HTTP_400_BAD_REQUEST:
|
||||
return (
|
||||
_anthropic_error_frame(
|
||||
error_dialect, status_code=produced.status_code, raw_message=body.decode(errors="replace")
|
||||
).encode(),
|
||||
)
|
||||
case _:
|
||||
return b"data: " + body + b"\n\n", b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _chunk_bytes(chunk: str | bytes | memoryview) -> bytes:
|
||||
return chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk)
|
||||
|
||||
|
||||
async def _late_stream_error_frames(produced: StreamingResponse, error_dialect: AnthropicSseErrors) -> tuple[str, ...]:
|
||||
if isinstance(produced, _FirstChunkFailedResponse):
|
||||
return _late_failure_frames(produced.error, error_dialect)
|
||||
body: Final = b"".join([_chunk_bytes(chunk) async for chunk in produced.body_iterator])
|
||||
return (
|
||||
_anthropic_error_frame(
|
||||
error_dialect, status_code=produced.status_code, raw_message=body.decode(errors="replace")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _late_stream_chunks(produced: StreamingResponse, error_dialect: SseErrorDialect) -> AsyncIterator[bytes]:
|
||||
match error_dialect:
|
||||
case AnthropicSseErrors() if produced.status_code >= status.HTTP_400_BAD_REQUEST:
|
||||
for frame in await _late_stream_error_frames(produced, error_dialect):
|
||||
yield frame.encode()
|
||||
case _:
|
||||
async for chunk in produced.body_iterator:
|
||||
yield _chunk_bytes(chunk)
|
||||
|
||||
|
||||
async def _relay_late_response(produced: Response, error_dialect: SseErrorDialect) -> AsyncGenerator[bytes, None]:
|
||||
|
|
@ -1410,8 +1518,8 @@ async def _relay_late_response(produced: Response, error_dialect: SseErrorDialec
|
|||
return
|
||||
|
||||
try:
|
||||
async for chunk in produced.body_iterator:
|
||||
yield chunk.encode("utf-8") if isinstance(chunk, str) else bytes(chunk)
|
||||
async for chunk in _late_stream_chunks(produced, error_dialect):
|
||||
yield chunk
|
||||
finally:
|
||||
# Starlette never called this response, so the cleanup its __call__ would
|
||||
# have run has to happen here or the upstream LLM connection leaks.
|
||||
|
|
@ -1448,7 +1556,7 @@ async def open_sse_before_first_byte(
|
|||
ping_interval_seconds: float | str | None,
|
||||
media_type: str = "text/event-stream",
|
||||
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
|
||||
error_dialect: SseErrorDialect = "openai",
|
||||
error_dialect: SseErrorDialect = _OPENAI_SSE_ERRORS,
|
||||
) -> _LateResponseT | StreamingResponse:
|
||||
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
|
||||
|
||||
|
|
@ -2933,7 +3041,14 @@ class ProxyBaseLLMRequestProcessing:
|
|||
),
|
||||
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
|
||||
on_late_failure=_audit_late_failure,
|
||||
error_dialect="anthropic" if route_type == "anthropic_messages" else "openai",
|
||||
error_dialect=(
|
||||
AnthropicSseErrors(
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
call_id=lambda: error_body_call_id(general_settings, self.litellm_call_id),
|
||||
)
|
||||
if route_type == "anthropic_messages"
|
||||
else _OPENAI_SSE_ERRORS
|
||||
),
|
||||
)
|
||||
|
||||
async def _process_llm_request(
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ async def _keepalive_ping_stream(
|
|||
# A relayed chunk can be a raw transport read rather than a whole
|
||||
# SSE frame, so an upstream that stalls halfway through a frame
|
||||
# must not have a ping spliced into it.
|
||||
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
|
||||
if ends_sse_frame(recent_tail):
|
||||
yield ping_chunk
|
||||
continue
|
||||
try:
|
||||
|
|
@ -131,6 +131,10 @@ def wrap_passthrough_sse_bytes_with_keepalive_pings(
|
|||
return _keepalive_ping_stream(stream=stream, ping_interval_seconds=interval, ping_chunk=SSE_COMMENT_PING_BYTES)
|
||||
|
||||
|
||||
def ends_sse_frame(recent_tail: bytes) -> bool:
|
||||
return recent_tail.endswith(_SSE_FRAME_DELIMITERS)
|
||||
|
||||
|
||||
def advance_sse_tail(recent_tail: bytes, chunk: object) -> bytes:
|
||||
written: Final = _sse_tail_bytes(chunk)
|
||||
if not written:
|
||||
|
|
@ -147,7 +151,7 @@ def _sse_tail_bytes(chunk: object) -> bytes:
|
|||
|
||||
|
||||
def seal_open_sse_frame(recent_tail: bytes) -> str:
|
||||
if recent_tail.endswith(_SSE_FRAME_DELIMITERS):
|
||||
if ends_sse_frame(recent_tail):
|
||||
return ""
|
||||
line_break: Final = "" if recent_tail.endswith((b"\n", b"\r")) else "\n"
|
||||
return f"{line_break}{ANTHROPIC_PING_SSE_CHUNK}"
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ from litellm.proxy.auth.auth_utils import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import (
|
||||
AnthropicSseErrors,
|
||||
OpenAISseErrors,
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
log_llm_api_exception,
|
||||
open_sse_before_first_byte,
|
||||
|
|
@ -2338,6 +2340,11 @@ def create_pass_through_route(
|
|||
return await open_sse_before_first_byte(
|
||||
_relay(),
|
||||
ping_interval_seconds=(litellm.sse_keepalive_ping_interval_seconds if is_stream else None),
|
||||
error_dialect=(
|
||||
AnthropicSseErrors(request_id=request.headers.get("x-request-id"), call_id=lambda: None)
|
||||
if custom_llm_provider == "anthropic"
|
||||
else OpenAISseErrors()
|
||||
),
|
||||
)
|
||||
|
||||
setattr(endpoint_func, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, True)
|
||||
|
|
|
|||
|
|
@ -980,7 +980,10 @@ class TestAgenticStreamingIteratorHoldBack:
|
|||
async def test_should_emit_pings_while_the_follow_up_stream_is_slow(self):
|
||||
"""The corrected answer can be slow to generate, so the follow-up stream gets keepalives too."""
|
||||
chunks = _build_tool_use_stream()
|
||||
phase2_chunks = [b"follow-up-chunk-1", b"follow-up-chunk-2"]
|
||||
phase2_chunks = [
|
||||
_sse_event("message_start", {"type": "message_start", "message": {"id": "follow-up"}}),
|
||||
_sse_event("message_stop", {"type": "message_stop"}),
|
||||
]
|
||||
|
||||
mock_handler = MagicMock()
|
||||
mock_handler._call_agentic_completion_hooks = AsyncMock(
|
||||
|
|
@ -1001,6 +1004,28 @@ class TestAgenticStreamingIteratorHoldBack:
|
|||
assert collected[first_follow_up_index + 1] == STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_not_ping_inside_a_follow_up_event_that_stalls(self):
|
||||
"""A ping written into a half-sent event corrupts that event for the client."""
|
||||
event = _sse_event("content_block_delta", {"type": "content_block_delta", "delta": {"text": "hello"}})
|
||||
halves = [event[:20], event[20:]]
|
||||
|
||||
mock_handler = MagicMock()
|
||||
mock_handler._call_agentic_completion_hooks = AsyncMock(
|
||||
return_value=MockSlowAsyncStream(halves, delay_seconds=0.06)
|
||||
)
|
||||
|
||||
iterator = _build_hold_back_iterator(
|
||||
MockAsyncStream(_build_tool_use_stream()),
|
||||
mock_handler,
|
||||
ping_interval_seconds=0.02,
|
||||
)
|
||||
|
||||
collected = [chunk async for chunk in iterator]
|
||||
|
||||
first_half_index = collected.index(halves[0])
|
||||
assert collected[first_half_index + 1] == halves[1]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_propagate_follow_up_stream_error(self):
|
||||
"""A failing follow-up stream surfaces its error instead of hanging on pings forever."""
|
||||
|
|
|
|||
|
|
@ -74,7 +74,9 @@ class TestBlockedResponseUsage:
|
|||
"""Blocked responses report the blocked LLM response's real usage."""
|
||||
|
||||
def test_uses_original_response_usage(self):
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
blocked_response_usage as _blocked_response_usage,
|
||||
)
|
||||
|
||||
# original_response is the AnthropicMessagesResponse the LLM produced
|
||||
# before the guardrail blocked it; its usage is real.
|
||||
|
|
@ -85,7 +87,9 @@ class TestBlockedResponseUsage:
|
|||
}
|
||||
|
||||
def test_zero_usage_when_no_original_response(self):
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import _blocked_response_usage
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
blocked_response_usage as _blocked_response_usage,
|
||||
)
|
||||
|
||||
# Pre-call blocks never invoked the LLM -> nothing consumed.
|
||||
assert _blocked_response_usage(None) == {
|
||||
|
|
|
|||
|
|
@ -8031,6 +8031,76 @@ async def test_pass_through_route_pings_while_the_upstream_call_is_still_running
|
|||
assert collected[-1] in (MESSAGE_START_SSE_FRAME, MESSAGE_START_SSE_FRAME.decode())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("custom_llm_provider, expect_error_event", [("anthropic", True), (None, False)])
|
||||
async def test_pass_through_route_sends_a_late_upstream_rejection_in_the_providers_error_shape(
|
||||
custom_llm_provider, expect_error_event
|
||||
):
|
||||
"""Once a ping is out the upstream's 401 status never reaches the client, and an
|
||||
Anthropic client only sees an error that arrives as an `event: error`."""
|
||||
from fastapi import Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
module = "litellm.proxy.pass_through_endpoints.pass_through_endpoints"
|
||||
envelope = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}}
|
||||
payload = json.dumps(envelope).encode()
|
||||
|
||||
async def _rejection():
|
||||
yield payload
|
||||
|
||||
async def slow_rejection(**kwargs):
|
||||
await asyncio.sleep(0.25)
|
||||
return StreamingResponse(_rejection(), status_code=401, media_type="application/json")
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
patch(
|
||||
f"{module}.InitPassThroughEndpointHelpers.is_registered_pass_through_route",
|
||||
return_value=True,
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch(
|
||||
f"{module}.InitPassThroughEndpointHelpers.get_registered_pass_through_route",
|
||||
return_value=None,
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch(f"{module}.pass_through_request", slow_rejection))
|
||||
stack.enter_context(patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05))
|
||||
|
||||
endpoint_func = create_pass_through_route(
|
||||
endpoint="/anthropic/v1/messages",
|
||||
target="https://api.anthropic.com/v1/messages",
|
||||
custom_headers={},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
is_streaming_request=True,
|
||||
)
|
||||
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request.method = "POST"
|
||||
mock_request.url = httpx.URL("http://test-proxy.com/anthropic/v1/messages")
|
||||
mock_request.scope = {}
|
||||
mock_request.body = AsyncMock(return_value=b'{"model": "claude-3", "stream": true}')
|
||||
mock_request.headers = Headers({"content-type": "application/json"})
|
||||
mock_request.query_params = QueryParams({})
|
||||
mock_request.state = SimpleNamespace()
|
||||
|
||||
response = await endpoint_func(
|
||||
request=mock_request,
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=MagicMock(),
|
||||
)
|
||||
wire = b"".join([chunk async for chunk in response.body_iterator])
|
||||
|
||||
error_events = [
|
||||
json.loads(frame.split(b"data: ", 1)[1])
|
||||
for frame in wire.split(b"\n\n")
|
||||
if frame.startswith(b"event: error\n")
|
||||
]
|
||||
assert error_events == ([envelope] if expect_error_event else [])
|
||||
assert wire.endswith(payload) is not expect_error_event
|
||||
|
||||
|
||||
def test_passthrough_carries_the_per_model_budgets():
|
||||
"""
|
||||
Native passthrough builds its logging metadata from
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
|
|||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame, anthropic_error_sse_frame
|
||||
from litellm.exceptions import ModifyResponseException
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
DISABLE_ENV_VAR,
|
||||
|
|
@ -37,6 +38,7 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.integrations.opentelemetry import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.common_request_processing import (
|
||||
AnthropicSseErrors,
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
ProxyConfig,
|
||||
_await_llm_call_cancelling_on_disconnect,
|
||||
|
|
@ -10541,16 +10543,23 @@ async def test_ttft_keepalive_relays_a_late_non_streaming_body_as_an_sse_frame()
|
|||
assert collected[-1] == b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError:
|
||||
_ANTHROPIC_ERRORS: Final = AnthropicSseErrors(request_id=None, call_id=lambda: None)
|
||||
|
||||
|
||||
def _anthropic_sdk_over(wire: bytes) -> anthropic.Anthropic:
|
||||
def serve(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=wire)
|
||||
|
||||
client: Final = anthropic.Anthropic(
|
||||
return anthropic.Anthropic(
|
||||
api_key="sk-test",
|
||||
base_url="http://proxy.test",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(serve)),
|
||||
max_retries=0,
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_sdk_stream_error(wire: bytes) -> anthropic.APIStatusError:
|
||||
client: Final = _anthropic_sdk_over(wire)
|
||||
with pytest.raises(anthropic.APIStatusError) as raised:
|
||||
for _ in client.messages.create(
|
||||
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True
|
||||
|
|
@ -10580,7 +10589,7 @@ async def test_ttft_keepalive_raises_a_late_failure_in_the_anthropic_sdk(late_fa
|
|||
raise late_failure
|
||||
|
||||
response = await open_sse_before_first_byte(
|
||||
slow_failure(), ping_interval_seconds=0.05, error_dialect="anthropic"
|
||||
slow_failure(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
|
||||
)
|
||||
collected = await _drain(response)
|
||||
|
||||
|
|
@ -10602,13 +10611,111 @@ async def test_ttft_keepalive_raises_a_late_error_body_in_the_anthropic_sdk():
|
|||
await asyncio.sleep(0.2)
|
||||
return JSONResponse(status_code=529, content=envelope)
|
||||
|
||||
response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect="anthropic")
|
||||
response = await open_sse_before_first_byte(slow_json(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS)
|
||||
collected = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_raises_a_late_upstream_error_stream_in_the_anthropic_sdk():
|
||||
"""A pass-through relays the upstream's rejection as a stream under its own
|
||||
status, which the keepalive already replaced with a 200."""
|
||||
envelope = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}}
|
||||
payload = json.dumps(envelope).encode()
|
||||
|
||||
async def slow_rejection():
|
||||
await asyncio.sleep(0.2)
|
||||
return StreamingResponse(iter([payload[:10], payload[10:]]), status_code=401, media_type="application/json")
|
||||
|
||||
response = await open_sse_before_first_byte(
|
||||
slow_rejection(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
|
||||
)
|
||||
collected = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
assert _anthropic_sdk_stream_error(b"".join(collected)).body == envelope
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_raises_a_first_chunk_failure_in_the_anthropic_sdk():
|
||||
"""A guardrail that rejects the first streamed chunk makes create_response answer
|
||||
with an error stream, whose status the keepalive already replaced with a 200."""
|
||||
|
||||
async def rejected_stream():
|
||||
for chunk in ():
|
||||
yield chunk
|
||||
raise HTTPException(status_code=400, detail="blocked by guardrail")
|
||||
|
||||
async def slow_rejected_stream():
|
||||
await asyncio.sleep(0.2)
|
||||
return await create_response(rejected_stream(), media_type="text/event-stream", headers={})
|
||||
|
||||
response = await open_sse_before_first_byte(
|
||||
slow_rejected_stream(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
|
||||
)
|
||||
collected = await _drain(response)
|
||||
|
||||
assert collected[0] == TTFT_PING
|
||||
body = _anthropic_sdk_stream_error(b"".join(collected)).body
|
||||
assert isinstance(body, dict)
|
||||
assert body["error"] == {"type": "invalid_request_error", "message": "blocked by guardrail"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_late_anthropic_error_keeps_the_provider_fields():
|
||||
failure = ProxyException(
|
||||
message="quota exhausted",
|
||||
type="rate_limit_error",
|
||||
param=None,
|
||||
code=429,
|
||||
provider_specific_fields={"retry_after_seconds": 7},
|
||||
)
|
||||
|
||||
async def slow_failure():
|
||||
await asyncio.sleep(0.2)
|
||||
raise failure
|
||||
|
||||
response = await open_sse_before_first_byte(
|
||||
slow_failure(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
|
||||
)
|
||||
body = _anthropic_sdk_stream_error(b"".join(await _drain(response))).body
|
||||
|
||||
assert isinstance(body, dict)
|
||||
assert body["error"]["provider_specific_fields"] == {"retry_after_seconds": 7}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_streams_a_late_guardrail_block_as_the_assistant_message():
|
||||
"""/v1/messages answers a guardrail block with a 200 message carrying the
|
||||
violation text, so the keepalive path has to stream that message too."""
|
||||
block = ModifyResponseException(
|
||||
message="Blocked by policy",
|
||||
model="claude-sonnet-4-5",
|
||||
request_data={},
|
||||
original_response={"usage": {"input_tokens": 31, "output_tokens": 9}},
|
||||
)
|
||||
|
||||
async def slow_block():
|
||||
await asyncio.sleep(0.2)
|
||||
raise block
|
||||
|
||||
response = await open_sse_before_first_byte(
|
||||
slow_block(), ping_interval_seconds=0.05, error_dialect=_ANTHROPIC_ERRORS
|
||||
)
|
||||
wire = b"".join(await _drain(response))
|
||||
|
||||
with _anthropic_sdk_over(wire).messages.stream(
|
||||
model="claude-sonnet-4-5", max_tokens=16, messages=[{"role": "user", "content": "hi"}]
|
||||
) as stream:
|
||||
message = stream.get_final_message()
|
||||
|
||||
assert [block.text for block in message.content if block.type == "text"] == ["Blocked by policy"]
|
||||
assert message.stop_reason == "end_turn"
|
||||
assert message.usage.output_tokens == 9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ttft_keepalive_closes_the_upstream_stream_it_relayed():
|
||||
"""Starlette never calls the produced response, so its own cleanup never runs
|
||||
|
|
@ -10897,17 +11004,19 @@ async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_an
|
|||
|
||||
proxy_logging_obj = MagicMock(spec=ProxyLogging)
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
processor = ProxyBaseLLMRequestProcessing(data={"model": "claude-sonnet-4-5", "stream": True})
|
||||
processor = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": "claude-sonnet-4-5", "stream": True, "litellm_call_id": "call-1"}
|
||||
)
|
||||
|
||||
with patch.object(litellm, "sse_keepalive_ping_interval_seconds", 0.05):
|
||||
with patch.object(ProxyBaseLLMRequestProcessing, "_process_llm_request", slow_failure):
|
||||
response = await processor.base_process_llm_request(
|
||||
request=MagicMock(spec=Request),
|
||||
request=MagicMock(spec=Request, headers={"x-request-id": "req_1"}),
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=MagicMock(spec=UserAPIKeyAuth),
|
||||
route_type="anthropic_messages",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
general_settings={},
|
||||
general_settings={"include_call_id_in_error_body": True},
|
||||
proxy_config=MagicMock(spec=ProxyConfig),
|
||||
)
|
||||
collected = await _drain(response)
|
||||
|
|
@ -10916,6 +11025,8 @@ async def test_base_process_llm_request_raises_a_late_messages_failure_in_the_an
|
|||
body = _anthropic_sdk_stream_error(b"".join(collected)).body
|
||||
assert isinstance(body, dict)
|
||||
assert body["error"]["message"] == "upstream exploded"
|
||||
assert body["error"]["litellm_call_id"] == "call-1"
|
||||
assert body["request_id"] == "req_1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue