diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 48b035e6ce2..31e69b8a275 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1297,7 +1297,7 @@ _ANTHROPIC_LATE_ERROR_BODY_LIMIT: Final = 64 * 1024 async def _read_late_error_body(response: Response) -> bytes: """Read at most one page of an error body. A read that fails yields what it got.""" - read: bytes = b"" + read: bytes = b"" # rebind-ok: accumulates streamed chunks, then takes the buffered body try: if isinstance(response, StreamingResponse): async for chunk in response.body_iterator: @@ -1305,20 +1305,20 @@ async def _read_late_error_body(response: Response) -> bytes: if len(read) >= _ANTHROPIC_LATE_ERROR_BODY_LIMIT: break else: - read = bytes(response.body or b"") + read = bytes(response.body or b"") # rebind-ok: the buffered body arrives whole, nothing to accumulate except Exception: # noqa: BLE001 # the stream is already broken; the caller still needs a frame verbose_proxy_logger.exception("could not read the body of an upstream late failure") return read[:_ANTHROPIC_LATE_ERROR_BODY_LIMIT] class _LateErrorDetail(BaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", frozen=True) message: str | None = None class _LateErrorEnvelope(BaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", frozen=True) error: _LateErrorDetail | None = None diff --git a/tests/unit/proxy/test_passthrough_late_error_frames.py b/tests/unit/proxy/test_passthrough_late_error_frames.py index 4c49ffcb0be..e39ae88e9f7 100644 --- a/tests/unit/proxy/test_passthrough_late_error_frames.py +++ b/tests/unit/proxy/test_passthrough_late_error_frames.py @@ -11,7 +11,7 @@ import json from collections.abc import AsyncIterator import pytest -from starlette.responses import StreamingResponse +from starlette.responses import Response, StreamingResponse from litellm.proxy._types import ProxyException from litellm.proxy.common_request_processing import open_sse_before_first_byte @@ -36,8 +36,8 @@ async def _drain(response: StreamingResponse) -> str: return b"".join(chunks).decode("utf-8") -async def _relay(produced: StreamingResponse, *, answers_as_anthropic: bool = False) -> str: - async def deliver() -> StreamingResponse: +async def _relay(produced: Response, *, answers_as_anthropic: bool = False) -> str: + async def deliver() -> Response: await asyncio.sleep(0.05) return produced @@ -126,6 +126,20 @@ async def test_a_large_error_body_is_read_with_a_bound(): assert len(body) < 128 * 1024 +@pytest.mark.asyncio +async def test_an_error_body_that_arrived_whole_still_reaches_the_client_as_an_event(): + produced = Response( + content=json.dumps({"error": {"message": "buffered upstream said no", "type": "server_error"}}), + status_code=502, + media_type="application/json", + ) + + body = await _relay(produced, answers_as_anthropic=True) + + assert body.count("event: error") == 1 + assert "buffered upstream said no" in body + + FAKE_KEY = "sk-test-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" INTERNAL_PATH = "/home/runner/litellm/proxy/internal_state.py"