From ae4e3eeff1f4c2e94e6b98ddf133cd7afbb4f04a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:29:46 -0700 Subject: [PATCH] fix(proxy): answer the Anthropic envelope for 404, 405 and unhandled 500 errors on Messages routes A disabled Claude Code gateway 404, a wrong-method 405 and an unhandled 500 on the Messages and /anthropic/ routes kept FastAPI's detail body or the OpenAI shape. The central HTTP and unhandled-exception handlers now pick the Anthropic envelope by route. The dashboard playground toast titles Anthropic error types when the body has no code. --- litellm/proxy/common_request_processing.py | 3 +- .../common_utils/anthropic_error_payload.py | 35 ++---- litellm/proxy/proxy_server.py | 8 ++ .../anthropic/test_anthropic_gateway_wire.py | 7 +- .../proxy_server/test_exception_handlers.py | 112 ++++++++++++------ ui/litellm-dashboard/src/lib/toast.test.ts | 19 +++ ui/litellm-dashboard/src/lib/toast.ts | 14 ++- 7 files changed, 131 insertions(+), 67 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a36a7f2dc..67768e00f08 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -29,6 +29,7 @@ import orjson from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, Response, StreamingResponse from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.types import Receive, Scope, Send from typing_extensions import Never @@ -748,7 +749,7 @@ def resolve_counted_tool_fields( return resolved -def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException: +def proxy_exception_from_http_exception(exc: StarletteHTTPException, headers: dict[str, str]) -> ProxyException: raw_detail: Final = attribute_of(exc, "detail", str(exc)) message, structured_fields = serialize_http_exception_detail(raw_detail) existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {} diff --git a/litellm/proxy/common_utils/anthropic_error_payload.py b/litellm/proxy/common_utils/anthropic_error_payload.py index 3f1924f78d9..de8c460ad43 100644 --- a/litellm/proxy/common_utils/anthropic_error_payload.py +++ b/litellm/proxy/common_utils/anthropic_error_payload.py @@ -1,10 +1,5 @@ -"""Renders the Anthropic error envelope for the Anthropic routes, wherever the failure is raised. - -The central ``ProxyException`` handler answers every route with OpenAI's ``{"error": {...}}`` -body, which an Anthropic SDK cannot read. Auth, key, budget and model-access failures are -raised by ``user_api_key_auth`` before the route handler runs, so the envelope is picked -from the request route rather than inside the handler. -""" +"""Auth, budget and model-access failures are raised by ``user_api_key_auth`` before the Messages +handler runs, so the central handlers pick the Anthropic envelope from the request route.""" from collections.abc import Mapping from typing import Final @@ -33,21 +28,10 @@ ANTHROPIC_PASS_THROUGH_PREFIX: Final = "/anthropic/" def uses_anthropic_error_envelope(route: str) -> bool: - """True for the Messages routes and the Anthropic pass-through, whose callers read the Anthropic envelope. - - Narrower than ``LiteLLMRoutes.anthropic_routes``: the Skills and marketplace routes there - answer their own body shape. The pass-through relays upstream errors as Anthropic sent them, - so the proxy's own refusals there answer the same envelope. - """ return route in ANTHROPIC_MESSAGES_ROUTES or route.startswith(ANTHROPIC_PASS_THROUGH_PREFIX) def anthropic_error_status_code(exc: ProxyException) -> int: - """The exception's stringified ``code`` as a status, else 500. - - ``ProxyException.__init__`` stringifies ``code``, so a code-less exception holds the - literal ``"None"``, which must not be parsed as a status. - """ code: Final = exc.code if code.isdigit(): return int(code) @@ -79,7 +63,6 @@ def anthropic_error_content( request_id: str | None, general_settings: Mapping[str, object], ) -> AnthropicErrorResponse: - """The Anthropic ``{"type": "error", "error": {...}}`` body for ``exc``, typed by status.""" envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error( status_code=status_code, raw_message=exc.message, @@ -94,11 +77,6 @@ def anthropic_error_content( def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: - """The Anthropic error response for a Messages request, span close included. - - Shared by the Messages handlers and the central handler, so a failure raised before the - handler and one raised inside it answer identically. - """ from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; a JSONResponse returned instead of a raised error must stamp the OTel server span like the global ProxyException handler does general_settings_view, @@ -110,3 +88,12 @@ def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSON exc, status_code, request.headers.get("x-request-id"), general_settings_view() ) return JSONResponse(status_code=status_code, content=content, headers=exc.headers) + + +def anthropic_internal_server_error_response(request: Request) -> JSONResponse: + content: Final = AnthropicExceptionMapping.transform_to_anthropic_error( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + raw_message="Internal server error", + request_id=request.headers.get("x-request-id"), + ) + return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content=content) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 38d1653f6ed..ada7e46da1b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -462,6 +462,7 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l is_azure_model_router_request, log_llm_api_exception, open_sse_before_first_byte, + proxy_exception_from_http_exception, proxy_exception_from_route_error, request_litellm_call_id, resolve_litellm_call_id, @@ -471,6 +472,7 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l ) from litellm.proxy.common_utils.anthropic_error_payload import ( anthropic_error_json_response, + anthropic_internal_server_error_response, uses_anthropic_error_envelope, ) from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( @@ -2229,6 +2231,10 @@ async def otlp_http_exception_handler(connection: Request | WebSocket, exc: Star if response is not None: _close_dangling_otel_server_span(connection, exc.status_code, exc=exc) return response + if uses_anthropic_error_envelope(get_request_route(connection)): + return anthropic_error_json_response( + proxy_exception_from_http_exception(exc, dict(exc.headers or {})), connection + ) return await http_exception_handler(connection, exc) @@ -2436,6 +2442,8 @@ async def _unhandled_exception_response(request: Request, exc: Exception) -> Res otlp_response: Final = tracing_endpoints.otlp_error_response(request, 500) if otlp_response is not None: return otlp_response + if uses_anthropic_error_envelope(get_request_route(request)): + return anthropic_internal_server_error_response(request) return JSONResponse( status_code=500, content={ diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py index 0f0f1e7676c..e104fad4b3f 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py @@ -97,9 +97,10 @@ def test_shared_gateway_route_is_disabled_without_reaching_provider(gateway: Gat headers=_cli_bearer_headers(gateway.key), ) assert response.status_code == 404, response.text - assert _JSON_OBJECT.validate_json(response.content) == {"detail": "Claude Code gateway is not enabled"}, ( - response.text - ) + assert _JSON_OBJECT.validate_json(response.content) == { + "type": "error", + "error": {"type": "not_found_error", "message": "Claude Code gateway is not enabled"}, + }, response.text assert wire.drain() == () diff --git a/tests/unit/proxy/proxy_server/test_exception_handlers.py b/tests/unit/proxy/proxy_server/test_exception_handlers.py index 7ced5b24d96..7676f82bd74 100644 --- a/tests/unit/proxy/proxy_server/test_exception_handlers.py +++ b/tests/unit/proxy/proxy_server/test_exception_handlers.py @@ -173,8 +173,7 @@ async def test_openai_exception_handler_never_fabricates_a_call_id(monkeypatch): # --------------------------------------------------------------------------- -def _model_access_denied_exception(): - """The 403 can_team_access_model raises: OpenAI type team_model_access_denied, param model.""" +def _model_access_denied_exception() -> ProxyException: return ProxyException( message="Team not allowed to access model. Tried to access gpt-5.6", type="team_model_access_denied", @@ -198,11 +197,11 @@ _ANTHROPIC_PASS_THROUGH_ROUTES: Final = ( @pytest.mark.asyncio @pytest.mark.parametrize("path", (*_MESSAGES_ROUTES, *_ANTHROPIC_PASS_THROUGH_ROUTES)) -async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_messages_routes(path): - """An auth failure raised before the Messages handler (user_api_key_auth) must reach an - Anthropic SDK as {"type": "error", "error": {...}}, not the OpenAI {"error": {...}} body.""" - response = await openai_exception_handler(request=_make_request(path=path), exc=_model_access_denied_exception()) - body = json.loads(response.body) +async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_messages_routes(path: str) -> None: + response: Final = await openai_exception_handler( + request=_make_request(path=path), exc=_model_access_denied_exception() + ) + body: Final = json.loads(response.body) assert response.status_code == 403 assert body == { @@ -224,25 +223,25 @@ async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_me (429, "rate_limit_error"), ], ) -async def test_openai_exception_handler_maps_messages_failures_to_anthropic_error_types(status_code, anthropic_type): - """The Anthropic error type follows the HTTP status, so a bad key, a denied model, an - unknown model and a rate limit each read the way an Anthropic client expects.""" - exc = ProxyException(message="denied", type="key_model_access_denied", param="model", code=status_code) +async def test_openai_exception_handler_maps_messages_failures_to_anthropic_error_types( + status_code: int, anthropic_type: str +) -> None: + exc: Final = ProxyException(message="denied", type="key_model_access_denied", param="model", code=status_code) - response = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc) + response: Final = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc) assert response.status_code == status_code assert json.loads(response.body)["error"]["type"] == anthropic_type @pytest.mark.asyncio -async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_messages(): - """A budget failure carries the proxy's own budget status; the Messages route must keep - it and answer the Anthropic envelope rather than the OpenAI body.""" - exc = ProxyException(message="Budget exceeded", type="budget_exceeded", param=None, code=422) +async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_messages() -> None: + exc: Final = ProxyException(message="Budget exceeded", type="budget_exceeded", param=None, code=422) - response = await openai_exception_handler(request=_make_request(path="/v1/messages/count_tokens"), exc=exc) - body = json.loads(response.body) + response: Final = await openai_exception_handler( + request=_make_request(path="/v1/messages/count_tokens"), exc=exc + ) + body: Final = json.loads(response.body) assert response.status_code == 422 assert body["type"] == "error" @@ -252,12 +251,11 @@ async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_ @pytest.mark.asyncio -async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messages_routes(): - """The envelope is chosen by route, so OpenAI routes are untouched.""" - response = await openai_exception_handler( +async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messages_routes() -> None: + response: Final = await openai_exception_handler( request=_make_request(path="/v1/chat/completions"), exc=_model_access_denied_exception() ) - body = json.loads(response.body) + body: Final = json.loads(response.body) assert response.status_code == 403 assert body == { @@ -271,12 +269,11 @@ async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messag @pytest.mark.asyncio -async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthropic_routes(): - """The Skills and marketplace routes are not Messages routes and answer their own body.""" - response = await openai_exception_handler( +async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthropic_routes() -> None: + response: Final = await openai_exception_handler( request=_make_request(path="/v1/skills"), exc=_model_access_denied_exception() ) - body = json.loads(response.body) + body: Final = json.loads(response.body) assert response.status_code == 403 assert "type" not in body @@ -284,11 +281,11 @@ async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthr @pytest.mark.asyncio -async def test_openai_exception_handler_messages_route_copies_the_call_id_when_opted_in(monkeypatch): - """The Messages envelope carries error.litellm_call_id under the same opt-in as the - OpenAI envelope, so an Anthropic SDK's e.body still names the request to look up.""" +async def test_openai_exception_handler_messages_route_copies_the_call_id_when_opted_in( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) - exc = ProxyException( + exc: Final = ProxyException( message="denied", type="team_model_access_denied", param="model", @@ -297,8 +294,8 @@ async def test_openai_exception_handler_messages_route_copies_the_call_id_when_o provider_specific_fields={"guardrail": "keyword-block"}, ) - response = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc) - body = json.loads(response.body) + response: Final = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc) + body: Final = json.loads(response.body) assert response.headers["x-litellm-call-id"] == "call-8302" assert body["error"] == { @@ -311,15 +308,14 @@ async def test_openai_exception_handler_messages_route_copies_the_call_id_when_o @pytest.mark.asyncio @pytest.mark.parametrize("path", ("/v1/messages", "/anthropic/v1/messages")) -async def test_a_messages_request_with_no_key_answers_the_anthropic_authentication_error(client, monkeypatch, path): - """End to end through the real user_api_key_auth: a /v1/messages request with no - credential is rejected before the handler, and the SDK must see Anthropic's - authentication_error envelope, not OpenAI's {"error": {...}} body.""" +async def test_a_messages_request_with_no_key_answers_the_anthropic_authentication_error( + client: TestClient, monkeypatch: pytest.MonkeyPatch, path: str +) -> None: monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key") monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) - response = client.post(path, json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}) - body = response.json() + response: Final = client.post(path, json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}) + body: Final = response.json() assert response.status_code == 401 assert body["type"] == "error" @@ -626,6 +622,22 @@ async def test_otel_unhandled_exception_handler_answers_a_db_outage_with_503_no_ assert json.loads(response.body) == _DB_OUTAGE_503_BODY +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/v1/messages", "/claude_code_gateway/v1/messages", "/anthropic/v1/messages")) +async def test_otel_unhandled_exception_handler_answers_the_anthropic_api_error_on_the_messages_routes( + path: str, +) -> None: + response: Final = await otel_unhandled_exception_handler( + request=_make_request(path=path), exc=RuntimeError("kaboom") + ) + + assert response.status_code == 500 + assert json.loads(response.body) == { + "type": "error", + "error": {"type": "api_error", "message": "Internal server error"}, + } + + @pytest.mark.asyncio async def test_otel_unhandled_exception_handler_reraises_proxy_exception_error(): """ProxyException / HTTPException / RequestValidationError are re-raised @@ -705,6 +717,30 @@ async def test_http_exception_on_a_plain_http_request_keeps_the_default_json_bod assert json.loads(bytes(response.body)) == {"detail": "not found"} +@pytest.mark.asyncio +async def test_a_disabled_claude_code_gateway_answers_the_anthropic_not_found_error() -> None: + request: Final = _make_request(path="/claude_code_gateway/v1/messages") + + response: Final = await otlp_http_exception_handler( + request, HTTPException(404, "Claude Code gateway is not enabled") + ) + + assert response is not None + assert response.status_code == 404 + assert json.loads(bytes(response.body)) == { + "type": "error", + "error": {"type": "not_found_error", "message": "Claude Code gateway is not enabled"}, + } + + +def test_a_get_on_messages_answers_the_anthropic_envelope_and_keeps_the_allow_header(client: TestClient) -> None: + response: Final = client.get("/v1/messages") + + assert response.status_code == 405 + assert response.headers["allow"] == "POST" + assert response.json() == {"type": "error", "error": {"type": "api_error", "message": "Method Not Allowed"}} + + @pytest.mark.asyncio async def test_http_exception_on_a_websocket_closed_before_accept_sends_nothing_more() -> None: sent: Final[list[Message]] = [] diff --git a/ui/litellm-dashboard/src/lib/toast.test.ts b/ui/litellm-dashboard/src/lib/toast.test.ts index b4a776f123a..e927c3e06ed 100644 --- a/ui/litellm-dashboard/src/lib/toast.test.ts +++ b/ui/litellm-dashboard/src/lib/toast.test.ts @@ -108,6 +108,25 @@ describe("toast", () => { }); }); + it.each([ + [403, "permission_error", sonner.error, "Access Denied"], + [401, "authentication_error", sonner.error, "Authentication Error"], + [429, "rate_limit_error", sonner.warning, "Rate Limit Exceeded"], + [422, "invalid_request_error", sonner.error, "Request Error"], + ] as const)( + "titles an Anthropic SDK %s error on the Messages playground by its %s type", + (status, type, fn, title) => { + const body = JSON.stringify({ type: "error", error: { type, message: "Team not allowed to access model" } }); + toast.fromError( + `Error occurred while generating model response. Please try again. Error: Error: ${status} ${body}`, + ); + expect(fn).toHaveBeenCalledWith(title, { + description: `Error occurred while generating model response. Please try again. Error: Error: ${status} Team not allowed to access model`, + duration: 6000, + }); + }, + ); + it("leaves a string alone when its braces are not a JSON envelope", () => { toast.fromError("Template {name} is invalid"); expect(sonner.error).toHaveBeenCalledWith("Error", { diff --git a/ui/litellm-dashboard/src/lib/toast.ts b/ui/litellm-dashboard/src/lib/toast.ts index a7a4ce45ad0..465ac7a1f6c 100644 --- a/ui/litellm-dashboard/src/lib/toast.ts +++ b/ui/litellm-dashboard/src/lib/toast.ts @@ -46,6 +46,17 @@ const STATUS_TITLES: Readonly> = { 503: "Service Unavailable", }; +const ANTHROPIC_TYPE_TITLES: Readonly> = { + invalid_request_error: "Request Error", + authentication_error: "Authentication Error", + permission_error: "Access Denied", + not_found_error: "Not Found", + request_too_large: "Request Error", + rate_limit_error: "Rate Limit Exceeded", + api_error: "Server Error", + overloaded_error: "Service Unavailable", +}; + const WARNING_TITLES: ReadonlySet = new Set(["Budget Exceeded", "Rate Limit Exceeded"]); const asRecord = (value: unknown): Record | undefined => @@ -123,7 +134,8 @@ const titleFor = ({ status, proxyType }: ErrorFacts): string => { if (proxyType?.endsWith("_access_denied")) return "Access Denied"; const byType = proxyType === undefined ? undefined : PROXY_TYPE_TITLES[proxyType]; if (byType !== undefined) return byType; - return status === undefined ? "Error" : titleForStatus(status); + if (status !== undefined) return titleForStatus(status); + return (proxyType === undefined ? undefined : ANTHROPIC_TYPE_TITLES[proxyType]) ?? "Error"; }; const show = (kind: ToastKind, message: ReactNode, options?: ToastOptions): void => {