mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-10 08:28:25 +00:00
fix(proxy): answer the Anthropic envelope for 404, 405 and unhandled 500 errors on Messages routes
A disabled Claude Code gateway 404, a wrong-method 405 and an unhandled 500 on the Messages and /anthropic/ routes kept FastAPI's detail body or the OpenAI shape. The central HTTP and unhandled-exception handlers now pick the Anthropic envelope by route. The dashboard playground toast titles Anthropic error types when the body has no code.
This commit is contained in:
mateo-berri
parent
8208298f69
commit
ae4e3eeff1
7 changed files with 131 additions and 67 deletions
|
|
@ -29,6 +29,7 @@ import orjson
|
|||
from fastapi import HTTPException, Request, status
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from starlette.types import Receive, Scope, Send
|
||||
from typing_extensions import Never
|
||||
|
||||
|
|
@ -748,7 +749,7 @@ def resolve_counted_tool_fields(
|
|||
return resolved
|
||||
|
||||
|
||||
def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
|
||||
def proxy_exception_from_http_exception(exc: StarletteHTTPException, headers: dict[str, str]) -> ProxyException:
|
||||
raw_detail: Final = attribute_of(exc, "detail", str(exc))
|
||||
message, structured_fields = serialize_http_exception_detail(raw_detail)
|
||||
existing_fields: Final = getattr(exc, "provider_specific_fields", None) or {}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,5 @@
|
|||
"""Renders the Anthropic error envelope for the Anthropic routes, wherever the failure is raised.
|
||||
|
||||
The central ``ProxyException`` handler answers every route with OpenAI's ``{"error": {...}}``
|
||||
body, which an Anthropic SDK cannot read. Auth, key, budget and model-access failures are
|
||||
raised by ``user_api_key_auth`` before the route handler runs, so the envelope is picked
|
||||
from the request route rather than inside the handler.
|
||||
"""
|
||||
"""Auth, budget and model-access failures are raised by ``user_api_key_auth`` before the Messages
|
||||
handler runs, so the central handlers pick the Anthropic envelope from the request route."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
|
@ -33,21 +28,10 @@ ANTHROPIC_PASS_THROUGH_PREFIX: Final = "/anthropic/"
|
|||
|
||||
|
||||
def uses_anthropic_error_envelope(route: str) -> bool:
|
||||
"""True for the Messages routes and the Anthropic pass-through, whose callers read the Anthropic envelope.
|
||||
|
||||
Narrower than ``LiteLLMRoutes.anthropic_routes``: the Skills and marketplace routes there
|
||||
answer their own body shape. The pass-through relays upstream errors as Anthropic sent them,
|
||||
so the proxy's own refusals there answer the same envelope.
|
||||
"""
|
||||
return route in ANTHROPIC_MESSAGES_ROUTES or route.startswith(ANTHROPIC_PASS_THROUGH_PREFIX)
|
||||
|
||||
|
||||
def anthropic_error_status_code(exc: ProxyException) -> int:
|
||||
"""The exception's stringified ``code`` as a status, else 500.
|
||||
|
||||
``ProxyException.__init__`` stringifies ``code``, so a code-less exception holds the
|
||||
literal ``"None"``, which must not be parsed as a status.
|
||||
"""
|
||||
code: Final = exc.code
|
||||
if code.isdigit():
|
||||
return int(code)
|
||||
|
|
@ -79,7 +63,6 @@ def anthropic_error_content(
|
|||
request_id: str | None,
|
||||
general_settings: Mapping[str, object],
|
||||
) -> AnthropicErrorResponse:
|
||||
"""The Anthropic ``{"type": "error", "error": {...}}`` body for ``exc``, typed by status."""
|
||||
envelope: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
status_code=status_code,
|
||||
raw_message=exc.message,
|
||||
|
|
@ -94,11 +77,6 @@ def anthropic_error_content(
|
|||
|
||||
|
||||
def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
|
||||
"""The Anthropic error response for a Messages request, span close included.
|
||||
|
||||
Shared by the Messages handlers and the central handler, so a failure raised before the
|
||||
handler and one raised inside it answer identically.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; a JSONResponse returned instead of a raised error must stamp the OTel server span like the global ProxyException handler does
|
||||
general_settings_view,
|
||||
|
|
@ -110,3 +88,12 @@ def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSON
|
|||
exc, status_code, request.headers.get("x-request-id"), general_settings_view()
|
||||
)
|
||||
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
|
||||
|
||||
|
||||
def anthropic_internal_server_error_response(request: Request) -> JSONResponse:
|
||||
content: Final = AnthropicExceptionMapping.transform_to_anthropic_error(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
raw_message="Internal server error",
|
||||
request_id=request.headers.get("x-request-id"),
|
||||
)
|
||||
return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content=content)
|
||||
|
|
|
|||
|
|
@ -462,6 +462,7 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l
|
|||
is_azure_model_router_request,
|
||||
log_llm_api_exception,
|
||||
open_sse_before_first_byte,
|
||||
proxy_exception_from_http_exception,
|
||||
proxy_exception_from_route_error,
|
||||
request_litellm_call_id,
|
||||
resolve_litellm_call_id,
|
||||
|
|
@ -471,6 +472,7 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l
|
|||
)
|
||||
from litellm.proxy.common_utils.anthropic_error_payload import (
|
||||
anthropic_error_json_response,
|
||||
anthropic_internal_server_error_response,
|
||||
uses_anthropic_error_envelope,
|
||||
)
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
||||
|
|
@ -2229,6 +2231,10 @@ async def otlp_http_exception_handler(connection: Request | WebSocket, exc: Star
|
|||
if response is not None:
|
||||
_close_dangling_otel_server_span(connection, exc.status_code, exc=exc)
|
||||
return response
|
||||
if uses_anthropic_error_envelope(get_request_route(connection)):
|
||||
return anthropic_error_json_response(
|
||||
proxy_exception_from_http_exception(exc, dict(exc.headers or {})), connection
|
||||
)
|
||||
return await http_exception_handler(connection, exc)
|
||||
|
||||
|
||||
|
|
@ -2436,6 +2442,8 @@ async def _unhandled_exception_response(request: Request, exc: Exception) -> Res
|
|||
otlp_response: Final = tracing_endpoints.otlp_error_response(request, 500)
|
||||
if otlp_response is not None:
|
||||
return otlp_response
|
||||
if uses_anthropic_error_envelope(get_request_route(request)):
|
||||
return anthropic_internal_server_error_response(request)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
|
|
|
|||
|
|
@ -97,9 +97,10 @@ def test_shared_gateway_route_is_disabled_without_reaching_provider(gateway: Gat
|
|||
headers=_cli_bearer_headers(gateway.key),
|
||||
)
|
||||
assert response.status_code == 404, response.text
|
||||
assert _JSON_OBJECT.validate_json(response.content) == {"detail": "Claude Code gateway is not enabled"}, (
|
||||
response.text
|
||||
)
|
||||
assert _JSON_OBJECT.validate_json(response.content) == {
|
||||
"type": "error",
|
||||
"error": {"type": "not_found_error", "message": "Claude Code gateway is not enabled"},
|
||||
}, response.text
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -173,8 +173,7 @@ async def test_openai_exception_handler_never_fabricates_a_call_id(monkeypatch):
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _model_access_denied_exception():
|
||||
"""The 403 can_team_access_model raises: OpenAI type team_model_access_denied, param model."""
|
||||
def _model_access_denied_exception() -> ProxyException:
|
||||
return ProxyException(
|
||||
message="Team not allowed to access model. Tried to access gpt-5.6",
|
||||
type="team_model_access_denied",
|
||||
|
|
@ -198,11 +197,11 @@ _ANTHROPIC_PASS_THROUGH_ROUTES: Final = (
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", (*_MESSAGES_ROUTES, *_ANTHROPIC_PASS_THROUGH_ROUTES))
|
||||
async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_messages_routes(path):
|
||||
"""An auth failure raised before the Messages handler (user_api_key_auth) must reach an
|
||||
Anthropic SDK as {"type": "error", "error": {...}}, not the OpenAI {"error": {...}} body."""
|
||||
response = await openai_exception_handler(request=_make_request(path=path), exc=_model_access_denied_exception())
|
||||
body = json.loads(response.body)
|
||||
async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_messages_routes(path: str) -> None:
|
||||
response: Final = await openai_exception_handler(
|
||||
request=_make_request(path=path), exc=_model_access_denied_exception()
|
||||
)
|
||||
body: Final = json.loads(response.body)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert body == {
|
||||
|
|
@ -224,25 +223,25 @@ async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_me
|
|||
(429, "rate_limit_error"),
|
||||
],
|
||||
)
|
||||
async def test_openai_exception_handler_maps_messages_failures_to_anthropic_error_types(status_code, anthropic_type):
|
||||
"""The Anthropic error type follows the HTTP status, so a bad key, a denied model, an
|
||||
unknown model and a rate limit each read the way an Anthropic client expects."""
|
||||
exc = ProxyException(message="denied", type="key_model_access_denied", param="model", code=status_code)
|
||||
async def test_openai_exception_handler_maps_messages_failures_to_anthropic_error_types(
|
||||
status_code: int, anthropic_type: str
|
||||
) -> None:
|
||||
exc: Final = ProxyException(message="denied", type="key_model_access_denied", param="model", code=status_code)
|
||||
|
||||
response = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc)
|
||||
response: Final = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc)
|
||||
|
||||
assert response.status_code == status_code
|
||||
assert json.loads(response.body)["error"]["type"] == anthropic_type
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_messages():
|
||||
"""A budget failure carries the proxy's own budget status; the Messages route must keep
|
||||
it and answer the Anthropic envelope rather than the OpenAI body."""
|
||||
exc = ProxyException(message="Budget exceeded", type="budget_exceeded", param=None, code=422)
|
||||
async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_messages() -> None:
|
||||
exc: Final = ProxyException(message="Budget exceeded", type="budget_exceeded", param=None, code=422)
|
||||
|
||||
response = await openai_exception_handler(request=_make_request(path="/v1/messages/count_tokens"), exc=exc)
|
||||
body = json.loads(response.body)
|
||||
response: Final = await openai_exception_handler(
|
||||
request=_make_request(path="/v1/messages/count_tokens"), exc=exc
|
||||
)
|
||||
body: Final = json.loads(response.body)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert body["type"] == "error"
|
||||
|
|
@ -252,12 +251,11 @@ async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messages_routes():
|
||||
"""The envelope is chosen by route, so OpenAI routes are untouched."""
|
||||
response = await openai_exception_handler(
|
||||
async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messages_routes() -> None:
|
||||
response: Final = await openai_exception_handler(
|
||||
request=_make_request(path="/v1/chat/completions"), exc=_model_access_denied_exception()
|
||||
)
|
||||
body = json.loads(response.body)
|
||||
body: Final = json.loads(response.body)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert body == {
|
||||
|
|
@ -271,12 +269,11 @@ async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messag
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthropic_routes():
|
||||
"""The Skills and marketplace routes are not Messages routes and answer their own body."""
|
||||
response = await openai_exception_handler(
|
||||
async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthropic_routes() -> None:
|
||||
response: Final = await openai_exception_handler(
|
||||
request=_make_request(path="/v1/skills"), exc=_model_access_denied_exception()
|
||||
)
|
||||
body = json.loads(response.body)
|
||||
body: Final = json.loads(response.body)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert "type" not in body
|
||||
|
|
@ -284,11 +281,11 @@ async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthr
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_exception_handler_messages_route_copies_the_call_id_when_opted_in(monkeypatch):
|
||||
"""The Messages envelope carries error.litellm_call_id under the same opt-in as the
|
||||
OpenAI envelope, so an Anthropic SDK's e.body still names the request to look up."""
|
||||
async def test_openai_exception_handler_messages_route_copies_the_call_id_when_opted_in(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True})
|
||||
exc = ProxyException(
|
||||
exc: Final = ProxyException(
|
||||
message="denied",
|
||||
type="team_model_access_denied",
|
||||
param="model",
|
||||
|
|
@ -297,8 +294,8 @@ async def test_openai_exception_handler_messages_route_copies_the_call_id_when_o
|
|||
provider_specific_fields={"guardrail": "keyword-block"},
|
||||
)
|
||||
|
||||
response = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc)
|
||||
body = json.loads(response.body)
|
||||
response: Final = await openai_exception_handler(request=_make_request(path="/v1/messages"), exc=exc)
|
||||
body: Final = json.loads(response.body)
|
||||
|
||||
assert response.headers["x-litellm-call-id"] == "call-8302"
|
||||
assert body["error"] == {
|
||||
|
|
@ -311,15 +308,14 @@ async def test_openai_exception_handler_messages_route_copies_the_call_id_when_o
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/v1/messages", "/anthropic/v1/messages"))
|
||||
async def test_a_messages_request_with_no_key_answers_the_anthropic_authentication_error(client, monkeypatch, path):
|
||||
"""End to end through the real user_api_key_auth: a /v1/messages request with no
|
||||
credential is rejected before the handler, and the SDK must see Anthropic's
|
||||
authentication_error envelope, not OpenAI's {"error": {...}} body."""
|
||||
async def test_a_messages_request_with_no_key_answers_the_anthropic_authentication_error(
|
||||
client: TestClient, monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "test-master-key")
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
|
||||
|
||||
response = client.post(path, json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]})
|
||||
body = response.json()
|
||||
response: Final = client.post(path, json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]})
|
||||
body: Final = response.json()
|
||||
|
||||
assert response.status_code == 401
|
||||
assert body["type"] == "error"
|
||||
|
|
@ -626,6 +622,22 @@ async def test_otel_unhandled_exception_handler_answers_a_db_outage_with_503_no_
|
|||
assert json.loads(response.body) == _DB_OUTAGE_503_BODY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/v1/messages", "/claude_code_gateway/v1/messages", "/anthropic/v1/messages"))
|
||||
async def test_otel_unhandled_exception_handler_answers_the_anthropic_api_error_on_the_messages_routes(
|
||||
path: str,
|
||||
) -> None:
|
||||
response: Final = await otel_unhandled_exception_handler(
|
||||
request=_make_request(path=path), exc=RuntimeError("kaboom")
|
||||
)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert json.loads(response.body) == {
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": "Internal server error"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_otel_unhandled_exception_handler_reraises_proxy_exception_error():
|
||||
"""ProxyException / HTTPException / RequestValidationError are re-raised
|
||||
|
|
@ -705,6 +717,30 @@ async def test_http_exception_on_a_plain_http_request_keeps_the_default_json_bod
|
|||
assert json.loads(bytes(response.body)) == {"detail": "not found"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_disabled_claude_code_gateway_answers_the_anthropic_not_found_error() -> None:
|
||||
request: Final = _make_request(path="/claude_code_gateway/v1/messages")
|
||||
|
||||
response: Final = await otlp_http_exception_handler(
|
||||
request, HTTPException(404, "Claude Code gateway is not enabled")
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.status_code == 404
|
||||
assert json.loads(bytes(response.body)) == {
|
||||
"type": "error",
|
||||
"error": {"type": "not_found_error", "message": "Claude Code gateway is not enabled"},
|
||||
}
|
||||
|
||||
|
||||
def test_a_get_on_messages_answers_the_anthropic_envelope_and_keeps_the_allow_header(client: TestClient) -> None:
|
||||
response: Final = client.get("/v1/messages")
|
||||
|
||||
assert response.status_code == 405
|
||||
assert response.headers["allow"] == "POST"
|
||||
assert response.json() == {"type": "error", "error": {"type": "api_error", "message": "Method Not Allowed"}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_exception_on_a_websocket_closed_before_accept_sends_nothing_more() -> None:
|
||||
sent: Final[list[Message]] = []
|
||||
|
|
|
|||
|
|
@ -108,6 +108,25 @@ describe("toast", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
[403, "permission_error", sonner.error, "Access Denied"],
|
||||
[401, "authentication_error", sonner.error, "Authentication Error"],
|
||||
[429, "rate_limit_error", sonner.warning, "Rate Limit Exceeded"],
|
||||
[422, "invalid_request_error", sonner.error, "Request Error"],
|
||||
] as const)(
|
||||
"titles an Anthropic SDK %s error on the Messages playground by its %s type",
|
||||
(status, type, fn, title) => {
|
||||
const body = JSON.stringify({ type: "error", error: { type, message: "Team not allowed to access model" } });
|
||||
toast.fromError(
|
||||
`Error occurred while generating model response. Please try again. Error: Error: ${status} ${body}`,
|
||||
);
|
||||
expect(fn).toHaveBeenCalledWith(title, {
|
||||
description: `Error occurred while generating model response. Please try again. Error: Error: ${status} Team not allowed to access model`,
|
||||
duration: 6000,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("leaves a string alone when its braces are not a JSON envelope", () => {
|
||||
toast.fromError("Template {name} is invalid");
|
||||
expect(sonner.error).toHaveBeenCalledWith("Error", {
|
||||
|
|
|
|||
|
|
@ -46,6 +46,17 @@ const STATUS_TITLES: Readonly<Record<number, string>> = {
|
|||
503: "Service Unavailable",
|
||||
};
|
||||
|
||||
const ANTHROPIC_TYPE_TITLES: Readonly<Record<string, string>> = {
|
||||
invalid_request_error: "Request Error",
|
||||
authentication_error: "Authentication Error",
|
||||
permission_error: "Access Denied",
|
||||
not_found_error: "Not Found",
|
||||
request_too_large: "Request Error",
|
||||
rate_limit_error: "Rate Limit Exceeded",
|
||||
api_error: "Server Error",
|
||||
overloaded_error: "Service Unavailable",
|
||||
};
|
||||
|
||||
const WARNING_TITLES: ReadonlySet<string> = new Set(["Budget Exceeded", "Rate Limit Exceeded"]);
|
||||
|
||||
const asRecord = (value: unknown): Record<string, unknown> | undefined =>
|
||||
|
|
@ -123,7 +134,8 @@ const titleFor = ({ status, proxyType }: ErrorFacts): string => {
|
|||
if (proxyType?.endsWith("_access_denied")) return "Access Denied";
|
||||
const byType = proxyType === undefined ? undefined : PROXY_TYPE_TITLES[proxyType];
|
||||
if (byType !== undefined) return byType;
|
||||
return status === undefined ? "Error" : titleForStatus(status);
|
||||
if (status !== undefined) return titleForStatus(status);
|
||||
return (proxyType === undefined ? undefined : ANTHROPIC_TYPE_TITLES[proxyType]) ?? "Error";
|
||||
};
|
||||
|
||||
const show = (kind: ToastKind, message: ReactNode, options?: ToastOptions): void => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue