diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index af58b14ac57..9c726b97630 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -16,16 +16,18 @@ from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicE _PARSED_ERROR_BODY: Final = TypeAdapter(object) # HTTP status code -> Anthropic error type -# Source: https://docs.anthropic.com/en/api/errors +# Source: https://platform.claude.com/docs/en/api/errors ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = { 400: "invalid_request_error", 401: "authentication_error", + 402: "billing_error", 403: "permission_error", 404: "not_found_error", + 409: "conflict_error", 413: "request_too_large", - 422: "invalid_request_error", 429: "rate_limit_error", 500: "api_error", + 504: "timeout_error", 529: "overloaded_error", } @@ -40,7 +42,10 @@ class AnthropicExceptionMapping: @staticmethod def get_error_type(status_code: int) -> AnthropicErrorType: """Map HTTP status code to Anthropic error type.""" - return ANTHROPIC_ERROR_TYPE_MAP.get(status_code, "api_error") + named: Final = ANTHROPIC_ERROR_TYPE_MAP.get(status_code) + if named is not None: + return named + return "invalid_request_error" if 400 <= status_code < 500 else "api_error" @staticmethod def create_error_response( diff --git a/litellm/anthropic_interface/exceptions/exceptions.py b/litellm/anthropic_interface/exceptions/exceptions.py index b48cd2fee6f..18b3ca37a3b 100644 --- a/litellm/anthropic_interface/exceptions/exceptions.py +++ b/litellm/anthropic_interface/exceptions/exceptions.py @@ -6,15 +6,18 @@ from typing import Literal from typing_extensions import NotRequired, ReadOnly, Required, TypedDict # Known Anthropic error types -# Source: https://docs.anthropic.com/en/api/errors +# Source: https://platform.claude.com/docs/en/api/errors AnthropicErrorType = Literal[ "invalid_request_error", "authentication_error", + "billing_error", "permission_error", "not_found_error", + "conflict_error", "request_too_large", "rate_limit_error", "api_error", + "timeout_error", "overloaded_error", ] @@ -25,6 +28,7 @@ class AnthropicErrorDetail(TypedDict): type: AnthropicErrorType message: str provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]] + litellm_error_type: NotRequired[ReadOnly[str]] litellm_call_id: NotRequired[ReadOnly[str]] diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 743a659fc37..d245984933e 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -56,11 +56,6 @@ router: Final = APIRouter() def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: - """The Anthropic error envelope for a failure raised inside a Messages handler. - - Delegates to the shared renderer so a failure raised here and one raised by the auth - layer before the handler (see the central ProxyException handler) answer identically. - """ return anthropic_error_json_response(exc, request) diff --git a/litellm/proxy/common_utils/anthropic_error_payload.py b/litellm/proxy/common_utils/anthropic_error_payload.py index de8c460ad43..98342e945c2 100644 --- a/litellm/proxy/common_utils/anthropic_error_payload.py +++ b/litellm/proxy/common_utils/anthropic_error_payload.py @@ -6,15 +6,18 @@ from typing import Final from fastapi import Request, status from fastapi.responses import JSONResponse +from starlette.types import Scope from litellm.anthropic_interface.exceptions import ( AnthropicErrorDetail, AnthropicErrorResponse, + AnthropicErrorType, AnthropicExceptionMapping, ) from litellm.proxy._types import ProxyException +from litellm.proxy.auth.auth_utils import get_request_route from litellm.proxy.common_utils.error_body_call_id import error_body_call_id -from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER +from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER, openai_error_type_for_status ANTHROPIC_MESSAGES_ROUTES: Final[frozenset[str]] = frozenset( { @@ -25,10 +28,27 @@ ANTHROPIC_MESSAGES_ROUTES: Final[frozenset[str]] = frozenset( } ) ANTHROPIC_PASS_THROUGH_PREFIX: Final = "/anthropic/" +ANTHROPIC_SHAPED_MODEL_LIST_ROUTES: Final[frozenset[str]] = frozenset( + {"/v1/models", "/models", "/claude_code_gateway/v1/models"} +) -def uses_anthropic_error_envelope(route: str) -> bool: - return route in ANTHROPIC_MESSAGES_ROUTES or route.startswith(ANTHROPIC_PASS_THROUGH_PREFIX) +def uses_anthropic_error_envelope(request: Request) -> bool: + route: Final = get_request_route(request) + if route in ANTHROPIC_MESSAGES_ROUTES or route.startswith(ANTHROPIC_PASS_THROUGH_PREFIX): + return True + return route in ANTHROPIC_SHAPED_MODEL_LIST_ROUTES and "anthropic-version" in request.headers + + +def route_error_content( + scope: Scope, anthropic_type: AnthropicErrorType, message: str, openai_content: Mapping[str, object] +) -> Mapping[str, object]: + request: Final = Request(scope) + if not uses_anthropic_error_envelope(request): + return openai_content + content: Final[AnthropicErrorResponse] = {"type": "error", "error": {"type": anthropic_type, "message": message}} + request_id: Final = request.headers.get("x-request-id") + return content if request_id is None else {**content, "request_id": request_id} def anthropic_error_status_code(exc: ProxyException) -> int: @@ -45,15 +65,22 @@ def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDe return with_fields -def _anthropic_error_detail( - exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None +def _with_litellm_error_type( + exc: ProxyException, status_code: int, detail: AnthropicErrorDetail ) -> AnthropicErrorDetail: + if not exc.type or exc.type in (detail["type"], openai_error_type_for_status(status_code)): + return detail + with_type: Final[AnthropicErrorDetail] = {**detail, "litellm_error_type": exc.type} + return with_type + + +def _anthropic_error_detail( + exc: ProxyException, status_code: int, detail: AnthropicErrorDetail, call_id: str | None +) -> AnthropicErrorDetail: + with_proxy_fields: Final = _with_litellm_error_type(exc, status_code, _with_provider_specific_fields(exc, detail)) if call_id is None: - return _with_provider_specific_fields(exc, detail) - with_call_id: Final[AnthropicErrorDetail] = { - **_with_provider_specific_fields(exc, detail), - "litellm_call_id": call_id, - } + return with_proxy_fields + with_call_id: Final[AnthropicErrorDetail] = {**with_proxy_fields, "litellm_call_id": call_id} return with_call_id @@ -71,7 +98,7 @@ def anthropic_error_content( body_call_id: Final = error_body_call_id(general_settings, exc.headers.get(LITELLM_CALL_ID_HEADER)) content: Final[AnthropicErrorResponse] = { **envelope, - "error": _anthropic_error_detail(exc, envelope["error"], body_call_id), + "error": _anthropic_error_detail(exc, status_code, envelope["error"], body_call_id), } return content diff --git a/litellm/proxy/middleware/admission_control_middleware.py b/litellm/proxy/middleware/admission_control_middleware.py index 1bb94e23d15..83503d0c3d4 100644 --- a/litellm/proxy/middleware/admission_control_middleware.py +++ b/litellm/proxy/middleware/admission_control_middleware.py @@ -10,6 +10,7 @@ from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.anthropic_error_payload import route_error_content ADMISSION_LEASE_SCOPE_KEY: Final = "litellm.admission_lease" @@ -175,7 +176,7 @@ class AdmissionControlMiddleware: state.record_admission() elif state.get_stats().queued >= settings.max_queued_requests: state.record_rejection("queue_full") - await _overloaded_response(state)(scope, receive, send) + await _overloaded_response(state, scope)(scope, receive, send) return else: state.record_queue() @@ -187,7 +188,7 @@ class AdmissionControlMiddleware: except asyncio.TimeoutError: state.record_dequeue() state.record_rejection("queue_timeout") - await _overloaded_response(state)(scope, receive, send) + await _overloaded_response(state, scope)(scope, receive, send) return except asyncio.CancelledError: state.record_dequeue() @@ -313,18 +314,16 @@ def get_admission_control_settings(settings: Mapping[str, object]) -> AdmissionC ) -def _overloaded_response(state: AdmissionControlState) -> JSONResponse: +def _overloaded_response(state: AdmissionControlState, scope: Scope) -> JSONResponse: stats: Final = state.get_stats() + message: Final = f"Worker at capacity: {stats.admitted} in-flight, {stats.queued} queued requests. Retry later." return JSONResponse( status_code=503, headers={"retry-after": "1"}, - content={ - "error": { - "message": ( - f"Worker at capacity: {stats.admitted} in-flight, {stats.queued} queued requests. Retry later." - ), - "type": "overloaded_error", - "code": "503", - } - }, + content=route_error_content( + scope, + "overloaded_error", + message, + {"error": {"message": message, "type": "overloaded_error", "code": "503"}}, + ), ) diff --git a/litellm/proxy/middleware/request_body_decompression_middleware.py b/litellm/proxy/middleware/request_body_decompression_middleware.py index 5eb8a9bfc91..6a732cc1e8e 100644 --- a/litellm/proxy/middleware/request_body_decompression_middleware.py +++ b/litellm/proxy/middleware/request_body_decompression_middleware.py @@ -12,12 +12,14 @@ from starlette.requests import ClientDisconnect, Request from starlette.responses import JSONResponse from starlette.types import ASGIApp, Message, Receive, Scope, Send +from litellm.anthropic_interface.exceptions import AnthropicErrorType, AnthropicExceptionMapping from litellm.constants import ( MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB, MAX_DECOMPRESSED_REQUEST_BODY_SIZE_MB, MAX_REQUEST_BODY_DECOMPRESSION_RATIO, ) from litellm.proxy.auth.auth_utils import get_request_route +from litellm.proxy.common_utils.anthropic_error_payload import route_error_content from litellm.proxy.middleware.request_size_limit_middleware import MaxRequestSizeGetter, send_request_too_large RFC_9659_ZSTD_WINDOW_LOG_MAX: Final = 23 @@ -291,13 +293,24 @@ def _replay_body(body: bytes, receive: Receive, more_body: bool = False) -> Rece def _error_response( + scope: Scope, message: str, status_code: int, error_type: str = "invalid_request_error", headers: Mapping[str, str] | None = None, ) -> JSONResponse: + anthropic_type: Final[AnthropicErrorType] = ( + "overloaded_error" + if error_type == "overloaded_error" + else AnthropicExceptionMapping.get_error_type(status_code) + ) return JSONResponse( - {"error": {"message": message, "type": error_type, "param": None, "code": str(status_code)}}, + route_error_content( + scope, + anthropic_type, + message, + {"error": {"message": message, "type": error_type, "param": None, "code": str(status_code)}}, + ), status_code=status_code, headers=headers, ) @@ -356,6 +369,7 @@ class RequestBodyDecompressionMiddleware: if codec is None: supported: Final = ", ".join(sorted(self.codecs)) await _error_response( + scope, f"This proxy cannot decode {coding} request bodies. Supported encodings: {supported}", 415, headers={"accept-encoding": supported}, @@ -372,7 +386,7 @@ class RequestBodyDecompressionMiddleware: case _Body(data=data): await self._decode_within_budget(scope, receive, send, codec, data, limits) case _TooLarge(): - await send_request_too_large(send=send, max_request_size_mb=max_mb) + await send_request_too_large(scope=scope, send=send, max_request_size_mb=max_mb) case _ClientDisconnected(): return @@ -382,6 +396,7 @@ class RequestBodyDecompressionMiddleware: reservation: Final = _max_decoded_length(codec, data, limits) if not self.budget.try_reserve(reservation): await _error_response( + scope, "The proxy is decoding too many compressed request bodies right now. " "Retry later, or send this body uncompressed", 503, @@ -407,12 +422,12 @@ class RequestBodyDecompressionMiddleware: case _Body(data=body): await self.app(_identity_scope(scope, len(body)), _replay_body(body, receive), send) case _DecodedTooLarge(): - await _error_response(f"Request body is larger than {limits.max_mb:g} MB when decoded", 413)( + await _error_response(scope, f"Request body is larger than {limits.max_mb:g} MB when decoded", 413)( scope, receive, send ) case _OverRatio(): - await _error_response(f"Request body expands more than {limits.max_ratio} times when decoded", 413)( - scope, receive, send - ) + await _error_response( + scope, f"Request body expands more than {limits.max_ratio} times when decoded", 413 + )(scope, receive, send) case _Undecodable(message=message): - await _error_response(message, 400)(scope, receive, send) + await _error_response(scope, message, 400)(scope, receive, send) diff --git a/litellm/proxy/middleware/request_size_limit_middleware.py b/litellm/proxy/middleware/request_size_limit_middleware.py index cbbc79cc2ca..15bbf05bd01 100644 --- a/litellm/proxy/middleware/request_size_limit_middleware.py +++ b/litellm/proxy/middleware/request_size_limit_middleware.py @@ -4,6 +4,8 @@ from typing import Final from starlette.types import ASGIApp, Message, Receive, Scope, Send +from litellm.proxy.common_utils.anthropic_error_payload import route_error_content + MaxRequestSizeGetter = Callable[[], int | float | None] RequestSizeLimitEnabledGetter = Callable[[], bool] @@ -44,7 +46,7 @@ class RequestSizeLimitMiddleware: content_length: Final = _get_content_length(scope=scope) if content_length is not None and content_length > max_request_size_bytes: - await send_request_too_large(send=send, max_request_size_mb=max_request_size_mb) + await send_request_too_large(scope=scope, send=send, max_request_size_mb=max_request_size_mb) return received_body_bytes = 0 @@ -74,7 +76,7 @@ class RequestSizeLimitMiddleware: except RequestEntityTooLarge: if response_started: raise - await send_request_too_large(send=send, max_request_size_mb=max_request_size_mb) + await send_request_too_large(scope=scope, send=send, max_request_size_mb=max_request_size_mb) def _mb_to_bytes(max_request_size_mb: float | None) -> int | None: @@ -98,11 +100,13 @@ def _get_content_length(scope: Scope) -> int | None: async def send_request_too_large( + scope: Scope, send: Send, max_request_size_mb: float | None, ) -> None: + message: Final = f"Request size is too large. Max size is {max_request_size_mb} MB" body: Final = json.dumps( - {"error": f"Request size is too large. Max size is {max_request_size_mb} MB"}, + route_error_content(scope, "request_too_large", message, {"error": message}), separators=(",", ":"), ).encode("utf-8") await send( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ada7e46da1b..95807cf71c4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -405,7 +405,6 @@ from litellm.proxy.auth.auth_object_prefetch import AUTH_OBJECTS_TARGET from litellm.proxy.auth.auth_utils import ( MAX_REQUEST_SIZE_MB_SETTING, check_response_size_is_safe, - get_request_route, is_request_body_safe, log_model_access_denial, log_once_if_budget_reservation_disabled, @@ -2194,7 +2193,7 @@ async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions log_model_access_denial(exc) headers: Final = exc.headers - if uses_anthropic_error_envelope(get_request_route(request)): + if uses_anthropic_error_envelope(request): return anthropic_error_json_response(exc, request) error_dict: Final = with_call_id( JSON_OBJECT.validate_python(exc.to_dict()), @@ -2231,7 +2230,7 @@ async def otlp_http_exception_handler(connection: Request | WebSocket, exc: Star if response is not None: _close_dangling_otel_server_span(connection, exc.status_code, exc=exc) return response - if uses_anthropic_error_envelope(get_request_route(connection)): + if uses_anthropic_error_envelope(connection): return anthropic_error_json_response( proxy_exception_from_http_exception(exc, dict(exc.headers or {})), connection ) @@ -2442,7 +2441,7 @@ async def _unhandled_exception_response(request: Request, exc: Exception) -> Res otlp_response: Final = tracing_endpoints.otlp_error_response(request, 500) if otlp_response is not None: return otlp_response - if uses_anthropic_error_envelope(get_request_route(request)): + if uses_anthropic_error_envelope(request): return anthropic_internal_server_error_response(request) return JSONResponse( status_code=500, diff --git a/tests/integration/authorization/test_claude_code_gateway_oauth.py b/tests/integration/authorization/test_claude_code_gateway_oauth.py index 700816c8d31..cb5ae4c3ba4 100644 --- a/tests/integration/authorization/test_claude_code_gateway_oauth.py +++ b/tests/integration/authorization/test_claude_code_gateway_oauth.py @@ -481,6 +481,7 @@ def test_device_authorization_polls_complete_sso_and_mints_a_model_restricted_be f"The requested model '{model_b}' is not available for this API key, or the model name is " "invalid. Check the models available to you and try again." ), + "litellm_error_type": "team_model_access_denied", }, }, model_b_response.text assert upstream.drain() == () diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py index e104fad4b3f..1f57584572b 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_gateway_wire.py @@ -276,6 +276,7 @@ def test_enabled_gateway_count_tokens_alias_matches_native_count_tokens_route(en f"The requested model '{restricted_model}' is not available for this API key, or the model name " "is invalid. Check the models available to you and try again." ), + "litellm_error_type": "key_model_access_denied", }, }, restricted_response.text assert wire.drain() == () @@ -292,6 +293,7 @@ def test_enabled_gateway_refuses_model_outside_virtual_key(enabled_gateway: Gate f"The requested model '{result[4]}' is not available for this API key, or the model name is invalid. " "Check the models available to you and try again." ), + "litellm_error_type": "key_model_access_denied", }, } assert result[0] == result[2] == 403, (result[1], result[3]) diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py index a448a8cd2da..fe6b8efb423 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_wire.py @@ -357,10 +357,14 @@ def test_anthropic_messages_preserves_cli_beta_safeguards_and_result(gateway: Ga @pytest.mark.parametrize( - ("status", "error_type", "error_class", "litellm_error"), + ("status", "error_type", "error_class", "litellm_error", "litellm_error_type"), [ - pytest.param(429, "rate_limit_error", anthropic.RateLimitError, "RateLimitError", id="rate_limit"), - pytest.param(400, "invalid_request_error", anthropic.BadRequestError, "BadRequestError", id="invalid_request"), + pytest.param( + 429, "rate_limit_error", anthropic.RateLimitError, "RateLimitError", "throttling_error", id="rate_limit" + ), + pytest.param( + 400, "invalid_request_error", anthropic.BadRequestError, "BadRequestError", None, id="invalid_request" + ), ], ) def test_anthropic_messages_preserves_upstream_error_status_and_body( @@ -369,6 +373,7 @@ def test_anthropic_messages_preserves_upstream_error_status_and_body( error_type: str, error_class: type[anthropic.APIStatusError], litellm_error: str, + litellm_error_type: str | None, ) -> None: message: Final = f"synthetic {error_type}" expected_error: Final = {"type": "error", "error": {"type": error_type, "message": message}} @@ -416,6 +421,7 @@ def test_anthropic_messages_preserves_upstream_error_status_and_body( f"litellm.{litellm_error}: AnthropicException - {json.dumps(expected_error)}\n\n" f"LiteLLM: model group '{model}' failed with the error above. No fallback was attempted." ), + **({"litellm_error_type": litellm_error_type} if litellm_error_type else {}), }, }, caught.value.response.text assert len(wire.drain()) == 1 @@ -470,6 +476,7 @@ def test_anthropic_messages_preserves_upstream_rate_limit_retry_after(gateway: G f"litellm.RateLimitError: AnthropicException - {json.dumps(upstream_error)}\n\n" f"LiteLLM: model group '{model}' failed with the error above. No fallback was attempted." ), + "litellm_error_type": "throttling_error", }, }, caught.value.response.text assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_count_tokens_error_shapes_wire.py b/tests/integration/providers/test_count_tokens_error_shapes_wire.py index a6c3a6dc62f..a5299547ec3 100644 --- a/tests/integration/providers/test_count_tokens_error_shapes_wire.py +++ b/tests/integration/providers/test_count_tokens_error_shapes_wire.py @@ -140,7 +140,11 @@ def test_anthropic_count_without_a_provider_credential_is_an_anthropic_401( _anthropic(gateway).messages.count_tokens(model=model, messages=_MESSAGES) assert raised.value.body == { "type": "error", - "error": {"type": "authentication_error", "message": raised.value.body["error"]["message"]}, + "error": { + "type": "authentication_error", + "message": raised.value.body["error"]["message"], + "litellm_error_type": "token_counting_error", + }, } assert missing in raised.value.message, raised.value.message assert wire.drain() == () @@ -210,7 +214,7 @@ def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy assert rejected.status_code == 400, rejected.text body: Final = _JSON_OBJECT.validate_json(rejected.content) assert set(body) <= {"type", "error", "request_id"}, body - assert body["error"] == upstream_error["error"], body + assert body["error"] == {**upstream_error["error"], "litellm_error_type": "token_counting_error"}, body assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_vertex_partner_count_tokens_wire.py b/tests/integration/providers/test_vertex_partner_count_tokens_wire.py index c679311e4cc..809e8877312 100644 --- a/tests/integration/providers/test_vertex_partner_count_tokens_wire.py +++ b/tests/integration/providers/test_vertex_partner_count_tokens_wire.py @@ -258,7 +258,10 @@ class _CountRoute: _MESSAGES_ROUTE: Final = _CountRoute( path=lambda _model: "/v1/messages/count_tokens", body=_bare, - rejection=lambda message: {"type": "error", "error": {"type": "invalid_request_error", "message": message}}, + rejection=lambda message: { + "type": "error", + "error": {"type": "invalid_request_error", "message": message, "litellm_error_type": "token_counting_error"}, + }, count_field="input_tokens", ) _RESPONSES_ROUTE: Final = _CountRoute( diff --git a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py index 5c40bfafa08..7019e68edf6 100644 --- a/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py +++ b/tests/unit/anthropic_interface/exceptions/test_exception_mapping_utils.py @@ -65,13 +65,22 @@ class TestCreateErrorResponse: @pytest.mark.parametrize( ("status_code", "expected_error_type"), - [(422, "invalid_request_error"), (418, "api_error"), (502, "api_error"), (599, "api_error")], + [ + (402, "billing_error"), + (409, "conflict_error"), + (504, "timeout_error"), + (405, "invalid_request_error"), + (418, "invalid_request_error"), + (422, "invalid_request_error"), + (499, "invalid_request_error"), + (502, "api_error"), + (503, "api_error"), + (599, "api_error"), + ], ) - def test_a_budget_status_is_a_request_error_and_a_status_the_table_does_not_name_is_api_error( + def test_a_status_the_docs_do_not_name_is_a_request_error_below_500_and_an_api_error_from_500( self, status_code: int, expected_error_type: str ): - """The proxy's budget status is a caller-fault request error, so the table names 422. - A status the table does not name, such as a 418 or a 5xx, keeps api_error.""" response = AnthropicExceptionMapping.create_error_response(status_code, "scripted status") assert response["error"]["type"] == expected_error_type diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index c7bb05e5d30..5d18c883db3 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -172,6 +172,7 @@ class TestProxyExceptionAnthropicEnvelope: "error": { "type": "invalid_request_error", "message": "Invalid type for 'metadata': expected an object, but got a string instead.", + "litellm_error_type": "bad_request_error", }, "request_id": "req_test_6468", } diff --git a/tests/unit/proxy/middleware/test_admission_control_middleware.py b/tests/unit/proxy/middleware/test_admission_control_middleware.py index e449ebc83bf..66549b0c351 100644 --- a/tests/unit/proxy/middleware/test_admission_control_middleware.py +++ b/tests/unit/proxy/middleware/test_admission_control_middleware.py @@ -510,3 +510,32 @@ async def test_admission_lease_cannot_be_reused_by_another_worker(state: Admissi response: Final = await _call(parent) assert response[1]["body"] == b"1" assert state.get_stats() == other_state.get_stats() == AdmissionControlStats(0, 0, 0) + + +@pytest.mark.asyncio +async def test_a_messages_request_turned_away_at_capacity_gets_the_anthropic_overloaded_error( + state: AdmissionControlState, +) -> None: + started: Final = asyncio.Event() + release: Final = asyncio.Event() + middleware: Final = AdmissionControlMiddleware( + _handler_with_release(started, release), + lambda: AdmissionControlSettings(1, 0, 1.0), + state, + ) + + first: Final = asyncio.create_task(_call(middleware, path="/v1/messages")) + await started.wait() + rejected: Final = await _call(middleware, path="/v1/messages") + release.set() + await first + + assert rejected[0]["status"] == 503 + assert dict(rejected[0]["headers"])[b"retry-after"] == b"1" + assert json.loads(rejected[1]["body"]) == { + "type": "error", + "error": { + "type": "overloaded_error", + "message": "Worker at capacity: 1 in-flight, 0 queued requests. Retry later.", + }, + } diff --git a/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py b/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py index ca5774b6be0..f91b3690637 100644 --- a/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py +++ b/tests/unit/proxy/middleware/test_request_body_decompression_middleware.py @@ -665,3 +665,33 @@ async def test_non_http_scopes_reach_the_app_untouched(scope: Scope): [(app_scope, app_receive)] = seen assert app_scope is scope assert app_receive is receive + + +@pytest.mark.parametrize( + ("body", "max_mb", "status_code", "anthropic_type", "message"), + [ + pytest.param( + zstd.compress(_low_entropy_text(2 * int(CAP_MB * MIB))), + CAP_MB, + 413, + "request_too_large", + f"Request body is larger than {CAP_MB:g} MB when decoded", + id="decoded-past-the-cap", + ), + pytest.param(b"not zstd at all", 50.0, 400, "invalid_request_error", None, id="undecodable"), + ], +) +def test_a_messages_body_the_proxy_cannot_take_gets_the_anthropic_envelope( + body: bytes, max_mb: float, status_code: int, anthropic_type: str, message: str | None +): + response: Final = _post( + _client(max_decompressed_body_size_mb=max_mb), + ZSTD_FRAME_MAGIC + body if message is None else body, + ("zstd",), + path="/claude_code_gateway/v1/messages", + ) + + assert response.status_code == status_code + assert response.json()["type"] == "error" + assert response.json()["error"]["type"] == anthropic_type + assert message is None or response.json()["error"]["message"] == message diff --git a/tests/unit/proxy/middleware/test_request_size_limit_middleware.py b/tests/unit/proxy/middleware/test_request_size_limit_middleware.py index 3e8792e0179..29b4a6a43a1 100644 --- a/tests/unit/proxy/middleware/test_request_size_limit_middleware.py +++ b/tests/unit/proxy/middleware/test_request_size_limit_middleware.py @@ -133,3 +133,22 @@ async def test_request_size_limit_middleware_rejects_streamed_body_without_conte "more_body": False, } assert received_body_bytes == 1024 * 1024 + + +def test_an_oversized_messages_request_gets_the_anthropic_request_too_large_error(): + async def app(scope, receive, send): + await JSONResponse({"ok": True})(scope, receive, send) + + client = TestClient( + RequestSizeLimitMiddleware(app, get_max_request_size_mb=lambda: 1, is_request_size_limit_enabled=lambda: True) + ) + + response = client.post( + "/v1/messages", content=b"x" * (1024 * 1024 + 1), headers={"content-type": "application/json"} + ) + + assert response.status_code == 413 + assert response.json() == { + "type": "error", + "error": {"type": "request_too_large", "message": "Request size is too large. Max size is 1 MB"}, + } diff --git a/tests/unit/proxy/proxy_server/test_exception_handlers.py b/tests/unit/proxy/proxy_server/test_exception_handlers.py index 7676f82bd74..8a8916407d1 100644 --- a/tests/unit/proxy/proxy_server/test_exception_handlers.py +++ b/tests/unit/proxy/proxy_server/test_exception_handlers.py @@ -36,9 +36,9 @@ from litellm.proxy.proxy_server import ( from .conftest import normalize -def _make_request(parent_otel_span=None, path="/chat/completions"): +def _make_request(parent_otel_span=None, path="/chat/completions", headers=()): return Request({ - "type": "http", "method": "POST", "path": path, "headers": [], + "type": "http", "method": "POST", "path": path, "headers": list(headers), "state": {"parent_otel_span": parent_otel_span}, }) @@ -209,6 +209,7 @@ async def test_openai_exception_handler_answers_the_anthropic_envelope_on_the_me "error": { "type": "permission_error", "message": "Team not allowed to access model. Tried to access gpt-5.6", + "litellm_error_type": "team_model_access_denied", }, } @@ -247,6 +248,7 @@ async def test_openai_exception_handler_keeps_the_status_of_a_budget_failure_on_ assert body["type"] == "error" assert body["error"]["type"] == "invalid_request_error" assert body["error"]["message"] == "Budget exceeded" + assert body["error"]["litellm_error_type"] == "budget_exceeded" assert "param" not in body["error"] and "code" not in body["error"] @@ -268,6 +270,23 @@ async def test_openai_exception_handler_keeps_the_openai_envelope_off_the_messag } +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/v1/models", "/models", "/claude_code_gateway/v1/models")) +async def test_a_model_list_request_answers_errors_in_the_shape_its_listing_takes(path: str) -> None: + anthropic: Final = await openai_exception_handler( + request=_make_request(path=path, headers=[(b"anthropic-version", b"2023-06-01")]), + exc=_model_access_denied_exception(), + ) + openai: Final = await openai_exception_handler( + request=_make_request(path=path), exc=_model_access_denied_exception() + ) + + assert json.loads(anthropic.body)["error"]["type"] == "permission_error" + assert json.loads(anthropic.body)["type"] == "error" + assert json.loads(openai.body)["error"]["type"] == "team_model_access_denied" + assert "type" not in json.loads(openai.body) + + @pytest.mark.asyncio async def test_openai_exception_handler_keeps_the_openai_envelope_on_other_anthropic_routes() -> None: response: Final = await openai_exception_handler( @@ -302,6 +321,7 @@ async def test_openai_exception_handler_messages_route_copies_the_call_id_when_o "type": "permission_error", "message": "denied", "provider_specific_fields": {"guardrail": "keyword-block"}, + "litellm_error_type": "team_model_access_denied", "litellm_call_id": "call-8302", } @@ -738,7 +758,10 @@ def test_a_get_on_messages_answers_the_anthropic_envelope_and_keeps_the_allow_he assert response.status_code == 405 assert response.headers["allow"] == "POST" - assert response.json() == {"type": "error", "error": {"type": "api_error", "message": "Method Not Allowed"}} + assert response.json() == { + "type": "error", + "error": {"type": "invalid_request_error", "message": "Method Not Allowed"}, + } @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..f8134aed27f 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1525,6 +1525,7 @@ async def test_count_tokens_without_a_count_api_rejects_a_non_string_tool_result assert json.loads(bytes(response.body))["error"] == { "type": "invalid_request_error", "message": "tool_result toolu_log content.1.text: Input should be a valid string", + "litellm_error_type": "token_counting_error", } diff --git a/ui/litellm-dashboard/src/lib/toast.test.ts b/ui/litellm-dashboard/src/lib/toast.test.ts index e927c3e06ed..bb0edf791f3 100644 --- a/ui/litellm-dashboard/src/lib/toast.test.ts +++ b/ui/litellm-dashboard/src/lib/toast.test.ts @@ -109,14 +109,55 @@ describe("toast", () => { }); it.each([ - [403, "permission_error", sonner.error, "Access Denied"], - [401, "authentication_error", sonner.error, "Authentication Error"], - [429, "rate_limit_error", sonner.warning, "Rate Limit Exceeded"], - [422, "invalid_request_error", sonner.error, "Request Error"], - ] as const)( - "titles an Anthropic SDK %s error on the Messages playground by its %s type", - (status, type, fn, title) => { - const body = JSON.stringify({ type: "error", error: { type, message: "Team not allowed to access model" } }); + { + status: 403, + type: "permission_error", + litellmType: "team_model_access_denied", + fn: sonner.error, + title: "Access Denied", + }, + { + status: 401, + type: "authentication_error", + litellmType: "auth_error", + fn: sonner.error, + title: "Authentication Error", + }, + { + status: 422, + type: "invalid_request_error", + litellmType: "budget_exceeded", + fn: sonner.warning, + title: "Budget Exceeded", + }, + { + status: 503, + type: "api_error", + litellmType: "no_db_connection", + fn: sonner.error, + title: "Service Unavailable", + }, + { + status: 429, + type: "rate_limit_error", + litellmType: undefined, + fn: sonner.warning, + title: "Rate Limit Exceeded", + }, + { + status: 422, + type: "invalid_request_error", + litellmType: undefined, + fn: sonner.error, + title: "Validation Error", + }, + ])( + "titles an Anthropic SDK $status $type error from the Messages playground by its proxy type $litellmType, else its status", + ({ status, type, litellmType, fn, title }) => { + const body = JSON.stringify({ + type: "error", + error: { type, message: "Team not allowed to access model", litellm_error_type: litellmType }, + }); toast.fromError( `Error occurred while generating model response. Please try again. Error: Error: ${status} ${body}`, ); @@ -127,6 +168,11 @@ describe("toast", () => { }, ); + it("titles an Anthropic error body with no status by its Anthropic type", () => { + toast.fromError(JSON.stringify({ type: "error", error: { type: "permission_error", message: "denied" } })); + expect(sonner.error).toHaveBeenCalledWith("Access Denied", { description: "denied", duration: 6000 }); + }); + it("leaves a string alone when its braces are not a JSON envelope", () => { toast.fromError("Template {name} is invalid"); expect(sonner.error).toHaveBeenCalledWith("Error", { diff --git a/ui/litellm-dashboard/src/lib/toast.ts b/ui/litellm-dashboard/src/lib/toast.ts index 465ac7a1f6c..8ef545ba0ef 100644 --- a/ui/litellm-dashboard/src/lib/toast.ts +++ b/ui/litellm-dashboard/src/lib/toast.ts @@ -82,11 +82,16 @@ const proxyEnvelope = (payload: unknown): Record | undefined => }; const proxyTypeOf = (payload: unknown): string | undefined => { - const type = proxyEnvelope(payload)?.type; + const envelope = proxyEnvelope(payload); + const type = envelope?.litellm_error_type ?? envelope?.type; return typeof type === "string" ? type : undefined; }; const EMBEDDED_JSON = /\{[\s\S]*\}/; +const STATUS_BEFORE_JSON = /(\d{3})\s*$/; + +const statusBefore = (text: string, embedded: string): number | undefined => + toStatus(text.slice(0, text.indexOf(embedded)).match(STATUS_BEFORE_JSON)?.[1]); const describeText = (text: string): ErrorFacts => { const embedded = text.match(EMBEDDED_JSON)?.[0]; @@ -95,7 +100,7 @@ const describeText = (text: string): ErrorFacts => { return { status: undefined, proxyType: undefined, text: unwrapProxyErrorMessage(text) }; } return { - status: toStatus(proxyEnvelope(parsed)?.code), + status: toStatus(proxyEnvelope(parsed)?.code) ?? statusBefore(text, embedded), proxyType: proxyTypeOf(parsed), text: text.replace(embedded, unwrapProxyErrorMessage(deriveErrorMessage(parsed))).trim(), };