diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 13a1844a6de..94261f2063f 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -75,7 +75,7 @@ def _anthropic_error_detail( return with_call_id -def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: +def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does general_settings_view, @@ -286,7 +286,7 @@ async def anthropic_response( log_llm_api_exception(e, base_llm_response_processor.litellm_call_id) if isinstance(e, ProxyException): - return _anthropic_error_json_response( + return anthropic_error_json_response( with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request ) @@ -309,10 +309,10 @@ async def anthropic_response( ) if isinstance(e, HTTPException): - return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request) + return anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request) error_msg: Final = f"{e}" - return _anthropic_error_json_response( + return anthropic_error_json_response( ProxyException( message=getattr(e, "message", error_msg), type=openai_error_type(e, error_status_code(e, 500)), @@ -406,7 +406,7 @@ async def count_tokens( token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields) ) if isinstance(token_request, ProxyException): - return _anthropic_error_json_response(token_request, request) + return anthropic_error_json_response(token_request, request) token_response: Final = await count_request_tokens( request=token_request, @@ -419,12 +419,12 @@ async def count_tokens( ProxyException(message=str(e), type="invalid_request_error", param="messages", code=400), request ) except HTTPException as e: - return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request) + return anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request) except ProxyException as e: - return _anthropic_error_json_response(e, request) + return anthropic_error_json_response(e, request) except Exception as e: log_llm_api_exception(e, litellm_call_id) - return _anthropic_error_json_response(proxy_exception_from_route_error(e), request) + return anthropic_error_json_response(proxy_exception_from_route_error(e), request) else: return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..59cf1ab2727 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -829,6 +829,7 @@ except ImportError: build_billing_metrics_recorder = None shutdown_billing_metrics_recorder = None from fastapi.exception_handlers import http_exception_handler +from starlette._utils import get_route_path from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.websockets import WebSocketState @@ -2186,6 +2187,10 @@ class UserAPIKeyCacheTTLEnum(enum.Enum): async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions log_model_access_denial(exc) + if get_route_path(request.scope) in ("/v1/messages", "/v1/messages/count_tokens"): + from litellm.proxy.anthropic_endpoints.endpoints import anthropic_error_json_response + + return anthropic_error_json_response(exc, request) headers: Final = exc.headers error_dict: Final = with_call_id( JSON_OBJECT.validate_python(exc.to_dict()), diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index c7bb05e5d30..2b6e776b25f 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -187,14 +187,14 @@ class TestProxyExceptionAnthropicEnvelope: request = MagicMock() request.headers = {} - response = ep._anthropic_error_json_response( + response = ep.anthropic_error_json_response( ProxyException(message="Rate limit exceeded", type="rate_limit_error", param=None, code=429), request, ) assert response.status_code == 429 assert json.loads(response.body)["error"]["type"] == "rate_limit_error" - fallback = ep._anthropic_error_json_response( + fallback = ep.anthropic_error_json_response( ProxyException(message="boom", type="None", param=None, code=None), request, ) @@ -217,7 +217,7 @@ class TestProxyExceptionAnthropicEnvelope: provider_specific_fields=provider_specific_fields, ) with patch("litellm.proxy.proxy_server.general_settings", general_settings): - return ep._anthropic_error_json_response(exc, request) + return ep.anthropic_error_json_response(exc, request) def test_anthropic_error_copies_the_call_id_into_the_error_when_opted_in(self): """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index a26a64c6fb0..7066ef1eeaa 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -49,6 +49,113 @@ from litellm.proxy.proxy_server import ( # Replace with the actual module where ) from litellm.proxy.utils import ProxyLogging + +@pytest.mark.parametrize("root_path", ["", "/gateway"]) +@pytest.mark.parametrize( + ("path", "anthropic_envelope"), + [ + ("/v1/messages", True), + ("/v1/messages/count_tokens", True), + ("/v1/chat/completions", False), + ("/v1/responses", False), + ], +) +@pytest.mark.parametrize("failure", ["missing_key", "malformed_json"]) +async def test_pre_handler_errors_use_endpoint_envelope( + monkeypatch: pytest.MonkeyPatch, + root_path: str, + path: str, + anthropic_envelope: bool, + failure: str, +) -> None: + import httpx2 + + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "master_key", "sk-local-envelope-regression") + monkeypatch.setattr(proxy_server, "general_settings", {}) + application: Final = FastAPI() + application.mount(root_path or "/", app) + headers: Final = {"Authorization": "Bearer sk-local-envelope-regression"} if failure == "malformed_json" else {} + payload: Final = b'{"model":' if failure == "malformed_json" else b'{"model":"local-only"}' + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=application), + base_url="http://testserver", + ) as client: + response: Final = await client.post(root_path + path, content=payload, headers=headers) + + assert response.status_code == (400 if failure == "malformed_json" else 401), response.text + message: Final = response.json()["error"]["message"] + assert message.startswith( + "Invalid JSON payload:" if failure == "malformed_json" else "Authentication Error, No api key passed in." + ) + expected: Final = ( + { + "type": "error", + "error": { + "type": "invalid_request_error" if failure == "malformed_json" else "authentication_error", + "message": message, + }, + } + if anthropic_envelope + else { + "error": { + "type": "invalid_request_error" if failure == "malformed_json" else "auth_error", + "message": message, + "param": "request_body" if failure == "malformed_json" else "None", + "code": "400" if failure == "malformed_json" else "401", + }, + } + ) + assert response.json() == expected + + +@pytest.mark.parametrize("path", ["/v1/messages", "/v1/messages/count_tokens"]) +@pytest.mark.parametrize("include_call_id", [False, True]) +async def test_messages_global_error_preserves_status_and_diagnostics( + monkeypatch: pytest.MonkeyPatch, + path: str, + include_call_id: bool, +) -> None: + from fastapi import Request + + from litellm.proxy import proxy_server + from litellm.proxy._types import ProxyException + + monkeypatch.setattr(proxy_server, "general_settings", {"include_call_id_in_error_body": include_call_id}) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": path, + "headers": [(b"x-request-id", b"request-envelope")], + } + ) + error: Final = ProxyException( + message="Model access denied", + type="user_model_access_denied", + param="model", + code=403, + headers={"x-litellm-call-id": "call-envelope", "retry-after": "12"}, + provider_specific_fields={"model": "restricted-model"}, + ) + response: Final = await proxy_server.openai_exception_handler(request, error) + + assert response.status_code == 403 + assert response.headers["x-litellm-call-id"] == "call-envelope" + assert response.headers["retry-after"] == "12" + assert json.loads(response.body) == { + "type": "error", + "request_id": "request-envelope", + "error": { + "type": "permission_error", + "message": "Model access denied", + "provider_specific_fields": {"model": "restricted-model"}, + **({"litellm_call_id": "call-envelope"} if include_call_id else {}), + }, + } + + @pytest.fixture def admin_mcp_proxy(monkeypatch: pytest.MonkeyPatch) -> FastAPI: from litellm.proxy import proxy_server