Merge b361c799f0 into 220a70653e

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -75,7 +75,7 @@ def _anthropic_error_detail(
     return with_call_id
 
 
-def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
+def anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse:
     from litellm.proxy.proxy_server import (
         _close_dangling_otel_server_span,  # pyright: ignore[reportPrivateUsage]  # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
         general_settings_view,
@@ -286,7 +286,7 @@ async def anthropic_response(
         log_llm_api_exception(e, base_llm_response_processor.litellm_call_id)
 
         if isinstance(e, ProxyException):
-            return _anthropic_error_json_response(
+            return anthropic_error_json_response(
                 with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request
             )
 
@@ -309,10 +309,10 @@ async def anthropic_response(
         )
 
         if isinstance(e, HTTPException):
-            return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request)
+            return anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request)
 
         error_msg: Final = f"{e}"
-        return _anthropic_error_json_response(
+        return anthropic_error_json_response(
             ProxyException(
                 message=getattr(e, "message", error_msg),
                 type=openai_error_type(e, error_status_code(e, 500)),
@@ -406,7 +406,7 @@ async def count_tokens(
             token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields)
         )
         if isinstance(token_request, ProxyException):
-            return _anthropic_error_json_response(token_request, request)
+            return anthropic_error_json_response(token_request, request)
 
         token_response: Final = await count_request_tokens(
             request=token_request,
@@ -419,12 +419,12 @@ async def count_tokens(
             ProxyException(message=str(e), type="invalid_request_error", param="messages", code=400), request
         )
     except HTTPException as e:
-        return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)
+        return anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)
     except ProxyException as e:
-        return _anthropic_error_json_response(e, request)
+        return anthropic_error_json_response(e, request)
     except Exception as e:
         log_llm_api_exception(e, litellm_call_id)
-        return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
+        return anthropic_error_json_response(proxy_exception_from_route_error(e), request)
     else:
         return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)}
 

litellm/proxy/proxy_server.py

@@ -829,6 +829,7 @@ except ImportError:
     build_billing_metrics_recorder = None
     shutdown_billing_metrics_recorder = None
 from fastapi.exception_handlers import http_exception_handler
+from starlette._utils import get_route_path
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.websockets import WebSocketState
 
@@ -2186,6 +2187,10 @@ class UserAPIKeyCacheTTLEnum(enum.Enum):
 async def openai_exception_handler(request: Request, exc: ProxyException):
     # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions
     log_model_access_denial(exc)
+    if get_route_path(request.scope) in ("/v1/messages", "/v1/messages/count_tokens"):
+        from litellm.proxy.anthropic_endpoints.endpoints import anthropic_error_json_response
+
+        return anthropic_error_json_response(exc, request)
     headers: Final = exc.headers
     error_dict: Final = with_call_id(
         JSON_OBJECT.validate_python(exc.to_dict()),

tests/unit/proxy/anthropic_endpoints/test_endpoints.py

@@ -187,14 +187,14 @@ class TestProxyExceptionAnthropicEnvelope:
         request = MagicMock()
         request.headers = {}
 
-        response = ep._anthropic_error_json_response(
+        response = ep.anthropic_error_json_response(
             ProxyException(message="Rate limit exceeded", type="rate_limit_error", param=None, code=429),
             request,
         )
         assert response.status_code == 429
         assert json.loads(response.body)["error"]["type"] == "rate_limit_error"
 
-        fallback = ep._anthropic_error_json_response(
+        fallback = ep.anthropic_error_json_response(
             ProxyException(message="boom", type="None", param=None, code=None),
             request,
         )
@@ -217,7 +217,7 @@ class TestProxyExceptionAnthropicEnvelope:
             provider_specific_fields=provider_specific_fields,
         )
         with patch("litellm.proxy.proxy_server.general_settings", general_settings):
-            return ep._anthropic_error_json_response(exc, request)
+            return ep.anthropic_error_json_response(exc, request)
 
     def test_anthropic_error_copies_the_call_id_into_the_error_when_opted_in(self):
         """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to

tests/unit/proxy/test_proxy_server.py

@@ -49,6 +49,113 @@ from litellm.proxy.proxy_server import (  # Replace with the actual module where
 )
 from litellm.proxy.utils import ProxyLogging
 
+
+@pytest.mark.parametrize("root_path", ["", "/gateway"])
+@pytest.mark.parametrize(
+    ("path", "anthropic_envelope"),
+    [
+        ("/v1/messages", True),
+        ("/v1/messages/count_tokens", True),
+        ("/v1/chat/completions", False),
+        ("/v1/responses", False),
+    ],
+)
+@pytest.mark.parametrize("failure", ["missing_key", "malformed_json"])
+async def test_pre_handler_errors_use_endpoint_envelope(
+    monkeypatch: pytest.MonkeyPatch,
+    root_path: str,
+    path: str,
+    anthropic_envelope: bool,
+    failure: str,
+) -> None:
+    import httpx2
+
+    from litellm.proxy import proxy_server
+
+    monkeypatch.setattr(proxy_server, "master_key", "sk-local-envelope-regression")
+    monkeypatch.setattr(proxy_server, "general_settings", {})
+    application: Final = FastAPI()
+    application.mount(root_path or "/", app)
+    headers: Final = {"Authorization": "Bearer sk-local-envelope-regression"} if failure == "malformed_json" else {}
+    payload: Final = b'{"model":' if failure == "malformed_json" else b'{"model":"local-only"}'
+    async with httpx2.AsyncClient(
+        transport=httpx2.ASGITransport(app=application),
+        base_url="http://testserver",
+    ) as client:
+        response: Final = await client.post(root_path + path, content=payload, headers=headers)
+
+    assert response.status_code == (400 if failure == "malformed_json" else 401), response.text
+    message: Final = response.json()["error"]["message"]
+    assert message.startswith(
+        "Invalid JSON payload:" if failure == "malformed_json" else "Authentication Error, No api key passed in."
+    )
+    expected: Final = (
+        {
+            "type": "error",
+            "error": {
+                "type": "invalid_request_error" if failure == "malformed_json" else "authentication_error",
+                "message": message,
+            },
+        }
+        if anthropic_envelope
+        else {
+            "error": {
+                "type": "invalid_request_error" if failure == "malformed_json" else "auth_error",
+                "message": message,
+                "param": "request_body" if failure == "malformed_json" else "None",
+                "code": "400" if failure == "malformed_json" else "401",
+            },
+        }
+    )
+    assert response.json() == expected
+
+
+@pytest.mark.parametrize("path", ["/v1/messages", "/v1/messages/count_tokens"])
+@pytest.mark.parametrize("include_call_id", [False, True])
+async def test_messages_global_error_preserves_status_and_diagnostics(
+    monkeypatch: pytest.MonkeyPatch,
+    path: str,
+    include_call_id: bool,
+) -> None:
+    from fastapi import Request
+
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import ProxyException
+
+    monkeypatch.setattr(proxy_server, "general_settings", {"include_call_id_in_error_body": include_call_id})
+    request: Final = Request(
+        {
+            "type": "http",
+            "method": "POST",
+            "path": path,
+            "headers": [(b"x-request-id", b"request-envelope")],
+        }
+    )
+    error: Final = ProxyException(
+        message="Model access denied",
+        type="user_model_access_denied",
+        param="model",
+        code=403,
+        headers={"x-litellm-call-id": "call-envelope", "retry-after": "12"},
+        provider_specific_fields={"model": "restricted-model"},
+    )
+    response: Final = await proxy_server.openai_exception_handler(request, error)
+
+    assert response.status_code == 403
+    assert response.headers["x-litellm-call-id"] == "call-envelope"
+    assert response.headers["retry-after"] == "12"
+    assert json.loads(response.body) == {
+        "type": "error",
+        "request_id": "request-envelope",
+        "error": {
+            "type": "permission_error",
+            "message": "Model access denied",
+            "provider_specific_fields": {"model": "restricted-model"},
+            **({"litellm_call_id": "call-envelope"} if include_call_id else {}),
+        },
+    }
+
+
 @pytest.fixture
 def admin_mcp_proxy(monkeypatch: pytest.MonkeyPatch) -> FastAPI:
     from litellm.proxy import proxy_server