mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
fix(mcp): map a DB outage during bridge key reload to a retryable 503
get_key_object's raw transport error propagated uncaught out of _reload_admitted_key as an opaque 500; classify it via the shared _raise_503_if_db_unavailable helper (also used by the live-policy gate) so a database outage is a retryable 503, while a key-not-found ProxyException stays the fail-closed 401.
This commit is contained in:
parent
ea64ef7a2a
commit
f5f03cbd63
2 changed files with 40 additions and 7 deletions
|
|
@ -581,11 +581,28 @@ class MCPRequestHandler:
|
|||
)
|
||||
except (ProxyException, HTTPException):
|
||||
raise HTTPException(status_code=401, detail="Invalid or expired credential") from None
|
||||
except Exception as e: # noqa: BLE001 # a DB outage during reload is a retryable 503, not an opaque 500
|
||||
MCPRequestHandler._raise_503_if_db_unavailable(e)
|
||||
raise
|
||||
if not MCPRequestHandler._admitted_key_is_active(key_object):
|
||||
raise HTTPException(status_code=401, detail="Invalid or expired credential")
|
||||
await MCPRequestHandler._reject_if_admitted_owner_scim_deactivated(key_object)
|
||||
return key_object
|
||||
|
||||
@staticmethod
|
||||
def _raise_503_if_db_unavailable(e: Exception) -> None:
|
||||
"""Raise a retryable 503 when ``e`` means the auth database is unreachable, else return so the
|
||||
caller applies its own fail-closed mapping. A DB outage must not masquerade as an auth failure
|
||||
(401) or surface as an opaque 500; the caller retries. Mirrors ``UserAPIKeyAuthExceptionHandler``,
|
||||
which renders a service-unavailable database error as 503 on the standard pipeline."""
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
|
||||
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.",
|
||||
) from None
|
||||
|
||||
@staticmethod
|
||||
async def _reject_if_admitted_owner_scim_deactivated(key_object: UserAPIKeyAuth) -> None:
|
||||
"""Fail closed with a 401 when the key's owning user was deactivated via SCIM.
|
||||
|
|
@ -636,8 +653,6 @@ class MCPRequestHandler:
|
|||
it told an over-budget but validly-authenticated caller their credential was invalid,
|
||||
which on a DCR client reads as broken auth and can trigger a pointless re-authorize
|
||||
loop that cannot fix a budget problem, and it masked a DB outage as an auth error."""
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
|
||||
try:
|
||||
await _run_centralized_common_checks(
|
||||
user_api_key_auth_obj=admitted,
|
||||
|
|
@ -650,11 +665,7 @@ class MCPRequestHandler:
|
|||
except litellm.BudgetExceededError as e:
|
||||
raise HTTPException(status_code=getattr(e, "status_code", 429), detail=str(e)) from None
|
||||
except Exception as e: # noqa: BLE001 # untyped gate failure: retryable 503 for a DB outage, else fail closed 401
|
||||
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.",
|
||||
) from None
|
||||
MCPRequestHandler._raise_503_if_db_unavailable(e)
|
||||
raise HTTPException(status_code=401, detail="Invalid or expired credential") from None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -5275,6 +5275,28 @@ class TestMCPDcrBridgeDelegateAdmission:
|
|||
mapped = await self._enforce_with_gate_error(ConnectionError("could not reach database server"))
|
||||
assert mapped.status_code == 503
|
||||
|
||||
async def test_db_outage_during_key_reload_surfaces_503_not_500(self):
|
||||
"""A DB outage while reloading the admitted key surfaces a retryable 503, not the opaque 500 a
|
||||
raw get_key_object transport error would otherwise propagate as, and not a 401 that masks the
|
||||
outage as an auth failure. Regression for the reload-path exception gap."""
|
||||
envelope = self._mint_bridge_envelope(key_hash=self._KEY_HASH)
|
||||
scope = {
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/mcp/bridge_delegate_server",
|
||||
"headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))],
|
||||
}
|
||||
with (
|
||||
patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr,
|
||||
patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY),
|
||||
self._patch_key_reload(side_effect=ConnectionError("could not reach database server")),
|
||||
):
|
||||
mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server()
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await MCPRequestHandler.process_mcp_request(scope)
|
||||
|
||||
assert exc_info.value.status_code == 503
|
||||
|
||||
async def test_blocked_state_bare_exception_stays_401(self):
|
||||
"""A blocked team/project raises a bare Exception (no status) in common_checks, which the
|
||||
standard pipeline renders as 401; the arm keeps failing those closed as 401, never a 500."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue