From f5f03cbd635456f48eb72a65567b0e90596bbb97 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 11 Jul 2026 12:46:09 -0700 Subject: [PATCH] fix(mcp): map a DB outage during bridge key reload to a retryable 503 get_key_object's raw transport error propagated uncaught out of _reload_admitted_key as an opaque 500; classify it via the shared _raise_503_if_db_unavailable helper (also used by the live-policy gate) so a database outage is a retryable 503, while a key-not-found ProxyException stays the fail-closed 401. --- .../mcp_server/auth/user_api_key_auth_mcp.py | 25 +++++++++++++------ .../auth/test_user_api_key_auth_mcp.py | 22 ++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index ddeab918fcc..e785b6b8da4 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -581,11 +581,28 @@ class MCPRequestHandler: ) except (ProxyException, HTTPException): raise HTTPException(status_code=401, detail="Invalid or expired credential") from None + except Exception as e: # noqa: BLE001 # a DB outage during reload is a retryable 503, not an opaque 500 + MCPRequestHandler._raise_503_if_db_unavailable(e) + raise if not MCPRequestHandler._admitted_key_is_active(key_object): raise HTTPException(status_code=401, detail="Invalid or expired credential") await MCPRequestHandler._reject_if_admitted_owner_scim_deactivated(key_object) return key_object + @staticmethod + def _raise_503_if_db_unavailable(e: Exception) -> None: + """Raise a retryable 503 when ``e`` means the auth database is unreachable, else return so the + caller applies its own fail-closed mapping. A DB outage must not masquerade as an auth failure + (401) or surface as an opaque 500; the caller retries. Mirrors ``UserAPIKeyAuthExceptionHandler``, + which renders a service-unavailable database error as 503 on the standard pipeline.""" + from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler + + if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): + raise HTTPException( + status_code=503, + detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + ) from None + @staticmethod async def _reject_if_admitted_owner_scim_deactivated(key_object: UserAPIKeyAuth) -> None: """Fail closed with a 401 when the key's owning user was deactivated via SCIM. @@ -636,8 +653,6 @@ class MCPRequestHandler: it told an over-budget but validly-authenticated caller their credential was invalid, which on a DCR client reads as broken auth and can trigger a pointless re-authorize loop that cannot fix a budget problem, and it masked a DB outage as an auth error.""" - from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler - try: await _run_centralized_common_checks( user_api_key_auth_obj=admitted, @@ -650,11 +665,7 @@ class MCPRequestHandler: except litellm.BudgetExceededError as e: raise HTTPException(status_code=getattr(e, "status_code", 429), detail=str(e)) from None except Exception as e: # noqa: BLE001 # untyped gate failure: retryable 503 for a DB outage, else fail closed 401 - if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - raise HTTPException( - status_code=503, - detail="Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", - ) from None + MCPRequestHandler._raise_503_if_db_unavailable(e) raise HTTPException(status_code=401, detail="Invalid or expired credential") from None @staticmethod diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index f9234cecde4..da3e3250c59 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -5275,6 +5275,28 @@ class TestMCPDcrBridgeDelegateAdmission: mapped = await self._enforce_with_gate_error(ConnectionError("could not reach database server")) assert mapped.status_code == 503 + async def test_db_outage_during_key_reload_surfaces_503_not_500(self): + """A DB outage while reloading the admitted key surfaces a retryable 503, not the opaque 500 a + raw get_key_object transport error would otherwise propagate as, and not a 401 that masks the + outage as an auth failure. Regression for the reload-path exception gap.""" + envelope = self._mint_bridge_envelope(key_hash=self._KEY_HASH) + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/bridge_delegate_server", + "headers": [(b"authorization", f"Bearer {envelope}".encode("latin-1"))], + } + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), + self._patch_key_reload(side_effect=ConnectionError("could not reach database server")), + ): + mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server() + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(scope) + + assert exc_info.value.status_code == 503 + async def test_blocked_state_bare_exception_stays_401(self): """A blocked team/project raises a bare Exception (no status) in common_checks, which the standard pipeline renders as 401; the arm keeps failing those closed as 401, never a 500."""