This commit is contained in:
Jeevan Mohan Pawar 2026-10-25 16:44:36 +00:00 • committed by GitHub
commit 1fd139667b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 120 additions and 28 deletions

View file

@ -144,6 +144,7 @@ class UserAPIKeyAuthExceptionHandler:
parent_otel_span: Span | None,
api_key: str,
resolved_identity: UserAPIKeyAuth | None = None,
is_custom_auth_error: bool = False,
) -> UserAPIKeyAuth:
"""
Handles Connection Errors when reading a Virtual Key from LiteLLM DB
@ -166,6 +167,7 @@ class UserAPIKeyAuthExceptionHandler:
if (
PrismaDBExceptionHandler.should_allow_request_on_db_unavailable()
and not is_custom_auth_error
and PrismaDBExceptionHandler.is_database_connection_error(e)
):
# log this as a DB failure on prometheus

View file

@ -2123,6 +2123,7 @@ async def user_api_key_auth_builder(
route: Final[str] = get_request_route(request=request)
valid_token: UserAPIKeyAuth | None = None
custom_auth_api_key: bool = False
custom_auth_error: bool = False # rebind-ok: callback failure provenance must reach the shared error handler
try:
with tracer.trace("litellm.proxy.auth.pre_db_read_auth_checks"):
@ -2160,10 +2161,14 @@ async def user_api_key_auth_builder(
### USER-DEFINED AUTH FUNCTION ###
if enterprise_custom_auth is not None:
with tracer.trace("litellm.proxy.auth.enterprise_custom_auth"):
response = await enterprise_custom_auth(
request=request, api_key=api_key, user_custom_auth=user_custom_auth
)
try:
with tracer.trace("litellm.proxy.auth.enterprise_custom_auth"):
response = await enterprise_custom_auth(
request=request, api_key=api_key, user_custom_auth=user_custom_auth
)
except Exception:
custom_auth_error = True # rebind-ok: record callback failure for the shared handler
raise
if response is not None and isinstance(response, UserAPIKeyAuth):
validated = UserAPIKeyAuth.model_validate(response)
if getattr(litellm, "enable_post_custom_auth_checks", False):
@ -2180,7 +2185,11 @@ async def user_api_key_auth_builder(
api_key = response
custom_auth_api_key = True
elif user_custom_auth is not None:
response = await user_custom_auth(request=request, api_key=api_key)
try:
response = await user_custom_auth(request=request, api_key=api_key)
except Exception:
custom_auth_error = True # rebind-ok: record callback failure for the shared handler
raise
validated = UserAPIKeyAuth.model_validate(response)
if getattr(litellm, "enable_post_custom_auth_checks", False):
validated = await _run_post_custom_auth_checks(
@ -2831,6 +2840,7 @@ async def user_api_key_auth_builder(
parent_otel_span=parent_otel_span,
api_key=api_key,
resolved_identity=valid_token,
is_custom_auth_error=custom_auth_error,
)

View file

@ -113,15 +113,17 @@ class PrismaDBExceptionHandler:
Reporting decisions want the opposite breadth; use
``is_database_infrastructure_error`` for those.
"""
import prisma.engine.errors
if isinstance(e, (*DB_CONNECTION_ERROR_TYPES, DBLookupDeadlineExceeded)):
return True
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
return True
try:
import prisma.engine.errors
except ImportError:
return False
if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)):
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return True
return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection
return PrismaDBExceptionHandler.is_database_capacity_error(e)
@staticmethod
def is_database_infrastructure_error(e: Exception) -> bool:
@ -139,8 +141,14 @@ class PrismaDBExceptionHandler:
``RecordNotFoundError``, etc.) are excluded — the DB IS reachable and
the request itself is what failed.
"""
import prisma
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
return True
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
return True
try:
import prisma
except ImportError:
return False
data_layer_errors: Final = _exception_types(
prisma.errors.DataError,
prisma.errors.UniqueViolationError,
@ -152,12 +160,8 @@ class PrismaDBExceptionHandler:
)
if isinstance(e, data_layer_errors):
return False
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
return True
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
return True
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
return True
return False
@staticmethod
@ -179,8 +183,10 @@ class PrismaDBExceptionHandler:
per-row data rejection has to additionally consult
``is_database_service_unavailable_error`` before acting on a True here.
"""
import prisma
try:
import prisma
except ImportError:
return False
return type(e) is prisma.errors.DataError
@staticmethod
@ -192,10 +198,14 @@ class PrismaDBExceptionHandler:
Use this for reconnect logic — data-layer errors like UniqueViolationError
mean the DB IS reachable, so reconnecting would be pointless.
"""
import prisma
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
return True
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
return True
try:
import prisma
except ImportError:
return False
if isinstance(
e,
_exception_types(
@ -224,21 +234,24 @@ class PrismaDBExceptionHandler:
)
if any(keyword in error_message for keyword in connection_keywords):
return True
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
return True
return False
@staticmethod
def is_prisma_error(e: Exception) -> bool:
import prisma
try:
import prisma
except ImportError:
return False
return isinstance(e, _exception_types(prisma.errors.PrismaError))
@staticmethod
def is_deadlock_error(e: Exception) -> bool:
"""True iff ``e`` is a Postgres deadlock (P2034 / 40P01) surfaced through prisma."""
import prisma
try:
import prisma
except ImportError:
return False
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
if getattr(e, "code", None) == "P2034":
@ -258,7 +271,10 @@ class PrismaDBExceptionHandler:
but full, so the failure is neither a transport error (which would tear
the engine down and open yet more connections against it) nor a
rejection of the rows being written."""
import prisma
try:
import prisma
except ImportError:
return False
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
@ -328,8 +344,10 @@ class PrismaDBExceptionHandler:
are already classified by type/keyword above, and data-layer ones
(the DB IS reachable) must stay 401.
"""
import prisma
try:
import prisma
except ImportError:
return False
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
tb = e.__traceback__ if hasattr(e, "__traceback__") else None

View file

@ -666,6 +666,39 @@ async def test_user_custom_auth_skips_post_custom_auth_checks_by_default():
litellm.enable_post_custom_auth_checks = original_flag
@pytest.mark.asyncio
@pytest.mark.parametrize("enterprise", [False, True])
async def test_user_custom_auth_transport_error_does_not_use_db_fallback(
monkeypatch: pytest.MonkeyPatch, enterprise: bool
) -> None:
from fastapi import Request
import httpx
import litellm.proxy.proxy_server as proxy_server
import litellm.proxy.auth.user_api_key_auth as auth_module
custom_auth: Final = AsyncMock(side_effect=httpx.ConnectError("auth service unavailable"))
for name, value in _proxy_server_attrs_for_custom_auth(user_custom_auth=None if enterprise else custom_auth).items():
monkeypatch.setattr(proxy_server, name, value)
monkeypatch.setattr(proxy_server, "general_settings", {"allow_requests_on_db_unavailable": True})
monkeypatch.setattr(auth_module, "enterprise_custom_auth", custom_auth if enterprise else None)
request: Final = Request(scope={"type": "http", "path": "/chat/completions", "headers": []})
with pytest.raises(ProxyException) as exc_info:
await user_api_key_auth_builder(
request=request,
api_key="Bearer sk-custom-auth-unavailable",
azure_api_key_header="",
anthropic_api_key_header=None,
google_ai_studio_api_key_header=None,
azure_apim_header=None,
request_data={},
)
assert exc_info.value.type == ProxyErrorTypes.no_db_connection
assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE
custom_auth.assert_awaited_once()
@pytest.mark.asyncio
async def test_user_custom_auth_runs_post_custom_auth_checks_when_opt_in():
"""

View file

@ -64,6 +64,18 @@ def test_is_database_transport_error_non_connection_prisma_errors(prisma_error):
assert PrismaDBExceptionHandler.is_database_transport_error(prisma_error) == False
@pytest.mark.parametrize(
"transport_error",
[
httpx.ConnectError("connection refused"),
ClientNotConnectedError(),
HTTPClientClosedError(),
],
)
def test_is_database_transport_error_connection_errors(transport_error):
assert PrismaDBExceptionHandler.is_database_transport_error(transport_error) is True
def test_is_database_connection_generic_errors():
"""
Test non-Prisma error cases for database connection checking
@ -77,12 +89,29 @@ def test_is_database_connection_generic_errors():
param="test-param",
)
assert PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception) == True
assert PrismaDBExceptionHandler.is_database_infrastructure_error(db_proxy_exception)
assert PrismaDBExceptionHandler.is_database_transport_error(db_proxy_exception)
# Test with non-DB error
regular_exception = Exception("Regular error")
assert PrismaDBExceptionHandler.is_database_connection_error(regular_exception) == False
def test_db_less_proxy_auth_error_does_not_require_prisma() -> None:
"""A missing Prisma dependency must not mask a normal authentication error."""
with patch.dict(sys.modules, {"prisma": None}):
error: Final = Exception("No api key passed in.")
assert PrismaDBExceptionHandler.is_database_connection_error(error) is False
assert PrismaDBExceptionHandler.is_database_infrastructure_error(error) is False
assert PrismaDBExceptionHandler.is_prisma_data_error(error) is False
assert PrismaDBExceptionHandler.is_prisma_error(error) is False
assert PrismaDBExceptionHandler.is_database_transport_error(error) is False
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
assert PrismaDBExceptionHandler.is_deadlock_error(error) is False
assert PrismaDBExceptionHandler.is_prisma_engine_internal_error(error) is False
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False
@pytest.mark.parametrize(
"error",
[