mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
Merge 8a7348e784 into dddbaa803d
This commit is contained in:
commit
1fd139667b
5 changed files with 120 additions and 28 deletions
|
|
@ -144,6 +144,7 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
parent_otel_span: Span | None,
|
||||
api_key: str,
|
||||
resolved_identity: UserAPIKeyAuth | None = None,
|
||||
is_custom_auth_error: bool = False,
|
||||
) -> UserAPIKeyAuth:
|
||||
"""
|
||||
Handles Connection Errors when reading a Virtual Key from LiteLLM DB
|
||||
|
|
@ -166,6 +167,7 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
|
||||
if (
|
||||
PrismaDBExceptionHandler.should_allow_request_on_db_unavailable()
|
||||
and not is_custom_auth_error
|
||||
and PrismaDBExceptionHandler.is_database_connection_error(e)
|
||||
):
|
||||
# log this as a DB failure on prometheus
|
||||
|
|
|
|||
|
|
@ -2123,6 +2123,7 @@ async def user_api_key_auth_builder(
|
|||
route: Final[str] = get_request_route(request=request)
|
||||
valid_token: UserAPIKeyAuth | None = None
|
||||
custom_auth_api_key: bool = False
|
||||
custom_auth_error: bool = False # rebind-ok: callback failure provenance must reach the shared error handler
|
||||
|
||||
try:
|
||||
with tracer.trace("litellm.proxy.auth.pre_db_read_auth_checks"):
|
||||
|
|
@ -2160,10 +2161,14 @@ async def user_api_key_auth_builder(
|
|||
|
||||
### USER-DEFINED AUTH FUNCTION ###
|
||||
if enterprise_custom_auth is not None:
|
||||
with tracer.trace("litellm.proxy.auth.enterprise_custom_auth"):
|
||||
response = await enterprise_custom_auth(
|
||||
request=request, api_key=api_key, user_custom_auth=user_custom_auth
|
||||
)
|
||||
try:
|
||||
with tracer.trace("litellm.proxy.auth.enterprise_custom_auth"):
|
||||
response = await enterprise_custom_auth(
|
||||
request=request, api_key=api_key, user_custom_auth=user_custom_auth
|
||||
)
|
||||
except Exception:
|
||||
custom_auth_error = True # rebind-ok: record callback failure for the shared handler
|
||||
raise
|
||||
if response is not None and isinstance(response, UserAPIKeyAuth):
|
||||
validated = UserAPIKeyAuth.model_validate(response)
|
||||
if getattr(litellm, "enable_post_custom_auth_checks", False):
|
||||
|
|
@ -2180,7 +2185,11 @@ async def user_api_key_auth_builder(
|
|||
api_key = response
|
||||
custom_auth_api_key = True
|
||||
elif user_custom_auth is not None:
|
||||
response = await user_custom_auth(request=request, api_key=api_key)
|
||||
try:
|
||||
response = await user_custom_auth(request=request, api_key=api_key)
|
||||
except Exception:
|
||||
custom_auth_error = True # rebind-ok: record callback failure for the shared handler
|
||||
raise
|
||||
validated = UserAPIKeyAuth.model_validate(response)
|
||||
if getattr(litellm, "enable_post_custom_auth_checks", False):
|
||||
validated = await _run_post_custom_auth_checks(
|
||||
|
|
@ -2831,6 +2840,7 @@ async def user_api_key_auth_builder(
|
|||
parent_otel_span=parent_otel_span,
|
||||
api_key=api_key,
|
||||
resolved_identity=valid_token,
|
||||
is_custom_auth_error=custom_auth_error,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -113,15 +113,17 @@ class PrismaDBExceptionHandler:
|
|||
Reporting decisions want the opposite breadth; use
|
||||
``is_database_infrastructure_error`` for those.
|
||||
"""
|
||||
import prisma.engine.errors
|
||||
|
||||
if isinstance(e, (*DB_CONNECTION_ERROR_TYPES, DBLookupDeadlineExceeded)):
|
||||
return True
|
||||
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
|
||||
return True
|
||||
try:
|
||||
import prisma.engine.errors
|
||||
except ImportError:
|
||||
return False
|
||||
if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.is_database_capacity_error(e):
|
||||
return True
|
||||
return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection
|
||||
return PrismaDBExceptionHandler.is_database_capacity_error(e)
|
||||
|
||||
@staticmethod
|
||||
def is_database_infrastructure_error(e: Exception) -> bool:
|
||||
|
|
@ -139,8 +141,14 @@ class PrismaDBExceptionHandler:
|
|||
``RecordNotFoundError``, etc.) are excluded — the DB IS reachable and
|
||||
the request itself is what failed.
|
||||
"""
|
||||
import prisma
|
||||
|
||||
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
||||
return True
|
||||
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
|
||||
return True
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
data_layer_errors: Final = _exception_types(
|
||||
prisma.errors.DataError,
|
||||
prisma.errors.UniqueViolationError,
|
||||
|
|
@ -152,12 +160,8 @@ class PrismaDBExceptionHandler:
|
|||
)
|
||||
if isinstance(e, data_layer_errors):
|
||||
return False
|
||||
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
||||
return True
|
||||
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return True
|
||||
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -179,8 +183,10 @@ class PrismaDBExceptionHandler:
|
|||
per-row data rejection has to additionally consult
|
||||
``is_database_service_unavailable_error`` before acting on a True here.
|
||||
"""
|
||||
import prisma
|
||||
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
return type(e) is prisma.errors.DataError
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -192,10 +198,14 @@ class PrismaDBExceptionHandler:
|
|||
Use this for reconnect logic — data-layer errors like UniqueViolationError
|
||||
mean the DB IS reachable, so reconnecting would be pointless.
|
||||
"""
|
||||
import prisma
|
||||
|
||||
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
||||
return True
|
||||
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
|
||||
return True
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
if isinstance(
|
||||
e,
|
||||
_exception_types(
|
||||
|
|
@ -224,21 +234,24 @@ class PrismaDBExceptionHandler:
|
|||
)
|
||||
if any(keyword in error_message for keyword in connection_keywords):
|
||||
return True
|
||||
if isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_prisma_error(e: Exception) -> bool:
|
||||
import prisma
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
return isinstance(e, _exception_types(prisma.errors.PrismaError))
|
||||
|
||||
@staticmethod
|
||||
def is_deadlock_error(e: Exception) -> bool:
|
||||
"""True iff ``e`` is a Postgres deadlock (P2034 / 40P01) surfaced through prisma."""
|
||||
import prisma
|
||||
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return False
|
||||
if getattr(e, "code", None) == "P2034":
|
||||
|
|
@ -258,7 +271,10 @@ class PrismaDBExceptionHandler:
|
|||
but full, so the failure is neither a transport error (which would tear
|
||||
the engine down and open yet more connections against it) nor a
|
||||
rejection of the rows being written."""
|
||||
import prisma
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return False
|
||||
|
|
@ -328,8 +344,10 @@ class PrismaDBExceptionHandler:
|
|||
are already classified by type/keyword above, and data-layer ones
|
||||
(the DB IS reachable) must stay 401.
|
||||
"""
|
||||
import prisma
|
||||
|
||||
try:
|
||||
import prisma
|
||||
except ImportError:
|
||||
return False
|
||||
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return False
|
||||
tb = e.__traceback__ if hasattr(e, "__traceback__") else None
|
||||
|
|
|
|||
|
|
@ -666,6 +666,39 @@ async def test_user_custom_auth_skips_post_custom_auth_checks_by_default():
|
|||
litellm.enable_post_custom_auth_checks = original_flag
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enterprise", [False, True])
|
||||
async def test_user_custom_auth_transport_error_does_not_use_db_fallback(
|
||||
monkeypatch: pytest.MonkeyPatch, enterprise: bool
|
||||
) -> None:
|
||||
from fastapi import Request
|
||||
import httpx
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
import litellm.proxy.auth.user_api_key_auth as auth_module
|
||||
|
||||
custom_auth: Final = AsyncMock(side_effect=httpx.ConnectError("auth service unavailable"))
|
||||
for name, value in _proxy_server_attrs_for_custom_auth(user_custom_auth=None if enterprise else custom_auth).items():
|
||||
monkeypatch.setattr(proxy_server, name, value)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"allow_requests_on_db_unavailable": True})
|
||||
monkeypatch.setattr(auth_module, "enterprise_custom_auth", custom_auth if enterprise else None)
|
||||
request: Final = Request(scope={"type": "http", "path": "/chat/completions", "headers": []})
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await user_api_key_auth_builder(
|
||||
request=request,
|
||||
api_key="Bearer sk-custom-auth-unavailable",
|
||||
azure_api_key_header="",
|
||||
anthropic_api_key_header=None,
|
||||
google_ai_studio_api_key_header=None,
|
||||
azure_apim_header=None,
|
||||
request_data={},
|
||||
)
|
||||
|
||||
assert exc_info.value.type == ProxyErrorTypes.no_db_connection
|
||||
assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
custom_auth.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_custom_auth_runs_post_custom_auth_checks_when_opt_in():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -64,6 +64,18 @@ def test_is_database_transport_error_non_connection_prisma_errors(prisma_error):
|
|||
assert PrismaDBExceptionHandler.is_database_transport_error(prisma_error) == False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"transport_error",
|
||||
[
|
||||
httpx.ConnectError("connection refused"),
|
||||
ClientNotConnectedError(),
|
||||
HTTPClientClosedError(),
|
||||
],
|
||||
)
|
||||
def test_is_database_transport_error_connection_errors(transport_error):
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(transport_error) is True
|
||||
|
||||
|
||||
def test_is_database_connection_generic_errors():
|
||||
"""
|
||||
Test non-Prisma error cases for database connection checking
|
||||
|
|
@ -77,12 +89,29 @@ def test_is_database_connection_generic_errors():
|
|||
param="test-param",
|
||||
)
|
||||
assert PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception) == True
|
||||
assert PrismaDBExceptionHandler.is_database_infrastructure_error(db_proxy_exception)
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(db_proxy_exception)
|
||||
|
||||
# Test with non-DB error
|
||||
regular_exception = Exception("Regular error")
|
||||
assert PrismaDBExceptionHandler.is_database_connection_error(regular_exception) == False
|
||||
|
||||
|
||||
def test_db_less_proxy_auth_error_does_not_require_prisma() -> None:
|
||||
"""A missing Prisma dependency must not mask a normal authentication error."""
|
||||
with patch.dict(sys.modules, {"prisma": None}):
|
||||
error: Final = Exception("No api key passed in.")
|
||||
assert PrismaDBExceptionHandler.is_database_connection_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_database_infrastructure_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_prisma_data_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_prisma_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_deadlock_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_prisma_engine_internal_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue