diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b1e8bdbe787..5862dbef712 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2759,6 +2759,23 @@ async def user_api_key_auth_builder( if ( prisma_client is None ): # if both master key + user key submitted, and user key != master key, and no db connected, raise an error + if ( + not PrismaDBExceptionHandler.is_database_configured() + and PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() + ): + # No database was ever configured, so there is no outage to + # ride out. ``allow_requests_on_db_unavailable`` must not fail + # open here: a non-master key cannot be verified and is + # rejected, instead of being admitted via the + # ``no_db_connection`` fallback token. See #47119. When the + # flag is off no fallback token is issued anyway, so the + # established ``no_db_connection`` error below stands. + raise ProxyException( + message="Authentication Error - No database configured. Virtual keys require a configured database; only the master key can be used without one.", + type=ProxyErrorTypes.auth_error, + code=status.HTTP_401_UNAUTHORIZED, + param=None, + ) raise ProxyException( message="No connected db.", type=ProxyErrorTypes.no_db_connection, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 5dfcaa4b2f8..75c85815915 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,4 +1,6 @@ +import os import re +import sys from collections.abc import Awaitable, Callable, Iterator from http import HTTPStatus from typing import Final, Protocol, TypeVar @@ -24,6 +26,7 @@ _TRANSIENT_DB_UNAVAILABLE_MESSAGE: Final = ( ) _DATABASE_ERROR_META: Final = TypeAdapter(dict[str, object]) +_GENERAL_SETTINGS_ADAPTER: Final = TypeAdapter(dict[str, object]) _BATCH_POSTGRES_ERROR_CODE: Final = re.compile(r'PostgresError \{ code: "([0-9A-Z]{5})"') _CONNECTION_CAPACITY_PHRASES: Final = ( "too many clients already", @@ -91,6 +94,39 @@ class PrismaDBExceptionHandler: return True return False + @staticmethod + def is_database_configured() -> bool: + """True when a database was configured for this proxy. + + A database is configured when ``DATABASE_URL`` (or ``DIRECT_URL``) + is present in the environment, or a ``database_url`` is present in + ``general_settings``. The CLI / entrypoints assemble ``DATABASE_URL`` + into ``os.environ`` from discrete ``DATABASE_*`` settings before the + proxy starts, so the environment is the canonical signal by request + time. When neither is present there is no database to be + ``unavailable``: that state never resolves by itself and must not be + treated as a transient outage by + ``allow_requests_on_db_unavailable``. + """ + if os.getenv("DATABASE_URL") or os.getenv("DIRECT_URL"): + return True + # Read ``general_settings`` through ``sys.modules``: importing + # ``proxy_server`` here could fail or recurse while it is still + # being imported, and a proxy server that has not been imported + # yet has not configured a database through it either. The raw + # value is ``object`` until validated into a typed dict, because + # ``general_settings`` is declared as a bare ``dict`` and reading + # it directly surfaces as partially unknown to basedpyright. + proxy_server_module: Final = sys.modules.get("litellm.proxy.proxy_server") + raw_settings: Final[object] = getattr(proxy_server_module, "general_settings", None) + if not isinstance(raw_settings, dict): + return False + try: + settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(raw_settings) + except ValidationError: + return False + return bool(settings.get("database_url")) + @staticmethod def is_database_connection_error(e: Exception) -> bool: """True only for a database that is temporarily unreachable and is diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 60a8dd73a34..3e370bd7229 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -476,9 +476,10 @@ def test_is_api_route_allowed(route, user_role, expected_result): @pytest.mark.asyncio -async def test_auth_not_connected_to_db(): +async def test_auth_not_connected_to_db(monkeypatch: pytest.MonkeyPatch): """ - ensure requests don't fail when `prisma_client` = None + ensure requests don't fail when `prisma_client` = None but a database + IS configured (configured-but-unreachable / transient outage) """ from fastapi import Request from starlette.datastructures import URL @@ -486,6 +487,7 @@ async def test_auth_not_connected_to_db(): from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import hash_token, user_api_key_cache + monkeypatch.setenv("DATABASE_URL", "postgresql://user:pass@localhost:5432/litellm") user_key = "sk-98765678" setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) @@ -506,6 +508,71 @@ async def test_auth_not_connected_to_db(): assert valid_token.token == "failed-to-connect-to-db" +@pytest.mark.asyncio +async def test_auth_no_db_configured_rejects_non_master_key(monkeypatch: pytest.MonkeyPatch): + """Regression for #47119: with no database configured, + `allow_requests_on_db_unavailable` must not accept any API key. + + There is no outage to ride out, so a non-master key is rejected (401) + instead of receiving the DB-unavailable fallback token. + """ + from fastapi import Request + from starlette.datastructures import URL + + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import user_api_key_cache + + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.delenv("DIRECT_URL", raising=False) + + setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + setattr(litellm.proxy.proxy_server, "prisma_client", None) + setattr( + litellm.proxy.proxy_server, + "general_settings", + {"allow_requests_on_db_unavailable": True}, + ) + + request = Request(scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []}) + request._url = URL(url="/chat/completions") + + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth(request=request, api_key="Bearer sk-anything-1234") + + assert str(exc_info.value.code) == "401" + + +@pytest.mark.asyncio +async def test_auth_no_db_configured_master_key_still_works(monkeypatch: pytest.MonkeyPatch): + """With no database configured, the master key still authenticates.""" + from fastapi import Request + from starlette.datastructures import URL + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import user_api_key_cache + + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.delenv("DIRECT_URL", raising=False) + + setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + setattr(litellm.proxy.proxy_server, "prisma_client", None) + setattr( + litellm.proxy.proxy_server, + "general_settings", + {"allow_requests_on_db_unavailable": True}, + ) + + request = Request(scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []}) + request._url = URL(url="/chat/completions") + + valid_token = await user_api_key_auth(request=request, api_key="Bearer " + MASTER_KEY) + assert valid_token is not None + assert valid_token.key_name != "failed-to-connect-to-db" + + def _assert_api_key_from_custom_header(headers, custom_header_name, expected_api_key): verbose_proxy_logger.setLevel(logging.DEBUG) request = MagicMock(spec=Request) diff --git a/tests/unit/proxy/db/test_exception_handler.py b/tests/unit/proxy/db/test_exception_handler.py index 75346ca08a6..a0391781cd5 100644 --- a/tests/unit/proxy/db/test_exception_handler.py +++ b/tests/unit/proxy/db/test_exception_handler.py @@ -951,3 +951,22 @@ def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeo ) def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None: assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False + + +def test_is_database_configured(monkeypatch): + """No DATABASE_URL/DIRECT_URL and no general_settings database_url + means no database was configured (#47119); setting DATABASE_URL means + a database is configured even if it is currently unreachable.""" + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.delenv("DIRECT_URL", raising=False) + with patch("litellm.proxy.proxy_server.general_settings", {}): + assert PrismaDBExceptionHandler.is_database_configured() is False + monkeypatch.setenv("DATABASE_URL", "postgresql://user:pass@localhost:5432/litellm") + with patch("litellm.proxy.proxy_server.general_settings", {}): + assert PrismaDBExceptionHandler.is_database_configured() is True + monkeypatch.delenv("DATABASE_URL", raising=False) + with patch( + "litellm.proxy.proxy_server.general_settings", + {"database_url": "postgresql://localhost:5432/litellm"}, + ): + assert PrismaDBExceptionHandler.is_database_configured() is True