From 9642a439fdc61205052bc94925fcf2d13aacd4cc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:14:51 +0000 Subject: [PATCH] fix(auth): distinguish end-user lookup failures from missing end users An exception while loading the end-user row was collapsed into None, the same value returned for an end user that has no row, so the centralized auth path dropped _check_end_user_budget entirely and a transient DB outage silently disabled end-user budget enforcement. Lookup failures now log a warning and, when fail_closed_budget_enforcement is enabled, reject the request with 503 instead of admitting it on an unevaluated budget; a genuinely missing end user still returns None and stays unbudgeted. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 52 +++++++++++--- litellm/proxy/auth/user_api_key_auth.py | 8 +-- .../proxy/auth/test_auth_checks.py | 67 +++++++++++++++++++ 3 files changed, 115 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f876b303510..4dc9db6796e 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1114,6 +1114,37 @@ async def _check_end_user_budget( ) +def _handle_end_user_lookup_failure(end_user_id: str, error: Exception) -> None: + """An end-user row could not be loaded because the lookup itself failed. + + This is not the same as "end user has no row": the budget recorded against + that end user is unknown, so returning ``None`` silently drops + ``_check_end_user_budget`` from the auth path. Reject instead when the + deployment opted into ``fail_closed_budget_enforcement``, matching how + unverifiable spend counters and unwritable budget reservations behave. + """ + from litellm.proxy.proxy_server import general_settings + + verbose_proxy_logger.warning( + "end-user lookup for %s failed (%s: %s); its budget cannot be enforced for this request", + end_user_id, + type(error).__name__, + error, + ) + if general_settings.get("fail_closed_budget_enforcement") is True: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={ + "error": ( + "Budget enforcement unavailable: the end user object could not be loaded, " + "so its budget could not be evaluated, and fail_closed_budget_enforcement " + "is enabled, so the request was rejected. Retry shortly." + ) + }, + ) + return None + + @log_db_metrics async def get_end_user_object( end_user_id: str | None, @@ -1171,11 +1202,13 @@ async def get_end_user_object( where={"user_id": end_user_id}, include={"litellm_budget_table": True, "object_permission": True}, ) + except Exception as e: + return _handle_end_user_lookup_failure(end_user_id=end_user_id, error=e) - if response is None: - raise Exception + if response is None: + return None - # Convert to LiteLLM_EndUserTable object + try: _response = LiteLLM_EndUserTable.model_validate(response.dict()) # Apply default budget if needed @@ -1185,18 +1218,19 @@ async def get_end_user_object( user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, ) + except Exception as e: + return _handle_end_user_lookup_failure(end_user_id=end_user_id, error=e) - # Save to cache + try: await user_api_key_cache.async_set_cache( key=f"end_user_id:{end_user_id}", value=_response, model_type=LiteLLM_EndUserTable, ) + except Exception as e: + verbose_proxy_logger.debug("failed caching end user %s: %s", end_user_id, e) - return _response - - except Exception: - return None + return _response _END_USER_VALIDATION_NEGATIVE_TTL = 60 @@ -1288,6 +1322,8 @@ async def _end_user_id_exists_in_db( ) if end_user_obj is not None: return True + except HTTPException: + raise except Exception as e: verbose_proxy_logger.debug(f"end_user validation: get_end_user_object lookup failed: {e}") diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 72450453174..0ee402fda40 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1478,9 +1478,9 @@ async def _user_api_key_auth_builder( budget_info=default_budget, end_user_id=end_user_id, ) + except (HTTPException, ProxyException, litellm.BudgetExceededError): + raise except Exception as e: - if isinstance(e, litellm.BudgetExceededError): - raise e verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e!s}") ### CHECK IF ADMIN ### @@ -2751,9 +2751,9 @@ async def _lookup_end_user_and_apply_budget( valid_token = update_valid_token_with_end_user_params( valid_token=valid_token, end_user_params=end_user_params ) + except (HTTPException, ProxyException, litellm.BudgetExceededError): + raise except Exception as e: - if isinstance(e, litellm.BudgetExceededError): - raise e verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e!s}") return valid_token, end_user_object diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 5f3b0f36b95..f07b0c7fd54 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -5038,6 +5038,73 @@ async def test_get_end_user_object_db_fetch_returns_validated_end_user(): assert result.spend == 3.0 +def _end_user_lookup_mocks(find_unique: AsyncMock): + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_endusertable.find_unique = find_unique + + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_set_cache = AsyncMock() + return mock_prisma_client, mock_cache + + +@pytest.mark.asyncio +async def test_get_end_user_object_missing_row_is_not_a_lookup_failure(monkeypatch): + """A genuinely absent end user stays unbudgeted even under fail-closed enforcement.""" + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.auth.auth_checks import get_end_user_object + + monkeypatch.setattr(proxy_server, "general_settings", {"fail_closed_budget_enforcement": True}) + mock_prisma_client, mock_cache = _end_user_lookup_mocks(AsyncMock(return_value=None)) + + result = await get_end_user_object( + end_user_id="eu-missing", + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_get_end_user_object_db_error_fails_closed(monkeypatch): + """#35529: a DB error must not look like 'no such end user', which silently + drops _check_end_user_budget from the auth path.""" + import litellm.proxy.proxy_server as proxy_server + from fastapi import HTTPException + + from litellm.proxy.auth.auth_checks import get_end_user_object + + monkeypatch.setattr(proxy_server, "general_settings", {"fail_closed_budget_enforcement": True}) + mock_prisma_client, mock_cache = _end_user_lookup_mocks(AsyncMock(side_effect=RuntimeError("database unavailable"))) + + with pytest.raises(HTTPException) as exc_info: + await get_end_user_object( + end_user_id="eu-1", + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + + assert exc_info.value.status_code == 503 + + +@pytest.mark.asyncio +async def test_get_end_user_object_db_error_defaults_to_open(monkeypatch): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.auth.auth_checks import get_end_user_object + + monkeypatch.setattr(proxy_server, "general_settings", {}) + mock_prisma_client, mock_cache = _end_user_lookup_mocks(AsyncMock(side_effect=RuntimeError("database unavailable"))) + + result = await get_end_user_object( + end_user_id="eu-1", + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + + assert result is None + + @pytest.mark.asyncio async def test_get_team_membership_db_fetch_returns_validated_membership(): from litellm.proxy._types import LiteLLM_TeamMembership