fix(auth): distinguish end-user lookup failures from missing end users

An exception while loading the end-user row was collapsed into None, the same
value returned for an end user that has no row, so the centralized auth path
dropped _check_end_user_budget entirely and a transient DB outage silently
disabled end-user budget enforcement.

Lookup failures now log a warning and, when fail_closed_budget_enforcement is
enabled, reject the request with 503 instead of admitting it on an unevaluated
budget; a genuinely missing end user still returns None and stays unbudgeted.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/auth/auth_checks.py

@@ -1114,6 +1114,37 @@ async def _check_end_user_budget(
         )
 
 
+def _handle_end_user_lookup_failure(end_user_id: str, error: Exception) -> None:
+    """An end-user row could not be loaded because the lookup itself failed.
+
+    This is not the same as "end user has no row": the budget recorded against
+    that end user is unknown, so returning ``None`` silently drops
+    ``_check_end_user_budget`` from the auth path. Reject instead when the
+    deployment opted into ``fail_closed_budget_enforcement``, matching how
+    unverifiable spend counters and unwritable budget reservations behave.
+    """
+    from litellm.proxy.proxy_server import general_settings
+
+    verbose_proxy_logger.warning(
+        "end-user lookup for %s failed (%s: %s); its budget cannot be enforced for this request",
+        end_user_id,
+        type(error).__name__,
+        error,
+    )
+    if general_settings.get("fail_closed_budget_enforcement") is True:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail={
+                "error": (
+                    "Budget enforcement unavailable: the end user object could not be loaded, "
+                    "so its budget could not be evaluated, and fail_closed_budget_enforcement "
+                    "is enabled, so the request was rejected. Retry shortly."
+                )
+            },
+        )
+    return None
+
+
 @log_db_metrics
 async def get_end_user_object(
     end_user_id: str | None,
@@ -1171,11 +1202,13 @@ async def get_end_user_object(
             where={"user_id": end_user_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
+    except Exception as e:
+        return _handle_end_user_lookup_failure(end_user_id=end_user_id, error=e)
 
-        if response is None:
-            raise Exception
+    if response is None:
+        return None
 
-        # Convert to LiteLLM_EndUserTable object
+    try:
         _response = LiteLLM_EndUserTable.model_validate(response.dict())
 
         # Apply default budget if needed
@@ -1185,18 +1218,19 @@ async def get_end_user_object(
             user_api_key_cache=user_api_key_cache,
             parent_otel_span=parent_otel_span,
         )
+    except Exception as e:
+        return _handle_end_user_lookup_failure(end_user_id=end_user_id, error=e)
 
-        # Save to cache
+    try:
         await user_api_key_cache.async_set_cache(
             key=f"end_user_id:{end_user_id}",
             value=_response,
             model_type=LiteLLM_EndUserTable,
         )
+    except Exception as e:
+        verbose_proxy_logger.debug("failed caching end user %s: %s", end_user_id, e)
 
-        return _response
-
-    except Exception:
-        return None
+    return _response
 
 
 _END_USER_VALIDATION_NEGATIVE_TTL = 60
@@ -1288,6 +1322,8 @@ async def _end_user_id_exists_in_db(
         )
         if end_user_obj is not None:
             return True
+    except HTTPException:
+        raise
     except Exception as e:
         verbose_proxy_logger.debug(f"end_user validation: get_end_user_object lookup failed: {e}")
 

litellm/proxy/auth/user_api_key_auth.py

@@ -1478,9 +1478,9 @@ async def _user_api_key_auth_builder(
                             budget_info=default_budget,
                             end_user_id=end_user_id,
                         )
+            except (HTTPException, ProxyException, litellm.BudgetExceededError):
+                raise
             except Exception as e:
-                if isinstance(e, litellm.BudgetExceededError):
-                    raise e
                 verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e!s}")
 
         ### CHECK IF ADMIN ###
@@ -2751,9 +2751,9 @@ async def _lookup_end_user_and_apply_budget(
                 valid_token = update_valid_token_with_end_user_params(
                     valid_token=valid_token, end_user_params=end_user_params
                 )
+    except (HTTPException, ProxyException, litellm.BudgetExceededError):
+        raise
     except Exception as e:
-        if isinstance(e, litellm.BudgetExceededError):
-            raise e
         verbose_proxy_logger.debug(f"Unable to find user in db. Error - {e!s}")
     return valid_token, end_user_object
 

tests/test_litellm/proxy/auth/test_auth_checks.py

@@ -5038,6 +5038,73 @@ async def test_get_end_user_object_db_fetch_returns_validated_end_user():
     assert result.spend == 3.0
 
 
+def _end_user_lookup_mocks(find_unique: AsyncMock):
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.litellm_endusertable.find_unique = find_unique
+
+    mock_cache = MagicMock()
+    mock_cache.async_get_cache = AsyncMock(return_value=None)
+    mock_cache.async_set_cache = AsyncMock()
+    return mock_prisma_client, mock_cache
+
+
+@pytest.mark.asyncio
+async def test_get_end_user_object_missing_row_is_not_a_lookup_failure(monkeypatch):
+    """A genuinely absent end user stays unbudgeted even under fail-closed enforcement."""
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.auth.auth_checks import get_end_user_object
+
+    monkeypatch.setattr(proxy_server, "general_settings", {"fail_closed_budget_enforcement": True})
+    mock_prisma_client, mock_cache = _end_user_lookup_mocks(AsyncMock(return_value=None))
+
+    result = await get_end_user_object(
+        end_user_id="eu-missing",
+        prisma_client=mock_prisma_client,
+        user_api_key_cache=mock_cache,
+    )
+
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_get_end_user_object_db_error_fails_closed(monkeypatch):
+    """#35529: a DB error must not look like 'no such end user', which silently
+    drops _check_end_user_budget from the auth path."""
+    import litellm.proxy.proxy_server as proxy_server
+    from fastapi import HTTPException
+
+    from litellm.proxy.auth.auth_checks import get_end_user_object
+
+    monkeypatch.setattr(proxy_server, "general_settings", {"fail_closed_budget_enforcement": True})
+    mock_prisma_client, mock_cache = _end_user_lookup_mocks(AsyncMock(side_effect=RuntimeError("database unavailable")))
+
+    with pytest.raises(HTTPException) as exc_info:
+        await get_end_user_object(
+            end_user_id="eu-1",
+            prisma_client=mock_prisma_client,
+            user_api_key_cache=mock_cache,
+        )
+
+    assert exc_info.value.status_code == 503
+
+
+@pytest.mark.asyncio
+async def test_get_end_user_object_db_error_defaults_to_open(monkeypatch):
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.proxy.auth.auth_checks import get_end_user_object
+
+    monkeypatch.setattr(proxy_server, "general_settings", {})
+    mock_prisma_client, mock_cache = _end_user_lookup_mocks(AsyncMock(side_effect=RuntimeError("database unavailable")))
+
+    result = await get_end_user_object(
+        end_user_id="eu-1",
+        prisma_client=mock_prisma_client,
+        user_api_key_cache=mock_cache,
+    )
+
+    assert result is None
+
+
 @pytest.mark.asyncio
 async def test_get_team_membership_db_fetch_returns_validated_membership():
     from litellm.proxy._types import LiteLLM_TeamMembership