From 52f3ff13f0a7da9f5a2cbe7333e9b7dcd8643780 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Thu, 6 Aug 2026 13:26:11 -0700 Subject: [PATCH 1/2] feat(proxy): limit repeated failed Admin UI sign-in attempts The Admin UI sign-in endpoints accept an unbounded number of password attempts. All three call authenticate_user, and none of them keeps any record of how many times a given caller has already been refused, so a misbehaving or misconfigured client can retry indefinitely at full speed. A LoginThrottle is now a required argument to authenticate_user, so the accounting lives at the one function all three endpoints share and a fourth endpoint cannot be added without deciding what to pass. Failures are counted per username and source address over a fixed window and further attempts are refused with 429 and a Retry-After header. The check runs before the database lookup and before the password comparison, so a refused caller does no further work. Only genuine credential rejections count. Configuration errors do not, a refused attempt does not extend the window, and a successful sign-in clears the bucket. The username is case folded because the user lookup is case insensitive, so casing cannot multiply the allowance. Both credential rejections now return one identical message. SSO is unaffected; it never calls this function. max_failed_login_attempts (10) and failed_login_window_seconds (900) are read from config.yaml, with LITELLM_DISABLE_LOGIN_RATE_LIMIT to turn the accounting off. They are deliberately not database backed, so editing YAML always wins and an operator refused by a bad value can recover. --- litellm/proxy/_types.py | 10 + litellm/proxy/auth/login_throttle.py | 177 +++++++ litellm/proxy/auth/login_utils.py | 18 +- litellm/proxy/proxy_server.py | 7 +- .../proxy/auth/test_login_utils.py | 477 ++++++++++++++++++ .../proxy/proxy_server/conftest.py | 25 + .../proxy_server/test_routes_login_sso.py | 92 +++- tests/test_litellm/proxy/test_proxy_server.py | 39 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 + 9 files changed, 843 insertions(+), 12 deletions(-) create mode 100644 litellm/proxy/auth/login_throttle.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ed35691c6ec..1fd2781bd95 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2525,6 +2525,16 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="sends alerts if requests hang for 5min+", ) ui_access_mode: Literal["admin_only", "all"] | None = Field("all", description="Control access to the Proxy UI") + max_failed_login_attempts: int | None = Field( + None, + ge=1, + description="Number of failed Admin UI sign-in attempts allowed for one username from one source address within `failed_login_window_seconds`, before further attempts are refused with 429. Configurable from config.yaml only. Defaults to 10", + ) + failed_login_window_seconds: int | None = Field( + None, + ge=1, + description="Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Configurable from config.yaml only. Defaults to 900", + ) allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access") reject_clientside_metadata_tags: bool | None = Field( None, diff --git a/litellm/proxy/auth/login_throttle.py b/litellm/proxy/auth/login_throttle.py new file mode 100644 index 00000000000..e2befd15a35 --- /dev/null +++ b/litellm/proxy/auth/login_throttle.py @@ -0,0 +1,177 @@ +"""Failed-login accounting for the Admin UI sign-in path. + +Counts failed credential checks per (username, source address) over a fixed window and +denies further attempts with 429 once the count reaches the limit. Built per request by +``LoginThrottle.from_request`` because it carries that request's resolved source address, +and because the coordination cache is assigned at startup and can be reassigned later. +""" + +import hashlib +from collections.abc import Awaitable +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from fastapi import Request + +from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.auth.network import TrustedProxyConfig, normalize_cidr_ranges, resolve_client_ip +from litellm.proxy.auth.trusted_proxy_utils import TRUSTED_PROXY_RANGES_KEY +from litellm.secret_managers.main import get_secret_bool + +DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS: Final = 10 +DEFAULT_FAILED_LOGIN_WINDOW_SECONDS: Final = 900 + +_CACHE_KEY_PREFIX: Final = "login_fail" +_UNKNOWN_SOURCE: Final = "unknown" +_MAX_LOGGED_USERNAME_CHARS: Final = 128 + +_MAX_TRACKED_LOGIN_SOURCES: Final = 10_000 + +_FAILED_LOGIN_CACHE: Final = DualCache( + in_memory_cache=InMemoryCache(max_size_in_memory=_MAX_TRACKED_LOGIN_SOURCES), + default_in_memory_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS, +) +_NO_SETTINGS: Final = MappingProxyType({}) + + +def _int_setting(name: str, value: object, default: int, minimum: int) -> int: + if value is None: + return default + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + verbose_proxy_logger.warning( + "general_settings.%s=%r is not an integer >= %s; using the default of %s", name, value, minimum, default + ) + return default + return value + + +def _as_count(cached: object) -> int: + return int(cached) if isinstance(cached, int | float) and not isinstance(cached, bool) else 0 + + +@dataclass(frozen=True, slots=True) +class LoginThrottle: + """Fixed-window failed-login accounting for one request's source address.""" + + client_ip: str + max_attempts: int + window_seconds: int + cache: DualCache + redis_cache: RedisCache | None = None + enabled: bool = True + + @classmethod + def from_request(cls, request: Request) -> "LoginThrottle": + """Build the throttle for this request from the live proxy settings and caches.""" + from litellm.proxy.proxy_server import general_settings, redis_usage_cache + + settings: Final = general_settings or _NO_SETTINGS + cidrs: Final = normalize_cidr_ranges( + settings.get(TRUSTED_PROXY_RANGES_KEY), setting_name=TRUSTED_PROXY_RANGES_KEY + ) + resolved, _ = resolve_client_ip( + request, TrustedProxyConfig(use_forwarded_for=bool(cidrs), trusted_proxy_cidrs=cidrs) + ) + return cls( + client_ip=resolved or _UNKNOWN_SOURCE, + max_attempts=_int_setting( + "max_failed_login_attempts", + settings.get("max_failed_login_attempts"), + DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS, + 1, + ), + window_seconds=_int_setting( + "failed_login_window_seconds", + settings.get("failed_login_window_seconds"), + DEFAULT_FAILED_LOGIN_WINDOW_SECONDS, + 1, + ), + cache=_FAILED_LOGIN_CACHE, + redis_cache=redis_usage_cache, + enabled=not get_secret_bool("LITELLM_DISABLE_LOGIN_RATE_LIMIT"), + ) + + @staticmethod + def _loggable(username: str) -> str: + """The username with anything that could forge a log line removed.""" + return "".join(c for c in username if c.isprintable())[:_MAX_LOGGED_USERNAME_CHARS] + + def _key(self, username: str) -> str: + identity: Final = hashlib.sha256(username.casefold().encode("utf-8")).hexdigest() + return f"{_CACHE_KEY_PREFIX}:{identity}:{self.client_ip}" + + async def _outcome(self, work: Awaitable[object]) -> object: + try: + return await work + except Exception as exc: # noqa: BLE001 # an unreachable cache must never deny a valid credential + verbose_proxy_logger.warning("login attempt accounting unavailable: %s", exc) + return None + + async def _failures(self, key: str) -> int: + store: Final = self.cache if self.redis_cache is None else self.redis_cache + return _as_count(await self._outcome(store.async_get_cache(key=key))) + + async def _ensure_expiry(self, key: str) -> None: + """Give the counter an expiry if it somehow has none. + + Redis commits the increment before setting the TTL, so a failure in between can + leave a counter that never expires. Nothing increments the key again once the + limit is reached, so without this the pair would stay refused indefinitely. + """ + redis_cache: Final = self.redis_cache + if redis_cache is None: + return + if isinstance(await self._outcome(redis_cache.async_get_ttl(key)), int): + return + await self._outcome(redis_cache.async_increment(key, 0, ttl=self.window_seconds)) + + async def raise_if_blocked(self, username: str) -> None: + """Deny before the database lookup and before the password comparison.""" + if not self.enabled: + return + key: Final = self._key(username) + if await self._failures(key) < self.max_attempts: + return + await self._ensure_expiry(key) + verbose_proxy_logger.warning( + "Admin UI sign-in attempts exhausted for username=%s source=%s; %s attempts in %ss, retry after %ss", + self._loggable(username), + self.client_ip, + self.max_attempts, + self.window_seconds, + self.window_seconds, + ) + raise ProxyException( + message="Too many failed sign-in attempts. Try again later.", + type=ProxyErrorTypes.auth_error, + param="max_failed_login_attempts", + code=429, + headers={"Retry-After": str(self.window_seconds)}, # mutable-ok: ProxyException coerces header values + ) + + async def record_failure(self, username: str) -> None: + """Count one rejected credential guess against this username and source.""" + if not self.enabled: + return + key: Final = self._key(username) + redis_cache: Final = self.redis_cache + if redis_cache is not None: + await self._outcome(redis_cache.async_increment(key, 1, ttl=self.window_seconds)) + await self._ensure_expiry(key) + return + await self._outcome(self.cache.async_increment_cache(key=key, value=1, ttl=self.window_seconds)) + + async def clear(self, username: str) -> None: + """Drop the bucket after a successful sign-in.""" + if not self.enabled: + return + key: Final = self._key(username) + redis_cache: Final = self.redis_cache + if redis_cache is not None: + await self._outcome(redis_cache.async_delete_cache(key)) + await self._outcome(self.cache.async_delete_cache(key=key)) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index fba95972944..98780a4692a 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -24,6 +24,7 @@ from litellm.proxy._types import ( UpdateUserRequest, UserAPIKeyAuth, ) +from litellm.proxy.auth.login_throttle import LoginThrottle from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -41,6 +42,10 @@ from litellm.repositories.user_repository import UserRepository from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.ui_sso import ReturnedUITokenObject +INVALID_UI_CREDENTIALS_MESSAGE: Final = ( + "Invalid credentials used to access UI. Check 'UI_USERNAME' and 'UI_PASSWORD', or the password set for your user" +) + async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: """Rehash legacy password (SHA256) to scrypt on successful login.""" @@ -111,6 +116,7 @@ async def authenticate_user( password: str, master_key: str | None, prisma_client: PrismaClient | None, + throttle: LoginThrottle, ) -> LoginResult: """ Authenticate a user and generate an API key for UI access. @@ -139,6 +145,8 @@ async def authenticate_user( code=500, ) + await throttle.raise_if_blocked(username) + ui_username, ui_password = get_ui_credentials(master_key) # Check if we can find the `username` in the db. On the UI, users can enter username=their email @@ -240,6 +248,8 @@ async def authenticate_user( key = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(user_info) + await throttle.clear(username) + return LoginResult( user_id=user_id, key=key, @@ -294,6 +304,8 @@ async def authenticate_user( key = response["token"] + await throttle.clear(username) + return LoginResult( user_id=user_id, key=key, @@ -302,15 +314,17 @@ async def authenticate_user( login_method="username_password", ) else: + await throttle.record_failure(username) raise ProxyException( - message=f"Invalid credentials used to access UI.\nNot valid credentials for {username}", + message=INVALID_UI_CREDENTIALS_MESSAGE, type=ProxyErrorTypes.auth_error, param="invalid_credentials", code=401, ) else: + await throttle.record_failure(username) raise ProxyException( - message="Invalid credentials used to access UI.\nCheck 'UI_USERNAME', 'UI_PASSWORD' in .env file", + message=INVALID_UI_CREDENTIALS_MESSAGE, type=ProxyErrorTypes.auth_error, param="invalid_credentials", code=401, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9b5d1b9bfea..39a419df73f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -293,6 +293,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.litellm_license import LicenseCheck +from litellm.proxy.auth.login_throttle import LoginThrottle from litellm.proxy.auth.model_checks import ( expand_wildcard_deployments_for_model_info, get_all_fallbacks, @@ -723,6 +724,7 @@ from fastapi.openapi.docs import get_swagger_ui_html from fastapi.openapi.utils import get_openapi from fastapi.responses import ( FileResponse, + HTMLResponse, JSONResponse, ORJSONResponse, RedirectResponse, @@ -14730,8 +14732,6 @@ async def fallback_login(request: Request): else: redirect_url += "/sso/callback" - from fastapi.responses import HTMLResponse - hide_default_credentials_hint: Final = ( os.getenv("LITELLM_HIDE_DEFAULT_CREDENTIALS_HINT", "false").lower() == "true" or general_settings.get("hide_default_credentials_hint", False) is True @@ -14761,6 +14761,7 @@ async def login(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + throttle=LoginThrottle.from_request(request), ) # Create UI token object @@ -14835,6 +14836,7 @@ async def login_v2(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + throttle=LoginThrottle.from_request(request), ) returned_ui_token_object: Final = create_ui_token_object( @@ -14905,6 +14907,7 @@ async def login_v3(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + throttle=LoginThrottle.from_request(request), ) returned_ui_token_object: Final = create_ui_token_object( diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 1c66acf8678..1e3fcaf5d78 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -10,6 +10,16 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest + +def _unlimited_throttle(): + """A throttle wired to a real in-memory store with a limit no test can reach.""" + from litellm.caching.dual_cache import DualCache + from litellm.proxy.auth.login_throttle import LoginThrottle + + return LoginThrottle(client_ip="1.2.3.4", max_attempts=10_000, window_seconds=900, cache=DualCache()) + + + from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._types import ( LiteLLM_UserTable, @@ -98,6 +108,7 @@ async def test_authenticate_user_admin_login_with_ui_credentials(): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -155,6 +166,7 @@ async def test_authenticate_user_admin_login_with_master_key_as_password(monkeyp password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -179,6 +191,7 @@ async def test_authenticate_user_invalid_credentials(): password=wrong_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -197,6 +210,7 @@ async def test_authenticate_user_missing_master_key(): password="password", master_key=None, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -237,6 +251,7 @@ async def test_authenticate_user_wrong_password(): password=wrong_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -295,12 +310,14 @@ async def test_authenticate_user_email_case_insensitive_login(): password=correct_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) result_lower = await authenticate_user( username=stored_email, password=correct_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert result_mixed.user_id == result_lower.user_id == "test-user-123" @@ -342,6 +359,7 @@ async def test_authenticate_user_database_required_for_admin(monkeypatch): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -393,6 +411,7 @@ async def test_authenticate_user_admin_login_with_non_ascii_characters(): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -469,18 +488,21 @@ async def test_authenticate_user_multiple_logins_generate_unique_tokens(): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) result2 = await authenticate_user( username=ui_username, password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) result3 = await authenticate_user( username=ui_username, password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) # Each login should return a unique token @@ -538,6 +560,7 @@ async def test_authenticate_user_database_login_with_non_ascii_password(): password=password_with_special_char, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -598,3 +621,457 @@ class TestEncodeUiSessionJwt: request.cookies = {"token": token} with patch("litellm.proxy.proxy_server.master_key", "sk-master-for-tests"): assert _user_id_from_session_cookie(request) == "cornell-user" + + +# --------------------------------------------------------------------------- +# Failed-login accounting (LIT-5285) +# --------------------------------------------------------------------------- + + +def _throttle(max_attempts: int = 3, window_seconds: int = 900, client_ip: str = "1.2.3.4", cache=None, redis_cache=None): + """A throttle over a real in-memory store, so the tests exercise the true counters.""" + from litellm.caching.dual_cache import DualCache + from litellm.proxy.auth.login_throttle import LoginThrottle + + return LoginThrottle( + client_ip=client_ip, + max_attempts=max_attempts, + window_seconds=window_seconds, + cache=cache if cache is not None else DualCache(), + redis_cache=redis_cache, + ) + + +async def _guess(throttle, username: str = "admin", password: str = "wrong"): + from litellm.proxy.auth.login_utils import authenticate_user + + return await authenticate_user( + username=username, + password=password, + master_key="sk-master", + prisma_client=None, + throttle=throttle, + ) + + +@pytest.mark.asyncio +async def test_attempts_are_refused_once_the_limit_is_reached(monkeypatch): + """The limit denies further attempts for the window, and the denial carries Retry-After.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=3, window_seconds=77) + + for _ in range(3): + with pytest.raises(ProxyException) as first: + await _guess(throttle) + assert first.value.code == "401" + + with pytest.raises(ProxyException) as blocked: + await _guess(throttle) + assert blocked.value.code == "429" + assert blocked.value.headers.get("Retry-After") == "77" + + +@pytest.mark.asyncio +async def test_a_correct_password_is_refused_while_blocked(monkeypatch): + """The check precedes the credential comparison, so being over the limit wins.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=2) + + for _ in range(2): + with pytest.raises(ProxyException): + await _guess(throttle) + + with pytest.raises(ProxyException) as blocked: + await _guess(throttle, password="right") + assert blocked.value.code == "429" + + +@pytest.mark.asyncio +async def test_a_blocked_attempt_does_not_extend_the_window(monkeypatch): + """Hammering while blocked must not push the counter or refresh its TTL.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=2) + key = throttle._key("admin") + + for _ in range(2): + with pytest.raises(ProxyException): + await _guess(throttle) + counted_at_limit = await throttle._failures(key) + + for _ in range(5): + with pytest.raises(ProxyException): + await _guess(throttle) + + assert await throttle._failures(key) == counted_at_limit == 2 + + +@pytest.mark.asyncio +async def test_a_successful_sign_in_clears_the_bucket(monkeypatch): + """Success resets the budget rather than leaving the operator near the limit.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(max_attempts=3) + + for _ in range(2): + with pytest.raises(ProxyException): + await _guess(throttle) + + with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch( + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ): + await _guess(throttle, password="right") + + assert await throttle._failures(throttle._key("admin")) == 0 + + +@pytest.mark.asyncio +async def test_a_configuration_error_never_counts(monkeypatch): + """A 500 from an unset master key is not a guess and must not consume the budget.""" + from litellm.proxy._types import ProxyException + + throttle = _throttle(max_attempts=2) + for _ in range(5): + with pytest.raises(ProxyException) as exc: + await authenticate_user( + username="admin", password="x", master_key=None, prisma_client=None, throttle=throttle + ) + assert exc.value.code == "500" + + assert await throttle._failures(throttle._key("admin")) == 0 + + +@pytest.mark.asyncio +async def test_the_username_is_case_folded_into_one_bucket(monkeypatch): + """The DB lookup is case-insensitive, so casing must not multiply the budget.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=4) + + for name in ("admin@corp.com", "ADMIN@corp.com", "Admin@corp.com", "aDmIn@corp.com"): + with pytest.raises(ProxyException) as exc: + await _guess(throttle, username=name) + assert exc.value.code == "401" + + with pytest.raises(ProxyException) as blocked: + await _guess(throttle, username="admin@CORP.com") + assert blocked.value.code == "429" + + +@pytest.mark.asyncio +async def test_a_different_username_from_the_same_source_is_unaffected(monkeypatch): + """The bucket is the pair, so one username's failures do not block another.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=2) + + for _ in range(3): + with pytest.raises(ProxyException): + await _guess(throttle, username="admin") + + with pytest.raises(ProxyException) as other: + await _guess(throttle, username="someone-else@example.com") + assert other.value.code == "401", "a second username must still reach the credential check" + + +@pytest.mark.asyncio +async def test_both_credential_rejections_are_indistinguishable(monkeypatch): + """One message for the known and the unknown username, so responses do not enumerate.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + + with pytest.raises(ProxyException) as unknown: + await _guess(_throttle(max_attempts=99), username="nobody@example.com") + + fake_user = MagicMock() + fake_user.user_id = "u-1" + fake_user.user_email = "known@example.com" + fake_user.user_role = "internal_user" + fake_user.password = "scrypt:fake" + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=fake_user) + with patch("litellm.proxy.auth.login_utils.UserRepository", repo), patch( + "litellm.proxy.auth.login_utils.verify_password", return_value=False + ): + with pytest.raises(ProxyException) as known: + await authenticate_user( + username="known@example.com", + password="wrong", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=_throttle(max_attempts=99), + ) + + assert unknown.value.message == known.value.message + assert "known@example.com" not in unknown.value.message + known.value.message + + +@pytest.mark.asyncio +async def test_a_user_with_no_password_set_does_not_consume_the_budget(monkeypatch): + """That 401 is deterministic and guards no secret, so counting it would only let + someone burn a passwordless account's bucket.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=2) + + passwordless = MagicMock() + passwordless.user_id = "u-2" + passwordless.user_email = "nopass@example.com" + passwordless.user_role = "internal_user" + passwordless.password = None + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=passwordless) + + with patch("litellm.proxy.auth.login_utils.UserRepository", repo): + for _ in range(5): + with pytest.raises(ProxyException) as exc: + await authenticate_user( + username="nopass@example.com", + password="x", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + ) + assert exc.value.code == "401" + + assert await throttle._failures(throttle._key("nopass@example.com")) == 0 + + +@pytest.mark.asyncio +async def test_a_wrong_password_for_a_known_user_also_counts(monkeypatch): + """The database-user branch must charge the bucket too, not just the unknown-user branch.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=3) + + known = MagicMock() + known.user_id = "u-1" + known.user_email = "known@example.com" + known.user_role = "internal_user" + known.password = "scrypt:stored" + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=known) + + async def _attempt(): + return await authenticate_user( + username="known@example.com", + password="wrong", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + ) + + with patch("litellm.proxy.auth.login_utils.UserRepository", repo), patch( + "litellm.proxy.auth.login_utils.verify_password", return_value=False + ): + for _ in range(3): + with pytest.raises(ProxyException) as rejected: + await _attempt() + assert rejected.value.code == "401" + + with pytest.raises(ProxyException) as blocked: + await _attempt() + assert blocked.value.code == "429" + + +@pytest.mark.asyncio +async def test_two_source_addresses_do_not_share_a_bucket(monkeypatch): + """The key is the pair, so one address exhausting its budget must not block another. + + Dropping the address from the key would turn this into the username-only counter the + design rejects, where anyone can lock a named admin out from anywhere. + """ + from litellm.caching.dual_cache import DualCache + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + shared_store = DualCache() + attacker = _throttle(max_attempts=2, client_ip="203.0.113.9", cache=shared_store) + operator = _throttle(max_attempts=2, client_ip="198.51.100.7", cache=shared_store) + + for _ in range(3): + with pytest.raises(ProxyException): + await _guess(attacker, username="admin") + + with pytest.raises(ProxyException) as blocked: + await _guess(attacker, username="admin") + assert blocked.value.code == "429" + + with pytest.raises(ProxyException) as unaffected: + await _guess(operator, username="admin") + assert unaffected.value.code == "401", "the real operator must still reach the credential check" + + +class _NoExpiryRedis: + """Redis that stores the counter but never records an expiry for it. + + Models the window between INCRBYFLOAT committing and the TTL call failing. + """ + + def __init__(self): + self.values: dict = {} + self.expiry_repairs = 0 + + async def async_get_cache(self, key, **kwargs): + return self.values.get(key) + + async def async_increment(self, key, value, ttl=None, **kwargs): + if int(value) == 0: + self.expiry_repairs += 1 + self.values[key] = self.values.get(key, 0) + int(value) + return self.values[key] + + async def async_get_ttl(self, key): + return None + + async def async_delete_cache(self, key): + self.values.pop(key, None) + + +@pytest.mark.asyncio +async def test_a_counter_left_without_an_expiry_is_repaired(monkeypatch): + """Regression: a counter with no TTL would refuse the pair forever. + + Redis commits the increment before setting the expiry, and nothing increments the key + again once the limit is reached, so a TTL that never landed is never repaired on its + own and the username and source pair stays refused with no way back. + """ + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + redis = _NoExpiryRedis() + throttle = _throttle(max_attempts=2, redis_cache=redis) + + for _ in range(2): + with pytest.raises(ProxyException): + await _guess(throttle) + + assert redis.expiry_repairs >= 1, "each recorded failure must leave the counter with an expiry" + + repairs_before_block = redis.expiry_repairs + with pytest.raises(ProxyException) as blocked: + await _guess(throttle) + assert blocked.value.code == "429" + assert redis.expiry_repairs > repairs_before_block, "the refusal path must repair a missing expiry too" + + +@pytest.mark.asyncio +async def test_counters_do_not_share_the_key_authentication_cache(monkeypatch): + """Regression: throttle entries must not evict cached credentials. + + user_api_key_cache holds at most 200 in-memory entries and evicts the soonest to + expire first, so parking 900s sign-in counters there let a stream of made-up usernames + push out the much shorter lived credential entries, sending every ordinary API request + back to the database. + """ + from litellm.proxy import proxy_server as ps + from litellm.proxy.auth.login_throttle import _CACHE_KEY_PREFIX, LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setattr(ps, "redis_usage_cache", None) + + auth_cache_keys_before = set(ps.user_api_key_cache.in_memory_cache.cache_dict) + + request = MagicMock() + request.headers = {} + request.client = MagicMock() + request.client.host = "1.2.3.4" + throttle = LoginThrottle.from_request(request) + + for i in range(25): + with pytest.raises(Exception): + await _guess(throttle, username=f"made-up-{i}@example.com") + + added = set(ps.user_api_key_cache.in_memory_cache.cache_dict) - auth_cache_keys_before + assert not [k for k in added if str(k).startswith(_CACHE_KEY_PREFIX)], ( + "sign-in counters must live in their own cache, not the key-authentication cache" + ) + + +@pytest.mark.asyncio +async def test_a_refused_username_cannot_forge_log_lines(monkeypatch): + """The username reaches a warning log, so it must not carry newlines or control bytes.""" + import logging + + from litellm._logging import verbose_proxy_logger + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(max_attempts=1) + forged = "victim@example.com\nWARNING: sign-in succeeded for attacker\x00" + + with pytest.raises(ProxyException): + await _guess(throttle, username=forged) + + records: list[logging.LogRecord] = [] + handler = logging.Handler() + handler.emit = records.append + verbose_proxy_logger.addHandler(handler) + try: + with pytest.raises(ProxyException) as blocked: + await _guess(throttle, username=forged) + finally: + verbose_proxy_logger.removeHandler(handler) + + assert blocked.value.code == "429" + emitted = [r.getMessage() for r in records if "sign-in attempts exhausted" in r.getMessage()] + assert emitted, "the refusal must be logged" + assert "\n" not in emitted[0] and "\x00" not in emitted[0] + assert "victim@example.com" in emitted[0] + + +@pytest.mark.asyncio +async def test_a_username_spray_cannot_evict_an_existing_counter(monkeypatch): + """Regression: the in-memory tier must hold more counters than a spray can create. + + The default in-memory cache keeps 200 entries and evicts the soonest to expire, and + every counter shares one window, so eviction was effectively oldest-first. A few + hundred made-up usernames therefore pushed out the attacker's own counter and handed + back a fresh allowance against the real account. + """ + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.login_throttle import _FAILED_LOGIN_CACHE, _MAX_TRACKED_LOGIN_SOURCES + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + assert _MAX_TRACKED_LOGIN_SOURCES >= 10_000 + assert _FAILED_LOGIN_CACHE.in_memory_cache.max_size_in_memory == _MAX_TRACKED_LOGIN_SOURCES + + throttle = _throttle(max_attempts=3, cache=_FAILED_LOGIN_CACHE, client_ip="10.9.9.9") + victim = "spray-victim@corp.com" + for _ in range(3): + with pytest.raises(ProxyException): + await _guess(throttle, username=victim) + + for i in range(500): + await throttle.record_failure(f"spray-filler-{i}@corp.com") + + assert await throttle._failures(throttle._key(victim)) == 3, "the counter must survive a spray" + with pytest.raises(ProxyException) as blocked: + await _guess(throttle, username=victim) + assert blocked.value.code == "429" diff --git a/tests/test_litellm/proxy/proxy_server/conftest.py b/tests/test_litellm/proxy/proxy_server/conftest.py index c545965f9a9..cdfd3549544 100644 --- a/tests/test_litellm/proxy/proxy_server/conftest.py +++ b/tests/test_litellm/proxy/proxy_server/conftest.py @@ -511,3 +511,28 @@ def make_key( max_budget=max_budget, **kwargs, ) + + +@pytest.fixture(autouse=True) +def reset_login_throttle(monkeypatch): + """Clear the Admin UI failed-login counters between tests. + + `client` is session scoped and the counters live in the shared `user_api_key_cache` + with a 900s window, so without this any test that fails a sign-in enough times would + start returning 429 from unrelated tests later in the same process. Only the throttle's + own keys are removed, so nothing else in that cache is disturbed. + """ + from litellm.proxy import proxy_server as ps + from litellm.proxy.auth.login_throttle import _FAILED_LOGIN_CACHE, _CACHE_KEY_PREFIX + + def _drop_throttle_keys() -> None: + in_memory = getattr(_FAILED_LOGIN_CACHE, "in_memory_cache", None) + cache_dict = getattr(in_memory, "cache_dict", None) + if isinstance(cache_dict, dict): + for key in [k for k in cache_dict if str(k).startswith(_CACHE_KEY_PREFIX)]: + cache_dict.pop(key, None) + + monkeypatch.setattr(ps, "redis_usage_cache", None) + _drop_throttle_keys() + yield _drop_throttle_keys + _drop_throttle_keys() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index af37dbe85fe..faeb7750e03 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -10,6 +10,7 @@ Routes covered: from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor from unittest.mock import AsyncMock, MagicMock import pytest @@ -29,7 +30,7 @@ def _install_login_mocks(monkeypatch, raise_on_auth: bool = False) -> None: """ from litellm.proxy import proxy_server as ps - async def _fake_auth(username, password, master_key, prisma_client): + async def _fake_auth(username, password, master_key, prisma_client, throttle=None): if raise_on_auth: raise Exception("boom-auth-failure") fake = MagicMock() @@ -460,3 +461,92 @@ def test_login_form_ignores_open_redirect_return_to(client, monkeypatch): location = response.headers.get("location", "") assert "evil.example.com" not in location assert "/ui" in location # dashboard fallback + + +# --------------------------------------------------------------------------- +# Failed-login accounting across the login routes (LIT-5285) +# --------------------------------------------------------------------------- + + +def _install_real_auth(monkeypatch, **settings): + """Run the real authenticate_user so the throttle inside it is exercised. + + prisma_client stays None, so every guess falls through to the credential rejection. + """ + from litellm.proxy import proxy_server as ps + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right-password") + monkeypatch.setattr(ps, "master_key", "sk-test-master") + monkeypatch.setattr(ps, "prisma_client", None) + monkeypatch.setattr(ps, "premium_user", False) + monkeypatch.setattr(ps, "general_settings", dict(settings)) + + +def _form_login(client, username="admin", password="wrong"): + return client.post( + "/login", data={"username": username, "password": password}, follow_redirects=False + ).status_code + + +def _json_login(client, path, username="admin", password="wrong"): + return client.post(path, json={"username": username, "password": password}).status_code + + +def test_budget_is_shared_across_every_login_endpoint(client, monkeypatch, reset_login_throttle): + """The endpoint is not part of the key, so spending the budget on one route blocks the rest. + + Partitioning the counter per endpoint would silently triple the real allowance. + """ + _install_real_auth( + monkeypatch, + max_failed_login_attempts=10, + control_plane_url="https://cp.example.com", + ) + + assert [_form_login(client) for _ in range(5)] == [401] * 5 + assert [_json_login(client, "/v2/login") for _ in range(5)] == [401] * 5 + + assert _json_login(client, "/v3/login") == 429, "the eleventh attempt must be refused on a third route" + + +def test_budget_is_shared_across_username_casing(client, monkeypatch, reset_login_throttle): + """The database lookup is case-insensitive, so casing must not partition the counter.""" + _install_real_auth(monkeypatch, max_failed_login_attempts=10) + + assert [_json_login(client, "/v2/login", username="admin@corp.com") for _ in range(5)] == [401] * 5 + assert [_json_login(client, "/v2/login", username="ADMIN@corp.com") for _ in range(5)] == [401] * 5 + + assert _json_login(client, "/v2/login", username="Admin@corp.com") == 429 + + +def test_a_refused_attempt_carries_retry_after(client, monkeypatch, reset_login_throttle): + """The 429 tells the caller how long the window has left.""" + _install_real_auth(monkeypatch, max_failed_login_attempts=2, failed_login_window_seconds=77) + + assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401] + + refused = client.post("/v2/login", json={"username": "admin", "password": "wrong"}) + assert refused.status_code == 429 + assert refused.headers.get("retry-after") == "77" + + +def test_a_second_username_from_the_same_source_still_gets_through(client, monkeypatch, reset_login_throttle): + """The bucket is the username and source pair, so one account cannot block another.""" + _install_real_auth(monkeypatch, max_failed_login_attempts=2) + + for _ in range(3): + _json_login(client, "/v2/login", username="admin") + + assert _json_login(client, "/v2/login", username="someone-else@example.com") == 401 + + +def test_sign_in_succeeds_again_once_the_budget_is_restored(client, monkeypatch, reset_login_throttle): + """A cleared bucket lets the same username straight back in.""" + _install_real_auth(monkeypatch, max_failed_login_attempts=2) + + assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401] + assert _json_login(client, "/v2/login") == 429 + + reset_login_throttle() + assert _json_login(client, "/v2/login") == 401 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index afc42e8db45..ac2e84b8474 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -19,7 +19,6 @@ from fastapi import FastAPI from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient - import litellm import litellm.proxy.proxy_server as proxy_server_module from litellm.caching.caching import RedisCache @@ -27,6 +26,7 @@ from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.caching.dual_cache import DualCache from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.login_throttle import LoginThrottle from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.proxy_server import app, initialize from litellm.utils import _invalidate_model_cost_lowercase_map @@ -124,12 +124,13 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): } assert response.cookies.get("token") == "signed-token" - mock_authenticate_user.assert_awaited_once_with( - username="alice", - password="secret", - master_key="test-master-key", - prisma_client=mock_prisma_client, - ) + mock_authenticate_user.assert_awaited_once() + auth_kwargs = mock_authenticate_user.call_args.kwargs + assert auth_kwargs["username"] == "alice" + assert auth_kwargs["password"] == "secret" + assert auth_kwargs["master_key"] == "test-master-key" + assert auth_kwargs["prisma_client"] is mock_prisma_client + assert isinstance(auth_kwargs["throttle"], LoginThrottle), "the endpoint must thread a throttle through" mock_create_ui_token_object.assert_called_once_with( login_result=mock_login_result, general_settings={}, @@ -11424,3 +11425,27 @@ async def test_authoritative_floor_spend_keeps_a_reset_marker_written_during_the assert real_spend_counter_cache.in_memory_cache.get_cache(key=marker_key) == 0.0, ( "the in-flight DB read clobbered the post-reset floor marker with the stale pre-reset value" ) + + +@pytest.mark.asyncio +async def test_login_throttle_settings_are_not_overridable_from_the_database(): + """LIT-5285: the sign-in limits stay config.yaml only. + + _update_general_settings copies an allowlist of keys out of the DB row. Adding these + to it would let a stored value outrank config.yaml, so an operator refused by a bad + value could not fix it by editing YAML and restarting. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy.proxy_server import ProxyConfig + + original = dict(ps.general_settings) + try: + ps.general_settings.clear() + await ProxyConfig()._update_general_settings( + db_general_settings={"max_failed_login_attempts": 999, "failed_login_window_seconds": 1} + ) + assert "max_failed_login_attempts" not in ps.general_settings + assert "failed_login_window_seconds" not in ps.general_settings + finally: + ps.general_settings.clear() + ps.general_settings.update(original) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e0b8cf19159..bb2841b6388 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -24150,6 +24150,11 @@ export interface components { * @default false */ enable_public_model_hub: boolean; + /** + * Failed Login Window Seconds + * @description Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Configurable from config.yaml only. Defaults to 900 + */ + failed_login_window_seconds?: number | null; /** * Forward Client Headers To Llm Api * @description If True, forwards client headers (e.g. Authorization) to the LLM API. Required for Claude Code with Max subscription. @@ -24194,6 +24199,11 @@ export interface components { * @description max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider */ max_batch_file_size_mb?: number | null; + /** + * Max Failed Login Attempts + * @description Number of failed Admin UI sign-in attempts allowed for one username from one source address within `failed_login_window_seconds`, before further attempts are refused with 429. Configurable from config.yaml only. Defaults to 10 + */ + max_failed_login_attempts?: number | null; /** * Max Parallel Requests * @description maximum parallel requests for each api key From a39efb1a1d59fbf122e8ddfc5929f256bda9f9c0 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 26 Aug 2026 10:24:32 -0700 Subject: [PATCH 2/2] test(proxy): clear failed-login TTLs when resetting the throttle between tests --- tests/test_litellm/proxy/proxy_server/conftest.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/proxy/proxy_server/conftest.py b/tests/test_litellm/proxy/proxy_server/conftest.py index cdfd3549544..22f2e5b3feb 100644 --- a/tests/test_litellm/proxy/proxy_server/conftest.py +++ b/tests/test_litellm/proxy/proxy_server/conftest.py @@ -527,10 +527,17 @@ def reset_login_throttle(monkeypatch): def _drop_throttle_keys() -> None: in_memory = getattr(_FAILED_LOGIN_CACHE, "in_memory_cache", None) - cache_dict = getattr(in_memory, "cache_dict", None) - if isinstance(cache_dict, dict): - for key in [k for k in cache_dict if str(k).startswith(_CACHE_KEY_PREFIX)]: - cache_dict.pop(key, None) + if in_memory is None: + return + tracked = tuple( + key + for store in (getattr(in_memory, "cache_dict", None), getattr(in_memory, "ttl_dict", None)) + if isinstance(store, dict) + for key in tuple(store) + if str(key).startswith(_CACHE_KEY_PREFIX) + ) + for key in tracked: + in_memory.delete_cache(key) monkeypatch.setattr(ps, "redis_usage_cache", None) _drop_throttle_keys()