diff --git a/litellm/constants.py b/litellm/constants.py index a80d09ff59f..fb40b5a1283 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1657,6 +1657,11 @@ LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS: Final = int( LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE: Final = int( os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000) ) +LOGIN_THROTTLE_CACHE_KEY_PREFIX: Final = "login_fail" +LOGIN_THROTTLE_UNKNOWN_SOURCE: Final = "unknown" +LOGIN_THROTTLE_MAX_TRACKED_COUNTERS: Final = 20_000 +LOGIN_THROTTLE_MAX_TRACKED_BLOCKS: Final = 10_000 +LOGIN_THROTTLE_NOT_BLOCKED: Final = (0, 0) LITELLM_PROXY_ADMIN_NAME: Final = "default_user_id" LITELLM_PROXY_BUDGET_NAME: Final = "litellm-proxy-budget" GLOBAL_PROXY_SPEND_CACHE_KEY: Final = f"{LITELLM_PROXY_ADMIN_NAME}:spend" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7d94ef400cc..882007e1745 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2768,6 +2768,25 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="sends alerts if requests hang for 5min+", ) ui_access_mode: Literal["admin_only", "all"] | None = Field("all", description="Control access to the Proxy UI") + max_failed_login_attempts_per_source: int | None = Field( + None, + ge=1, + description="Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Half this value, rounded down but at least 1, is the allowance for one username from that address; one more blocks that address for that username only, and its further failures stop counting toward the address limit, so a script stuck on one account does not block everyone behind a shared address. The per-address limit is only enforced when `trusted_proxy_ranges` is set: to the proxies in front of LiteLLM, or to an empty list when clients connect directly. Left unset, the peer address may be a shared ingress and only the per-username half runs. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10", + ) + max_failed_login_attempts_per_source_overrides: dict[str, int] | None = Field( + None, + description="Per-address overrides of `max_failed_login_attempts_per_source`, keyed by IP address or CIDR range, e.g. {'1.2.3.4': 200, '5.6.0.0/24': 500}. The most specific matching range wins (between equivalent keys such as '1.2.3.4' and '1.2.3.4/32', an exemption wins, then the higher limit), and the per-username allowance for that address follows as half the override. A value of 0 exempts the address from both limits. Set under `general_settings` in config.yaml", + ) + failed_login_window_seconds: int | None = Field( + None, + ge=1, + description="Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Set under `general_settings` in config.yaml. Defaults to 60", + ) + failed_login_block_seconds: int | None = Field( + None, + ge=1, + description="How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300", + ) allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access") reject_clientside_metadata_tags: bool | None = Field( None, @@ -2881,7 +2900,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): ) trusted_proxy_ranges: list[str] | None = Field( None, - description="CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler.", + description="CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler, and whose X-Forwarded-For is used to attribute Admin UI sign-in attempts to a source address. Set it to an empty list when clients connect directly, so the peer address is the source. Left unset, or containing an entry that is not an address or CIDR range, the per-source sign-in limit is off.", ) store_model_in_db: bool | None = Field( None, diff --git a/litellm/proxy/auth/login_throttle.py b/litellm/proxy/auth/login_throttle.py new file mode 100644 index 00000000000..b7f7eaceff4 --- /dev/null +++ b/litellm/proxy/auth/login_throttle.py @@ -0,0 +1,445 @@ +"""Failed-login accounting for the Admin UI sign-in path. + +Wrong passwords are counted over a short window per source address and per source-and-username +pair; too many in one window blocks that key for a fixed time. While a key is blocked every attempt +from it, right or wrong, is refused with 429 before the password is checked. A blocked pair stops +counting against its source, so one script stuck on one account does not block the whole office. +Recovery is the master key over the API, which never passes through here, or waiting out the block. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import ipaddress +import math +import time +from collections.abc import Mapping +from dataclasses import dataclass +from functools import cache +from typing import Final, Literal, NamedTuple, Protocol, TypeAlias + +from fastapi import Request, status +from pydantic import TypeAdapter, ValidationError +from redis.exceptions import RedisError + +from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError +from litellm.constants import ( + EMPTY_MAPPING, + LOGIN_THROTTLE_CACHE_KEY_PREFIX, + LOGIN_THROTTLE_MAX_TRACKED_BLOCKS, + LOGIN_THROTTLE_MAX_TRACKED_COUNTERS, + LOGIN_THROTTLE_NOT_BLOCKED, + LOGIN_THROTTLE_UNKNOWN_SOURCE, +) +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.auth.network import TrustedProxyConfig, resolve_client_ip +from litellm.secret_managers.main import get_secret_bool + +DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE: Final = 10 +DEFAULT_FAILED_LOGIN_WINDOW_SECONDS: Final = 60 +DEFAULT_FAILED_LOGIN_BLOCK_SECONDS: Final = 300 + +IPV6_SOURCE_PREFIX_LENGTH: Final = 64 +EXEMPT: Final = 0 + +SOURCE_LIMIT_KEY: Final = "max_failed_login_attempts_per_source" +SOURCE_LIMIT_OVERRIDES_KEY: Final = "max_failed_login_attempts_per_source_overrides" +WINDOW_KEY: Final = "failed_login_window_seconds" +BLOCK_KEY: Final = "failed_login_block_seconds" +TRUSTED_PROXY_RANGES_KEY: Final = "trusted_proxy_ranges" + +_REDIS_FAILURES: Final = (RedisError, RedisCircuitBreakerOpenError, OSError, asyncio.TimeoutError) +_LOCAL_BLOCK_EXPIRY: Final = TypeAdapter[float | None](float | None) +_SOURCE_LIMIT_OVERRIDES: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) +_RANGE_ENTRIES: Final = TypeAdapter[tuple[object, ...]](tuple[object, ...]) + +Scope: TypeAlias = Literal["user", "source"] + +_BlockTtls: TypeAlias = tuple[int, int] +_LUA_BLOCK_TTLS: Final = TypeAdapter[_BlockTtls](_BlockTtls) +_Network: TypeAlias = ipaddress.IPv4Network | ipaddress.IPv6Network + + +class LocalStore(Protocol): + """The per-worker store behind the counters and blocks; ``InMemoryCache`` satisfies it.""" + + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: float, *, ttl: int) -> None: ... + + def increment_cache(self, key: str, value: float, *, ttl: int) -> float: ... + + def delete_cache(self, key: str) -> None: ... + + +# KEYS: pair counter, pair block, source counter, source block (one cluster slot via the source hash tag) +# ARGV: pair limit, source limit (0 = source scope off), window seconds, block seconds +# Both scripts return {pair block TTL, source block TTL}; 0 or below means not blocked +_BLOCK_TTLS_LUA: Final = "return {redis.call('TTL', KEYS[2]), redis.call('TTL', KEYS[4])}" +_RECORD_FAILURE_LUA: Final = ( + "local function bump(count_key, block_key, limit) " + "local blocked = redis.call('TTL', block_key) " + "if blocked > 0 then return blocked end " + "local count = redis.call('INCR', count_key) " + "if redis.call('TTL', count_key) < 0 then redis.call('EXPIRE', count_key, ARGV[3]) end " + "if count > limit then redis.call('SET', block_key, '1', 'EX', ARGV[4]) return tonumber(ARGV[4]) end " + "return 0 end " + "local user_block = bump(KEYS[1], KEYS[2], tonumber(ARGV[1])) " + "local source_block = 0 " + "if tonumber(ARGV[2]) > 0 and user_block == 0 then " + "source_block = bump(KEYS[3], KEYS[4], tonumber(ARGV[2])) end " + "return {user_block, source_block}" +) + +_COUNTERS: Final = InMemoryCache( + max_size_in_memory=LOGIN_THROTTLE_MAX_TRACKED_COUNTERS, default_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS +) +_BLOCKS: Final = InMemoryCache( + max_size_in_memory=LOGIN_THROTTLE_MAX_TRACKED_BLOCKS, default_ttl=DEFAULT_FAILED_LOGIN_BLOCK_SECONDS +) + + +@cache +def _rate_limit_disabled() -> bool: + return get_secret_bool("LITELLM_DISABLE_LOGIN_RATE_LIMIT", default_value=False) is True + + +@cache +def warn_login_counters_are_per_worker(num_workers: str) -> None: + verbose_proxy_logger.warning( + "Running %s workers but Redis is not configured. Failed Admin UI sign-in attempts are counted " + "per worker, so the effective limits are %s times the configured values. Configure Redis " + "to share one count across workers.", + num_workers, + num_workers, + ) + + +@cache +def warn_source_login_limit_is_off() -> None: + verbose_proxy_logger.warning( + "%s is not set or not a valid list of ranges, so failed Admin UI sign-in attempts are limited per " + "source address and username only. Set it to the address ranges of the proxies in front of LiteLLM, " + "or to an empty list when clients connect directly, to also limit each source address across usernames.", + TRUSTED_PROXY_RANGES_KEY, + ) + + +def declared_proxy_ranges(settings: Mapping[str, object]) -> tuple[str, ...] | None: + """What the operator says fronts LiteLLM: the proxy ranges, an empty tuple for none, None when unsaid. + + Only a declared topology makes the source address trustworthy enough to limit across usernames. + An unset key, a value that is not a list of ranges, or a list with an entry that is not an address + or range leaves it unknown and the source scope off. + """ + entries: Final = _configured_range_entries(settings.get(TRUSTED_PROXY_RANGES_KEY)) + if entries is None or any(_parse_network(entry, TRUSTED_PROXY_RANGES_KEY) is None for entry in entries): + return None + return entries + + +def _configured_range_entries(raw_ranges: object) -> tuple[str, ...] | None: + """Every configured entry, blanks included, so a stray empty string fails validation like any other typo.""" + if raw_ranges is None: + return None + if isinstance(raw_ranges, str): + return tuple(part.strip() for part in raw_ranges.split(",")) + try: + return tuple(str(entry).strip() for entry in _RANGE_ENTRIES.validate_python(raw_ranges)) + except ValidationError: + verbose_proxy_logger.warning( + "Invalid %s value: expected a list of address ranges, got %s", + TRUSTED_PROXY_RANGES_KEY, + type(raw_ranges).__name__, + ) + return None + + +def _positive_int(raw: object, key: str, default: int) -> int: + if raw is None: + return default + try: + value: Final = int(str(raw)) + except (TypeError, ValueError): + verbose_proxy_logger.warning("Invalid %s value %r; using %s", key, raw, default) + return default + if value < 1: + verbose_proxy_logger.warning("Invalid %s value %s (must be >= 1); using %s", key, value, default) + return default + return value + + +def _int_setting(settings: Mapping[str, object], key: str, default: int) -> int: + return _positive_int(settings.get(key), key, default) + + +def _override_limit(raw: object, default: int) -> int: + """A per-address override: a limit of 1 or more, or ``EXEMPT`` (0) to leave that address unlimited.""" + if str(raw).strip() == str(EXEMPT): + return EXEMPT + return _positive_int(raw, SOURCE_LIMIT_OVERRIDES_KEY, default) + + +def _parse_address(client_ip: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None: + """The address as it is limited and counted: an IPv4-mapped IPv6 address is its IPv4 address.""" + try: + address: Final = ipaddress.ip_address(client_ip) + except ValueError: + return None + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None: + return address.ipv4_mapped + return address + + +def _parse_network(raw_range: str, setting_name: str = SOURCE_LIMIT_OVERRIDES_KEY) -> _Network | None: + try: + return ipaddress.ip_network(raw_range.strip(), strict=False) + except ValueError: + verbose_proxy_logger.warning("Invalid address or range %r in %s; skipping", raw_range, setting_name) + return None + + +def _precedence(network: _Network, limit: int) -> tuple[int, bool, int]: + """Sort key for competing overrides: the longest prefix wins, then an exemption, then the higher limit.""" + return (network.prefixlen, limit == EXEMPT, limit) + + +def _source_limit(settings: Mapping[str, object], client_ip: str) -> int: + """Failure allowance for this address: the most specific configured range containing it, else the default. + + ``EXEMPT`` (0) means the operator opted this address out of both limits. Between equivalent keys such as + ``1.2.3.4`` and ``1.2.3.4/32`` an exemption wins, then the higher limit. + """ + default: Final = _int_setting(settings, SOURCE_LIMIT_KEY, DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE) + raw_overrides: Final = settings.get(SOURCE_LIMIT_OVERRIDES_KEY) + if raw_overrides is None: + return default + try: + overrides: Final = _SOURCE_LIMIT_OVERRIDES.validate_python(raw_overrides) + except ValidationError: + verbose_proxy_logger.warning( + "Invalid %s value; expected a mapping of address or range to limit", SOURCE_LIMIT_OVERRIDES_KEY + ) + return default + address: Final = _parse_address(client_ip) + if address is None: + return default + matches: Final = sorted( + _precedence(network, _override_limit(raw_limit, default)) + for raw_range, raw_limit in overrides.items() + if (network := _parse_network(raw_range)) is not None and address in network + ) + return matches[-1][-1] if matches else default + + +def user_limit_for(source_limit: int) -> int: + """Failures allowed for one username from one address: half the address allowance, rounded down, at least 1.""" + return max(source_limit // 2, 1) + + +def source_group(client_ip: str) -> str: + """The bucket an address is counted in: IPv4 as is, IPv6 by its /64, so one prefix holder cannot rotate.""" + address: Final = _parse_address(client_ip) + if address is None: + return client_ip + if isinstance(address, ipaddress.IPv6Address): + return str(ipaddress.ip_network((address, IPV6_SOURCE_PREFIX_LENGTH), strict=False)) + return str(address) + + +class _Keys(NamedTuple): + pair_counter: str + pair_block: str + source_counter: str + source_block: str + + +@dataclass(frozen=True, slots=True) +class Block: + scope: Scope + retry_after: int + + +@dataclass(frozen=True, slots=True) +class LoginThrottle: + """Failed-login limits for one request's source address. + + ``source_limit`` is None when the source scope is off: ``trusted_proxy_ranges`` is unset, so the peer + address may be a shared ingress. An empty list means clients connect directly and the peer is the source. + ``user_limit`` is derived from the address allowance either way, see ``user_limit_for``. An address whose + override is ``EXEMPT`` gets a disabled throttle: nothing is counted or blocked for it. + """ + + client_ip: str + source_limit: int | None + user_limit: int + window_seconds: int + block_seconds: int + counters: LocalStore + blocks: LocalStore + redis_cache: RedisCache | None = None + enabled: bool = True + + @classmethod + def from_request( + cls, + request: Request, + general_settings: Mapping[str, object] | None, + redis_cache: RedisCache | None, + ) -> LoginThrottle: + settings: Final[Mapping[str, object]] = general_settings if general_settings is not None else EMPTY_MAPPING + proxies: Final = declared_proxy_ranges(settings) + resolved, _ = resolve_client_ip( + request, TrustedProxyConfig(use_forwarded_for=bool(proxies), trusted_proxy_cidrs=proxies or ()) + ) + source_limit: Final = _source_limit(settings, resolved or LOGIN_THROTTLE_UNKNOWN_SOURCE) + exempt: Final = source_limit == EXEMPT + return cls( + client_ip=resolved or LOGIN_THROTTLE_UNKNOWN_SOURCE, + source_limit=source_limit if proxies is not None and resolved is not None and not exempt else None, + user_limit=user_limit_for(source_limit), + window_seconds=_int_setting(settings, WINDOW_KEY, DEFAULT_FAILED_LOGIN_WINDOW_SECONDS), + block_seconds=_int_setting(settings, BLOCK_KEY, DEFAULT_FAILED_LOGIN_BLOCK_SECONDS), + counters=_COUNTERS, + blocks=_BLOCKS, + redis_cache=redis_cache, + enabled=not exempt and not _rate_limit_disabled(), + ) + + def _keys(self, username: str) -> _Keys: + group: Final = source_group(self.client_ip) + user: Final = hashlib.sha256(username.casefold().encode("utf-8")).hexdigest() + return _Keys( + pair_counter=f"{LOGIN_THROTTLE_CACHE_KEY_PREFIX}:{{{group}}}:user:{user}", + pair_block=f"{LOGIN_THROTTLE_CACHE_KEY_PREFIX}:{{{group}}}:block:user:{user}", + source_counter=f"{LOGIN_THROTTLE_CACHE_KEY_PREFIX}:{{{group}}}:source", + source_block=f"{LOGIN_THROTTLE_CACHE_KEY_PREFIX}:{{{group}}}:block:source", + ) + + async def attempt(self, username: str) -> LoginAttempt: + """Refuses a blocked key before any credential is looked at; otherwise hands back the attempt to settle.""" + if not self.enabled: + return LoginAttempt(throttle=self, username=username) + block: Final = await self._active_block(self._keys(username)) + if block is None: + return LoginAttempt(throttle=self, username=username) + verbose_proxy_logger.warning( + "Admin UI sign-in refused: the %s is blocked for %s more seconds; username=%r source=%s", + block.scope, + block.retry_after, + username, + self.client_ip, + ) + raise self.refused(block.retry_after) + + async def _active_block(self, keys: _Keys) -> Block | None: + local: Final = self._local_block_ttls(keys) + shared: Final = await self._shared_block_ttls(keys) + user_ttl: Final = max(local[0], shared[0]) + source_ttl: Final = max(local[1], shared[1]) + if self.source_limit is not None and source_ttl > 0: + return Block(scope="source", retry_after=source_ttl) + if user_ttl > 0: + return Block(scope="user", retry_after=user_ttl) + return None + + async def _shared_block_ttls(self, keys: _Keys) -> _BlockTtls: + if self.redis_cache is None: + return LOGIN_THROTTLE_NOT_BLOCKED + try: + return _LUA_BLOCK_TTLS.validate_python( + await self.redis_cache.async_register_script(_BLOCK_TTLS_LUA)(keys, ()) + ) + except _REDIS_FAILURES as err: + self._warn_redis(err) + return LOGIN_THROTTLE_NOT_BLOCKED + + def _local_block_ttls(self, keys: _Keys) -> _BlockTtls: + return self._local_block_ttl(keys.pair_block), self._local_block_ttl(keys.source_block) + + def _local_block_ttl(self, block_key: str) -> int: + expires_at: Final = _LOCAL_BLOCK_EXPIRY.validate_python(self.blocks.get_cache(block_key)) + if expires_at is None: + return 0 + return max(math.ceil(expires_at - time.time()), 0) + + async def record_failure(self, username: str) -> _BlockTtls: + keys: Final = self._keys(username) + source_limit: Final = self.source_limit or 0 + if self.redis_cache is not None: + try: + return _LUA_BLOCK_TTLS.validate_python( + await self.redis_cache.async_register_script(_RECORD_FAILURE_LUA)( + keys, (self.user_limit, source_limit, self.window_seconds, self.block_seconds) + ) + ) + except _REDIS_FAILURES as err: + self._warn_redis(err) + user_block: Final = self._local_bump(keys.pair_counter, keys.pair_block, self.user_limit) + if source_limit == 0 or user_block > 0: + return user_block, 0 + return user_block, self._local_bump(keys.source_counter, keys.source_block, source_limit) + + def _local_bump(self, count_key: str, block_key: str, limit: int) -> int: + blocked: Final = self._local_block_ttl(block_key) + if blocked > 0: + return blocked + count: Final = int(self.counters.increment_cache(count_key, 1, ttl=self.window_seconds)) + if count <= limit: + return 0 + self.blocks.set_cache(block_key, time.time() + self.block_seconds, ttl=self.block_seconds) + return self.block_seconds + + async def clear_pair(self, username: str) -> None: + pair_counter: Final = self._keys(username).pair_counter + if self.redis_cache is not None: + try: + await self.redis_cache.async_delete_cache(pair_counter) + except _REDIS_FAILURES as err: + self._warn_redis(err) + self.counters.delete_cache(pair_counter) + + def _warn_redis(self, err: Exception) -> None: + verbose_proxy_logger.warning( + "Redis failed while counting Admin UI sign-in attempts; using this worker's own counters " + "until it recovers: %s", + err, + ) + + @staticmethod + def refused(retry_after: int) -> ProxyException: + return ProxyException( + message="Too many failed sign-in attempts. Try again later.", + type=ProxyErrorTypes.auth_error, + param="username", + code=status.HTTP_429_TOO_MANY_REQUESTS, + headers={"Retry-After": str(retry_after)}, # mutable-ok: ProxyException writes into its headers dict + ) + + +@dataclass(frozen=True, slots=True) +class LoginAttempt: + throttle: LoginThrottle + username: str + + async def succeeded(self) -> None: + if not self.throttle.enabled: + return + await self.throttle.clear_pair(self.username) + + async def failed(self) -> None: + if not self.throttle.enabled: + return + user_block, source_block = await self.throttle.record_failure(self.username) + if user_block == 0 and source_block == 0: + return + verbose_proxy_logger.warning( + "Admin UI sign-in blocked for %s seconds after too many failures; scope=%s username=%r source=%s", + user_block or source_block, + "user" if user_block else "source", + self.username, + self.throttle.client_ip, + ) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index b7064802878..e0d599b0017 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -27,6 +27,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured +from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -44,6 +45,11 @@ from litellm.repositories.user_repository import UserRepository from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.ui_sso import ReturnedUITokenObject +INVALID_UI_CREDENTIALS_MESSAGE: Final = ( + "Invalid credentials used to access UI. Check 'UI_USERNAME' and 'UI_PASSWORD', or the password set for your user" +) +INVALID_USER_PASSWORD_MESSAGE: Final = "Invalid credentials used to access UI. Check the password set for your user" + async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: """Rehash legacy password (SHA256) to scrypt on successful login.""" @@ -92,6 +98,21 @@ def _matches_env_credentials(username: str, password: str, master_key: str | Non ) +def _admin_credentials_match( + username: str, password: str, master_key: str, general_settings: Mapping[str, object] +) -> bool: + return general_settings.get("disable_env_credential_login") is not True and _matches_env_credentials( + username, password, master_key + ) + + +def _invalid_credentials_message(general_settings: Mapping[str, object]) -> str: + """One rejection message for unknown usernames and wrong passwords alike, so neither can be enumerated.""" + if is_env_credential_login_enabled(general_settings): + return INVALID_UI_CREDENTIALS_MESSAGE + return INVALID_USER_PASSWORD_MESSAGE + + def is_env_credential_login_enabled(general_settings: Mapping[str, object]) -> bool: """Whether a login with UI_USERNAME/UI_PASSWORD (or the master-key fallback) can succeed. @@ -137,6 +158,7 @@ async def authenticate_user( password: str, master_key: str | None, prisma_client: PrismaClient | None, + throttle: LoginThrottle, general_settings: Mapping[str, object] = MappingProxyType({}), ) -> LoginResult: """ @@ -151,6 +173,7 @@ async def authenticate_user( password: Password from the login form master_key: Master key for the proxy (required) prisma_client: Prisma database client (optional) + throttle: Failed sign-in accounting for this request's source address general_settings: Proxy general_settings, checked for `disable_password_login_when_sso_enabled` and `disable_env_credential_login` @@ -163,9 +186,11 @@ async def authenticate_user( or if username/password login is disabled while SSO is configured Recovery: an admin locked out of the UI by - `disable_password_login_when_sso_enabled` can still administer the proxy over - the API with the master key (Authorization: Bearer ), which never - goes through this function. To restore UI username/password login, unset the + `disable_password_login_when_sso_enabled`, or by the failed sign-in block in + `throttle`, can still administer the proxy over the API with the master key + (Authorization: Bearer ), which never goes through this function. + No credential, the env admin credentials and the master key included, is + exempt from the block. To restore UI username/password login, unset the setting in config.yaml (or the DB-persisted general_settings) and restart the proxy; this is a deliberate, auditable config change rather than a hidden bypass. @@ -194,6 +219,19 @@ async def authenticate_user( code=500, ) + attempt: Final = await throttle.attempt(username) + return await _sign_in(username, password, master_key, prisma_client, attempt, general_settings) + + +async def _sign_in( + username: str, + password: str, + master_key: str, + prisma_client: PrismaClient | None, + attempt: LoginAttempt, + general_settings: Mapping[str, object], +) -> LoginResult: + admin_credentials_match: Final = _admin_credentials_match(username, password, master_key, general_settings) # Check if we can find the `username` in the db. On the UI, users can enter username=their email _user_row: LiteLLM_UserTable | None = None user_role: ( @@ -219,20 +257,13 @@ async def authenticate_user( - Login with UI_USERNAME and UI_PASSWORD - Login with Invite Link `user_email` and `password` combination """ - if general_settings.get("disable_env_credential_login") is not True and _matches_env_credentials( - username, password, master_key - ): + if admin_credentials_match: # Non SSO -> If user is using UI_USERNAME and UI_PASSWORD they are Proxy admin user_role = LitellmUserRoles.PROXY_ADMIN user_id = LITELLM_PROXY_ADMIN_NAME # we want the key created to have PROXY_ADMIN_PERMISSIONS - key_user_id = LITELLM_PROXY_ADMIN_NAME - if ( - os.getenv("PROXY_ADMIN_ID", None) is not None and os.environ["PROXY_ADMIN_ID"] == user_id - ) or user_id == LITELLM_PROXY_ADMIN_NAME: - # checks if user is admin - key_user_id = os.getenv("PROXY_ADMIN_ID", LITELLM_PROXY_ADMIN_NAME) + key_user_id: Final = os.getenv("PROXY_ADMIN_ID", LITELLM_PROXY_ADMIN_NAME) # Admin is Authe'd in - generate key for the UI to access Proxy @@ -294,6 +325,8 @@ async def authenticate_user( key = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(user_info) + await attempt.succeeded() + return LoginResult( user_id=user_id, key=key, @@ -349,6 +382,8 @@ async def authenticate_user( key = response["token"] + await attempt.succeeded() + return LoginResult( user_id=user_id, key=key, @@ -357,20 +392,17 @@ async def authenticate_user( login_method="username_password", ) else: + await attempt.failed() raise ProxyException( - message=f"Invalid credentials used to access UI.\nNot valid credentials for {username}", + message=_invalid_credentials_message(general_settings), type=ProxyErrorTypes.auth_error, param="invalid_credentials", code=401, ) else: - env_credentials_hint: Final = ( - "\nCheck 'UI_USERNAME', 'UI_PASSWORD' in .env file" - if is_env_credential_login_enabled(general_settings) - else "" - ) + await attempt.failed() raise ProxyException( - message=f"Invalid credentials used to access UI.{env_credentials_hint}", + message=_invalid_credentials_message(general_settings), type=ProxyErrorTypes.auth_error, param="invalid_credentials", code=401, diff --git a/litellm/proxy/auth/network.py b/litellm/proxy/auth/network.py index 32ad18d4deb..4e8ab7512a7 100644 --- a/litellm/proxy/auth/network.py +++ b/litellm/proxy/auth/network.py @@ -1,6 +1,7 @@ from __future__ import annotations import ipaddress +from collections.abc import Sequence from typing import Any, Final from fastapi import Request @@ -19,7 +20,7 @@ class NetworkContext(BaseModel): class TrustedProxyConfig(BaseModel): use_forwarded_for: bool = False - trusted_proxy_cidrs: list[str] = Field(default_factory=list) + trusted_proxy_cidrs: Sequence[str] = Field(default_factory=tuple) def normalize_cidr_ranges(configured_ranges: Any, *, setting_name: str = "trusted_proxy_cidrs") -> list[str]: @@ -49,6 +50,12 @@ def parse_trusted_proxy_ranges( return networks +def _unmapped(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> ipaddress.IPv4Address | ipaddress.IPv6Address: + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + return addr.ipv4_mapped + return addr + + def ip_in_networks(client_ip: str | None, networks: list[TrustedProxyNetwork]) -> bool: if not client_ip or not networks: return False @@ -56,7 +63,8 @@ def ip_in_networks(client_ip: str | None, networks: list[TrustedProxyNetwork]) - addr: Final = ipaddress.ip_address(client_ip.strip()) except ValueError: return False - return any(addr in network for network in networks) + candidates: Final = (addr, _unmapped(addr)) + return any(candidate in network for candidate in candidates for network in networks) def _is_valid_ip(value: str) -> bool: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index b25c77f6828..9f2e4c9802e 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1411,6 +1411,8 @@ def run_server( # DO NOT DELETE - enables global variables to work across files from litellm.proxy.proxy_server import app + os.environ["NUM_WORKERS"] = str(num_workers) + # Auto-create PROMETHEUS_MULTIPROC_DIR for multi-worker setups prometheus_multiproc_dir: Final = ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( num_workers=num_workers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b5f4236d22..dc2aa2d6fc7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -330,6 +330,12 @@ from litellm.proxy.auth.fallback_budget import router_fallback_budget_check from litellm.proxy.auth.fallback_model_access import router_fallback_access_check from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY, LicenseCheck +from litellm.proxy.auth.login_throttle import ( + LoginThrottle, + declared_proxy_ranges, + warn_login_counters_are_per_worker, + warn_source_login_limit_is_off, +) from litellm.proxy.auth.model_checks import ( expand_wildcard_deployments_for_model_info, get_all_fallbacks, @@ -827,6 +833,7 @@ from fastapi.openapi.docs import get_swagger_ui_html from fastapi.openapi.utils import get_openapi from fastapi.responses import ( FileResponse, + HTMLResponse, JSONResponse, ORJSONResponse, RedirectResponse, @@ -6050,6 +6057,12 @@ class ProxyConfig: general_settings = config.get("general_settings", {}) if general_settings is None: general_settings = {} + + if os.getenv("NUM_WORKERS", "1") != "1" and redis_usage_cache is None: + warn_login_counters_are_per_worker(os.getenv("NUM_WORKERS", "1")) + if declared_proxy_ranges(general_settings) is None: + warn_source_login_limit_is_off() + _bg_hc_model_groups: Final = parse_background_health_check_model_groups(general_settings) _enable_hc_routing = False _hc_staleness = None @@ -15901,8 +15914,6 @@ async def fallback_login(request: Request): else: redirect_url += "/sso/callback" - from fastapi.responses import HTMLResponse - hide_default_credentials_hint: Final = should_hide_default_credentials_hint(general_settings) return HTMLResponse( content=build_ui_login_form( @@ -15924,13 +15935,27 @@ async def login(request: Request): password: Final = str(form.get("password")) # Authenticate user and get login result - login_result: Final = await authenticate_user( - username=username, - password=password, - master_key=master_key, - prisma_client=prisma_client, - general_settings=general_settings, - ) + try: + login_result: Final = await authenticate_user( + username=username, + password=password, + master_key=master_key, + prisma_client=prisma_client, + throttle=LoginThrottle.from_request(request, general_settings, redis_usage_cache), + general_settings=general_settings, + ) + except ProxyException as exc: + if int(exc.code) != status.HTTP_429_TOO_MANY_REQUESTS: + raise + retry_after: Final = exc.headers.get("Retry-After", "30") + return HTMLResponse( + content=( + "

Too many sign-in attempts

" + f"

Try again in about {retry_after} seconds

" + ), + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + headers=exc.headers, + ) # Create UI token object returned_ui_token_object: Final = create_ui_token_object( @@ -16009,6 +16034,7 @@ async def login_v2(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + throttle=LoginThrottle.from_request(request, general_settings, redis_usage_cache), general_settings=general_settings, ) @@ -16080,6 +16106,7 @@ async def login_v3(request: Request): password=password, master_key=master_key, prisma_client=prisma_client, + throttle=LoginThrottle.from_request(request, general_settings, redis_usage_cache), general_settings=general_settings, ) diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index e209a491b0a..55ece36252d 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -6,11 +6,33 @@ to login_utils.py for better reusability. """ import os +from collections.abc import Mapping from contextlib import ExitStack +from typing import TYPE_CHECKING, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +if TYPE_CHECKING: + from litellm.proxy.auth.login_throttle import LoginThrottle + + +def _unlimited_throttle(): + """A throttle wired to real in-memory stores with limits no test can reach.""" + from litellm.caching.in_memory_cache import InMemoryCache + from litellm.proxy.auth.login_throttle import LoginThrottle + + return LoginThrottle( + client_ip="1.2.3.4", + source_limit=None, + user_limit=10_000, + window_seconds=60, + block_seconds=300, + counters=InMemoryCache(), + blocks=InMemoryCache(), + ) + + from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._types import ( LiteLLM_UserTable, @@ -100,6 +122,7 @@ async def test_authenticate_user_admin_login_with_ui_credentials(): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -157,6 +180,7 @@ async def test_authenticate_user_admin_login_with_master_key_as_password(monkeyp password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -181,6 +205,7 @@ async def test_authenticate_user_invalid_credentials(): password=wrong_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -200,6 +225,7 @@ async def test_authenticate_user_missing_master_key(): password="password", master_key=None, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -240,6 +266,7 @@ async def test_authenticate_user_wrong_password(): password=wrong_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -298,12 +325,14 @@ async def test_authenticate_user_email_case_insensitive_login(): password=correct_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) result_lower = await authenticate_user( username=stored_email, password=correct_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert result_mixed.user_id == result_lower.user_id == "test-user-123" @@ -345,6 +374,7 @@ async def test_authenticate_user_database_required_for_admin(monkeypatch): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert exc_info.value.type == ProxyErrorTypes.auth_error @@ -396,6 +426,7 @@ async def test_authenticate_user_admin_login_with_non_ascii_characters(): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -472,18 +503,21 @@ async def test_authenticate_user_multiple_logins_generate_unique_tokens(): password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) result2 = await authenticate_user( username=ui_username, password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) result3 = await authenticate_user( username=ui_username, password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) # Each login should return a unique token @@ -541,6 +575,7 @@ async def test_authenticate_user_database_login_with_non_ascii_password(): password=password_with_special_char, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), ) assert isinstance(result, LoginResult) @@ -603,6 +638,1070 @@ class TestEncodeUiSessionJwt: assert _user_id_from_session_cookie(request) == "cornell-user" +def _throttle( + user_limit: int = 2, + source_limit: int | None = None, + window_seconds: int = 60, + block_seconds: int = 300, + client_ip: str = "1.2.3.4", + stores=None, + redis_cache=None, +): + """A throttle over real in-memory stores, so the tests exercise the true counters and blocks.""" + from litellm.caching.in_memory_cache import InMemoryCache + from litellm.proxy.auth.login_throttle import LoginThrottle + + counters, blocks = stores if stores is not None else (InMemoryCache(), InMemoryCache()) + return LoginThrottle( + client_ip=client_ip, + source_limit=source_limit, + user_limit=user_limit, + window_seconds=window_seconds, + block_seconds=block_seconds, + counters=counters, + blocks=blocks, + redis_cache=redis_cache, + ) + + +def _stores(): + from litellm.caching.in_memory_cache import InMemoryCache + + return InMemoryCache(), InMemoryCache() + + +async def _guess(throttle, username: str = "admin", password: str = "wrong"): + from litellm.proxy.auth.login_utils import authenticate_user + + return await authenticate_user( + username=username, + password=password, + master_key="sk-master", + prisma_client=None, + throttle=throttle, + ) + + +async def _fail(throttle, username: str = "admin") -> str: + """One wrong guess; returns the status code it was answered with.""" + from litellm.proxy._types import ProxyException + + with pytest.raises(ProxyException) as exc: + await _guess(throttle, username=username) + return exc.value.code + + +def _known_user(email: str = "known@example.com"): + user = MagicMock() + user.user_id = "u-1" + user.user_email = email + user.user_role = "internal_user" + user.password = "scrypt:stored" + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=user) + return repo + + +async def _db_login(throttle, username: str, password: str, *, correct: bool): + """A database user's sign-in with the stored hash faked, so no database or scrypt is needed.""" + from litellm.proxy.auth.login_utils import authenticate_user + + with ( + patch( # test-quality-ok: the user lookup is the database boundary; faked so no DB is needed + "litellm.proxy.auth.login_utils.UserRepository", _known_user(username) + ), + patch( # test-quality-ok: reaches the known-DB-user branch without a database + "litellm.proxy.auth.login_utils.verify_password", return_value=correct + ), + patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed + "litellm.proxy.auth.login_utils._rehash_password_if_needed", new=AsyncMock() + ), + patch( # test-quality-ok: success mints a UI key; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + return await authenticate_user( + username=username, password=password, master_key="sk-master", prisma_client=MagicMock(), throttle=throttle + ) + + +def _local_count(throttle, key: str) -> int: + return int(throttle.counters.get_cache(key) or 0) + + +@pytest.mark.asyncio +async def test_too_many_failures_for_one_username_block_that_pair_and_carry_retry_after(monkeypatch): + """One failure past the pair limit blocks the source for that username; the next guess is answered 429 + with the block's remaining time, and the counter is not touched by blocked guesses.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2, block_seconds=77) + keys = throttle._keys("admin") + + assert [await _fail(throttle) for _ in range(3)] == ["401", "401", "401"], "the limit itself is a plain 401" + assert throttle._local_block_ttl(keys.pair_block) == 77 + + with pytest.raises(ProxyException) as blocked: + await _guess(throttle) + assert blocked.value.code == "429" + assert blocked.value.headers.get("Retry-After") == "77" + assert _local_count(throttle, keys.pair_counter) == 3, "a blocked guess is not counted again" + + +@pytest.mark.asyncio +async def test_a_blocked_key_is_refused_before_the_password_is_looked_at(monkeypatch): + """The block is the rate cap: once a key is blocked, nothing from it reaches the user lookup or the + password check, so a guessing script gets no verification work out of the proxy.""" + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.login_utils import authenticate_user + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=1) + + assert [await _fail(throttle, username="user@corp.com") for _ in range(3)] == ["401", "401", "429"] + + lookup = _known_user("user@corp.com") + verify = MagicMock(return_value=True) + with ( + patch( # test-quality-ok: the user lookup is the database boundary; a blocked attempt must not reach it + "litellm.proxy.auth.login_utils.UserRepository", lookup + ), + patch( # test-quality-ok: the password check is the expensive step; a blocked attempt must not reach it + "litellm.proxy.auth.login_utils.verify_password", verify + ), + pytest.raises(ProxyException) as refused, + ): + await authenticate_user( + username="user@corp.com", + password="right", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + ) + + assert refused.value.code == "429" + assert lookup.return_value.table.find_first.await_count == 0 + assert verify.call_count == 0 + + +@pytest.mark.asyncio +async def test_a_correct_password_is_refused_while_its_pair_is_blocked(monkeypatch): + """Letting the right password through would give a guesser unlimited tries, so the block is hard: the + real user waits it out, or uses the master key over the API, which never passes through here.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=1, block_seconds=90) + + assert [await _fail(throttle, username="user@corp.com") for _ in range(3)] == ["401", "401", "429"] + + with pytest.raises(ProxyException) as refused: + await _db_login(throttle, "user@corp.com", "right", correct=True) + assert refused.value.code == "429" + assert refused.value.headers.get("Retry-After") == "90" + + +@pytest.mark.asyncio +async def test_a_correct_password_is_refused_while_its_source_is_blocked(monkeypatch): + """Same for the source-wide block: every username from that address is refused until it lapses.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=100, source_limit=2) + + for i in range(3): + assert await _fail(throttle, username=f"other-{i}@corp.com") == "401" + assert await _fail(throttle, username="other-9@corp.com") == "429", "the source is blocked for everyone" + + with pytest.raises(ProxyException) as refused: + await _db_login(throttle, "user@corp.com", "right", correct=True) + assert refused.value.code == "429" + + +@pytest.mark.asyncio +async def test_a_successful_sign_in_clears_the_pair_counter_but_not_the_source_counter(monkeypatch): + """One account's success says nothing about the other guesses the address is making.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=5, source_limit=50) + keys = throttle._keys("user@corp.com") + + for _ in range(2): + assert await _fail(throttle, username="user@corp.com") == "401" + assert _local_count(throttle, keys.pair_counter) == 2 + assert _local_count(throttle, keys.source_counter) == 2 + + await _db_login(throttle, "user@corp.com", "right", correct=True) + + assert _local_count(throttle, keys.pair_counter) == 0 + assert _local_count(throttle, keys.source_counter) == 2 + + +@pytest.mark.asyncio +async def test_once_a_pair_is_blocked_its_failures_stop_counting_against_the_source(monkeypatch): + """A script stuck on one account trips the pair block and then leaves the office's shared address alone.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2, source_limit=4) + keys = throttle._keys("stuck-script@corp.com") + + assert [await _fail(throttle, username="stuck-script@corp.com") for _ in range(3)] == ["401"] * 3 + assert _local_count(throttle, keys.source_counter) == 2, "failures before the pair block count for the source" + + for _ in range(5): + assert await _fail(throttle, username="stuck-script@corp.com") == "429" + assert _local_count(throttle, keys.source_counter) == 2, "blocked-pair failures must not reach the source" + + assert await _fail(throttle, username="colleague@corp.com") == "401", "a colleague still signs in normally" + assert throttle._local_block_ttl(keys.source_block) == 0 + + +@pytest.mark.asyncio +async def test_the_blocking_failure_itself_does_not_count_against_the_source(monkeypatch): + """The guess that installs the pair block is the first one that stops counting, so a pair limit of B + costs the source exactly B, not B plus one.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2, source_limit=2) + keys = throttle._keys("stuck@corp.com") + + assert [await _fail(throttle, username="stuck@corp.com") for _ in range(3)] == ["401", "401", "401"] + + assert _local_count(throttle, keys.source_counter) == 2 + assert throttle._local_block_ttl(keys.source_block) == 0, "the third guess blocked the pair, not the source" + + +@pytest.mark.asyncio +async def test_too_many_failures_across_usernames_block_the_whole_source(monkeypatch): + """A spray of one guess per username never trips a pair; the source counter is what stops it.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=5, source_limit=3, block_seconds=200) + + assert [await _fail(throttle, username=f"sprayed-{i}@corp.com") for i in range(4)] == ["401"] * 4 + + assert await _fail(throttle, username="sprayed-99@corp.com") == "429" + assert throttle._local_block_ttl(throttle._keys("x").source_block) == 200 + + +@pytest.mark.asyncio +async def test_without_trusted_proxy_ranges_the_source_scope_is_off(monkeypatch): + """Behind an ingress every client shares the peer address, so a source-wide block would block them all. + The pair scope still applies.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.9"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + throttle = LoginThrottle.from_request( + request, general_settings={"max_failed_login_attempts_per_source": 1}, redis_cache=None + ) + + assert throttle.source_limit is None + assert throttle.client_ip == "10.0.0.1", "the header is not trusted without a configured proxy range" + assert [await _fail(throttle, username=f"user-{i}@corp.com") for i in range(6)] == ["401"] * 6 + + +@pytest.mark.asyncio +async def test_an_empty_trusted_proxy_ranges_means_the_peer_is_the_client_and_the_source_scope_is_on(monkeypatch): + """An explicit empty list says there are no proxies: the peer address is the client, the forwarded header + is ignored, and the source-wide limit applies. Only an unset key means the topology is unknown.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.9"} + request.client = MagicMock() + request.client.host = "198.51.100.7" + throttle = LoginThrottle.from_request( + request, + general_settings={"trusted_proxy_ranges": [], "max_failed_login_attempts_per_source": 3}, + redis_cache=None, + ) + + assert throttle.client_ip == "198.51.100.7" + assert throttle.source_limit == 3 + assert [await _fail(throttle, username=f"user-{i}@corp.com") for i in range(4)] == ["401"] * 4 + assert await _fail(throttle, username="user-99@corp.com") == "429", "the spray is stopped by the source limit" + + +@pytest.mark.parametrize( + "configured", + [ + None, + 5, + {"10.0.0.0/8": True}, + ["", " "], + ["not-a-range"], + ["10.0.0.0/8, 172.16.0.0/12"], + ["10.0.0.0/8", "10.0.0.0/33"], + ["10.0.0.0/8", " "], + ["10.0.0.0/8", ""], + ["10.0.0.0/8", None], + "10.0.0.0/8;172.16.0.0/12", + "10.0.0.0/8,", + "", + ], +) +def test_a_trusted_proxy_ranges_value_that_names_no_ranges_leaves_the_topology_unknown(configured): + """Only a list of valid ranges or an explicit empty list counts as a declaration; anything else, including a + list with one bad entry, is the same as unset, so a typo cannot switch the source-wide block on against + the shared ingress address and lock out everyone behind it.""" + from litellm.proxy.auth.login_throttle import LoginThrottle, declared_proxy_ranges + + settings = {"trusted_proxy_ranges": configured} if configured is not None else {} + assert declared_proxy_ranges(settings) is None + + request = MagicMock() + request.headers = {} + request.client = MagicMock() + request.client.host = "198.51.100.7" + throttle = LoginThrottle.from_request(request, general_settings=settings, redis_cache=None) + assert throttle.source_limit is None + assert throttle.client_ip == "198.51.100.7" + + +def test_declared_proxy_ranges_distinguishes_none_from_empty_from_configured(): + from litellm.proxy.auth.login_throttle import declared_proxy_ranges + + assert declared_proxy_ranges({}) is None + assert declared_proxy_ranges({"trusted_proxy_ranges": []}) == () + assert declared_proxy_ranges({"trusted_proxy_ranges": ["10.0.0.0/8", " 192.168.1.1 "]}) == ( + "10.0.0.0/8", + "192.168.1.1", + ) + assert declared_proxy_ranges({"trusted_proxy_ranges": "10.0.0.0/8,172.16.0.0/12"}) == ( + "10.0.0.0/8", + "172.16.0.0/12", + ) + + +@pytest.mark.asyncio +async def test_with_trusted_proxy_ranges_the_source_is_the_forwarded_client(monkeypatch): + """The header is walked right to left past the trusted hops, so a forged left-most entry cannot pick the bucket.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + settings = {"trusted_proxy_ranges": ["10.0.0.0/8"], "max_failed_login_attempts_per_source": 2} + + def _from(peer: str, forwarded: str): + request = MagicMock() + request.headers = {"x-forwarded-for": forwarded} + request.client = MagicMock() + request.client.host = peer + return LoginThrottle.from_request(request, general_settings=settings, redis_cache=None) + + via_proxy = _from("10.0.0.1", "1.1.1.1, 203.0.113.9, 10.0.0.2") + assert via_proxy.client_ip == "203.0.113.9" + assert via_proxy.source_limit == 2 + + direct = _from("198.51.100.7", "203.0.113.9") + assert direct.client_ip == "198.51.100.7", "a peer outside the trusted ranges cannot forward anything" + + +def test_source_overrides_pick_the_most_specific_matching_range(): + """An exact address beats a /16 beats a /8; an address in none of them keeps the default.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + settings = { + "trusted_proxy_ranges": ["10.0.0.0/8"], + "max_failed_login_attempts_per_source": 7, + "max_failed_login_attempts_per_source_overrides": { + "203.0.0.0/8": 100, + "203.0.113.0/24": 200, + "203.0.113.9": 300, + "not-an-address": 999, + "198.51.100.0/24": "not-a-number", + }, + } + + def _limit(client: str) -> int | None: + request = MagicMock() + request.headers = {"x-forwarded-for": client} + request.client = MagicMock() + request.client.host = "10.0.0.1" + return LoginThrottle.from_request(request, general_settings=settings, redis_cache=None).source_limit + + assert _limit("203.0.113.9") == 300 + assert _limit("203.0.113.10") == 200 + assert _limit("203.0.1.1") == 100 + assert _limit("192.0.2.1") == 7 + assert _limit("198.51.100.1") == 7, "a garbage limit falls back to the default rather than a huge or zero budget" + assert _limit("::ffff:203.0.113.9") == 300, "a mapped address gets the limit of the IPv4 bucket it is counted in" + assert _limit("::ffff:203.0.113.10") == 200 + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + ({"203.0.113.7": 0, "203.0.113.7/32": 5}, None), + ({"203.0.113.7/32": 5, "203.0.113.7": 0}, None), + ({"203.0.113.0/24": 3, "203.0.113.9/24": 8}, 8), + ({"203.0.113.9/24": 8, "203.0.113.0/24": 3}, 8), + ], + ids=["exact-then-slash32", "slash32-then-exact", "low-then-high", "high-then-low"], +) +def test_equivalent_override_keys_resolve_to_the_exemption_then_the_higher_limit(overrides, expected): + """Two spellings of the same network are a config mistake, so precedence must not depend on dict order.""" + settings = {"trusted_proxy_ranges": ["10.0.0.0/8"], "max_failed_login_attempts_per_source_overrides": overrides} + + assert _throttle_behind_trusted_proxy("203.0.113.7", settings).source_limit == expected + + +def test_ipv6_sources_are_grouped_by_their_64_bit_prefix(): + """A /64 holder has 2^64 addresses; counting each one separately would hand them unlimited fresh buckets.""" + from litellm.proxy.auth.login_throttle import source_group + + assert source_group("2001:db8:1:2::1") == source_group("2001:db8:1:2:ffff:ffff:ffff:ffff") == "2001:db8:1:2::/64" + assert source_group("2001:db8:1:3::1") != source_group("2001:db8:1:2::1") + assert source_group("::ffff:203.0.113.9") == source_group("203.0.113.9") == "203.0.113.9" + assert source_group("unknown") == "unknown" + + +@pytest.mark.asyncio +async def test_two_ipv6_addresses_in_one_64_share_the_source_budget(monkeypatch): + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + stores = _stores() + first = _throttle(user_limit=50, source_limit=2, client_ip="2001:db8:1:2::1", stores=stores) + second = _throttle(user_limit=50, source_limit=2, client_ip="2001:db8:1:2::2", stores=stores) + + assert [await _fail(first, username=f"a-{i}@corp.com") for i in range(3)] == ["401"] * 3 + assert await _fail(second, username="b@corp.com") == "429" + + +@pytest.mark.asyncio +async def test_one_source_being_blocked_does_not_touch_another(monkeypatch): + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + stores = _stores() + attacker = _throttle(user_limit=50, source_limit=2, client_ip="203.0.113.9", stores=stores) + neighbour = _throttle(user_limit=50, source_limit=2, client_ip="198.51.100.7", stores=stores) + + assert [await _fail(attacker, username=f"t-{i}@corp.com") for i in range(3)] == ["401"] * 3 + assert await _fail(attacker, username="t-9@corp.com") == "429" + assert await _fail(neighbour, username="t-9@corp.com") == "401" + + +@pytest.mark.asyncio +async def test_the_same_username_from_another_source_has_its_own_budget(monkeypatch): + """The pair carries the address on purpose: an attacker elsewhere cannot lock a user out of their own office.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + stores = _stores() + attacker = _throttle(user_limit=1, client_ip="203.0.113.9", stores=stores) + office = _throttle(user_limit=1, client_ip="198.51.100.7", stores=stores) + + assert [await _fail(attacker, username="victim@corp.com") for _ in range(3)] == ["401", "401", "429"] + assert await _fail(office, username="victim@corp.com") == "401" + + +@pytest.mark.asyncio +async def test_the_counting_window_is_anchored_at_the_first_failure(monkeypatch): + """Later failures must not push the expiry out, or a slow guesser keeps their own count alive forever.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=50, window_seconds=60) + key = throttle._keys("admin").pair_counter + + await _fail(throttle) + first_expiry = throttle.counters.ttl_dict[key] + for _ in range(3): + await _fail(throttle) + + assert throttle.counters.ttl_dict[key] == first_expiry + + +@pytest.mark.asyncio +async def test_the_block_outlives_the_counting_window(monkeypatch): + """Counters expire after the window and blocks after the block time; the two are separate keys.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1, window_seconds=10, block_seconds=300) + keys = throttle._keys("admin") + + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"] + + throttle.counters.delete_cache(keys.pair_counter) + + assert await _fail(throttle) == "429", "an expired counter must not lift an active block" + assert 290 <= throttle._local_block_ttl(keys.pair_block) <= 300 + + +@pytest.mark.asyncio +async def test_the_block_time_is_fixed_and_not_refreshed_by_blocked_guesses(monkeypatch): + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1, block_seconds=300) + key = throttle._keys("admin").pair_block + + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"] + installed_at = throttle.blocks.ttl_dict[key] + + for _ in range(4): + assert await _fail(throttle) == "429" + + assert throttle.blocks.ttl_dict[key] == installed_at + + +@pytest.mark.asyncio +async def test_the_configured_admin_credentials_are_not_exempt_from_the_block(monkeypatch): + """Exempting the env credentials would make them the one password worth guessing without limit, so the + right UI_PASSWORD is refused while its pair is blocked, and signs in normally once the block lapses.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=1) + + assert [await _fail(throttle) for _ in range(3)] == ["401", "401", "429"] + + with ( + patch( # test-quality-ok: the admin sign-in upserts the admin row; faked so no DB is needed + "litellm.proxy.auth.login_utils.user_update", new=AsyncMock() + ), + patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + with pytest.raises(ProxyException) as refused: + await _guess(throttle, password="right") + assert refused.value.code == "429" + + throttle.blocks.delete_cache(throttle._keys("admin").pair_block) + result = await _guess(throttle, password="right") + assert result.key == "sk-ui" + + +@pytest.mark.asyncio +async def test_the_master_key_used_as_the_ui_password_is_not_exempt_from_the_block(monkeypatch): + """Without UI_PASSWORD the master key doubles as the admin password; it gets no special treatment here + either. Lockout recovery is the master key as a bearer token over the API, which never enters this path.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.delenv("UI_PASSWORD", raising=False) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=1) + + assert [await _fail(throttle) for _ in range(3)] == ["401", "401", "429"] + + with pytest.raises(ProxyException) as refused: + await _guess(throttle, password="sk-master") + assert refused.value.code == "429" + + +@pytest.mark.asyncio +async def test_a_configuration_error_never_counts(monkeypatch): + """A 500 from an unset master key is not a guess and must not consume the budget.""" + from litellm.proxy._types import ProxyException + + throttle = _throttle(user_limit=2) + for _ in range(5): + with pytest.raises(ProxyException) as exc: + await authenticate_user( + username="admin", password="x", master_key=None, prisma_client=None, throttle=throttle + ) + assert exc.value.code == "500" + + assert _local_count(throttle, throttle._keys("admin").pair_counter) == 0 + + +@pytest.mark.asyncio +async def test_the_username_is_case_folded_into_one_pair(monkeypatch): + """The DB lookup is case-insensitive, so casing must not multiply the budget.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=4) + + for name in ("admin@corp.com", "ADMIN@corp.com", "Admin@corp.com", "aDmIn@corp.com", "admin@CORP.com"): + assert await _fail(throttle, username=name) == "401" + + assert await _fail(throttle, username="admin@Corp.com") == "429" + + +@pytest.mark.asyncio +async def test_both_credential_rejections_are_indistinguishable(monkeypatch): + """One message for the known and the unknown username, so responses do not enumerate.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + + with pytest.raises(ProxyException) as unknown: + await _guess(_throttle(user_limit=99), username="nobody@example.com") + with pytest.raises(ProxyException) as known: + await _db_login(_throttle(user_limit=99), "known@example.com", "wrong", correct=False) + + assert unknown.value.message == known.value.message + assert "known@example.com" not in unknown.value.message + known.value.message + + +@pytest.mark.asyncio +async def test_a_user_with_no_password_set_does_not_consume_the_budget(monkeypatch): + """That 401 is deterministic and guards no secret, so counting it would only let someone burn the pair.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2) + + passwordless = MagicMock() + passwordless.user_id = "u-2" + passwordless.user_email = "nopass@example.com" + passwordless.user_role = "internal_user" + passwordless.password = None + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=passwordless) + + with patch( # test-quality-ok: reaches the passwordless-DB-user branch without a database + "litellm.proxy.auth.login_utils.UserRepository", repo + ): + for _ in range(5): + with pytest.raises(ProxyException) as exc: + await authenticate_user( + username="nopass@example.com", + password="x", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + ) + assert exc.value.code == "401" + + assert _local_count(throttle, throttle._keys("nopass@example.com").pair_counter) == 0 + + +@pytest.mark.asyncio +async def test_a_wrong_password_for_a_known_user_also_counts(monkeypatch): + """The database-user branch must charge the pair too, not just the unknown-user branch.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2) + + for _ in range(3): + with pytest.raises(ProxyException) as rejected: + await _db_login(throttle, "known@example.com", "wrong", correct=False) + assert rejected.value.code == "401" + + with pytest.raises(ProxyException) as blocked: + await _db_login(throttle, "known@example.com", "wrong", correct=False) + assert blocked.value.code == "429" + + +@pytest.mark.asyncio +async def test_a_source_block_outranks_a_pair_block_in_the_retry_after(monkeypatch): + """When both scopes are blocked, the answer carries the source block's time, which is the one that + still applies to every other username from that address.""" + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1, source_limit=3, block_seconds=120, client_ip="203.0.113.45") + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"], "the admin pair is now blocked" + throttle.blocks.set_cache(throttle._keys("admin").pair_block, 1, ttl=30) + assert [await _fail(throttle, username=f"spray-{i}@corp.com") for i in range(3)] == ["401"] * 3 + assert throttle._local_block_ttl(throttle._keys("admin").source_block) == 120, "the source is now blocked too" + + for name in ("admin", "spray-0@corp.com", "never-seen@corp.com"): + with pytest.raises(ProxyException) as refused: + await _guess(throttle, username=name) + assert refused.value.code == "429" + assert refused.value.headers.get("Retry-After") == "120", name + + +@pytest.mark.asyncio +async def test_disabling_the_control_lets_every_attempt_through(monkeypatch): + """The escape hatch has to turn off the whole control: no counting and no refusal.""" + import dataclasses + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = dataclasses.replace(_throttle(user_limit=1), enabled=False) + + assert [await _fail(throttle) for _ in range(6)] == ["401"] * 6 + assert _local_count(throttle, throttle._keys("admin").pair_counter) == 0 + + +class _FakeRedis: + """Redis whose only writes are the throttle's two scripts, run atomically as one call each. + + Mirrors the Lua: a blocked key returns its remaining block time and is not counted; a counter + is expired on first write; one over the limit installs the block; a blocked pair stops the + source from being counted. The real scripts are exercised against a live Redis in the PR's + proof, this fake only has to be faithful enough for the worker-sharing tests. + """ + + def __init__(self): + self.values: dict = {} + self.ttls: dict = {} + self.scripts: list[str] = [] + + def async_register_script(self, script: str): + from litellm.proxy.auth import login_throttle as lt + + async def _run(keys, args): + self.scripts.append(script) + if script == lt._BLOCK_TTLS_LUA: + return [self._ttl(keys[1]), self._ttl(keys[3])] + assert script == lt._RECORD_FAILURE_LUA + user_limit, source_limit, window, block = (int(a) for a in args) + user_block = self._bump(keys[0], keys[1], user_limit, window, block) + if source_limit > 0 and user_block == 0: + return [user_block, self._bump(keys[2], keys[3], source_limit, window, block)] + return [user_block, 0] + + return _run + + def _ttl(self, key: str) -> int: + return self.ttls.get(key, -2) if key in self.values else -2 + + def _bump(self, count_key: str, block_key: str, limit: int, window: int, block: int) -> int: + if self._ttl(block_key) > 0: + return self._ttl(block_key) + self.values[count_key] = self.values.get(count_key, 0) + 1 + self.ttls.setdefault(count_key, window) + if self.values[count_key] > limit: + self.values[block_key] = 1 + self.ttls[block_key] = block + return block + return 0 + + async def async_delete_cache(self, key): + self.values.pop(key, None) + self.ttls.pop(key, None) + + +class _DownRedis(_FakeRedis): + """Redis whose every call fails, as during an outage or an open circuit breaker.""" + + def async_register_script(self, script: str): + async def _run(keys, args): + raise ConnectionError("redis is down") + + return _run + + async def async_delete_cache(self, key): + raise ConnectionError("redis is down") + + +@pytest.mark.asyncio +async def test_redis_is_the_only_counter_while_it_answers(monkeypatch): + """Every worker must spend the same budget, see the same block, and a success must clear the pair for all.""" + from litellm.constants import LOGIN_THROTTLE_CACHE_KEY_PREFIX + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + redis = _FakeRedis() + first_worker = _throttle(user_limit=2, stores=_stores(), redis_cache=redis) + second_worker = _throttle(user_limit=2, stores=_stores(), redis_cache=redis) + + assert [await _fail(first_worker, username="user@corp.com") for _ in range(3)] == ["401"] * 3 + assert not [k for k in first_worker.counters.cache_dict if str(k).startswith(LOGIN_THROTTLE_CACHE_KEY_PREFIX)], ( + "with Redis answering, no worker may keep a counter of its own" + ) + assert not first_worker.blocks.cache_dict + + assert await _fail(second_worker, username="user@corp.com") == "429", "the second worker sees the block" + + block_keys = [k for k in redis.values if ":block:user:" in k] + assert block_keys, "the block lives in Redis, where every worker reads it" + for key in block_keys: + await redis.async_delete_cache(key) + await _db_login(second_worker, "user@corp.com", "right", correct=True) + + assert not [k for k in redis.values if ":user:" in k and ":block:" not in k], ( + "success clears the shared pair counter" + ) + + +@pytest.mark.asyncio +async def test_a_redis_outage_falls_back_to_this_workers_own_counter(monkeypatch): + """With Redis raising, guesses are still counted and blocked per worker, with a warning, instead of unbounded.""" + import logging + + from litellm._logging import verbose_proxy_logger + from litellm.proxy._types import ProxyException + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2, block_seconds=300, redis_cache=_DownRedis()) + + records: list[logging.LogRecord] = [] + handler = logging.Handler() + handler.emit = records.append + verbose_proxy_logger.addHandler(handler) + try: + assert [await _fail(throttle) for _ in range(3)] == ["401"] * 3 + with pytest.raises(ProxyException) as blocked: + await _guess(throttle) + finally: + verbose_proxy_logger.removeHandler(handler) + + assert blocked.value.code == "429" + assert blocked.value.headers.get("Retry-After") == "300" + assert any("Redis failed while counting Admin UI sign-in attempts" in r.getMessage() for r in records) + + +@pytest.mark.asyncio +async def test_a_failed_redis_delete_still_clears_this_workers_counter(monkeypatch): + """The fail-open tradeoff: when Redis cannot clear the pair, the worker clears what it holds and moves on.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=5, redis_cache=_DownRedis()) + key = throttle._keys("user@corp.com").pair_counter + + assert [await _fail(throttle, username="user@corp.com") for _ in range(2)] == ["401", "401"] + assert _local_count(throttle, key) == 2 + + await _db_login(throttle, "user@corp.com", "right", correct=True) + assert _local_count(throttle, key) == 0 + + +@pytest.mark.asyncio +async def test_counters_do_not_share_the_key_authentication_cache(monkeypatch): + """Regression: throttle entries must not evict cached credentials from user_api_key_cache.""" + from litellm.constants import LOGIN_THROTTLE_CACHE_KEY_PREFIX + from litellm.proxy import proxy_server as ps + from litellm.proxy.auth.login_throttle import LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + auth_cache_keys_before = set(ps.user_api_key_cache.in_memory_cache.cache_dict) + + request = MagicMock() + request.headers = {} + request.client = MagicMock() + request.client.host = "1.2.3.4" + throttle = LoginThrottle.from_request(request, general_settings={}, redis_cache=None) + + for i in range(25): + assert await _fail(throttle, username=f"made-up-{i}@example.com") == "401" + + added = set(ps.user_api_key_cache.in_memory_cache.cache_dict) - auth_cache_keys_before + assert not [k for k in added if str(k).startswith(LOGIN_THROTTLE_CACHE_KEY_PREFIX)] + + +def test_settings_that_arrive_as_environment_strings_are_honored(): + """An `os.environ/VAR` reference in general_settings resolves to a string, not an int.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.9"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + + throttle = LoginThrottle.from_request( + request, + general_settings={ + "trusted_proxy_ranges": "10.0.0.0/8", + "max_failed_login_attempts_per_source": " 70 ", + "failed_login_window_seconds": "not-a-number", + "failed_login_block_seconds": "-5", + }, + redis_cache=None, + ) + + assert throttle.source_limit == 70 + assert throttle.user_limit == 35, "the per-username allowance is half the address allowance" + assert throttle.window_seconds == 60, "garbage falls back to the default" + assert throttle.block_seconds == 300, "a value below one would block nothing or forever" + + +def test_the_defaults_are_the_agreed_ones(): + from litellm.proxy.auth.login_throttle import LoginThrottle + + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.9"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + throttle = LoginThrottle.from_request( + request, general_settings={"trusted_proxy_ranges": ["10.0.0.0/8"]}, redis_cache=None + ) + + assert (throttle.source_limit, throttle.user_limit, throttle.window_seconds, throttle.block_seconds) == ( + 10, + 5, + 60, + 300, + ) + + +@pytest.mark.parametrize( + ("source_limit", "expected_user_limit"), + [(1, 1), (2, 1), (3, 1), (10, 5), (11, 5), (70, 35)], + ids=["one-stays-one", "two-halves-to-one", "odd-rounds-down", "default", "eleven-rounds-down", "even"], +) +def test_the_per_username_allowance_is_half_the_address_allowance_rounded_down_at_least_one( + source_limit, expected_user_limit +): + from litellm.proxy.auth.login_throttle import user_limit_for + + assert user_limit_for(source_limit) == expected_user_limit + + +def _throttle_behind_trusted_proxy(client_ip: str, settings: Mapping[str, object]) -> "LoginThrottle": + from litellm.proxy.auth.login_throttle import LoginThrottle + + request = MagicMock() + request.headers = {"x-forwarded-for": client_ip} + request.client = MagicMock() + request.client.host = "10.0.0.1" + return LoginThrottle.from_request(request, general_settings=settings, redis_cache=None) + + +def test_a_per_address_override_also_raises_that_address_per_username_allowance(): + """One override sizes both limits for an address, so operators need no second override table.""" + settings = { + "trusted_proxy_ranges": ["10.0.0.0/8"], + "max_failed_login_attempts_per_source": 10, + "max_failed_login_attempts_per_source_overrides": {"203.0.113.0/24": 50}, + } + + raised = _throttle_behind_trusted_proxy("203.0.113.9", settings) + assert (raised.source_limit, raised.user_limit) == (50, 25) + + ordinary = _throttle_behind_trusted_proxy("198.51.100.4", settings) + assert (ordinary.source_limit, ordinary.user_limit) == (10, 5) + + +@pytest.mark.asyncio +async def test_an_override_of_zero_exempts_that_address_from_both_limits(): + """Regression: opting an address out used to mean guessing a large enough number.""" + settings = { + "trusted_proxy_ranges": ["10.0.0.0/8"], + "max_failed_login_attempts_per_source": 1, + "max_failed_login_attempts_per_source_overrides": {"203.0.113.7": 0, "203.0.113.0/24": 3}, + } + + exempt = _throttle_behind_trusted_proxy("203.0.113.7", settings) + assert exempt.enabled is False + assert exempt.source_limit is None + attempt = await exempt.attempt("scanner@example.com") + for _ in range(5): + await attempt.failed() + await exempt.attempt("scanner@example.com") + + sibling = _throttle_behind_trusted_proxy("203.0.113.8", settings) + assert sibling.enabled is True + assert (sibling.source_limit, sibling.user_limit) == (3, 1) + + +def test_the_per_username_allowance_follows_the_peer_override_when_the_source_scope_is_off(): + """Without trusted_proxy_ranges the address is not blocked, but its override still sizes the pair limit.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + request = MagicMock() + request.headers = {} + request.client = MagicMock() + request.client.host = "192.0.2.8" + throttle = LoginThrottle.from_request( + request, + general_settings={"max_failed_login_attempts_per_source_overrides": {"192.0.2.8": 40}}, + redis_cache=None, + ) + + assert throttle.source_limit is None + assert throttle.user_limit == 20 + + +def test_the_disable_flag_is_read_once_not_per_login_attempt(monkeypatch): + """Regression: the kill switch was read through the secret manager on every unauthenticated request.""" + from litellm.proxy.auth import login_throttle + + reads: Final[list[str]] = [] # mutable-ok: test-only call recorder + monkeypatch.setattr( + login_throttle, "get_secret_bool", lambda name, default_value: reads.append(name) or default_value + ) + login_throttle._rate_limit_disabled.cache_clear() + request = MagicMock() + request.headers = {} + request.client = MagicMock() + request.client.host = "1.2.3.4" + + for _ in range(50): + assert login_throttle.LoginThrottle.from_request(request, general_settings={}, redis_cache=None).enabled is True + + login_throttle._rate_limit_disabled.cache_clear() + assert reads == ["LITELLM_DISABLE_LOGIN_RATE_LIMIT"] + + +@pytest.mark.asyncio +async def test_a_blocked_username_cannot_forge_log_lines(monkeypatch): + """The username reaches a warning log, so it must not carry newlines or control bytes.""" + import logging + + from litellm._logging import verbose_proxy_logger + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1) + forged = "victim@example.com\nWARNING: sign-in succeeded for attacker\x00" + + assert await _fail(throttle, username=forged) == "401" + + records: list[logging.LogRecord] = [] + handler = logging.Handler() + handler.emit = records.append + verbose_proxy_logger.addHandler(handler) + try: + assert await _fail(throttle, username=forged) == "401" + finally: + verbose_proxy_logger.removeHandler(handler) + + emitted = [r.getMessage() for r in records if "Admin UI sign-in blocked" in r.getMessage()] + assert emitted, "installing the block must be logged" + assert "\n" not in emitted[0] and "\x00" not in emitted[0] + assert "victim@example.com" in emitted[0] + + +@pytest.mark.asyncio +async def test_a_username_spray_cannot_evict_an_active_block(monkeypatch): + """Counters and blocks live in separate bounded stores, so a flood of made-up pairs fills the counter + store while the blocks it already earned stay in force.""" + from litellm.caching.in_memory_cache import InMemoryCache + from litellm.constants import LOGIN_THROTTLE_MAX_TRACKED_BLOCKS, LOGIN_THROTTLE_MAX_TRACKED_COUNTERS + from litellm.proxy.auth.login_throttle import _BLOCKS, _COUNTERS, LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + assert LOGIN_THROTTLE_MAX_TRACKED_COUNTERS >= 10_000 and LOGIN_THROTTLE_MAX_TRACKED_BLOCKS >= 10_000 + assert _COUNTERS is not _BLOCKS + counters, blocks = InMemoryCache(max_size_in_memory=50), InMemoryCache(max_size_in_memory=50) + throttle = LoginThrottle( + client_ip="10.9.9.9", + source_limit=None, + user_limit=1, + window_seconds=60, + block_seconds=300, + counters=counters, + blocks=blocks, + ) + victim = "spray-victim@corp.com" + assert [await _fail(throttle, username=victim) for _ in range(2)] == ["401", "401"] + + for i in range(200): + await throttle.record_failure(f"spray-filler-{i}@corp.com") + + assert len(counters.cache_dict) <= 50, "the counter store is bounded" + assert counters.get_cache(throttle._keys(victim).pair_counter) is None, "the victim's counter was evicted" + assert await _fail(throttle, username=victim) == "429", "the block survived the spray" + + def _patch_sso_configured(stack: ExitStack, *, configured: bool) -> None: stack.enter_context( patch( # test-quality-ok: no HTTP boundary here; same internal the pre-existing tests above already mock @@ -662,6 +1761,7 @@ class TestDisablePasswordLoginWhenSSOEnabled: password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_password_login_when_sso_enabled": True}, ) @@ -694,6 +1794,7 @@ class TestDisablePasswordLoginWhenSSOEnabled: password=password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_password_login_when_sso_enabled": True}, ) @@ -727,6 +1828,7 @@ class TestDisablePasswordLoginWhenSSOEnabled: password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_password_login_when_sso_enabled": True}, ) @@ -763,6 +1865,7 @@ class TestDisablePasswordLoginWhenSSOEnabled: password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_password_login_when_sso_enabled": True}, ) @@ -796,6 +1899,7 @@ class TestDisablePasswordLoginWhenSSOEnabled: password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={}, ) @@ -825,6 +1929,7 @@ class TestDisableEnvCredentialLogin: password=ui_password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_env_credential_login": True}, ) @@ -849,6 +1954,7 @@ class TestDisableEnvCredentialLogin: password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_env_credential_login": True}, ) @@ -891,6 +1997,7 @@ class TestDisableEnvCredentialLogin: password=password, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={"disable_env_credential_login": True}, ) @@ -924,6 +2031,7 @@ class TestDisableEnvCredentialLogin: password=master_key, master_key=master_key, prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), general_settings={}, ) diff --git a/tests/test_litellm/proxy/auth/test_network.py b/tests/test_litellm/proxy/auth/test_network.py index b67723305e4..e743ce8cd23 100644 --- a/tests/test_litellm/proxy/auth/test_network.py +++ b/tests/test_litellm/proxy/auth/test_network.py @@ -57,6 +57,21 @@ def test_xff_honored_from_trusted_peer(): assert via_proxy is True +def test_ipv4_mapped_peer_and_hop_match_ipv4_trusted_ranges(): + request = make_request(headers={"x-forwarded-for": "203.0.113.9, ::ffff:10.0.0.5"}, client=("::ffff:10.0.0.1", 1)) + ip, via_proxy = resolve_client_ip(request, TRUSTED) + assert ip == "203.0.113.9" + assert via_proxy is True + + +def test_ipv4_mapped_peer_still_matches_mapped_notation_trusted_range(): + config = TrustedProxyConfig(use_forwarded_for=True, trusted_proxy_cidrs=["::ffff:10.0.0.0/104"]) + request = make_request(headers={"x-forwarded-for": "203.0.113.9"}, client=("::ffff:10.0.0.1", 1)) + ip, via_proxy = resolve_client_ip(request, config) + assert ip == "203.0.113.9" + assert via_proxy is True + + def test_spoofed_xff_from_untrusted_peer_is_ignored(): request = make_request( headers={"x-forwarded-for": "203.0.113.9"}, client=("8.8.8.8", 1) diff --git a/tests/test_litellm/proxy/proxy_server/conftest.py b/tests/test_litellm/proxy/proxy_server/conftest.py index c545965f9a9..ae1b42363ef 100644 --- a/tests/test_litellm/proxy/proxy_server/conftest.py +++ b/tests/test_litellm/proxy/proxy_server/conftest.py @@ -511,3 +511,27 @@ def make_key( max_budget=max_budget, **kwargs, ) + + +@pytest.fixture(autouse=True) +def reset_login_throttle(monkeypatch): + """Clear the Admin UI failed-login counters between tests. + + `client` is session scoped and the counters live in shared module stores with a 300s block + window, so without this a failed sign-in test could block unrelated tests later. + Only the throttle's own keys are removed, so other cache entries remain untouched. + """ + from litellm.constants import LOGIN_THROTTLE_CACHE_KEY_PREFIX + from litellm.proxy import proxy_server as ps + from litellm.proxy.auth.login_throttle import _BLOCKS, _COUNTERS + + def _drop_throttle_keys() -> None: + for store in (_COUNTERS, _BLOCKS): + for key in tuple(store.cache_dict) + tuple(store.ttl_dict): + if key.startswith(LOGIN_THROTTLE_CACHE_KEY_PREFIX): + store.delete_cache(key) + + monkeypatch.setattr(ps, "redis_usage_cache", None) + _drop_throttle_keys() + yield _drop_throttle_keys + _drop_throttle_keys() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index 79c23b11f3e..88f8be4e49a 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -12,8 +12,6 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock -import pytest - from .conftest import normalize # --------------------------------------------------------------------------- @@ -29,7 +27,7 @@ def _install_login_mocks(monkeypatch, raise_on_auth: bool = False) -> None: """ from litellm.proxy import proxy_server as ps - async def _fake_auth(username, password, master_key, prisma_client, general_settings=None): + async def _fake_auth(username, password, master_key, prisma_client, throttle=None, general_settings=None): if raise_on_auth: raise Exception("boom-auth-failure") fake = MagicMock() @@ -471,3 +469,222 @@ def test_login_form_ignores_open_redirect_return_to(client, monkeypatch): location = response.headers.get("location", "") assert "evil.example.com" not in location assert "/ui" in location # dashboard fallback + + +# --------------------------------------------------------------------------- +# Failed-login accounting across the login routes (LIT-5285) +# --------------------------------------------------------------------------- + + +def _install_real_auth(monkeypatch, **settings): + """Run the real authenticate_user so the throttle inside it is exercised. + + prisma_client stays None, so every guess falls through to the credential rejection. + """ + from litellm.proxy import proxy_server as ps + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right-password") + monkeypatch.setattr(ps, "master_key", "sk-test-master") + monkeypatch.setattr(ps, "prisma_client", None) + monkeypatch.setattr(ps, "premium_user", False) + monkeypatch.setattr(ps, "general_settings", dict(settings)) + + +def _form_login(client, username="admin", password="wrong"): + return client.post("/login", data={"username": username, "password": password}, follow_redirects=False).status_code + + +def _json_login(client, path, username="admin", password="wrong"): + return client.post(path, json={"username": username, "password": password}).status_code + + +def _db_user(monkeypatch, email: str): + """A database user with a stored hash, faked so the route reaches the known-user branch without Postgres.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server as ps + + user = MagicMock() + user.user_id = "u-1" + user.user_email = email + user.user_role = "internal_user" + user.password = "scrypt:stored" + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=user) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr("litellm.proxy.auth.login_utils.UserRepository", repo) + monkeypatch.setattr("litellm.proxy.auth.login_utils._rehash_password_if_needed", AsyncMock()) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.verify_password", lambda given, stored: given == "right-db-password" + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.generate_key_helper_fn", AsyncMock(return_value={"token": "sk-ui"}) + ) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + + +def test_budget_is_shared_across_every_login_endpoint(client, monkeypatch, reset_login_throttle): + """The endpoint is not part of the key, so spending the budget on one route blocks the rest. + + Partitioning the counter per endpoint would silently triple the real allowance. + """ + _install_real_auth( + monkeypatch, + max_failed_login_attempts_per_source=20, + control_plane_url="https://cp.example.com", + ) + + assert [_form_login(client) for _ in range(5)] == [401] * 5 + assert [_json_login(client, "/v2/login") for _ in range(5)] == [401] * 5 + + assert _json_login(client, "/v3/login") == 401, "the eleventh failure crosses the limit and installs the block" + assert _json_login(client, "/v3/login") == 429, "the twelfth attempt must be refused on a third route" + + +def test_budget_is_shared_across_username_casing(client, monkeypatch, reset_login_throttle): + """The database lookup is case-insensitive, so casing must not partition the counter.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=6) + + assert [_json_login(client, "/v2/login", username="admin@corp.com") for _ in range(2)] == [401] * 2 + assert [_json_login(client, "/v2/login", username="ADMIN@corp.com") for _ in range(2)] == [401] * 2 + + assert _json_login(client, "/v2/login", username="Admin@corp.com") == 429 + + +def test_a_refused_attempt_carries_retry_after(client, monkeypatch, reset_login_throttle): + """The 429 tells the caller how long the block has left.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2, failed_login_block_seconds=77) + + assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401] + + refused = client.post("/v2/login", json={"username": "admin", "password": "wrong"}) + assert refused.status_code == 429 + assert refused.headers.get("retry-after") == "77" + + +def test_the_form_returns_a_human_readable_lockout_page(client, monkeypatch, reset_login_throttle): + """The no-JavaScript form must render a wait page when its POST is throttled.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2, failed_login_block_seconds=77) + + assert [_form_login(client) for _ in range(2)] == [401, 401] + + refused = client.post("/login", data={"username": "admin", "password": "wrong"}) + assert refused.status_code == 429 + assert refused.headers.get("content-type", "").startswith("text/html") + assert "Try again in about 77 seconds" in refused.text + assert refused.headers.get("retry-after") == "77" + + +def test_a_second_username_from_the_same_source_still_gets_through(client, monkeypatch, reset_login_throttle): + """The pair block is per username, so one account's block cannot take the office down with it.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2) + + assert [_json_login(client, "/v2/login", username="admin") for _ in range(3)] == [401, 401, 429] + + assert _json_login(client, "/v2/login", username="someone-else@example.com") == 401 + + +def test_a_spray_across_usernames_is_blocked_on_the_source_when_the_source_is_attributable( + client, monkeypatch, reset_login_throttle +): + """A fresh username per guess keeps every pair at one, so the address is what stops it.""" + _install_real_auth(monkeypatch, trusted_proxy_ranges=["10.0.0.0/8"], max_failed_login_attempts_per_source=4) + + sprayed = [_json_login(client, "/v2/login", username=f"sprayed-{i}@corp.com") for i in range(5)] + assert sprayed == [401] * 5 + + assert _json_login(client, "/v2/login", username="sprayed-6@corp.com") == 429 + + +def test_a_spray_across_usernames_is_not_blocked_without_trusted_proxy_ranges( + client, monkeypatch, reset_login_throttle +): + """Without a configured proxy range the peer address is whoever fronts the proxy, shared by every + client, so a source-wide block would block them all and the source scope stays off.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=4) + + sprayed = [_json_login(client, "/v2/login", username=f"sprayed-{i}@corp.com") for i in range(8)] + assert sprayed == [401] * 8 + + +def test_a_spray_across_usernames_is_blocked_on_the_source_with_an_empty_trusted_proxy_ranges( + client, monkeypatch, reset_login_throttle +): + """An explicit empty list says nothing fronts the proxy, so the peer address is the client and the + source scope is on. A forwarded header from an untrusted peer is ignored rather than trusted.""" + _install_real_auth(monkeypatch, trusted_proxy_ranges=[], max_failed_login_attempts_per_source=4) + + sprayed = [ + client.post( + "/v2/login", + json={"username": f"sprayed-{i}@corp.com", "password": "wrong"}, + headers={"x-forwarded-for": f"203.0.113.{i}"}, + ).status_code + for i in range(5) + ] + assert sprayed == [401] * 5 + + assert _json_login(client, "/v2/login", username="sprayed-6@corp.com") == 429 + + +def test_the_configured_admin_password_is_refused_while_blocked(client, monkeypatch, reset_login_throttle): + """The env credentials get no bypass: a bypass would make them the one password worth guessing without + limit. An operator who is blocked administers the proxy with the master key over the API meanwhile.""" + from unittest.mock import AsyncMock, patch + + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + + assert [_json_login(client, "/v2/login") for _ in range(3)] == [401, 401, 429] + + with ( + patch( # test-quality-ok: the admin sign-in upserts the admin row; faked so no DB is needed + "litellm.proxy.auth.login_utils.user_update", new=AsyncMock() + ), + patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + assert _json_login(client, "/v2/login", password="right-password") == 429 + reset_login_throttle() + assert _json_login(client, "/v2/login", password="right-password") == 200 + + +def test_the_master_key_as_a_bearer_token_still_works_while_the_ui_password_is_blocked( + client, monkeypatch, reset_login_throttle +): + """Lockout recovery: the API path with the master key never enters the sign-in throttle.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2) + + assert [_json_login(client, "/v2/login") for _ in range(3)] == [401, 401, 429] + + assert client.get("/models", headers={"Authorization": "Bearer sk-not-the-master"}).status_code >= 400 + assert client.get("/models", headers={"Authorization": "Bearer sk-test-master"}).status_code == 200 + assert _json_login(client, "/v2/login", password="right-password") == 429, "the UI block is unaffected" + + +def test_a_database_users_correct_password_is_refused_while_blocked(client, monkeypatch, reset_login_throttle): + """The block is hard: while it lasts, nothing from that source signs in as that user, right password or not, + and the block is not extended by the refused attempts.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2, failed_login_block_seconds=64) + _db_user(monkeypatch, "user@corp.com") + + assert [_json_login(client, "/v2/login", username="user@corp.com") for _ in range(3)] == [401, 401, 429] + + refused = client.post("/v2/login", json={"username": "user@corp.com", "password": "right-db-password"}) + assert refused.status_code == 429 + assert refused.headers.get("retry-after") == "64" + + reset_login_throttle() + assert _json_login(client, "/v2/login", username="user@corp.com", password="right-db-password") == 200 + + +def test_sign_in_succeeds_again_once_the_block_is_cleared(client, monkeypatch, reset_login_throttle): + """A cleared store lets the same username straight back to a plain credential check.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=2) + + assert [_json_login(client, "/v2/login") for _ in range(3)] == [401, 401, 429] + + reset_login_throttle() + assert _json_login(client, "/v2/login") == 401 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index ce809eda847..6dc088f676f 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -26,7 +26,6 @@ from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient - import litellm import litellm.proxy.proxy_server as proxy_server_module from litellm.caching.caching import RedisCache @@ -41,6 +40,7 @@ from litellm.proxy._types import ( TokenCountRequest, UserAPIKeyAuth, ) +from litellm.proxy.auth.login_throttle import LoginThrottle from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.hooks.parallel_request_limiter_v3 import RequestRateLimiterStash from litellm.proxy.proxy_server import app, initialize, openai_exception_handler @@ -139,13 +139,14 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): } assert response.cookies.get("token") == "signed-token" - mock_authenticate_user.assert_awaited_once_with( - username="alice", - password="secret", - master_key="test-master-key", - prisma_client=mock_prisma_client, - general_settings={}, - ) + mock_authenticate_user.assert_awaited_once() + auth_kwargs = mock_authenticate_user.call_args.kwargs + assert auth_kwargs["username"] == "alice" + assert auth_kwargs["password"] == "secret" + assert auth_kwargs["master_key"] == "test-master-key" + assert auth_kwargs["prisma_client"] is mock_prisma_client + assert auth_kwargs["general_settings"] == {} + assert isinstance(auth_kwargs["throttle"], LoginThrottle), "the endpoint must thread a throttle through" mock_create_ui_token_object.assert_called_once_with( login_result=mock_login_result, general_settings={}, @@ -3410,6 +3411,60 @@ async def test_load_config_user_url_validation_handles_null_and_string_false(tmp assert litellm.user_url_validation is False +@pytest.mark.asyncio +async def test_load_config_warns_per_worker_login_counters_without_general_settings(tmp_path, monkeypatch, caplog): + """Regression: the failed-login throttle is on by default, so a multi-worker proxy with no + Redis must hear that its counters are per worker even when the config has no general_settings.""" + import logging + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.auth.login_throttle import warn_login_counters_are_per_worker + from litellm.proxy.proxy_server import ProxyConfig + + for redis_var in ("REDIS_HOST", "REDIS_URL", "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(redis_var, raising=False) + monkeypatch.setenv("NUM_WORKERS", "4") + monkeypatch.setattr(proxy_server, "redis_usage_cache", None) + warn_login_counters_are_per_worker.cache_clear() + config_file = tmp_path / "config.yaml" + config_file.write_text("model_list: []\n") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + + assert "Running 4 workers but Redis is not configured" in caplog.text + + +@pytest.mark.asyncio +async def test_load_config_warns_that_the_source_login_limit_is_off_without_trusted_proxy_ranges( + tmp_path, monkeypatch, caplog +): + """The per-source failed-login limit is skipped when the source cannot be attributed, and the + operator must be told so at startup. Both a configured range and an explicit empty list (no + proxies, the peer is the source) silence it, since both keep the limit on.""" + import logging + + from litellm.proxy.auth.login_throttle import warn_source_login_limit_is_off + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setenv("NUM_WORKERS", "1") + warn_source_login_limit_is_off.cache_clear() + config_file = tmp_path / "config.yaml" + config_file.write_text("model_list: []\n") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert "trusted_proxy_ranges is not set" in caplog.text + + for configured in ("['10.0.0.0/8']", "[]"): + caplog.clear() + warn_source_login_limit_is_off.cache_clear() + config_file.write_text(f"model_list: []\ngeneral_settings:\n trusted_proxy_ranges: {configured}\n") + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert "trusted_proxy_ranges is not set" not in caplog.text, configured + + @pytest.mark.asyncio async def test_load_environment_variables_direct_and_os_environ(): """ @@ -13952,6 +14007,35 @@ async def test_authoritative_floor_spend_keeps_a_reset_marker_written_during_the ) +@pytest.mark.asyncio +async def test_login_throttle_settings_are_not_hot_applied_from_the_database(): + """LIT-5285: a stored sign-in limit does not take effect on a live worker. + + _update_general_settings copies an allowlist of keys out of the DB row on every config + poll. Adding these to it would let a stored value outrank config.yaml without a restart, + so an operator locked out by a bad value could not fix it by editing YAML and restarting. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy.proxy_server import ProxyConfig + + original = dict(ps.general_settings) + try: + ps.general_settings.clear() + await ProxyConfig()._update_general_settings( + db_general_settings={ + "max_failed_login_attempts_per_source": 999, + "failed_login_window_seconds": 1, + "failed_login_block_seconds": 1, + } + ) + assert "max_failed_login_attempts_per_source" not in ps.general_settings + assert "failed_login_window_seconds" not in ps.general_settings + assert "failed_login_block_seconds" not in ps.general_settings + finally: + ps.general_settings.clear() + ps.general_settings.update(original) + + @pytest.mark.asyncio async def test_load_config_router_authorizes_fallback_targets_against_the_calling_key(tmp_path): from litellm.proxy.auth.fallback_model_access import router_fallback_access_check diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8c6805aa622..7767cd5f5a8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26692,6 +26692,16 @@ export interface components { * @description If True, router fallbacks configured in router_settings are only attempted when the calling key (and its team and project) is allowed to call the fallback model; unauthorized fallback targets are skipped and the primary model's error is returned. Default is False. */ enforce_fallback_model_access?: boolean | null; + /** + * Failed Login Block Seconds + * @description How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300 + */ + failed_login_block_seconds?: number | null; + /** + * Failed Login Window Seconds + * @description Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Set under `general_settings` in config.yaml. Defaults to 60 + */ + failed_login_window_seconds?: number | null; /** * Forward Client Headers To Llm Api * @description If True, forwards client headers (e.g. Authorization) to the LLM API. Required for Claude Code with Max subscription. @@ -26736,6 +26746,18 @@ export interface components { * @description max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider */ max_batch_file_size_mb?: number | null; + /** + * Max Failed Login Attempts Per Source + * @description Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Half this value, rounded down but at least 1, is the allowance for one username from that address; one more blocks that address for that username only, and its further failures stop counting toward the address limit, so a script stuck on one account does not block everyone behind a shared address. The per-address limit is only enforced when `trusted_proxy_ranges` is set: to the proxies in front of LiteLLM, or to an empty list when clients connect directly. Left unset, the peer address may be a shared ingress and only the per-username half runs. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10 + */ + max_failed_login_attempts_per_source?: number | null; + /** + * Max Failed Login Attempts Per Source Overrides + * @description Per-address overrides of `max_failed_login_attempts_per_source`, keyed by IP address or CIDR range, e.g. {'1.2.3.4': 200, '5.6.0.0/24': 500}. The most specific matching range wins (between equivalent keys such as '1.2.3.4' and '1.2.3.4/32', an exemption wins, then the higher limit), and the per-username allowance for that address follows as half the override. A value of 0 exempts the address from both limits. Set under `general_settings` in config.yaml + */ + max_failed_login_attempts_per_source_overrides?: { + [key: string]: number; + } | null; /** * Max File Size Mb * @description max file size in MB for /v1/files uploads, for any purpose, if a file is larger than this size it will be rejected before being forwarded to the provider @@ -26916,7 +26938,7 @@ export interface components { transcribe_media_buckets?: string[] | null; /** * Trusted Proxy Ranges - * @description CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler. + * @description CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler, and whose X-Forwarded-For is used to attribute Admin UI sign-in attempts to a source address. Set it to an empty list when clients connect directly, so the peer address is the source. Left unset, or containing an entry that is not an address or CIDR range, the per-source sign-in limit is off. */ trusted_proxy_ranges?: string[] | null; /**