From aa7f1e16b8810a9321fd12539732ff738aeebd38 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 15 Sep 2026 23:53:14 +0000 Subject: [PATCH] feat(proxy): throttle failed Admin UI sign-ins per source and source/username Replace the username-global lockout with counters keyed by source address and by source/username pair. Each has a fixed counting window (60s) and a separate block TTL (300s). Blocks are soft: a correct password still signs in, wrong passwords from a blocked key take one of 5 held slots per worker and are held 30s before a 429. Once a pair is blocked its failures stop counting against the source. The source scope runs only when trusted_proxy_ranges is set, IPv6 is grouped by /64, and per-source limits accept IP and CIDR overrides with longest-prefix matching. Redis is authoritative through one Lua script per failure, with bounded per-worker fallback when Redis raises. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 4 +- litellm/proxy/_types.py | 23 +- litellm/proxy/auth/login_throttle.py | 578 +++++---- litellm/proxy/auth/login_utils.py | 26 +- litellm/proxy/proxy_server.py | 12 +- .../proxy/auth/test_login_utils.py | 1072 +++++++++-------- .../proxy/proxy_server/conftest.py | 31 +- .../proxy_server/test_routes_login_sso.py | 121 +- tests/test_litellm/proxy/test_proxy_server.py | 34 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 26 +- 10 files changed, 1043 insertions(+), 884 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 3f1bac12563..b4b2b1a334c 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -874,7 +874,7 @@ class RedisCache(BaseCache): return _LUA_COUNT.validate_python(count) @_redis_circuit_breaker_guard_sync - def batch_get_counts(self, key_list: list[str] | tuple[str, ...]) -> tuple[int | None, ...]: + def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]: """Read integer counters for ``key_list``, in order, raising when Redis cannot answer. ``batch_get_cache`` swallows every failure and returns an empty dict, which the caller @@ -885,7 +885,7 @@ class RedisCache(BaseCache): return _decoded_counts(self._run_redis_mget_operation(keys=namespaced_keys)) @_redis_circuit_breaker_guard - async def async_batch_get_counts(self, key_list: list[str] | tuple[str, ...]) -> tuple[int | None, ...]: + async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]: """Async twin of ``batch_get_counts``, raising on failure the same way.""" namespaced_keys: Final = [self.check_and_fix_namespace(key=key) for key in key_list] return _decoded_counts(await self._async_run_redis_mget_operation(keys=namespaced_keys)) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ddcc7b2dece..2537555f316 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2736,20 +2736,29 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="sends alerts if requests hang for 5min+", ) ui_access_mode: Literal["admin_only", "all"] | None = Field("all", description="Control access to the Proxy UI") - max_failed_login_attempts: int | None = Field( - None, - ge=1, - description="Number of failed Admin UI sign-in attempts allowed for one username, from any source address, within `failed_login_window_seconds`, before further attempts for that username are refused with 429. Attempts are answered with a doubling delay well before this ceiling. Set under `general_settings` in config.yaml. Defaults to 50", - ) max_failed_login_attempts_per_source: int | None = Field( None, ge=1, - description="Number of failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`, before further attempts from that address are refused with 429. Counted independently of `max_failed_login_attempts`. Set under `general_settings` in config.yaml. Defaults to 250", + description="Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Only enforced when `trusted_proxy_ranges` is set, since otherwise every client behind an ingress shares one address. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10", + ) + max_failed_login_attempts_per_source_overrides: dict[str, int] | None = Field( + None, + description="Per-address overrides of `max_failed_login_attempts_per_source`, keyed by IP address or CIDR range, e.g. {'1.2.3.4': 200, '5.6.0.0/24': 500}. The most specific matching range wins. Set under `general_settings` in config.yaml", + ) + max_failed_login_attempts_per_user: int | None = Field( + None, + ge=1, + description="Failed Admin UI sign-in attempts allowed from one source address for one username within `failed_login_window_seconds`. One more blocks that address for that username for `failed_login_block_seconds`, and its further failures stop counting against `max_failed_login_attempts_per_source`, so a script stuck on one account does not block everyone behind the same address. Set under `general_settings` in config.yaml. Defaults to 5", ) failed_login_window_seconds: int | None = Field( None, ge=1, - description="Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Set under `general_settings` in config.yaml. Defaults to 900", + description="Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Set under `general_settings` in config.yaml. Defaults to 60", + ) + failed_login_block_seconds: int | None = Field( + None, + ge=1, + description="How long a blocked source address, or source address and username, stays blocked. The block is soft: a correct password still signs in, but each attempt from a blocked key takes one of 5 held slots per worker and a wrong password is held for 30 seconds before it is refused with 429. Set under `general_settings` in config.yaml. Defaults to 300", ) allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access") reject_clientside_metadata_tags: bool | None = Field( diff --git a/litellm/proxy/auth/login_throttle.py b/litellm/proxy/auth/login_throttle.py index 50333ffad05..fb708ecf872 100644 --- a/litellm/proxy/auth/login_throttle.py +++ b/litellm/proxy/auth/login_throttle.py @@ -1,142 +1,231 @@ """Failed-login accounting for the Admin UI sign-in path. -Counts failed credential checks over a fixed window against two independent keys, the -username on its own and the source address on its own, so that one username attacked from -many sources and one source spraying many usernames are both counted. Repeated failures -are answered slowly, doubling from one second, and refused with 429 once either counter -reaches its limit. Built per request by ``LoginThrottle.from_request`` because it carries -that request's resolved source address, and because the coordination cache is assigned at -startup and can be reassigned later. +Wrong passwords are counted over a short window per source address and per source-and-username +pair; too many in one window blocks that key for a fixed time. Blocks are soft: a correct password +still signs in, while a wrong one from a blocked key is held open before its 429 and only a few can +be held at once, which bounds how many guesses a blocked key gets checked. A blocked pair stops +counting against its source, so one script stuck on one account does not block the whole office. """ +from __future__ import annotations + import asyncio import hashlib -from collections.abc import Awaitable, Callable, Mapping +import ipaddress +import math +import time +from collections.abc import AsyncGenerator, Mapping +from contextlib import asynccontextmanager from dataclasses import dataclass from functools import cache from types import MappingProxyType -from typing import Final, NamedTuple, NoReturn +from typing import Final, Literal, NamedTuple, NoReturn -from fastapi import Request +from fastapi import Request, status +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger -from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.proxy.auth.network import TrustedProxyConfig, normalize_cidr_ranges, resolve_client_ip -from litellm.proxy.auth.trusted_proxy_utils import TRUSTED_PROXY_RANGES_KEY from litellm.secret_managers.main import get_secret_bool -DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS: Final = 50 -DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE: Final = 250 -DEFAULT_FAILED_LOGIN_WINDOW_SECONDS: Final = 900 +DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE: Final = 10 +DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_USER: Final = 5 +DEFAULT_FAILED_LOGIN_WINDOW_SECONDS: Final = 60 +DEFAULT_FAILED_LOGIN_BLOCK_SECONDS: Final = 300 -USERNAME_DELAY_ONSET: Final = 3 -SOURCE_DELAY_ONSET: Final = 25 -FIRST_DELAY_SECONDS: Final = 1.0 -MAX_DELAY_SECONDS: Final = 30.0 -MAX_CONCURRENT_DELAYS_PER_SOURCE: Final = 5 +BLOCKED_ATTEMPT_HOLD_SECONDS: Final = 30 +MAX_HELD_ATTEMPTS_PER_KEY: Final = 5 +IPV6_SOURCE_PREFIX_LENGTH: Final = 64 -_MAX_DELAY_DOUBLINGS: Final = 16 +SOURCE_LIMIT_KEY: Final = "max_failed_login_attempts_per_source" +SOURCE_LIMIT_OVERRIDES_KEY: Final = "max_failed_login_attempts_per_source_overrides" +USER_LIMIT_KEY: Final = "max_failed_login_attempts_per_user" +WINDOW_KEY: Final = "failed_login_window_seconds" +BLOCK_KEY: Final = "failed_login_block_seconds" +TRUSTED_PROXY_RANGES_KEY: Final = "trusted_proxy_ranges" _CACHE_KEY_PREFIX: Final = "login_fail" _UNKNOWN_SOURCE: Final = "unknown" -_MAX_LOGGED_USERNAME_CHARS: Final = 128 +_MAX_TRACKED_COUNTERS: Final = 20_000 +_MAX_TRACKED_BLOCKS: Final = 10_000 +_NO_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) +_NOT_BLOCKED: Final = (0, 0) +_LOCAL_BLOCK_EXPIRY: Final = TypeAdapter[float | None](float | None) +_SOURCE_LIMIT_OVERRIDES: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object]) -_MAX_TRACKED_LOGIN_USERNAMES: Final = 10_000 -_MAX_TRACKED_LOGIN_SOURCES: Final = 10_000 +Scope = Literal["user", "source"] + +_BlockTtls = tuple[int, int] +_LUA_BLOCK_TTLS: Final = TypeAdapter[_BlockTtls](_BlockTtls) +_Network = ipaddress.IPv4Network | ipaddress.IPv6Network + +# KEYS: pair counter, pair block, source counter, source block (one cluster slot via the source hash tag) +# ARGV: pair limit, source limit (0 = source scope off), window seconds, block seconds +# Both scripts return {pair block TTL, source block TTL}; 0 or below means not blocked +_BLOCK_TTLS_LUA: Final = "return {redis.call('TTL', KEYS[2]), redis.call('TTL', KEYS[4])}" +_RECORD_FAILURE_LUA: Final = ( + "local function bump(count_key, block_key, limit) " + "local blocked = redis.call('TTL', block_key) " + "if blocked > 0 then return blocked end " + "local count = redis.call('INCR', count_key) " + "if redis.call('TTL', count_key) < 0 then redis.call('EXPIRE', count_key, ARGV[3]) end " + "if count > limit then redis.call('SET', block_key, '1', 'EX', ARGV[4]) return tonumber(ARGV[4]) end " + "return 0 end " + "local user_block = bump(KEYS[1], KEYS[2], tonumber(ARGV[1])) " + "local source_block = 0 " + "if tonumber(ARGV[2]) > 0 and user_block == 0 then " + "source_block = bump(KEYS[3], KEYS[4], tonumber(ARGV[2])) end " + "return {user_block, source_block}" +) + +_COUNTERS: Final = InMemoryCache( + max_size_in_memory=_MAX_TRACKED_COUNTERS, default_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS +) +_BLOCKS: Final = InMemoryCache(max_size_in_memory=_MAX_TRACKED_BLOCKS, default_ttl=DEFAULT_FAILED_LOGIN_BLOCK_SECONDS) +_HELD_ATTEMPTS: Final[dict[str, int]] = {} -def _bounded_store(max_entries: int) -> DualCache: - return DualCache( - in_memory_cache=InMemoryCache(max_size_in_memory=max_entries), - default_in_memory_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS, - ) +async def _sleep(seconds: float) -> None: + await asyncio.sleep(seconds) -_FAILED_LOGIN_USERNAME_CACHE: Final = _bounded_store(_MAX_TRACKED_LOGIN_USERNAMES) -_FAILED_LOGIN_SOURCE_CACHE: Final = _bounded_store(_MAX_TRACKED_LOGIN_SOURCES) -_NO_SETTINGS: Final = MappingProxyType({}) -_UNAVAILABLE: Final = object() - -_DELAYS_IN_FLIGHT: Final[dict[str, int]] = {} # mutable-ok: per-source slots taken and released around each held delay +@cache +def _rate_limit_disabled() -> bool: + return get_secret_bool("LITELLM_DISABLE_LOGIN_RATE_LIMIT", default_value=False) is True @cache def warn_login_counters_are_per_worker(num_workers: str) -> None: - """Warn once per process that failed sign-in counters are not shared across workers.""" verbose_proxy_logger.warning( - "Running %s workers but Redis is not configured for LiteLLM caching. " - "Failed Admin UI sign-in attempts are counted per worker, so an attacker " - "gets max_failed_login_attempts guesses per worker instead of overall. " - "Configure Redis via the 'cache' section in your proxy config.", + "Running %s workers but Redis is not configured. Failed Admin UI sign-in attempts are counted " + "per worker, so the effective limits are %s times the configured values. Configure Redis " + "to share one count across workers.", + num_workers, num_workers, ) @cache -def _rate_limit_disabled() -> bool: - """Resolved once per process so an unauthenticated flood never reaches the secret manager.""" - return bool(get_secret_bool("LITELLM_DISABLE_LOGIN_RATE_LIMIT", False)) +def warn_source_login_limit_is_off() -> None: + verbose_proxy_logger.warning( + "%s is not set, so failed Admin UI sign-in attempts are limited per source address and username " + "only. Set it to the address ranges of the proxies in front of LiteLLM to also limit each " + "source address across usernames.", + TRUSTED_PROXY_RANGES_KEY, + ) -async def _sleep(seconds: float) -> None: - """The wait a rejected sign-in is held for. Replaced in tests so the suite pays no wall clock.""" - await asyncio.sleep(seconds) - - -class FailureCounts(NamedTuple): - """Failures recorded so far in this window against each of the two keys.""" - - username: int - source: int - - -def _parse_int_setting(value: object) -> object: - if not isinstance(value, str): - return value - try: - return int(value.strip()) - except ValueError: - return value - - -def _int_setting(name: str, value: object, default: int, minimum: int) -> int: - if value is None: +def _positive_int(raw: object, key: str, default: int) -> int: + if raw is None: return default - parsed: Final = _parse_int_setting(value) - if isinstance(parsed, bool) or not isinstance(parsed, int) or parsed < minimum: + try: + value: Final = int(str(raw)) + except (TypeError, ValueError): + verbose_proxy_logger.warning("Invalid %s value %r; using %s", key, raw, default) + return default + if value < 1: + verbose_proxy_logger.warning("Invalid %s value %s (must be >= 1); using %s", key, value, default) + return default + return value + + +def _int_setting(settings: Mapping[str, object], key: str, default: int) -> int: + return _positive_int(settings.get(key), key, default) + + +def _parse_address(client_ip: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None: + try: + return ipaddress.ip_address(client_ip) + except ValueError: + return None + + +def _parse_network(raw_range: str) -> _Network | None: + try: + return ipaddress.ip_network(raw_range.strip(), strict=False) + except ValueError: verbose_proxy_logger.warning( - "general_settings.%s=%r is not an integer >= %s; using the default of %s", name, value, minimum, default + "Invalid address or range %r in %s; skipping", raw_range, SOURCE_LIMIT_OVERRIDES_KEY + ) + return None + + +def _source_limit(settings: Mapping[str, object], client_ip: str) -> int: + """Failure allowance for this address: the most specific configured range containing it, else the default.""" + default: Final = _int_setting(settings, SOURCE_LIMIT_KEY, DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE) + raw_overrides: Final = settings.get(SOURCE_LIMIT_OVERRIDES_KEY) + if raw_overrides is None: + return default + try: + overrides: Final = _SOURCE_LIMIT_OVERRIDES.validate_python(raw_overrides) + except ValidationError: + verbose_proxy_logger.warning( + "Invalid %s value; expected a mapping of address or range to limit", SOURCE_LIMIT_OVERRIDES_KEY ) return default - return parsed + address: Final = _parse_address(client_ip) + if address is None: + return default + matches: Final = sorted( + (network.prefixlen, _positive_int(raw_limit, SOURCE_LIMIT_OVERRIDES_KEY, default)) + for raw_range, raw_limit in overrides.items() + if (network := _parse_network(raw_range)) is not None and address in network + ) + return matches[-1][1] if matches else default -def _as_count(cached: object) -> int: - return int(cached) if isinstance(cached, int | float) and not isinstance(cached, bool) else 0 +def source_group(client_ip: str) -> str: + """The bucket an address is counted in: IPv4 as is, IPv6 by its /64, so one prefix holder cannot rotate.""" + address: Final = _parse_address(client_ip) + if address is None: + return client_ip + if isinstance(address, ipaddress.IPv6Address): + mapped: Final = address.ipv4_mapped + if mapped is not None: + return str(mapped) + return str(ipaddress.ip_network((address, IPV6_SOURCE_PREFIX_LENGTH), strict=False)) + return str(address) + + +class _Keys(NamedTuple): + pair_counter: str + pair_block: str + source_counter: str + source_block: str + + +@dataclass(frozen=True, slots=True) +class Block: + scope: Scope + retry_after: int @dataclass(frozen=True, slots=True) class LoginThrottle: - """Fixed-window failed-login accounting for one request's username and source address.""" + """Failed-login limits for one request's source address; ``source_limit`` is None when the + source scope is off because ``trusted_proxy_ranges`` is unset and the peer address is the ingress.""" client_ip: str - max_attempts: int - max_attempts_per_source: int + source_limit: int | None + user_limit: int window_seconds: int - username_cache: DualCache - source_cache: DualCache + block_seconds: int + counters: InMemoryCache + blocks: InMemoryCache redis_cache: RedisCache | None = None enabled: bool = True @classmethod def from_request( - cls, request: Request, general_settings: Mapping[str, object] | None, redis_cache: RedisCache | None - ) -> "LoginThrottle": - """Build the throttle for this request from the proxy's general_settings and shared Redis cache.""" - settings: Final = general_settings or _NO_SETTINGS + cls, + request: Request, + general_settings: Mapping[str, object] | None, + redis_cache: RedisCache | None, + ) -> LoginThrottle: + settings: Final = general_settings if general_settings is not None else _NO_SETTINGS cidrs: Final = normalize_cidr_ranges( settings.get(TRUSTED_PROXY_RANGES_KEY), setting_name=TRUSTED_PROXY_RANGES_KEY ) @@ -145,199 +234,166 @@ class LoginThrottle: ) return cls( client_ip=resolved or _UNKNOWN_SOURCE, - max_attempts=_int_setting( - "max_failed_login_attempts", - settings.get("max_failed_login_attempts"), - DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS, - 1, - ), - max_attempts_per_source=_int_setting( - "max_failed_login_attempts_per_source", - settings.get("max_failed_login_attempts_per_source"), - DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE, - 1, - ), - window_seconds=_int_setting( - "failed_login_window_seconds", - settings.get("failed_login_window_seconds"), - DEFAULT_FAILED_LOGIN_WINDOW_SECONDS, - 1, - ), - username_cache=_FAILED_LOGIN_USERNAME_CACHE, - source_cache=_FAILED_LOGIN_SOURCE_CACHE, + source_limit=_source_limit(settings, resolved) if cidrs and resolved is not None else None, + user_limit=_int_setting(settings, USER_LIMIT_KEY, DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_USER), + window_seconds=_int_setting(settings, WINDOW_KEY, DEFAULT_FAILED_LOGIN_WINDOW_SECONDS), + block_seconds=_int_setting(settings, BLOCK_KEY, DEFAULT_FAILED_LOGIN_BLOCK_SECONDS), + counters=_COUNTERS, + blocks=_BLOCKS, redis_cache=redis_cache, enabled=not _rate_limit_disabled(), ) - @staticmethod - def _loggable(username: str) -> str: - """The username with anything that could forge a log line removed.""" - return "".join(c for c in username if c.isprintable())[:_MAX_LOGGED_USERNAME_CHARS] + def _keys(self, username: str) -> _Keys: + group: Final = source_group(self.client_ip) + user: Final = hashlib.sha256(username.casefold().encode("utf-8")).hexdigest() + return _Keys( + pair_counter=f"{_CACHE_KEY_PREFIX}:{{{group}}}:user:{user}", + pair_block=f"{_CACHE_KEY_PREFIX}:{{{group}}}:block:user:{user}", + source_counter=f"{_CACHE_KEY_PREFIX}:{{{group}}}:source", + source_block=f"{_CACHE_KEY_PREFIX}:{{{group}}}:block:source", + ) - @staticmethod - def _username_key(username: str) -> str: - identity: Final = hashlib.sha256(username.casefold().encode("utf-8")).hexdigest() - return f"{_CACHE_KEY_PREFIX}:user:{identity}" - - def _source_key(self) -> str: - return f"{_CACHE_KEY_PREFIX}:source:{self.client_ip}" - - async def _outcome(self, work: Awaitable[object]) -> object: + @asynccontextmanager + async def attempt(self, username: str, *, exempt: bool = False) -> AsyncGenerator[LoginAttempt]: + if not self.enabled or exempt: + yield LoginAttempt(throttle=self, username=username, block=None) + return + keys: Final = self._keys(username) + block: Final = await self._active_block(keys) + if block is None: + yield LoginAttempt(throttle=self, username=username, block=None) + return + slot: Final = keys.pair_block if block.scope == "user" else keys.source_block + held: Final = _HELD_ATTEMPTS.get(slot, 0) + if held >= MAX_HELD_ATTEMPTS_PER_KEY: + verbose_proxy_logger.warning( + "Admin UI sign-in refused: %s attempts already held for a blocked %s; username=%r source=%s", + held, + block.scope, + username, + self.client_ip, + ) + self.refuse(BLOCKED_ATTEMPT_HOLD_SECONDS) + _HELD_ATTEMPTS[slot] = held + 1 try: - return await work - except Exception as exc: # noqa: BLE001 # an unreachable cache must never deny a valid credential - verbose_proxy_logger.warning("login attempt accounting unavailable: %s", exc) - return _UNAVAILABLE + yield LoginAttempt(throttle=self, username=username, block=block) + finally: + remaining: Final = _HELD_ATTEMPTS.get(slot, 1) - 1 + if remaining > 0: + _HELD_ATTEMPTS[slot] = remaining + else: + _HELD_ATTEMPTS.pop(slot, None) - async def _shared(self, work: Callable[[RedisCache], Awaitable[object]]) -> object: - """The Redis result, or ``_UNAVAILABLE`` when Redis is not configured or the call raised.""" - redis_cache: Final = self.redis_cache - if redis_cache is None: - return _UNAVAILABLE - return await self._outcome(work(redis_cache)) + async def _active_block(self, keys: _Keys) -> Block | None: + local: Final = self._local_block_ttls(keys) + shared: Final = await self._shared_block_ttls(keys) + user_ttl: Final = max(local[0], shared[0]) + source_ttl: Final = max(local[1], shared[1]) + if user_ttl > 0: + return Block(scope="user", retry_after=user_ttl) + if self.source_limit is not None and source_ttl > 0: + return Block(scope="source", retry_after=source_ttl) + return None - async def _failures(self, store: DualCache, key: str) -> int: - """The shared count plus this worker's own. - - A failure is written to exactly one of the two: Redis, or this worker's store when Redis - refused it. So the local store is empty while Redis is healthy, and once Redis answers - again the guesses it missed still count. Read through ``async_batch_get_counts`` because - ``async_get_cache`` turns a failed GET into ``None``, which would pass as an empty counter. - """ - local: Final = _as_count(await self._outcome(store.async_get_cache(key=key))) - shared: Final = await self._shared(lambda redis_cache: redis_cache.async_batch_get_counts((key,))) - if not isinstance(shared, tuple): - return local - return _as_count(shared[0]) + local - - async def _remaining_window(self, key: str) -> int: - """Seconds until this counter expires. - - Counters are only ever written together with their expiry, so a counter without one - was stripped out of band (PERSIST, a restore). It is given the full window again, - since nothing increments a key once the limit is reached. - """ + async def _shared_block_ttls(self, keys: _Keys) -> _BlockTtls: if self.redis_cache is None: - return self.window_seconds - ttl: Final = await self._shared(lambda redis_cache: redis_cache.async_get_ttl(key)) - if isinstance(ttl, int) and ttl > 0: - return min(ttl, self.window_seconds) - await self._shared(lambda redis_cache: redis_cache.async_increment_with_floor(key, 0, self.window_seconds)) - return self.window_seconds + return _NOT_BLOCKED + try: + return _LUA_BLOCK_TTLS.validate_python( + await self.redis_cache.async_register_script(_BLOCK_TTLS_LUA)(list(keys), []) + ) + except Exception as err: + self._warn_redis(err) + return _NOT_BLOCKED - def _refused(self, retry_after: int, param: str) -> ProxyException: - return ProxyException( + def _local_block_ttls(self, keys: _Keys) -> _BlockTtls: + return self._local_block_ttl(keys.pair_block), self._local_block_ttl(keys.source_block) + + def _local_block_ttl(self, block_key: str) -> int: + expires_at: Final = _LOCAL_BLOCK_EXPIRY.validate_python(self.blocks.get_cache(block_key)) + if expires_at is None: + return 0 + return max(math.ceil(expires_at - time.time()), 0) + + async def record_failure(self, username: str) -> _BlockTtls: + keys: Final = self._keys(username) + source_limit: Final = self.source_limit or 0 + if self.redis_cache is not None: + try: + return _LUA_BLOCK_TTLS.validate_python( + await self.redis_cache.async_register_script(_RECORD_FAILURE_LUA)( + list(keys), [self.user_limit, source_limit, self.window_seconds, self.block_seconds] + ) + ) + except Exception as err: + self._warn_redis(err) + user_block: Final = self._local_bump(keys.pair_counter, keys.pair_block, self.user_limit) + if source_limit == 0 or user_block > 0: + return user_block, 0 + return user_block, self._local_bump(keys.source_counter, keys.source_block, source_limit) + + def _local_bump(self, count_key: str, block_key: str, limit: int) -> int: + blocked: Final = self._local_block_ttl(block_key) + if blocked > 0: + return blocked + count: Final = int(self.counters.increment_cache(count_key, 1, ttl=self.window_seconds)) + if count <= limit: + return 0 + self.blocks.set_cache(block_key, time.time() + self.block_seconds, ttl=self.block_seconds) + return self.block_seconds + + async def clear_pair(self, username: str) -> None: + pair_counter: Final = self._keys(username).pair_counter + if self.redis_cache is not None: + try: + await self.redis_cache.async_delete_cache(pair_counter) + except Exception as err: + self._warn_redis(err) + self.counters.delete_cache(pair_counter) + + def _warn_redis(self, err: Exception) -> None: + verbose_proxy_logger.warning( + "Redis failed while counting Admin UI sign-in attempts; using this worker's own counters " + "until it recovers: %s", + err, + ) + + @staticmethod + def refuse(retry_after: int) -> NoReturn: + raise ProxyException( message="Too many failed sign-in attempts. Try again later.", type=ProxyErrorTypes.auth_error, - param=param, - code=429, - headers={"Retry-After": str(retry_after)}, # mutable-ok: ProxyException coerces header values + param="username", + code=status.HTTP_429_TOO_MANY_REQUESTS, + headers={"Retry-After": str(retry_after)}, ) - async def _refuse(self, key: str, scope: str, param: str, username: str, failures: int, limit: int) -> NoReturn: - retry_after: Final = await self._remaining_window(key) + +@dataclass(frozen=True, slots=True) +class LoginAttempt: + throttle: LoginThrottle + username: str + block: Block | None + + async def succeeded(self) -> None: + if not self.throttle.enabled: + return + await self.throttle.clear_pair(self.username) + + async def failed(self) -> None: + if not self.throttle.enabled: + return + if self.block is not None: + await _sleep(BLOCKED_ATTEMPT_HOLD_SECONDS) + self.throttle.refuse(max(self.block.retry_after - BLOCKED_ATTEMPT_HOLD_SECONDS, 1)) + user_block, source_block = await self.throttle.record_failure(self.username) + if user_block == 0 and source_block == 0: + return verbose_proxy_logger.warning( - "Admin UI sign-in attempts exhausted for %s; username=%s source=%s failures=%s limit=%s window=%ss", - scope, - self._loggable(username), - self.client_ip, - failures, - limit, - self.window_seconds, + "Admin UI sign-in blocked for %s seconds after too many failures; scope=%s username=%r source=%s", + user_block or source_block, + "user" if user_block else "source", + self.username, + self.throttle.client_ip, ) - raise self._refused(retry_after, param) - - async def raise_if_blocked(self, username: str) -> None: - """Refuse before the database lookup and before the invite-link password hash.""" - if not self.enabled: - return - username_key: Final = self._username_key(username) - source_key: Final = self._source_key() - username_failures: Final = await self._failures(self.username_cache, username_key) - if username_failures >= self.max_attempts: - await self._refuse( - username_key, "username", "max_failed_login_attempts", username, username_failures, self.max_attempts - ) - source_failures: Final = await self._failures(self.source_cache, source_key) - if source_failures >= self.max_attempts_per_source: - await self._refuse( - source_key, - "source address", - "max_failed_login_attempts_per_source", - username, - source_failures, - self.max_attempts_per_source, - ) - - async def _bump(self, store: DualCache, key: str) -> int: - shared: Final = await self._shared( - lambda redis_cache: redis_cache.async_increment_with_floor(key, 1, self.window_seconds) - ) - if shared is _UNAVAILABLE: - return _as_count( - await self._outcome(store.async_increment_cache(key=key, value=1, ttl=self.window_seconds)) - ) - return _as_count(shared) + _as_count(await self._outcome(store.async_get_cache(key=key))) - - async def record_failure(self, username: str) -> FailureCounts: - """Count one rejected credential guess against this username and against this source.""" - if not self.enabled: - return FailureCounts(username=0, source=0) - return FailureCounts( - username=await self._bump(self.username_cache, self._username_key(username)), - source=await self._bump(self.source_cache, self._source_key()), - ) - - @staticmethod - def delay_seconds(counts: FailureCounts) -> float: - """Seconds to hold a rejected attempt for, doubling per failure past whichever onset is further along.""" - steps: Final = min( - max(counts.username - USERNAME_DELAY_ONSET, counts.source - SOURCE_DELAY_ONSET), - _MAX_DELAY_DOUBLINGS, - ) - if steps < 0: - return 0.0 - return min(FIRST_DELAY_SECONDS * float(2**steps), MAX_DELAY_SECONDS) - - async def delay_for(self, username: str, counts: FailureCounts) -> None: - """Hold this rejected attempt open before answering it, so guessing costs wall-clock time. - - Only ever reached once the credentials are known to be wrong, so a valid password is - never delayed. Sources are capped at ``MAX_CONCURRENT_DELAYS_PER_SOURCE`` held - connections; over that, the attempt is refused immediately instead of parking a socket. - """ - if not self.enabled: - return - delay: Final = self.delay_seconds(counts) - if delay <= 0: - return - in_flight: Final = _DELAYS_IN_FLIGHT.get(self.client_ip, 0) - if in_flight >= MAX_CONCURRENT_DELAYS_PER_SOURCE: - verbose_proxy_logger.warning( - "Admin UI sign-in attempts held concurrently exhausted; username=%s source=%s in_flight=%s", - self._loggable(username), - self.client_ip, - in_flight, - ) - raise self._refused(int(MAX_DELAY_SECONDS), "concurrent_failed_logins") - _DELAYS_IN_FLIGHT[self.client_ip] = in_flight + 1 - try: - await _sleep(delay) - finally: - remaining: Final = _DELAYS_IN_FLIGHT.get(self.client_ip, 1) - 1 - if remaining > 0: - _DELAYS_IN_FLIGHT[self.client_ip] = remaining - else: - _DELAYS_IN_FLIGHT.pop(self.client_ip, None) - - async def clear(self, username: str) -> None: - """Drop the username counter after a successful sign-in. - - The source counter is left alone. It is shared by every account behind that address, - so one success there says nothing about the other attempts it is counting. - """ - if not self.enabled: - return - key: Final = self._username_key(username) - await self._shared(lambda redis_cache: redis_cache.async_delete_cache(key)) - await self._outcome(self.username_cache.async_delete_cache(key=key)) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index b88a7d14ad8..6f52babb255 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -27,7 +27,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured -from litellm.proxy.auth.login_throttle import LoginThrottle +from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -219,9 +219,21 @@ async def authenticate_user( admin_credentials_match: Final = _admin_credentials_match(username, password, master_key, general_settings) - if not admin_credentials_match: - await throttle.raise_if_blocked(username) + async with throttle.attempt(username, exempt=admin_credentials_match) as attempt: + return await _sign_in( + username, password, master_key, prisma_client, attempt, general_settings, admin_credentials_match + ) + +async def _sign_in( + username: str, + password: str, + master_key: str, + prisma_client: PrismaClient | None, + attempt: LoginAttempt, + general_settings: Mapping[str, object], + admin_credentials_match: bool, +) -> LoginResult: # Check if we can find the `username` in the db. On the UI, users can enter username=their email _user_row: LiteLLM_UserTable | None = None user_role: ( @@ -315,7 +327,7 @@ async def authenticate_user( key = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(user_info) - await throttle.clear(username) + await attempt.succeeded() return LoginResult( user_id=user_id, @@ -372,7 +384,7 @@ async def authenticate_user( key = response["token"] - await throttle.clear(username) + await attempt.succeeded() return LoginResult( user_id=user_id, @@ -382,7 +394,7 @@ async def authenticate_user( login_method="username_password", ) else: - await throttle.delay_for(username, await throttle.record_failure(username)) + await attempt.failed() raise ProxyException( message=_invalid_credentials_message(general_settings), type=ProxyErrorTypes.auth_error, @@ -390,7 +402,7 @@ async def authenticate_user( code=401, ) else: - await throttle.delay_for(username, await throttle.record_failure(username)) + await attempt.failed() raise ProxyException( message=_invalid_credentials_message(general_settings), type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 43013e85e05..5b89beb38d2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -325,7 +325,12 @@ from litellm.proxy.auth.auth_utils import ( from litellm.proxy.auth.fallback_model_access import router_fallback_access_check from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY, LicenseCheck -from litellm.proxy.auth.login_throttle import LoginThrottle, warn_login_counters_are_per_worker +from litellm.proxy.auth.login_throttle import ( + TRUSTED_PROXY_RANGES_KEY, + LoginThrottle, + warn_login_counters_are_per_worker, + warn_source_login_limit_is_off, +) from litellm.proxy.auth.model_checks import ( expand_wildcard_deployments_for_model_info, get_all_fallbacks, @@ -5803,11 +5808,10 @@ class ProxyConfig: if general_settings is None: general_settings = {} - ### FAILED-LOGIN ACCOUNTING MULTI-INSTANCE PREREQUISITE CHECK ### - # Failed Admin UI sign-in counters live in redis_usage_cache when available so a - # brute-force run is counted once across workers instead of once per worker. if os.getenv("NUM_WORKERS", "1") != "1" and redis_usage_cache is None: warn_login_counters_are_per_worker(os.getenv("NUM_WORKERS", "1")) + if not general_settings.get(TRUSTED_PROXY_RANGES_KEY): + warn_source_login_limit_is_off() _bg_hc_model_groups: Final = parse_background_health_check_model_groups(general_settings) _enable_hc_routing = False diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index c53476f5148..22fcedaa19d 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -25,33 +25,32 @@ class _RecordedSleeps: @pytest.fixture(autouse=True) def login_delays(monkeypatch): - """Replace the failed-login wait, so the suite pays no wall clock and can read it back.""" + """Replace the hold on a blocked wrong password, so the suite pays no wall clock and can read it back.""" from litellm.proxy.auth import login_throttle recorded = _RecordedSleeps() monkeypatch.setattr(login_throttle, "_sleep", recorded) - login_throttle._DELAYS_IN_FLIGHT.clear() + login_throttle._HELD_ATTEMPTS.clear() yield recorded - login_throttle._DELAYS_IN_FLIGHT.clear() + login_throttle._HELD_ATTEMPTS.clear() def _unlimited_throttle(): - """A throttle wired to a real in-memory store with a limit no test can reach.""" - from litellm.caching.dual_cache import DualCache + """A throttle wired to real in-memory stores with limits no test can reach.""" + from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.auth.login_throttle import LoginThrottle - store: Final = DualCache() return LoginThrottle( client_ip="1.2.3.4", - max_attempts=10_000, - max_attempts_per_source=10_000, - window_seconds=900, - username_cache=store, - source_cache=store, + source_limit=None, + user_limit=10_000, + window_seconds=60, + block_seconds=300, + counters=InMemoryCache(), + blocks=InMemoryCache(), ) - from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._types import ( LiteLLM_UserTable, @@ -658,29 +657,37 @@ class TestEncodeUiSessionJwt: def _throttle( - max_attempts: int = 3, - window_seconds: int = 900, + user_limit: int = 2, + source_limit: int | None = None, + window_seconds: int = 60, + block_seconds: int = 300, client_ip: str = "1.2.3.4", - cache=None, + stores=None, redis_cache=None, - max_attempts_per_source: int = 10_000, ): - """A throttle over a real in-memory store, so the tests exercise the true counters.""" - from litellm.caching.dual_cache import DualCache + """A throttle over real in-memory stores, so the tests exercise the true counters and blocks.""" + from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.auth.login_throttle import LoginThrottle - store: Final = cache if cache is not None else DualCache() + counters, blocks = stores if stores is not None else (InMemoryCache(), InMemoryCache()) return LoginThrottle( client_ip=client_ip, - max_attempts=max_attempts, - max_attempts_per_source=max_attempts_per_source, + source_limit=source_limit, + user_limit=user_limit, window_seconds=window_seconds, - username_cache=store, - source_cache=store, + block_seconds=block_seconds, + counters=counters, + blocks=blocks, redis_cache=redis_cache, ) +def _stores(): + from litellm.caching.in_memory_cache import InMemoryCache + + return InMemoryCache(), InMemoryCache() + + async def _guess(throttle, username: str = "admin", password: str = "wrong"): from litellm.proxy.auth.login_utils import authenticate_user @@ -693,97 +700,376 @@ async def _guess(throttle, username: str = "admin", password: str = "wrong"): ) +async def _fail(throttle, username: str = "admin") -> str: + """One wrong guess; returns the status code it was answered with.""" + from litellm.proxy._types import ProxyException + + with pytest.raises(ProxyException) as exc: + await _guess(throttle, username=username) + return exc.value.code + + +def _known_user(email: str = "known@example.com"): + user = MagicMock() + user.user_id = "u-1" + user.user_email = email + user.user_role = "internal_user" + user.password = "scrypt:stored" + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=user) + return repo + + +async def _db_login(throttle, username: str, password: str, *, correct: bool): + """A database user's sign-in with the stored hash faked, so no database or scrypt is needed.""" + from litellm.proxy.auth.login_utils import authenticate_user + + with ( + patch("litellm.proxy.auth.login_utils.UserRepository", _known_user(username)), + patch( # test-quality-ok: reaches the known-DB-user branch without a database + "litellm.proxy.auth.login_utils.verify_password", return_value=correct + ), + patch("litellm.proxy.auth.login_utils._rehash_password_if_needed", new=AsyncMock()), + patch( # test-quality-ok: success mints a UI key; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + return await authenticate_user( + username=username, password=password, master_key="sk-master", prisma_client=MagicMock(), throttle=throttle + ) + + +def _local_count(throttle, key: str) -> int: + return int(throttle.counters.get_cache(key) or 0) + + @pytest.mark.asyncio -async def test_attempts_are_refused_once_the_limit_is_reached(monkeypatch): - """The limit denies further attempts for the window, and the denial carries Retry-After.""" +async def test_too_many_failures_for_one_username_block_that_pair_and_carry_retry_after(monkeypatch): + """One failure past the pair limit blocks the source for that username; the next wrong guess is held + and answered 429 with the block's remaining time, and the counter is not touched by blocked guesses.""" from litellm.proxy._types import ProxyException monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=3, window_seconds=77) + throttle = _throttle(user_limit=2, block_seconds=77) + keys = throttle._keys("admin") - for _ in range(3): - with pytest.raises(ProxyException) as first: - await _guess(throttle) - assert first.value.code == "401" + assert [await _fail(throttle) for _ in range(3)] == ["401", "401", "401"], "the limit itself is a plain 401" + assert throttle._local_block_ttl(keys.pair_block) == 77 with pytest.raises(ProxyException) as blocked: await _guess(throttle) assert blocked.value.code == "429" - assert blocked.value.headers.get("Retry-After") == "77" + assert blocked.value.headers.get("Retry-After") == "47", "the 30s hold is taken off the remaining block" + assert _local_count(throttle, keys.pair_counter) == 3, "a blocked guess is not counted again" @pytest.mark.asyncio -async def test_a_correct_admin_password_is_accepted_while_blocked(monkeypatch): - """The configured admin credentials are compared before the gate, so the operator gets in. - - A throttle that refuses a valid password hands anyone who can reach the login form a - denial of service against the one account that can fix it. - """ - from litellm.proxy._types import ProxyException +async def test_a_wrong_password_from_a_blocked_key_is_held_before_it_is_refused(monkeypatch, login_delays): + """The hold is the rate cap: a blocked key gets one verified guess per held slot per 30 seconds.""" + from litellm.proxy.auth.login_throttle import BLOCKED_ATTEMPT_HOLD_SECONDS + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1) + + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"] + assert login_delays.seconds == [], "an unblocked wrong password is answered at once" + + assert await _fail(throttle) == "429" + assert login_delays.seconds == [BLOCKED_ATTEMPT_HOLD_SECONDS] + + +@pytest.mark.asyncio +async def test_a_correct_password_signs_in_while_its_pair_is_blocked(monkeypatch): + """The block is soft: the real user is still verified and gets in, so nobody can be locked out by + guessing at their account.""" monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") monkeypatch.setenv("DATABASE_URL", "postgresql://stub") - throttle = _throttle(max_attempts=2) + throttle = _throttle(user_limit=1) - for _ in range(2): - with pytest.raises(ProxyException): - await _guess(throttle) + assert [await _fail(throttle, username="user@corp.com") for _ in range(3)] == ["401", "401", "429"] - with pytest.raises(ProxyException) as still_blocked: - await _guess(throttle) - assert still_blocked.value.code == "429", "a wrong password is still refused" - - with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed - "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) - ): - result = await _guess(throttle, password="right") + result = await _db_login(throttle, "user@corp.com", "right", correct=True) assert result.key == "sk-ui" @pytest.mark.asyncio -async def test_a_blocked_attempt_does_not_extend_the_window(monkeypatch): - """Hammering while blocked must not push the counter or refresh its TTL.""" - from litellm.proxy._types import ProxyException - +async def test_a_correct_password_signs_in_while_its_source_is_blocked(monkeypatch): + """Same for the source-wide block: it slows guessing from that address, it does not refuse a user.""" monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=2) - key = throttle._username_key("admin") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=100, source_limit=2) - for _ in range(2): - with pytest.raises(ProxyException): - await _guess(throttle) - counted_at_limit = await throttle._failures(throttle.username_cache, key) + for i in range(3): + assert await _fail(throttle, username=f"other-{i}@corp.com") == "401" + assert await _fail(throttle, username="other-9@corp.com") == "429", "the source is blocked for everyone" - for _ in range(5): - with pytest.raises(ProxyException): - await _guess(throttle) - - assert await throttle._failures(throttle.username_cache, key) == counted_at_limit == 2 + result = await _db_login(throttle, "user@corp.com", "right", correct=True) + assert result.key == "sk-ui" @pytest.mark.asyncio -async def test_a_successful_sign_in_clears_the_bucket(monkeypatch): - """Success resets the budget rather than leaving the operator near the limit.""" - from litellm.proxy._types import ProxyException +async def test_a_successful_sign_in_clears_the_pair_counter_but_not_the_source_counter(monkeypatch): + """One account's success says nothing about the other guesses the address is making.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=5, source_limit=50) + keys = throttle._keys("user@corp.com") + + for _ in range(2): + assert await _fail(throttle, username="user@corp.com") == "401" + assert _local_count(throttle, keys.pair_counter) == 2 + assert _local_count(throttle, keys.source_counter) == 2 + + await _db_login(throttle, "user@corp.com", "right", correct=True) + + assert _local_count(throttle, keys.pair_counter) == 0 + assert _local_count(throttle, keys.source_counter) == 2 + + +@pytest.mark.asyncio +async def test_once_a_pair_is_blocked_its_failures_stop_counting_against_the_source(monkeypatch): + """A script stuck on one account trips the pair block and then leaves the office's shared address alone.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2, source_limit=4) + keys = throttle._keys("stuck-script@corp.com") + + assert [await _fail(throttle, username="stuck-script@corp.com") for _ in range(3)] == ["401"] * 3 + assert _local_count(throttle, keys.source_counter) == 2, "failures before the pair block count for the source" + + for _ in range(5): + assert await _fail(throttle, username="stuck-script@corp.com") == "429" + assert _local_count(throttle, keys.source_counter) == 2, "blocked-pair failures must not reach the source" + + assert await _fail(throttle, username="colleague@corp.com") == "401", "a colleague still signs in normally" + assert throttle._local_block_ttl(keys.source_block) == 0 + + +@pytest.mark.asyncio +async def test_the_blocking_failure_itself_does_not_count_against_the_source(monkeypatch): + """The guess that installs the pair block is the first one that stops counting, so a pair limit of B + costs the source exactly B, not B plus one.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=2, source_limit=2) + keys = throttle._keys("stuck@corp.com") + + assert [await _fail(throttle, username="stuck@corp.com") for _ in range(3)] == ["401", "401", "401"] + + assert _local_count(throttle, keys.source_counter) == 2 + assert throttle._local_block_ttl(keys.source_block) == 0, "the third guess blocked the pair, not the source" + + +@pytest.mark.asyncio +async def test_too_many_failures_across_usernames_block_the_whole_source(monkeypatch): + """A spray of one guess per username never trips a pair; the source counter is what stops it.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=5, source_limit=3, block_seconds=200) + + assert [await _fail(throttle, username=f"sprayed-{i}@corp.com") for i in range(4)] == ["401"] * 4 + + assert await _fail(throttle, username="sprayed-99@corp.com") == "429" + assert throttle._local_block_ttl(throttle._keys("x").source_block) == 200 + + +@pytest.mark.asyncio +async def test_without_trusted_proxy_ranges_the_source_scope_is_off(monkeypatch): + """Behind an ingress every client shares the peer address, so a source-wide block would block them all. + The pair scope still applies.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.9"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + throttle = LoginThrottle.from_request( + request, general_settings={"max_failed_login_attempts_per_source": 1}, redis_cache=None + ) + + assert throttle.source_limit is None + assert throttle.client_ip == "10.0.0.1", "the header is not trusted without a configured proxy range" + assert [await _fail(throttle, username=f"user-{i}@corp.com") for i in range(6)] == ["401"] * 6 + + +@pytest.mark.asyncio +async def test_with_trusted_proxy_ranges_the_source_is_the_forwarded_client(monkeypatch): + """The header is walked right to left past the trusted hops, so a forged left-most entry cannot pick the bucket.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + settings = {"trusted_proxy_ranges": ["10.0.0.0/8"], "max_failed_login_attempts_per_source": 2} + + def _from(peer: str, forwarded: str): + request = MagicMock() + request.headers = {"x-forwarded-for": forwarded} + request.client = MagicMock() + request.client.host = peer + return LoginThrottle.from_request(request, general_settings=settings, redis_cache=None) + + via_proxy = _from("10.0.0.1", "1.1.1.1, 203.0.113.9, 10.0.0.2") + assert via_proxy.client_ip == "203.0.113.9" + assert via_proxy.source_limit == 2 + + direct = _from("198.51.100.7", "203.0.113.9") + assert direct.client_ip == "198.51.100.7", "a peer outside the trusted ranges cannot forward anything" + + +def test_source_overrides_pick_the_most_specific_matching_range(): + """An exact address beats a /16 beats a /8; an address in none of them keeps the default.""" + from litellm.proxy.auth.login_throttle import LoginThrottle + + settings = { + "trusted_proxy_ranges": ["10.0.0.0/8"], + "max_failed_login_attempts_per_source": 7, + "max_failed_login_attempts_per_source_overrides": { + "203.0.0.0/8": 100, + "203.0.113.0/24": 200, + "203.0.113.9": 300, + "not-an-address": 999, + "198.51.100.0/24": "not-a-number", + }, + } + + def _limit(client: str) -> int | None: + request = MagicMock() + request.headers = {"x-forwarded-for": client} + request.client = MagicMock() + request.client.host = "10.0.0.1" + return LoginThrottle.from_request(request, general_settings=settings, redis_cache=None).source_limit + + assert _limit("203.0.113.9") == 300 + assert _limit("203.0.113.10") == 200 + assert _limit("203.0.1.1") == 100 + assert _limit("192.0.2.1") == 7 + assert _limit("198.51.100.1") == 7, "a garbage limit falls back to the default rather than a huge or zero budget" + + +def test_ipv6_sources_are_grouped_by_their_64_bit_prefix(): + """A /64 holder has 2^64 addresses; counting each one separately would hand them unlimited fresh buckets.""" + from litellm.proxy.auth.login_throttle import source_group + + assert source_group("2001:db8:1:2::1") == source_group("2001:db8:1:2:ffff:ffff:ffff:ffff") == "2001:db8:1:2::/64" + assert source_group("2001:db8:1:3::1") != source_group("2001:db8:1:2::1") + assert source_group("::ffff:203.0.113.9") == source_group("203.0.113.9") == "203.0.113.9" + assert source_group("unknown") == "unknown" + + +@pytest.mark.asyncio +async def test_two_ipv6_addresses_in_one_64_share_the_source_budget(monkeypatch): + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + stores = _stores() + first = _throttle(user_limit=50, source_limit=2, client_ip="2001:db8:1:2::1", stores=stores) + second = _throttle(user_limit=50, source_limit=2, client_ip="2001:db8:1:2::2", stores=stores) + + assert [await _fail(first, username=f"a-{i}@corp.com") for i in range(3)] == ["401"] * 3 + assert await _fail(second, username="b@corp.com") == "429" + + +@pytest.mark.asyncio +async def test_one_source_being_blocked_does_not_touch_another(monkeypatch): + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + stores = _stores() + attacker = _throttle(user_limit=50, source_limit=2, client_ip="203.0.113.9", stores=stores) + neighbour = _throttle(user_limit=50, source_limit=2, client_ip="198.51.100.7", stores=stores) + + assert [await _fail(attacker, username=f"t-{i}@corp.com") for i in range(3)] == ["401"] * 3 + assert await _fail(attacker, username="t-9@corp.com") == "429" + assert await _fail(neighbour, username="t-9@corp.com") == "401" + + +@pytest.mark.asyncio +async def test_the_same_username_from_another_source_has_its_own_budget(monkeypatch): + """The pair carries the address on purpose: an attacker elsewhere cannot lock a user out of their own office.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + stores = _stores() + attacker = _throttle(user_limit=1, client_ip="203.0.113.9", stores=stores) + office = _throttle(user_limit=1, client_ip="198.51.100.7", stores=stores) + + assert [await _fail(attacker, username="victim@corp.com") for _ in range(3)] == ["401", "401", "429"] + assert await _fail(office, username="victim@corp.com") == "401" + + +@pytest.mark.asyncio +async def test_the_counting_window_is_anchored_at_the_first_failure(monkeypatch): + """Later failures must not push the expiry out, or a slow guesser keeps their own count alive forever.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=50, window_seconds=60) + key = throttle._keys("admin").pair_counter + + await _fail(throttle) + first_expiry = throttle.counters.ttl_dict[key] + for _ in range(3): + await _fail(throttle) + + assert throttle.counters.ttl_dict[key] == first_expiry + + +@pytest.mark.asyncio +async def test_the_block_outlives_the_counting_window(monkeypatch): + """Counters expire after the window and blocks after the block time; the two are separate keys.""" + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1, window_seconds=10, block_seconds=300) + keys = throttle._keys("admin") + + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"] + + throttle.counters.delete_cache(keys.pair_counter) + + assert await _fail(throttle) == "429", "an expired counter must not lift an active block" + assert 290 <= throttle._local_block_ttl(keys.pair_block) <= 300 + + +@pytest.mark.asyncio +async def test_the_block_time_is_fixed_and_not_refreshed_by_blocked_guesses(monkeypatch): + monkeypatch.setenv("UI_USERNAME", "admin") + monkeypatch.setenv("UI_PASSWORD", "right") + throttle = _throttle(user_limit=1, block_seconds=300) + key = throttle._keys("admin").pair_block + + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"] + installed_at = throttle.blocks.ttl_dict[key] + + for _ in range(4): + assert await _fail(throttle) == "429" + + assert throttle.blocks.ttl_dict[key] == installed_at + + +@pytest.mark.asyncio +async def test_the_configured_admin_credentials_bypass_the_throttle(monkeypatch): + """The only account that can fix a misconfiguration is exempt: no hold, no slot, even while blocked.""" + from litellm.proxy.auth import login_throttle as lt monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") monkeypatch.setenv("DATABASE_URL", "postgresql://stub") - throttle = _throttle(max_attempts=3) + throttle = _throttle(user_limit=1) - for _ in range(2): - with pytest.raises(ProxyException): - await _guess(throttle) + assert [await _fail(throttle) for _ in range(3)] == ["401", "401", "429"] - with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed - "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + with ( + patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), + patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), ): - await _guess(throttle, password="right") - - assert await throttle._failures(throttle.username_cache, throttle._username_key("admin")) == 0 + result = await _guess(throttle, password="right") + assert result.key == "sk-ui" + assert lt._HELD_ATTEMPTS == {} @pytest.mark.asyncio @@ -791,7 +1077,7 @@ async def test_a_configuration_error_never_counts(monkeypatch): """A 500 from an unset master key is not a guess and must not consume the budget.""" from litellm.proxy._types import ProxyException - throttle = _throttle(max_attempts=2) + throttle = _throttle(user_limit=2) for _ in range(5): with pytest.raises(ProxyException) as exc: await authenticate_user( @@ -799,44 +1085,20 @@ async def test_a_configuration_error_never_counts(monkeypatch): ) assert exc.value.code == "500" - assert await throttle._failures(throttle.username_cache, throttle._username_key("admin")) == 0 + assert _local_count(throttle, throttle._keys("admin").pair_counter) == 0 @pytest.mark.asyncio -async def test_the_username_is_case_folded_into_one_bucket(monkeypatch): +async def test_the_username_is_case_folded_into_one_pair(monkeypatch): """The DB lookup is case-insensitive, so casing must not multiply the budget.""" - from litellm.proxy._types import ProxyException - monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=4) + throttle = _throttle(user_limit=4) - for name in ("admin@corp.com", "ADMIN@corp.com", "Admin@corp.com", "aDmIn@corp.com"): - with pytest.raises(ProxyException) as exc: - await _guess(throttle, username=name) - assert exc.value.code == "401" + for name in ("admin@corp.com", "ADMIN@corp.com", "Admin@corp.com", "aDmIn@corp.com", "admin@CORP.com"): + assert await _fail(throttle, username=name) == "401" - with pytest.raises(ProxyException) as blocked: - await _guess(throttle, username="admin@CORP.com") - assert blocked.value.code == "429" - - -@pytest.mark.asyncio -async def test_a_different_username_from_the_same_source_is_unaffected(monkeypatch): - """The counters are independent, so one username's failures do not exhaust another's.""" - from litellm.proxy._types import ProxyException - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=2) - - for _ in range(3): - with pytest.raises(ProxyException): - await _guess(throttle, username="admin") - - with pytest.raises(ProxyException) as other: - await _guess(throttle, username="someone-else@example.com") - assert other.value.code == "401", "a second username must still reach the credential check" + assert await _fail(throttle, username="admin@Corp.com") == "429" @pytest.mark.asyncio @@ -848,26 +1110,9 @@ async def test_both_credential_rejections_are_indistinguishable(monkeypatch): monkeypatch.setenv("UI_PASSWORD", "right") with pytest.raises(ProxyException) as unknown: - await _guess(_throttle(max_attempts=99), username="nobody@example.com") - - fake_user = MagicMock() - fake_user.user_id = "u-1" - fake_user.user_email = "known@example.com" - fake_user.user_role = "internal_user" - fake_user.password = "scrypt:fake" - repo = MagicMock() - repo.return_value.table.find_first = AsyncMock(return_value=fake_user) - with patch("litellm.proxy.auth.login_utils.UserRepository", repo), patch( # test-quality-ok: reaches the known-DB-user branch without a database - "litellm.proxy.auth.login_utils.verify_password", return_value=False - ): - with pytest.raises(ProxyException) as known: - await authenticate_user( - username="known@example.com", - password="wrong", - master_key="sk-master", - prisma_client=MagicMock(), - throttle=_throttle(max_attempts=99), - ) + await _guess(_throttle(user_limit=99), username="nobody@example.com") + with pytest.raises(ProxyException) as known: + await _db_login(_throttle(user_limit=99), "known@example.com", "wrong", correct=False) assert unknown.value.message == known.value.message assert "known@example.com" not in unknown.value.message + known.value.message @@ -875,13 +1120,12 @@ async def test_both_credential_rejections_are_indistinguishable(monkeypatch): @pytest.mark.asyncio async def test_a_user_with_no_password_set_does_not_consume_the_budget(monkeypatch): - """That 401 is deterministic and guards no secret, so counting it would only let - someone burn a passwordless account's bucket.""" + """That 401 is deterministic and guards no secret, so counting it would only let someone burn the pair.""" from litellm.proxy._types import ProxyException monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=2) + throttle = _throttle(user_limit=2) passwordless = MagicMock() passwordless.user_id = "u-2" @@ -891,7 +1135,9 @@ async def test_a_user_with_no_password_set_does_not_consume_the_budget(monkeypat repo = MagicMock() repo.return_value.table.find_first = AsyncMock(return_value=passwordless) - with patch("litellm.proxy.auth.login_utils.UserRepository", repo): # test-quality-ok: reaches the passwordless-DB-user branch without a database + with patch( + "litellm.proxy.auth.login_utils.UserRepository", repo + ): # test-quality-ok: reaches the passwordless-DB-user branch without a database for _ in range(5): with pytest.raises(ProxyException) as exc: await authenticate_user( @@ -903,185 +1149,36 @@ async def test_a_user_with_no_password_set_does_not_consume_the_budget(monkeypat ) assert exc.value.code == "401" - assert await throttle._failures(throttle.username_cache, throttle._username_key("nopass@example.com")) == 0 + assert _local_count(throttle, throttle._keys("nopass@example.com").pair_counter) == 0 @pytest.mark.asyncio async def test_a_wrong_password_for_a_known_user_also_counts(monkeypatch): - """The database-user branch must charge the bucket too, not just the unknown-user branch.""" + """The database-user branch must charge the pair too, not just the unknown-user branch.""" from litellm.proxy._types import ProxyException monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=3) + throttle = _throttle(user_limit=2) - known = MagicMock() - known.user_id = "u-1" - known.user_email = "known@example.com" - known.user_role = "internal_user" - known.password = "scrypt:stored" - repo = MagicMock() - repo.return_value.table.find_first = AsyncMock(return_value=known) - - async def _attempt(): - return await authenticate_user( - username="known@example.com", - password="wrong", - master_key="sk-master", - prisma_client=MagicMock(), - throttle=throttle, - ) - - with patch("litellm.proxy.auth.login_utils.UserRepository", repo), patch( # test-quality-ok: reaches the known-DB-user branch without a database - "litellm.proxy.auth.login_utils.verify_password", return_value=False - ): - for _ in range(3): - with pytest.raises(ProxyException) as rejected: - await _attempt() - assert rejected.value.code == "401" - - with pytest.raises(ProxyException) as blocked: - await _attempt() - assert blocked.value.code == "429" - - -@pytest.mark.asyncio -async def test_one_source_exhausting_its_own_budget_does_not_refuse_another_source(monkeypatch): - """The source counter is per address, so a noisy office does not take its neighbour down.""" - from litellm.caching.dual_cache import DualCache - from litellm.proxy._types import ProxyException - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - shared_store = DualCache() - attacker = _throttle( - max_attempts=10_000, max_attempts_per_source=2, client_ip="203.0.113.9", cache=shared_store - ) - operator = _throttle( - max_attempts=10_000, max_attempts_per_source=2, client_ip="198.51.100.7", cache=shared_store - ) - - for i in range(2): - with pytest.raises(ProxyException): - await _guess(attacker, username=f"target-{i}@corp.com") - - with pytest.raises(ProxyException) as blocked: - await _guess(attacker, username="target-2@corp.com") - assert blocked.value.code == "429" - - with pytest.raises(ProxyException) as unaffected: - await _guess(operator, username="target-3@corp.com") - assert unaffected.value.code == "401", "the other address must still reach the credential check" - - -@pytest.mark.asyncio -async def test_a_username_exhausted_from_one_source_is_refused_from_another(monkeypatch): - """The username counter carries no address, so spreading the guesses buys nothing. - - The pair key this replaced reset the budget for every new address, which is exactly the - shape of a credential-stuffing run from a proxy pool. - """ - from litellm.caching.dual_cache import DualCache - from litellm.proxy._types import ProxyException - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - shared_store = DualCache() - first_hop = _throttle(max_attempts=2, client_ip="203.0.113.9", cache=shared_store) - second_hop = _throttle(max_attempts=2, client_ip="198.51.100.7", cache=shared_store) - - for _ in range(2): - with pytest.raises(ProxyException): - await _guess(first_hop, username="victim@corp.com") - - with pytest.raises(ProxyException) as rotated: - await _guess(second_hop, username="victim@corp.com") - assert rotated.value.code == "429" - - -@pytest.mark.asyncio -async def test_a_source_wide_spray_is_counted_even_though_each_username_is_fresh(monkeypatch): - """One guess against each of many usernames never trips a username counter, only the source one.""" - from litellm.proxy._types import ProxyException - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=10_000, max_attempts_per_source=6, client_ip="203.0.113.11") - - for i in range(6): + for _ in range(3): with pytest.raises(ProxyException) as rejected: - await _guess(throttle, username=f"sprayed-{i}@corp.com") + await _db_login(throttle, "known@example.com", "wrong", correct=False) assert rejected.value.code == "401" with pytest.raises(ProxyException) as blocked: - await _guess(throttle, username="sprayed-7@corp.com") + await _db_login(throttle, "known@example.com", "wrong", correct=False) assert blocked.value.code == "429" - assert await throttle._failures(throttle.username_cache, throttle._username_key("sprayed-7@corp.com")) == 0 @pytest.mark.asyncio -async def test_a_successful_sign_in_leaves_the_source_counter_alone(monkeypatch): - """One account's success says nothing about the other attempts the address is making.""" - from litellm.proxy._types import ProxyException - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - monkeypatch.setenv("DATABASE_URL", "postgresql://stub") - throttle = _throttle(max_attempts=10) - - for _ in range(2): - with pytest.raises(ProxyException): - await _guess(throttle) - - with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed - "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) - ): - await _guess(throttle, password="right") - - assert await throttle._failures(throttle.username_cache, throttle._username_key("admin")) == 0 - assert await throttle._failures(throttle.source_cache, throttle._source_key()) == 2 - - -@pytest.mark.asyncio -async def test_the_delay_doubles_from_one_second_and_is_capped(monkeypatch, login_delays): - """Guessing has to cost wall clock, and the cost has to stop short of an unbounded hang.""" - from litellm.proxy._types import ProxyException - from litellm.proxy.auth.login_throttle import MAX_DELAY_SECONDS - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=10_000) - - for _ in range(9): - with pytest.raises(ProxyException): - await _guess(throttle) - - assert login_delays.seconds == [1.0, 2.0, 4.0, 8.0, 16.0, 30.0, 30.0], ( - "the first two failures answer immediately, then the wait doubles up to the cap" - ) - assert max(login_delays.seconds) == MAX_DELAY_SECONDS - - -@pytest.mark.asyncio -async def test_the_delay_tracks_whichever_counter_is_further_past_its_onset(monkeypatch): - """A source deep into a spray must not be answered instantly just because the username is fresh.""" - from litellm.proxy.auth.login_throttle import FailureCounts, LoginThrottle - - assert LoginThrottle.delay_seconds(FailureCounts(username=1, source=1)) == 0.0 - assert LoginThrottle.delay_seconds(FailureCounts(username=2, source=24)) == 0.0 - assert LoginThrottle.delay_seconds(FailureCounts(username=3, source=1)) == 1.0 - assert LoginThrottle.delay_seconds(FailureCounts(username=1, source=25)) == 1.0 - assert LoginThrottle.delay_seconds(FailureCounts(username=4, source=28)) == 8.0 - - -@pytest.mark.asyncio -async def test_held_attempts_from_one_source_are_capped(monkeypatch): - """Holding a rejected attempt open must not let one address park unlimited sockets.""" +async def test_held_attempts_from_one_blocked_key_are_capped(monkeypatch): + """Holding a wrong guess open must not let one blocked key park unlimited sockets in password checks.""" import asyncio from litellm.proxy._types import ProxyException from litellm.proxy.auth import login_throttle as lt - from litellm.proxy.auth.login_throttle import MAX_CONCURRENT_DELAYS_PER_SOURCE + from litellm.proxy.auth.login_throttle import MAX_HELD_ATTEMPTS_PER_KEY monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") @@ -1091,134 +1188,102 @@ async def test_held_attempts_from_one_source_are_capped(monkeypatch): await release.wait() monkeypatch.setattr(lt, "_sleep", _park) - throttle = _throttle(max_attempts=10_000, client_ip="203.0.113.44") - await throttle.record_failure("admin") - await throttle.record_failure("admin") + throttle = _throttle(user_limit=1, client_ip="203.0.113.44") + slot = throttle._keys("admin").pair_block + assert [await _fail(throttle) for _ in range(2)] == ["401", "401"] - held = [asyncio.create_task(_guess(throttle)) for _ in range(MAX_CONCURRENT_DELAYS_PER_SOURCE)] + held = [asyncio.create_task(_guess(throttle)) for _ in range(MAX_HELD_ATTEMPTS_PER_KEY)] for _ in range(1000): - if lt._DELAYS_IN_FLIGHT.get("203.0.113.44") == MAX_CONCURRENT_DELAYS_PER_SOURCE: + if lt._HELD_ATTEMPTS.get(slot) == MAX_HELD_ATTEMPTS_PER_KEY: break await asyncio.sleep(0) - assert lt._DELAYS_IN_FLIGHT.get("203.0.113.44") == MAX_CONCURRENT_DELAYS_PER_SOURCE + assert lt._HELD_ATTEMPTS.get(slot) == MAX_HELD_ATTEMPTS_PER_KEY try: with pytest.raises(ProxyException) as over_cap: await _guess(throttle) assert over_cap.value.code == "429" assert over_cap.value.headers.get("Retry-After") == "30" + assert await _fail(throttle, username="someone-else@corp.com") == "401", "other keys are not affected" finally: release.set() for task in held: with pytest.raises(ProxyException): await task - with pytest.raises(ProxyException) as after_drain: - await _guess(throttle) - assert after_drain.value.code == "401", "the cap must release once the held attempts answer" + assert lt._HELD_ATTEMPTS.get(slot) is None, "the slots are released once the held attempts answer" @pytest.mark.asyncio -async def test_disabling_the_control_removes_the_delay_as_well(monkeypatch, login_delays): +async def test_disabling_the_control_removes_the_hold_as_well(monkeypatch, login_delays): """The escape hatch has to turn off the whole control, not only the refusal.""" import dataclasses - from litellm.proxy._types import ProxyException - monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = dataclasses.replace(_throttle(max_attempts=2), enabled=False) - - for _ in range(6): - with pytest.raises(ProxyException) as rejected: - await _guess(throttle) - assert rejected.value.code == "401" + throttle = dataclasses.replace(_throttle(user_limit=1), enabled=False) + assert [await _fail(throttle) for _ in range(6)] == ["401"] * 6 assert login_delays.seconds == [] class _FakeRedis: - """Redis whose only counter write is the atomic INCRBY-plus-EXPIRE Lua call. + """Redis whose only writes are the throttle's two scripts, run atomically as one call each. - `async_increment` is deliberately absent: a two-step increment would fail the test - with AttributeError, because Redis could then commit a count without its expiry. + Mirrors the Lua: a blocked key returns its remaining block time and is not counted; a counter + is expired on first write; one over the limit installs the block; a blocked pair stops the + source from being counted. The real scripts are exercised against a live Redis in the PR's + proof, this fake only has to be faithful enough for the worker-sharing tests. """ def __init__(self): self.values: dict = {} self.ttls: dict = {} + self.scripts: list[str] = [] - async def async_get_cache(self, key, **kwargs): - return self.values.get(key) + def async_register_script(self, script: str): + from litellm.proxy.auth import login_throttle as lt - async def async_batch_get_counts(self, key_list): - return tuple(self.values.get(key) for key in key_list) + async def _run(keys, args): + self.scripts.append(script) + if script == lt._BLOCK_TTLS_LUA: + return [self._ttl(keys[1]), self._ttl(keys[3])] + assert script == lt._RECORD_FAILURE_LUA + user_limit, source_limit, window, block = (int(a) for a in args) + user_block = self._bump(keys[0], keys[1], user_limit, window, block) + if source_limit > 0 and user_block == 0: + return [user_block, self._bump(keys[2], keys[3], source_limit, window, block)] + return [user_block, 0] - async def async_increment_with_floor(self, key, value, ttl): - self.values[key] = self.values.get(key, 0) + value - self.ttls.setdefault(key, ttl) - return self.values[key] + return _run - async def async_get_ttl(self, key): - return self.ttls.get(key) + def _ttl(self, key: str) -> int: + return self.ttls.get(key, -2) if key in self.values else -2 + + def _bump(self, count_key: str, block_key: str, limit: int, window: int, block: int) -> int: + if self._ttl(block_key) > 0: + return self._ttl(block_key) + self.values[count_key] = self.values.get(count_key, 0) + 1 + self.ttls.setdefault(count_key, window) + if self.values[count_key] > limit: + self.values[block_key] = 1 + self.ttls[block_key] = block + return block + return 0 async def async_delete_cache(self, key): self.values.pop(key, None) self.ttls.pop(key, None) - def persist(self): - self.ttls.clear() - - -@pytest.mark.asyncio -async def test_counters_are_written_with_their_expiry_and_re_armed_if_stripped(monkeypatch): - """Regression: a counter with no TTL would refuse the pair forever. - - Nothing increments a key once the limit is reached, so a counter that ever exists - without an expiry stays refused with no way back. Every write must therefore carry the - expiry, and a refusal that finds it stripped (PERSIST) must put the window back. - """ - from litellm.proxy._types import ProxyException - - monkeypatch.setenv("UI_USERNAME", "admin") - monkeypatch.setenv("UI_PASSWORD", "right") - redis = _FakeRedis() - throttle = _throttle(max_attempts=2, window_seconds=77, redis_cache=redis) - - for _ in range(2): - with pytest.raises(ProxyException): - await _guess(throttle) - - assert redis.values, "failures must land in the shared counter" - assert set(redis.ttls) == set(redis.values), "no counter may exist without its expiry" - assert set(redis.ttls.values()) == {77} - - redis.persist() - with pytest.raises(ProxyException) as blocked: - await _guess(throttle) - assert blocked.value.code == "429" - assert blocked.value.headers.get("Retry-After") == "77" - assert set(redis.ttls) >= {k for k in redis.values if ":user:" in k}, "the refusal must re-arm a stripped expiry" - class _DownRedis(_FakeRedis): - """Redis whose every call fails, as during an outage or an open circuit breaker. + """Redis whose every call fails, as during an outage or an open circuit breaker.""" - `async_get_cache` returns None rather than raising, as the real one does: it swallows the - error, so a failed GET is indistinguishable from an empty key to anyone reading through it. - """ + def async_register_script(self, script: str): + async def _run(keys, args): + raise ConnectionError("redis is down") - async def async_get_cache(self, key, **kwargs): - return None - - async def async_batch_get_counts(self, key_list): - raise ConnectionError("redis is down") - - async def async_increment_with_floor(self, key, value, ttl): - raise ConnectionError("redis is down") - - async def async_get_ttl(self, key): - raise ConnectionError("redis is down") + return _run async def async_delete_cache(self, key): raise ConnectionError("redis is down") @@ -1226,124 +1291,84 @@ class _DownRedis(_FakeRedis): @pytest.mark.asyncio async def test_redis_is_the_only_counter_while_it_answers(monkeypatch): - """Regression: every worker must spend the same budget, and a success must clear it for all. - - Counting in this worker's memory as well as in Redis let the two drift apart: a worker - whose Redis write failed kept its own count while the others gave the attacker fresh - guesses, and a stale local count outlived the shared clear after a correct password. - """ - from litellm.caching.dual_cache import DualCache - from litellm.proxy._types import ProxyException + """Every worker must spend the same budget, see the same block, and a success must clear the pair for all.""" from litellm.proxy.auth.login_throttle import _CACHE_KEY_PREFIX monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") redis = _FakeRedis() - first_worker_store = DualCache() - second_worker_store = DualCache() - first_worker = _throttle(max_attempts=2, cache=first_worker_store, redis_cache=redis) - second_worker = _throttle(max_attempts=2, cache=second_worker_store, redis_cache=redis) + first_worker = _throttle(user_limit=2, stores=_stores(), redis_cache=redis) + second_worker = _throttle(user_limit=2, stores=_stores(), redis_cache=redis) - for _ in range(2): - with pytest.raises(ProxyException, match="Invalid credentials"): - await _guess(first_worker) - - assert not [k for k in first_worker_store.in_memory_cache.cache_dict if str(k).startswith(_CACHE_KEY_PREFIX)], ( + assert [await _fail(first_worker, username="user@corp.com") for _ in range(3)] == ["401"] * 3 + assert not [k for k in first_worker.counters.cache_dict if str(k).startswith(_CACHE_KEY_PREFIX)], ( "with Redis answering, no worker may keep a counter of its own" ) - with pytest.raises(ProxyException) as blocked: - await _guess(second_worker) - assert blocked.value.code == "429", "the second worker must see the budget the first one spent" + assert not first_worker.blocks.cache_dict - monkeypatch.setenv("DATABASE_URL", "postgresql://stub") - with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed - "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) - ): - await _guess(second_worker, password="right") + assert await _fail(second_worker, username="user@corp.com") == "429", "the second worker sees the block" - assert not [k for k in redis.values if ":user:" in k], "a success must clear the shared username counter" - with pytest.raises(ProxyException, match="Invalid credentials"): - await _guess(first_worker) + await _db_login(second_worker, "user@corp.com", "right", correct=True) + + assert not [k for k in redis.values if ":user:" in k and ":block:" not in k], ( + "success clears the shared pair counter" + ) + assert [k for k in redis.values if ":block:user:" in k], "an active block is not lifted by one success" @pytest.mark.asyncio async def test_a_redis_outage_falls_back_to_this_workers_own_counter(monkeypatch): - """With Redis raising, guesses are still counted and refused, per worker, instead of unbounded.""" + """With Redis raising, guesses are still counted and blocked per worker, with a warning, instead of unbounded.""" + import logging + + from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=2, redis_cache=_DownRedis()) + throttle = _throttle(user_limit=2, block_seconds=300, redis_cache=_DownRedis()) - for _ in range(2): - with pytest.raises(ProxyException, match="Invalid credentials"): + records: list[logging.LogRecord] = [] + handler = logging.Handler() + handler.emit = records.append + verbose_proxy_logger.addHandler(handler) + try: + assert [await _fail(throttle) for _ in range(3)] == ["401"] * 3 + with pytest.raises(ProxyException) as blocked: await _guess(throttle) + finally: + verbose_proxy_logger.removeHandler(handler) - with pytest.raises(ProxyException) as blocked: - await _guess(throttle) assert blocked.value.code == "429" - assert blocked.value.headers.get("Retry-After") == "900" - - -class _WriteRefusingRedis(_FakeRedis): - """Redis that answers reads but raises on writes until `recover()` is called.""" - - def __init__(self): - super().__init__() - self.writable = False - - def recover(self): - self.writable = True - - async def async_increment_with_floor(self, key, value, ttl): - if not self.writable: - raise ConnectionError("redis write failed") - return await super().async_increment_with_floor(key, value, ttl) + assert blocked.value.headers.get("Retry-After") == "270" + assert any("Redis failed while counting Admin UI sign-in attempts" in r.getMessage() for r in records) @pytest.mark.asyncio -async def test_failures_redis_refused_still_count_once_redis_recovers(monkeypatch): - """Regression: a guess Redis could not record must not be forgotten when Redis comes back. - - Such a guess lands in this worker's own store. Reading only Redis afterwards handed the - attacker that guess again, so the budget was the limit plus however many writes failed. - """ - from litellm.proxy._types import ProxyException - +async def test_a_failed_redis_delete_still_clears_this_workers_counter(monkeypatch): + """The fail-open tradeoff: when Redis cannot clear the pair, the worker clears what it holds and moves on.""" monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - redis = _WriteRefusingRedis() - throttle = _throttle(max_attempts=2, redis_cache=redis) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + throttle = _throttle(user_limit=5, redis_cache=_DownRedis()) + key = throttle._keys("user@corp.com").pair_counter - with pytest.raises(ProxyException, match="Invalid credentials"): - await _guess(throttle) - assert not redis.values, "the refused write must not have reached Redis" + assert [await _fail(throttle, username="user@corp.com") for _ in range(2)] == ["401", "401"] + assert _local_count(throttle, key) == 2 - redis.recover() - with pytest.raises(ProxyException, match="Invalid credentials"): - await _guess(throttle) - assert [v for k, v in redis.values.items() if ":user:" in k] == [1], "only the recorded guess is in Redis" - - with pytest.raises(ProxyException) as blocked: - await _guess(throttle) - assert blocked.value.code == "429", "the guess Redis missed and the one it took must add up to the limit" + await _db_login(throttle, "user@corp.com", "right", correct=True) + assert _local_count(throttle, key) == 0 @pytest.mark.asyncio async def test_counters_do_not_share_the_key_authentication_cache(monkeypatch): - """Regression: throttle entries must not evict cached credentials. - - user_api_key_cache holds at most 200 in-memory entries and evicts the soonest to - expire first, so parking 900s sign-in counters there let a stream of made-up usernames - push out the much shorter lived credential entries, sending every ordinary API request - back to the database. - """ + """Regression: throttle entries must not evict cached credentials from user_api_key_cache.""" from litellm.proxy import proxy_server as ps from litellm.proxy.auth.login_throttle import _CACHE_KEY_PREFIX, LoginThrottle monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - auth_cache_keys_before = set(ps.user_api_key_cache.in_memory_cache.cache_dict) request = MagicMock() @@ -1353,53 +1378,66 @@ async def test_counters_do_not_share_the_key_authentication_cache(monkeypatch): throttle = LoginThrottle.from_request(request, general_settings={}, redis_cache=None) for i in range(25): - with pytest.raises(ProxyException, match="Invalid credentials"): - await _guess(throttle, username=f"made-up-{i}@example.com") + assert await _fail(throttle, username=f"made-up-{i}@example.com") == "401" added = set(ps.user_api_key_cache.in_memory_cache.cache_dict) - auth_cache_keys_before - assert not [k for k in added if str(k).startswith(_CACHE_KEY_PREFIX)], ( - "sign-in counters must live in their own cache, not the key-authentication cache" - ) + assert not [k for k in added if str(k).startswith(_CACHE_KEY_PREFIX)] def test_settings_that_arrive_as_environment_strings_are_honored(): - """An `os.environ/VAR` reference in general_settings resolves to a string, not an int. - - Regression: a digit string fell back to the default with only a log line, so an operator - tightening the limits through environment substitution silently kept the stock ceilings. - """ + """An `os.environ/VAR` reference in general_settings resolves to a string, not an int.""" from litellm.proxy.auth.login_throttle import LoginThrottle request = MagicMock() - request.headers = {} + request.headers = {"x-forwarded-for": "203.0.113.9"} request.client = MagicMock() - request.client.host = "1.2.3.4" + request.client.host = "10.0.0.1" throttle = LoginThrottle.from_request( request, general_settings={ - "max_failed_login_attempts": "7", + "trusted_proxy_ranges": "10.0.0.0/8", "max_failed_login_attempts_per_source": " 70 ", + "max_failed_login_attempts_per_user": "7", "failed_login_window_seconds": "not-a-number", + "failed_login_block_seconds": "-5", }, redis_cache=None, ) - assert throttle.max_attempts == 7 - assert throttle.max_attempts_per_source == 70 - assert throttle.window_seconds == 900, "garbage still falls back to the default" + assert throttle.source_limit == 70 + assert throttle.user_limit == 7 + assert throttle.window_seconds == 60, "garbage falls back to the default" + assert throttle.block_seconds == 300, "a value below one would block nothing or forever" + + +def test_the_defaults_are_the_agreed_ones(): + from litellm.proxy.auth.login_throttle import LoginThrottle + + request = MagicMock() + request.headers = {"x-forwarded-for": "203.0.113.9"} + request.client = MagicMock() + request.client.host = "10.0.0.1" + throttle = LoginThrottle.from_request( + request, general_settings={"trusted_proxy_ranges": ["10.0.0.0/8"]}, redis_cache=None + ) + + assert (throttle.source_limit, throttle.user_limit, throttle.window_seconds, throttle.block_seconds) == ( + 10, + 5, + 60, + 300, + ) def test_the_disable_flag_is_read_once_not_per_login_attempt(monkeypatch): - """Regression: the kill switch was read through the secret manager on every unauthenticated request. - - With a hosted secret manager in read mode that is a synchronous network call per guess, so a - flood of wrong passwords could exhaust the secret manager even after the source was refused. - """ + """Regression: the kill switch was read through the secret manager on every unauthenticated request.""" from litellm.proxy.auth import login_throttle reads: Final[list[str]] = [] # mutable-ok: test-only call recorder - monkeypatch.setattr(login_throttle, "get_secret_bool", lambda name, default: reads.append(name) or default) + monkeypatch.setattr( + login_throttle, "get_secret_bool", lambda name, default_value: reads.append(name) or default_value + ) login_throttle._rate_limit_disabled.cache_clear() request = MagicMock() request.headers = {} @@ -1413,105 +1451,71 @@ def test_the_disable_flag_is_read_once_not_per_login_attempt(monkeypatch): assert reads == ["LITELLM_DISABLE_LOGIN_RATE_LIMIT"] -def test_a_negative_or_boolean_setting_falls_back_to_the_default(): - """A limit below one would refuse everyone; a bool is a typo, not a count.""" - from litellm.proxy.auth.login_throttle import LoginThrottle - - request = MagicMock() - request.headers = {} - request.client = MagicMock() - request.client.host = "1.2.3.4" - - throttle = LoginThrottle.from_request( - request, - general_settings={"max_failed_login_attempts": "-7", "max_failed_login_attempts_per_source": True}, - redis_cache=None, - ) - - assert throttle.max_attempts == 50 - assert throttle.max_attempts_per_source == 250 - - @pytest.mark.asyncio -async def test_a_refused_username_cannot_forge_log_lines(monkeypatch): +async def test_a_blocked_username_cannot_forge_log_lines(monkeypatch): """The username reaches a warning log, so it must not carry newlines or control bytes.""" import logging from litellm._logging import verbose_proxy_logger - from litellm.proxy._types import ProxyException monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - throttle = _throttle(max_attempts=1) + throttle = _throttle(user_limit=1) forged = "victim@example.com\nWARNING: sign-in succeeded for attacker\x00" - with pytest.raises(ProxyException): - await _guess(throttle, username=forged) + assert await _fail(throttle, username=forged) == "401" records: list[logging.LogRecord] = [] handler = logging.Handler() handler.emit = records.append verbose_proxy_logger.addHandler(handler) try: - with pytest.raises(ProxyException) as blocked: - await _guess(throttle, username=forged) + assert await _fail(throttle, username=forged) == "401" finally: verbose_proxy_logger.removeHandler(handler) - assert blocked.value.code == "429" - emitted = [r.getMessage() for r in records if "sign-in attempts exhausted" in r.getMessage()] - assert emitted, "the refusal must be logged" + emitted = [r.getMessage() for r in records if "Admin UI sign-in blocked" in r.getMessage()] + assert emitted, "installing the block must be logged" assert "\n" not in emitted[0] and "\x00" not in emitted[0] assert "victim@example.com" in emitted[0] @pytest.mark.asyncio -async def test_a_username_spray_cannot_evict_an_existing_counter(monkeypatch): - """Regression: the in-memory tier must hold more counters than a spray can create. - - The default in-memory cache keeps 200 entries and evicts the soonest to expire, and - every counter shares one window, so eviction was effectively oldest-first. A few - hundred made-up usernames therefore pushed out the attacker's own counter and handed - back a fresh allowance against the real account. Username and source counters must also - live in separate stores, or the same spray evicts the source counter meant to stop it. - """ - from litellm.proxy._types import ProxyException +async def test_a_username_spray_cannot_evict_an_active_block(monkeypatch): + """Counters and blocks live in separate bounded stores, so a flood of made-up pairs fills the counter + store while the blocks it already earned stay in force.""" + from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.auth.login_throttle import ( + _BLOCKS, + _COUNTERS, + _MAX_TRACKED_BLOCKS, + _MAX_TRACKED_COUNTERS, LoginThrottle, - _FAILED_LOGIN_SOURCE_CACHE, - _FAILED_LOGIN_USERNAME_CACHE, - _MAX_TRACKED_LOGIN_SOURCES, - _MAX_TRACKED_LOGIN_USERNAMES, ) monkeypatch.setenv("UI_USERNAME", "admin") monkeypatch.setenv("UI_PASSWORD", "right") - assert _MAX_TRACKED_LOGIN_SOURCES >= 10_000 - assert _MAX_TRACKED_LOGIN_USERNAMES >= 10_000 - assert _FAILED_LOGIN_SOURCE_CACHE.in_memory_cache.max_size_in_memory == _MAX_TRACKED_LOGIN_SOURCES - assert _FAILED_LOGIN_USERNAME_CACHE.in_memory_cache.max_size_in_memory == _MAX_TRACKED_LOGIN_USERNAMES - assert _FAILED_LOGIN_SOURCE_CACHE.in_memory_cache is not _FAILED_LOGIN_USERNAME_CACHE.in_memory_cache - + assert _MAX_TRACKED_COUNTERS >= 10_000 and _MAX_TRACKED_BLOCKS >= 10_000 + assert _COUNTERS is not _BLOCKS + counters, blocks = InMemoryCache(max_size_in_memory=50), InMemoryCache(max_size_in_memory=50) throttle = LoginThrottle( client_ip="10.9.9.9", - max_attempts=3, - max_attempts_per_source=10_000, - window_seconds=900, - username_cache=_FAILED_LOGIN_USERNAME_CACHE, - source_cache=_FAILED_LOGIN_SOURCE_CACHE, + source_limit=None, + user_limit=1, + window_seconds=60, + block_seconds=300, + counters=counters, + blocks=blocks, ) victim = "spray-victim@corp.com" - for _ in range(3): - with pytest.raises(ProxyException): - await _guess(throttle, username=victim) + assert [await _fail(throttle, username=victim) for _ in range(2)] == ["401", "401"] - for i in range(500): + for i in range(200): await throttle.record_failure(f"spray-filler-{i}@corp.com") - assert await throttle._failures(throttle.username_cache, throttle._username_key(victim)) == 3, "the counter must survive a spray" - with pytest.raises(ProxyException) as blocked: - await _guess(throttle, username=victim) - assert blocked.value.code == "429" + assert len(counters.cache_dict) <= 50, "the counter store is bounded" + assert counters.get_cache(throttle._keys(victim).pair_counter) is None, "the victim's counter was evicted" + assert await _fail(throttle, username=victim) == "429", "the block survived the spray" def _patch_sso_configured(stack: ExitStack, *, configured: bool) -> None: diff --git a/tests/test_litellm/proxy/proxy_server/conftest.py b/tests/test_litellm/proxy/proxy_server/conftest.py index 56aa87f1e50..a349d378985 100644 --- a/tests/test_litellm/proxy/proxy_server/conftest.py +++ b/tests/test_litellm/proxy/proxy_server/conftest.py @@ -517,38 +517,25 @@ def make_key( def reset_login_throttle(monkeypatch): """Clear the Admin UI failed-login counters between tests. - `client` is session scoped and the counters live in shared module stores with a 900s - window, so without this a failed sign-in test could return 429 in unrelated tests later. + `client` is session scoped and the counters live in shared module stores with a 300s block + window, so without this a failed sign-in test could block unrelated tests later. Only the throttle's own keys are removed, so other cache entries remain untouched. """ from litellm.proxy import proxy_server as ps from litellm.proxy.auth import login_throttle - from litellm.proxy.auth.login_throttle import ( - _CACHE_KEY_PREFIX, - _FAILED_LOGIN_SOURCE_CACHE, - _FAILED_LOGIN_USERNAME_CACHE, - ) + from litellm.proxy.auth.login_throttle import _BLOCKS, _CACHE_KEY_PREFIX, _COUNTERS async def _no_delay(_seconds: float) -> None: - """The escalating wait on a rejected sign-in, replaced so the route tests stay fast.""" + """The hold on a rejected sign-in from a blocked key, replaced so the route tests stay fast.""" monkeypatch.setattr(login_throttle, "_sleep", _no_delay) def _drop_throttle_keys() -> None: - login_throttle._DELAYS_IN_FLIGHT.clear() - for cache in (_FAILED_LOGIN_USERNAME_CACHE, _FAILED_LOGIN_SOURCE_CACHE): - in_memory = getattr(cache, "in_memory_cache", None) - if in_memory is None: - continue - tracked = tuple( - key - for store in (getattr(in_memory, "cache_dict", None), getattr(in_memory, "ttl_dict", None)) - if isinstance(store, dict) - for key in tuple(store) - if str(key).startswith(_CACHE_KEY_PREFIX) - ) - for key in tracked: - in_memory.delete_cache(key) + login_throttle._HELD_ATTEMPTS.clear() + for store in (_COUNTERS, _BLOCKS): + for key in tuple(store.cache_dict) + tuple(store.ttl_dict): + if key.startswith(_CACHE_KEY_PREFIX): + store.delete_cache(key) monkeypatch.setattr(ps, "redis_usage_cache", None) _drop_throttle_keys() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index f8382c64e6a..7399e64c421 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -10,11 +10,8 @@ Routes covered: from __future__ import annotations -from concurrent.futures import ThreadPoolExecutor from unittest.mock import AsyncMock, MagicMock -import pytest - from .conftest import normalize # --------------------------------------------------------------------------- @@ -495,15 +492,38 @@ def _install_real_auth(monkeypatch, **settings): def _form_login(client, username="admin", password="wrong"): - return client.post( - "/login", data={"username": username, "password": password}, follow_redirects=False - ).status_code + return client.post("/login", data={"username": username, "password": password}, follow_redirects=False).status_code def _json_login(client, path, username="admin", password="wrong"): return client.post(path, json={"username": username, "password": password}).status_code +def _db_user(monkeypatch, email: str): + """A database user with a stored hash, faked so the route reaches the known-user branch without Postgres.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server as ps + + user = MagicMock() + user.user_id = "u-1" + user.user_email = email + user.user_role = "internal_user" + user.password = "scrypt:stored" + repo = MagicMock() + repo.return_value.table.find_first = AsyncMock(return_value=user) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr("litellm.proxy.auth.login_utils.UserRepository", repo) + monkeypatch.setattr("litellm.proxy.auth.login_utils._rehash_password_if_needed", AsyncMock()) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.verify_password", lambda given, stored: given == "right-db-password" + ) + monkeypatch.setattr( + "litellm.proxy.auth.login_utils.generate_key_helper_fn", AsyncMock(return_value={"token": "sk-ui"}) + ) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + + def test_budget_is_shared_across_every_login_endpoint(client, monkeypatch, reset_login_throttle): """The endpoint is not part of the key, so spending the budget on one route blocks the rest. @@ -511,92 +531,117 @@ def test_budget_is_shared_across_every_login_endpoint(client, monkeypatch, reset """ _install_real_auth( monkeypatch, - max_failed_login_attempts=10, + max_failed_login_attempts_per_user=10, control_plane_url="https://cp.example.com", ) assert [_form_login(client) for _ in range(5)] == [401] * 5 assert [_json_login(client, "/v2/login") for _ in range(5)] == [401] * 5 - assert _json_login(client, "/v3/login") == 429, "the eleventh attempt must be refused on a third route" + assert _json_login(client, "/v3/login") == 401, "the eleventh failure crosses the limit and installs the block" + assert _json_login(client, "/v3/login") == 429, "the twelfth attempt must be refused on a third route" def test_budget_is_shared_across_username_casing(client, monkeypatch, reset_login_throttle): """The database lookup is case-insensitive, so casing must not partition the counter.""" - _install_real_auth(monkeypatch, max_failed_login_attempts=10) + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=3) - assert [_json_login(client, "/v2/login", username="admin@corp.com") for _ in range(5)] == [401] * 5 - assert [_json_login(client, "/v2/login", username="ADMIN@corp.com") for _ in range(5)] == [401] * 5 + assert [_json_login(client, "/v2/login", username="admin@corp.com") for _ in range(2)] == [401] * 2 + assert [_json_login(client, "/v2/login", username="ADMIN@corp.com") for _ in range(2)] == [401] * 2 assert _json_login(client, "/v2/login", username="Admin@corp.com") == 429 def test_a_refused_attempt_carries_retry_after(client, monkeypatch, reset_login_throttle): - """The 429 tells the caller how long the window has left.""" - _install_real_auth(monkeypatch, max_failed_login_attempts=2, failed_login_window_seconds=77) + """The 429 tells the caller how long the block has left, after the 30 seconds it was already held.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1, failed_login_block_seconds=77) assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401] refused = client.post("/v2/login", json={"username": "admin", "password": "wrong"}) assert refused.status_code == 429 - assert refused.headers.get("retry-after") == "77" + assert refused.headers.get("retry-after") == "47" def test_the_form_returns_a_human_readable_lockout_page(client, monkeypatch, reset_login_throttle): """The no-JavaScript form must render a wait page when its POST is throttled.""" - _install_real_auth(monkeypatch, max_failed_login_attempts=2, failed_login_window_seconds=77) + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1, failed_login_block_seconds=77) assert [_form_login(client) for _ in range(2)] == [401, 401] refused = client.post("/login", data={"username": "admin", "password": "wrong"}) assert refused.status_code == 429 assert refused.headers.get("content-type", "").startswith("text/html") - assert "Try again in about 77 seconds" in refused.text - assert refused.headers.get("retry-after") == "77" + assert "Try again in about 47 seconds" in refused.text + assert refused.headers.get("retry-after") == "47" def test_a_second_username_from_the_same_source_still_gets_through(client, monkeypatch, reset_login_throttle): - """The username counter carries no address, so one account exhausting it cannot block another.""" - _install_real_auth(monkeypatch, max_failed_login_attempts=2) + """The pair block is per username, so one account's block cannot take the office down with it.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1) - for _ in range(3): - _json_login(client, "/v2/login", username="admin") + assert [_json_login(client, "/v2/login", username="admin") for _ in range(3)] == [401, 401, 429] assert _json_login(client, "/v2/login", username="someone-else@example.com") == 401 -def test_a_spray_across_usernames_is_refused_on_the_source_counter(client, monkeypatch, reset_login_throttle): - """A fresh username per guess keeps every username counter at one, so the address is what stops it.""" - _install_real_auth(monkeypatch, max_failed_login_attempts=100, max_failed_login_attempts_per_source=4) +def test_a_spray_across_usernames_is_blocked_on_the_source_when_the_source_is_attributable( + client, monkeypatch, reset_login_throttle +): + """A fresh username per guess keeps every pair at one, so the address is what stops it.""" + _install_real_auth(monkeypatch, trusted_proxy_ranges=["10.0.0.0/8"], max_failed_login_attempts_per_source=4) - sprayed = [_json_login(client, "/v2/login", username=f"sprayed-{i}@corp.com") for i in range(4)] - assert sprayed == [401] * 4 + sprayed = [_json_login(client, "/v2/login", username=f"sprayed-{i}@corp.com") for i in range(5)] + assert sprayed == [401] * 5 - assert _json_login(client, "/v2/login", username="sprayed-5@corp.com") == 429 + assert _json_login(client, "/v2/login", username="sprayed-6@corp.com") == 429 -def test_the_configured_admin_password_still_signs_in_while_refused(client, monkeypatch, reset_login_throttle): +def test_a_spray_across_usernames_is_not_blocked_without_trusted_proxy_ranges( + client, monkeypatch, reset_login_throttle +): + """Without a configured proxy range the peer address is whoever fronts the proxy, shared by every + client, so a source-wide block would block them all and the source scope stays off.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_source=4) + + sprayed = [_json_login(client, "/v2/login", username=f"sprayed-{i}@corp.com") for i in range(8)] + assert sprayed == [401] * 8 + + +def test_the_configured_admin_password_still_signs_in_while_blocked(client, monkeypatch, reset_login_throttle): """The operator must never be locked out of the console by traffic aimed at it.""" from unittest.mock import AsyncMock, patch - _install_real_auth(monkeypatch, max_failed_login_attempts=2) + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1) monkeypatch.setenv("DATABASE_URL", "postgresql://stub") - assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401] - assert _json_login(client, "/v2/login") == 429 + assert [_json_login(client, "/v2/login") for _ in range(3)] == [401, 401, 429] - with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed - "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + with ( + patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), + patch( # test-quality-ok: success mints a UI key and persists the user; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), ): assert _json_login(client, "/v2/login", password="right-password") == 200 -def test_sign_in_succeeds_again_once_the_budget_is_restored(client, monkeypatch, reset_login_throttle): - """A cleared bucket lets the same username straight back in.""" - _install_real_auth(monkeypatch, max_failed_login_attempts=2) +def test_a_database_users_correct_password_signs_in_while_blocked(client, monkeypatch, reset_login_throttle): + """The block is soft: guessing at an account slows the guesser down, it does not lock the owner out.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1) + _db_user(monkeypatch, "user@corp.com") - assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401] - assert _json_login(client, "/v2/login") == 429 + assert [_json_login(client, "/v2/login", username="user@corp.com") for _ in range(3)] == [401, 401, 429] + + assert _json_login(client, "/v2/login", username="user@corp.com", password="right-db-password") == 200 + assert _json_login(client, "/v2/login", username="user@corp.com") == 429, "the block itself is still in force" + + +def test_sign_in_succeeds_again_once_the_block_is_cleared(client, monkeypatch, reset_login_throttle): + """A cleared store lets the same username straight back to a plain credential check.""" + _install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1) + + assert [_json_login(client, "/v2/login") for _ in range(3)] == [401, 401, 429] reset_login_throttle() assert _json_login(client, "/v2/login") == 401 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 2833686f115..07696f23da7 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3438,6 +3438,34 @@ async def test_load_config_warns_per_worker_login_counters_without_general_setti assert "Running 4 workers but Redis is not configured" in caplog.text +@pytest.mark.asyncio +async def test_load_config_warns_that_the_source_login_limit_is_off_without_trusted_proxy_ranges( + tmp_path, monkeypatch, caplog +): + """The per-source failed-login limit is skipped when the source cannot be attributed, and the + operator must be told so at startup; a configured range silences it.""" + import logging + + from litellm.proxy.auth.login_throttle import warn_source_login_limit_is_off + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setenv("NUM_WORKERS", "1") + warn_source_login_limit_is_off.cache_clear() + config_file = tmp_path / "config.yaml" + config_file.write_text("model_list: []\n") + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert "trusted_proxy_ranges is not set" in caplog.text + + caplog.clear() + warn_source_login_limit_is_off.cache_clear() + config_file.write_text("model_list: []\ngeneral_settings:\n trusted_proxy_ranges: ['10.0.0.0/8']\n") + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert "trusted_proxy_ranges is not set" not in caplog.text + + @pytest.mark.asyncio async def test_load_environment_variables_direct_and_os_environ(): """ @@ -13331,14 +13359,16 @@ async def test_login_throttle_settings_are_not_hot_applied_from_the_database(): ps.general_settings.clear() await ProxyConfig()._update_general_settings( db_general_settings={ - "max_failed_login_attempts": 999, + "max_failed_login_attempts_per_user": 999, "max_failed_login_attempts_per_source": 999, "failed_login_window_seconds": 1, + "failed_login_block_seconds": 1, } ) - assert "max_failed_login_attempts" not in ps.general_settings + assert "max_failed_login_attempts_per_user" not in ps.general_settings assert "max_failed_login_attempts_per_source" not in ps.general_settings assert "failed_login_window_seconds" not in ps.general_settings + assert "failed_login_block_seconds" not in ps.general_settings finally: ps.general_settings.clear() ps.general_settings.update(original) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 099ce397ffe..6203e79bccd 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26447,9 +26447,14 @@ export interface components { * @description If True, router fallbacks configured in router_settings are only attempted when the calling key (and its team and project) is allowed to call the fallback model; unauthorized fallback targets are skipped and the primary model's error is returned. Default is False. */ enforce_fallback_model_access?: boolean | null; + /** + * Failed Login Block Seconds + * @description How long a blocked source address, or source address and username, stays blocked. The block is soft: a correct password still signs in, but each attempt from a blocked key takes one of 5 held slots per worker and a wrong password is held for 30 seconds before it is refused with 429. Set under `general_settings` in config.yaml. Defaults to 300 + */ + failed_login_block_seconds?: number | null; /** * Failed Login Window Seconds - * @description Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Set under `general_settings` in config.yaml. Defaults to 900 + * @description Fixed window in seconds over which failed Admin UI sign-in attempts are counted. The window starts at the first failure and is not extended by later ones. Set under `general_settings` in config.yaml. Defaults to 60 */ failed_login_window_seconds?: number | null; /** @@ -26496,16 +26501,23 @@ export interface components { * @description max batch input file size in MB for /v1/files uploads with purpose=batch, if a file is larger than this size it will be rejected before being forwarded to the provider */ max_batch_file_size_mb?: number | null; - /** - * Max Failed Login Attempts - * @description Number of failed Admin UI sign-in attempts allowed for one username, from any source address, within `failed_login_window_seconds`, before further attempts for that username are refused with 429. Attempts are answered with a doubling delay well before this ceiling. Set under `general_settings` in config.yaml. Defaults to 50 - */ - max_failed_login_attempts?: number | null; /** * Max Failed Login Attempts Per Source - * @description Number of failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`, before further attempts from that address are refused with 429. Counted independently of `max_failed_login_attempts`. Set under `general_settings` in config.yaml. Defaults to 250 + * @description Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Only enforced when `trusted_proxy_ranges` is set, since otherwise every client behind an ingress shares one address. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10 */ max_failed_login_attempts_per_source?: number | null; + /** + * Max Failed Login Attempts Per Source Overrides + * @description Per-address overrides of `max_failed_login_attempts_per_source`, keyed by IP address or CIDR range, e.g. {'1.2.3.4': 200, '5.6.0.0/24': 500}. The most specific matching range wins. Set under `general_settings` in config.yaml + */ + max_failed_login_attempts_per_source_overrides?: { + [key: string]: number; + } | null; + /** + * Max Failed Login Attempts Per User + * @description Failed Admin UI sign-in attempts allowed from one source address for one username within `failed_login_window_seconds`. One more blocks that address for that username for `failed_login_block_seconds`, and its further failures stop counting against `max_failed_login_attempts_per_source`, so a script stuck on one account does not block everyone behind the same address. Set under `general_settings` in config.yaml. Defaults to 5 + */ + max_failed_login_attempts_per_user?: number | null; /** * Max File Size Mb * @description max file size in MB for /v1/files uploads, for any purpose, if a file is larger than this size it will be rejected before being forwarded to the provider