mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
feat(proxy): harden Admin UI login throttling
This commit is contained in:
Yucheng Zhu
parent
a39efb1a1d
commit
2d33c949cb
10 changed files with 581 additions and 120 deletions
|
|
@ -2528,7 +2528,12 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
max_failed_login_attempts: int | None = Field(
|
||||
None,
|
||||
ge=1,
|
||||
description="Number of failed Admin UI sign-in attempts allowed for one username from one source address within `failed_login_window_seconds`, before further attempts are refused with 429. Configurable from config.yaml only. Defaults to 10",
|
||||
description="Number of failed Admin UI sign-in attempts allowed for one username, from any source address, within `failed_login_window_seconds`, before further attempts for that username are refused with 429. Attempts are answered with a doubling delay well before this ceiling. Configurable from config.yaml only. Defaults to 50",
|
||||
)
|
||||
max_failed_login_attempts_per_source: int | None = Field(
|
||||
None,
|
||||
ge=1,
|
||||
description="Number of failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`, before further attempts from that address are refused with 429. Counted independently of `max_failed_login_attempts`. Configurable from config.yaml only. Defaults to 250",
|
||||
)
|
||||
failed_login_window_seconds: int | None = Field(
|
||||
None,
|
||||
|
|
|
|||
|
|
@ -1,16 +1,20 @@
|
|||
"""Failed-login accounting for the Admin UI sign-in path.
|
||||
|
||||
Counts failed credential checks per (username, source address) over a fixed window and
|
||||
denies further attempts with 429 once the count reaches the limit. Built per request by
|
||||
``LoginThrottle.from_request`` because it carries that request's resolved source address,
|
||||
and because the coordination cache is assigned at startup and can be reassigned later.
|
||||
Counts failed credential checks over a fixed window against two independent keys, the
|
||||
username on its own and the source address on its own, so that one username attacked from
|
||||
many sources and one source spraying many usernames are both counted. Repeated failures
|
||||
are answered slowly, doubling from one second, and refused with 429 once either counter
|
||||
reaches its limit. Built per request by ``LoginThrottle.from_request`` because it carries
|
||||
that request's resolved source address, and because the coordination cache is assigned at
|
||||
startup and can be reassigned later.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from collections.abc import Awaitable
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, NamedTuple, NoReturn
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
|
|
@ -23,21 +27,53 @@ from litellm.proxy.auth.network import TrustedProxyConfig, normalize_cidr_ranges
|
|||
from litellm.proxy.auth.trusted_proxy_utils import TRUSTED_PROXY_RANGES_KEY
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
|
||||
DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS: Final = 10
|
||||
DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS: Final = 50
|
||||
DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE: Final = 250
|
||||
DEFAULT_FAILED_LOGIN_WINDOW_SECONDS: Final = 900
|
||||
|
||||
USERNAME_DELAY_ONSET: Final = 3
|
||||
SOURCE_DELAY_ONSET: Final = 25
|
||||
FIRST_DELAY_SECONDS: Final = 1.0
|
||||
MAX_DELAY_SECONDS: Final = 30.0
|
||||
MAX_CONCURRENT_DELAYS_PER_SOURCE: Final = 5
|
||||
|
||||
_MAX_DELAY_DOUBLINGS: Final = 16
|
||||
|
||||
_CACHE_KEY_PREFIX: Final = "login_fail"
|
||||
_UNKNOWN_SOURCE: Final = "unknown"
|
||||
_MAX_LOGGED_USERNAME_CHARS: Final = 128
|
||||
|
||||
_MAX_TRACKED_LOGIN_USERNAMES: Final = 10_000
|
||||
_MAX_TRACKED_LOGIN_SOURCES: Final = 10_000
|
||||
|
||||
_FAILED_LOGIN_CACHE: Final = DualCache(
|
||||
in_memory_cache=InMemoryCache(max_size_in_memory=_MAX_TRACKED_LOGIN_SOURCES),
|
||||
default_in_memory_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS,
|
||||
)
|
||||
|
||||
def _bounded_store(max_entries: int) -> DualCache:
|
||||
return DualCache(
|
||||
in_memory_cache=InMemoryCache(max_size_in_memory=max_entries),
|
||||
default_in_memory_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
# Separate stores: eviction is earliest-expiring-first, so in one shared store a spray of
|
||||
# fresh usernames would evict the source counter that is meant to stop that same spray.
|
||||
_FAILED_LOGIN_USERNAME_CACHE: Final = _bounded_store(_MAX_TRACKED_LOGIN_USERNAMES)
|
||||
_FAILED_LOGIN_SOURCE_CACHE: Final = _bounded_store(_MAX_TRACKED_LOGIN_SOURCES)
|
||||
_NO_SETTINGS: Final = MappingProxyType({})
|
||||
|
||||
_DELAYS_IN_FLIGHT: Final[dict[str, int]] = {}
|
||||
|
||||
|
||||
async def _sleep(seconds: float) -> None:
|
||||
"""The wait a rejected sign-in is held for. Replaced in tests so the suite pays no wall clock."""
|
||||
await asyncio.sleep(seconds)
|
||||
|
||||
|
||||
class FailureCounts(NamedTuple):
|
||||
"""Failures recorded so far in this window against each of the two keys."""
|
||||
|
||||
username: int
|
||||
source: int
|
||||
|
||||
|
||||
def _int_setting(name: str, value: object, default: int, minimum: int) -> int:
|
||||
if value is None:
|
||||
|
|
@ -56,12 +92,14 @@ def _as_count(cached: object) -> int:
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoginThrottle:
|
||||
"""Fixed-window failed-login accounting for one request's source address."""
|
||||
"""Fixed-window failed-login accounting for one request's username and source address."""
|
||||
|
||||
client_ip: str
|
||||
max_attempts: int
|
||||
max_attempts_per_source: int
|
||||
window_seconds: int
|
||||
cache: DualCache
|
||||
username_cache: DualCache
|
||||
source_cache: DualCache
|
||||
redis_cache: RedisCache | None = None
|
||||
enabled: bool = True
|
||||
|
||||
|
|
@ -85,13 +123,20 @@ class LoginThrottle:
|
|||
DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS,
|
||||
1,
|
||||
),
|
||||
max_attempts_per_source=_int_setting(
|
||||
"max_failed_login_attempts_per_source",
|
||||
settings.get("max_failed_login_attempts_per_source"),
|
||||
DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_SOURCE,
|
||||
1,
|
||||
),
|
||||
window_seconds=_int_setting(
|
||||
"failed_login_window_seconds",
|
||||
settings.get("failed_login_window_seconds"),
|
||||
DEFAULT_FAILED_LOGIN_WINDOW_SECONDS,
|
||||
1,
|
||||
),
|
||||
cache=_FAILED_LOGIN_CACHE,
|
||||
username_cache=_FAILED_LOGIN_USERNAME_CACHE,
|
||||
source_cache=_FAILED_LOGIN_SOURCE_CACHE,
|
||||
redis_cache=redis_usage_cache,
|
||||
enabled=not get_secret_bool("LITELLM_DISABLE_LOGIN_RATE_LIMIT"),
|
||||
)
|
||||
|
|
@ -101,9 +146,13 @@ class LoginThrottle:
|
|||
"""The username with anything that could forge a log line removed."""
|
||||
return "".join(c for c in username if c.isprintable())[:_MAX_LOGGED_USERNAME_CHARS]
|
||||
|
||||
def _key(self, username: str) -> str:
|
||||
@staticmethod
|
||||
def _username_key(username: str) -> str:
|
||||
identity: Final = hashlib.sha256(username.casefold().encode("utf-8")).hexdigest()
|
||||
return f"{_CACHE_KEY_PREFIX}:{identity}:{self.client_ip}"
|
||||
return f"{_CACHE_KEY_PREFIX}:user:{identity}"
|
||||
|
||||
def _source_key(self) -> str:
|
||||
return f"{_CACHE_KEY_PREFIX}:source:{self.client_ip}"
|
||||
|
||||
async def _outcome(self, work: Awaitable[object]) -> object:
|
||||
try:
|
||||
|
|
@ -112,66 +161,147 @@ class LoginThrottle:
|
|||
verbose_proxy_logger.warning("login attempt accounting unavailable: %s", exc)
|
||||
return None
|
||||
|
||||
async def _failures(self, key: str) -> int:
|
||||
store: Final = self.cache if self.redis_cache is None else self.redis_cache
|
||||
return _as_count(await self._outcome(store.async_get_cache(key=key)))
|
||||
async def _failures(self, store: DualCache, key: str) -> int:
|
||||
"""The larger of the shared and the process-local count, so a Redis outage degrades
|
||||
to per-worker accounting instead of switching the control off."""
|
||||
local: Final = _as_count(await self._outcome(store.async_get_cache(key=key)))
|
||||
redis_cache: Final = self.redis_cache
|
||||
if redis_cache is None:
|
||||
return local
|
||||
return max(local, _as_count(await self._outcome(redis_cache.async_get_cache(key))))
|
||||
|
||||
async def _ensure_expiry(self, key: str) -> None:
|
||||
"""Give the counter an expiry if it somehow has none.
|
||||
async def _remaining_window(self, key: str) -> int:
|
||||
"""Seconds until this counter expires, repairing a counter left without an expiry.
|
||||
|
||||
Redis commits the increment before setting the TTL, so a failure in between can
|
||||
leave a counter that never expires. Nothing increments the key again once the
|
||||
limit is reached, so without this the pair would stay refused indefinitely.
|
||||
limit is reached, so without the repair the key would stay refused indefinitely.
|
||||
"""
|
||||
redis_cache: Final = self.redis_cache
|
||||
if redis_cache is None:
|
||||
return
|
||||
if isinstance(await self._outcome(redis_cache.async_get_ttl(key)), int):
|
||||
return
|
||||
return self.window_seconds
|
||||
ttl: Final = await self._outcome(redis_cache.async_get_ttl(key))
|
||||
if isinstance(ttl, int) and ttl > 0:
|
||||
return min(ttl, self.window_seconds)
|
||||
await self._outcome(redis_cache.async_increment(key, 0, ttl=self.window_seconds))
|
||||
return self.window_seconds
|
||||
|
||||
async def raise_if_blocked(self, username: str) -> None:
|
||||
"""Deny before the database lookup and before the password comparison."""
|
||||
if not self.enabled:
|
||||
return
|
||||
key: Final = self._key(username)
|
||||
if await self._failures(key) < self.max_attempts:
|
||||
return
|
||||
await self._ensure_expiry(key)
|
||||
verbose_proxy_logger.warning(
|
||||
"Admin UI sign-in attempts exhausted for username=%s source=%s; %s attempts in %ss, retry after %ss",
|
||||
self._loggable(username),
|
||||
self.client_ip,
|
||||
self.max_attempts,
|
||||
self.window_seconds,
|
||||
self.window_seconds,
|
||||
)
|
||||
raise ProxyException(
|
||||
def _refused(self, retry_after: int, param: str) -> ProxyException:
|
||||
return ProxyException(
|
||||
message="Too many failed sign-in attempts. Try again later.",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="max_failed_login_attempts",
|
||||
param=param,
|
||||
code=429,
|
||||
headers={"Retry-After": str(self.window_seconds)}, # mutable-ok: ProxyException coerces header values
|
||||
headers={"Retry-After": str(retry_after)}, # mutable-ok: ProxyException coerces header values
|
||||
)
|
||||
|
||||
async def record_failure(self, username: str) -> None:
|
||||
"""Count one rejected credential guess against this username and source."""
|
||||
async def _refuse(self, key: str, scope: str, param: str, username: str, failures: int, limit: int) -> NoReturn:
|
||||
retry_after: Final = await self._remaining_window(key)
|
||||
verbose_proxy_logger.warning(
|
||||
"Admin UI sign-in attempts exhausted for %s; username=%s source=%s failures=%s limit=%s window=%ss",
|
||||
scope,
|
||||
self._loggable(username),
|
||||
self.client_ip,
|
||||
failures,
|
||||
limit,
|
||||
self.window_seconds,
|
||||
)
|
||||
raise self._refused(retry_after, param)
|
||||
|
||||
async def raise_if_blocked(self, username: str) -> None:
|
||||
"""Refuse before the database lookup and before the invite-link password hash."""
|
||||
if not self.enabled:
|
||||
return
|
||||
key: Final = self._key(username)
|
||||
username_key: Final = self._username_key(username)
|
||||
source_key: Final = self._source_key()
|
||||
username_failures: Final = await self._failures(self.username_cache, username_key)
|
||||
if username_failures >= self.max_attempts:
|
||||
await self._refuse(
|
||||
username_key, "username", "max_failed_login_attempts", username, username_failures, self.max_attempts
|
||||
)
|
||||
source_failures: Final = await self._failures(self.source_cache, source_key)
|
||||
if source_failures >= self.max_attempts_per_source:
|
||||
await self._refuse(
|
||||
source_key,
|
||||
"source address",
|
||||
"max_failed_login_attempts_per_source",
|
||||
username,
|
||||
source_failures,
|
||||
self.max_attempts_per_source,
|
||||
)
|
||||
|
||||
async def _bump(self, store: DualCache, key: str) -> int:
|
||||
local: Final = _as_count(
|
||||
await self._outcome(store.async_increment_cache(key=key, value=1, ttl=self.window_seconds))
|
||||
)
|
||||
redis_cache: Final = self.redis_cache
|
||||
if redis_cache is not None:
|
||||
await self._outcome(redis_cache.async_increment(key, 1, ttl=self.window_seconds))
|
||||
await self._ensure_expiry(key)
|
||||
if redis_cache is None:
|
||||
return local
|
||||
shared: Final = _as_count(await self._outcome(redis_cache.async_increment(key, 1, ttl=self.window_seconds)))
|
||||
await self._remaining_window(key)
|
||||
return max(local, shared)
|
||||
|
||||
async def record_failure(self, username: str) -> FailureCounts:
|
||||
"""Count one rejected credential guess against this username and against this source."""
|
||||
if not self.enabled:
|
||||
return FailureCounts(username=0, source=0)
|
||||
return FailureCounts(
|
||||
username=await self._bump(self.username_cache, self._username_key(username)),
|
||||
source=await self._bump(self.source_cache, self._source_key()),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def delay_seconds(counts: FailureCounts) -> float:
|
||||
"""Seconds to hold a rejected attempt for, doubling per failure past whichever onset is further along."""
|
||||
steps: Final = min(
|
||||
max(counts.username - USERNAME_DELAY_ONSET, counts.source - SOURCE_DELAY_ONSET),
|
||||
_MAX_DELAY_DOUBLINGS,
|
||||
)
|
||||
if steps < 0:
|
||||
return 0.0
|
||||
return min(FIRST_DELAY_SECONDS * float(2**steps), MAX_DELAY_SECONDS)
|
||||
|
||||
async def delay_for(self, username: str, counts: FailureCounts) -> None:
|
||||
"""Hold this rejected attempt open before answering it, so guessing costs wall-clock time.
|
||||
|
||||
Only ever reached once the credentials are known to be wrong, so a valid password is
|
||||
never delayed. Sources are capped at ``MAX_CONCURRENT_DELAYS_PER_SOURCE`` held
|
||||
connections; over that, the attempt is refused immediately instead of parking a socket.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return
|
||||
await self._outcome(self.cache.async_increment_cache(key=key, value=1, ttl=self.window_seconds))
|
||||
delay: Final = self.delay_seconds(counts)
|
||||
if delay <= 0:
|
||||
return
|
||||
in_flight: Final = _DELAYS_IN_FLIGHT.get(self.client_ip, 0)
|
||||
if in_flight >= MAX_CONCURRENT_DELAYS_PER_SOURCE:
|
||||
verbose_proxy_logger.warning(
|
||||
"Admin UI sign-in attempts held concurrently exhausted; username=%s source=%s in_flight=%s",
|
||||
self._loggable(username),
|
||||
self.client_ip,
|
||||
in_flight,
|
||||
)
|
||||
raise self._refused(int(MAX_DELAY_SECONDS), "concurrent_failed_logins")
|
||||
_DELAYS_IN_FLIGHT[self.client_ip] = in_flight + 1
|
||||
try:
|
||||
await _sleep(delay)
|
||||
finally:
|
||||
remaining: Final = _DELAYS_IN_FLIGHT.get(self.client_ip, 1) - 1
|
||||
if remaining > 0:
|
||||
_DELAYS_IN_FLIGHT[self.client_ip] = remaining
|
||||
else:
|
||||
_DELAYS_IN_FLIGHT.pop(self.client_ip, None)
|
||||
|
||||
async def clear(self, username: str) -> None:
|
||||
"""Drop the bucket after a successful sign-in."""
|
||||
"""Drop the username counter after a successful sign-in.
|
||||
|
||||
The source counter is left alone. It is shared by every account behind that address,
|
||||
so one success there says nothing about the other attempts it is counting.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return
|
||||
key: Final = self._key(username)
|
||||
key: Final = self._username_key(username)
|
||||
redis_cache: Final = self.redis_cache
|
||||
if redis_cache is not None:
|
||||
await self._outcome(redis_cache.async_delete_cache(key))
|
||||
await self._outcome(self.cache.async_delete_cache(key=key))
|
||||
await self._outcome(self.username_cache.async_delete_cache(key=key))
|
||||
|
|
|
|||
|
|
@ -145,10 +145,15 @@ async def authenticate_user(
|
|||
code=500,
|
||||
)
|
||||
|
||||
await throttle.raise_if_blocked(username)
|
||||
|
||||
ui_username, ui_password = get_ui_credentials(master_key)
|
||||
|
||||
admin_credentials_match: Final = secrets.compare_digest(
|
||||
username.encode("utf-8"), ui_username.encode("utf-8")
|
||||
) and secrets.compare_digest(password.encode("utf-8"), ui_password.encode("utf-8"))
|
||||
|
||||
if not admin_credentials_match:
|
||||
await throttle.raise_if_blocked(username)
|
||||
|
||||
# Check if we can find the `username` in the db. On the UI, users can enter username=their email
|
||||
_user_row: LiteLLM_UserTable | None = None
|
||||
user_role: (
|
||||
|
|
@ -174,9 +179,7 @@ async def authenticate_user(
|
|||
- Login with UI_USERNAME and UI_PASSWORD
|
||||
- Login with Invite Link `user_email` and `password` combination
|
||||
"""
|
||||
if secrets.compare_digest(username.encode("utf-8"), ui_username.encode("utf-8")) and secrets.compare_digest(
|
||||
password.encode("utf-8"), ui_password.encode("utf-8")
|
||||
):
|
||||
if admin_credentials_match:
|
||||
# Non SSO -> If user is using UI_USERNAME and UI_PASSWORD they are Proxy admin
|
||||
user_role = LitellmUserRoles.PROXY_ADMIN
|
||||
user_id = LITELLM_PROXY_ADMIN_NAME
|
||||
|
|
@ -314,7 +317,7 @@ async def authenticate_user(
|
|||
login_method="username_password",
|
||||
)
|
||||
else:
|
||||
await throttle.record_failure(username)
|
||||
await throttle.delay_for(username, await throttle.record_failure(username))
|
||||
raise ProxyException(
|
||||
message=INVALID_UI_CREDENTIALS_MESSAGE,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
|
|
@ -322,7 +325,7 @@ async def authenticate_user(
|
|||
code=401,
|
||||
)
|
||||
else:
|
||||
await throttle.record_failure(username)
|
||||
await throttle.delay_for(username, await throttle.record_failure(username))
|
||||
raise ProxyException(
|
||||
message=INVALID_UI_CREDENTIALS_MESSAGE,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
|
|
|
|||
|
|
@ -1358,6 +1358,10 @@ def run_server(
|
|||
# DO NOT DELETE - enables global variables to work across files
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
# Write the resolved --num_workers back to its env var so worker processes can read
|
||||
# the fleet size at startup (the failed-login accounting warning keys off it)
|
||||
os.environ["NUM_WORKERS"] = str(num_workers)
|
||||
|
||||
# Auto-create PROMETHEUS_MULTIPROC_DIR for multi-worker setups
|
||||
ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir(
|
||||
num_workers=num_workers,
|
||||
|
|
|
|||
|
|
@ -2264,6 +2264,7 @@ user_custom_key_generate = None
|
|||
# Tests that need to reset it can patch 'litellm.proxy.proxy_server._pkce_no_redis_warning_emitted'.
|
||||
_pkce_no_redis_warning_emitted: bool = False
|
||||
_cp_no_redis_warning_emitted: bool = False
|
||||
_login_throttle_no_redis_warning_emitted: bool = False
|
||||
user_custom_key_update = None
|
||||
user_custom_sso = None
|
||||
user_custom_ui_sso_sign_in_handler = None
|
||||
|
|
@ -5317,6 +5318,21 @@ class ProxyConfig:
|
|||
"or ensure sticky sessions for single-instance deployments."
|
||||
)
|
||||
|
||||
### FAILED-LOGIN ACCOUNTING MULTI-INSTANCE PREREQUISITE CHECK ###
|
||||
# Failed Admin UI sign-in counters live in redis_usage_cache when available so a
|
||||
# brute-force run is counted once across workers instead of once per worker.
|
||||
if os.getenv("NUM_WORKERS", "1") != "1" and redis_usage_cache is None:
|
||||
global _login_throttle_no_redis_warning_emitted
|
||||
if not _login_throttle_no_redis_warning_emitted:
|
||||
_login_throttle_no_redis_warning_emitted = True
|
||||
verbose_proxy_logger.warning(
|
||||
"Running %s workers but Redis is not configured for LiteLLM caching. "
|
||||
"Failed Admin UI sign-in attempts are counted per worker, so an attacker "
|
||||
"gets max_failed_login_attempts guesses per worker instead of overall. "
|
||||
"Configure Redis via the 'cache' section in your proxy config.",
|
||||
os.getenv("NUM_WORKERS", "1"),
|
||||
)
|
||||
|
||||
### STORE MODEL IN DB ### feature flag for `/model/new`
|
||||
store_model_in_db = general_settings.get("store_model_in_db", False)
|
||||
if store_model_in_db is None:
|
||||
|
|
@ -14756,13 +14772,26 @@ async def login(request: Request):
|
|||
password: Final = str(form.get("password"))
|
||||
|
||||
# Authenticate user and get login result
|
||||
login_result: Final = await authenticate_user(
|
||||
username=username,
|
||||
password=password,
|
||||
master_key=master_key,
|
||||
prisma_client=prisma_client,
|
||||
throttle=LoginThrottle.from_request(request),
|
||||
)
|
||||
try:
|
||||
login_result: Final = await authenticate_user(
|
||||
username=username,
|
||||
password=password,
|
||||
master_key=master_key,
|
||||
prisma_client=prisma_client,
|
||||
throttle=LoginThrottle.from_request(request),
|
||||
)
|
||||
except ProxyException as exc:
|
||||
if int(exc.code) != status.HTTP_429_TOO_MANY_REQUESTS:
|
||||
raise
|
||||
retry_after: Final = exc.headers.get("Retry-After", "30")
|
||||
return HTMLResponse(
|
||||
content=(
|
||||
"<html><body><h1>Too many sign-in attempts</h1>"
|
||||
f"<p>Try again in about {retry_after} seconds</p></body></html>"
|
||||
),
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
headers=exc.headers,
|
||||
)
|
||||
|
||||
# Create UI token object
|
||||
returned_ui_token_object: Final = create_ui_token_object(
|
||||
|
|
|
|||
|
|
@ -6,17 +6,48 @@ to login_utils.py for better reusability.
|
|||
"""
|
||||
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class _RecordedSleeps:
|
||||
"""A sleep that records what it was asked to wait for instead of waiting."""
|
||||
|
||||
def __init__(self):
|
||||
self.seconds: list[float] = []
|
||||
|
||||
async def __call__(self, seconds: float) -> None:
|
||||
self.seconds.append(seconds)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def login_delays(monkeypatch):
|
||||
"""Replace the failed-login wait, so the suite pays no wall clock and can read it back."""
|
||||
from litellm.proxy.auth import login_throttle
|
||||
|
||||
recorded = _RecordedSleeps()
|
||||
monkeypatch.setattr(login_throttle, "_sleep", recorded)
|
||||
login_throttle._DELAYS_IN_FLIGHT.clear()
|
||||
yield recorded
|
||||
login_throttle._DELAYS_IN_FLIGHT.clear()
|
||||
|
||||
|
||||
def _unlimited_throttle():
|
||||
"""A throttle wired to a real in-memory store with a limit no test can reach."""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy.auth.login_throttle import LoginThrottle
|
||||
|
||||
return LoginThrottle(client_ip="1.2.3.4", max_attempts=10_000, window_seconds=900, cache=DualCache())
|
||||
store: Final = DualCache()
|
||||
return LoginThrottle(
|
||||
client_ip="1.2.3.4",
|
||||
max_attempts=10_000,
|
||||
max_attempts_per_source=10_000,
|
||||
window_seconds=900,
|
||||
username_cache=store,
|
||||
source_cache=store,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
|
@ -628,16 +659,26 @@ class TestEncodeUiSessionJwt:
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _throttle(max_attempts: int = 3, window_seconds: int = 900, client_ip: str = "1.2.3.4", cache=None, redis_cache=None):
|
||||
def _throttle(
|
||||
max_attempts: int = 3,
|
||||
window_seconds: int = 900,
|
||||
client_ip: str = "1.2.3.4",
|
||||
cache=None,
|
||||
redis_cache=None,
|
||||
max_attempts_per_source: int = 10_000,
|
||||
):
|
||||
"""A throttle over a real in-memory store, so the tests exercise the true counters."""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy.auth.login_throttle import LoginThrottle
|
||||
|
||||
store: Final = cache if cache is not None else DualCache()
|
||||
return LoginThrottle(
|
||||
client_ip=client_ip,
|
||||
max_attempts=max_attempts,
|
||||
max_attempts_per_source=max_attempts_per_source,
|
||||
window_seconds=window_seconds,
|
||||
cache=cache if cache is not None else DualCache(),
|
||||
username_cache=store,
|
||||
source_cache=store,
|
||||
redis_cache=redis_cache,
|
||||
)
|
||||
|
||||
|
|
@ -675,21 +716,32 @@ async def test_attempts_are_refused_once_the_limit_is_reached(monkeypatch):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_correct_password_is_refused_while_blocked(monkeypatch):
|
||||
"""The check precedes the credential comparison, so being over the limit wins."""
|
||||
async def test_a_correct_admin_password_is_accepted_while_blocked(monkeypatch):
|
||||
"""The configured admin credentials are compared before the gate, so the operator gets in.
|
||||
|
||||
A throttle that refuses a valid password hands anyone who can reach the login form a
|
||||
denial of service against the one account that can fix it.
|
||||
"""
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
|
||||
throttle = _throttle(max_attempts=2)
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(throttle)
|
||||
|
||||
with pytest.raises(ProxyException) as blocked:
|
||||
await _guess(throttle, password="right")
|
||||
assert blocked.value.code == "429"
|
||||
with pytest.raises(ProxyException) as still_blocked:
|
||||
await _guess(throttle)
|
||||
assert still_blocked.value.code == "429", "a wrong password is still refused"
|
||||
|
||||
with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch(
|
||||
"litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
|
||||
):
|
||||
result = await _guess(throttle, password="right")
|
||||
assert result.key == "sk-ui"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -700,18 +752,18 @@ async def test_a_blocked_attempt_does_not_extend_the_window(monkeypatch):
|
|||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
throttle = _throttle(max_attempts=2)
|
||||
key = throttle._key("admin")
|
||||
key = throttle._username_key("admin")
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(throttle)
|
||||
counted_at_limit = await throttle._failures(key)
|
||||
counted_at_limit = await throttle._failures(throttle.username_cache, key)
|
||||
|
||||
for _ in range(5):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(throttle)
|
||||
|
||||
assert await throttle._failures(key) == counted_at_limit == 2
|
||||
assert await throttle._failures(throttle.username_cache, key) == counted_at_limit == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -733,7 +785,7 @@ async def test_a_successful_sign_in_clears_the_bucket(monkeypatch):
|
|||
):
|
||||
await _guess(throttle, password="right")
|
||||
|
||||
assert await throttle._failures(throttle._key("admin")) == 0
|
||||
assert await throttle._failures(throttle.username_cache, throttle._username_key("admin")) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -749,7 +801,7 @@ async def test_a_configuration_error_never_counts(monkeypatch):
|
|||
)
|
||||
assert exc.value.code == "500"
|
||||
|
||||
assert await throttle._failures(throttle._key("admin")) == 0
|
||||
assert await throttle._failures(throttle.username_cache, throttle._username_key("admin")) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -773,7 +825,7 @@ async def test_the_username_is_case_folded_into_one_bucket(monkeypatch):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_different_username_from_the_same_source_is_unaffected(monkeypatch):
|
||||
"""The bucket is the pair, so one username's failures do not block another."""
|
||||
"""The counters are independent, so one username's failures do not exhaust another's."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
|
|
@ -853,7 +905,7 @@ async def test_a_user_with_no_password_set_does_not_consume_the_budget(monkeypat
|
|||
)
|
||||
assert exc.value.code == "401"
|
||||
|
||||
assert await throttle._failures(throttle._key("nopass@example.com")) == 0
|
||||
assert await throttle._failures(throttle.username_cache, throttle._username_key("nopass@example.com")) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -896,11 +948,40 @@ async def test_a_wrong_password_for_a_known_user_also_counts(monkeypatch):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_source_addresses_do_not_share_a_bucket(monkeypatch):
|
||||
"""The key is the pair, so one address exhausting its budget must not block another.
|
||||
async def test_one_source_exhausting_its_own_budget_does_not_refuse_another_source(monkeypatch):
|
||||
"""The source counter is per address, so a noisy office does not take its neighbour down."""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
Dropping the address from the key would turn this into the username-only counter the
|
||||
design rejects, where anyone can lock a named admin out from anywhere.
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
shared_store = DualCache()
|
||||
attacker = _throttle(
|
||||
max_attempts=10_000, max_attempts_per_source=2, client_ip="203.0.113.9", cache=shared_store
|
||||
)
|
||||
operator = _throttle(
|
||||
max_attempts=10_000, max_attempts_per_source=2, client_ip="198.51.100.7", cache=shared_store
|
||||
)
|
||||
|
||||
for i in range(2):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(attacker, username=f"target-{i}@corp.com")
|
||||
|
||||
with pytest.raises(ProxyException) as blocked:
|
||||
await _guess(attacker, username="target-2@corp.com")
|
||||
assert blocked.value.code == "429"
|
||||
|
||||
with pytest.raises(ProxyException) as unaffected:
|
||||
await _guess(operator, username="target-3@corp.com")
|
||||
assert unaffected.value.code == "401", "the other address must still reach the credential check"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_username_exhausted_from_one_source_is_refused_from_another(monkeypatch):
|
||||
"""The username counter carries no address, so spreading the guesses buys nothing.
|
||||
|
||||
The pair key this replaced reset the budget for every new address, which is exactly the
|
||||
shape of a credential-stuffing run from a proxy pool.
|
||||
"""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
|
@ -908,20 +989,154 @@ async def test_two_source_addresses_do_not_share_a_bucket(monkeypatch):
|
|||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
shared_store = DualCache()
|
||||
attacker = _throttle(max_attempts=2, client_ip="203.0.113.9", cache=shared_store)
|
||||
operator = _throttle(max_attempts=2, client_ip="198.51.100.7", cache=shared_store)
|
||||
first_hop = _throttle(max_attempts=2, client_ip="203.0.113.9", cache=shared_store)
|
||||
second_hop = _throttle(max_attempts=2, client_ip="198.51.100.7", cache=shared_store)
|
||||
|
||||
for _ in range(3):
|
||||
for _ in range(2):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(attacker, username="admin")
|
||||
await _guess(first_hop, username="victim@corp.com")
|
||||
|
||||
with pytest.raises(ProxyException) as rotated:
|
||||
await _guess(second_hop, username="victim@corp.com")
|
||||
assert rotated.value.code == "429"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_source_wide_spray_is_counted_even_though_each_username_is_fresh(monkeypatch):
|
||||
"""One guess against each of many usernames never trips a username counter, only the source one."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
throttle = _throttle(max_attempts=10_000, max_attempts_per_source=6, client_ip="203.0.113.11")
|
||||
|
||||
for i in range(6):
|
||||
with pytest.raises(ProxyException) as rejected:
|
||||
await _guess(throttle, username=f"sprayed-{i}@corp.com")
|
||||
assert rejected.value.code == "401"
|
||||
|
||||
with pytest.raises(ProxyException) as blocked:
|
||||
await _guess(attacker, username="admin")
|
||||
await _guess(throttle, username="sprayed-7@corp.com")
|
||||
assert blocked.value.code == "429"
|
||||
assert await throttle._failures(throttle.username_cache, throttle._username_key("sprayed-7@corp.com")) == 0
|
||||
|
||||
with pytest.raises(ProxyException) as unaffected:
|
||||
await _guess(operator, username="admin")
|
||||
assert unaffected.value.code == "401", "the real operator must still reach the credential check"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_successful_sign_in_leaves_the_source_counter_alone(monkeypatch):
|
||||
"""One account's success says nothing about the other attempts the address is making."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
|
||||
throttle = _throttle(max_attempts=10)
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(throttle)
|
||||
|
||||
with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch(
|
||||
"litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
|
||||
):
|
||||
await _guess(throttle, password="right")
|
||||
|
||||
assert await throttle._failures(throttle.username_cache, throttle._username_key("admin")) == 0
|
||||
assert await throttle._failures(throttle.source_cache, throttle._source_key()) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_delay_doubles_from_one_second_and_is_capped(monkeypatch, login_delays):
|
||||
"""Guessing has to cost wall clock, and the cost has to stop short of an unbounded hang."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.login_throttle import MAX_DELAY_SECONDS
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
throttle = _throttle(max_attempts=10_000)
|
||||
|
||||
for _ in range(9):
|
||||
with pytest.raises(ProxyException):
|
||||
await _guess(throttle)
|
||||
|
||||
assert login_delays.seconds == [1.0, 2.0, 4.0, 8.0, 16.0, 30.0, 30.0], (
|
||||
"the first two failures answer immediately, then the wait doubles up to the cap"
|
||||
)
|
||||
assert max(login_delays.seconds) == MAX_DELAY_SECONDS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_delay_tracks_whichever_counter_is_further_past_its_onset(monkeypatch):
|
||||
"""A source deep into a spray must not be answered instantly just because the username is fresh."""
|
||||
from litellm.proxy.auth.login_throttle import FailureCounts, LoginThrottle
|
||||
|
||||
assert LoginThrottle.delay_seconds(FailureCounts(username=1, source=1)) == 0.0
|
||||
assert LoginThrottle.delay_seconds(FailureCounts(username=2, source=24)) == 0.0
|
||||
assert LoginThrottle.delay_seconds(FailureCounts(username=3, source=1)) == 1.0
|
||||
assert LoginThrottle.delay_seconds(FailureCounts(username=1, source=25)) == 1.0
|
||||
assert LoginThrottle.delay_seconds(FailureCounts(username=4, source=28)) == 8.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_held_attempts_from_one_source_are_capped(monkeypatch):
|
||||
"""Holding a rejected attempt open must not let one address park unlimited sockets."""
|
||||
import asyncio
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth import login_throttle as lt
|
||||
from litellm.proxy.auth.login_throttle import MAX_CONCURRENT_DELAYS_PER_SOURCE
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
release = asyncio.Event()
|
||||
|
||||
async def _park(_seconds: float) -> None:
|
||||
await release.wait()
|
||||
|
||||
monkeypatch.setattr(lt, "_sleep", _park)
|
||||
throttle = _throttle(max_attempts=10_000, client_ip="203.0.113.44")
|
||||
await throttle.record_failure("admin")
|
||||
await throttle.record_failure("admin")
|
||||
|
||||
held = [asyncio.create_task(_guess(throttle)) for _ in range(MAX_CONCURRENT_DELAYS_PER_SOURCE)]
|
||||
for _ in range(1000):
|
||||
if lt._DELAYS_IN_FLIGHT.get("203.0.113.44") == MAX_CONCURRENT_DELAYS_PER_SOURCE:
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
assert lt._DELAYS_IN_FLIGHT.get("203.0.113.44") == MAX_CONCURRENT_DELAYS_PER_SOURCE
|
||||
|
||||
try:
|
||||
with pytest.raises(ProxyException) as over_cap:
|
||||
await _guess(throttle)
|
||||
assert over_cap.value.code == "429"
|
||||
assert over_cap.value.headers.get("Retry-After") == "30"
|
||||
finally:
|
||||
release.set()
|
||||
for task in held:
|
||||
with pytest.raises(ProxyException):
|
||||
await task
|
||||
|
||||
with pytest.raises(ProxyException) as after_drain:
|
||||
await _guess(throttle)
|
||||
assert after_drain.value.code == "401", "the cap must release once the held attempts answer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabling_the_control_removes_the_delay_as_well(monkeypatch, login_delays):
|
||||
"""The escape hatch has to turn off the whole control, not only the refusal."""
|
||||
import dataclasses
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
throttle = dataclasses.replace(_throttle(max_attempts=2), enabled=False)
|
||||
|
||||
for _ in range(6):
|
||||
with pytest.raises(ProxyException) as rejected:
|
||||
await _guess(throttle)
|
||||
assert rejected.value.code == "401"
|
||||
|
||||
assert login_delays.seconds == []
|
||||
|
||||
|
||||
class _NoExpiryRedis:
|
||||
|
|
@ -1003,7 +1218,7 @@ async def test_counters_do_not_share_the_key_authentication_cache(monkeypatch):
|
|||
throttle = LoginThrottle.from_request(request)
|
||||
|
||||
for i in range(25):
|
||||
with pytest.raises(Exception):
|
||||
with pytest.raises(ProxyException, match="Invalid credentials"):
|
||||
await _guess(throttle, username=f"made-up-{i}@example.com")
|
||||
|
||||
added = set(ps.user_api_key_cache.in_memory_cache.cache_dict) - auth_cache_keys_before
|
||||
|
|
@ -1055,14 +1270,29 @@ async def test_a_username_spray_cannot_evict_an_existing_counter(monkeypatch):
|
|||
back a fresh allowance against the real account.
|
||||
"""
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.login_throttle import _FAILED_LOGIN_CACHE, _MAX_TRACKED_LOGIN_SOURCES
|
||||
from litellm.proxy.auth.login_throttle import (
|
||||
LoginThrottle,
|
||||
_FAILED_LOGIN_SOURCE_CACHE,
|
||||
_FAILED_LOGIN_USERNAME_CACHE,
|
||||
_MAX_TRACKED_LOGIN_SOURCES,
|
||||
_MAX_TRACKED_LOGIN_USERNAMES,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("UI_USERNAME", "admin")
|
||||
monkeypatch.setenv("UI_PASSWORD", "right")
|
||||
assert _MAX_TRACKED_LOGIN_SOURCES >= 10_000
|
||||
assert _FAILED_LOGIN_CACHE.in_memory_cache.max_size_in_memory == _MAX_TRACKED_LOGIN_SOURCES
|
||||
assert _MAX_TRACKED_LOGIN_USERNAMES >= 10_000
|
||||
assert _FAILED_LOGIN_SOURCE_CACHE.in_memory_cache.max_size_in_memory == _MAX_TRACKED_LOGIN_SOURCES
|
||||
assert _FAILED_LOGIN_USERNAME_CACHE.in_memory_cache.max_size_in_memory == _MAX_TRACKED_LOGIN_USERNAMES
|
||||
|
||||
throttle = _throttle(max_attempts=3, cache=_FAILED_LOGIN_CACHE, client_ip="10.9.9.9")
|
||||
throttle = LoginThrottle(
|
||||
client_ip="10.9.9.9",
|
||||
max_attempts=3,
|
||||
max_attempts_per_source=10_000,
|
||||
window_seconds=900,
|
||||
username_cache=_FAILED_LOGIN_USERNAME_CACHE,
|
||||
source_cache=_FAILED_LOGIN_SOURCE_CACHE,
|
||||
)
|
||||
victim = "spray-victim@corp.com"
|
||||
for _ in range(3):
|
||||
with pytest.raises(ProxyException):
|
||||
|
|
@ -1071,7 +1301,7 @@ async def test_a_username_spray_cannot_evict_an_existing_counter(monkeypatch):
|
|||
for i in range(500):
|
||||
await throttle.record_failure(f"spray-filler-{i}@corp.com")
|
||||
|
||||
assert await throttle._failures(throttle._key(victim)) == 3, "the counter must survive a spray"
|
||||
assert await throttle._failures(throttle.username_cache, throttle._username_key(victim)) == 3, "the counter must survive a spray"
|
||||
with pytest.raises(ProxyException) as blocked:
|
||||
await _guess(throttle, username=victim)
|
||||
assert blocked.value.code == "429"
|
||||
|
|
|
|||
|
|
@ -517,27 +517,38 @@ def make_key(
|
|||
def reset_login_throttle(monkeypatch):
|
||||
"""Clear the Admin UI failed-login counters between tests.
|
||||
|
||||
`client` is session scoped and the counters live in the shared `user_api_key_cache`
|
||||
with a 900s window, so without this any test that fails a sign-in enough times would
|
||||
start returning 429 from unrelated tests later in the same process. Only the throttle's
|
||||
own keys are removed, so nothing else in that cache is disturbed.
|
||||
`client` is session scoped and the counters live in shared module stores with a 900s
|
||||
window, so without this a failed sign-in test could return 429 in unrelated tests later.
|
||||
Only the throttle's own keys are removed, so other cache entries remain untouched.
|
||||
"""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy.auth.login_throttle import _FAILED_LOGIN_CACHE, _CACHE_KEY_PREFIX
|
||||
from litellm.proxy.auth import login_throttle
|
||||
from litellm.proxy.auth.login_throttle import (
|
||||
_CACHE_KEY_PREFIX,
|
||||
_FAILED_LOGIN_SOURCE_CACHE,
|
||||
_FAILED_LOGIN_USERNAME_CACHE,
|
||||
)
|
||||
|
||||
async def _no_delay(_seconds: float) -> None:
|
||||
"""The escalating wait on a rejected sign-in, replaced so the route tests stay fast."""
|
||||
|
||||
monkeypatch.setattr(login_throttle, "_sleep", _no_delay)
|
||||
|
||||
def _drop_throttle_keys() -> None:
|
||||
in_memory = getattr(_FAILED_LOGIN_CACHE, "in_memory_cache", None)
|
||||
if in_memory is None:
|
||||
return
|
||||
tracked = tuple(
|
||||
key
|
||||
for store in (getattr(in_memory, "cache_dict", None), getattr(in_memory, "ttl_dict", None))
|
||||
if isinstance(store, dict)
|
||||
for key in tuple(store)
|
||||
if str(key).startswith(_CACHE_KEY_PREFIX)
|
||||
)
|
||||
for key in tracked:
|
||||
in_memory.delete_cache(key)
|
||||
login_throttle._DELAYS_IN_FLIGHT.clear()
|
||||
for cache in (_FAILED_LOGIN_USERNAME_CACHE, _FAILED_LOGIN_SOURCE_CACHE):
|
||||
in_memory = getattr(cache, "in_memory_cache", None)
|
||||
if in_memory is None:
|
||||
continue
|
||||
tracked = tuple(
|
||||
key
|
||||
for store in (getattr(in_memory, "cache_dict", None), getattr(in_memory, "ttl_dict", None))
|
||||
if isinstance(store, dict)
|
||||
for key in tuple(store)
|
||||
if str(key).startswith(_CACHE_KEY_PREFIX)
|
||||
)
|
||||
for key in tracked:
|
||||
in_memory.delete_cache(key)
|
||||
|
||||
monkeypatch.setattr(ps, "redis_usage_cache", None)
|
||||
_drop_throttle_keys()
|
||||
|
|
|
|||
|
|
@ -531,8 +531,21 @@ def test_a_refused_attempt_carries_retry_after(client, monkeypatch, reset_login_
|
|||
assert refused.headers.get("retry-after") == "77"
|
||||
|
||||
|
||||
def test_the_form_returns_a_human_readable_lockout_page(client, monkeypatch, reset_login_throttle):
|
||||
"""The no-JavaScript form must render a wait page when its POST is throttled."""
|
||||
_install_real_auth(monkeypatch, max_failed_login_attempts=2, failed_login_window_seconds=77)
|
||||
|
||||
assert [_form_login(client) for _ in range(2)] == [401, 401]
|
||||
|
||||
refused = client.post("/login", data={"username": "admin", "password": "wrong"})
|
||||
assert refused.status_code == 429
|
||||
assert refused.headers.get("content-type", "").startswith("text/html")
|
||||
assert "Try again in about 77 seconds" in refused.text
|
||||
assert refused.headers.get("retry-after") == "77"
|
||||
|
||||
|
||||
def test_a_second_username_from_the_same_source_still_gets_through(client, monkeypatch, reset_login_throttle):
|
||||
"""The bucket is the username and source pair, so one account cannot block another."""
|
||||
"""The username counter carries no address, so one account exhausting it cannot block another."""
|
||||
_install_real_auth(monkeypatch, max_failed_login_attempts=2)
|
||||
|
||||
for _ in range(3):
|
||||
|
|
@ -541,6 +554,32 @@ def test_a_second_username_from_the_same_source_still_gets_through(client, monke
|
|||
assert _json_login(client, "/v2/login", username="someone-else@example.com") == 401
|
||||
|
||||
|
||||
def test_a_spray_across_usernames_is_refused_on_the_source_counter(client, monkeypatch, reset_login_throttle):
|
||||
"""A fresh username per guess keeps every username counter at one, so the address is what stops it."""
|
||||
_install_real_auth(monkeypatch, max_failed_login_attempts=100, max_failed_login_attempts_per_source=4)
|
||||
|
||||
sprayed = [_json_login(client, "/v2/login", username=f"sprayed-{i}@corp.com") for i in range(4)]
|
||||
assert sprayed == [401] * 4
|
||||
|
||||
assert _json_login(client, "/v2/login", username="sprayed-5@corp.com") == 429
|
||||
|
||||
|
||||
def test_the_configured_admin_password_still_signs_in_while_refused(client, monkeypatch, reset_login_throttle):
|
||||
"""The operator must never be locked out of the console by traffic aimed at it."""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
_install_real_auth(monkeypatch, max_failed_login_attempts=2)
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
|
||||
|
||||
assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401]
|
||||
assert _json_login(client, "/v2/login") == 429
|
||||
|
||||
with patch("litellm.proxy.auth.login_utils.user_update", new=AsyncMock()), patch(
|
||||
"litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
|
||||
):
|
||||
assert _json_login(client, "/v2/login", password="right-password") == 200
|
||||
|
||||
|
||||
def test_sign_in_succeeds_again_once_the_budget_is_restored(client, monkeypatch, reset_login_throttle):
|
||||
"""A cleared bucket lets the same username straight back in."""
|
||||
_install_real_auth(monkeypatch, max_failed_login_attempts=2)
|
||||
|
|
|
|||
|
|
@ -11442,9 +11442,14 @@ async def test_login_throttle_settings_are_not_overridable_from_the_database():
|
|||
try:
|
||||
ps.general_settings.clear()
|
||||
await ProxyConfig()._update_general_settings(
|
||||
db_general_settings={"max_failed_login_attempts": 999, "failed_login_window_seconds": 1}
|
||||
db_general_settings={
|
||||
"max_failed_login_attempts": 999,
|
||||
"max_failed_login_attempts_per_source": 999,
|
||||
"failed_login_window_seconds": 1,
|
||||
}
|
||||
)
|
||||
assert "max_failed_login_attempts" not in ps.general_settings
|
||||
assert "max_failed_login_attempts_per_source" not in ps.general_settings
|
||||
assert "failed_login_window_seconds" not in ps.general_settings
|
||||
finally:
|
||||
ps.general_settings.clear()
|
||||
|
|
|
|||
7
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
7
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -24201,9 +24201,14 @@ export interface components {
|
|||
max_batch_file_size_mb?: number | null;
|
||||
/**
|
||||
* Max Failed Login Attempts
|
||||
* @description Number of failed Admin UI sign-in attempts allowed for one username from one source address within `failed_login_window_seconds`, before further attempts are refused with 429. Configurable from config.yaml only. Defaults to 10
|
||||
* @description Number of failed Admin UI sign-in attempts allowed for one username, from any source address, within `failed_login_window_seconds`, before further attempts for that username are refused with 429. Attempts are answered with a doubling delay well before this ceiling. Configurable from config.yaml only. Defaults to 50
|
||||
*/
|
||||
max_failed_login_attempts?: number | null;
|
||||
/**
|
||||
* Max Failed Login Attempts Per Source
|
||||
* @description Number of failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`, before further attempts from that address are refused with 429. Counted independently of `max_failed_login_attempts`. Configurable from config.yaml only. Defaults to 250
|
||||
*/
|
||||
max_failed_login_attempts_per_source?: number | null;
|
||||
/**
|
||||
* Max Parallel Requests
|
||||
* @description maximum parallel requests for each api key
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue