scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

feat(proxy): hard-block throttled Admin UI sign-ins with no credential bypass

Browse source 
A blocked source, or source and username pair, is now refused with 429 before the database lookup and password check, in place of the soft block that held wrong guesses for 30 seconds and let a correct password through. The env admin credentials and the master key typed into the login form are refused like any other credential while blocked; recovery is the master key as an API bearer token, which never goes through the sign-in path

trusted_proxy_ranges: [] now means clients connect directly, so the peer address is the source and the per-source limit stays on. Only an unset or malformed value leaves the topology unknown, warns at startup and turns the per-source limit off

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-17 16:10:23 +00:00
parent c05095373d
commit 9a365d2021
10 changed files with 273 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/proxy/_types.py
View file

@ -2755,7 +2755,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
max_failed_login_attempts_per_source: int | None = Field(
None,
ge=1,
description="Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Only enforced when `trusted_proxy_ranges` is set, since otherwise every client behind an ingress shares one address. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10",
description="Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Only enforced when `trusted_proxy_ranges` is set: to the proxies in front of LiteLLM, or to an empty list when clients connect directly. Left unset, the peer address may be a shared ingress and this limit is off. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10",
)
max_failed_login_attempts_per_source_overrides: dict[str, int] | None = Field(
None,
@ -2774,7 +2774,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
failed_login_block_seconds: int | None = Field(
None,
ge=1,
description="How long a blocked source address, or source address and username, stays blocked. The block is soft: a correct password still signs in, but each attempt from a blocked key takes one of 5 held slots per worker and a wrong password is held for 30 seconds before it is refused with 429. Set under `general_settings` in config.yaml. Defaults to 300",
description="How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300",
)
allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access")
reject_clientside_metadata_tags: bool | None = Field(
@ -2885,7 +2885,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
)
trusted_proxy_ranges: list[str] | None = Field(
None,
description="CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler.",
description="CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler, and whose X-Forwarded-For is used to attribute Admin UI sign-in attempts to a source address. Set it to an empty list when clients connect directly, so the peer address is the source. Left unset, the per-source sign-in limit is off.",
)
store_model_in_db: bool | None = Field(
None,

100
litellm/proxy/auth/login_throttle.py
View file

@ -1,10 +1,10 @@
"""Failed-login accounting for the Admin UI sign-in path.
Wrong passwords are counted over a short window per source address and per source-and-username
pair; too many in one window blocks that key for a fixed time. Blocks are soft: a correct password
still signs in, while a wrong one from a blocked key is held open before its 429 and only a few can
be held at once, which bounds how many guesses a blocked key gets checked. A blocked pair stops
pair; too many in one window blocks that key for a fixed time. While a key is blocked every attempt
from it, right or wrong, is refused with 429 before the password is checked. A blocked pair stops
counting against its source, so one script stuck on one account does not block the whole office.
Recovery is the master key over the API, which never passes through here, or waiting out the block.
"""
from __future__ import annotations
@ -14,8 +14,7 @@ import hashlib
import ipaddress
import math
import time
from collections.abc import AsyncGenerator, Mapping
from contextlib import asynccontextmanager
from collections.abc import Mapping
from dataclasses import dataclass
from functools import cache
from types import MappingProxyType
@ -37,8 +36,6 @@ DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_USER: Final = 5
DEFAULT_FAILED_LOGIN_WINDOW_SECONDS: Final = 60
DEFAULT_FAILED_LOGIN_BLOCK_SECONDS: Final = 300
BLOCKED_ATTEMPT_HOLD_SECONDS: Final = 30
MAX_HELD_ATTEMPTS_PER_KEY: Final = 5
IPV6_SOURCE_PREFIX_LENGTH: Final = 64
SOURCE_LIMIT_KEY: Final = "max_failed_login_attempts_per_source"
@ -100,11 +97,6 @@ _COUNTERS: Final = InMemoryCache(
max_size_in_memory=_MAX_TRACKED_COUNTERS, default_ttl=DEFAULT_FAILED_LOGIN_WINDOW_SECONDS
)
_BLOCKS: Final = InMemoryCache(max_size_in_memory=_MAX_TRACKED_BLOCKS, default_ttl=DEFAULT_FAILED_LOGIN_BLOCK_SECONDS)
_HELD_ATTEMPTS: Final[dict[str, int]] = {} # mutable-ok: in-flight hold counts rise on entry and fall on exit
async def _sleep(seconds: float) -> None:
await asyncio.sleep(seconds)
@cache
@ -127,12 +119,25 @@ def warn_login_counters_are_per_worker(num_workers: str) -> None:
def warn_source_login_limit_is_off() -> None:
verbose_proxy_logger.warning(
"%s is not set, so failed Admin UI sign-in attempts are limited per source address and username "
"only. Set it to the address ranges of the proxies in front of LiteLLM to also limit each "
"source address across usernames.",
"only. Set it to the address ranges of the proxies in front of LiteLLM, or to an empty list when "
"clients connect directly, to also limit each source address across usernames.",
TRUSTED_PROXY_RANGES_KEY,
)
def declared_proxy_ranges(settings: Mapping[str, object]) -> tuple[str, ...] | None:
"""What the operator says fronts LiteLLM: the proxy ranges, an empty tuple for none, None when unsaid.
Only a declared topology makes the source address trustworthy enough to limit across usernames.
An unset key, or a value that is not a list of ranges, leaves it unknown and the source scope off.
"""
raw_ranges: Final = settings.get(TRUSTED_PROXY_RANGES_KEY)
if isinstance(raw_ranges, (list, tuple, set)) and not raw_ranges:
return ()
cidrs: Final = tuple(normalize_cidr_ranges(raw_ranges, setting_name=TRUSTED_PROXY_RANGES_KEY))
return cidrs or None
def _positive_int(raw: object, key: str, default: int) -> int:
if raw is None:
return default
@ -221,8 +226,11 @@ class Block:
@dataclass(frozen=True, slots=True)
class LoginThrottle:
"""Failed-login limits for one request's source address; ``source_limit`` is None when the
source scope is off because ``trusted_proxy_ranges`` is unset and the peer address is the ingress."""
"""Failed-login limits for one request's source address.
``source_limit`` is None when the source scope is off: ``trusted_proxy_ranges`` is unset, so the peer
address may be a shared ingress. An empty list means clients connect directly and the peer is the source.
"""
client_ip: str
source_limit: int | None
@ -242,15 +250,13 @@ class LoginThrottle:
redis_cache: RedisCache | None,
) -> LoginThrottle:
settings: Final = general_settings if general_settings is not None else _NO_SETTINGS
cidrs: Final = normalize_cidr_ranges(
settings.get(TRUSTED_PROXY_RANGES_KEY), setting_name=TRUSTED_PROXY_RANGES_KEY
)
proxies: Final = declared_proxy_ranges(settings)
resolved, _ = resolve_client_ip(
request, TrustedProxyConfig(use_forwarded_for=bool(cidrs), trusted_proxy_cidrs=cidrs)
request, TrustedProxyConfig(use_forwarded_for=bool(proxies), trusted_proxy_cidrs=proxies or ())
)
return cls(
client_ip=resolved or _UNKNOWN_SOURCE,
source_limit=_source_limit(settings, resolved) if cidrs and resolved is not None else None,
source_limit=_source_limit(settings, resolved) if proxies is not None and resolved is not None else None,
user_limit=_int_setting(settings, USER_LIMIT_KEY, DEFAULT_MAX_FAILED_LOGIN_ATTEMPTS_PER_USER),
window_seconds=_int_setting(settings, WINDOW_KEY, DEFAULT_FAILED_LOGIN_WINDOW_SECONDS),
block_seconds=_int_setting(settings, BLOCK_KEY, DEFAULT_FAILED_LOGIN_BLOCK_SECONDS),
@ -270,18 +276,21 @@ class LoginThrottle:
source_block=f"{_CACHE_KEY_PREFIX}:{{{group}}}:block:source",
)
@asynccontextmanager
async def attempt(self, username: str, *, exempt: bool = False) -> AsyncGenerator[LoginAttempt]:
if not self.enabled or exempt:
yield LoginAttempt(throttle=self, username=username, block=None)
return
keys: Final = self._keys(username)
block: Final = await self._active_block(keys)
async def attempt(self, username: str) -> LoginAttempt:
"""Refuses a blocked key before any credential is looked at; otherwise hands back the attempt to settle."""
if not self.enabled:
return LoginAttempt(throttle=self, username=username)
block: Final = await self._active_block(self._keys(username))
if block is None:
yield LoginAttempt(throttle=self, username=username, block=None)
return
slot: Final = keys.pair_block if block.scope == "user" else keys.source_block
yield LoginAttempt(throttle=self, username=username, block=block, slot=slot)
return LoginAttempt(throttle=self, username=username)
verbose_proxy_logger.warning(
"Admin UI sign-in refused: the %s is blocked for %s more seconds; username=%r source=%s",
block.scope,
block.retry_after,
username,
self.client_ip,
)
self.refuse(block.retry_after)
async def _active_block(self, keys: _Keys) -> Block | None:
local: Final = self._local_block_ttls(keys)
@ -372,8 +381,6 @@ class LoginThrottle:
class LoginAttempt:
throttle: LoginThrottle
username: str
block: Block | None
slot: str | None = None
async def succeeded(self) -> None:
if not self.throttle.enabled:
@ -383,8 +390,6 @@ class LoginAttempt:
async def failed(self) -> None:
if not self.throttle.enabled:
return
if self.block is not None and self.slot is not None:
await self._hold_then_refuse(self.block, self.slot)
user_block, source_block = await self.throttle.record_failure(self.username)
if user_block == 0 and source_block == 0:
return
@ -395,26 +400,3 @@ class LoginAttempt:
self.username,
self.throttle.client_ip,
)
async def _hold_then_refuse(self, block: Block, slot: str) -> NoReturn:
held: Final = _HELD_ATTEMPTS.get(slot, 0)
if held >= MAX_HELD_ATTEMPTS_PER_KEY:
verbose_proxy_logger.warning(
"Admin UI sign-in refused at once: %s wrong attempts already held for a blocked %s; "
"username=%r source=%s",
held,
block.scope,
self.username,
self.throttle.client_ip,
)
self.throttle.refuse(block.retry_after)
_HELD_ATTEMPTS[slot] = held + 1
try:
await _sleep(BLOCKED_ATTEMPT_HOLD_SECONDS)
finally:
remaining: Final = _HELD_ATTEMPTS.get(slot, 1) - 1
if remaining > 0:
_HELD_ATTEMPTS[slot] = remaining
else:
_HELD_ATTEMPTS.pop(slot, None)
self.throttle.refuse(max(block.retry_after - BLOCKED_ATTEMPT_HOLD_SECONDS, 1))

18
litellm/proxy/auth/login_utils.py
View file

@ -186,9 +186,11 @@ async def authenticate_user(
or if username/password login is disabled while SSO is configured
Recovery: an admin locked out of the UI by
`disable_password_login_when_sso_enabled` can still administer the proxy over
the API with the master key (Authorization: Bearer <master_key>), which never
goes through this function. To restore UI username/password login, unset the
`disable_password_login_when_sso_enabled`, or by the failed sign-in block in
`throttle`, can still administer the proxy over the API with the master key
(Authorization: Bearer <master_key>), which never goes through this function.
No credential, the env admin credentials and the master key included, is
exempt from the block. To restore UI username/password login, unset the
setting in config.yaml (or the DB-persisted general_settings) and restart the
proxy; this is a deliberate, auditable config change rather than a hidden
bypass.
@ -217,12 +219,8 @@ async def authenticate_user(
code=500,
)
admin_credentials_match: Final = _admin_credentials_match(username, password, master_key, general_settings)
async with throttle.attempt(username, exempt=admin_credentials_match) as attempt:
return await _sign_in(
username, password, master_key, prisma_client, attempt, general_settings, admin_credentials_match
)
attempt: Final = await throttle.attempt(username)
return await _sign_in(username, password, master_key, prisma_client, attempt, general_settings)
async def _sign_in(
@ -232,8 +230,8 @@ async def _sign_in(
prisma_client: PrismaClient | None,
attempt: LoginAttempt,
general_settings: Mapping[str, object],
admin_credentials_match: bool,
) -> LoginResult:
admin_credentials_match: Final = _admin_credentials_match(username, password, master_key, general_settings)
# Check if we can find the `username` in the db. On the UI, users can enter username=their email
_user_row: LiteLLM_UserTable | None = None
user_role: (

3
litellm/proxy/auth/network.py
View file

@ -1,6 +1,7 @@
from __future__ import annotations
import ipaddress
from collections.abc import Sequence
from typing import Any, Final
from fastapi import Request
@ -19,7 +20,7 @@ class NetworkContext(BaseModel):
class TrustedProxyConfig(BaseModel):
use_forwarded_for: bool = False
trusted_proxy_cidrs: list[str] = Field(default_factory=list)
trusted_proxy_cidrs: Sequence[str] = Field(default_factory=tuple)
def normalize_cidr_ranges(configured_ranges: Any, *, setting_name: str = "trusted_proxy_cidrs") -> list[str]:

4
litellm/proxy/proxy_server.py
View file

@ -328,8 +328,8 @@ from litellm.proxy.auth.fallback_model_access import router_fallback_access_chec
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY, LicenseCheck
from litellm.proxy.auth.login_throttle import (
TRUSTED_PROXY_RANGES_KEY,
LoginThrottle,
declared_proxy_ranges,
warn_login_counters_are_per_worker,
warn_source_login_limit_is_off,
)
@ -5819,7 +5819,7 @@ class ProxyConfig:
if os.getenv("NUM_WORKERS", "1") != "1" and redis_usage_cache is None:
warn_login_counters_are_per_worker(os.getenv("NUM_WORKERS", "1"))
if not general_settings.get(TRUSTED_PROXY_RANGES_KEY):
if declared_proxy_ranges(general_settings) is None:
warn_source_login_limit_is_off()
_bg_hc_model_groups: Final = parse_background_health_check_model_groups(general_settings)

333
tests/test_litellm/proxy/auth/test_login_utils.py
View file

@ -13,28 +13,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
class _RecordedSleeps:
"""A sleep that records what it was asked to wait for instead of waiting."""
def __init__(self):
self.seconds: list[float] = []
async def __call__(self, seconds: float) -> None:
self.seconds.append(seconds)
@pytest.fixture(autouse=True)
def login_delays(monkeypatch):
"""Replace the hold on a blocked wrong password, so the suite pays no wall clock and can read it back."""
from litellm.proxy.auth import login_throttle
recorded = _RecordedSleeps()
monkeypatch.setattr(login_throttle, "_sleep", recorded)
login_throttle._HELD_ATTEMPTS.clear()
yield recorded
login_throttle._HELD_ATTEMPTS.clear()
def _unlimited_throttle():
"""A throttle wired to real in-memory stores with limits no test can reach."""
from litellm.caching.in_memory_cache import InMemoryCache
@ -749,8 +727,8 @@ def _local_count(throttle, key: str) -> int:
@pytest.mark.asyncio
async def test_too_many_failures_for_one_username_block_that_pair_and_carry_retry_after(monkeypatch):
"""One failure past the pair limit blocks the source for that username; the next wrong guess is held
and answered 429 with the block's remaining time, and the counter is not touched by blocked guesses."""
"""One failure past the pair limit blocks the source for that username; the next guess is answered 429
with the block's remaining time, and the counter is not touched by blocked guesses."""
from litellm.proxy._types import ProxyException
monkeypatch.setenv("UI_USERNAME", "admin")
@ -764,30 +742,17 @@ async def test_too_many_failures_for_one_username_block_that_pair_and_carry_retr
with pytest.raises(ProxyException) as blocked:
await _guess(throttle)
assert blocked.value.code == "429"
assert blocked.value.headers.get("Retry-After") == "47", "the 30s hold is taken off the remaining block"
assert blocked.value.headers.get("Retry-After") == "77"
assert _local_count(throttle, keys.pair_counter) == 3, "a blocked guess is not counted again"
@pytest.mark.asyncio
async def test_a_wrong_password_from_a_blocked_key_is_held_before_it_is_refused(monkeypatch, login_delays):
"""The hold is the rate cap: a blocked key gets one verified guess per held slot per 30 seconds."""
from litellm.proxy.auth.login_throttle import BLOCKED_ATTEMPT_HOLD_SECONDS
async def test_a_blocked_key_is_refused_before_the_password_is_looked_at(monkeypatch):
"""The block is the rate cap: once a key is blocked, nothing from it reaches the user lookup or the
password check, so a guessing script gets no verification work out of the proxy."""
from litellm.proxy._types import ProxyException
from litellm.proxy.auth.login_utils import authenticate_user
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
throttle = _throttle(user_limit=1)
assert [await _fail(throttle) for _ in range(2)] == ["401", "401"]
assert login_delays.seconds == [], "an unblocked wrong password is answered at once"
assert await _fail(throttle) == "429"
assert login_delays.seconds == [BLOCKED_ATTEMPT_HOLD_SECONDS]
@pytest.mark.asyncio
async def test_a_correct_password_signs_in_while_its_pair_is_blocked(monkeypatch):
"""The block is soft: the real user is still verified and gets in, so nobody can be locked out by
guessing at their account."""
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
@ -795,13 +760,50 @@ async def test_a_correct_password_signs_in_while_its_pair_is_blocked(monkeypatch
assert [await _fail(throttle, username="user@corp.com") for _ in range(3)] == ["401", "401", "429"]
result = await _db_login(throttle, "user@corp.com", "right", correct=True)
assert result.key == "sk-ui"
lookup = _known_user("user@corp.com")
verify = MagicMock(return_value=True)
with (
patch( # test-quality-ok: the user lookup is the database boundary; a blocked attempt must not reach it
"litellm.proxy.auth.login_utils.UserRepository", lookup
),
patch( # test-quality-ok: the password check is the expensive step; a blocked attempt must not reach it
"litellm.proxy.auth.login_utils.verify_password", verify
),
pytest.raises(ProxyException) as refused,
):
await authenticate_user(
username="user@corp.com",
password="right",
master_key="sk-master",
prisma_client=MagicMock(),
throttle=throttle,
)
assert refused.value.code == "429"
assert lookup.return_value.table.find_first.await_count == 0
assert verify.call_count == 0
@pytest.mark.asyncio
async def test_a_correct_password_signs_in_while_its_source_is_blocked(monkeypatch):
"""Same for the source-wide block: it slows guessing from that address, it does not refuse a user."""
async def test_a_correct_password_is_refused_while_its_pair_is_blocked(monkeypatch):
"""Letting the right password through would give a guesser unlimited tries, so the block is hard: the
real user waits it out, or uses the master key over the API, which never passes through here."""
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
throttle = _throttle(user_limit=1, block_seconds=90)
assert [await _fail(throttle, username="user@corp.com") for _ in range(3)] == ["401", "401", "429"]
with pytest.raises(ProxyException) as refused:
await _db_login(throttle, "user@corp.com", "right", correct=True)
assert refused.value.code == "429"
assert refused.value.headers.get("Retry-After") == "90"
@pytest.mark.asyncio
async def test_a_correct_password_is_refused_while_its_source_is_blocked(monkeypatch):
"""Same for the source-wide block: every username from that address is refused until it lapses."""
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
@ -811,8 +813,9 @@ async def test_a_correct_password_signs_in_while_its_source_is_blocked(monkeypat
assert await _fail(throttle, username=f"other-{i}@corp.com") == "401"
assert await _fail(throttle, username="other-9@corp.com") == "429", "the source is blocked for everyone"
result = await _db_login(throttle, "user@corp.com", "right", correct=True)
assert result.key == "sk-ui"
with pytest.raises(ProxyException) as refused:
await _db_login(throttle, "user@corp.com", "right", correct=True)
assert refused.value.code == "429"
@pytest.mark.asyncio
@ -903,6 +906,63 @@ async def test_without_trusted_proxy_ranges_the_source_scope_is_off(monkeypatch)
assert [await _fail(throttle, username=f"user-{i}@corp.com") for i in range(6)] == ["401"] * 6
@pytest.mark.asyncio
async def test_an_empty_trusted_proxy_ranges_means_the_peer_is_the_client_and_the_source_scope_is_on(monkeypatch):
"""An explicit empty list says there are no proxies: the peer address is the client, the forwarded header
is ignored, and the source-wide limit applies. Only an unset key means the topology is unknown."""
from litellm.proxy.auth.login_throttle import LoginThrottle
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
request = MagicMock()
request.headers = {"x-forwarded-for": "203.0.113.9"}
request.client = MagicMock()
request.client.host = "198.51.100.7"
throttle = LoginThrottle.from_request(
request,
general_settings={"trusted_proxy_ranges": [], "max_failed_login_attempts_per_source": 3},
redis_cache=None,
)
assert throttle.client_ip == "198.51.100.7"
assert throttle.source_limit == 3
assert [await _fail(throttle, username=f"user-{i}@corp.com") for i in range(4)] == ["401"] * 4
assert await _fail(throttle, username="user-99@corp.com") == "429", "the spray is stopped by the source limit"
@pytest.mark.parametrize("configured", [None, 5, {"10.0.0.0/8": True}, ["", " "]])
def test_a_trusted_proxy_ranges_value_that_names_no_ranges_leaves_the_topology_unknown(configured):
"""Only a real list of ranges or an explicit empty list counts as a declaration; anything else is the same
as unset, so a typo cannot switch the source-wide block on behind a shared ingress."""
from litellm.proxy.auth.login_throttle import LoginThrottle, declared_proxy_ranges
settings = {"trusted_proxy_ranges": configured} if configured is not None else {}
assert declared_proxy_ranges(settings) is None
request = MagicMock()
request.headers = {}
request.client = MagicMock()
request.client.host = "198.51.100.7"
throttle = LoginThrottle.from_request(request, general_settings=settings, redis_cache=None)
assert throttle.source_limit is None
assert throttle.client_ip == "198.51.100.7"
def test_declared_proxy_ranges_distinguishes_none_from_empty_from_configured():
from litellm.proxy.auth.login_throttle import declared_proxy_ranges
assert declared_proxy_ranges({}) is None
assert declared_proxy_ranges({"trusted_proxy_ranges": []}) == ()
assert declared_proxy_ranges({"trusted_proxy_ranges": ["10.0.0.0/8", " 192.168.1.1 "]}) == (
"10.0.0.0/8",
"192.168.1.1",
)
assert declared_proxy_ranges({"trusted_proxy_ranges": "10.0.0.0/8,172.16.0.0/12"}) == (
"10.0.0.0/8",
"172.16.0.0/12",
)
@pytest.mark.asyncio
async def test_with_trusted_proxy_ranges_the_source_is_the_forwarded_client(monkeypatch):
"""The header is walked right to left past the trusted hops, so a forged left-most entry cannot pick the bucket."""
@ -1056,9 +1116,10 @@ async def test_the_block_time_is_fixed_and_not_refreshed_by_blocked_guesses(monk
@pytest.mark.asyncio
async def test_the_configured_admin_credentials_bypass_the_throttle(monkeypatch):
"""The only account that can fix a misconfiguration is exempt: no hold, no slot, even while blocked."""
from litellm.proxy.auth import login_throttle as lt
async def test_the_configured_admin_credentials_are_not_exempt_from_the_block(monkeypatch):
"""Exempting the env credentials would make them the one password worth guessing without limit, so the
right UI_PASSWORD is refused while its pair is blocked, and signs in normally once the block lapses."""
from litellm.proxy._types import ProxyException
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
@ -1075,9 +1136,31 @@ async def test_the_configured_admin_credentials_bypass_the_throttle(monkeypatch)
"litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
),
):
with pytest.raises(ProxyException) as refused:
await _guess(throttle, password="right")
assert refused.value.code == "429"
throttle.blocks.delete_cache(throttle._keys("admin").pair_block)
result = await _guess(throttle, password="right")
assert result.key == "sk-ui"
assert lt._HELD_ATTEMPTS == {}
@pytest.mark.asyncio
async def test_the_master_key_used_as_the_ui_password_is_not_exempt_from_the_block(monkeypatch):
"""Without UI_PASSWORD the master key doubles as the admin password; it gets no special treatment here
either. Lockout recovery is the master key as a bearer token over the API, which never enters this path."""
from litellm.proxy._types import ProxyException
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.delenv("UI_PASSWORD", raising=False)
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
throttle = _throttle(user_limit=1)
assert [await _fail(throttle) for _ in range(3)] == ["401", "401", "429"]
with pytest.raises(ProxyException) as refused:
await _guess(throttle, password="sk-master")
assert refused.value.code == "429"
@pytest.mark.asyncio
@ -1180,138 +1263,29 @@ async def test_a_wrong_password_for_a_known_user_also_counts(monkeypatch):
@pytest.mark.asyncio
async def test_held_attempts_from_one_blocked_key_are_capped(monkeypatch):
"""Holding a wrong guess open must not let one blocked key park unlimited sockets in password checks."""
import asyncio
async def test_a_source_block_outranks_a_pair_block_in_the_retry_after(monkeypatch):
"""When both scopes are blocked, the answer carries the source block's time, which is the one that
still applies to every other username from that address."""
from litellm.proxy._types import ProxyException
from litellm.proxy.auth import login_throttle as lt
from litellm.proxy.auth.login_throttle import MAX_HELD_ATTEMPTS_PER_KEY
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
release = asyncio.Event()
async def _park(_seconds: float) -> None:
await release.wait()
monkeypatch.setattr(lt, "_sleep", _park)
throttle = _throttle(user_limit=1, client_ip="203.0.113.44")
slot = throttle._keys("admin").pair_block
assert [await _fail(throttle) for _ in range(2)] == ["401", "401"]
held = [asyncio.create_task(_guess(throttle)) for _ in range(MAX_HELD_ATTEMPTS_PER_KEY)]
for _ in range(1000):
if lt._HELD_ATTEMPTS.get(slot) == MAX_HELD_ATTEMPTS_PER_KEY:
break
await asyncio.sleep(0)
assert lt._HELD_ATTEMPTS.get(slot) == MAX_HELD_ATTEMPTS_PER_KEY
try:
with pytest.raises(ProxyException) as over_cap:
await _guess(throttle)
assert over_cap.value.code == "429"
assert over_cap.value.headers.get("Retry-After") == "300", "refused at once, for the whole block"
assert await _fail(throttle, username="someone-else@corp.com") == "401", "other keys are not affected"
finally:
release.set()
for task in held:
with pytest.raises(ProxyException):
await task
assert lt._HELD_ATTEMPTS.get(slot) is None, "the slots are released once the held attempts answer"
@pytest.mark.asyncio
async def test_a_full_hold_pool_still_lets_the_right_password_in(monkeypatch):
"""Five parked wrong guesses from the office must not turn the soft block into a lockout for the real user."""
import asyncio
from litellm.proxy.auth import login_throttle as lt
from litellm.proxy.auth.login_throttle import MAX_HELD_ATTEMPTS_PER_KEY
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
release = asyncio.Event()
async def _park(_seconds: float) -> None:
await release.wait()
monkeypatch.setattr(lt, "_sleep", _park)
throttle = _throttle(user_limit=1, source_limit=3, client_ip="203.0.113.46")
assert [await _fail(throttle, username=f"spray-{i}@corp.com") for i in range(4)] == ["401"] * 4
source_slot = throttle._keys("known@example.com").source_block
assert throttle._local_block_ttl(source_slot) > 0, "the source is blocked"
held = [
asyncio.create_task(_guess(throttle, username="known@example.com")) for _ in range(MAX_HELD_ATTEMPTS_PER_KEY)
]
for _ in range(1000):
if lt._HELD_ATTEMPTS.get(source_slot) == MAX_HELD_ATTEMPTS_PER_KEY:
break
await asyncio.sleep(0)
assert lt._HELD_ATTEMPTS == {source_slot: MAX_HELD_ATTEMPTS_PER_KEY}
try:
signed_in = await _db_login(throttle, "known@example.com", "right", correct=True)
assert signed_in.user_id == "u-1"
finally:
release.set()
for task in held:
with pytest.raises(ProxyException):
await task
@pytest.mark.asyncio
async def test_a_blocked_source_shares_one_held_slot_pool_across_its_blocked_usernames(monkeypatch):
"""Once the source is blocked, a pair block for a username must not hand that username its own five slots."""
import asyncio
from litellm.proxy._types import ProxyException
from litellm.proxy.auth import login_throttle as lt
from litellm.proxy.auth.login_throttle import MAX_HELD_ATTEMPTS_PER_KEY
monkeypatch.setenv("UI_USERNAME", "admin")
monkeypatch.setenv("UI_PASSWORD", "right")
release = asyncio.Event()
async def _park(_seconds: float) -> None:
await release.wait()
monkeypatch.setattr(lt, "_sleep", _park)
throttle = _throttle(user_limit=1, source_limit=3, client_ip="203.0.113.45")
throttle = _throttle(user_limit=1, source_limit=3, block_seconds=120, client_ip="203.0.113.45")
assert [await _fail(throttle) for _ in range(2)] == ["401", "401"], "the admin pair is now blocked"
throttle.blocks.set_cache(throttle._keys("admin").pair_block, 1, ttl=30)
assert [await _fail(throttle, username=f"spray-{i}@corp.com") for i in range(3)] == ["401"] * 3
source_slot = throttle._keys("admin").source_block
assert throttle._local_block_ttl(source_slot) > 0, "the source is now blocked as well"
assert throttle._local_block_ttl(throttle._keys("admin").source_block) == 120, "the source is now blocked too"
usernames = ["admin", *(f"fresh-{i}@corp.com" for i in range(MAX_HELD_ATTEMPTS_PER_KEY - 1))]
held = [asyncio.create_task(_guess(throttle, username=name)) for name in usernames]
for _ in range(1000):
if lt._HELD_ATTEMPTS.get(source_slot) == MAX_HELD_ATTEMPTS_PER_KEY:
break
await asyncio.sleep(0)
assert lt._HELD_ATTEMPTS == {source_slot: MAX_HELD_ATTEMPTS_PER_KEY}
try:
for name in ("admin", "fresh-0@corp.com", "never-seen@corp.com"):
with pytest.raises(ProxyException) as over_cap:
await _guess(throttle, username=name)
assert over_cap.value.code == "429"
assert over_cap.value.headers.get("Retry-After") == "300"
finally:
release.set()
for task in held:
with pytest.raises(ProxyException):
await task
assert lt._HELD_ATTEMPTS == {}
for name in ("admin", "spray-0@corp.com", "never-seen@corp.com"):
with pytest.raises(ProxyException) as refused:
await _guess(throttle, username=name)
assert refused.value.code == "429"
assert refused.value.headers.get("Retry-After") == "120", name
@pytest.mark.asyncio
async def test_disabling_the_control_removes_the_hold_as_well(monkeypatch, login_delays):
"""The escape hatch has to turn off the whole control, not only the refusal."""
async def test_disabling_the_control_lets_every_attempt_through(monkeypatch):
"""The escape hatch has to turn off the whole control: no counting and no refusal."""
import dataclasses
monkeypatch.setenv("UI_USERNAME", "admin")
@ -1319,7 +1293,7 @@ async def test_disabling_the_control_removes_the_hold_as_well(monkeypatch, login
throttle = dataclasses.replace(_throttle(user_limit=1), enabled=False)
assert [await _fail(throttle) for _ in range(6)] == ["401"] * 6
assert login_delays.seconds == []
assert _local_count(throttle, throttle._keys("admin").pair_counter) == 0
class _FakeRedis:
@ -1404,12 +1378,15 @@ async def test_redis_is_the_only_counter_while_it_answers(monkeypatch):
assert await _fail(second_worker, username="user@corp.com") == "429", "the second worker sees the block"
block_keys = [k for k in redis.values if ":block:user:" in k]
assert block_keys, "the block lives in Redis, where every worker reads it"
for key in block_keys:
await redis.async_delete_cache(key)
await _db_login(second_worker, "user@corp.com", "right", correct=True)
assert not [k for k in redis.values if ":user:" in k and ":block:" not in k], (
"success clears the shared pair counter"
)
assert [k for k in redis.values if ":block:user:" in k], "an active block is not lifted by one success"
@pytest.mark.asyncio
@ -1436,7 +1413,7 @@ async def test_a_redis_outage_falls_back_to_this_workers_own_counter(monkeypatch
verbose_proxy_logger.removeHandler(handler)
assert blocked.value.code == "429"
assert blocked.value.headers.get("Retry-After") == "270"
assert blocked.value.headers.get("Retry-After") == "300"
assert any("Redis failed while counting Admin UI sign-in attempts" in r.getMessage() for r in records)

7
tests/test_litellm/proxy/proxy_server/conftest.py
View file

@ -522,16 +522,9 @@ def reset_login_throttle(monkeypatch):
Only the throttle's own keys are removed, so other cache entries remain untouched.
"""
from litellm.proxy import proxy_server as ps
from litellm.proxy.auth import login_throttle
from litellm.proxy.auth.login_throttle import _BLOCKS, _CACHE_KEY_PREFIX, _COUNTERS
async def _no_delay(_seconds: float) -> None:
"""The hold on a rejected sign-in from a blocked key, replaced so the route tests stay fast."""
monkeypatch.setattr(login_throttle, "_sleep", _no_delay)
def _drop_throttle_keys() -> None:
login_throttle._HELD_ATTEMPTS.clear()
for store in (_COUNTERS, _BLOCKS):
for key in tuple(store.cache_dict) + tuple(store.ttl_dict):
if key.startswith(_CACHE_KEY_PREFIX):

59
tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py
View file

@ -553,14 +553,14 @@ def test_budget_is_shared_across_username_casing(client, monkeypatch, reset_logi
def test_a_refused_attempt_carries_retry_after(client, monkeypatch, reset_login_throttle):
"""The 429 tells the caller how long the block has left, after the 30 seconds it was already held."""
"""The 429 tells the caller how long the block has left."""
_install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1, failed_login_block_seconds=77)
assert [_json_login(client, "/v2/login") for _ in range(2)] == [401, 401]
refused = client.post("/v2/login", json={"username": "admin", "password": "wrong"})
assert refused.status_code == 429
assert refused.headers.get("retry-after") == "47"
assert refused.headers.get("retry-after") == "77"
def test_the_form_returns_a_human_readable_lockout_page(client, monkeypatch, reset_login_throttle):
@ -572,8 +572,8 @@ def test_the_form_returns_a_human_readable_lockout_page(client, monkeypatch, res
refused = client.post("/login", data={"username": "admin", "password": "wrong"})
assert refused.status_code == 429
assert refused.headers.get("content-type", "").startswith("text/html")
assert "Try again in about 47 seconds" in refused.text
assert refused.headers.get("retry-after") == "47"
assert "Try again in about 77 seconds" in refused.text
assert refused.headers.get("retry-after") == "77"
def test_a_second_username_from_the_same_source_still_gets_through(client, monkeypatch, reset_login_throttle):
@ -608,8 +608,29 @@ def test_a_spray_across_usernames_is_not_blocked_without_trusted_proxy_ranges(
assert sprayed == [401] * 8
def test_the_configured_admin_password_still_signs_in_while_blocked(client, monkeypatch, reset_login_throttle):
"""The operator must never be locked out of the console by traffic aimed at it."""
def test_a_spray_across_usernames_is_blocked_on_the_source_with_an_empty_trusted_proxy_ranges(
client, monkeypatch, reset_login_throttle
):
"""An explicit empty list says nothing fronts the proxy, so the peer address is the client and the
source scope is on. A forwarded header from an untrusted peer is ignored rather than trusted."""
_install_real_auth(monkeypatch, trusted_proxy_ranges=[], max_failed_login_attempts_per_source=4)
sprayed = [
client.post(
"/v2/login",
json={"username": f"sprayed-{i}@corp.com", "password": "wrong"},
headers={"x-forwarded-for": f"203.0.113.{i}"},
).status_code
for i in range(5)
]
assert sprayed == [401] * 5
assert _json_login(client, "/v2/login", username="sprayed-6@corp.com") == 429
def test_the_configured_admin_password_is_refused_while_blocked(client, monkeypatch, reset_login_throttle):
"""The env credentials get no bypass: a bypass would make them the one password worth guessing without
limit. An operator who is blocked administers the proxy with the master key over the API meanwhile."""
from unittest.mock import AsyncMock, patch
_install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1)
@ -625,18 +646,38 @@ def test_the_configured_admin_password_still_signs_in_while_blocked(client, monk
"litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
),
):
assert _json_login(client, "/v2/login", password="right-password") == 429
reset_login_throttle()
assert _json_login(client, "/v2/login", password="right-password") == 200
def test_a_database_users_correct_password_signs_in_while_blocked(client, monkeypatch, reset_login_throttle):
"""The block is soft: guessing at an account slows the guesser down, it does not lock the owner out."""
def test_the_master_key_as_a_bearer_token_still_works_while_the_ui_password_is_blocked(
client, monkeypatch, reset_login_throttle
):
"""Lockout recovery: the API path with the master key never enters the sign-in throttle."""
_install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1)
assert [_json_login(client, "/v2/login") for _ in range(3)] == [401, 401, 429]
assert client.get("/models", headers={"Authorization": "Bearer sk-not-the-master"}).status_code >= 400
assert client.get("/models", headers={"Authorization": "Bearer sk-test-master"}).status_code == 200
assert _json_login(client, "/v2/login", password="right-password") == 429, "the UI block is unaffected"
def test_a_database_users_correct_password_is_refused_while_blocked(client, monkeypatch, reset_login_throttle):
"""The block is hard: while it lasts, nothing from that source signs in as that user, right password or not,
and the block is not extended by the refused attempts."""
_install_real_auth(monkeypatch, max_failed_login_attempts_per_user=1, failed_login_block_seconds=64)
_db_user(monkeypatch, "user@corp.com")
assert [_json_login(client, "/v2/login", username="user@corp.com") for _ in range(3)] == [401, 401, 429]
refused = client.post("/v2/login", json={"username": "user@corp.com", "password": "right-db-password"})
assert refused.status_code == 429
assert refused.headers.get("retry-after") == "64"
reset_login_throttle()
assert _json_login(client, "/v2/login", username="user@corp.com", password="right-db-password") == 200
assert _json_login(client, "/v2/login", username="user@corp.com") == 429, "the block itself is still in force"
def test_sign_in_succeeds_again_once_the_block_is_cleared(client, monkeypatch, reset_login_throttle):

16
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -3450,7 +3450,8 @@ async def test_load_config_warns_that_the_source_login_limit_is_off_without_trus
tmp_path, monkeypatch, caplog
):
"""The per-source failed-login limit is skipped when the source cannot be attributed, and the
operator must be told so at startup; a configured range silences it."""
operator must be told so at startup. Both a configured range and an explicit empty list (no
proxies, the peer is the source) silence it, since both keep the limit on."""
import logging
from litellm.proxy.auth.login_throttle import warn_source_login_limit_is_off
@ -3465,12 +3466,13 @@ async def test_load_config_warns_that_the_source_login_limit_is_off_without_trus
await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file))
assert "trusted_proxy_ranges is not set" in caplog.text
caplog.clear()
warn_source_login_limit_is_off.cache_clear()
config_file.write_text("model_list: []\ngeneral_settings:\n trusted_proxy_ranges: ['10.0.0.0/8']\n")
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file))
assert "trusted_proxy_ranges is not set" not in caplog.text
for configured in ("['10.0.0.0/8']", "[]"):
caplog.clear()
warn_source_login_limit_is_off.cache_clear()
config_file.write_text(f"model_list: []\ngeneral_settings:\n trusted_proxy_ranges: {configured}\n")
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file))
assert "trusted_proxy_ranges is not set" not in caplog.text, configured
@pytest.mark.asyncio

6
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -26512,7 +26512,7 @@ export interface components {
enforce_fallback_model_access?: boolean | null;
/**
* Failed Login Block Seconds
* @description How long a blocked source address, or source address and username, stays blocked. The block is soft: a correct password still signs in, but each attempt from a blocked key takes one of 5 held slots per worker and a wrong password is held for 30 seconds before it is refused with 429. Set under `general_settings` in config.yaml. Defaults to 300
* @description How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300
*/
failed_login_block_seconds?: number | null;
/**
@ -26566,7 +26566,7 @@ export interface components {
max_batch_file_size_mb?: number | null;
/**
* Max Failed Login Attempts Per Source
* @description Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Only enforced when `trusted_proxy_ranges` is set, since otherwise every client behind an ingress shares one address. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10
* @description Failed Admin UI sign-in attempts allowed from one source address, across every username, within `failed_login_window_seconds`. One more blocks that address for `failed_login_block_seconds`. Only enforced when `trusted_proxy_ranges` is set: to the proxies in front of LiteLLM, or to an empty list when clients connect directly. Left unset, the peer address may be a shared ingress and this limit is off. IPv6 addresses are grouped by /64. Set under `general_settings` in config.yaml. Defaults to 10
*/
max_failed_login_attempts_per_source?: number | null;
/**
@ -26756,7 +26756,7 @@ export interface components {
supported_db_objects?: components["schemas"]["SupportedDBObjectType"][] | null;
/**
* Trusted Proxy Ranges
* @description CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler.
* @description CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler, and whose X-Forwarded-For is used to attribute Admin UI sign-in attempts to a source address. Set it to an empty list when clients connect directly, so the peer address is the source. Left unset, the per-source sign-in limit is off.
*/
trusted_proxy_ranges?: string[] | null;
/**