fix(proxy): validate budget fallbacks against every model budget scope and evict bulk user fallbacks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
jesus 2026-09-18 21:58:42 +00:00
parent 37f5b4323b
commit 93dc85b15f
4 changed files with 300 additions and 28 deletions

View file

@ -11,7 +11,7 @@ import asyncio
import fnmatch
import re
import secrets
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from datetime import datetime, timezone
from functools import partial
from typing import Any, Final, NamedTuple, Protocol, Union, cast
@ -318,6 +318,12 @@ class _TeamModelBudgetLimiter(Protocol):
) -> bool: ...
class _ModelBudgetLimiter(_KeyModelBudgetLimiter, _UserModelBudgetLimiter, _TeamModelBudgetLimiter, Protocol):
async def is_end_user_within_model_budget(
self, end_user_id: str, end_user_model_max_budget: Mapping[str, object], model: str
) -> bool: ...
class _TokenTeamModels(Protocol):
@property
def team_models(self) -> list[str]: ...
@ -368,7 +374,7 @@ async def _read_user_budget_row(
async def _check_user_model_budget(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _UserModelBudgetLimiter,
model_max_budget_limiter: _ModelBudgetLimiter,
models: list[str],
request_data: dict | None = None,
request: Request | None = None,
@ -392,6 +398,7 @@ async def _check_user_model_budget(
if request_data is not None and request is not None:
await _check_model_budget_with_fallback(
check=check,
model_max_budget_limiter=model_max_budget_limiter,
budget_fallbacks=valid_token.user_budget_fallbacks,
model_name=model_name,
request_data=request_data,
@ -406,7 +413,7 @@ async def _check_user_model_budget(
async def _check_team_model_budget(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _TeamModelBudgetLimiter,
model_max_budget_limiter: _ModelBudgetLimiter,
models: list[str],
request_data: dict | None = None,
request: Request | None = None,
@ -429,6 +436,7 @@ async def _check_team_model_budget(
if request_data is not None and request is not None:
await _check_model_budget_with_fallback(
check=check,
model_max_budget_limiter=model_max_budget_limiter,
budget_fallbacks=valid_token.team_budget_fallbacks,
model_name=model_name,
request_data=request_data,
@ -441,8 +449,39 @@ async def _check_team_model_budget(
await check(model=model_name)
def _iter_model_budget_scope_checks(
valid_token: UserAPIKeyAuth, model_max_budget_limiter: _ModelBudgetLimiter
) -> Iterator[Callable[..., Awaitable[object]]]:
key_budget: Final = valid_token.model_max_budget
if valid_token.token is not None and isinstance(key_budget, Mapping) and key_budget:
yield partial(model_max_budget_limiter.is_key_within_model_budget, user_api_key_dict=valid_token)
user_budget: Final = valid_token.user_model_max_budget
if valid_token.user_id is not None and isinstance(user_budget, Mapping) and user_budget:
yield partial(
model_max_budget_limiter.is_user_within_model_budget,
user_id=valid_token.user_id,
user_model_max_budget=user_budget,
)
end_user_budget: Final = valid_token.end_user_model_max_budget
if valid_token.end_user_id is not None and isinstance(end_user_budget, Mapping) and end_user_budget:
yield partial(
model_max_budget_limiter.is_end_user_within_model_budget,
end_user_id=valid_token.end_user_id,
end_user_model_max_budget=end_user_budget,
)
team_budget: Final = valid_token.team_model_max_budget
if valid_token.team_id is not None and isinstance(team_budget, Mapping) and team_budget:
yield partial(
model_max_budget_limiter.is_team_within_model_budget,
team_id=valid_token.team_id,
team_model_max_budget=team_budget,
key_model_max_budget=valid_token.model_max_budget,
)
async def _check_model_budget_with_fallback(
check: Callable[..., Awaitable[object]],
model_max_budget_limiter: _ModelBudgetLimiter,
budget_fallbacks: Mapping[str, Sequence[str]] | None,
model_name: str,
request_data: dict,
@ -454,8 +493,10 @@ async def _check_model_budget_with_fallback(
"""
Run `check` for `model_name`. If it exceeds its per-model budget and
`budget_fallbacks` has a chain for `model_name`, reroute the request to
the first fallback model still within its own budget instead of
rejecting the request.
the first fallback model still within budget instead of
rejecting the request. Every candidate fallback is validated against all
applicable per-model budget scopes (key, internal user, end user, team),
not only the scope that failed.
The selected fallback is validated against the key's model-access
allowlist and the team's model restrictions so that budget_fallbacks
@ -481,10 +522,18 @@ async def _check_model_budget_with_fallback(
original_error: Final = e
if request_data.get("model") != model_name:
raise original_error
scope_checks: Final = (check, *_iter_model_budget_scope_checks(valid_token, model_max_budget_limiter))
async def _fallback_within_every_scope(fallback_model: str) -> bool:
for scope_check in scope_checks:
try:
await scope_check(model=fallback_model)
except litellm.BudgetExceededError:
return False
return True
for fallback_model in budget_fallbacks.get(model_name, ()) if budget_fallbacks is not None else ():
try:
await check(model=fallback_model)
except litellm.BudgetExceededError:
if not await _fallback_within_every_scope(fallback_model):
continue
try:
await can_key_call_model(
@ -517,7 +566,7 @@ async def _check_model_budget_with_fallback(
async def _check_key_model_budget_with_fallback(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _KeyModelBudgetLimiter,
model_max_budget_limiter: _ModelBudgetLimiter,
model_name: str,
request_data: dict,
request: Request,
@ -533,6 +582,7 @@ async def _check_key_model_budget_with_fallback(
model_max_budget_limiter.is_key_within_model_budget,
user_api_key_dict=valid_token,
),
model_max_budget_limiter=model_max_budget_limiter,
budget_fallbacks=valid_token.budget_fallbacks,
model_name=model_name,
request_data=request_data,
@ -2406,14 +2456,19 @@ async def _user_api_key_auth_builder(
# Check 5. Token Model Spend is under Model budget
max_budget_per_model: Final = valid_token.model_max_budget
current_model = _get_model_from_request_context(
request_data=request_data,
route=route,
request=request,
llm_router=llm_router,
team_id=valid_token.team_id,
)
current_models = _get_model_names_for_budget_checks(model=current_model)
def _budget_models() -> list[str]:
return _get_model_names_for_budget_checks(
model=_get_model_from_request_context(
request_data=request_data,
route=route,
request=request,
llm_router=llm_router,
team_id=valid_token.team_id,
)
)
current_models = _budget_models()
if (
max_budget_per_model is not None
@ -2435,16 +2490,7 @@ async def _user_api_key_auth_builder(
llm_router=llm_router,
)
# Recompute after a potential budget-fallback rewrite so
# the end-user check below validates the final model
current_model = _get_model_from_request_context(
request_data=request_data,
route=route,
request=request,
llm_router=llm_router,
team_id=valid_token.team_id,
)
current_models = _get_model_names_for_budget_checks(model=current_model)
current_models = _budget_models() # rebind-ok: refresh after a budget-fallback rewrite
# Check 5a. Internal user model_max_budget
if current_models:
@ -2457,6 +2503,7 @@ async def _user_api_key_auth_builder(
llm_model_list=llm_model_list,
llm_router=llm_router,
)
current_models = _budget_models() # rebind-ok: refresh after a budget-fallback rewrite
# Check 5b. End-user model max budget
end_user_mmb: Final = valid_token.end_user_model_max_budget

View file

@ -1929,7 +1929,7 @@ async def bulk_user_update(
),
)
if "model_max_budget" in non_default_values:
if "model_max_budget" in non_default_values or "budget_fallbacks" in non_default_values:
for start in range(0, len(all_users_in_db), _USER_BUDGET_CACHE_INVALIDATION_BATCH_SIZE):
await asyncio.gather(
*(

View file

@ -7708,6 +7708,191 @@ class TestEntityModelBudgetFallbacks:
assert request_data["model"] == "claude-haiku"
@pytest.mark.asyncio
async def test_team_fallback_over_key_budget_raises_original_error(self):
"""A team fallback must still satisfy the key's per-model budget;
without that check a team chain could reroute to a model the key
already spent out on."""
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
async def is_team_within_model_budget(
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
):
if model == "gpt-4o-mini":
return True
raise original_error
async def is_key_within_model_budget(user_api_key_dict=None, model=""):
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
limiter = MagicMock()
limiter.is_team_within_model_budget = is_team_within_model_budget
limiter.is_key_within_model_budget = is_key_within_model_budget
valid_token = UserAPIKeyAuth(
token="sk-test",
model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
team_id="team-1",
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
)
request_data = {"model": "gpt-4o"}
request = self._make_request()
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
new=AsyncMock(return_value=True),
):
with pytest.raises(litellm.BudgetExceededError) as exc_info:
await _check_team_model_budget(
valid_token=valid_token,
model_max_budget_limiter=limiter,
models=["gpt-4o"],
request_data=request_data,
request=request,
)
assert exc_info.value is original_error
assert request_data["model"] == "gpt-4o"
@pytest.mark.asyncio
async def test_team_fallback_over_user_budget_skips_to_next_fallback(self):
"""A fallback over the internal user's budget must be skipped so the
next in-scope fallback in the chain is used."""
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
async def is_team_within_model_budget(
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
):
if model in ("gpt-4o-mini", "claude-haiku"):
return True
raise litellm.BudgetExceededError(current_cost=10, max_budget=5)
async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""):
if model == "claude-haiku":
return True
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
limiter = MagicMock()
limiter.is_team_within_model_budget = is_team_within_model_budget
limiter.is_user_within_model_budget = is_user_within_model_budget
valid_token = UserAPIKeyAuth(
user_id="user-1",
user_model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
team_id="team-1",
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]},
)
request_data = {"model": "gpt-4o"}
request = self._make_request()
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
new=AsyncMock(return_value=True),
):
await _check_team_model_budget(
valid_token=valid_token,
model_max_budget_limiter=limiter,
models=["gpt-4o"],
request_data=request_data,
request=request,
)
assert request_data["model"] == "claude-haiku"
@pytest.mark.asyncio
async def test_user_fallback_over_end_user_budget_raises_original_error(self):
"""A user-level fallback must still satisfy the end user's per-model
budget; without that check it reroutes to a model the end user
already spent out on."""
from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""):
if model == "gpt-4o-mini":
return True
raise original_error
async def is_end_user_within_model_budget(end_user_id=None, end_user_model_max_budget=None, model=""):
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
limiter = MagicMock()
limiter.is_user_within_model_budget = is_user_within_model_budget
limiter.is_end_user_within_model_budget = is_end_user_within_model_budget
valid_token = UserAPIKeyAuth(
user_id="user-1",
user_model_max_budget={"gpt-4o": {"max_budget": 5}},
user_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
end_user_id="eu-1",
end_user_model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
)
request_data = {"model": "gpt-4o"}
request = self._make_request()
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
new=AsyncMock(return_value=True),
):
with pytest.raises(litellm.BudgetExceededError) as exc_info:
await _check_user_model_budget(
valid_token=valid_token,
model_max_budget_limiter=limiter,
models=["gpt-4o"],
request_data=request_data,
request=request,
)
assert exc_info.value is original_error
assert request_data["model"] == "gpt-4o"
@pytest.mark.asyncio
async def test_key_fallback_over_team_budget_raises_original_error(self):
"""A key-level fallback must still satisfy the team's per-model
budget; without that check a key chain could reroute to a model the
team already spent out on."""
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
async def is_key_within_model_budget(user_api_key_dict=None, model=""):
if model == "gpt-4o-mini":
return True
raise original_error
async def is_team_within_model_budget(
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
):
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
limiter = MagicMock()
limiter.is_key_within_model_budget = is_key_within_model_budget
limiter.is_team_within_model_budget = is_team_within_model_budget
valid_token = UserAPIKeyAuth(
token="sk-test",
model_max_budget={"gpt-4o": {"max_budget": 5}},
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
team_id="team-1",
team_model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
)
request_data = {"model": "gpt-4o"}
request = self._make_request()
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
new=AsyncMock(return_value=True),
):
with pytest.raises(litellm.BudgetExceededError) as exc_info:
await _check_key_model_budget_with_fallback(
valid_token=valid_token,
model_max_budget_limiter=limiter,
model_name="gpt-4o",
request_data=request_data,
request=request,
)
assert exc_info.value is original_error
assert request_data["model"] == "gpt-4o"
@pytest.mark.asyncio
async def test_global_proxy_spend_reads_resettable_proxy_budget_row():

View file

@ -2272,6 +2272,46 @@ async def test_bulk_user_model_budget_clear_serializes_and_refreshes_cache(mocke
broadcast.assert_awaited_once_with(cache_key=saved_user.user_id)
@pytest.mark.asyncio
async def test_bulk_user_budget_fallbacks_update_serializes_and_refreshes_cache(mocker: MockerFixture) -> None:
from litellm.proxy._types import LiteLLM_UserTable
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update
from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkUpdateUserRequest
saved_user: Final = LiteLLM_UserTable(
user_id="user-spruce", budget_fallbacks={"gpt-4o": ["claude-haiku"]}
)
prisma_client: Final = mocker.MagicMock()
prisma_client.db.litellm_usertable.find_many = mocker.AsyncMock(return_value=[saved_user])
prisma_client.db.litellm_usertable.update_many = mocker.AsyncMock(return_value=1)
mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) # test-quality-ok: substitute the database dependency
cache: Final = UserApiKeyCache()
await cache.async_set_cache(key=saved_user.user_id, value=saved_user, model_type=LiteLLM_UserTable)
mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) # test-quality-ok: exercise a real isolated cache
broadcast: Final = mocker.patch( # test-quality-ok: observe the Redis publication boundary
"litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation",
new_callable=mocker.AsyncMock,
)
response: Final = await bulk_user_update(
data=BulkUpdateUserRequest(
all_users=True, user_updates={"budget_fallbacks": {"gpt-4o": ["gpt-4o-mini"]}}
),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-spruce", user_role=LitellmUserRoles.PROXY_ADMIN),
litellm_changed_by=None,
)
prisma_client.db.litellm_usertable.update_many.assert_awaited_once_with(
where={}, data={"budget_fallbacks": json.dumps({"gpt-4o": ["gpt-4o-mini"]})}
)
prisma_client.update_data.assert_not_called()
assert response.successful_updates == 1
assert response.results[0].updated_user["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]}
assert await cache.async_get_cache(key=saved_user.user_id, model_type=LiteLLM_UserTable) is None
broadcast.assert_awaited_once_with(cache_key=saved_user.user_id)
def test_generate_request_base_validator():
"""
Test that GenerateRequestBase validator converts empty string to None for max_budget