From 93dc85b15fade879b625d8a70ea9d54403a384e6 Mon Sep 17 00:00:00 2001 From: jesus Date: Fri, 18 Sep 2026 21:58:42 +0000 Subject: [PATCH] fix(proxy): validate budget fallbacks against every model budget scope and evict bulk user fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/user_api_key_auth.py | 101 +++++++--- .../internal_user_endpoints.py | 2 +- .../proxy/auth/test_user_api_key_auth.py | 185 ++++++++++++++++++ .../test_internal_user_endpoints.py | 40 ++++ 4 files changed, 300 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 0b42fa6d08b..4efcdb13d13 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -11,7 +11,7 @@ import asyncio import fnmatch import re import secrets -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from datetime import datetime, timezone from functools import partial from typing import Any, Final, NamedTuple, Protocol, Union, cast @@ -318,6 +318,12 @@ class _TeamModelBudgetLimiter(Protocol): ) -> bool: ... +class _ModelBudgetLimiter(_KeyModelBudgetLimiter, _UserModelBudgetLimiter, _TeamModelBudgetLimiter, Protocol): + async def is_end_user_within_model_budget( + self, end_user_id: str, end_user_model_max_budget: Mapping[str, object], model: str + ) -> bool: ... + + class _TokenTeamModels(Protocol): @property def team_models(self) -> list[str]: ... @@ -368,7 +374,7 @@ async def _read_user_budget_row( async def _check_user_model_budget( valid_token: UserAPIKeyAuth, - model_max_budget_limiter: _UserModelBudgetLimiter, + model_max_budget_limiter: _ModelBudgetLimiter, models: list[str], request_data: dict | None = None, request: Request | None = None, @@ -392,6 +398,7 @@ async def _check_user_model_budget( if request_data is not None and request is not None: await _check_model_budget_with_fallback( check=check, + model_max_budget_limiter=model_max_budget_limiter, budget_fallbacks=valid_token.user_budget_fallbacks, model_name=model_name, request_data=request_data, @@ -406,7 +413,7 @@ async def _check_user_model_budget( async def _check_team_model_budget( valid_token: UserAPIKeyAuth, - model_max_budget_limiter: _TeamModelBudgetLimiter, + model_max_budget_limiter: _ModelBudgetLimiter, models: list[str], request_data: dict | None = None, request: Request | None = None, @@ -429,6 +436,7 @@ async def _check_team_model_budget( if request_data is not None and request is not None: await _check_model_budget_with_fallback( check=check, + model_max_budget_limiter=model_max_budget_limiter, budget_fallbacks=valid_token.team_budget_fallbacks, model_name=model_name, request_data=request_data, @@ -441,8 +449,39 @@ async def _check_team_model_budget( await check(model=model_name) +def _iter_model_budget_scope_checks( + valid_token: UserAPIKeyAuth, model_max_budget_limiter: _ModelBudgetLimiter +) -> Iterator[Callable[..., Awaitable[object]]]: + key_budget: Final = valid_token.model_max_budget + if valid_token.token is not None and isinstance(key_budget, Mapping) and key_budget: + yield partial(model_max_budget_limiter.is_key_within_model_budget, user_api_key_dict=valid_token) + user_budget: Final = valid_token.user_model_max_budget + if valid_token.user_id is not None and isinstance(user_budget, Mapping) and user_budget: + yield partial( + model_max_budget_limiter.is_user_within_model_budget, + user_id=valid_token.user_id, + user_model_max_budget=user_budget, + ) + end_user_budget: Final = valid_token.end_user_model_max_budget + if valid_token.end_user_id is not None and isinstance(end_user_budget, Mapping) and end_user_budget: + yield partial( + model_max_budget_limiter.is_end_user_within_model_budget, + end_user_id=valid_token.end_user_id, + end_user_model_max_budget=end_user_budget, + ) + team_budget: Final = valid_token.team_model_max_budget + if valid_token.team_id is not None and isinstance(team_budget, Mapping) and team_budget: + yield partial( + model_max_budget_limiter.is_team_within_model_budget, + team_id=valid_token.team_id, + team_model_max_budget=team_budget, + key_model_max_budget=valid_token.model_max_budget, + ) + + async def _check_model_budget_with_fallback( check: Callable[..., Awaitable[object]], + model_max_budget_limiter: _ModelBudgetLimiter, budget_fallbacks: Mapping[str, Sequence[str]] | None, model_name: str, request_data: dict, @@ -454,8 +493,10 @@ async def _check_model_budget_with_fallback( """ Run `check` for `model_name`. If it exceeds its per-model budget and `budget_fallbacks` has a chain for `model_name`, reroute the request to - the first fallback model still within its own budget instead of - rejecting the request. + the first fallback model still within budget instead of + rejecting the request. Every candidate fallback is validated against all + applicable per-model budget scopes (key, internal user, end user, team), + not only the scope that failed. The selected fallback is validated against the key's model-access allowlist and the team's model restrictions so that budget_fallbacks @@ -481,10 +522,18 @@ async def _check_model_budget_with_fallback( original_error: Final = e if request_data.get("model") != model_name: raise original_error + scope_checks: Final = (check, *_iter_model_budget_scope_checks(valid_token, model_max_budget_limiter)) + + async def _fallback_within_every_scope(fallback_model: str) -> bool: + for scope_check in scope_checks: + try: + await scope_check(model=fallback_model) + except litellm.BudgetExceededError: + return False + return True + for fallback_model in budget_fallbacks.get(model_name, ()) if budget_fallbacks is not None else (): - try: - await check(model=fallback_model) - except litellm.BudgetExceededError: + if not await _fallback_within_every_scope(fallback_model): continue try: await can_key_call_model( @@ -517,7 +566,7 @@ async def _check_model_budget_with_fallback( async def _check_key_model_budget_with_fallback( valid_token: UserAPIKeyAuth, - model_max_budget_limiter: _KeyModelBudgetLimiter, + model_max_budget_limiter: _ModelBudgetLimiter, model_name: str, request_data: dict, request: Request, @@ -533,6 +582,7 @@ async def _check_key_model_budget_with_fallback( model_max_budget_limiter.is_key_within_model_budget, user_api_key_dict=valid_token, ), + model_max_budget_limiter=model_max_budget_limiter, budget_fallbacks=valid_token.budget_fallbacks, model_name=model_name, request_data=request_data, @@ -2406,14 +2456,19 @@ async def _user_api_key_auth_builder( # Check 5. Token Model Spend is under Model budget max_budget_per_model: Final = valid_token.model_max_budget - current_model = _get_model_from_request_context( - request_data=request_data, - route=route, - request=request, - llm_router=llm_router, - team_id=valid_token.team_id, - ) - current_models = _get_model_names_for_budget_checks(model=current_model) + + def _budget_models() -> list[str]: + return _get_model_names_for_budget_checks( + model=_get_model_from_request_context( + request_data=request_data, + route=route, + request=request, + llm_router=llm_router, + team_id=valid_token.team_id, + ) + ) + + current_models = _budget_models() if ( max_budget_per_model is not None @@ -2435,16 +2490,7 @@ async def _user_api_key_auth_builder( llm_router=llm_router, ) - # Recompute after a potential budget-fallback rewrite so - # the end-user check below validates the final model - current_model = _get_model_from_request_context( - request_data=request_data, - route=route, - request=request, - llm_router=llm_router, - team_id=valid_token.team_id, - ) - current_models = _get_model_names_for_budget_checks(model=current_model) + current_models = _budget_models() # rebind-ok: refresh after a budget-fallback rewrite # Check 5a. Internal user model_max_budget if current_models: @@ -2457,6 +2503,7 @@ async def _user_api_key_auth_builder( llm_model_list=llm_model_list, llm_router=llm_router, ) + current_models = _budget_models() # rebind-ok: refresh after a budget-fallback rewrite # Check 5b. End-user model max budget end_user_mmb: Final = valid_token.end_user_model_max_budget diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index a442db6ee1d..ec5fbbc9228 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1929,7 +1929,7 @@ async def bulk_user_update( ), ) - if "model_max_budget" in non_default_values: + if "model_max_budget" in non_default_values or "budget_fallbacks" in non_default_values: for start in range(0, len(all_users_in_db), _USER_BUDGET_CACHE_INVALIDATION_BATCH_SIZE): await asyncio.gather( *( diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 76a3d8a47f2..100044d9960 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -7708,6 +7708,191 @@ class TestEntityModelBudgetFallbacks: assert request_data["model"] == "claude-haiku" + @pytest.mark.asyncio + async def test_team_fallback_over_key_budget_raises_original_error(self): + """A team fallback must still satisfy the key's per-model budget; + without that check a team chain could reroute to a model the key + already spent out on.""" + from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget + + original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def is_team_within_model_budget( + team_id=None, team_model_max_budget=None, key_model_max_budget=None, model="" + ): + if model == "gpt-4o-mini": + return True + raise original_error + + async def is_key_within_model_budget(user_api_key_dict=None, model=""): + raise litellm.BudgetExceededError(current_cost=10, max_budget=1) + + limiter = MagicMock() + limiter.is_team_within_model_budget = is_team_within_model_budget + limiter.is_key_within_model_budget = is_key_within_model_budget + valid_token = UserAPIKeyAuth( + token="sk-test", + model_max_budget={"gpt-4o-mini": {"max_budget": 1}}, + team_id="team-1", + team_model_max_budget={"gpt-4o": {"max_budget": 5}}, + team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_model_budget( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert exc_info.value is original_error + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_team_fallback_over_user_budget_skips_to_next_fallback(self): + """A fallback over the internal user's budget must be skipped so the + next in-scope fallback in the chain is used.""" + from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget + + async def is_team_within_model_budget( + team_id=None, team_model_max_budget=None, key_model_max_budget=None, model="" + ): + if model in ("gpt-4o-mini", "claude-haiku"): + return True + raise litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""): + if model == "claude-haiku": + return True + raise litellm.BudgetExceededError(current_cost=10, max_budget=1) + + limiter = MagicMock() + limiter.is_team_within_model_budget = is_team_within_model_budget + limiter.is_user_within_model_budget = is_user_within_model_budget + valid_token = UserAPIKeyAuth( + user_id="user-1", + user_model_max_budget={"gpt-4o-mini": {"max_budget": 1}}, + team_id="team-1", + team_model_max_budget={"gpt-4o": {"max_budget": 5}}, + team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + await _check_team_model_budget( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert request_data["model"] == "claude-haiku" + + @pytest.mark.asyncio + async def test_user_fallback_over_end_user_budget_raises_original_error(self): + """A user-level fallback must still satisfy the end user's per-model + budget; without that check it reroutes to a model the end user + already spent out on.""" + from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget + + original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""): + if model == "gpt-4o-mini": + return True + raise original_error + + async def is_end_user_within_model_budget(end_user_id=None, end_user_model_max_budget=None, model=""): + raise litellm.BudgetExceededError(current_cost=10, max_budget=1) + + limiter = MagicMock() + limiter.is_user_within_model_budget = is_user_within_model_budget + limiter.is_end_user_within_model_budget = is_end_user_within_model_budget + valid_token = UserAPIKeyAuth( + user_id="user-1", + user_model_max_budget={"gpt-4o": {"max_budget": 5}}, + user_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + end_user_id="eu-1", + end_user_model_max_budget={"gpt-4o-mini": {"max_budget": 1}}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_user_model_budget( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert exc_info.value is original_error + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_key_fallback_over_team_budget_raises_original_error(self): + """A key-level fallback must still satisfy the team's per-model + budget; without that check a key chain could reroute to a model the + team already spent out on.""" + original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def is_key_within_model_budget(user_api_key_dict=None, model=""): + if model == "gpt-4o-mini": + return True + raise original_error + + async def is_team_within_model_budget( + team_id=None, team_model_max_budget=None, key_model_max_budget=None, model="" + ): + raise litellm.BudgetExceededError(current_cost=10, max_budget=1) + + limiter = MagicMock() + limiter.is_key_within_model_budget = is_key_within_model_budget + limiter.is_team_within_model_budget = is_team_within_model_budget + valid_token = UserAPIKeyAuth( + token="sk-test", + model_max_budget={"gpt-4o": {"max_budget": 5}}, + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + team_id="team-1", + team_model_max_budget={"gpt-4o-mini": {"max_budget": 1}}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_key_model_budget_with_fallback( + valid_token=valid_token, + model_max_budget_limiter=limiter, + model_name="gpt-4o", + request_data=request_data, + request=request, + ) + + assert exc_info.value is original_error + assert request_data["model"] == "gpt-4o" + @pytest.mark.asyncio async def test_global_proxy_spend_reads_resettable_proxy_budget_row(): diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 08e89980742..a305729849c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -2272,6 +2272,46 @@ async def test_bulk_user_model_budget_clear_serializes_and_refreshes_cache(mocke broadcast.assert_awaited_once_with(cache_key=saved_user.user_id) +@pytest.mark.asyncio +async def test_bulk_user_budget_fallbacks_update_serializes_and_refreshes_cache(mocker: MockerFixture) -> None: + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update + from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkUpdateUserRequest + + saved_user: Final = LiteLLM_UserTable( + user_id="user-spruce", budget_fallbacks={"gpt-4o": ["claude-haiku"]} + ) + prisma_client: Final = mocker.MagicMock() + prisma_client.db.litellm_usertable.find_many = mocker.AsyncMock(return_value=[saved_user]) + prisma_client.db.litellm_usertable.update_many = mocker.AsyncMock(return_value=1) + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) # test-quality-ok: substitute the database dependency + cache: Final = UserApiKeyCache() + await cache.async_set_cache(key=saved_user.user_id, value=saved_user, model_type=LiteLLM_UserTable) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) # test-quality-ok: exercise a real isolated cache + broadcast: Final = mocker.patch( # test-quality-ok: observe the Redis publication boundary + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation", + new_callable=mocker.AsyncMock, + ) + + response: Final = await bulk_user_update( + data=BulkUpdateUserRequest( + all_users=True, user_updates={"budget_fallbacks": {"gpt-4o": ["gpt-4o-mini"]}} + ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-spruce", user_role=LitellmUserRoles.PROXY_ADMIN), + litellm_changed_by=None, + ) + + prisma_client.db.litellm_usertable.update_many.assert_awaited_once_with( + where={}, data={"budget_fallbacks": json.dumps({"gpt-4o": ["gpt-4o-mini"]})} + ) + prisma_client.update_data.assert_not_called() + assert response.successful_updates == 1 + assert response.results[0].updated_user["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]} + assert await cache.async_get_cache(key=saved_user.user_id, model_type=LiteLLM_UserTable) is None + broadcast.assert_awaited_once_with(cache_key=saved_user.user_id) + + def test_generate_request_base_validator(): """ Test that GenerateRequestBase validator converts empty string to None for max_budget