mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): validate budget fallbacks against every model budget scope and evict bulk user fallbacks
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
37f5b4323b
commit
93dc85b15f
4 changed files with 300 additions and 28 deletions
|
|
@ -11,7 +11,7 @@ import asyncio
|
|||
import fnmatch
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from functools import partial
|
||||
from typing import Any, Final, NamedTuple, Protocol, Union, cast
|
||||
|
|
@ -318,6 +318,12 @@ class _TeamModelBudgetLimiter(Protocol):
|
|||
) -> bool: ...
|
||||
|
||||
|
||||
class _ModelBudgetLimiter(_KeyModelBudgetLimiter, _UserModelBudgetLimiter, _TeamModelBudgetLimiter, Protocol):
|
||||
async def is_end_user_within_model_budget(
|
||||
self, end_user_id: str, end_user_model_max_budget: Mapping[str, object], model: str
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
class _TokenTeamModels(Protocol):
|
||||
@property
|
||||
def team_models(self) -> list[str]: ...
|
||||
|
|
@ -368,7 +374,7 @@ async def _read_user_budget_row(
|
|||
|
||||
async def _check_user_model_budget(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _UserModelBudgetLimiter,
|
||||
model_max_budget_limiter: _ModelBudgetLimiter,
|
||||
models: list[str],
|
||||
request_data: dict | None = None,
|
||||
request: Request | None = None,
|
||||
|
|
@ -392,6 +398,7 @@ async def _check_user_model_budget(
|
|||
if request_data is not None and request is not None:
|
||||
await _check_model_budget_with_fallback(
|
||||
check=check,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
budget_fallbacks=valid_token.user_budget_fallbacks,
|
||||
model_name=model_name,
|
||||
request_data=request_data,
|
||||
|
|
@ -406,7 +413,7 @@ async def _check_user_model_budget(
|
|||
|
||||
async def _check_team_model_budget(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _TeamModelBudgetLimiter,
|
||||
model_max_budget_limiter: _ModelBudgetLimiter,
|
||||
models: list[str],
|
||||
request_data: dict | None = None,
|
||||
request: Request | None = None,
|
||||
|
|
@ -429,6 +436,7 @@ async def _check_team_model_budget(
|
|||
if request_data is not None and request is not None:
|
||||
await _check_model_budget_with_fallback(
|
||||
check=check,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
budget_fallbacks=valid_token.team_budget_fallbacks,
|
||||
model_name=model_name,
|
||||
request_data=request_data,
|
||||
|
|
@ -441,8 +449,39 @@ async def _check_team_model_budget(
|
|||
await check(model=model_name)
|
||||
|
||||
|
||||
def _iter_model_budget_scope_checks(
|
||||
valid_token: UserAPIKeyAuth, model_max_budget_limiter: _ModelBudgetLimiter
|
||||
) -> Iterator[Callable[..., Awaitable[object]]]:
|
||||
key_budget: Final = valid_token.model_max_budget
|
||||
if valid_token.token is not None and isinstance(key_budget, Mapping) and key_budget:
|
||||
yield partial(model_max_budget_limiter.is_key_within_model_budget, user_api_key_dict=valid_token)
|
||||
user_budget: Final = valid_token.user_model_max_budget
|
||||
if valid_token.user_id is not None and isinstance(user_budget, Mapping) and user_budget:
|
||||
yield partial(
|
||||
model_max_budget_limiter.is_user_within_model_budget,
|
||||
user_id=valid_token.user_id,
|
||||
user_model_max_budget=user_budget,
|
||||
)
|
||||
end_user_budget: Final = valid_token.end_user_model_max_budget
|
||||
if valid_token.end_user_id is not None and isinstance(end_user_budget, Mapping) and end_user_budget:
|
||||
yield partial(
|
||||
model_max_budget_limiter.is_end_user_within_model_budget,
|
||||
end_user_id=valid_token.end_user_id,
|
||||
end_user_model_max_budget=end_user_budget,
|
||||
)
|
||||
team_budget: Final = valid_token.team_model_max_budget
|
||||
if valid_token.team_id is not None and isinstance(team_budget, Mapping) and team_budget:
|
||||
yield partial(
|
||||
model_max_budget_limiter.is_team_within_model_budget,
|
||||
team_id=valid_token.team_id,
|
||||
team_model_max_budget=team_budget,
|
||||
key_model_max_budget=valid_token.model_max_budget,
|
||||
)
|
||||
|
||||
|
||||
async def _check_model_budget_with_fallback(
|
||||
check: Callable[..., Awaitable[object]],
|
||||
model_max_budget_limiter: _ModelBudgetLimiter,
|
||||
budget_fallbacks: Mapping[str, Sequence[str]] | None,
|
||||
model_name: str,
|
||||
request_data: dict,
|
||||
|
|
@ -454,8 +493,10 @@ async def _check_model_budget_with_fallback(
|
|||
"""
|
||||
Run `check` for `model_name`. If it exceeds its per-model budget and
|
||||
`budget_fallbacks` has a chain for `model_name`, reroute the request to
|
||||
the first fallback model still within its own budget instead of
|
||||
rejecting the request.
|
||||
the first fallback model still within budget instead of
|
||||
rejecting the request. Every candidate fallback is validated against all
|
||||
applicable per-model budget scopes (key, internal user, end user, team),
|
||||
not only the scope that failed.
|
||||
|
||||
The selected fallback is validated against the key's model-access
|
||||
allowlist and the team's model restrictions so that budget_fallbacks
|
||||
|
|
@ -481,10 +522,18 @@ async def _check_model_budget_with_fallback(
|
|||
original_error: Final = e
|
||||
if request_data.get("model") != model_name:
|
||||
raise original_error
|
||||
scope_checks: Final = (check, *_iter_model_budget_scope_checks(valid_token, model_max_budget_limiter))
|
||||
|
||||
async def _fallback_within_every_scope(fallback_model: str) -> bool:
|
||||
for scope_check in scope_checks:
|
||||
try:
|
||||
await scope_check(model=fallback_model)
|
||||
except litellm.BudgetExceededError:
|
||||
return False
|
||||
return True
|
||||
|
||||
for fallback_model in budget_fallbacks.get(model_name, ()) if budget_fallbacks is not None else ():
|
||||
try:
|
||||
await check(model=fallback_model)
|
||||
except litellm.BudgetExceededError:
|
||||
if not await _fallback_within_every_scope(fallback_model):
|
||||
continue
|
||||
try:
|
||||
await can_key_call_model(
|
||||
|
|
@ -517,7 +566,7 @@ async def _check_model_budget_with_fallback(
|
|||
|
||||
async def _check_key_model_budget_with_fallback(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _KeyModelBudgetLimiter,
|
||||
model_max_budget_limiter: _ModelBudgetLimiter,
|
||||
model_name: str,
|
||||
request_data: dict,
|
||||
request: Request,
|
||||
|
|
@ -533,6 +582,7 @@ async def _check_key_model_budget_with_fallback(
|
|||
model_max_budget_limiter.is_key_within_model_budget,
|
||||
user_api_key_dict=valid_token,
|
||||
),
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
budget_fallbacks=valid_token.budget_fallbacks,
|
||||
model_name=model_name,
|
||||
request_data=request_data,
|
||||
|
|
@ -2406,14 +2456,19 @@ async def _user_api_key_auth_builder(
|
|||
|
||||
# Check 5. Token Model Spend is under Model budget
|
||||
max_budget_per_model: Final = valid_token.model_max_budget
|
||||
current_model = _get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
request=request,
|
||||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
current_models = _get_model_names_for_budget_checks(model=current_model)
|
||||
|
||||
def _budget_models() -> list[str]:
|
||||
return _get_model_names_for_budget_checks(
|
||||
model=_get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
request=request,
|
||||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
)
|
||||
|
||||
current_models = _budget_models()
|
||||
|
||||
if (
|
||||
max_budget_per_model is not None
|
||||
|
|
@ -2435,16 +2490,7 @@ async def _user_api_key_auth_builder(
|
|||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
# Recompute after a potential budget-fallback rewrite so
|
||||
# the end-user check below validates the final model
|
||||
current_model = _get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
request=request,
|
||||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
current_models = _get_model_names_for_budget_checks(model=current_model)
|
||||
current_models = _budget_models() # rebind-ok: refresh after a budget-fallback rewrite
|
||||
|
||||
# Check 5a. Internal user model_max_budget
|
||||
if current_models:
|
||||
|
|
@ -2457,6 +2503,7 @@ async def _user_api_key_auth_builder(
|
|||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
current_models = _budget_models() # rebind-ok: refresh after a budget-fallback rewrite
|
||||
|
||||
# Check 5b. End-user model max budget
|
||||
end_user_mmb: Final = valid_token.end_user_model_max_budget
|
||||
|
|
|
|||
|
|
@ -1929,7 +1929,7 @@ async def bulk_user_update(
|
|||
),
|
||||
)
|
||||
|
||||
if "model_max_budget" in non_default_values:
|
||||
if "model_max_budget" in non_default_values or "budget_fallbacks" in non_default_values:
|
||||
for start in range(0, len(all_users_in_db), _USER_BUDGET_CACHE_INVALIDATION_BATCH_SIZE):
|
||||
await asyncio.gather(
|
||||
*(
|
||||
|
|
|
|||
|
|
@ -7708,6 +7708,191 @@ class TestEntityModelBudgetFallbacks:
|
|||
|
||||
assert request_data["model"] == "claude-haiku"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_fallback_over_key_budget_raises_original_error(self):
|
||||
"""A team fallback must still satisfy the key's per-model budget;
|
||||
without that check a team chain could reroute to a model the key
|
||||
already spent out on."""
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
|
||||
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def is_team_within_model_budget(
|
||||
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
|
||||
):
|
||||
if model == "gpt-4o-mini":
|
||||
return True
|
||||
raise original_error
|
||||
|
||||
async def is_key_within_model_budget(user_api_key_dict=None, model=""):
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_team_within_model_budget = is_team_within_model_budget
|
||||
limiter.is_key_within_model_budget = is_key_within_model_budget
|
||||
valid_token = UserAPIKeyAuth(
|
||||
token="sk-test",
|
||||
model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
|
||||
team_id="team-1",
|
||||
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
with pytest.raises(litellm.BudgetExceededError) as exc_info:
|
||||
await _check_team_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert exc_info.value is original_error
|
||||
assert request_data["model"] == "gpt-4o"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_fallback_over_user_budget_skips_to_next_fallback(self):
|
||||
"""A fallback over the internal user's budget must be skipped so the
|
||||
next in-scope fallback in the chain is used."""
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
|
||||
|
||||
async def is_team_within_model_budget(
|
||||
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
|
||||
):
|
||||
if model in ("gpt-4o-mini", "claude-haiku"):
|
||||
return True
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""):
|
||||
if model == "claude-haiku":
|
||||
return True
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_team_within_model_budget = is_team_within_model_budget
|
||||
limiter.is_user_within_model_budget = is_user_within_model_budget
|
||||
valid_token = UserAPIKeyAuth(
|
||||
user_id="user-1",
|
||||
user_model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
|
||||
team_id="team-1",
|
||||
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
await _check_team_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert request_data["model"] == "claude-haiku"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_fallback_over_end_user_budget_raises_original_error(self):
|
||||
"""A user-level fallback must still satisfy the end user's per-model
|
||||
budget; without that check it reroutes to a model the end user
|
||||
already spent out on."""
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget
|
||||
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""):
|
||||
if model == "gpt-4o-mini":
|
||||
return True
|
||||
raise original_error
|
||||
|
||||
async def is_end_user_within_model_budget(end_user_id=None, end_user_model_max_budget=None, model=""):
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_user_within_model_budget = is_user_within_model_budget
|
||||
limiter.is_end_user_within_model_budget = is_end_user_within_model_budget
|
||||
valid_token = UserAPIKeyAuth(
|
||||
user_id="user-1",
|
||||
user_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
user_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
end_user_id="eu-1",
|
||||
end_user_model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
with pytest.raises(litellm.BudgetExceededError) as exc_info:
|
||||
await _check_user_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert exc_info.value is original_error
|
||||
assert request_data["model"] == "gpt-4o"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_fallback_over_team_budget_raises_original_error(self):
|
||||
"""A key-level fallback must still satisfy the team's per-model
|
||||
budget; without that check a key chain could reroute to a model the
|
||||
team already spent out on."""
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def is_key_within_model_budget(user_api_key_dict=None, model=""):
|
||||
if model == "gpt-4o-mini":
|
||||
return True
|
||||
raise original_error
|
||||
|
||||
async def is_team_within_model_budget(
|
||||
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
|
||||
):
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=1)
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_key_within_model_budget = is_key_within_model_budget
|
||||
limiter.is_team_within_model_budget = is_team_within_model_budget
|
||||
valid_token = UserAPIKeyAuth(
|
||||
token="sk-test",
|
||||
model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
team_id="team-1",
|
||||
team_model_max_budget={"gpt-4o-mini": {"max_budget": 1}},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch( # test-quality-ok: the fallback check reads the limiter seam from module scope; no injection point
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
with pytest.raises(litellm.BudgetExceededError) as exc_info:
|
||||
await _check_key_model_budget_with_fallback(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
model_name="gpt-4o",
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert exc_info.value is original_error
|
||||
assert request_data["model"] == "gpt-4o"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_proxy_spend_reads_resettable_proxy_budget_row():
|
||||
|
|
|
|||
|
|
@ -2272,6 +2272,46 @@ async def test_bulk_user_model_budget_clear_serializes_and_refreshes_cache(mocke
|
|||
broadcast.assert_awaited_once_with(cache_key=saved_user.user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_user_budget_fallbacks_update_serializes_and_refreshes_cache(mocker: MockerFixture) -> None:
|
||||
from litellm.proxy._types import LiteLLM_UserTable
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update
|
||||
from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkUpdateUserRequest
|
||||
|
||||
saved_user: Final = LiteLLM_UserTable(
|
||||
user_id="user-spruce", budget_fallbacks={"gpt-4o": ["claude-haiku"]}
|
||||
)
|
||||
prisma_client: Final = mocker.MagicMock()
|
||||
prisma_client.db.litellm_usertable.find_many = mocker.AsyncMock(return_value=[saved_user])
|
||||
prisma_client.db.litellm_usertable.update_many = mocker.AsyncMock(return_value=1)
|
||||
mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) # test-quality-ok: substitute the database dependency
|
||||
cache: Final = UserApiKeyCache()
|
||||
await cache.async_set_cache(key=saved_user.user_id, value=saved_user, model_type=LiteLLM_UserTable)
|
||||
mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) # test-quality-ok: exercise a real isolated cache
|
||||
broadcast: Final = mocker.patch( # test-quality-ok: observe the Redis publication boundary
|
||||
"litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation",
|
||||
new_callable=mocker.AsyncMock,
|
||||
)
|
||||
|
||||
response: Final = await bulk_user_update(
|
||||
data=BulkUpdateUserRequest(
|
||||
all_users=True, user_updates={"budget_fallbacks": {"gpt-4o": ["gpt-4o-mini"]}}
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-spruce", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
litellm_changed_by=None,
|
||||
)
|
||||
|
||||
prisma_client.db.litellm_usertable.update_many.assert_awaited_once_with(
|
||||
where={}, data={"budget_fallbacks": json.dumps({"gpt-4o": ["gpt-4o-mini"]})}
|
||||
)
|
||||
prisma_client.update_data.assert_not_called()
|
||||
assert response.successful_updates == 1
|
||||
assert response.results[0].updated_user["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]}
|
||||
assert await cache.async_get_cache(key=saved_user.user_id, model_type=LiteLLM_UserTable) is None
|
||||
broadcast.assert_awaited_once_with(cache_key=saved_user.user_id)
|
||||
|
||||
|
||||
def test_generate_request_base_validator():
|
||||
"""
|
||||
Test that GenerateRequestBase validator converts empty string to None for max_budget
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue