From 8421fd2efce691bf1aeac8eb061266c80fb7a820 Mon Sep 17 00:00:00 2001 From: jesus Date: Fri, 18 Sep 2026 01:22:25 +0000 Subject: [PATCH] feat(proxy): budget fallbacks on teams and internal users Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration.sql | 2 + .../litellm_proxy_extras/schema.prisma | 2 + litellm/models/team.py | 2 + litellm/models/user.py | 1 + litellm/proxy/_types.py | 5 + litellm/proxy/auth/handle_jwt.py | 1 + litellm/proxy/auth/team_grants.py | 2 + litellm/proxy/auth/user_api_key_auth.py | 171 ++++++++++--- .../internal_user_endpoints.py | 30 ++- .../key_management_endpoints.py | 2 + .../management_endpoints/team_endpoints.py | 1 + litellm/proxy/schema.prisma | 2 + .../spend_tracking/carried_budget_state.py | 4 + litellm/proxy/utils.py | 2 + schema.prisma | 2 + .../test_user_api_key_auth.py | 18 +- .../proxy/auth/test_handle_jwt.py | 60 +++++ .../proxy/auth/test_user_api_key_auth.py | 229 ++++++++++++++++-- .../test_internal_user_endpoints.py | 105 ++++++++ .../test_team_endpoints.py | 98 ++++++++ 20 files changed, 668 insertions(+), 71 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260918010429_add_team_user_budget_fallbacks/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260918010429_add_team_user_budget_fallbacks/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260918010429_add_team_user_budget_fallbacks/migration.sql new file mode 100644 index 00000000000..aed93ae6623 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260918010429_add_team_user_budget_fallbacks/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}'; +ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}'; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 1894518e51d..24a28f685b5 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -148,6 +148,7 @@ model LiteLLM_TeamTable { policies String[] @default([]) default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction budget_limits Json? // per-model budget limits for the team + budget_fallbacks Json @default("{}") // per-model fallback chain when the team's model budget is exhausted model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -260,6 +261,7 @@ model LiteLLM_UserTable { policies String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") + budget_fallbacks Json @default("{}") // per-model fallback chain when the user's model budget is exhausted created_at DateTime? @default(now()) @map("created_at") updated_at DateTime? @default(now()) @updatedAt @map("updated_at") diff --git a/litellm/models/team.py b/litellm/models/team.py index 8edf10703b1..b8ddee38265 100644 --- a/litellm/models/team.py +++ b/litellm/models/team.py @@ -76,6 +76,7 @@ class TeamBase(LiteLLMPydanticObjectBase): soft_budget: float | None = None budget_duration: str | None = None budget_limits: list[BudgetLimitEntry] | None = None + budget_fallbacks: dict[str, list[str]] | None = None models: list[str] = [] blocked: bool = False router_settings: dict | None = None @@ -114,6 +115,7 @@ class LiteLLM_TeamTable(TeamBase): "model_aliases", "router_settings", "budget_limits", + "budget_fallbacks", ] if isinstance(values, BaseModel): diff --git a/litellm/models/user.py b/litellm/models/user.py index 82f78c28078..6891b1564e2 100644 --- a/litellm/models/user.py +++ b/litellm/models/user.py @@ -40,6 +40,7 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase): policies: list[str] = [] model_spend: dict | None = {} model_max_budget: dict | None = {} + budget_fallbacks: dict[str, list[str]] | None = None created_at: datetime | None = None updated_at: datetime | None = None organization_memberships: list[LiteLLM_OrganizationMembershipTable] | None = None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b0d31df92ce..6322c355cfe 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2117,6 +2117,9 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): router_settings: RouterSettingsDict | None = None access_group_ids: list[str] | None = None budget_limits: list[BudgetLimitEntry] | None = None # multiple concurrent budget windows + budget_fallbacks: dict[str, list[str]] | None = ( + None # per-model fallback chain when the team's model budget is exhausted + ) default_team_member_models: list[str] | None = None # default allowed_models seeded onto new team members model_max_budget: GenericBudgetConfigType | None = Field( default=None, @@ -3053,6 +3056,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): team_max_budget: float | None = None team_soft_budget: float | None = None team_model_max_budget: dict[str, object] | None = None + team_budget_fallbacks: dict[str, list[str]] | None = None team_models: list = [] team_blocked: bool = False soft_budget: float | None = None @@ -3133,6 +3137,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob # and validating it here would make one malformed row fail auth outright. # resolve_model_budget validates the single entry a request actually needs. user_model_max_budget: Mapping[str, object] | None = None + user_budget_fallbacks: dict[str, list[str]] | None = None request_route: str | None = None is_session_token: bool = False # Server-only marker set exclusively by the MCP gateway admission path diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 6a28cd7ff99..402faa9abf0 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -2734,6 +2734,7 @@ class JWTAuthManager: user_tpm_limit=user.tpm_limit if user is not None and not admin else None, user_rpm_limit=user.rpm_limit if user is not None and not admin else None, user_model_max_budget=user.model_max_budget if user is not None and not admin else None, + user_budget_fallbacks=user.budget_fallbacks if user is not None and not admin else None, **team_grants( team_object=result["team_object"], team_membership=result.get("team_membership"), diff --git a/litellm/proxy/auth/team_grants.py b/litellm/proxy/auth/team_grants.py index 2029ee342ae..25061ad5056 100644 --- a/litellm/proxy/auth/team_grants.py +++ b/litellm/proxy/auth/team_grants.py @@ -60,6 +60,7 @@ class TeamGrants(TypedDict, total=False): team_max_budget: ReadOnly[float | None] team_soft_budget: ReadOnly[float | None] team_model_max_budget: ReadOnly[dict[str, object] | None] + team_budget_fallbacks: ReadOnly[dict[str, list[str]] | None] team_spend: ReadOnly[float | None] team_models: ReadOnly[Sequence[str]] team_blocked: ReadOnly[bool] @@ -103,6 +104,7 @@ def team_grants( team_max_budget=team_object.max_budget, team_soft_budget=team_object.soft_budget, team_model_max_budget=team_object.model_max_budget, + team_budget_fallbacks=team_object.budget_fallbacks, team_spend=team_object.spend, team_models=tuple(team_object.models), team_blocked=team_object.blocked, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 4cbd4213463..1c09e6ea3a1 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -11,8 +11,9 @@ import asyncio import fnmatch import re import secrets -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime, timezone +from functools import partial from typing import Any, Final, NamedTuple, Protocol, Union, cast import fastapi @@ -334,14 +335,14 @@ def _token_team_models(valid_token: _TokenTeamModels) -> list[str]: return valid_token.team_models -async def _read_user_model_max_budget( +async def _read_user_budget_row( user_id: str | None, prisma_client: PrismaClient | None, user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None, proxy_logging_obj: ProxyLogging, -) -> Mapping[str, object] | None: - """The user row's `model_max_budget`, or None when the row cannot be read. +) -> LiteLLM_UserTable | None: + """The user row behind the per-model budget check, or None when it cannot be read. A user whose row is missing must not be refused: this is a budget lookup, and the main auth path likewise treats an unreadable user as no user. @@ -349,7 +350,7 @@ async def _read_user_model_max_budget( if user_id is None or prisma_client is None: return None try: - user_obj: Final = await get_user_object( + return await get_user_object( user_id=user_id, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, @@ -360,13 +361,16 @@ async def _read_user_model_max_budget( except Exception as e: # noqa: BLE001 # mirrors the main path's tolerance verbose_logger.debug("Unable to read user for the per-model budget check: %s", e) return None - return user_obj.model_max_budget if user_obj is not None else None async def _check_user_model_budget( valid_token: UserAPIKeyAuth, model_max_budget_limiter: _UserModelBudgetLimiter, models: list[str], + request_data: dict | None = None, + request: Request | None = None, + llm_model_list: list | None = None, + llm_router: litellm.Router | None = None, ) -> None: """Enforce the internal user's own `model_max_budget` across the request's models. @@ -376,47 +380,79 @@ async def _check_user_model_budget( user_model_max_budget: Final = valid_token.user_model_max_budget if valid_token.user_id is None or not isinstance(user_model_max_budget, Mapping) or not user_model_max_budget: return + check: Final = partial( + model_max_budget_limiter.is_user_within_model_budget, + user_id=cast(str, valid_token.user_id), + user_model_max_budget=user_model_max_budget, + ) for model_name in models: - await model_max_budget_limiter.is_user_within_model_budget( - user_id=valid_token.user_id, - user_model_max_budget=user_model_max_budget, - model=model_name, - ) + if request_data is not None and request is not None: + await _check_model_budget_with_fallback( + check=check, + budget_fallbacks=valid_token.user_budget_fallbacks, + model_name=model_name, + request_data=request_data, + request=request, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + else: + await check(model=model_name) async def _check_team_model_budget( valid_token: UserAPIKeyAuth, model_max_budget_limiter: _TeamModelBudgetLimiter, models: list[str], + request_data: dict | None = None, + request: Request | None = None, + llm_model_list: list | None = None, + llm_router: litellm.Router | None = None, ) -> None: """Enforce the team's `model_max_budget` for every requested model the key does not override.""" team_model_max_budget: Final = valid_token.team_model_max_budget if valid_token.team_id is None or not team_model_max_budget: return key_model_max_budget: Final[Mapping[str, object] | None] = valid_token.model_max_budget + team_id: Final = valid_token.team_id + check: Final = partial( + model_max_budget_limiter.is_team_within_model_budget, + team_id=team_id, + team_model_max_budget=team_model_max_budget, + key_model_max_budget=key_model_max_budget, + ) for model_name in models: - await model_max_budget_limiter.is_team_within_model_budget( - team_id=valid_token.team_id, - team_model_max_budget=team_model_max_budget, - key_model_max_budget=key_model_max_budget, - model=model_name, - ) + if request_data is not None and request is not None: + await _check_model_budget_with_fallback( + check=check, + budget_fallbacks=valid_token.team_budget_fallbacks, + model_name=model_name, + request_data=request_data, + request=request, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + else: + await check(model=model_name) -async def _check_key_model_budget_with_fallback( - valid_token: UserAPIKeyAuth, - model_max_budget_limiter: _KeyModelBudgetLimiter, +async def _check_model_budget_with_fallback( + check: Callable[[str], Awaitable[object]], + budget_fallbacks: Mapping[str, Sequence[str]] | None, model_name: str, request_data: dict, request: Request, + valid_token: UserAPIKeyAuth, llm_model_list: list | None = None, llm_router: litellm.Router | None = None, ) -> None: """ - Enforce the key's per-model budget for `model_name`. If exceeded and the - key has a `budget_fallbacks` chain configured for `model_name`, reroute - the request to the first fallback model still within its own budget - instead of rejecting the request. + Run `check` for `model_name`. If it exceeds its per-model budget and + `budget_fallbacks` has a chain for `model_name`, reroute the request to + the first fallback model still within its own budget instead of + rejecting the request. The selected fallback is validated against the key's model-access allowlist and the team's model restrictions so that budget_fallbacks @@ -436,19 +472,17 @@ async def _check_key_model_budget_with_fallback( fallback is within budget either (or the fallback is not authorized). """ try: - await model_max_budget_limiter.is_key_within_model_budget( - user_api_key_dict=valid_token, - model=model_name, - ) + await check(model=model_name) + return except litellm.BudgetExceededError as e: - if request_data.get("model") != model_name: - raise e - fallback_model: Final = await model_max_budget_limiter.get_fallback_model_within_budget( - user_api_key_dict=valid_token, - model=model_name, - ) - if fallback_model is None: - raise e + original_error: Final = e + if request_data.get("model") != model_name: + raise original_error + for fallback_model in (budget_fallbacks or {}).get(model_name, ()): + try: + await check(model=fallback_model) + except litellm.BudgetExceededError: + continue try: await can_key_call_model( model=fallback_model, @@ -466,7 +500,7 @@ async def _check_key_model_budget_with_fallback( object_type="team", ) except ProxyException: - raise e + raise original_error request_data["model"] = fallback_model _safe_set_request_parsed_body(request=request, parsed_body=request_data) request._json = request_data @@ -474,6 +508,36 @@ async def _check_key_model_budget_with_fallback( path_params: Final = request.scope.get("path_params") if isinstance(path_params, dict) and "model" in path_params: path_params["model"] = fallback_model + return + raise original_error + + +async def _check_key_model_budget_with_fallback( + valid_token: UserAPIKeyAuth, + model_max_budget_limiter: _KeyModelBudgetLimiter, + model_name: str, + request_data: dict, + request: Request, + llm_model_list: list | None = None, + llm_router: litellm.Router | None = None, +) -> None: + """ + Enforce the key's per-model budget for `model_name`, rerouting to the + key's `budget_fallbacks` chain when the budget is exceeded. + """ + await _check_model_budget_with_fallback( + check=partial( + model_max_budget_limiter.is_key_within_model_budget, + user_api_key_dict=valid_token, + ), + budget_fallbacks=valid_token.budget_fallbacks, + model_name=model_name, + request_data=request_data, + request=request, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) def _get_bearer_token_or_received_api_key(api_key: str) -> str: @@ -1420,6 +1484,7 @@ async def _refresh_session_token_grants( **team_grants(team_object, team_membership, user_object.user_id), "user_role": _get_user_role(user_object), "models": () if team_object is not None else user_models(user_object), + "user_budget_fallbacks": user_object.budget_fallbacks, } ) ) @@ -1757,6 +1822,9 @@ async def _user_api_key_auth_builder( auto_registered.user_model_max_budget = ( user_object.model_max_budget if user_object is not None else None ) + auto_registered.user_budget_fallbacks = ( + user_object.budget_fallbacks if user_object is not None else None + ) valid_token = auto_registered api_key = valid_token.token or "" @@ -1810,6 +1878,10 @@ async def _user_api_key_auth_builder( team_id=valid_token.team_id, ) ), + request_data=request_data, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, ) return cast(UserAPIKeyAuth, valid_token) @@ -2192,6 +2264,7 @@ async def _user_api_key_auth_builder( # user's own per-model budget reaches enforcement and the post-call # increment through the row fetched here. valid_token.user_model_max_budget = user_obj.model_max_budget + valid_token.user_budget_fallbacks = user_obj.budget_fallbacks if ( user_obj is not None @@ -2368,6 +2441,10 @@ async def _user_api_key_auth_builder( valid_token=valid_token, model_max_budget_limiter=model_max_budget_limiter, models=current_models, + request_data=request_data, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, ) # Check 5b. End-user model max budget @@ -2406,6 +2483,7 @@ async def _user_api_key_auth_builder( max_budget=valid_token.team_max_budget, soft_budget=valid_token.team_soft_budget, model_max_budget=valid_token.team_model_max_budget, + budget_fallbacks=valid_token.team_budget_fallbacks, spend=valid_token.team_spend, tpm_limit=valid_token.team_tpm_limit, rpm_limit=valid_token.team_rpm_limit, @@ -2561,6 +2639,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached max_budget=valid_token.team_max_budget, soft_budget=valid_token.team_soft_budget, model_max_budget=valid_token.team_model_max_budget, + budget_fallbacks=valid_token.team_budget_fallbacks, spend=valid_token.team_spend, tpm_limit=valid_token.team_tpm_limit, rpm_limit=valid_token.team_rpm_limit, @@ -2919,6 +2998,10 @@ async def _run_centralized_common_checks( team_id=user_api_key_auth_obj.team_id, ) ), + request_data=request_data, + request=request, + llm_model_list=llm_router.model_list if llm_router is not None else None, + llm_router=llm_router, ) await _reserve_budget_after_common_checks( @@ -3294,6 +3377,7 @@ async def _return_user_api_key_auth_obj( user_spend=getattr(user_obj, "spend", None), user_max_budget=getattr(user_obj, "max_budget", None), user_model_max_budget=getattr(user_obj, "model_max_budget", None), + user_budget_fallbacks=getattr(user_obj, "budget_fallbacks", None), ) if user_obj is not None and _is_user_proxy_admin(user_obj=user_obj): user_api_key_kwargs.update( @@ -3609,19 +3693,28 @@ async def _run_post_custom_auth_checks( # spend hook reads this field off the token: gating it on the same condition # as enforcement would leave the user's counter uncharged whenever this # request was not itself enforceable, so its spend would go untracked. - user_budget: Final = await _read_user_model_max_budget( + user_budget_row: Final = await _read_user_budget_row( user_id=valid_token.user_id, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, proxy_logging_obj=proxy_logging_obj, ) - valid_token.user_model_max_budget = user_budget # rebind-ok: the spend hook reads it off this token + valid_token.user_model_max_budget = ( + user_budget_row.model_max_budget if user_budget_row is not None else None + ) # rebind-ok: the spend hook reads it off this token + valid_token.user_budget_fallbacks = ( + user_budget_row.budget_fallbacks if user_budget_row is not None else None + ) # rebind-ok: pinned alongside the budget it guards if not skip_budget_checks and current_models: await _check_user_model_budget( valid_token=valid_token, model_max_budget_limiter=model_max_budget_limiter, models=current_models, + request_data=request_data, + request=request, + llm_model_list=llm_model_list, + llm_router=llm_router, ) # 4. Check end-user model_max_budget diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index ba7a3309a90..ff48c23f9f0 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -100,6 +100,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _USER_MODEL_BUDGET_ADAPTER: Final = TypeAdapter(dict[str, float | BudgetConfig]) +_BUDGET_FALLBACKS_ADAPTER: Final = TypeAdapter(dict[str, list[str]]) _USER_BUDGET_CACHE_INVALIDATION_BATCH_SIZE: Final = 50 @@ -1263,6 +1264,13 @@ def _update_internal_user_params(data_json: dict, data: UpdateUserRequest | Upda except ValidationError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc non_default_values[k] = {} if v is None else v + elif k == "budget_fallbacks": + if k in fields_set: + try: + _BUDGET_FALLBACKS_ADAPTER.validate_python({} if v is None else v) + except ValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + non_default_values[k] = {} if v is None else v elif ( v is not None and v @@ -1477,7 +1485,14 @@ async def _update_single_user_helper( # because `_update_internal_user_params` drops empty values, and `object_permission: {}` is # precisely the clear-my-own-ceiling case this must refuse. _sent_fields: Final = user_request.fields_set() if hasattr(user_request, "fields_set") else set() - _protected_fields: Final = ("max_budget", "model_max_budget", "soft_budget", "spend", "object_permission") + _protected_fields: Final = ( + "max_budget", + "model_max_budget", + "budget_fallbacks", + "soft_budget", + "spend", + "object_permission", + ) for _field in _protected_fields: if _field in non_default_values or _field in _sent_fields: raise HTTPException( @@ -1561,7 +1576,7 @@ async def _update_single_user_helper( await _invalidate_user_spend_counter_if_changed(non_default_values) - if "model_max_budget" in non_default_values: + if "model_max_budget" in non_default_values or "budget_fallbacks" in non_default_values: await evict_and_broadcast( cache_keys=(non_default_values["user_id"],), user_api_key_cache=user_api_key_cache, @@ -1886,9 +1901,14 @@ async def bulk_user_update( await UserRepository(prisma_client).table.update_many( where={}, data=( - {**non_default_values, "model_max_budget": json.dumps(non_default_values["model_max_budget"])} - if "model_max_budget" in non_default_values - else non_default_values + { + **non_default_values, + **{ + column: json.dumps(non_default_values[column]) + for column in ("model_max_budget", "budget_fallbacks") + if column in non_default_values + }, + } ), ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 802a7c3e469..e2714cd14c8 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4507,6 +4507,8 @@ async def generate_key_helper_fn( # Only when supplied: the SSO and default-key callers reach this with the # empty default, and writing that would clear an existing user's budgets. user_data["model_max_budget"] = model_max_budget_json + if budget_fallbacks is not None: + user_data["budget_fallbacks"] = budget_fallbacks_json key_data: Final = { "token": token, "key_alias": key_alias, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 216480e298b..88e906c6ebd 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -4458,6 +4458,7 @@ def _transform_teams_to_deleted_records( "object_permission", "id", "budget_limits", # not in LiteLLM_DeletedTeamTable schema + "budget_fallbacks", # not in LiteLLM_DeletedTeamTable schema "default_team_member_models", # not in LiteLLM_DeletedTeamTable schema ): record.pop(rel_key, None) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 1894518e51d..24a28f685b5 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -148,6 +148,7 @@ model LiteLLM_TeamTable { policies String[] @default([]) default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction budget_limits Json? // per-model budget limits for the team + budget_fallbacks Json @default("{}") // per-model fallback chain when the team's model budget is exhausted model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -260,6 +261,7 @@ model LiteLLM_UserTable { policies String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") + budget_fallbacks Json @default("{}") // per-model fallback chain when the user's model budget is exhausted created_at DateTime? @default(now()) @map("created_at") updated_at DateTime? @default(now()) @updatedAt @map("updated_at") diff --git a/litellm/proxy/spend_tracking/carried_budget_state.py b/litellm/proxy/spend_tracking/carried_budget_state.py index da8bf60ebda..49634b1b0a2 100644 --- a/litellm/proxy/spend_tracking/carried_budget_state.py +++ b/litellm/proxy/spend_tracking/carried_budget_state.py @@ -26,12 +26,16 @@ def carry_team_and_user_budget_state( max_budget=team_object.max_budget, ) valid_token.team_model_max_budget = team_object.model_max_budget # rebind-ok: caller keeps this object + valid_token.team_budget_fallbacks = team_object.budget_fallbacks # rebind-ok: caller keeps this object if user_object is not None: valid_token.user_budget_snapshot = UserBudgetSnapshot( # rebind-ok: same object the caller keeps using budget_reset_at=user_object.budget_reset_at, max_budget=user_object.max_budget, user_alias=user_object.user_alias, ) + valid_token.user_budget_fallbacks = ( + user_object.budget_fallbacks + ) # rebind-ok: same object the caller keeps using def carry_organization_budget_state(valid_token: UserAPIKeyAuth, org_table: LiteLLM_OrganizationTable) -> None: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 950ac5e9906..7afae13aa0a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4363,6 +4363,7 @@ class PrismaClient: t.spend AS team_spend, t.max_budget AS team_max_budget, t.model_max_budget AS team_model_max_budget, + t.budget_fallbacks AS team_budget_fallbacks, t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit, t.tpd_limit AS team_tpd_limit @@ -4803,6 +4804,7 @@ class PrismaClient: t.max_budget AS team_max_budget, t.soft_budget AS team_soft_budget, t.model_max_budget AS team_model_max_budget, + t.budget_fallbacks AS team_budget_fallbacks, t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit, t.tpd_limit AS team_tpd_limit, diff --git a/schema.prisma b/schema.prisma index 1894518e51d..24a28f685b5 100644 --- a/schema.prisma +++ b/schema.prisma @@ -148,6 +148,7 @@ model LiteLLM_TeamTable { policies String[] @default([]) default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction budget_limits Json? // per-model budget limits for the team + budget_fallbacks Json @default("{}") // per-model fallback chain when the team's model budget is exhausted model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id]) @@ -260,6 +261,7 @@ model LiteLLM_UserTable { policies String[] @default([]) model_spend Json @default("{}") model_max_budget Json @default("{}") + budget_fallbacks Json @default("{}") // per-model fallback chain when the user's model budget is exhausted created_at DateTime? @default(now()) @map("created_at") updated_at DateTime? @default(now()) @updatedAt @map("updated_at") diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index a8fce58c60b..1574b75845c 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -1508,7 +1508,7 @@ async def test_user_budget_lookup_tolerates_an_unreadable_user(): the budget being looked up lives on the row that could not be read. """ from litellm.caching.dual_cache import DualCache - from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget + from litellm.proxy.auth.user_api_key_auth import _read_user_budget_row prisma_client = MagicMock() @@ -1516,7 +1516,7 @@ async def test_user_budget_lookup_tolerates_an_unreadable_user(): "litellm.proxy.auth.user_api_key_auth.get_user_object", new=AsyncMock(side_effect=Exception("No user table row")), ): - budget = await _read_user_model_max_budget( + row = await _read_user_budget_row( user_id="user-with-no-row", prisma_client=prisma_client, user_api_key_cache=DualCache(), @@ -1524,7 +1524,7 @@ async def test_user_budget_lookup_tolerates_an_unreadable_user(): proxy_logging_obj=MagicMock(), ) - assert budget is None + assert row is None @pytest.mark.asyncio @@ -1546,7 +1546,7 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down() the absent case and a change to both auth paths. """ from litellm.caching.dual_cache import DualCache - from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget + from litellm.proxy.auth.user_api_key_auth import _read_user_budget_row db_down = ValueError("User doesn't exist in db. 'user_id'=u-1. Got error - Connection refused") @@ -1554,7 +1554,7 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down() "litellm.proxy.auth.user_api_key_auth.get_user_object", new=AsyncMock(side_effect=db_down), ): - budget = await _read_user_model_max_budget( + row = await _read_user_budget_row( user_id="u-1", prisma_client=MagicMock(), user_api_key_cache=DualCache(), @@ -1562,14 +1562,14 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down() proxy_logging_obj=MagicMock(), ) - assert budget is None + assert row is None @pytest.mark.asyncio async def test_user_budget_lookup_returns_the_budget_when_the_row_reads(): """Positive control: the tolerance above must not be swallowing every result.""" from litellm.caching.dual_cache import DualCache - from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget + from litellm.proxy.auth.user_api_key_auth import _read_user_budget_row stored = {"claude-opus-4-8": {"budget_limit": 1.0, "time_period": "18h"}} user_obj = MagicMock() @@ -1579,7 +1579,7 @@ async def test_user_budget_lookup_returns_the_budget_when_the_row_reads(): "litellm.proxy.auth.user_api_key_auth.get_user_object", new=AsyncMock(return_value=user_obj), ): - budget = await _read_user_model_max_budget( + row = await _read_user_budget_row( user_id="user-1", prisma_client=MagicMock(), user_api_key_cache=DualCache(), @@ -1587,7 +1587,7 @@ async def test_user_budget_lookup_returns_the_budget_when_the_row_reads(): proxy_logging_obj=MagicMock(), ) - assert budget == stored + assert row is not None and row.model_max_budget == stored def test_zero_cost_models_skip_the_user_budget_check_on_every_path(): diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 15defb196af..f5c8d9579e3 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -13,6 +13,7 @@ import litellm from litellm.proxy._types import ( DEFAULT_JWKS_STALE_TTL, + JWTAuthBuilderResult, JWTLiteLLMRoleMap, LiteLLM_JWTAuth, LiteLLM_ModelTable, @@ -7144,3 +7145,62 @@ async def test_admin_jwt_team_header_only_provisions_during_admission(monkeypatc else: create_team.assert_not_awaited() assert result["team_id"] is None + + +@pytest.mark.asyncio +async def test_jwt_built_user_api_key_auth_carries_team_and_user_budget_fallbacks(): + """A JWT-authenticated caller has no key row, so the fallback chains have + to ride team_grants and the user fields on UserAPIKeyAuth for the + auth-time reroute to find them.""" + result: JWTAuthBuilderResult = { + "is_proxy_admin": False, + "team_object": LiteLLM_TeamTable( + team_id="team-1", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ), + "user_object": LiteLLM_UserTable( + user_id="user-1", + budget_fallbacks={"gpt-4o": ["claude-haiku"]}, + ), + "end_user_object": None, + "org_object": None, + "token": "jwt-token", + "team_id": "team-1", + "user_id": "user-1", + "user_email": None, + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user-1"}, + } + + auth = JWTAuthManager.user_api_key_auth_from_result(result) + + assert auth.team_budget_fallbacks == {"gpt-4o": ["gpt-4o-mini"]} + assert auth.user_budget_fallbacks == {"gpt-4o": ["claude-haiku"]} + + +@pytest.mark.asyncio +async def test_jwt_admin_does_not_inherit_user_budget_fallbacks(): + result: JWTAuthBuilderResult = { + "is_proxy_admin": True, + "team_object": None, + "user_object": LiteLLM_UserTable( + user_id="admin-1", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ), + "end_user_object": None, + "org_object": None, + "token": "jwt-token", + "team_id": None, + "user_id": "admin-1", + "user_email": None, + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "admin-1"}, + } + + auth = JWTAuthManager.user_api_key_auth_from_result(result) + + assert auth.user_budget_fallbacks is None diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ba3e98ee718..cb8f6b81383 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -7083,6 +7083,16 @@ class TestCheckKeyModelBudgetWithFallback: request.scope = {} return request + def _exceeded_limiter(self, within: frozenset, error=None): + async def check(user_api_key_dict=None, model="", **_kwargs): + if model in within: + return True + raise error or litellm.BudgetExceededError(current_cost=10, max_budget=5) + + limiter = AsyncMock() + limiter.is_key_within_model_budget.side_effect = check + return limiter + @pytest.mark.asyncio async def test_within_budget_does_not_reroute(self): valid_token = UserAPIKeyAuth( @@ -7111,11 +7121,7 @@ class TestCheckKeyModelBudgetWithFallback: token="test-key", budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]}, ) - limiter = AsyncMock() - limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError( - current_cost=10, max_budget=5 - ) - limiter.get_fallback_model_within_budget.return_value = "gpt-4o-mini" + limiter = self._exceeded_limiter(within=frozenset({"gpt-4o-mini"})) request_data = {"model": "gpt-4o"} request = self._make_request() @@ -7128,8 +7134,8 @@ class TestCheckKeyModelBudgetWithFallback: ) assert request_data["model"] == "gpt-4o-mini" - limiter.get_fallback_model_within_budget.assert_awaited_once_with( - user_api_key_dict=valid_token, model="gpt-4o" + limiter.is_key_within_model_budget.assert_any_await( + user_api_key_dict=valid_token, model="gpt-4o-mini" ) # the rerouted model must be visible to a later, separate # `_read_request_body` call on the same `request` (route handlers @@ -7145,7 +7151,6 @@ class TestCheckKeyModelBudgetWithFallback: limiter = AsyncMock() original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) limiter.is_key_within_model_budget.side_effect = original_error - limiter.get_fallback_model_within_budget.return_value = None request_data = {"model": "gpt-4o"} request = self._make_request() @@ -7174,7 +7179,9 @@ class TestCheckKeyModelBudgetWithFallback: limiter = AsyncMock() original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) limiter.is_key_within_model_budget.side_effect = original_error - limiter.get_fallback_model_within_budget.return_value = "restricted-model" + limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter( + within=frozenset({"restricted-model"}), error=original_error + ).is_key_within_model_budget.side_effect request_data = {"model": "gpt-4o"} request = self._make_request() @@ -7211,10 +7218,9 @@ class TestCheckKeyModelBudgetWithFallback: budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, ) limiter = AsyncMock() - limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError( - current_cost=10, max_budget=5 - ) - limiter.get_fallback_model_within_budget.return_value = "gpt-4o-mini" + limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter( + within=frozenset({"gpt-4o-mini"}) + ).is_key_within_model_budget.side_effect request_data = {"model": "gpt-4o"} request = self._make_request() @@ -7248,7 +7254,9 @@ class TestCheckKeyModelBudgetWithFallback: limiter = AsyncMock() original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) limiter.is_key_within_model_budget.side_effect = original_error - limiter.get_fallback_model_within_budget.return_value = "restricted-model" + limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter( + within=frozenset({"restricted-model"}), error=original_error + ).is_key_within_model_budget.side_effect request_data = {"model": "gpt-4o"} request = self._make_request() @@ -7281,10 +7289,9 @@ class TestCheckKeyModelBudgetWithFallback: budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, ) limiter = AsyncMock() - limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError( - current_cost=10, max_budget=5 - ) - limiter.get_fallback_model_within_budget.return_value = "gpt-4o-mini" + limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter( + within=frozenset({"gpt-4o-mini"}) + ).is_key_within_model_budget.side_effect request_data = {"model": "gpt-4o"} request = self._make_request() request.scope["path_params"] = {"model": "gpt-4o"} @@ -7335,6 +7342,192 @@ class TestCheckKeyModelBudgetWithFallback: assert "model" not in request_data +class TestEntityModelBudgetFallbacks: + """Team and user `budget_fallbacks` must reroute a request whose model + exceeds that entity's per-model budget to the first fallback still within + the same entity's budget, using the same authorization and request + rewrite the key-level fallback applies.""" + + def _make_request(self): + request = MagicMock() + request.scope = {} + return request + + def _exceeded_except(self, within: frozenset): + async def is_team_within_model_budget( + team_id=None, team_model_max_budget=None, key_model_max_budget=None, model="" + ): + if model in within: + return True + raise litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""): + if model in within: + return True + raise litellm.BudgetExceededError(current_cost=10, max_budget=5) + + limiter = MagicMock() + limiter.is_team_within_model_budget = is_team_within_model_budget + limiter.is_user_within_model_budget = is_user_within_model_budget + return limiter + + @pytest.mark.asyncio + async def test_team_fallback_reroutes_request_to_fallback_model(self): + from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget + + valid_token = UserAPIKeyAuth( + team_id="team-1", + team_model_max_budget={"gpt-4o": {"max_budget": 5}}, + team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + request.scope["path_params"] = {"model": "gpt-4o"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + await _check_team_model_budget( + valid_token=valid_token, + model_max_budget_limiter=self._exceeded_except(frozenset({"gpt-4o-mini"})), + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert request_data["model"] == "gpt-4o-mini" + _, cached_body = request.scope["parsed_body"] + assert cached_body["model"] == "gpt-4o-mini" + assert request.scope["path_params"]["model"] == "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_user_fallback_reroutes_request_to_fallback_model(self): + from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget + + valid_token = UserAPIKeyAuth( + user_id="user-1", + user_model_max_budget={"gpt-4o": {"max_budget": 5}}, + user_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + await _check_user_model_budget( + valid_token=valid_token, + model_max_budget_limiter=self._exceeded_except(frozenset({"gpt-4o-mini"})), + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert request_data["model"] == "gpt-4o-mini" + _, cached_body = request.scope["parsed_body"] + assert cached_body["model"] == "gpt-4o-mini" + + @pytest.mark.asyncio + async def test_team_fallback_not_in_team_models_raises_original_error(self): + from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget + + original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def over_budget(team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""): + if model == "restricted-model": + return True + raise original_error + + limiter = MagicMock() + limiter.is_team_within_model_budget = over_budget + valid_token = UserAPIKeyAuth( + team_id="team-1", + team_models=["gpt-4o"], + team_model_max_budget={"gpt-4o": {"max_budget": 5}}, + team_budget_fallbacks={"gpt-4o": ["restricted-model"]}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_team_model_budget( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + llm_model_list=None, + llm_router=None, + ) + + assert exc_info.value is original_error + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_user_without_fallbacks_raises_original_error(self): + from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget + + original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5) + + async def over_budget(**_kwargs): + raise original_error + + limiter = MagicMock() + limiter.is_user_within_model_budget = over_budget + valid_token = UserAPIKeyAuth( + user_id="user-1", + user_model_max_budget={"gpt-4o": {"max_budget": 5}}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await _check_user_model_budget( + valid_token=valid_token, + model_max_budget_limiter=limiter, + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert exc_info.value is original_error + assert request_data["model"] == "gpt-4o" + + @pytest.mark.asyncio + async def test_team_fallback_only_tries_first_within_budget_fallback(self): + """A fallback that is itself over budget must be skipped, matching the + key-level semantics of first-in-order chain resolution.""" + from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget + + valid_token = UserAPIKeyAuth( + team_id="team-1", + team_model_max_budget={"gpt-4o": {"max_budget": 5}}, + team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]}, + ) + request_data = {"model": "gpt-4o"} + request = self._make_request() + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", + new=AsyncMock(return_value=True), + ): + await _check_team_model_budget( + valid_token=valid_token, + model_max_budget_limiter=self._exceeded_except(frozenset({"claude-haiku"})), + models=["gpt-4o"], + request_data=request_data, + request=request, + ) + + assert request_data["model"] == "claude-haiku" + + @pytest.mark.asyncio async def test_global_proxy_spend_reads_resettable_proxy_budget_row(): """Regression for the global proxy budget ignoring budget_duration diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 0d8b19345f1..4a699ecfe0c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4519,3 +4519,108 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo written_data = mock_prisma_client.update_data.call_args.kwargs["data"] assert written_data.get("password") is not None assert written_data["password"] != strong_password + + +@pytest.mark.asyncio +async def test_new_user_forwards_budget_fallbacks_into_user_persistence(mocker): + """/user/new must carry budget_fallbacks into generate_key_helper_fn so it + lands on the user row (previously the field was accepted but dropped).""" + from litellm.proxy.management_endpoints.internal_user_endpoints import new_user + + async def _noop(*args, **kwargs): + return None + + mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints._check_duplicate_user_email", + _noop, + ) + mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints._check_duplicate_user_id", + _noop, + ) + key_gen = mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints.generate_key_helper_fn", + new=mocker.AsyncMock(side_effect=RuntimeError("reached key generation")), + ) + prisma_client = mocker.MagicMock() + prisma_client.db.litellm_usertable.count = mocker.AsyncMock(return_value=0) + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + request = NewUserRequest( + user_role="internal_user", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]}, + ) + + with pytest.raises(ProxyException): + await new_user(data=request, user_api_key_dict=admin) + + assert key_gen.call_count == 1 + assert key_gen.call_args.kwargs["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini", "claude-haiku"]} + + +@pytest.mark.asyncio +async def test_update_user_replaces_budget_fallbacks(_admin_prisma, mocker): + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "target-user"} + existing_user.user_id = "target-user" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"}) + + user_request = UpdateUserRequest( + user_id="target-user", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + written_data = mock_prisma_client.update_data.call_args.kwargs["data"] + assert written_data["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]} + + +@pytest.mark.asyncio +async def test_update_user_clears_budget_fallbacks_with_empty_map(_admin_prisma, mocker): + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "target-user"} + existing_user.user_id = "target-user" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"}) + + user_request = UpdateUserRequest(user_id="target-user", budget_fallbacks={}) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + written_data = mock_prisma_client.update_data.call_args.kwargs["data"] + assert written_data["budget_fallbacks"] == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_value", + [["gpt-4o-mini"], {"gpt-4o": "gpt-4o-mini"}, {"gpt-4o": [1, 2]}], +) +async def test_update_internal_user_params_rejects_malformed_budget_fallbacks(bad_value): + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_internal_user_params, + ) + + data = UpdateUserRequest(user_id="u-1", budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}) + data_json = data.model_dump(exclude_unset=True) + data_json["budget_fallbacks"] = bad_value + + with pytest.raises(HTTPException) as exc_info: + _update_internal_user_params(data_json=data_json, data=data) + + assert exc_info.value.status_code == 400 diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a89bc9a8a3e..7ac1395e85a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -15422,3 +15422,101 @@ async def test_team_info_reports_what_the_caller_may_edit(caller, org_admin, ena ) assert response["team_info"].caller_edit_access.model_dump(mode="json") == expected + + +@pytest.mark.asyncio +async def test_new_team_persists_budget_fallbacks(mock_db_client, mock_admin_auth): + mock_db_client.jsonify_team_object = lambda db_data: db_data + mock_db_client.get_data = AsyncMock(return_value=None) + mock_db_client.update_data = AsyncMock(return_value=MagicMock()) + mock_db_client.db = MagicMock() + mock_db_client.db.litellm_modeltable = MagicMock() + mock_db_client.db.litellm_modeltable.create = AsyncMock(return_value=MagicMock(id="model123")) + + team_create_result = MagicMock(team_id="team-fallbacks") + team_create_result.model_dump.return_value = {"team_id": "team-fallbacks"} + mock_team_create = AsyncMock(return_value=team_create_result) + mock_db_client.db.litellm_teamtable = MagicMock() + mock_db_client.db.litellm_teamtable.create = mock_team_create + _wire_team_create_tx(mock_db_client) + mock_db_client.db.litellm_teamtable.count = AsyncMock(return_value=0) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=team_create_result) + mock_db_client.db.litellm_usertable = MagicMock() + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + from fastapi import Request + + from litellm.proxy._types import NewTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import new_team + + await new_team( + data=NewTeamRequest( + team_alias="fallbacks-team", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ), + http_request=MagicMock(spec=Request), + user_api_key_dict=mock_admin_auth, + ) + + team_data = mock_team_create.call_args.kwargs["data"] + assert team_data["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]} + + +@pytest.mark.asyncio +async def test_update_team_replaces_budget_fallbacks(disable_audit_logging_for_mocked_team): + from unittest.mock import Mock + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UpdateTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import update_team + + mock_user_api_key_dict = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1") + + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client, + patch("litellm.proxy.proxy_server.llm_router"), + patch("litellm.proxy.proxy_server.user_api_key_cache"), + patch("litellm.proxy.proxy_server.proxy_logging_obj"), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch("litellm.proxy.management_endpoints.team_endpoints._cache_team_object"), + patch( + "litellm.proxy.management_endpoints.team_endpoints.TeamMemberBudgetHandler.upsert_team_member_budget_table", + new=AsyncMock(side_effect=lambda **kwargs: kwargs["updated_kv"]), + ), + ): + mock_existing_team = MagicMock() + mock_existing_team.model_dump.return_value = { + "team_id": "team-1", + "team_alias": "team", + "metadata": {}, + "budget_fallbacks": {"gpt-4o": ["claude-haiku"]}, + } + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=mock_existing_team) + mock_updated_team = MagicMock() + mock_updated_team.team_id = "team-1" + mock_updated_team.model_dump.return_value = {"team_id": "team-1"} + mock_prisma_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_updated_team) + + import json as _json + + def _jsonify(db_data): + return { + k: (_json.dumps(v) if isinstance(v, dict) else v) + for k, v in db_data.items() + } + + mock_prisma_client.jsonify_team_object = MagicMock(side_effect=_jsonify) + + await update_team( + data=UpdateTeamRequest( + team_id="team-1", + budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]}, + ), + http_request=Mock(spec=Request), + user_api_key_dict=mock_user_api_key_dict, + ) + + update_call = mock_prisma_client.db.litellm_teamtable.update.call_args + written = update_call.kwargs["data"] + assert _json.loads(written["budget_fallbacks"]) == {"gpt-4o": ["gpt-4o-mini"]}