mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(proxy): budget fallbacks on teams and internal users
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
3424390101
commit
8421fd2efc
20 changed files with 668 additions and 71 deletions
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';
|
||||
|
|
@ -148,6 +148,7 @@ model LiteLLM_TeamTable {
|
|||
policies String[] @default([])
|
||||
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
|
||||
budget_limits Json? // per-model budget limits for the team
|
||||
budget_fallbacks Json @default("{}") // per-model fallback chain when the team's model budget is exhausted
|
||||
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
|
||||
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
|
||||
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
|
||||
|
|
@ -260,6 +261,7 @@ model LiteLLM_UserTable {
|
|||
policies String[] @default([])
|
||||
model_spend Json @default("{}")
|
||||
model_max_budget Json @default("{}")
|
||||
budget_fallbacks Json @default("{}") // per-model fallback chain when the user's model budget is exhausted
|
||||
created_at DateTime? @default(now()) @map("created_at")
|
||||
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
|
||||
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
|
|||
soft_budget: float | None = None
|
||||
budget_duration: str | None = None
|
||||
budget_limits: list[BudgetLimitEntry] | None = None
|
||||
budget_fallbacks: dict[str, list[str]] | None = None
|
||||
models: list[str] = []
|
||||
blocked: bool = False
|
||||
router_settings: dict | None = None
|
||||
|
|
@ -114,6 +115,7 @@ class LiteLLM_TeamTable(TeamBase):
|
|||
"model_aliases",
|
||||
"router_settings",
|
||||
"budget_limits",
|
||||
"budget_fallbacks",
|
||||
]
|
||||
|
||||
if isinstance(values, BaseModel):
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase):
|
|||
policies: list[str] = []
|
||||
model_spend: dict | None = {}
|
||||
model_max_budget: dict | None = {}
|
||||
budget_fallbacks: dict[str, list[str]] | None = None
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
organization_memberships: list[LiteLLM_OrganizationMembershipTable] | None = None
|
||||
|
|
|
|||
|
|
@ -2117,6 +2117,9 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
|
|||
router_settings: RouterSettingsDict | None = None
|
||||
access_group_ids: list[str] | None = None
|
||||
budget_limits: list[BudgetLimitEntry] | None = None # multiple concurrent budget windows
|
||||
budget_fallbacks: dict[str, list[str]] | None = (
|
||||
None # per-model fallback chain when the team's model budget is exhausted
|
||||
)
|
||||
default_team_member_models: list[str] | None = None # default allowed_models seeded onto new team members
|
||||
model_max_budget: GenericBudgetConfigType | None = Field(
|
||||
default=None,
|
||||
|
|
@ -3053,6 +3056,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
|
|||
team_max_budget: float | None = None
|
||||
team_soft_budget: float | None = None
|
||||
team_model_max_budget: dict[str, object] | None = None
|
||||
team_budget_fallbacks: dict[str, list[str]] | None = None
|
||||
team_models: list = []
|
||||
team_blocked: bool = False
|
||||
soft_budget: float | None = None
|
||||
|
|
@ -3133,6 +3137,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
|
|||
# and validating it here would make one malformed row fail auth outright.
|
||||
# resolve_model_budget validates the single entry a request actually needs.
|
||||
user_model_max_budget: Mapping[str, object] | None = None
|
||||
user_budget_fallbacks: dict[str, list[str]] | None = None
|
||||
request_route: str | None = None
|
||||
is_session_token: bool = False
|
||||
# Server-only marker set exclusively by the MCP gateway admission path
|
||||
|
|
|
|||
|
|
@ -2734,6 +2734,7 @@ class JWTAuthManager:
|
|||
user_tpm_limit=user.tpm_limit if user is not None and not admin else None,
|
||||
user_rpm_limit=user.rpm_limit if user is not None and not admin else None,
|
||||
user_model_max_budget=user.model_max_budget if user is not None and not admin else None,
|
||||
user_budget_fallbacks=user.budget_fallbacks if user is not None and not admin else None,
|
||||
**team_grants(
|
||||
team_object=result["team_object"],
|
||||
team_membership=result.get("team_membership"),
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ class TeamGrants(TypedDict, total=False):
|
|||
team_max_budget: ReadOnly[float | None]
|
||||
team_soft_budget: ReadOnly[float | None]
|
||||
team_model_max_budget: ReadOnly[dict[str, object] | None]
|
||||
team_budget_fallbacks: ReadOnly[dict[str, list[str]] | None]
|
||||
team_spend: ReadOnly[float | None]
|
||||
team_models: ReadOnly[Sequence[str]]
|
||||
team_blocked: ReadOnly[bool]
|
||||
|
|
@ -103,6 +104,7 @@ def team_grants(
|
|||
team_max_budget=team_object.max_budget,
|
||||
team_soft_budget=team_object.soft_budget,
|
||||
team_model_max_budget=team_object.model_max_budget,
|
||||
team_budget_fallbacks=team_object.budget_fallbacks,
|
||||
team_spend=team_object.spend,
|
||||
team_models=tuple(team_object.models),
|
||||
team_blocked=team_object.blocked,
|
||||
|
|
|
|||
|
|
@ -11,8 +11,9 @@ import asyncio
|
|||
import fnmatch
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from functools import partial
|
||||
from typing import Any, Final, NamedTuple, Protocol, Union, cast
|
||||
|
||||
import fastapi
|
||||
|
|
@ -334,14 +335,14 @@ def _token_team_models(valid_token: _TokenTeamModels) -> list[str]:
|
|||
return valid_token.team_models
|
||||
|
||||
|
||||
async def _read_user_model_max_budget(
|
||||
async def _read_user_budget_row(
|
||||
user_id: str | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
parent_otel_span: Span | None,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
) -> Mapping[str, object] | None:
|
||||
"""The user row's `model_max_budget`, or None when the row cannot be read.
|
||||
) -> LiteLLM_UserTable | None:
|
||||
"""The user row behind the per-model budget check, or None when it cannot be read.
|
||||
|
||||
A user whose row is missing must not be refused: this is a budget lookup,
|
||||
and the main auth path likewise treats an unreadable user as no user.
|
||||
|
|
@ -349,7 +350,7 @@ async def _read_user_model_max_budget(
|
|||
if user_id is None or prisma_client is None:
|
||||
return None
|
||||
try:
|
||||
user_obj: Final = await get_user_object(
|
||||
return await get_user_object(
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
|
|
@ -360,13 +361,16 @@ async def _read_user_model_max_budget(
|
|||
except Exception as e: # noqa: BLE001 # mirrors the main path's tolerance
|
||||
verbose_logger.debug("Unable to read user for the per-model budget check: %s", e)
|
||||
return None
|
||||
return user_obj.model_max_budget if user_obj is not None else None
|
||||
|
||||
|
||||
async def _check_user_model_budget(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _UserModelBudgetLimiter,
|
||||
models: list[str],
|
||||
request_data: dict | None = None,
|
||||
request: Request | None = None,
|
||||
llm_model_list: list | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> None:
|
||||
"""Enforce the internal user's own `model_max_budget` across the request's models.
|
||||
|
||||
|
|
@ -376,47 +380,79 @@ async def _check_user_model_budget(
|
|||
user_model_max_budget: Final = valid_token.user_model_max_budget
|
||||
if valid_token.user_id is None or not isinstance(user_model_max_budget, Mapping) or not user_model_max_budget:
|
||||
return
|
||||
check: Final = partial(
|
||||
model_max_budget_limiter.is_user_within_model_budget,
|
||||
user_id=cast(str, valid_token.user_id),
|
||||
user_model_max_budget=user_model_max_budget,
|
||||
)
|
||||
for model_name in models:
|
||||
await model_max_budget_limiter.is_user_within_model_budget(
|
||||
user_id=valid_token.user_id,
|
||||
user_model_max_budget=user_model_max_budget,
|
||||
model=model_name,
|
||||
)
|
||||
if request_data is not None and request is not None:
|
||||
await _check_model_budget_with_fallback(
|
||||
check=check,
|
||||
budget_fallbacks=valid_token.user_budget_fallbacks,
|
||||
model_name=model_name,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
valid_token=valid_token,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
else:
|
||||
await check(model=model_name)
|
||||
|
||||
|
||||
async def _check_team_model_budget(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _TeamModelBudgetLimiter,
|
||||
models: list[str],
|
||||
request_data: dict | None = None,
|
||||
request: Request | None = None,
|
||||
llm_model_list: list | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> None:
|
||||
"""Enforce the team's `model_max_budget` for every requested model the key does not override."""
|
||||
team_model_max_budget: Final = valid_token.team_model_max_budget
|
||||
if valid_token.team_id is None or not team_model_max_budget:
|
||||
return
|
||||
key_model_max_budget: Final[Mapping[str, object] | None] = valid_token.model_max_budget
|
||||
team_id: Final = valid_token.team_id
|
||||
check: Final = partial(
|
||||
model_max_budget_limiter.is_team_within_model_budget,
|
||||
team_id=team_id,
|
||||
team_model_max_budget=team_model_max_budget,
|
||||
key_model_max_budget=key_model_max_budget,
|
||||
)
|
||||
for model_name in models:
|
||||
await model_max_budget_limiter.is_team_within_model_budget(
|
||||
team_id=valid_token.team_id,
|
||||
team_model_max_budget=team_model_max_budget,
|
||||
key_model_max_budget=key_model_max_budget,
|
||||
model=model_name,
|
||||
)
|
||||
if request_data is not None and request is not None:
|
||||
await _check_model_budget_with_fallback(
|
||||
check=check,
|
||||
budget_fallbacks=valid_token.team_budget_fallbacks,
|
||||
model_name=model_name,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
valid_token=valid_token,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
else:
|
||||
await check(model=model_name)
|
||||
|
||||
|
||||
async def _check_key_model_budget_with_fallback(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _KeyModelBudgetLimiter,
|
||||
async def _check_model_budget_with_fallback(
|
||||
check: Callable[[str], Awaitable[object]],
|
||||
budget_fallbacks: Mapping[str, Sequence[str]] | None,
|
||||
model_name: str,
|
||||
request_data: dict,
|
||||
request: Request,
|
||||
valid_token: UserAPIKeyAuth,
|
||||
llm_model_list: list | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Enforce the key's per-model budget for `model_name`. If exceeded and the
|
||||
key has a `budget_fallbacks` chain configured for `model_name`, reroute
|
||||
the request to the first fallback model still within its own budget
|
||||
instead of rejecting the request.
|
||||
Run `check` for `model_name`. If it exceeds its per-model budget and
|
||||
`budget_fallbacks` has a chain for `model_name`, reroute the request to
|
||||
the first fallback model still within its own budget instead of
|
||||
rejecting the request.
|
||||
|
||||
The selected fallback is validated against the key's model-access
|
||||
allowlist and the team's model restrictions so that budget_fallbacks
|
||||
|
|
@ -436,19 +472,17 @@ async def _check_key_model_budget_with_fallback(
|
|||
fallback is within budget either (or the fallback is not authorized).
|
||||
"""
|
||||
try:
|
||||
await model_max_budget_limiter.is_key_within_model_budget(
|
||||
user_api_key_dict=valid_token,
|
||||
model=model_name,
|
||||
)
|
||||
await check(model=model_name)
|
||||
return
|
||||
except litellm.BudgetExceededError as e:
|
||||
if request_data.get("model") != model_name:
|
||||
raise e
|
||||
fallback_model: Final = await model_max_budget_limiter.get_fallback_model_within_budget(
|
||||
user_api_key_dict=valid_token,
|
||||
model=model_name,
|
||||
)
|
||||
if fallback_model is None:
|
||||
raise e
|
||||
original_error: Final = e
|
||||
if request_data.get("model") != model_name:
|
||||
raise original_error
|
||||
for fallback_model in (budget_fallbacks or {}).get(model_name, ()):
|
||||
try:
|
||||
await check(model=fallback_model)
|
||||
except litellm.BudgetExceededError:
|
||||
continue
|
||||
try:
|
||||
await can_key_call_model(
|
||||
model=fallback_model,
|
||||
|
|
@ -466,7 +500,7 @@ async def _check_key_model_budget_with_fallback(
|
|||
object_type="team",
|
||||
)
|
||||
except ProxyException:
|
||||
raise e
|
||||
raise original_error
|
||||
request_data["model"] = fallback_model
|
||||
_safe_set_request_parsed_body(request=request, parsed_body=request_data)
|
||||
request._json = request_data
|
||||
|
|
@ -474,6 +508,36 @@ async def _check_key_model_budget_with_fallback(
|
|||
path_params: Final = request.scope.get("path_params")
|
||||
if isinstance(path_params, dict) and "model" in path_params:
|
||||
path_params["model"] = fallback_model
|
||||
return
|
||||
raise original_error
|
||||
|
||||
|
||||
async def _check_key_model_budget_with_fallback(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _KeyModelBudgetLimiter,
|
||||
model_name: str,
|
||||
request_data: dict,
|
||||
request: Request,
|
||||
llm_model_list: list | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Enforce the key's per-model budget for `model_name`, rerouting to the
|
||||
key's `budget_fallbacks` chain when the budget is exceeded.
|
||||
"""
|
||||
await _check_model_budget_with_fallback(
|
||||
check=partial(
|
||||
model_max_budget_limiter.is_key_within_model_budget,
|
||||
user_api_key_dict=valid_token,
|
||||
),
|
||||
budget_fallbacks=valid_token.budget_fallbacks,
|
||||
model_name=model_name,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
valid_token=valid_token,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
|
||||
def _get_bearer_token_or_received_api_key(api_key: str) -> str:
|
||||
|
|
@ -1420,6 +1484,7 @@ async def _refresh_session_token_grants(
|
|||
**team_grants(team_object, team_membership, user_object.user_id),
|
||||
"user_role": _get_user_role(user_object),
|
||||
"models": () if team_object is not None else user_models(user_object),
|
||||
"user_budget_fallbacks": user_object.budget_fallbacks,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
|
@ -1757,6 +1822,9 @@ async def _user_api_key_auth_builder(
|
|||
auto_registered.user_model_max_budget = (
|
||||
user_object.model_max_budget if user_object is not None else None
|
||||
)
|
||||
auto_registered.user_budget_fallbacks = (
|
||||
user_object.budget_fallbacks if user_object is not None else None
|
||||
)
|
||||
valid_token = auto_registered
|
||||
api_key = valid_token.token or ""
|
||||
|
||||
|
|
@ -1810,6 +1878,10 @@ async def _user_api_key_auth_builder(
|
|||
team_id=valid_token.team_id,
|
||||
)
|
||||
),
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
return cast(UserAPIKeyAuth, valid_token)
|
||||
|
|
@ -2192,6 +2264,7 @@ async def _user_api_key_auth_builder(
|
|||
# user's own per-model budget reaches enforcement and the post-call
|
||||
# increment through the row fetched here.
|
||||
valid_token.user_model_max_budget = user_obj.model_max_budget
|
||||
valid_token.user_budget_fallbacks = user_obj.budget_fallbacks
|
||||
|
||||
if (
|
||||
user_obj is not None
|
||||
|
|
@ -2368,6 +2441,10 @@ async def _user_api_key_auth_builder(
|
|||
valid_token=valid_token,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
models=current_models,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
# Check 5b. End-user model max budget
|
||||
|
|
@ -2406,6 +2483,7 @@ async def _user_api_key_auth_builder(
|
|||
max_budget=valid_token.team_max_budget,
|
||||
soft_budget=valid_token.team_soft_budget,
|
||||
model_max_budget=valid_token.team_model_max_budget,
|
||||
budget_fallbacks=valid_token.team_budget_fallbacks,
|
||||
spend=valid_token.team_spend,
|
||||
tpm_limit=valid_token.team_tpm_limit,
|
||||
rpm_limit=valid_token.team_rpm_limit,
|
||||
|
|
@ -2561,6 +2639,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
|
|||
max_budget=valid_token.team_max_budget,
|
||||
soft_budget=valid_token.team_soft_budget,
|
||||
model_max_budget=valid_token.team_model_max_budget,
|
||||
budget_fallbacks=valid_token.team_budget_fallbacks,
|
||||
spend=valid_token.team_spend,
|
||||
tpm_limit=valid_token.team_tpm_limit,
|
||||
rpm_limit=valid_token.team_rpm_limit,
|
||||
|
|
@ -2919,6 +2998,10 @@ async def _run_centralized_common_checks(
|
|||
team_id=user_api_key_auth_obj.team_id,
|
||||
)
|
||||
),
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=llm_router.model_list if llm_router is not None else None,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
await _reserve_budget_after_common_checks(
|
||||
|
|
@ -3294,6 +3377,7 @@ async def _return_user_api_key_auth_obj(
|
|||
user_spend=getattr(user_obj, "spend", None),
|
||||
user_max_budget=getattr(user_obj, "max_budget", None),
|
||||
user_model_max_budget=getattr(user_obj, "model_max_budget", None),
|
||||
user_budget_fallbacks=getattr(user_obj, "budget_fallbacks", None),
|
||||
)
|
||||
if user_obj is not None and _is_user_proxy_admin(user_obj=user_obj):
|
||||
user_api_key_kwargs.update(
|
||||
|
|
@ -3609,19 +3693,28 @@ async def _run_post_custom_auth_checks(
|
|||
# spend hook reads this field off the token: gating it on the same condition
|
||||
# as enforcement would leave the user's counter uncharged whenever this
|
||||
# request was not itself enforceable, so its spend would go untracked.
|
||||
user_budget: Final = await _read_user_model_max_budget(
|
||||
user_budget_row: Final = await _read_user_budget_row(
|
||||
user_id=valid_token.user_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
valid_token.user_model_max_budget = user_budget # rebind-ok: the spend hook reads it off this token
|
||||
valid_token.user_model_max_budget = (
|
||||
user_budget_row.model_max_budget if user_budget_row is not None else None
|
||||
) # rebind-ok: the spend hook reads it off this token
|
||||
valid_token.user_budget_fallbacks = (
|
||||
user_budget_row.budget_fallbacks if user_budget_row is not None else None
|
||||
) # rebind-ok: pinned alongside the budget it guards
|
||||
if not skip_budget_checks and current_models:
|
||||
await _check_user_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
models=current_models,
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
# 4. Check end-user model_max_budget
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ if TYPE_CHECKING:
|
|||
|
||||
router: Final = APIRouter()
|
||||
_USER_MODEL_BUDGET_ADAPTER: Final = TypeAdapter(dict[str, float | BudgetConfig])
|
||||
_BUDGET_FALLBACKS_ADAPTER: Final = TypeAdapter(dict[str, list[str]])
|
||||
_USER_BUDGET_CACHE_INVALIDATION_BATCH_SIZE: Final = 50
|
||||
|
||||
|
||||
|
|
@ -1263,6 +1264,13 @@ def _update_internal_user_params(data_json: dict, data: UpdateUserRequest | Upda
|
|||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
non_default_values[k] = {} if v is None else v
|
||||
elif k == "budget_fallbacks":
|
||||
if k in fields_set:
|
||||
try:
|
||||
_BUDGET_FALLBACKS_ADAPTER.validate_python({} if v is None else v)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
non_default_values[k] = {} if v is None else v
|
||||
elif (
|
||||
v is not None
|
||||
and v
|
||||
|
|
@ -1477,7 +1485,14 @@ async def _update_single_user_helper(
|
|||
# because `_update_internal_user_params` drops empty values, and `object_permission: {}` is
|
||||
# precisely the clear-my-own-ceiling case this must refuse.
|
||||
_sent_fields: Final = user_request.fields_set() if hasattr(user_request, "fields_set") else set()
|
||||
_protected_fields: Final = ("max_budget", "model_max_budget", "soft_budget", "spend", "object_permission")
|
||||
_protected_fields: Final = (
|
||||
"max_budget",
|
||||
"model_max_budget",
|
||||
"budget_fallbacks",
|
||||
"soft_budget",
|
||||
"spend",
|
||||
"object_permission",
|
||||
)
|
||||
for _field in _protected_fields:
|
||||
if _field in non_default_values or _field in _sent_fields:
|
||||
raise HTTPException(
|
||||
|
|
@ -1561,7 +1576,7 @@ async def _update_single_user_helper(
|
|||
|
||||
await _invalidate_user_spend_counter_if_changed(non_default_values)
|
||||
|
||||
if "model_max_budget" in non_default_values:
|
||||
if "model_max_budget" in non_default_values or "budget_fallbacks" in non_default_values:
|
||||
await evict_and_broadcast(
|
||||
cache_keys=(non_default_values["user_id"],),
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
|
|
@ -1886,9 +1901,14 @@ async def bulk_user_update(
|
|||
await UserRepository(prisma_client).table.update_many(
|
||||
where={},
|
||||
data=(
|
||||
{**non_default_values, "model_max_budget": json.dumps(non_default_values["model_max_budget"])}
|
||||
if "model_max_budget" in non_default_values
|
||||
else non_default_values
|
||||
{
|
||||
**non_default_values,
|
||||
**{
|
||||
column: json.dumps(non_default_values[column])
|
||||
for column in ("model_max_budget", "budget_fallbacks")
|
||||
if column in non_default_values
|
||||
},
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -4507,6 +4507,8 @@ async def generate_key_helper_fn(
|
|||
# Only when supplied: the SSO and default-key callers reach this with the
|
||||
# empty default, and writing that would clear an existing user's budgets.
|
||||
user_data["model_max_budget"] = model_max_budget_json
|
||||
if budget_fallbacks is not None:
|
||||
user_data["budget_fallbacks"] = budget_fallbacks_json
|
||||
key_data: Final = {
|
||||
"token": token,
|
||||
"key_alias": key_alias,
|
||||
|
|
|
|||
|
|
@ -4458,6 +4458,7 @@ def _transform_teams_to_deleted_records(
|
|||
"object_permission",
|
||||
"id",
|
||||
"budget_limits", # not in LiteLLM_DeletedTeamTable schema
|
||||
"budget_fallbacks", # not in LiteLLM_DeletedTeamTable schema
|
||||
"default_team_member_models", # not in LiteLLM_DeletedTeamTable schema
|
||||
):
|
||||
record.pop(rel_key, None)
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ model LiteLLM_TeamTable {
|
|||
policies String[] @default([])
|
||||
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
|
||||
budget_limits Json? // per-model budget limits for the team
|
||||
budget_fallbacks Json @default("{}") // per-model fallback chain when the team's model budget is exhausted
|
||||
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
|
||||
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
|
||||
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
|
||||
|
|
@ -260,6 +261,7 @@ model LiteLLM_UserTable {
|
|||
policies String[] @default([])
|
||||
model_spend Json @default("{}")
|
||||
model_max_budget Json @default("{}")
|
||||
budget_fallbacks Json @default("{}") // per-model fallback chain when the user's model budget is exhausted
|
||||
created_at DateTime? @default(now()) @map("created_at")
|
||||
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
|
||||
|
||||
|
|
|
|||
|
|
@ -26,12 +26,16 @@ def carry_team_and_user_budget_state(
|
|||
max_budget=team_object.max_budget,
|
||||
)
|
||||
valid_token.team_model_max_budget = team_object.model_max_budget # rebind-ok: caller keeps this object
|
||||
valid_token.team_budget_fallbacks = team_object.budget_fallbacks # rebind-ok: caller keeps this object
|
||||
if user_object is not None:
|
||||
valid_token.user_budget_snapshot = UserBudgetSnapshot( # rebind-ok: same object the caller keeps using
|
||||
budget_reset_at=user_object.budget_reset_at,
|
||||
max_budget=user_object.max_budget,
|
||||
user_alias=user_object.user_alias,
|
||||
)
|
||||
valid_token.user_budget_fallbacks = (
|
||||
user_object.budget_fallbacks
|
||||
) # rebind-ok: same object the caller keeps using
|
||||
|
||||
|
||||
def carry_organization_budget_state(valid_token: UserAPIKeyAuth, org_table: LiteLLM_OrganizationTable) -> None:
|
||||
|
|
|
|||
|
|
@ -4363,6 +4363,7 @@ class PrismaClient:
|
|||
t.spend AS team_spend,
|
||||
t.max_budget AS team_max_budget,
|
||||
t.model_max_budget AS team_model_max_budget,
|
||||
t.budget_fallbacks AS team_budget_fallbacks,
|
||||
t.tpm_limit AS team_tpm_limit,
|
||||
t.rpm_limit AS team_rpm_limit,
|
||||
t.tpd_limit AS team_tpd_limit
|
||||
|
|
@ -4803,6 +4804,7 @@ class PrismaClient:
|
|||
t.max_budget AS team_max_budget,
|
||||
t.soft_budget AS team_soft_budget,
|
||||
t.model_max_budget AS team_model_max_budget,
|
||||
t.budget_fallbacks AS team_budget_fallbacks,
|
||||
t.tpm_limit AS team_tpm_limit,
|
||||
t.rpm_limit AS team_rpm_limit,
|
||||
t.tpd_limit AS team_tpd_limit,
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ model LiteLLM_TeamTable {
|
|||
policies String[] @default([])
|
||||
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
|
||||
budget_limits Json? // per-model budget limits for the team
|
||||
budget_fallbacks Json @default("{}") // per-model fallback chain when the team's model budget is exhausted
|
||||
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
|
||||
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
|
||||
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
|
||||
|
|
@ -260,6 +261,7 @@ model LiteLLM_UserTable {
|
|||
policies String[] @default([])
|
||||
model_spend Json @default("{}")
|
||||
model_max_budget Json @default("{}")
|
||||
budget_fallbacks Json @default("{}") // per-model fallback chain when the user's model budget is exhausted
|
||||
created_at DateTime? @default(now()) @map("created_at")
|
||||
updated_at DateTime? @default(now()) @updatedAt @map("updated_at")
|
||||
|
||||
|
|
|
|||
|
|
@ -1508,7 +1508,7 @@ async def test_user_budget_lookup_tolerates_an_unreadable_user():
|
|||
the budget being looked up lives on the row that could not be read.
|
||||
"""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget
|
||||
from litellm.proxy.auth.user_api_key_auth import _read_user_budget_row
|
||||
|
||||
prisma_client = MagicMock()
|
||||
|
||||
|
|
@ -1516,7 +1516,7 @@ async def test_user_budget_lookup_tolerates_an_unreadable_user():
|
|||
"litellm.proxy.auth.user_api_key_auth.get_user_object",
|
||||
new=AsyncMock(side_effect=Exception("No user table row")),
|
||||
):
|
||||
budget = await _read_user_model_max_budget(
|
||||
row = await _read_user_budget_row(
|
||||
user_id="user-with-no-row",
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=DualCache(),
|
||||
|
|
@ -1524,7 +1524,7 @@ async def test_user_budget_lookup_tolerates_an_unreadable_user():
|
|||
proxy_logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert budget is None
|
||||
assert row is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1546,7 +1546,7 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down()
|
|||
the absent case and a change to both auth paths.
|
||||
"""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget
|
||||
from litellm.proxy.auth.user_api_key_auth import _read_user_budget_row
|
||||
|
||||
db_down = ValueError("User doesn't exist in db. 'user_id'=u-1. Got error - Connection refused")
|
||||
|
||||
|
|
@ -1554,7 +1554,7 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down()
|
|||
"litellm.proxy.auth.user_api_key_auth.get_user_object",
|
||||
new=AsyncMock(side_effect=db_down),
|
||||
):
|
||||
budget = await _read_user_model_max_budget(
|
||||
row = await _read_user_budget_row(
|
||||
user_id="u-1",
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=DualCache(),
|
||||
|
|
@ -1562,14 +1562,14 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down()
|
|||
proxy_logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert budget is None
|
||||
assert row is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_budget_lookup_returns_the_budget_when_the_row_reads():
|
||||
"""Positive control: the tolerance above must not be swallowing every result."""
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget
|
||||
from litellm.proxy.auth.user_api_key_auth import _read_user_budget_row
|
||||
|
||||
stored = {"claude-opus-4-8": {"budget_limit": 1.0, "time_period": "18h"}}
|
||||
user_obj = MagicMock()
|
||||
|
|
@ -1579,7 +1579,7 @@ async def test_user_budget_lookup_returns_the_budget_when_the_row_reads():
|
|||
"litellm.proxy.auth.user_api_key_auth.get_user_object",
|
||||
new=AsyncMock(return_value=user_obj),
|
||||
):
|
||||
budget = await _read_user_model_max_budget(
|
||||
row = await _read_user_budget_row(
|
||||
user_id="user-1",
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=DualCache(),
|
||||
|
|
@ -1587,7 +1587,7 @@ async def test_user_budget_lookup_returns_the_budget_when_the_row_reads():
|
|||
proxy_logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert budget == stored
|
||||
assert row is not None and row.model_max_budget == stored
|
||||
|
||||
|
||||
def test_zero_cost_models_skip_the_user_budget_check_on_every_path():
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import litellm
|
|||
|
||||
from litellm.proxy._types import (
|
||||
DEFAULT_JWKS_STALE_TTL,
|
||||
JWTAuthBuilderResult,
|
||||
JWTLiteLLMRoleMap,
|
||||
LiteLLM_JWTAuth,
|
||||
LiteLLM_ModelTable,
|
||||
|
|
@ -7144,3 +7145,62 @@ async def test_admin_jwt_team_header_only_provisions_during_admission(monkeypatc
|
|||
else:
|
||||
create_team.assert_not_awaited()
|
||||
assert result["team_id"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jwt_built_user_api_key_auth_carries_team_and_user_budget_fallbacks():
|
||||
"""A JWT-authenticated caller has no key row, so the fallback chains have
|
||||
to ride team_grants and the user fields on UserAPIKeyAuth for the
|
||||
auth-time reroute to find them."""
|
||||
result: JWTAuthBuilderResult = {
|
||||
"is_proxy_admin": False,
|
||||
"team_object": LiteLLM_TeamTable(
|
||||
team_id="team-1",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
),
|
||||
"user_object": LiteLLM_UserTable(
|
||||
user_id="user-1",
|
||||
budget_fallbacks={"gpt-4o": ["claude-haiku"]},
|
||||
),
|
||||
"end_user_object": None,
|
||||
"org_object": None,
|
||||
"token": "jwt-token",
|
||||
"team_id": "team-1",
|
||||
"user_id": "user-1",
|
||||
"user_email": None,
|
||||
"end_user_id": None,
|
||||
"org_id": None,
|
||||
"team_membership": None,
|
||||
"jwt_claims": {"sub": "user-1"},
|
||||
}
|
||||
|
||||
auth = JWTAuthManager.user_api_key_auth_from_result(result)
|
||||
|
||||
assert auth.team_budget_fallbacks == {"gpt-4o": ["gpt-4o-mini"]}
|
||||
assert auth.user_budget_fallbacks == {"gpt-4o": ["claude-haiku"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jwt_admin_does_not_inherit_user_budget_fallbacks():
|
||||
result: JWTAuthBuilderResult = {
|
||||
"is_proxy_admin": True,
|
||||
"team_object": None,
|
||||
"user_object": LiteLLM_UserTable(
|
||||
user_id="admin-1",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
),
|
||||
"end_user_object": None,
|
||||
"org_object": None,
|
||||
"token": "jwt-token",
|
||||
"team_id": None,
|
||||
"user_id": "admin-1",
|
||||
"user_email": None,
|
||||
"end_user_id": None,
|
||||
"org_id": None,
|
||||
"team_membership": None,
|
||||
"jwt_claims": {"sub": "admin-1"},
|
||||
}
|
||||
|
||||
auth = JWTAuthManager.user_api_key_auth_from_result(result)
|
||||
|
||||
assert auth.user_budget_fallbacks is None
|
||||
|
|
|
|||
|
|
@ -7083,6 +7083,16 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
request.scope = {}
|
||||
return request
|
||||
|
||||
def _exceeded_limiter(self, within: frozenset, error=None):
|
||||
async def check(user_api_key_dict=None, model="", **_kwargs):
|
||||
if model in within:
|
||||
return True
|
||||
raise error or litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
limiter = AsyncMock()
|
||||
limiter.is_key_within_model_budget.side_effect = check
|
||||
return limiter
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_within_budget_does_not_reroute(self):
|
||||
valid_token = UserAPIKeyAuth(
|
||||
|
|
@ -7111,11 +7121,7 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
token="test-key",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]},
|
||||
)
|
||||
limiter = AsyncMock()
|
||||
limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError(
|
||||
current_cost=10, max_budget=5
|
||||
)
|
||||
limiter.get_fallback_model_within_budget.return_value = "gpt-4o-mini"
|
||||
limiter = self._exceeded_limiter(within=frozenset({"gpt-4o-mini"}))
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
|
|
@ -7128,8 +7134,8 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
)
|
||||
|
||||
assert request_data["model"] == "gpt-4o-mini"
|
||||
limiter.get_fallback_model_within_budget.assert_awaited_once_with(
|
||||
user_api_key_dict=valid_token, model="gpt-4o"
|
||||
limiter.is_key_within_model_budget.assert_any_await(
|
||||
user_api_key_dict=valid_token, model="gpt-4o-mini"
|
||||
)
|
||||
# the rerouted model must be visible to a later, separate
|
||||
# `_read_request_body` call on the same `request` (route handlers
|
||||
|
|
@ -7145,7 +7151,6 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
limiter = AsyncMock()
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
limiter.is_key_within_model_budget.side_effect = original_error
|
||||
limiter.get_fallback_model_within_budget.return_value = None
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
|
|
@ -7174,7 +7179,9 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
limiter = AsyncMock()
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
limiter.is_key_within_model_budget.side_effect = original_error
|
||||
limiter.get_fallback_model_within_budget.return_value = "restricted-model"
|
||||
limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter(
|
||||
within=frozenset({"restricted-model"}), error=original_error
|
||||
).is_key_within_model_budget.side_effect
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
|
|
@ -7211,10 +7218,9 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
)
|
||||
limiter = AsyncMock()
|
||||
limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError(
|
||||
current_cost=10, max_budget=5
|
||||
)
|
||||
limiter.get_fallback_model_within_budget.return_value = "gpt-4o-mini"
|
||||
limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter(
|
||||
within=frozenset({"gpt-4o-mini"})
|
||||
).is_key_within_model_budget.side_effect
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
|
|
@ -7248,7 +7254,9 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
limiter = AsyncMock()
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
limiter.is_key_within_model_budget.side_effect = original_error
|
||||
limiter.get_fallback_model_within_budget.return_value = "restricted-model"
|
||||
limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter(
|
||||
within=frozenset({"restricted-model"}), error=original_error
|
||||
).is_key_within_model_budget.side_effect
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
|
|
@ -7281,10 +7289,9 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
)
|
||||
limiter = AsyncMock()
|
||||
limiter.is_key_within_model_budget.side_effect = litellm.BudgetExceededError(
|
||||
current_cost=10, max_budget=5
|
||||
)
|
||||
limiter.get_fallback_model_within_budget.return_value = "gpt-4o-mini"
|
||||
limiter.is_key_within_model_budget.side_effect = self._exceeded_limiter(
|
||||
within=frozenset({"gpt-4o-mini"})
|
||||
).is_key_within_model_budget.side_effect
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
request.scope["path_params"] = {"model": "gpt-4o"}
|
||||
|
|
@ -7335,6 +7342,192 @@ class TestCheckKeyModelBudgetWithFallback:
|
|||
assert "model" not in request_data
|
||||
|
||||
|
||||
class TestEntityModelBudgetFallbacks:
|
||||
"""Team and user `budget_fallbacks` must reroute a request whose model
|
||||
exceeds that entity's per-model budget to the first fallback still within
|
||||
the same entity's budget, using the same authorization and request
|
||||
rewrite the key-level fallback applies."""
|
||||
|
||||
def _make_request(self):
|
||||
request = MagicMock()
|
||||
request.scope = {}
|
||||
return request
|
||||
|
||||
def _exceeded_except(self, within: frozenset):
|
||||
async def is_team_within_model_budget(
|
||||
team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""
|
||||
):
|
||||
if model in within:
|
||||
return True
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def is_user_within_model_budget(user_id=None, user_model_max_budget=None, model=""):
|
||||
if model in within:
|
||||
return True
|
||||
raise litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_team_within_model_budget = is_team_within_model_budget
|
||||
limiter.is_user_within_model_budget = is_user_within_model_budget
|
||||
return limiter
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_fallback_reroutes_request_to_fallback_model(self):
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
|
||||
|
||||
valid_token = UserAPIKeyAuth(
|
||||
team_id="team-1",
|
||||
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
request.scope["path_params"] = {"model": "gpt-4o"}
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
await _check_team_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=self._exceeded_except(frozenset({"gpt-4o-mini"})),
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert request_data["model"] == "gpt-4o-mini"
|
||||
_, cached_body = request.scope["parsed_body"]
|
||||
assert cached_body["model"] == "gpt-4o-mini"
|
||||
assert request.scope["path_params"]["model"] == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_fallback_reroutes_request_to_fallback_model(self):
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget
|
||||
|
||||
valid_token = UserAPIKeyAuth(
|
||||
user_id="user-1",
|
||||
user_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
user_budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
await _check_user_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=self._exceeded_except(frozenset({"gpt-4o-mini"})),
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert request_data["model"] == "gpt-4o-mini"
|
||||
_, cached_body = request.scope["parsed_body"]
|
||||
assert cached_body["model"] == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_fallback_not_in_team_models_raises_original_error(self):
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
|
||||
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def over_budget(team_id=None, team_model_max_budget=None, key_model_max_budget=None, model=""):
|
||||
if model == "restricted-model":
|
||||
return True
|
||||
raise original_error
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_team_within_model_budget = over_budget
|
||||
valid_token = UserAPIKeyAuth(
|
||||
team_id="team-1",
|
||||
team_models=["gpt-4o"],
|
||||
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
team_budget_fallbacks={"gpt-4o": ["restricted-model"]},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
with pytest.raises(litellm.BudgetExceededError) as exc_info:
|
||||
await _check_team_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
)
|
||||
|
||||
assert exc_info.value is original_error
|
||||
assert request_data["model"] == "gpt-4o"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_without_fallbacks_raises_original_error(self):
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_user_model_budget
|
||||
|
||||
original_error = litellm.BudgetExceededError(current_cost=10, max_budget=5)
|
||||
|
||||
async def over_budget(**_kwargs):
|
||||
raise original_error
|
||||
|
||||
limiter = MagicMock()
|
||||
limiter.is_user_within_model_budget = over_budget
|
||||
valid_token = UserAPIKeyAuth(
|
||||
user_id="user-1",
|
||||
user_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with pytest.raises(litellm.BudgetExceededError) as exc_info:
|
||||
await _check_user_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=limiter,
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert exc_info.value is original_error
|
||||
assert request_data["model"] == "gpt-4o"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_fallback_only_tries_first_within_budget_fallback(self):
|
||||
"""A fallback that is itself over budget must be skipped, matching the
|
||||
key-level semantics of first-in-order chain resolution."""
|
||||
from litellm.proxy.auth.user_api_key_auth import _check_team_model_budget
|
||||
|
||||
valid_token = UserAPIKeyAuth(
|
||||
team_id="team-1",
|
||||
team_model_max_budget={"gpt-4o": {"max_budget": 5}},
|
||||
team_budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]},
|
||||
)
|
||||
request_data = {"model": "gpt-4o"}
|
||||
request = self._make_request()
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.auth.user_api_key_auth.can_key_call_model",
|
||||
new=AsyncMock(return_value=True),
|
||||
):
|
||||
await _check_team_model_budget(
|
||||
valid_token=valid_token,
|
||||
model_max_budget_limiter=self._exceeded_except(frozenset({"claude-haiku"})),
|
||||
models=["gpt-4o"],
|
||||
request_data=request_data,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert request_data["model"] == "claude-haiku"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_proxy_spend_reads_resettable_proxy_budget_row():
|
||||
"""Regression for the global proxy budget ignoring budget_duration
|
||||
|
|
|
|||
|
|
@ -4519,3 +4519,108 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo
|
|||
written_data = mock_prisma_client.update_data.call_args.kwargs["data"]
|
||||
assert written_data.get("password") is not None
|
||||
assert written_data["password"] != strong_password
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_user_forwards_budget_fallbacks_into_user_persistence(mocker):
|
||||
"""/user/new must carry budget_fallbacks into generate_key_helper_fn so it
|
||||
lands on the user row (previously the field was accepted but dropped)."""
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
|
||||
|
||||
async def _noop(*args, **kwargs):
|
||||
return None
|
||||
|
||||
mocker.patch(
|
||||
"litellm.proxy.management_endpoints.internal_user_endpoints._check_duplicate_user_email",
|
||||
_noop,
|
||||
)
|
||||
mocker.patch(
|
||||
"litellm.proxy.management_endpoints.internal_user_endpoints._check_duplicate_user_id",
|
||||
_noop,
|
||||
)
|
||||
key_gen = mocker.patch(
|
||||
"litellm.proxy.management_endpoints.internal_user_endpoints.generate_key_helper_fn",
|
||||
new=mocker.AsyncMock(side_effect=RuntimeError("reached key generation")),
|
||||
)
|
||||
prisma_client = mocker.MagicMock()
|
||||
prisma_client.db.litellm_usertable.count = mocker.AsyncMock(return_value=0)
|
||||
mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client)
|
||||
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
request = NewUserRequest(
|
||||
user_role="internal_user",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini", "claude-haiku"]},
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException):
|
||||
await new_user(data=request, user_api_key_dict=admin)
|
||||
|
||||
assert key_gen.call_count == 1
|
||||
assert key_gen.call_args.kwargs["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini", "claude-haiku"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_user_replaces_budget_fallbacks(_admin_prisma, mocker):
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import (
|
||||
_update_single_user_helper,
|
||||
)
|
||||
|
||||
mock_prisma_client = _admin_prisma
|
||||
existing_user = mocker.MagicMock()
|
||||
existing_user.model_dump.return_value = {"user_id": "target-user"}
|
||||
existing_user.user_id = "target-user"
|
||||
mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
|
||||
mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"})
|
||||
|
||||
user_request = UpdateUserRequest(
|
||||
user_id="target-user",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
)
|
||||
admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
|
||||
|
||||
written_data = mock_prisma_client.update_data.call_args.kwargs["data"]
|
||||
assert written_data["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_user_clears_budget_fallbacks_with_empty_map(_admin_prisma, mocker):
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import (
|
||||
_update_single_user_helper,
|
||||
)
|
||||
|
||||
mock_prisma_client = _admin_prisma
|
||||
existing_user = mocker.MagicMock()
|
||||
existing_user.model_dump.return_value = {"user_id": "target-user"}
|
||||
existing_user.user_id = "target-user"
|
||||
mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user)
|
||||
mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"})
|
||||
|
||||
user_request = UpdateUserRequest(user_id="target-user", budget_fallbacks={})
|
||||
admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
|
||||
|
||||
written_data = mock_prisma_client.update_data.call_args.kwargs["data"]
|
||||
assert written_data["budget_fallbacks"] == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"bad_value",
|
||||
[["gpt-4o-mini"], {"gpt-4o": "gpt-4o-mini"}, {"gpt-4o": [1, 2]}],
|
||||
)
|
||||
async def test_update_internal_user_params_rejects_malformed_budget_fallbacks(bad_value):
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import (
|
||||
_update_internal_user_params,
|
||||
)
|
||||
|
||||
data = UpdateUserRequest(user_id="u-1", budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]})
|
||||
data_json = data.model_dump(exclude_unset=True)
|
||||
data_json["budget_fallbacks"] = bad_value
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_update_internal_user_params(data_json=data_json, data=data)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
|
|
|
|||
|
|
@ -15422,3 +15422,101 @@ async def test_team_info_reports_what_the_caller_may_edit(caller, org_admin, ena
|
|||
)
|
||||
|
||||
assert response["team_info"].caller_edit_access.model_dump(mode="json") == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_team_persists_budget_fallbacks(mock_db_client, mock_admin_auth):
|
||||
mock_db_client.jsonify_team_object = lambda db_data: db_data
|
||||
mock_db_client.get_data = AsyncMock(return_value=None)
|
||||
mock_db_client.update_data = AsyncMock(return_value=MagicMock())
|
||||
mock_db_client.db = MagicMock()
|
||||
mock_db_client.db.litellm_modeltable = MagicMock()
|
||||
mock_db_client.db.litellm_modeltable.create = AsyncMock(return_value=MagicMock(id="model123"))
|
||||
|
||||
team_create_result = MagicMock(team_id="team-fallbacks")
|
||||
team_create_result.model_dump.return_value = {"team_id": "team-fallbacks"}
|
||||
mock_team_create = AsyncMock(return_value=team_create_result)
|
||||
mock_db_client.db.litellm_teamtable = MagicMock()
|
||||
mock_db_client.db.litellm_teamtable.create = mock_team_create
|
||||
_wire_team_create_tx(mock_db_client)
|
||||
mock_db_client.db.litellm_teamtable.count = AsyncMock(return_value=0)
|
||||
mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=team_create_result)
|
||||
mock_db_client.db.litellm_usertable = MagicMock()
|
||||
mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock())
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import NewTeamRequest
|
||||
from litellm.proxy.management_endpoints.team_endpoints import new_team
|
||||
|
||||
await new_team(
|
||||
data=NewTeamRequest(
|
||||
team_alias="fallbacks-team",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
),
|
||||
http_request=MagicMock(spec=Request),
|
||||
user_api_key_dict=mock_admin_auth,
|
||||
)
|
||||
|
||||
team_data = mock_team_create.call_args.kwargs["data"]
|
||||
assert team_data["budget_fallbacks"] == {"gpt-4o": ["gpt-4o-mini"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_team_replaces_budget_fallbacks(disable_audit_logging_for_mocked_team):
|
||||
from unittest.mock import Mock
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UpdateTeamRequest, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.team_endpoints import update_team
|
||||
|
||||
mock_user_api_key_dict = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1")
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client,
|
||||
patch("litellm.proxy.proxy_server.llm_router"),
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache"),
|
||||
patch("litellm.proxy.proxy_server.proxy_logging_obj"),
|
||||
patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"),
|
||||
patch("litellm.proxy.management_endpoints.team_endpoints._cache_team_object"),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.team_endpoints.TeamMemberBudgetHandler.upsert_team_member_budget_table",
|
||||
new=AsyncMock(side_effect=lambda **kwargs: kwargs["updated_kv"]),
|
||||
),
|
||||
):
|
||||
mock_existing_team = MagicMock()
|
||||
mock_existing_team.model_dump.return_value = {
|
||||
"team_id": "team-1",
|
||||
"team_alias": "team",
|
||||
"metadata": {},
|
||||
"budget_fallbacks": {"gpt-4o": ["claude-haiku"]},
|
||||
}
|
||||
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=mock_existing_team)
|
||||
mock_updated_team = MagicMock()
|
||||
mock_updated_team.team_id = "team-1"
|
||||
mock_updated_team.model_dump.return_value = {"team_id": "team-1"}
|
||||
mock_prisma_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_updated_team)
|
||||
|
||||
import json as _json
|
||||
|
||||
def _jsonify(db_data):
|
||||
return {
|
||||
k: (_json.dumps(v) if isinstance(v, dict) else v)
|
||||
for k, v in db_data.items()
|
||||
}
|
||||
|
||||
mock_prisma_client.jsonify_team_object = MagicMock(side_effect=_jsonify)
|
||||
|
||||
await update_team(
|
||||
data=UpdateTeamRequest(
|
||||
team_id="team-1",
|
||||
budget_fallbacks={"gpt-4o": ["gpt-4o-mini"]},
|
||||
),
|
||||
http_request=Mock(spec=Request),
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
)
|
||||
|
||||
update_call = mock_prisma_client.db.litellm_teamtable.update.call_args
|
||||
written = update_call.kwargs["data"]
|
||||
assert _json.loads(written["budget_fallbacks"]) == {"gpt-4o": ["gpt-4o-mini"]}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue