mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 39cb563098 into dddbaa803d
This commit is contained in:
commit
b78c3b5f5a
6 changed files with 1289 additions and 71 deletions
|
|
@ -15,7 +15,11 @@ from collections.abc import Mapping, Sequence
|
|||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import TypeAdapter
|
||||
from litellm_enterprise.proxy.management_endpoints.project_model_quota import (
|
||||
describe_missing_quota,
|
||||
missing_project_model_quota,
|
||||
)
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -27,6 +31,15 @@ from litellm.proxy.management.teams.dependencies import get_team_access
|
|||
from litellm.proxy.management.users.service import org_admin_org_ids
|
||||
from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
|
||||
from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
|
||||
from litellm.proxy.management_helpers.model_quota_policy import (
|
||||
ENFORCE_PROJECT_MODEL_QUOTA_SETTING,
|
||||
REQUIRE_BOTH_LIMITS,
|
||||
ModelQuotaPolicy,
|
||||
RouteOf,
|
||||
no_route,
|
||||
parse_model_quota_setting,
|
||||
router_model_routes,
|
||||
)
|
||||
from litellm.proxy.management_helpers.utils import (
|
||||
management_endpoint_wrapper,
|
||||
)
|
||||
|
|
@ -276,26 +289,6 @@ def _check_team_project_limits(
|
|||
)
|
||||
|
||||
|
||||
def _project_models_missing_positive_quota(
|
||||
models: list[str] | None,
|
||||
rpm_limits: Mapping[str, object] | None,
|
||||
tpm_limits: Mapping[str, object] | None,
|
||||
) -> list[str]:
|
||||
"""Return the models that lack a positive `rpm` AND `tpm` quota.
|
||||
|
||||
A valid quota is a positive integer; null, zero, and negative are rejected
|
||||
because downstream rate limiters treat a non-positive limit as immediately
|
||||
exhausted (every request blocked).
|
||||
"""
|
||||
|
||||
def _is_positive(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value > 0
|
||||
|
||||
rpm = rpm_limits or {}
|
||||
tpm = tpm_limits or {}
|
||||
return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
|
||||
|
||||
|
||||
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
|
||||
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
|
||||
|
||||
|
|
@ -322,64 +315,79 @@ def _raise_on_project_models_expanding_at_request_time(
|
|||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
|
||||
"error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while '{ENFORCE_PROJECT_MODEL_QUOTA_SETTING}' is enabled. List concrete model names instead."
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _project_model_quota_policy(general_settings: Mapping[str, object]) -> ModelQuotaPolicy | None:
|
||||
try:
|
||||
return parse_model_quota_setting(general_settings.get(ENFORCE_PROJECT_MODEL_QUOTA_SETTING))
|
||||
except ValidationError as e:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": f"general_settings.{ENFORCE_PROJECT_MODEL_QUOTA_SETTING} is invalid: {e}"},
|
||||
) from e
|
||||
|
||||
|
||||
def _raise_on_missing_project_model_quota(
|
||||
data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
|
||||
data: NewProjectRequest | UpdateProjectRequest,
|
||||
access_group_names: frozenset[str] = frozenset(),
|
||||
policy: ModelQuotaPolicy = REQUIRE_BOTH_LIMITS,
|
||||
route_of: RouteOf = no_route,
|
||||
) -> None:
|
||||
"""Require a positive `rpm`/`tpm` quota for every model on project CREATE.
|
||||
"""Require the positive `rpm`/`tpm` quotas the policy asks of each model on project CREATE.
|
||||
|
||||
`model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
|
||||
model's `set_model_info` validator, so they are read from there.
|
||||
|
||||
Only invoked when `general_settings.enforce_project_model_quota` is enabled
|
||||
(default off), so it is opt-in and does not change behavior for existing users.
|
||||
"""
|
||||
_raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
|
||||
metadata = data.metadata or {}
|
||||
missing = _project_models_missing_positive_quota(
|
||||
data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
|
||||
missing: Final = missing_project_model_quota(
|
||||
data.models or (), metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit"), policy, route_of
|
||||
)
|
||||
if not missing:
|
||||
return
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
|
||||
"error": f"models added to project without a positive rpm/tpm quota: {describe_missing_quota(missing)}. Set a positive value for each listed limit."
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _raise_on_missing_project_model_quota_on_update(
|
||||
data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
|
||||
data: UpdateProjectRequest,
|
||||
existing_project: object,
|
||||
access_group_names: frozenset[str] = frozenset(),
|
||||
policy: ModelQuotaPolicy = REQUIRE_BOTH_LIMITS,
|
||||
route_of: RouteOf = no_route,
|
||||
) -> None:
|
||||
"""Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
|
||||
"""Require the positive `rpm`/`tpm` quotas the policy asks of each model over the RESULTING state on project UPDATE.
|
||||
|
||||
`/project/update` replaces `models` and `metadata` when they are provided, so the
|
||||
check runs on what the project WILL look like: a partial update that doesn't touch
|
||||
models/quota keeps the existing values, while one that adds a model or clears a
|
||||
model's quota must leave every resulting model with a positive limit.
|
||||
|
||||
Only invoked when `general_settings.enforce_project_model_quota` is enabled
|
||||
(default off), so it is opt-in and does not change behavior for existing users.
|
||||
model's quota must leave every resulting model with the limits it needs.
|
||||
"""
|
||||
resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
|
||||
resulting_metadata = (
|
||||
data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
|
||||
)
|
||||
_raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
|
||||
missing = _project_models_missing_positive_quota(
|
||||
resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
|
||||
missing: Final = missing_project_model_quota(
|
||||
resulting_models,
|
||||
resulting_metadata.get("model_rpm_limit"),
|
||||
resulting_metadata.get("model_tpm_limit"),
|
||||
policy,
|
||||
route_of,
|
||||
)
|
||||
if not missing:
|
||||
return
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
|
||||
"error": f"models would be left on the project without a positive rpm/tpm quota: {describe_missing_quota(missing)}. Set a positive value for each listed limit."
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -530,6 +538,7 @@ async def new_project(
|
|||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
general_settings_view,
|
||||
litellm_proxy_admin_name,
|
||||
llm_router,
|
||||
premium_user,
|
||||
|
|
@ -578,10 +587,6 @@ async def new_project(
|
|||
data=data,
|
||||
)
|
||||
|
||||
# Opt-in (default off): require rpm/tpm for every model added to the project.
|
||||
if general_settings.get("enforce_project_model_quota", False):
|
||||
_raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
|
||||
|
||||
# Check if user has permission to create projects for this team
|
||||
# only team admins can create projects for their team
|
||||
has_permission = await _check_user_permission_for_project(
|
||||
|
|
@ -600,6 +605,15 @@ async def new_project(
|
|||
},
|
||||
)
|
||||
|
||||
quota_policy: Final = _project_model_quota_policy(general_settings_view())
|
||||
if quota_policy is not None:
|
||||
_raise_on_missing_project_model_quota(
|
||||
data,
|
||||
_router_access_group_names(llm_router),
|
||||
quota_policy,
|
||||
router_model_routes(llm_router, data.team_id),
|
||||
)
|
||||
|
||||
# Generate project_id if not provided
|
||||
if data.project_id is None:
|
||||
data.project_id = str(uuid.uuid4())
|
||||
|
|
@ -721,6 +735,7 @@ async def update_project(
|
|||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
general_settings_view,
|
||||
litellm_proxy_admin_name,
|
||||
llm_router,
|
||||
premium_user,
|
||||
|
|
@ -828,10 +843,14 @@ async def update_project(
|
|||
data=data,
|
||||
)
|
||||
|
||||
# Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
|
||||
if general_settings.get("enforce_project_model_quota", False):
|
||||
quota_policy: Final = _project_model_quota_policy(general_settings_view())
|
||||
if quota_policy is not None:
|
||||
_raise_on_missing_project_model_quota_on_update(
|
||||
data, existing_project, _router_access_group_names(llm_router)
|
||||
data,
|
||||
existing_project,
|
||||
_router_access_group_names(llm_router),
|
||||
quota_policy,
|
||||
router_model_routes(llm_router, target_team_id),
|
||||
)
|
||||
|
||||
# Prepare update data
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.proxy.management_helpers.model_quota_policy import (
|
||||
ModelQuotaPolicy,
|
||||
QuotaLimit,
|
||||
RouteOf,
|
||||
required_limits,
|
||||
unmet_limits,
|
||||
)
|
||||
|
||||
_NO_LIMITS_SET: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MissingModelQuota:
|
||||
model: str
|
||||
limits: tuple[QuotaLimit, ...]
|
||||
|
||||
|
||||
def _limits_set_for(
|
||||
model: str, rpm_set: Mapping[str, object], tpm_set: Mapping[str, object]
|
||||
) -> Mapping[QuotaLimit, object]:
|
||||
return MappingProxyType({"rpm": rpm_set.get(model), "tpm": tpm_set.get(model)})
|
||||
|
||||
|
||||
def missing_project_model_quota(
|
||||
models: Sequence[str],
|
||||
rpm_limits: Mapping[str, object] | None,
|
||||
tpm_limits: Mapping[str, object] | None,
|
||||
policy: ModelQuotaPolicy,
|
||||
route_of: RouteOf,
|
||||
) -> tuple[MissingModelQuota, ...]:
|
||||
rpm_set: Final = rpm_limits or _NO_LIMITS_SET
|
||||
tpm_set: Final = tpm_limits or _NO_LIMITS_SET
|
||||
gaps: Final = tuple(
|
||||
MissingModelQuota(
|
||||
model,
|
||||
unmet_limits(required_limits(policy, model, route_of(model)), _limits_set_for(model, rpm_set, tpm_set)),
|
||||
)
|
||||
for model in models
|
||||
)
|
||||
return tuple(gap for gap in gaps if gap.limits)
|
||||
|
||||
|
||||
def describe_missing_quota(missing: Sequence[MissingModelQuota]) -> str:
|
||||
return ", ".join(f"{gap.model} ({' and '.join(f'model_{limit}_limit' for limit in gap.limits)})" for gap in missing)
|
||||
|
|
@ -42,6 +42,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
|
|||
ptu_config_error,
|
||||
)
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.proxy._types import (
|
||||
BlockModelRequest,
|
||||
CommonProxyErrors,
|
||||
|
|
@ -102,6 +103,16 @@ from litellm.proxy.management_helpers.auto_router_permissions import (
|
|||
authorize_member_auto_router_write,
|
||||
)
|
||||
from litellm.proxy.management_helpers.model_allowlist_rename_sync import sync_model_allowlists_for_renamed_model
|
||||
from litellm.proxy.management_helpers.model_quota_policy import (
|
||||
ENFORCE_PROJECT_MODEL_QUOTA_SETTING,
|
||||
ModelQuotaPolicy,
|
||||
ModelRoute,
|
||||
deployment_provider,
|
||||
limits_added_by_team_write,
|
||||
parse_model_quota_setting,
|
||||
required_limits,
|
||||
unmet_limits,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.ptu_feature_flag import (
|
||||
PTU_COST_ATTRIBUTION_ENV_VAR,
|
||||
is_ptu_cost_attribution_enabled,
|
||||
|
|
@ -111,6 +122,7 @@ from litellm.repositories.chunked_in import find_many_in
|
|||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.repositories.model_repository import ModelRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.project_repository import ProjectRepository
|
||||
from litellm.repositories.table_repositories import ModelTableRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.router import Router
|
||||
|
|
@ -151,6 +163,7 @@ from litellm.types.router import (
|
|||
SPECIAL_MODEL_INFO_PARAMS,
|
||||
Deployment,
|
||||
GenericLiteLLMParams,
|
||||
LiteLLM_Params,
|
||||
ModelInfo,
|
||||
updateDeployment,
|
||||
updateLiteLLMParams,
|
||||
|
|
@ -760,26 +773,53 @@ async def _auto_router_capability_slot(
|
|||
ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add"
|
||||
|
||||
|
||||
def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLMParams, enforced: bool) -> None:
|
||||
"""Require both rpm and tpm (each a positive value) when the operator opts in via config.yaml.
|
||||
_ROUTING_KEYS: Final = ("model", "custom_llm_provider", "api_base")
|
||||
_NOTHING_SENT: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
Off by default, so deployments keep adding models without limits. When
|
||||
``enforce_rpm_tpm_on_model_add: true`` is set under general_settings, a model added
|
||||
without both rpm and tpm set to a positive value is rejected rather than stored
|
||||
unbounded (or effectively excluded from routing by a zero/negative limit).
|
||||
"""
|
||||
if not enforced:
|
||||
|
||||
def _written_routing_params(
|
||||
incoming: GenericLiteLLMParams | None, existing: GenericLiteLLMParams | None
|
||||
) -> Mapping[str, object]:
|
||||
sent: Final = _NOTHING_SENT if incoming is None else _dump_without_none(incoming)
|
||||
stored: Final = _NOTHING_SENT if existing is None else _decrypted_litellm_params(existing)
|
||||
return MappingProxyType({key: sent.get(key, stored.get(key)) for key in _ROUTING_KEYS})
|
||||
|
||||
|
||||
def _quota_policy(general_settings: Mapping[str, object], setting: str) -> ModelQuotaPolicy | None:
|
||||
try:
|
||||
return parse_model_quota_setting(general_settings.get(setting))
|
||||
except ValidationError as e:
|
||||
raise ProxyException(
|
||||
message=f"general_settings.{setting} is invalid: {e}",
|
||||
type=ProxyErrorTypes.internal_server_error.value,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
param=None,
|
||||
) from e
|
||||
|
||||
|
||||
def _raise_if_rate_limits_required_but_missing(
|
||||
*, model_name: str, litellm_params: LiteLLM_Params, team_model: bool, policy: ModelQuotaPolicy | None
|
||||
) -> None:
|
||||
"""Off by default, so deployments keep adding models without limits. Once the operator opts in under
|
||||
general_settings, the policy decides per model name and provider which of rpm and tpm a new model needs,
|
||||
and a model missing one, or carrying a zero or negative one, is rejected rather than stored unbounded
|
||||
(or effectively excluded from routing)."""
|
||||
if policy is None:
|
||||
return
|
||||
missing: Final = tuple(
|
||||
field
|
||||
for field, value in (("rpm", litellm_params.rpm), ("tpm", litellm_params.tpm))
|
||||
if value is None or value <= 0
|
||||
provider: Final = deployment_provider(_written_routing_params(litellm_params, None))
|
||||
missing: Final = unmet_limits(
|
||||
required_limits(
|
||||
policy,
|
||||
model_name,
|
||||
ModelRoute(providers=frozenset() if provider is None else frozenset((provider,)), team_named=team_model),
|
||||
),
|
||||
MappingProxyType({"rpm": litellm_params.rpm, "tpm": litellm_params.tpm}),
|
||||
)
|
||||
if not missing:
|
||||
return
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"{' and '.join(missing)} must be set to a positive value when "
|
||||
f"{' and '.join(missing)} must be set to a positive value for '{model_name}' when "
|
||||
f"'{ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING}' is enabled in general_settings"
|
||||
),
|
||||
type=ProxyErrorTypes.validation_error.value,
|
||||
|
|
@ -788,6 +828,94 @@ def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLM
|
|||
)
|
||||
|
||||
|
||||
def _project_limit(metadata: object, limits_key: str, model: str) -> object:
|
||||
limits: Final = metadata.get(limits_key) if is_str_keyed_mapping(metadata) else None
|
||||
return limits.get(model) if is_str_keyed_mapping(limits) else None
|
||||
|
||||
|
||||
async def _raise_if_team_write_leaves_projects_short(
|
||||
*,
|
||||
prisma_client: PrismaClient,
|
||||
general_settings: Mapping[str, object],
|
||||
llm_router: Router | None,
|
||||
team_id: str | None,
|
||||
public_name: str,
|
||||
replaced_id: str | None,
|
||||
written_params: Mapping[str, object] | None,
|
||||
) -> None:
|
||||
if team_id is None:
|
||||
return
|
||||
policy: Final = _quota_policy(general_settings, ENFORCE_PROJECT_MODEL_QUOTA_SETTING)
|
||||
if policy is None:
|
||||
return
|
||||
added: Final = limits_added_by_team_write(policy, llm_router, team_id, public_name, replaced_id, written_params)
|
||||
if not added:
|
||||
return
|
||||
projects: Final = await ProjectRepository(prisma_client).table.find_many(
|
||||
where={"team_id": team_id, "models": {"has": public_name}}
|
||||
)
|
||||
short: Final = tuple(
|
||||
project.project_alias or project.project_id
|
||||
for project in projects
|
||||
if unmet_limits(
|
||||
added,
|
||||
MappingProxyType(
|
||||
{
|
||||
"rpm": _project_limit(project.metadata, "model_rpm_limit", public_name),
|
||||
"tpm": _project_limit(project.metadata, "model_tpm_limit", public_name),
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
if not short:
|
||||
return
|
||||
raise ProxyException(
|
||||
message=(
|
||||
f"projects {', '.join(short)} list '{public_name}' without a positive {' and '.join(sorted(added))} "
|
||||
f"quota, which '{ENFORCE_PROJECT_MODEL_QUOTA_SETTING}' requires once this team model change applies. "
|
||||
"Set those limits on the projects first"
|
||||
),
|
||||
type=ProxyErrorTypes.validation_error.value,
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
param=None,
|
||||
)
|
||||
|
||||
|
||||
async def _raise_if_team_update_leaves_projects_short(
|
||||
*,
|
||||
prisma_client: PrismaClient,
|
||||
general_settings: Mapping[str, object],
|
||||
llm_router: Router | None,
|
||||
existing: Deployment,
|
||||
written_team_id: str | None,
|
||||
written_name: str,
|
||||
incoming_params: GenericLiteLLMParams | None,
|
||||
) -> None:
|
||||
if written_team_id is None and existing.model_info.team_id is None:
|
||||
return
|
||||
stored_name: Final = existing.model_info.team_public_model_name or existing.model_name
|
||||
await _raise_if_team_write_leaves_projects_short(
|
||||
prisma_client=prisma_client,
|
||||
general_settings=general_settings,
|
||||
llm_router=llm_router,
|
||||
team_id=written_team_id,
|
||||
public_name=written_name,
|
||||
replaced_id=existing.model_info.id,
|
||||
written_params=_written_routing_params(incoming_params, existing.litellm_params),
|
||||
)
|
||||
if (written_team_id, written_name) == (existing.model_info.team_id, stored_name):
|
||||
return
|
||||
await _raise_if_team_write_leaves_projects_short(
|
||||
prisma_client=prisma_client,
|
||||
general_settings=general_settings,
|
||||
llm_router=llm_router,
|
||||
team_id=existing.model_info.team_id,
|
||||
public_name=stored_name,
|
||||
replaced_id=existing.model_info.id,
|
||||
written_params=None,
|
||||
)
|
||||
|
||||
|
||||
_PTU_PRICED_PAIR: Final = frozenset({"ptu_count", "cost_per_ptu_per_hour"})
|
||||
|
||||
|
||||
|
|
@ -1276,6 +1404,7 @@ async def patch_model(
|
|||
ProxyException: For various error conditions including authentication and database errors
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings_view,
|
||||
litellm_proxy_admin_name,
|
||||
llm_router,
|
||||
premium_user,
|
||||
|
|
@ -1385,6 +1514,17 @@ async def patch_model(
|
|||
existing_params=db_model.litellm_params,
|
||||
)
|
||||
|
||||
await _raise_if_team_update_leaves_projects_short(
|
||||
prisma_client=prisma_client,
|
||||
general_settings=general_settings_view(),
|
||||
llm_router=llm_router,
|
||||
existing=db_model,
|
||||
written_team_id=(patch_data.model_info.team_id if patch_data.model_info else None)
|
||||
or db_model.model_info.team_id,
|
||||
written_name=_get_public_model_name(effective_patch, db_model),
|
||||
incoming_params=patch_data.litellm_params,
|
||||
)
|
||||
|
||||
effective_params: Final = _effective_complexity_router_params(
|
||||
patch_data.litellm_params, db_model.litellm_params
|
||||
)
|
||||
|
|
@ -2460,6 +2600,7 @@ async def delete_model(
|
|||
|
||||
from litellm.proxy.proxy_server import (
|
||||
MODEL_RECONCILE_LOCK,
|
||||
general_settings_view,
|
||||
llm_router,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
|
|
@ -2493,6 +2634,15 @@ async def delete_model(
|
|||
incoming_params=None,
|
||||
allow_missing_team=True,
|
||||
)
|
||||
await _raise_if_team_write_leaves_projects_short(
|
||||
prisma_client=prisma_client,
|
||||
general_settings=general_settings_view(),
|
||||
llm_router=llm_router,
|
||||
team_id=model_params.model_info.team_id,
|
||||
public_name=model_params.model_info.team_public_model_name or model_params.model_name,
|
||||
replaced_id=model_params.model_info.id,
|
||||
written_params=None,
|
||||
)
|
||||
|
||||
# update DB
|
||||
if store_model_in_db is True:
|
||||
|
|
@ -2659,6 +2809,8 @@ async def add_new_model(
|
|||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
general_settings_view,
|
||||
llm_router,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
proxy_config,
|
||||
|
|
@ -2717,8 +2869,19 @@ async def add_new_model(
|
|||
)
|
||||
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name=model_params.model_name,
|
||||
litellm_params=model_params.litellm_params,
|
||||
enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)),
|
||||
team_model=model_params.model_info.team_id is not None,
|
||||
policy=_quota_policy(general_settings_view(), ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING),
|
||||
)
|
||||
await _raise_if_team_write_leaves_projects_short(
|
||||
prisma_client=prisma_client,
|
||||
general_settings=general_settings_view(),
|
||||
llm_router=llm_router,
|
||||
team_id=model_params.model_info.team_id,
|
||||
public_name=model_params.model_name,
|
||||
replaced_id=None,
|
||||
written_params=_written_routing_params(model_params.litellm_params, None),
|
||||
)
|
||||
|
||||
clean_model_info: Final = ModelInfo.model_validate(
|
||||
|
|
@ -2857,6 +3020,7 @@ async def update_model(
|
|||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
LITELLM_PROXY_ADMIN_NAME,
|
||||
general_settings_view,
|
||||
llm_router,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
|
|
@ -2921,6 +3085,15 @@ async def update_model(
|
|||
incoming_params=model_params.litellm_params,
|
||||
existing_params=deployment.litellm_params,
|
||||
)
|
||||
await _raise_if_team_update_leaves_projects_short(
|
||||
prisma_client=prisma_client,
|
||||
general_settings=general_settings_view(),
|
||||
llm_router=llm_router,
|
||||
existing=deployment,
|
||||
written_team_id=deployment.model_info.team_id,
|
||||
written_name=deployment.model_info.team_public_model_name or deployment.model_name,
|
||||
incoming_params=model_params.litellm_params,
|
||||
)
|
||||
effective_params: Final = _effective_complexity_router_params(
|
||||
model_params.litellm_params, deployment.litellm_params
|
||||
)
|
||||
|
|
|
|||
213
litellm/proxy/management_helpers/model_quota_policy.py
Normal file
213
litellm/proxy/management_helpers/model_quota_policy.py
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider, is_openai_compatible_endpoint
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.router_utils.common_utils import provider_for_generic_call
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm import Router
|
||||
|
||||
QuotaLimit: TypeAlias = Literal["rpm", "tpm"]
|
||||
|
||||
ENFORCE_PROJECT_MODEL_QUOTA_SETTING: Final = "enforce_project_model_quota"
|
||||
|
||||
_LIMITS: Final[tuple[QuotaLimit, ...]] = ("rpm", "tpm")
|
||||
_ROUTER_PATTERNS: Final = PatternMatchRouter()
|
||||
|
||||
|
||||
def _is_known_provider(provider: str) -> bool:
|
||||
return provider in litellm.provider_list or JSONProviderRegistry.exists(provider)
|
||||
|
||||
|
||||
class ModelQuotaPolicy(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
default: frozenset[QuotaLimit] = frozenset(_LIMITS)
|
||||
providers: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict)
|
||||
models: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("providers")
|
||||
@classmethod
|
||||
def _known_providers(cls, providers: Mapping[str, frozenset[QuotaLimit]]) -> Mapping[str, frozenset[QuotaLimit]]:
|
||||
unknown: Final = sorted(provider for provider in providers if not _is_known_provider(provider))
|
||||
if unknown:
|
||||
raise ValueError(f"unknown provider ids {unknown}")
|
||||
return providers
|
||||
|
||||
|
||||
REQUIRE_BOTH_LIMITS: Final = ModelQuotaPolicy()
|
||||
_POLICY_KEYS: Final = frozenset(ModelQuotaPolicy.model_fields)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelRoute:
|
||||
providers: frozenset[str] = frozenset()
|
||||
team_named: bool = False
|
||||
|
||||
|
||||
RouteOf: TypeAlias = Callable[[str], ModelRoute]
|
||||
NO_ROUTE: Final = ModelRoute()
|
||||
|
||||
|
||||
def parse_model_quota_setting(setting: object) -> ModelQuotaPolicy | None:
|
||||
"""Any falsy value is off and any other value requires both limits, as the boolean setting always
|
||||
behaved, so only a mapping that names a policy key is read as a per-provider and per-model policy."""
|
||||
if not setting:
|
||||
return None
|
||||
if is_str_keyed_mapping(setting) and not _POLICY_KEYS.isdisjoint(setting):
|
||||
return ModelQuotaPolicy.model_validate(setting)
|
||||
return REQUIRE_BOTH_LIMITS
|
||||
|
||||
|
||||
def _wildcard_matches(pattern: str, model: str) -> bool:
|
||||
return re.match(_ROUTER_PATTERNS.pattern_to_regex(pattern), model) is not None
|
||||
|
||||
|
||||
def _router_specificity(pattern: str) -> tuple[int, int]:
|
||||
return PatternUtils.calculate_pattern_specificity(_ROUTER_PATTERNS.pattern_to_regex(pattern))
|
||||
|
||||
|
||||
def _most_specific_model_pattern(model: str, entries: Mapping[str, frozenset[QuotaLimit]]) -> str | None:
|
||||
matching: Final = tuple(pattern for pattern in entries if "*" in pattern and _wildcard_matches(pattern, model))
|
||||
return max(matching, key=_router_specificity, default=None)
|
||||
|
||||
|
||||
def _model_entry(entries: Mapping[str, frozenset[QuotaLimit]], model: str) -> frozenset[QuotaLimit] | None:
|
||||
exact: Final = entries.get(model)
|
||||
if exact is not None:
|
||||
return exact
|
||||
pattern: Final = _most_specific_model_pattern(model, entries)
|
||||
return None if pattern is None else entries[pattern]
|
||||
|
||||
|
||||
def required_limits(policy: ModelQuotaPolicy, model: str, route: ModelRoute) -> frozenset[QuotaLimit]:
|
||||
"""A model entry (exact name, then the most specific `*` pattern) wins over the providers of the
|
||||
model's deployments, which win over the default. A model served by several providers needs every
|
||||
limit any of them requires. A team model's name is picked by its team, so model entries never match it."""
|
||||
entry: Final = None if route.team_named else _model_entry(policy.models, model)
|
||||
if entry is not None:
|
||||
return entry
|
||||
if not route.providers:
|
||||
return policy.default
|
||||
return frozenset[QuotaLimit]().union(
|
||||
*(policy.providers.get(provider, policy.default) for provider in route.providers)
|
||||
)
|
||||
|
||||
|
||||
def _is_positive(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and value > 0
|
||||
|
||||
|
||||
def unmet_limits(required: frozenset[QuotaLimit], values: Mapping[QuotaLimit, object]) -> tuple[QuotaLimit, ...]:
|
||||
return tuple(
|
||||
limit
|
||||
for limit in _LIMITS
|
||||
if (limit in required or values.get(limit) is not None) and not _is_positive(values.get(limit))
|
||||
)
|
||||
|
||||
|
||||
def no_route(_model: str) -> ModelRoute:
|
||||
return NO_ROUTE
|
||||
|
||||
|
||||
def _provider_behind_api_base(litellm_params: Mapping[str, object]) -> str | None:
|
||||
model: Final = litellm_params.get("model")
|
||||
api_base: Final = litellm_params.get("api_base")
|
||||
if litellm_params.get("custom_llm_provider") or not isinstance(model, str) or not isinstance(api_base, str):
|
||||
return None
|
||||
if model.split("/", 1)[0] in litellm.provider_list or not is_openai_compatible_endpoint(api_base):
|
||||
return None
|
||||
try:
|
||||
_, provider, _, _ = get_llm_provider(model=model, api_base=api_base)
|
||||
except BadRequestError:
|
||||
return None
|
||||
return provider
|
||||
|
||||
|
||||
def deployment_provider(litellm_params: object) -> str | None:
|
||||
if not is_str_keyed_mapping(litellm_params):
|
||||
return None
|
||||
return _provider_behind_api_base(litellm_params) or provider_for_generic_call(litellm_params)
|
||||
|
||||
|
||||
def _routed_deployments(llm_router: "Router", model: str, team_id: str | None) -> Sequence[Mapping[str, object]]:
|
||||
named: Final[tuple[Mapping[str, object], ...]] = (
|
||||
*llm_router.get_all_deployments(model_name=model, team_id=team_id),
|
||||
*llm_router.get_model_list_from_model_alias(model_name=model),
|
||||
*llm_router.get_model_list_from_routing_groups(model_name=model),
|
||||
)
|
||||
return named or llm_router.wildcard_route_deployments(model, team_id)
|
||||
|
||||
|
||||
def _is_team_deployment(deployment: Mapping[str, object]) -> bool:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return is_str_keyed_mapping(model_info) and model_info.get("team_id") is not None
|
||||
|
||||
|
||||
def router_model_routes(llm_router: "Router | None", team_id: str | None) -> RouteOf:
|
||||
if llm_router is None:
|
||||
return no_route
|
||||
|
||||
def route_of(model: str) -> ModelRoute:
|
||||
deployments: Final = _routed_deployments(llm_router, model, team_id)
|
||||
return ModelRoute(
|
||||
providers=frozenset(
|
||||
provider
|
||||
for deployment in deployments
|
||||
if (provider := deployment_provider(deployment.get("litellm_params"))) is not None
|
||||
),
|
||||
team_named=any(_is_team_deployment(deployment) for deployment in deployments),
|
||||
)
|
||||
|
||||
return route_of
|
||||
|
||||
|
||||
def _deployment_id(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return model_info.get("id") if is_str_keyed_mapping(model_info) else None
|
||||
|
||||
|
||||
def team_route_after_write(
|
||||
llm_router: "Router | None",
|
||||
team_id: str,
|
||||
public_name: str,
|
||||
replaced_id: str | None,
|
||||
written_params: Mapping[str, object] | None,
|
||||
) -> ModelRoute:
|
||||
deployments: Final = () if llm_router is None else llm_router.get_all_deployments(public_name, team_id=team_id)
|
||||
kept: Final = tuple(
|
||||
deployment.get("litellm_params")
|
||||
for deployment in deployments
|
||||
if _is_team_deployment(deployment) and _deployment_id(deployment) != replaced_id
|
||||
)
|
||||
team_params: Final = kept if written_params is None else (*kept, written_params)
|
||||
if not team_params:
|
||||
return router_model_routes(llm_router, None)(public_name)
|
||||
return ModelRoute(
|
||||
providers=frozenset(
|
||||
provider for litellm_params in team_params if (provider := deployment_provider(litellm_params)) is not None
|
||||
),
|
||||
team_named=True,
|
||||
)
|
||||
|
||||
|
||||
def limits_added_by_team_write(
|
||||
policy: ModelQuotaPolicy,
|
||||
llm_router: "Router | None",
|
||||
team_id: str,
|
||||
public_name: str,
|
||||
replaced_id: str | None,
|
||||
written_params: Mapping[str, object] | None,
|
||||
) -> frozenset[QuotaLimit]:
|
||||
after: Final = team_route_after_write(llm_router, team_id, public_name, replaced_id, written_params)
|
||||
now: Final = router_model_routes(llm_router, team_id)(public_name)
|
||||
return required_limits(policy, public_name, after) - required_limits(policy, public_name, now)
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import os
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
|
@ -30,6 +30,7 @@ from litellm_enterprise.proxy.management_endpoints.project_endpoints import (
|
|||
delete_project,
|
||||
project_info,
|
||||
)
|
||||
from litellm.proxy.management_helpers.model_quota_policy import ModelRoute, router_model_routes
|
||||
from litellm.proxy.proxy_server import (
|
||||
LitellmUserRoles,
|
||||
)
|
||||
|
|
@ -1205,7 +1206,9 @@ def test_update_quota_existing_quotaless_model_rejected():
|
|||
_raise_on_missing_project_model_quota_on_update(data, existing)
|
||||
|
||||
|
||||
def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router: mock.MagicMock | None) -> None:
|
||||
def _enforced_new_project_mocks(
|
||||
monkeypatch, team_models: list[str], llm_router: object, quota_setting: object = True
|
||||
) -> None:
|
||||
from litellm.proxy._types import LiteLLM_TeamTable
|
||||
from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe
|
||||
|
||||
|
|
@ -1213,7 +1216,7 @@ def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router:
|
|||
monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", mock.MagicMock())
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": True})
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": quota_setting})
|
||||
monkeypatch.setattr(pe, "_validate_team_exists", mock.AsyncMock(return_value=team))
|
||||
monkeypatch.setattr(pe, "_check_user_permission_for_project", mock.AsyncMock(return_value=True))
|
||||
|
||||
|
|
@ -1604,6 +1607,449 @@ async def test_new_project_flag_on_access_group_model_returns_400(monkeypatch):
|
|||
assert "expand to multiple models at request time" in str(exc_info.value)
|
||||
|
||||
|
||||
_CUSTOMER_QUOTA_POLICY: Final = {
|
||||
"providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []},
|
||||
"models": {"veo-*": ["rpm"], "jev-latest": []},
|
||||
}
|
||||
|
||||
_CUSTOMER_MODEL_PROVIDERS: Final = {
|
||||
"gemini-3.8-flash": frozenset({"vertex_ai"}),
|
||||
"veo-3.1-lite-generate-001": frozenset({"vertex_ai"}),
|
||||
"runway-gen4-turbo": frozenset({"runwayml"}),
|
||||
}
|
||||
|
||||
|
||||
def _customer_route_of(model: str) -> ModelRoute:
|
||||
return ModelRoute(providers=_CUSTOMER_MODEL_PROVIDERS.get(model, frozenset()))
|
||||
|
||||
|
||||
def _quota_error(
|
||||
models: list[str],
|
||||
rpm: dict[str, int] | None = None,
|
||||
tpm: dict[str, int] | None = None,
|
||||
setting: object = _CUSTOMER_QUOTA_POLICY,
|
||||
route_of: Callable[[str], ModelRoute] = _customer_route_of,
|
||||
) -> str | None:
|
||||
from litellm_enterprise.proxy.management_endpoints.project_endpoints import (
|
||||
_raise_on_missing_project_model_quota,
|
||||
)
|
||||
from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting
|
||||
|
||||
policy: Final = parse_model_quota_setting(setting)
|
||||
assert policy is not None
|
||||
data: Final = NewProjectRequest(team_id="test-team", models=models, model_rpm_limit=rpm, model_tpm_limit=tpm)
|
||||
try:
|
||||
_raise_on_missing_project_model_quota(data, frozenset(), policy, route_of)
|
||||
except HTTPException as e:
|
||||
rejection: Final = e
|
||||
else:
|
||||
return None
|
||||
assert rejection.status_code == 400
|
||||
return str(rejection.detail)
|
||||
|
||||
|
||||
def test_quota_policy_exempts_provider_and_passthrough_name_but_keeps_vertex_strict():
|
||||
assert _quota_error(["runway-gen4-turbo", "jev-latest"]) is None
|
||||
error = _quota_error(["gemini-3.8-flash"], rpm={"gemini-3.8-flash": 10})
|
||||
assert error is not None
|
||||
assert "gemini-3.8-flash (model_tpm_limit)" in error
|
||||
|
||||
|
||||
def test_quota_policy_model_pattern_overrides_its_provider_per_limit():
|
||||
assert _quota_error(["veo-3.1-lite-generate-001"], rpm={"veo-3.1-lite-generate-001": 5}) is None
|
||||
error = _quota_error(["veo-3.1-lite-generate-001"], tpm={"veo-3.1-lite-generate-001": 5000})
|
||||
assert error is not None
|
||||
assert "veo-3.1-lite-generate-001 (model_rpm_limit)" in error
|
||||
assert "model_tpm_limit" not in error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "rpm", "tpm", "expected_missing"),
|
||||
[
|
||||
("runway-gen4-turbo", {"runway-gen4-turbo": 0}, None, "runway-gen4-turbo (model_rpm_limit)"),
|
||||
("jev-latest", None, {"jev-latest": -5}, "jev-latest (model_tpm_limit)"),
|
||||
(
|
||||
"veo-3.1-lite-generate-001",
|
||||
{"veo-3.1-lite-generate-001": 5},
|
||||
{"veo-3.1-lite-generate-001": 0},
|
||||
"veo-3.1-lite-generate-001 (model_tpm_limit)",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_quota_policy_rejects_a_non_positive_limit_set_on_an_exempt_model(
|
||||
model: str, rpm: dict[str, int] | None, tpm: dict[str, int] | None, expected_missing: str
|
||||
):
|
||||
error: Final = _quota_error([model], rpm=rpm, tpm=tpm)
|
||||
assert error is not None
|
||||
assert expected_missing in error
|
||||
|
||||
|
||||
def test_quota_policy_error_names_each_model_with_the_limits_it_misses():
|
||||
error = _quota_error(["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo"])
|
||||
assert error is not None
|
||||
assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in error
|
||||
assert "veo-3.1-lite-generate-001 (model_rpm_limit)" in error
|
||||
assert "runway-gen4-turbo" not in error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"setting, model, expected_missing",
|
||||
[
|
||||
({"default": ["tpm"]}, "unrouted-name", "unrouted-name (model_tpm_limit)"),
|
||||
({"default": [], "providers": {"runwayml": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"),
|
||||
(
|
||||
{"providers": {"runwayml": []}, "models": {"runway-gen4-turbo": ["tpm"]}},
|
||||
"runway-gen4-turbo",
|
||||
"(model_tpm_limit)",
|
||||
),
|
||||
({"models": {"runway-*": [], "runway-gen4-turbo": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"),
|
||||
({"models": {"veo-*": [], "veo-3.1-lite-*": ["rpm"]}}, "veo-3.1-lite-generate-001", "(model_rpm_limit)"),
|
||||
({"models": {"gpt-5-mini*": [], "gpt-*-*-*": ["rpm"]}}, "gpt-5-mini-fast", "(model_rpm_limit)"),
|
||||
],
|
||||
)
|
||||
def test_quota_policy_precedence_is_model_then_pattern_then_provider_then_default(
|
||||
setting: object, model: str, expected_missing: str
|
||||
):
|
||||
error: Final = _quota_error([model], setting=setting)
|
||||
assert error is not None
|
||||
assert expected_missing in error
|
||||
|
||||
|
||||
def test_quota_policy_model_entry_matches_names_literally_apart_from_star():
|
||||
setting = {"models": {"claude-opus-5-5[1m]": [], "gpt.5": []}}
|
||||
assert _quota_error(["claude-opus-5-5[1m]"], setting=setting) is None
|
||||
assert _quota_error(["claude-opus-5-51"], setting=setting) is not None
|
||||
assert _quota_error(["gptx5"], setting=setting) is not None
|
||||
|
||||
|
||||
def test_quota_policy_model_on_several_providers_needs_every_required_limit():
|
||||
def mixed_providers(_model: str) -> ModelRoute:
|
||||
return ModelRoute(providers=frozenset({"runwayml", "vertex_ai"}))
|
||||
|
||||
error = _quota_error(["video-pool"], rpm={"video-pool": 10}, route_of=mixed_providers)
|
||||
assert error is not None
|
||||
assert "video-pool (model_tpm_limit)" in error
|
||||
|
||||
|
||||
def test_quota_policy_model_pattern_matches_from_the_start_like_the_router():
|
||||
setting: Final = {"providers": {"vertex_ai": ["rpm", "tpm"]}, "models": {"*-lite": []}}
|
||||
assert _quota_error(["veo-3.1-lite-generate-001"], setting=setting) is None
|
||||
assert _quota_error(["gemini-3.8-flash"], setting=setting) is not None
|
||||
|
||||
|
||||
def test_quota_policy_model_entries_never_match_a_team_model_name():
|
||||
def team_vertex_route(_model: str) -> ModelRoute:
|
||||
return ModelRoute(providers=frozenset({"vertex_ai"}), team_named=True)
|
||||
|
||||
exact: Final = _quota_error(["jev-latest"], route_of=team_vertex_route)
|
||||
pattern: Final = _quota_error(["veo-team"], rpm={"veo-team": 5}, route_of=team_vertex_route)
|
||||
assert exact is not None and "jev-latest (model_rpm_limit and model_tpm_limit)" in exact
|
||||
assert pattern is not None and "veo-team (model_tpm_limit)" in pattern
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setting", [True, "true", "false", 1, ["rpm"], {"enabled": True}, {"provider": {"runwayml": []}}])
|
||||
def test_quota_setting_truthy_non_policy_value_still_requires_both_limits(setting: object):
|
||||
error: Final = _quota_error(["runway-gen4-turbo"], setting=setting)
|
||||
assert error is not None
|
||||
assert "runway-gen4-turbo (model_rpm_limit and model_tpm_limit)" in error
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setting", [False, None, {}, "", 0, []])
|
||||
def test_quota_setting_off_enforces_nothing(setting: object):
|
||||
from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting
|
||||
|
||||
assert parse_model_quota_setting(setting) is None
|
||||
|
||||
|
||||
def test_update_quota_policy_resolves_providers_for_the_resulting_models():
|
||||
import types
|
||||
|
||||
from litellm_enterprise.proxy.management_endpoints.project_endpoints import (
|
||||
_raise_on_missing_project_model_quota_on_update,
|
||||
)
|
||||
from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting
|
||||
|
||||
policy = parse_model_quota_setting(_CUSTOMER_QUOTA_POLICY)
|
||||
assert policy is not None
|
||||
existing = types.SimpleNamespace(models=["runway-gen4-turbo"], metadata={})
|
||||
assert (
|
||||
_raise_on_missing_project_model_quota_on_update(
|
||||
UpdateProjectRequest(project_id="p", description="unrelated change"),
|
||||
existing,
|
||||
frozenset(),
|
||||
policy,
|
||||
_customer_route_of,
|
||||
)
|
||||
is None
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_raise_on_missing_project_model_quota_on_update(
|
||||
UpdateProjectRequest(project_id="p", models=["runway-gen4-turbo", "gemini-3.8-flash"]),
|
||||
existing,
|
||||
frozenset(),
|
||||
policy,
|
||||
_customer_route_of,
|
||||
)
|
||||
assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
def _customer_router() -> "litellm.Router":
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gemini-3.8-flash",
|
||||
"litellm_params": {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"},
|
||||
},
|
||||
{
|
||||
"model_name": "veo-3.1-lite-generate-001",
|
||||
"litellm_params": {"model": "vertex_ai/veo-3.1-lite-generate-001", "vertex_project": "qa-project"},
|
||||
},
|
||||
{
|
||||
"model_name": "runway-gen4-turbo",
|
||||
"litellm_params": {"model": "runwayml/gen4_turbo", "api_key": "qa-runway-key"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_quota_providers_come_from_the_one_wildcard_route_requests_use():
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gen4*", "litellm_params": {"model": "runwayml/gen4*", "api_key": "qa-runway-key"}},
|
||||
{
|
||||
"model_name": "media-team-gen",
|
||||
"litellm_params": {"model": "vertex_ai/*", "vertex_project": "qa-project"},
|
||||
"model_info": {"team_id": "media-team", "team_public_model_name": "gen*"},
|
||||
},
|
||||
]
|
||||
)
|
||||
route_of: Final = router_model_routes(router, "media-team")
|
||||
|
||||
assert route_of("gen4_turbo") == ModelRoute(providers=frozenset({"runwayml"}))
|
||||
assert route_of("gen3a_turbo") == ModelRoute(providers=frozenset({"vertex_ai"}), team_named=True)
|
||||
|
||||
|
||||
def test_quota_providers_read_the_provider_behind_a_known_api_base():
|
||||
groq_api_base: Final = "https://api.groq.com/openai/v1"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "qa-llama",
|
||||
"litellm_params": {"model": "qa-llama", "api_base": groq_api_base, "api_key": "qa-groq-key"},
|
||||
},
|
||||
{
|
||||
"model_name": "qa-runway",
|
||||
"litellm_params": {"model": "gen4_turbo", "custom_llm_provider": "runwayml", "api_base": groq_api_base},
|
||||
},
|
||||
]
|
||||
)
|
||||
route_of: Final = router_model_routes(router, None)
|
||||
|
||||
assert route_of("qa-llama").providers == frozenset({"groq"})
|
||||
assert route_of("qa-runway").providers == frozenset({"runwayml"})
|
||||
|
||||
|
||||
def test_quota_policy_rejects_an_unknown_provider_id():
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting
|
||||
|
||||
with pytest.raises(ValidationError, match=r"unknown provider ids \['runway'\]"):
|
||||
parse_model_quota_setting({"providers": {"vertex_ai": ["rpm"], "runway": []}})
|
||||
|
||||
|
||||
def test_quota_policy_accepts_a_provider_registered_at_runtime(monkeypatch):
|
||||
from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting
|
||||
|
||||
monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "qa-custom-llm"])
|
||||
policy: Final = parse_model_quota_setting({"providers": {"qa-custom-llm": []}})
|
||||
|
||||
assert policy is not None
|
||||
assert policy.providers["qa-custom-llm"] == frozenset()
|
||||
|
||||
|
||||
_TEAM_WRITE_POLICY: Final = {"providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []}, "models": {"jev-latest": []}}
|
||||
_VERTEX_PARAMS: Final = {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"}
|
||||
_RUNWAY_PARAMS: Final = {"model": "gen4_turbo", "custom_llm_provider": "runwayml"}
|
||||
|
||||
|
||||
def _team_deployment(public_name: str, model_id: str, litellm_params: Mapping[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": f"model_name_media-team_{model_id}",
|
||||
"litellm_params": dict(litellm_params),
|
||||
"model_info": {"id": model_id, "team_id": "media-team", "team_public_model_name": public_name},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("setting", "public_name", "proxy_params", "team_routes", "replaced_id", "written_params", "added"),
|
||||
[
|
||||
pytest.param(
|
||||
_TEAM_WRITE_POLICY,
|
||||
"jev-latest",
|
||||
{"model": "openai/jev-latest"},
|
||||
(),
|
||||
None,
|
||||
_VERTEX_PARAMS,
|
||||
{"rpm", "tpm"},
|
||||
id="team-add-on-vertex-behind-an-exempt-proxy-name",
|
||||
),
|
||||
pytest.param(
|
||||
_TEAM_WRITE_POLICY,
|
||||
"jev-latest",
|
||||
{"model": "openai/jev-latest"},
|
||||
(),
|
||||
None,
|
||||
_RUNWAY_PARAMS,
|
||||
set(),
|
||||
id="team-add-on-an-exempt-provider",
|
||||
),
|
||||
pytest.param(
|
||||
_TEAM_WRITE_POLICY,
|
||||
"media-latest",
|
||||
_VERTEX_PARAMS,
|
||||
(("t1", _RUNWAY_PARAMS),),
|
||||
"t1",
|
||||
None,
|
||||
{"rpm", "tpm"},
|
||||
id="team-delete-falls-back-to-the-proxy-vertex-route",
|
||||
),
|
||||
pytest.param(
|
||||
_TEAM_WRITE_POLICY,
|
||||
"media-latest",
|
||||
_VERTEX_PARAMS,
|
||||
(("t1", _RUNWAY_PARAMS), ("t2", _RUNWAY_PARAMS)),
|
||||
"t1",
|
||||
None,
|
||||
set(),
|
||||
id="team-delete-keeps-a-sibling-team-route",
|
||||
),
|
||||
pytest.param(
|
||||
_TEAM_WRITE_POLICY,
|
||||
"media-latest",
|
||||
_VERTEX_PARAMS,
|
||||
(("t1", _RUNWAY_PARAMS),),
|
||||
"t1",
|
||||
_VERTEX_PARAMS,
|
||||
{"rpm", "tpm"},
|
||||
id="team-update-moves-the-route-to-vertex",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"jev-latest",
|
||||
{"model": "openai/jev-latest"},
|
||||
(),
|
||||
None,
|
||||
_VERTEX_PARAMS,
|
||||
set(),
|
||||
id="true-already-requires-both-limits",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_team_model_write_adds_only_the_limits_its_new_route_requires(
|
||||
setting: object,
|
||||
public_name: str,
|
||||
proxy_params: Mapping[str, object],
|
||||
team_routes: tuple[tuple[str, Mapping[str, object]], ...],
|
||||
replaced_id: str | None,
|
||||
written_params: Mapping[str, object] | None,
|
||||
added: set[str],
|
||||
):
|
||||
from litellm.proxy.management_helpers.model_quota_policy import (
|
||||
limits_added_by_team_write,
|
||||
parse_model_quota_setting,
|
||||
)
|
||||
|
||||
policy: Final = parse_model_quota_setting(setting)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": public_name, "litellm_params": dict(proxy_params)},
|
||||
*(_team_deployment(public_name, model_id, params) for model_id, params in team_routes),
|
||||
]
|
||||
)
|
||||
|
||||
assert policy is not None
|
||||
assert limits_added_by_team_write(
|
||||
policy, router, "media-team", public_name, replaced_id, written_params
|
||||
) == frozenset(added)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_project_quota_policy_reads_providers_from_router_deployments(monkeypatch):
|
||||
team_models = ["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo", "jev-latest"]
|
||||
_enforced_new_project_mocks(
|
||||
monkeypatch, team_models=team_models, llm_router=_customer_router(), quota_setting=_CUSTOMER_QUOTA_POLICY
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_new_project(NewProjectRequest(team_id="test-team", models=team_models))
|
||||
|
||||
assert exc_info.value.code == "400"
|
||||
assert (
|
||||
"gemini-3.8-flash (model_rpm_limit and model_tpm_limit), veo-3.1-lite-generate-001 (model_rpm_limit)."
|
||||
in str(exc_info.value.message)
|
||||
)
|
||||
assert "runway-gen4-turbo" not in str(exc_info.value.message)
|
||||
assert "jev-latest" not in str(exc_info.value.message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_project_rejects_malformed_quota_setting_instead_of_skipping_it(monkeypatch):
|
||||
_enforced_new_project_mocks(
|
||||
monkeypatch,
|
||||
team_models=["runway-gen4-turbo"],
|
||||
llm_router=_customer_router(),
|
||||
quota_setting={"providers": {"runwayml": []}, "model": {"jev-latest": []}},
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_new_project(NewProjectRequest(team_id="test-team", models=["runway-gen4-turbo"]))
|
||||
|
||||
assert exc_info.value.code == "500"
|
||||
assert "general_settings.enforce_project_model_quota is invalid" in str(exc_info.value.message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_project_checks_permission_before_reading_the_quota_setting(monkeypatch):
|
||||
from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe
|
||||
|
||||
_enforced_new_project_mocks(
|
||||
monkeypatch,
|
||||
team_models=["runway-gen4-turbo"],
|
||||
llm_router=_customer_router(),
|
||||
quota_setting={"providers": {"runwayml": []}, "model": {"jev-latest": []}},
|
||||
)
|
||||
monkeypatch.setattr(pe, "_check_user_permission_for_project", mock.AsyncMock(return_value=False))
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_new_project(NewProjectRequest(team_id="test-team", models=["runway-gen4-turbo"]))
|
||||
|
||||
assert exc_info.value.code == "403"
|
||||
assert "enforce_project_model_quota" not in str(exc_info.value.message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_project_quota_policy_ignores_model_entries_for_a_team_model_name(monkeypatch):
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "model_name_test-team_qa",
|
||||
"litellm_params": {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"},
|
||||
"model_info": {"team_id": "test-team", "team_public_model_name": "jev-latest"},
|
||||
},
|
||||
]
|
||||
)
|
||||
_enforced_new_project_mocks(
|
||||
monkeypatch, team_models=["jev-latest"], llm_router=router, quota_setting=_CUSTOMER_QUOTA_POLICY
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await _run_new_project(NewProjectRequest(team_id="test-team", models=["jev-latest"]))
|
||||
|
||||
assert exc_info.value.code == "400"
|
||||
assert "jev-latest (model_rpm_limit and model_tpm_limit)" in str(exc_info.value.message)
|
||||
|
||||
|
||||
_ACCESS_NOW: Final = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from fastapi.testclient import TestClient
|
|||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.models.credentials import CredentialItem
|
||||
from litellm.models.project import LiteLLM_ProjectTable
|
||||
|
||||
from litellm.proxy._types import (
|
||||
LiteLLM_ModelTable,
|
||||
|
|
@ -38,6 +39,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
|
|||
patch_model,
|
||||
update_model,
|
||||
)
|
||||
from litellm.proxy.management_helpers.model_quota_policy import REQUIRE_BOTH_LIMITS, ModelQuotaPolicy
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.router import Router
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams
|
||||
|
|
@ -7352,12 +7354,19 @@ class TestAutoRouterClassifierDefaultPrompt:
|
|||
assert response.system_prompt == classification_system_prompt(5)
|
||||
|
||||
|
||||
_CUSTOMER_MODEL_ADD_POLICY: Final = ModelQuotaPolicy.model_validate(
|
||||
{
|
||||
"providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []},
|
||||
"models": {"veo-*": ["rpm"], "jev-latest": []},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestEnforceRpmTpmOnModelAdd:
|
||||
def test_passes_when_disabled_even_without_limits(self):
|
||||
assert (
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
litellm_params=LiteLLM_Params(model="azure/gpt-5.2"),
|
||||
enforced=False,
|
||||
model_name="gpt-5.2", team_model=False, litellm_params=LiteLLM_Params(model="azure/gpt-5.2"), policy=None
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
|
@ -7365,8 +7374,9 @@ class TestEnforceRpmTpmOnModelAdd:
|
|||
def test_passes_when_enabled_and_both_set(self):
|
||||
assert (
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name="gpt-5.2", team_model=False,
|
||||
litellm_params=LiteLLM_Params(model="azure/gpt-5.2", rpm=10, tpm=1000),
|
||||
enforced=True,
|
||||
policy=REQUIRE_BOTH_LIMITS,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
|
@ -7381,14 +7391,322 @@ class TestEnforceRpmTpmOnModelAdd:
|
|||
(LiteLLM_Params(model="azure/gpt-5.2", rpm=10, tpm=-1), "tpm"),
|
||||
],
|
||||
)
|
||||
def test_raises_when_enabled_and_missing(self, params, expected_missing):
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
def test_raises_when_enabled_and_missing(self, params: LiteLLM_Params, expected_missing: str):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
_raise_if_rate_limits_required_but_missing(litellm_params=params, enforced=True)
|
||||
assert expected_missing in str(exc_info.value.message)
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name="gpt-5.2", team_model=False, litellm_params=params, policy=REQUIRE_BOTH_LIMITS
|
||||
)
|
||||
assert exc_info.value.message.startswith(f"{expected_missing} must be set")
|
||||
assert exc_info.value.code == "400"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_name", "params"),
|
||||
[
|
||||
("runway-gen4", LiteLLM_Params(model="runwayml/gen4_turbo")),
|
||||
("my-runway", LiteLLM_Params(model="gen4_turbo", custom_llm_provider="runwayml")),
|
||||
("jev-latest", LiteLLM_Params(model="openai/jev-latest")),
|
||||
("veo-3.1-generate-001", LiteLLM_Params(model="vertex_ai/veo-3.1-generate-001", rpm=10)),
|
||||
("gemini-3-pro", LiteLLM_Params(model="vertex_ai/gemini-3-pro", rpm=10, tpm=1000)),
|
||||
],
|
||||
)
|
||||
def test_policy_asks_each_model_only_for_the_limits_its_entry_requires(
|
||||
self, model_name: str, params: LiteLLM_Params
|
||||
):
|
||||
assert (
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name=model_name, team_model=False, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_name", "params", "expected_missing"),
|
||||
[
|
||||
("veo-3.1-generate-001", LiteLLM_Params(model="vertex_ai/veo-3.1-generate-001", tpm=1000), "rpm"),
|
||||
("gemini-3-pro", LiteLLM_Params(model="vertex_ai/gemini-3-pro", rpm=10), "tpm"),
|
||||
("openai-gpt", LiteLLM_Params(model="openai/gpt-5.2"), "rpm and tpm"),
|
||||
("runway-gen4", LiteLLM_Params(model="runwayml/gen4_turbo", rpm=0), "rpm"),
|
||||
],
|
||||
)
|
||||
def test_policy_rejects_a_model_missing_a_required_or_carrying_a_non_positive_limit(
|
||||
self, model_name: str, params: LiteLLM_Params, expected_missing: str
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name=model_name, team_model=False, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY
|
||||
)
|
||||
assert exc_info.value.message.startswith(f"{expected_missing} must be set to a positive value for '{model_name}'")
|
||||
assert exc_info.value.code == "400"
|
||||
|
||||
def test_policy_reads_the_provider_behind_a_known_api_base(self):
|
||||
policy: Final = ModelQuotaPolicy.model_validate({"default": [], "providers": {"groq": ["rpm", "tpm"]}})
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name="qa-llama", team_model=False,
|
||||
litellm_params=LiteLLM_Params(model="qa-llama", api_base="https://api.groq.com/openai/v1"),
|
||||
policy=policy,
|
||||
)
|
||||
assert exc_info.value.message.startswith("rpm and tpm must be set to a positive value for 'qa-llama'")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_name", "params", "expected_missing"),
|
||||
[
|
||||
("jev-latest", LiteLLM_Params(model="vertex_ai/gemini-3.8-flash"), "rpm and tpm"),
|
||||
("veo-team", LiteLLM_Params(model="vertex_ai/veo-3.1-generate-001", rpm=5), "tpm"),
|
||||
],
|
||||
)
|
||||
def test_policy_model_entries_never_match_a_team_model_name(
|
||||
self, model_name: str, params: LiteLLM_Params, expected_missing: str
|
||||
):
|
||||
assert (
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name=model_name, team_model=False, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY
|
||||
)
|
||||
is None
|
||||
)
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
_raise_if_rate_limits_required_but_missing(
|
||||
model_name=model_name, team_model=True, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY
|
||||
)
|
||||
assert exc_info.value.message.startswith(f"{expected_missing} must be set to a positive value for '{model_name}'")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("setting", "expected_code", "expected_message"),
|
||||
[
|
||||
({"default": [], "models": {"veo-*": ["rpm"]}}, "400", "rpm must be set to a positive value for 'veo-3.1'"),
|
||||
("yes", "400", "rpm and tpm must be set to a positive value for 'veo-3.1'"),
|
||||
({"enabled": True}, "400", "rpm and tpm must be set to a positive value for 'veo-3.1'"),
|
||||
(
|
||||
{"providers": {"vertex_ai": ["qps"]}},
|
||||
"500",
|
||||
"general_settings.enforce_rpm_tpm_on_model_add is invalid",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_model_add_resolves_the_quota_policy_for_the_public_model_name(
|
||||
monkeypatch: pytest.MonkeyPatch, setting: object, expected_code: str, expected_message: str
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"enforce_rpm_tpm_on_model_add": setting})
|
||||
with pytest.raises(ProxyException) as error:
|
||||
await add_new_model(
|
||||
Deployment(model_name="veo-3.1", litellm_params={"model": "vertex_ai/veo-3.1"}),
|
||||
UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
assert error.value.code == expected_code
|
||||
assert error.value.message.startswith(expected_message)
|
||||
prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_model_add_cannot_borrow_a_model_entry_by_its_public_name(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=LiteLLM_TeamTable(team_id="media-team"))
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "premium_user", True)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"general_settings",
|
||||
{"enforce_rpm_tpm_on_model_add": {"providers": {"vertex_ai": ["rpm", "tpm"]}, "models": {"jev-latest": []}}},
|
||||
)
|
||||
with pytest.raises(ProxyException) as error:
|
||||
await add_new_model(
|
||||
Deployment(
|
||||
model_name="jev-latest",
|
||||
litellm_params={"model": "vertex_ai/gemini-3.8-flash"},
|
||||
model_info={"team_id": "media-team"},
|
||||
),
|
||||
UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
assert error.value.code == "400"
|
||||
assert error.value.message.startswith("rpm and tpm must be set to a positive value for 'jev-latest'")
|
||||
prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
|
||||
|
||||
_PROJECT_QUOTA_SETTINGS: Final = {
|
||||
"enforce_project_model_quota": {
|
||||
"providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []},
|
||||
"models": {"jev-latest": []},
|
||||
}
|
||||
}
|
||||
_TEAM_VERTEX: Final = {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"}
|
||||
_TEAM_RUNWAY: Final = {"model": "gen4_turbo", "custom_llm_provider": "runwayml"}
|
||||
|
||||
|
||||
def _media_team_db(public_name: str, project_metadata: Mapping[str, object]) -> MagicMock:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=LiteLLM_TeamTable(team_id="media-team"))
|
||||
prisma.db.litellm_projecttable.find_many = AsyncMock(
|
||||
return_value=[
|
||||
LiteLLM_ProjectTable(
|
||||
project_id="p-1",
|
||||
project_alias="video-gen",
|
||||
team_id="media-team",
|
||||
models=[public_name],
|
||||
metadata=dict(project_metadata),
|
||||
)
|
||||
]
|
||||
)
|
||||
return prisma
|
||||
|
||||
|
||||
def _team_model_row(public_name: str, litellm_params: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
|
||||
return LiteLLM_ProxyModelTable(
|
||||
model_id="team-row",
|
||||
model_name="model_name_media-team_team-row",
|
||||
litellm_params=dict(litellm_params),
|
||||
model_info={"id": "team-row", "team_id": "media-team", "team_public_model_name": public_name},
|
||||
created_by="admin",
|
||||
updated_by="admin",
|
||||
)
|
||||
|
||||
|
||||
def _quota_router(public_name: str, proxy_params: Mapping[str, object], *team_params: Mapping[str, object]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{"model_name": public_name, "litellm_params": dict(proxy_params)},
|
||||
*(
|
||||
{
|
||||
"model_name": "model_name_media-team_team-row",
|
||||
"litellm_params": dict(params),
|
||||
"model_info": {"id": "team-row", "team_id": "media-team", "team_public_model_name": public_name},
|
||||
}
|
||||
for params in team_params
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _use_quota_proxy(monkeypatch: pytest.MonkeyPatch, prisma: MagicMock, router: Router) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", _PROJECT_QUOTA_SETTINGS)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", False)
|
||||
monkeypatch.setattr(proxy_server, "premium_user", True)
|
||||
|
||||
|
||||
_ADMIN: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_model_add_rejects_a_route_its_projects_lack_the_quota_for(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
prisma: Final = _media_team_db("jev-latest", {})
|
||||
_use_quota_proxy(monkeypatch, prisma, _quota_router("jev-latest", {"model": "openai/jev-latest"}))
|
||||
|
||||
with pytest.raises(ProxyException) as error:
|
||||
await add_new_model(
|
||||
Deployment(
|
||||
model_name="jev-latest",
|
||||
litellm_params={**_TEAM_VERTEX, "rpm": 10, "tpm": 1000},
|
||||
model_info={"team_id": "media-team"},
|
||||
),
|
||||
_ADMIN,
|
||||
)
|
||||
|
||||
assert error.value.code == "400"
|
||||
assert error.value.message.startswith(
|
||||
"projects video-gen list 'jev-latest' without a positive rpm and tpm quota, "
|
||||
"which 'enforce_project_model_quota' requires once this team model change applies"
|
||||
)
|
||||
prisma.db.litellm_projecttable.find_many.assert_awaited_once_with(
|
||||
where={"team_id": "media-team", "models": {"has": "jev-latest"}}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "project_metadata"),
|
||||
[
|
||||
pytest.param(
|
||||
_TEAM_VERTEX,
|
||||
{"model_rpm_limit": {"jev-latest": 5}, "model_tpm_limit": {"jev-latest": 500}},
|
||||
id="projects-carry-the-added-limits",
|
||||
),
|
||||
pytest.param(_TEAM_RUNWAY, {}, id="new-route-is-exempt"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_model_add_goes_through_when_its_projects_meet_the_new_route(
|
||||
monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object], project_metadata: Mapping[str, object]
|
||||
) -> None:
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
_use_quota_proxy(
|
||||
monkeypatch, _media_team_db("jev-latest", project_metadata), _quota_router("jev-latest", {"model": "openai/jev-latest"})
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as error:
|
||||
await add_new_model(
|
||||
Deployment(model_name="jev-latest", litellm_params=dict(litellm_params), model_info={"team_id": "media-team"}),
|
||||
_ADMIN,
|
||||
)
|
||||
|
||||
assert "STORE_MODEL_IN_DB" in error.value.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["patch", "put"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_model_update_rejects_moving_its_route_where_its_projects_lack_quota(
|
||||
monkeypatch: pytest.MonkeyPatch, endpoint: str
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
runway_by_prefix: Final = {"model": "runwayml/gen4_turbo"}
|
||||
prisma: Final = _media_team_db("jev-latest", {})
|
||||
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(
|
||||
return_value=_team_model_row("jev-latest", runway_by_prefix)
|
||||
)
|
||||
_use_quota_proxy(monkeypatch, prisma, _quota_router("jev-latest", {"model": "openai/jev-latest"}, runway_by_prefix))
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
|
||||
moved_to_vertex: Final = updateLiteLLMParams(model="vertex_ai/gemini-3.8-flash", vertex_project="qa-project")
|
||||
|
||||
write: Final = (
|
||||
patch_model(
|
||||
model_id="team-row", patch_data=updateDeployment(litellm_params=moved_to_vertex), user_api_key_dict=_ADMIN
|
||||
)
|
||||
if endpoint == "patch"
|
||||
else update_model(
|
||||
model_params=updateDeployment(litellm_params=moved_to_vertex, model_info=ModelInfo(id="team-row")),
|
||||
user_api_key_dict=_ADMIN,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as error:
|
||||
await write
|
||||
|
||||
assert error.value.code == "400"
|
||||
assert error.value.message.startswith("projects video-gen list 'jev-latest' without a positive rpm and tpm quota")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_model_delete_rejects_falling_back_to_a_route_its_projects_lack_quota_for(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete, delete_model
|
||||
|
||||
prisma: Final = _media_team_db("media-latest", {})
|
||||
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=_team_model_row("media-latest", _TEAM_RUNWAY))
|
||||
_use_quota_proxy(monkeypatch, prisma, _quota_router("media-latest", _TEAM_VERTEX, _TEAM_RUNWAY))
|
||||
|
||||
with pytest.raises(ProxyException) as error:
|
||||
await delete_model(model_info=ModelInfoDelete(id="team-row"), user_api_key_dict=_ADMIN)
|
||||
|
||||
assert error.value.code == "400"
|
||||
assert error.value.message.startswith("projects video-gen list 'media-latest' without a positive rpm and tpm quota")
|
||||
prisma.db.litellm_proxymodeltable.delete.assert_not_called()
|
||||
|
||||
|
||||
class TestBlockModelResponseSerialization:
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue