From ba71f280d4785319fcbd6037c5c1d66aef979ea3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:19:43 -0700 Subject: [PATCH 1/6] feat(proxy): per-provider and per-model rpm/tpm requirements for enforce_project_model_quota --- .../management_endpoints/project_endpoints.py | 105 +++++---- .../project_model_quota.py | 125 +++++++++++ .../test_project_endpoints_prisma.py | 212 +++++++++++++++++- 3 files changed, 395 insertions(+), 47 deletions(-) create mode 100644 enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index db419fde1e6..8bca681ea12 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -15,7 +15,18 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, cast from fastapi import APIRouter, Depends, HTTPException, Request -from pydantic import TypeAdapter +from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + ENFORCE_PROJECT_MODEL_QUOTA_SETTING, + REQUIRE_BOTH_LIMITS, + ProjectModelQuotaPolicy, + ProvidersOf, + describe_missing_quota, + missing_project_model_quota, + no_providers, + parse_project_model_quota_setting, + router_deployment_providers, +) +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -265,26 +276,6 @@ def _check_team_project_limits( ) -def _project_models_missing_positive_quota( - models: list[str] | None, - rpm_limits: Mapping[str, object] | None, - tpm_limits: Mapping[str, object] | None, -) -> list[str]: - """Return the models that lack a positive `rpm` AND `tpm` quota. - - A valid quota is a positive integer; null, zero, and negative are rejected - because downstream rate limiters treat a non-positive limit as immediately - exhausted (every request blocked). - """ - - def _is_positive(value: object) -> bool: - return isinstance(value, int) and not isinstance(value, bool) and value > 0 - - rpm = rpm_limits or {} - tpm = tpm_limits or {} - return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))] - - def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]: return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset() @@ -311,64 +302,79 @@ def _raise_on_project_models_expanding_at_request_time( raise HTTPException( status_code=400, detail={ - "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead." + "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while '{ENFORCE_PROJECT_MODEL_QUOTA_SETTING}' is enabled. List concrete model names instead." }, ) +def _project_model_quota_policy(general_settings: Mapping[str, object]) -> ProjectModelQuotaPolicy | None: + try: + return parse_project_model_quota_setting(general_settings.get(ENFORCE_PROJECT_MODEL_QUOTA_SETTING)) + except ValidationError as e: + raise HTTPException( + status_code=500, + detail={"error": f"general_settings.{ENFORCE_PROJECT_MODEL_QUOTA_SETTING} is invalid: {e}"}, + ) from e + + def _raise_on_missing_project_model_quota( - data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset() + data: NewProjectRequest | UpdateProjectRequest, + access_group_names: frozenset[str] = frozenset(), + policy: ProjectModelQuotaPolicy = REQUIRE_BOTH_LIMITS, + providers_of: ProvidersOf = no_providers, ) -> None: - """Require a positive `rpm`/`tpm` quota for every model on project CREATE. + """Require the positive `rpm`/`tpm` quotas the policy asks of each model on project CREATE. `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request model's `set_model_info` validator, so they are read from there. - - Only invoked when `general_settings.enforce_project_model_quota` is enabled - (default off), so it is opt-in and does not change behavior for existing users. """ _raise_on_project_models_expanding_at_request_time(data.models, access_group_names) metadata = data.metadata or {} - missing = _project_models_missing_positive_quota( - data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit") + missing: Final = missing_project_model_quota( + data.models or (), metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit"), policy, providers_of ) if not missing: return raise HTTPException( status_code=400, detail={ - "error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model." + "error": f"models added to project without a positive rpm/tpm quota: {describe_missing_quota(missing)}. Set a positive value for each listed limit." }, ) def _raise_on_missing_project_model_quota_on_update( - data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset() + data: UpdateProjectRequest, + existing_project: object, + access_group_names: frozenset[str] = frozenset(), + policy: ProjectModelQuotaPolicy = REQUIRE_BOTH_LIMITS, + providers_of: ProvidersOf = no_providers, ) -> None: - """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE. + """Require the positive `rpm`/`tpm` quotas the policy asks of each model over the RESULTING state on project UPDATE. `/project/update` replaces `models` and `metadata` when they are provided, so the check runs on what the project WILL look like: a partial update that doesn't touch models/quota keeps the existing values, while one that adds a model or clears a - model's quota must leave every resulting model with a positive limit. - - Only invoked when `general_settings.enforce_project_model_quota` is enabled - (default off), so it is opt-in and does not change behavior for existing users. + model's quota must leave every resulting model with the limits it needs. """ resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or []) resulting_metadata = ( data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {}) ) _raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names) - missing = _project_models_missing_positive_quota( - resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit") + missing: Final = missing_project_model_quota( + resulting_models, + resulting_metadata.get("model_rpm_limit"), + resulting_metadata.get("model_tpm_limit"), + policy, + providers_of, ) if not missing: return raise HTTPException( status_code=400, detail={ - "error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model." + "error": f"models would be left on the project without a positive rpm/tpm quota: {describe_missing_quota(missing)}. Set a positive value for each listed limit." }, ) @@ -567,9 +573,14 @@ async def new_project( data=data, ) - # Opt-in (default off): require rpm/tpm for every model added to the project. - if general_settings.get("enforce_project_model_quota", False): - _raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router)) + quota_policy: Final = _project_model_quota_policy(general_settings) + if quota_policy is not None: + _raise_on_missing_project_model_quota( + data, + _router_access_group_names(llm_router), + quota_policy, + router_deployment_providers(llm_router, data.team_id), + ) # Check if user has permission to create projects for this team # only team admins can create projects for their team @@ -817,10 +828,14 @@ async def update_project( data=data, ) - # Opt-in (default off): require rpm/tpm for every model the update would leave on the project. - if general_settings.get("enforce_project_model_quota", False): + quota_policy: Final = _project_model_quota_policy(general_settings) + if quota_policy is not None: _raise_on_missing_project_model_quota_on_update( - data, existing_project, _router_access_group_names(llm_router) + data, + existing_project, + _router_access_group_names(llm_router), + quota_policy, + router_deployment_providers(llm_router, target_team_id), ) # Prepare update data diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py new file mode 100644 index 00000000000..5cdfe158fd1 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py @@ -0,0 +1,125 @@ +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter + +from litellm.router_utils.common_utils import provider_for_generic_call +from litellm.router_utils.pattern_match_deployments import PatternUtils + +if TYPE_CHECKING: + from litellm import Router + +QuotaLimit: TypeAlias = Literal["rpm", "tpm"] +ProvidersOf: TypeAlias = Callable[[str], frozenset[str]] + +ENFORCE_PROJECT_MODEL_QUOTA_SETTING: Final = "enforce_project_model_quota" +_BOTH_LIMITS: Final[frozenset[QuotaLimit]] = frozenset(("rpm", "tpm")) +_NO_LIMITS_SET: Final[Mapping[str, object]] = MappingProxyType({}) + + +class ProjectModelQuotaPolicy(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + default: frozenset[QuotaLimit] = _BOTH_LIMITS + providers: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) + models: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) + + +REQUIRE_BOTH_LIMITS: Final = ProjectModelQuotaPolicy() + +_QUOTA_SETTING: Final = TypeAdapter(bool | ProjectModelQuotaPolicy | None) + + +@dataclass(frozen=True, slots=True) +class MissingModelQuota: + model: str + limits: tuple[QuotaLimit, ...] + + +def parse_project_model_quota_setting(setting: object) -> ProjectModelQuotaPolicy | None: + parsed: Final = _QUOTA_SETTING.validate_python(setting) + match parsed: + case ProjectModelQuotaPolicy(): + return parsed + case True: + return REQUIRE_BOTH_LIMITS + case _: + return None + + +def _wildcard_matches(pattern: str, model: str) -> bool: + return re.fullmatch(re.escape(pattern).replace(r"\*", ".*"), model) is not None + + +def _most_specific_model_pattern(model: str, entries: Mapping[str, frozenset[QuotaLimit]]) -> str | None: + matching: Final = tuple(pattern for pattern in entries if "*" in pattern and _wildcard_matches(pattern, model)) + return max(matching, key=PatternUtils.calculate_pattern_specificity, default=None) + + +def required_limits(policy: ProjectModelQuotaPolicy, model: str, providers: frozenset[str]) -> frozenset[QuotaLimit]: + """A model entry (exact name, then the most specific `*` pattern) wins over the providers of the + model's deployments, which win over the default. A model served by several providers needs every + limit any of them requires.""" + exact: Final = policy.models.get(model) + if exact is not None: + return exact + pattern: Final = _most_specific_model_pattern(model, policy.models) + if pattern is not None: + return policy.models[pattern] + if not providers: + return policy.default + return frozenset(chain.from_iterable(policy.providers.get(provider, policy.default) for provider in providers)) + + +def _is_positive(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def _unset_limits( + model: str, required: frozenset[QuotaLimit], set_limits: Mapping[QuotaLimit, Mapping[str, object]] +) -> tuple[QuotaLimit, ...]: + return tuple(sorted(limit for limit in required if not _is_positive(set_limits[limit].get(model)))) + + +def missing_project_model_quota( + models: Sequence[str], + rpm_limits: Mapping[str, object] | None, + tpm_limits: Mapping[str, object] | None, + policy: ProjectModelQuotaPolicy, + providers_of: ProvidersOf, +) -> tuple[MissingModelQuota, ...]: + set_limits: Final[Mapping[QuotaLimit, Mapping[str, object]]] = MappingProxyType( + {"rpm": rpm_limits or _NO_LIMITS_SET, "tpm": tpm_limits or _NO_LIMITS_SET} + ) + gaps: Final = tuple( + MissingModelQuota(model, _unset_limits(model, required_limits(policy, model, providers_of(model)), set_limits)) + for model in models + ) + return tuple(gap for gap in gaps if gap.limits) + + +def describe_missing_quota(missing: Sequence[MissingModelQuota]) -> str: + return ", ".join(f"{gap.model} ({' and '.join(f'model_{limit}_limit' for limit in gap.limits)})" for gap in missing) + + +def no_providers(_model: str) -> frozenset[str]: + return frozenset() + + +def router_deployment_providers(llm_router: "Router | None", team_id: str | None) -> ProvidersOf: + if llm_router is None: + return no_providers + + def providers_of(model: str) -> frozenset[str]: + deployments: Final = llm_router.get_model_list(model_name=model, team_id=team_id) or () + return frozenset( + provider + for deployment in deployments + if (provider := provider_for_generic_call(deployment["litellm_params"])) is not None + ) + + return providers_of diff --git a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index 45697723af1..6a33959e634 100644 --- a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -1205,7 +1205,9 @@ def test_update_quota_existing_quotaless_model_rejected(): _raise_on_missing_project_model_quota_on_update(data, existing) -def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router: mock.MagicMock | None) -> None: +def _enforced_new_project_mocks( + monkeypatch, team_models: list[str], llm_router: object, quota_setting: object = True +) -> None: from litellm.proxy._types import LiteLLM_TeamTable from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe @@ -1213,7 +1215,7 @@ def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router: monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", mock.MagicMock()) monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": True}) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": quota_setting}) monkeypatch.setattr(pe, "_validate_team_exists", mock.AsyncMock(return_value=team)) monkeypatch.setattr(pe, "_check_user_permission_for_project", mock.AsyncMock(return_value=True)) @@ -1604,6 +1606,212 @@ async def test_new_project_flag_on_access_group_model_returns_400(monkeypatch): assert "expand to multiple models at request time" in str(exc_info.value) +_CUSTOMER_QUOTA_POLICY: Final = { + "providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []}, + "models": {"veo-*": ["rpm"], "jev-latest": []}, +} + +_CUSTOMER_MODEL_PROVIDERS: Final = { + "gemini-3.8-flash": frozenset({"vertex_ai"}), + "veo-3.1-lite-generate-001": frozenset({"vertex_ai"}), + "runway-gen4-turbo": frozenset({"runwayml"}), +} + + +def _customer_providers_of(model: str) -> frozenset[str]: + return _CUSTOMER_MODEL_PROVIDERS.get(model, frozenset()) + + +def _quota_error( + models: list[str], + rpm: dict[str, int] | None = None, + tpm: dict[str, int] | None = None, + setting: object = _CUSTOMER_QUOTA_POLICY, + providers_of=_customer_providers_of, +) -> str | None: + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + parse_project_model_quota_setting, + ) + + policy = parse_project_model_quota_setting(setting) + assert policy is not None + data = NewProjectRequest(team_id="test-team", models=models, model_rpm_limit=rpm, model_tpm_limit=tpm) + try: + _raise_on_missing_project_model_quota(data, frozenset(), policy, providers_of) + except HTTPException as e: + rejection = e + else: + return None + assert rejection.status_code == 400 + return str(rejection.detail) + + +def test_quota_policy_exempts_provider_and_passthrough_name_but_keeps_vertex_strict(): + assert _quota_error(["runway-gen4-turbo", "jev-latest"]) is None + error = _quota_error(["gemini-3.8-flash"], rpm={"gemini-3.8-flash": 10}) + assert error is not None + assert "gemini-3.8-flash (model_tpm_limit)" in error + + +def test_quota_policy_model_pattern_overrides_its_provider_per_limit(): + assert _quota_error(["veo-3.1-lite-generate-001"], rpm={"veo-3.1-lite-generate-001": 5}) is None + error = _quota_error(["veo-3.1-lite-generate-001"], tpm={"veo-3.1-lite-generate-001": 5000}) + assert error is not None + assert "veo-3.1-lite-generate-001 (model_rpm_limit)" in error + assert "model_tpm_limit" not in error + + +def test_quota_policy_error_names_each_model_with_the_limits_it_misses(): + error = _quota_error(["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo"]) + assert error is not None + assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in error + assert "veo-3.1-lite-generate-001 (model_rpm_limit)" in error + assert "runway-gen4-turbo" not in error + + +@pytest.mark.parametrize( + "setting, model, expected_missing", + [ + ({"default": ["tpm"]}, "unrouted-name", "unrouted-name (model_tpm_limit)"), + ({"default": [], "providers": {"runwayml": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"), + ( + {"providers": {"runwayml": []}, "models": {"runway-gen4-turbo": ["tpm"]}}, + "runway-gen4-turbo", + "(model_tpm_limit)", + ), + ({"models": {"runway-*": [], "runway-gen4-turbo": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"), + ({"models": {"veo-*": [], "veo-3.1-lite-*": ["rpm"]}}, "veo-3.1-lite-generate-001", "(model_rpm_limit)"), + ], +) +def test_quota_policy_precedence_is_model_then_pattern_then_provider_then_default(setting, model, expected_missing): + error = _quota_error([model], setting=setting) + assert error is not None + assert expected_missing in error + + +def test_quota_policy_model_entry_matches_names_literally_apart_from_star(): + setting = {"models": {"claude-opus-5-5[1m]": [], "gpt.5": []}} + assert _quota_error(["claude-opus-5-5[1m]"], setting=setting) is None + assert _quota_error(["claude-opus-5-51"], setting=setting) is not None + assert _quota_error(["gptx5"], setting=setting) is not None + + +def test_quota_policy_model_on_several_providers_needs_every_required_limit(): + def mixed_providers(_model: str) -> frozenset[str]: + return frozenset({"runwayml", "vertex_ai"}) + + error = _quota_error(["video-pool"], rpm={"video-pool": 10}, providers_of=mixed_providers) + assert error is not None + assert "video-pool (model_tpm_limit)" in error + + +@pytest.mark.parametrize("setting", [True, "true", {}]) +def test_quota_setting_true_or_empty_mapping_still_requires_both_limits(setting): + error = _quota_error(["runway-gen4-turbo"], setting=setting) + assert error is not None + assert "runway-gen4-turbo (model_rpm_limit and model_tpm_limit)" in error + + +@pytest.mark.parametrize("setting", [False, None]) +def test_quota_setting_off_enforces_nothing(setting): + from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + parse_project_model_quota_setting, + ) + + assert parse_project_model_quota_setting(setting) is None + + +def test_update_quota_policy_resolves_providers_for_the_resulting_models(): + import types + + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + parse_project_model_quota_setting, + ) + + policy = parse_project_model_quota_setting(_CUSTOMER_QUOTA_POLICY) + assert policy is not None + existing = types.SimpleNamespace(models=["runway-gen4-turbo"], metadata={}) + assert ( + _raise_on_missing_project_model_quota_on_update( + UpdateProjectRequest(project_id="p", description="unrelated change"), + existing, + frozenset(), + policy, + _customer_providers_of, + ) + is None + ) + with pytest.raises(HTTPException) as exc_info: + _raise_on_missing_project_model_quota_on_update( + UpdateProjectRequest(project_id="p", models=["runway-gen4-turbo", "gemini-3.8-flash"]), + existing, + frozenset(), + policy, + _customer_providers_of, + ) + assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in str(exc_info.value.detail) + + +def _customer_router() -> "litellm.Router": + return litellm.Router( + model_list=[ + { + "model_name": "gemini-3.8-flash", + "litellm_params": {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"}, + }, + { + "model_name": "veo-3.1-lite-generate-001", + "litellm_params": {"model": "vertex_ai/veo-3.1-lite-generate-001", "vertex_project": "qa-project"}, + }, + { + "model_name": "runway-gen4-turbo", + "litellm_params": {"model": "runwayml/gen4_turbo", "api_key": "qa-runway-key"}, + }, + ] + ) + + +@pytest.mark.asyncio +async def test_new_project_quota_policy_reads_providers_from_router_deployments(monkeypatch): + team_models = ["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo", "jev-latest"] + _enforced_new_project_mocks( + monkeypatch, team_models=team_models, llm_router=_customer_router(), quota_setting=_CUSTOMER_QUOTA_POLICY + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=team_models)) + + assert exc_info.value.code == "400" + assert ( + "gemini-3.8-flash (model_rpm_limit and model_tpm_limit), veo-3.1-lite-generate-001 (model_rpm_limit)." + in str(exc_info.value.message) + ) + assert "runway-gen4-turbo" not in str(exc_info.value.message) + assert "jev-latest" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_new_project_rejects_malformed_quota_setting_instead_of_skipping_it(monkeypatch): + _enforced_new_project_mocks( + monkeypatch, + team_models=["runway-gen4-turbo"], + llm_router=_customer_router(), + quota_setting={"provider": {"runwayml": []}}, + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=["runway-gen4-turbo"])) + + assert exc_info.value.code == "500" + assert "general_settings.enforce_project_model_quota is invalid" in str(exc_info.value.message) + + _ACCESS_NOW: Final = datetime.now(timezone.utc) From 07d3d938b8e69c899540bd1201fb369663d84f89 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:29:24 -0700 Subject: [PATCH 2/6] test(proxy): cover per-provider and per-model enforce_project_model_quota policies --- .../test_project_endpoints_prisma.py | 212 +++++++++++++++++- 1 file changed, 210 insertions(+), 2 deletions(-) diff --git a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index 45697723af1..6a33959e634 100644 --- a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -1205,7 +1205,9 @@ def test_update_quota_existing_quotaless_model_rejected(): _raise_on_missing_project_model_quota_on_update(data, existing) -def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router: mock.MagicMock | None) -> None: +def _enforced_new_project_mocks( + monkeypatch, team_models: list[str], llm_router: object, quota_setting: object = True +) -> None: from litellm.proxy._types import LiteLLM_TeamTable from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe @@ -1213,7 +1215,7 @@ def _enforced_new_project_mocks(monkeypatch, team_models: list[str], llm_router: monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", mock.MagicMock()) monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": True}) + monkeypatch.setattr(litellm.proxy.proxy_server, "general_settings", {"enforce_project_model_quota": quota_setting}) monkeypatch.setattr(pe, "_validate_team_exists", mock.AsyncMock(return_value=team)) monkeypatch.setattr(pe, "_check_user_permission_for_project", mock.AsyncMock(return_value=True)) @@ -1604,6 +1606,212 @@ async def test_new_project_flag_on_access_group_model_returns_400(monkeypatch): assert "expand to multiple models at request time" in str(exc_info.value) +_CUSTOMER_QUOTA_POLICY: Final = { + "providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []}, + "models": {"veo-*": ["rpm"], "jev-latest": []}, +} + +_CUSTOMER_MODEL_PROVIDERS: Final = { + "gemini-3.8-flash": frozenset({"vertex_ai"}), + "veo-3.1-lite-generate-001": frozenset({"vertex_ai"}), + "runway-gen4-turbo": frozenset({"runwayml"}), +} + + +def _customer_providers_of(model: str) -> frozenset[str]: + return _CUSTOMER_MODEL_PROVIDERS.get(model, frozenset()) + + +def _quota_error( + models: list[str], + rpm: dict[str, int] | None = None, + tpm: dict[str, int] | None = None, + setting: object = _CUSTOMER_QUOTA_POLICY, + providers_of=_customer_providers_of, +) -> str | None: + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota, + ) + from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + parse_project_model_quota_setting, + ) + + policy = parse_project_model_quota_setting(setting) + assert policy is not None + data = NewProjectRequest(team_id="test-team", models=models, model_rpm_limit=rpm, model_tpm_limit=tpm) + try: + _raise_on_missing_project_model_quota(data, frozenset(), policy, providers_of) + except HTTPException as e: + rejection = e + else: + return None + assert rejection.status_code == 400 + return str(rejection.detail) + + +def test_quota_policy_exempts_provider_and_passthrough_name_but_keeps_vertex_strict(): + assert _quota_error(["runway-gen4-turbo", "jev-latest"]) is None + error = _quota_error(["gemini-3.8-flash"], rpm={"gemini-3.8-flash": 10}) + assert error is not None + assert "gemini-3.8-flash (model_tpm_limit)" in error + + +def test_quota_policy_model_pattern_overrides_its_provider_per_limit(): + assert _quota_error(["veo-3.1-lite-generate-001"], rpm={"veo-3.1-lite-generate-001": 5}) is None + error = _quota_error(["veo-3.1-lite-generate-001"], tpm={"veo-3.1-lite-generate-001": 5000}) + assert error is not None + assert "veo-3.1-lite-generate-001 (model_rpm_limit)" in error + assert "model_tpm_limit" not in error + + +def test_quota_policy_error_names_each_model_with_the_limits_it_misses(): + error = _quota_error(["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo"]) + assert error is not None + assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in error + assert "veo-3.1-lite-generate-001 (model_rpm_limit)" in error + assert "runway-gen4-turbo" not in error + + +@pytest.mark.parametrize( + "setting, model, expected_missing", + [ + ({"default": ["tpm"]}, "unrouted-name", "unrouted-name (model_tpm_limit)"), + ({"default": [], "providers": {"runwayml": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"), + ( + {"providers": {"runwayml": []}, "models": {"runway-gen4-turbo": ["tpm"]}}, + "runway-gen4-turbo", + "(model_tpm_limit)", + ), + ({"models": {"runway-*": [], "runway-gen4-turbo": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"), + ({"models": {"veo-*": [], "veo-3.1-lite-*": ["rpm"]}}, "veo-3.1-lite-generate-001", "(model_rpm_limit)"), + ], +) +def test_quota_policy_precedence_is_model_then_pattern_then_provider_then_default(setting, model, expected_missing): + error = _quota_error([model], setting=setting) + assert error is not None + assert expected_missing in error + + +def test_quota_policy_model_entry_matches_names_literally_apart_from_star(): + setting = {"models": {"claude-opus-5-5[1m]": [], "gpt.5": []}} + assert _quota_error(["claude-opus-5-5[1m]"], setting=setting) is None + assert _quota_error(["claude-opus-5-51"], setting=setting) is not None + assert _quota_error(["gptx5"], setting=setting) is not None + + +def test_quota_policy_model_on_several_providers_needs_every_required_limit(): + def mixed_providers(_model: str) -> frozenset[str]: + return frozenset({"runwayml", "vertex_ai"}) + + error = _quota_error(["video-pool"], rpm={"video-pool": 10}, providers_of=mixed_providers) + assert error is not None + assert "video-pool (model_tpm_limit)" in error + + +@pytest.mark.parametrize("setting", [True, "true", {}]) +def test_quota_setting_true_or_empty_mapping_still_requires_both_limits(setting): + error = _quota_error(["runway-gen4-turbo"], setting=setting) + assert error is not None + assert "runway-gen4-turbo (model_rpm_limit and model_tpm_limit)" in error + + +@pytest.mark.parametrize("setting", [False, None]) +def test_quota_setting_off_enforces_nothing(setting): + from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + parse_project_model_quota_setting, + ) + + assert parse_project_model_quota_setting(setting) is None + + +def test_update_quota_policy_resolves_providers_for_the_resulting_models(): + import types + + from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( + _raise_on_missing_project_model_quota_on_update, + ) + from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( + parse_project_model_quota_setting, + ) + + policy = parse_project_model_quota_setting(_CUSTOMER_QUOTA_POLICY) + assert policy is not None + existing = types.SimpleNamespace(models=["runway-gen4-turbo"], metadata={}) + assert ( + _raise_on_missing_project_model_quota_on_update( + UpdateProjectRequest(project_id="p", description="unrelated change"), + existing, + frozenset(), + policy, + _customer_providers_of, + ) + is None + ) + with pytest.raises(HTTPException) as exc_info: + _raise_on_missing_project_model_quota_on_update( + UpdateProjectRequest(project_id="p", models=["runway-gen4-turbo", "gemini-3.8-flash"]), + existing, + frozenset(), + policy, + _customer_providers_of, + ) + assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in str(exc_info.value.detail) + + +def _customer_router() -> "litellm.Router": + return litellm.Router( + model_list=[ + { + "model_name": "gemini-3.8-flash", + "litellm_params": {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"}, + }, + { + "model_name": "veo-3.1-lite-generate-001", + "litellm_params": {"model": "vertex_ai/veo-3.1-lite-generate-001", "vertex_project": "qa-project"}, + }, + { + "model_name": "runway-gen4-turbo", + "litellm_params": {"model": "runwayml/gen4_turbo", "api_key": "qa-runway-key"}, + }, + ] + ) + + +@pytest.mark.asyncio +async def test_new_project_quota_policy_reads_providers_from_router_deployments(monkeypatch): + team_models = ["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo", "jev-latest"] + _enforced_new_project_mocks( + monkeypatch, team_models=team_models, llm_router=_customer_router(), quota_setting=_CUSTOMER_QUOTA_POLICY + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=team_models)) + + assert exc_info.value.code == "400" + assert ( + "gemini-3.8-flash (model_rpm_limit and model_tpm_limit), veo-3.1-lite-generate-001 (model_rpm_limit)." + in str(exc_info.value.message) + ) + assert "runway-gen4-turbo" not in str(exc_info.value.message) + assert "jev-latest" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_new_project_rejects_malformed_quota_setting_instead_of_skipping_it(monkeypatch): + _enforced_new_project_mocks( + monkeypatch, + team_models=["runway-gen4-turbo"], + llm_router=_customer_router(), + quota_setting={"provider": {"runwayml": []}}, + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=["runway-gen4-turbo"])) + + assert exc_info.value.code == "500" + assert "general_settings.enforce_project_model_quota is invalid" in str(exc_info.value.message) + + _ACCESS_NOW: Final = datetime.now(timezone.utc) From 232bbea1755a1a15755b91346bcc15725018bc14 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:09:59 -0700 Subject: [PATCH 3/6] fix(proxy): keep enforce_project_model_quota reads typed --- .../proxy/management_endpoints/project_endpoints.py | 6 ++++-- .../proxy/management_endpoints/project_model_quota.py | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index 8bca681ea12..8ad022a857a 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -525,6 +525,7 @@ async def new_project( """ from litellm.proxy.proxy_server import ( general_settings, + general_settings_view, litellm_proxy_admin_name, llm_router, premium_user, @@ -573,7 +574,7 @@ async def new_project( data=data, ) - quota_policy: Final = _project_model_quota_policy(general_settings) + quota_policy: Final = _project_model_quota_policy(general_settings_view()) if quota_policy is not None: _raise_on_missing_project_model_quota( data, @@ -721,6 +722,7 @@ async def update_project( """ from litellm.proxy.proxy_server import ( general_settings, + general_settings_view, litellm_proxy_admin_name, llm_router, premium_user, @@ -828,7 +830,7 @@ async def update_project( data=data, ) - quota_policy: Final = _project_model_quota_policy(general_settings) + quota_policy: Final = _project_model_quota_policy(general_settings_view()) if quota_policy is not None: _raise_on_missing_project_model_quota_on_update( data, diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py index 5cdfe158fd1..f0ddd3eed30 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py @@ -1,7 +1,6 @@ import re from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, TypeAlias @@ -31,7 +30,9 @@ class ProjectModelQuotaPolicy(BaseModel): REQUIRE_BOTH_LIMITS: Final = ProjectModelQuotaPolicy() -_QUOTA_SETTING: Final = TypeAdapter(bool | ProjectModelQuotaPolicy | None) +_QUOTA_SETTING: Final[TypeAdapter[bool | ProjectModelQuotaPolicy | None]] = TypeAdapter( + bool | ProjectModelQuotaPolicy | None +) @dataclass(frozen=True, slots=True) @@ -72,7 +73,7 @@ def required_limits(policy: ProjectModelQuotaPolicy, model: str, providers: froz return policy.models[pattern] if not providers: return policy.default - return frozenset(chain.from_iterable(policy.providers.get(provider, policy.default) for provider in providers)) + return frozenset[QuotaLimit]().union(*(policy.providers.get(provider, policy.default) for provider in providers)) def _is_positive(value: object) -> bool: From 7f04cbe781fccbbb285b23052ee2e2bca5094e57 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:09:26 -0700 Subject: [PATCH 4/6] fix(proxy): apply the quota policy to model add, follow router wildcard rules, keep old setting values --- .../management_endpoints/project_endpoints.py | 22 ++-- .../project_model_quota.py | 116 +++--------------- .../model_management_endpoints.py | 50 +++++--- .../management_helpers/model_quota_policy.py | 110 +++++++++++++++++ .../test_project_endpoints_prisma.py | 83 +++++++++---- .../test_model_management_endpoints.py | 96 +++++++++++++-- 6 files changed, 326 insertions(+), 151 deletions(-) create mode 100644 litellm/proxy/management_helpers/model_quota_policy.py diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index 8ad022a857a..883e7b8ae8e 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -17,14 +17,8 @@ from typing import TYPE_CHECKING, Final, cast from fastapi import APIRouter, Depends, HTTPException, Request from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( ENFORCE_PROJECT_MODEL_QUOTA_SETTING, - REQUIRE_BOTH_LIMITS, - ProjectModelQuotaPolicy, - ProvidersOf, describe_missing_quota, missing_project_model_quota, - no_providers, - parse_project_model_quota_setting, - router_deployment_providers, ) from pydantic import TypeAdapter, ValidationError @@ -38,6 +32,14 @@ from litellm.proxy.management.teams.dependencies import get_team_access from litellm.proxy.management.users.service import org_admin_org_ids from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects +from litellm.proxy.management_helpers.model_quota_policy import ( + REQUIRE_BOTH_LIMITS, + ModelQuotaPolicy, + ProvidersOf, + no_providers, + parse_model_quota_setting, + router_deployment_providers, +) from litellm.proxy.management_helpers.utils import ( management_endpoint_wrapper, ) @@ -307,9 +309,9 @@ def _raise_on_project_models_expanding_at_request_time( ) -def _project_model_quota_policy(general_settings: Mapping[str, object]) -> ProjectModelQuotaPolicy | None: +def _project_model_quota_policy(general_settings: Mapping[str, object]) -> ModelQuotaPolicy | None: try: - return parse_project_model_quota_setting(general_settings.get(ENFORCE_PROJECT_MODEL_QUOTA_SETTING)) + return parse_model_quota_setting(general_settings.get(ENFORCE_PROJECT_MODEL_QUOTA_SETTING)) except ValidationError as e: raise HTTPException( status_code=500, @@ -320,7 +322,7 @@ def _project_model_quota_policy(general_settings: Mapping[str, object]) -> Proje def _raise_on_missing_project_model_quota( data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset(), - policy: ProjectModelQuotaPolicy = REQUIRE_BOTH_LIMITS, + policy: ModelQuotaPolicy = REQUIRE_BOTH_LIMITS, providers_of: ProvidersOf = no_providers, ) -> None: """Require the positive `rpm`/`tpm` quotas the policy asks of each model on project CREATE. @@ -347,7 +349,7 @@ def _raise_on_missing_project_model_quota_on_update( data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset(), - policy: ProjectModelQuotaPolicy = REQUIRE_BOTH_LIMITS, + policy: ModelQuotaPolicy = REQUIRE_BOTH_LIMITS, providers_of: ProvidersOf = no_providers, ) -> None: """Require the positive `rpm`/`tpm` quotas the policy asks of each model over the RESULTING state on project UPDATE. diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py index f0ddd3eed30..e427e37f10b 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py @@ -1,103 +1,46 @@ -import re -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, TypeAlias +from typing import Final -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter - -from litellm.router_utils.common_utils import provider_for_generic_call -from litellm.router_utils.pattern_match_deployments import PatternUtils - -if TYPE_CHECKING: - from litellm import Router - -QuotaLimit: TypeAlias = Literal["rpm", "tpm"] -ProvidersOf: TypeAlias = Callable[[str], frozenset[str]] +from litellm.proxy.management_helpers.model_quota_policy import ( + ModelQuotaPolicy, + ProvidersOf, + QuotaLimit, + required_limits, + unmet_limits, +) ENFORCE_PROJECT_MODEL_QUOTA_SETTING: Final = "enforce_project_model_quota" -_BOTH_LIMITS: Final[frozenset[QuotaLimit]] = frozenset(("rpm", "tpm")) _NO_LIMITS_SET: Final[Mapping[str, object]] = MappingProxyType({}) -class ProjectModelQuotaPolicy(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - default: frozenset[QuotaLimit] = _BOTH_LIMITS - providers: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) - models: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) - - -REQUIRE_BOTH_LIMITS: Final = ProjectModelQuotaPolicy() - -_QUOTA_SETTING: Final[TypeAdapter[bool | ProjectModelQuotaPolicy | None]] = TypeAdapter( - bool | ProjectModelQuotaPolicy | None -) - - @dataclass(frozen=True, slots=True) class MissingModelQuota: model: str limits: tuple[QuotaLimit, ...] -def parse_project_model_quota_setting(setting: object) -> ProjectModelQuotaPolicy | None: - parsed: Final = _QUOTA_SETTING.validate_python(setting) - match parsed: - case ProjectModelQuotaPolicy(): - return parsed - case True: - return REQUIRE_BOTH_LIMITS - case _: - return None - - -def _wildcard_matches(pattern: str, model: str) -> bool: - return re.fullmatch(re.escape(pattern).replace(r"\*", ".*"), model) is not None - - -def _most_specific_model_pattern(model: str, entries: Mapping[str, frozenset[QuotaLimit]]) -> str | None: - matching: Final = tuple(pattern for pattern in entries if "*" in pattern and _wildcard_matches(pattern, model)) - return max(matching, key=PatternUtils.calculate_pattern_specificity, default=None) - - -def required_limits(policy: ProjectModelQuotaPolicy, model: str, providers: frozenset[str]) -> frozenset[QuotaLimit]: - """A model entry (exact name, then the most specific `*` pattern) wins over the providers of the - model's deployments, which win over the default. A model served by several providers needs every - limit any of them requires.""" - exact: Final = policy.models.get(model) - if exact is not None: - return exact - pattern: Final = _most_specific_model_pattern(model, policy.models) - if pattern is not None: - return policy.models[pattern] - if not providers: - return policy.default - return frozenset[QuotaLimit]().union(*(policy.providers.get(provider, policy.default) for provider in providers)) - - -def _is_positive(value: object) -> bool: - return isinstance(value, int) and not isinstance(value, bool) and value > 0 - - -def _unset_limits( - model: str, required: frozenset[QuotaLimit], set_limits: Mapping[QuotaLimit, Mapping[str, object]] -) -> tuple[QuotaLimit, ...]: - return tuple(sorted(limit for limit in required if not _is_positive(set_limits[limit].get(model)))) +def _limits_set_for( + model: str, rpm_set: Mapping[str, object], tpm_set: Mapping[str, object] +) -> Mapping[QuotaLimit, object]: + return MappingProxyType({"rpm": rpm_set.get(model), "tpm": tpm_set.get(model)}) def missing_project_model_quota( models: Sequence[str], rpm_limits: Mapping[str, object] | None, tpm_limits: Mapping[str, object] | None, - policy: ProjectModelQuotaPolicy, + policy: ModelQuotaPolicy, providers_of: ProvidersOf, ) -> tuple[MissingModelQuota, ...]: - set_limits: Final[Mapping[QuotaLimit, Mapping[str, object]]] = MappingProxyType( - {"rpm": rpm_limits or _NO_LIMITS_SET, "tpm": tpm_limits or _NO_LIMITS_SET} - ) + rpm_set: Final = rpm_limits or _NO_LIMITS_SET + tpm_set: Final = tpm_limits or _NO_LIMITS_SET gaps: Final = tuple( - MissingModelQuota(model, _unset_limits(model, required_limits(policy, model, providers_of(model)), set_limits)) + MissingModelQuota( + model, + unmet_limits(required_limits(policy, model, providers_of(model)), _limits_set_for(model, rpm_set, tpm_set)), + ) for model in models ) return tuple(gap for gap in gaps if gap.limits) @@ -105,22 +48,3 @@ def missing_project_model_quota( def describe_missing_quota(missing: Sequence[MissingModelQuota]) -> str: return ", ".join(f"{gap.model} ({' and '.join(f'model_{limit}_limit' for limit in gap.limits)})" for gap in missing) - - -def no_providers(_model: str) -> frozenset[str]: - return frozenset() - - -def router_deployment_providers(llm_router: "Router | None", team_id: str | None) -> ProvidersOf: - if llm_router is None: - return no_providers - - def providers_of(model: str) -> frozenset[str]: - deployments: Final = llm_router.get_model_list(model_name=model, team_id=team_id) or () - return frozenset( - provider - for deployment in deployments - if (provider := provider_for_generic_call(deployment["litellm_params"])) is not None - ) - - return providers_of diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fc0868a5fc9..504cc9afcbc 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -102,6 +102,13 @@ from litellm.proxy.management_helpers.auto_router_permissions import ( authorize_member_auto_router_write, ) from litellm.proxy.management_helpers.model_allowlist_rename_sync import sync_model_allowlists_for_renamed_model +from litellm.proxy.management_helpers.model_quota_policy import ( + ModelQuotaPolicy, + deployment_provider, + parse_model_quota_setting, + required_limits, + unmet_limits, +) from litellm.proxy.spend_tracking.ptu_feature_flag import ( PTU_COST_ATTRIBUTION_ENV_VAR, is_ptu_cost_attribution_enabled, @@ -151,6 +158,7 @@ from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, GenericLiteLLMParams, + LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams, @@ -760,26 +768,39 @@ async def _auto_router_capability_slot( ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add" -def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLMParams, enforced: bool) -> None: - """Require both rpm and tpm (each a positive value) when the operator opts in via config.yaml. +def _model_add_quota_policy(general_settings: Mapping[str, object]) -> ModelQuotaPolicy | None: + try: + return parse_model_quota_setting(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING)) + except ValidationError as e: + raise ProxyException( + message=f"general_settings.{ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING} is invalid: {e}", + type=ProxyErrorTypes.internal_server_error.value, + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + param=None, + ) from e - Off by default, so deployments keep adding models without limits. When - ``enforce_rpm_tpm_on_model_add: true`` is set under general_settings, a model added - without both rpm and tpm set to a positive value is rejected rather than stored - unbounded (or effectively excluded from routing by a zero/negative limit). - """ - if not enforced: + +def _raise_if_rate_limits_required_but_missing( + *, model_name: str, litellm_params: LiteLLM_Params, policy: ModelQuotaPolicy | None +) -> None: + """Off by default, so deployments keep adding models without limits. Once the operator opts in under + general_settings, the policy decides per model name and provider which of rpm and tpm a new model needs, + and a model missing one, or carrying a zero or negative one, is rejected rather than stored unbounded + (or effectively excluded from routing).""" + if policy is None: return - missing: Final = tuple( - field - for field, value in (("rpm", litellm_params.rpm), ("tpm", litellm_params.tpm)) - if value is None or value <= 0 + provider: Final = deployment_provider( + MappingProxyType({"model": litellm_params.model, "custom_llm_provider": litellm_params.custom_llm_provider}) + ) + missing: Final = unmet_limits( + required_limits(policy, model_name, frozenset() if provider is None else frozenset((provider,))), + MappingProxyType({"rpm": litellm_params.rpm, "tpm": litellm_params.tpm}), ) if not missing: return raise ProxyException( message=( - f"{' and '.join(missing)} must be set to a positive value when " + f"{' and '.join(missing)} must be set to a positive value for '{model_name}' when " f"'{ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING}' is enabled in general_settings" ), type=ProxyErrorTypes.validation_error.value, @@ -2717,8 +2738,9 @@ async def add_new_model( ) _raise_if_rate_limits_required_but_missing( + model_name=model_params.model_name, litellm_params=model_params.litellm_params, - enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)), + policy=_model_add_quota_policy(general_settings), ) clean_model_info: Final = ModelInfo.model_validate( diff --git a/litellm/proxy/management_helpers/model_quota_policy.py b/litellm/proxy/management_helpers/model_quota_policy.py new file mode 100644 index 00000000000..e2467e32997 --- /dev/null +++ b/litellm/proxy/management_helpers/model_quota_policy.py @@ -0,0 +1,110 @@ +import re +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field + +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping +from litellm.router_utils.common_utils import provider_for_generic_call +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils + +if TYPE_CHECKING: + from litellm import Router + +QuotaLimit: TypeAlias = Literal["rpm", "tpm"] +ProvidersOf: TypeAlias = Callable[[str], frozenset[str]] + +_LIMITS: Final[tuple[QuotaLimit, ...]] = ("rpm", "tpm") +_ROUTER_PATTERNS: Final = PatternMatchRouter() + + +class ModelQuotaPolicy(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + default: frozenset[QuotaLimit] = frozenset(_LIMITS) + providers: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) + models: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) + + +REQUIRE_BOTH_LIMITS: Final = ModelQuotaPolicy() + + +def parse_model_quota_setting(setting: object) -> ModelQuotaPolicy | None: + """Any falsy value is off and any other non-mapping value requires both limits, as the boolean + setting always behaved, so only a mapping is read as a per-provider and per-model policy.""" + if not setting: + return None + if is_str_keyed_mapping(setting): + return ModelQuotaPolicy.model_validate(setting) + return REQUIRE_BOTH_LIMITS + + +def _wildcard_matches(pattern: str, model: str) -> bool: + return re.fullmatch(_ROUTER_PATTERNS.pattern_to_regex(pattern), model) is not None + + +def _router_specificity(pattern: str) -> tuple[int, int]: + return PatternUtils.calculate_pattern_specificity(_ROUTER_PATTERNS.pattern_to_regex(pattern)) + + +def _most_specific_model_pattern(model: str, entries: Mapping[str, frozenset[QuotaLimit]]) -> str | None: + matching: Final = tuple(pattern for pattern in entries if "*" in pattern and _wildcard_matches(pattern, model)) + return max(matching, key=_router_specificity, default=None) + + +def required_limits(policy: ModelQuotaPolicy, model: str, providers: frozenset[str]) -> frozenset[QuotaLimit]: + """A model entry (exact name, then the most specific `*` pattern) wins over the providers of the + model's deployments, which win over the default. A model served by several providers needs every + limit any of them requires.""" + exact: Final = policy.models.get(model) + if exact is not None: + return exact + pattern: Final = _most_specific_model_pattern(model, policy.models) + if pattern is not None: + return policy.models[pattern] + if not providers: + return policy.default + return frozenset[QuotaLimit]().union(*(policy.providers.get(provider, policy.default) for provider in providers)) + + +def _is_positive(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def unmet_limits(required: frozenset[QuotaLimit], values: Mapping[QuotaLimit, object]) -> tuple[QuotaLimit, ...]: + return tuple( + limit + for limit in _LIMITS + if (limit in required or values.get(limit) is not None) and not _is_positive(values.get(limit)) + ) + + +def no_providers(_model: str) -> frozenset[str]: + return frozenset() + + +def deployment_provider(litellm_params: object) -> str | None: + return provider_for_generic_call(litellm_params) if is_str_keyed_mapping(litellm_params) else None + + +def _routed_deployments(llm_router: "Router", model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: + named: Final[tuple[Mapping[str, object], ...]] = ( + *llm_router.get_all_deployments(model_name=model, team_id=team_id), + *llm_router.get_model_list_from_model_alias(model_name=model), + *llm_router.get_model_list_from_routing_groups(model_name=model), + ) + return named or llm_router.wildcard_route_deployments(model, team_id) + + +def router_deployment_providers(llm_router: "Router | None", team_id: str | None) -> ProvidersOf: + if llm_router is None: + return no_providers + + def providers_of(model: str) -> frozenset[str]: + return frozenset( + provider + for deployment in _routed_deployments(llm_router, model, team_id) + if (provider := deployment_provider(deployment.get("litellm_params"))) is not None + ) + + return providers_of diff --git a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index 6a33959e634..c1e3c035b57 100644 --- a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -1,7 +1,7 @@ import asyncio import os import traceback -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import datetime, timezone from typing import Final @@ -1627,22 +1627,20 @@ def _quota_error( rpm: dict[str, int] | None = None, tpm: dict[str, int] | None = None, setting: object = _CUSTOMER_QUOTA_POLICY, - providers_of=_customer_providers_of, + providers_of: Callable[[str], frozenset[str]] = _customer_providers_of, ) -> str | None: from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _raise_on_missing_project_model_quota, ) - from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( - parse_project_model_quota_setting, - ) + from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting - policy = parse_project_model_quota_setting(setting) + policy: Final = parse_model_quota_setting(setting) assert policy is not None - data = NewProjectRequest(team_id="test-team", models=models, model_rpm_limit=rpm, model_tpm_limit=tpm) + data: Final = NewProjectRequest(team_id="test-team", models=models, model_rpm_limit=rpm, model_tpm_limit=tpm) try: _raise_on_missing_project_model_quota(data, frozenset(), policy, providers_of) except HTTPException as e: - rejection = e + rejection: Final = e else: return None assert rejection.status_code == 400 @@ -1664,6 +1662,27 @@ def test_quota_policy_model_pattern_overrides_its_provider_per_limit(): assert "model_tpm_limit" not in error +@pytest.mark.parametrize( + ("model", "rpm", "tpm", "expected_missing"), + [ + ("runway-gen4-turbo", {"runway-gen4-turbo": 0}, None, "runway-gen4-turbo (model_rpm_limit)"), + ("jev-latest", None, {"jev-latest": -5}, "jev-latest (model_tpm_limit)"), + ( + "veo-3.1-lite-generate-001", + {"veo-3.1-lite-generate-001": 5}, + {"veo-3.1-lite-generate-001": 0}, + "veo-3.1-lite-generate-001 (model_tpm_limit)", + ), + ], +) +def test_quota_policy_rejects_a_non_positive_limit_set_on_an_exempt_model( + model: str, rpm: dict[str, int] | None, tpm: dict[str, int] | None, expected_missing: str +): + error: Final = _quota_error([model], rpm=rpm, tpm=tpm) + assert error is not None + assert expected_missing in error + + def test_quota_policy_error_names_each_model_with_the_limits_it_misses(): error = _quota_error(["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo"]) assert error is not None @@ -1684,10 +1703,13 @@ def test_quota_policy_error_names_each_model_with_the_limits_it_misses(): ), ({"models": {"runway-*": [], "runway-gen4-turbo": ["rpm"]}}, "runway-gen4-turbo", "(model_rpm_limit)"), ({"models": {"veo-*": [], "veo-3.1-lite-*": ["rpm"]}}, "veo-3.1-lite-generate-001", "(model_rpm_limit)"), + ({"models": {"gpt-5-mini*": [], "gpt-*-*-*": ["rpm"]}}, "gpt-5-mini-fast", "(model_rpm_limit)"), ], ) -def test_quota_policy_precedence_is_model_then_pattern_then_provider_then_default(setting, model, expected_missing): - error = _quota_error([model], setting=setting) +def test_quota_policy_precedence_is_model_then_pattern_then_provider_then_default( + setting: object, model: str, expected_missing: str +): + error: Final = _quota_error([model], setting=setting) assert error is not None assert expected_missing in error @@ -1708,20 +1730,18 @@ def test_quota_policy_model_on_several_providers_needs_every_required_limit(): assert "video-pool (model_tpm_limit)" in error -@pytest.mark.parametrize("setting", [True, "true", {}]) -def test_quota_setting_true_or_empty_mapping_still_requires_both_limits(setting): - error = _quota_error(["runway-gen4-turbo"], setting=setting) +@pytest.mark.parametrize("setting", [True, "true", "false", 1, ["rpm"]]) +def test_quota_setting_truthy_non_mapping_still_requires_both_limits(setting: object): + error: Final = _quota_error(["runway-gen4-turbo"], setting=setting) assert error is not None assert "runway-gen4-turbo (model_rpm_limit and model_tpm_limit)" in error -@pytest.mark.parametrize("setting", [False, None]) -def test_quota_setting_off_enforces_nothing(setting): - from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( - parse_project_model_quota_setting, - ) +@pytest.mark.parametrize("setting", [False, None, {}, "", 0, []]) +def test_quota_setting_off_enforces_nothing(setting: object): + from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting - assert parse_project_model_quota_setting(setting) is None + assert parse_model_quota_setting(setting) is None def test_update_quota_policy_resolves_providers_for_the_resulting_models(): @@ -1730,11 +1750,9 @@ def test_update_quota_policy_resolves_providers_for_the_resulting_models(): from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _raise_on_missing_project_model_quota_on_update, ) - from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( - parse_project_model_quota_setting, - ) + from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting - policy = parse_project_model_quota_setting(_CUSTOMER_QUOTA_POLICY) + policy = parse_model_quota_setting(_CUSTOMER_QUOTA_POLICY) assert policy is not None existing = types.SimpleNamespace(models=["runway-gen4-turbo"], metadata={}) assert ( @@ -1777,6 +1795,25 @@ def _customer_router() -> "litellm.Router": ) +def test_quota_providers_come_from_the_one_wildcard_route_requests_use(): + from litellm.proxy.management_helpers.model_quota_policy import router_deployment_providers + + router: Final = litellm.Router( + model_list=[ + {"model_name": "gen4*", "litellm_params": {"model": "runwayml/gen4*", "api_key": "qa-runway-key"}}, + { + "model_name": "media-team-gen", + "litellm_params": {"model": "vertex_ai/*", "vertex_project": "qa-project"}, + "model_info": {"team_id": "media-team", "team_public_model_name": "gen*"}, + }, + ] + ) + providers_of: Final = router_deployment_providers(router, "media-team") + + assert providers_of("gen4_turbo") == frozenset({"runwayml"}) + assert providers_of("gen3a_turbo") == frozenset({"vertex_ai"}) + + @pytest.mark.asyncio async def test_new_project_quota_policy_reads_providers_from_router_deployments(monkeypatch): team_models = ["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo", "jev-latest"] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 303b8e060ed..5870e6fad9a 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -38,6 +38,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, update_model, ) +from litellm.proxy.management_helpers.model_quota_policy import REQUIRE_BOTH_LIMITS, ModelQuotaPolicy from litellm.proxy.utils import PrismaClient from litellm.router import Router from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams @@ -7352,12 +7353,19 @@ class TestAutoRouterClassifierDefaultPrompt: assert response.system_prompt == classification_system_prompt(5) +_CUSTOMER_MODEL_ADD_POLICY: Final = ModelQuotaPolicy.model_validate( + { + "providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []}, + "models": {"veo-*": ["rpm"], "jev-latest": []}, + } +) + + class TestEnforceRpmTpmOnModelAdd: def test_passes_when_disabled_even_without_limits(self): assert ( _raise_if_rate_limits_required_but_missing( - litellm_params=LiteLLM_Params(model="azure/gpt-5.2"), - enforced=False, + model_name="gpt-5.2", litellm_params=LiteLLM_Params(model="azure/gpt-5.2"), policy=None ) is None ) @@ -7365,8 +7373,9 @@ class TestEnforceRpmTpmOnModelAdd: def test_passes_when_enabled_and_both_set(self): assert ( _raise_if_rate_limits_required_but_missing( + model_name="gpt-5.2", litellm_params=LiteLLM_Params(model="azure/gpt-5.2", rpm=10, tpm=1000), - enforced=True, + policy=REQUIRE_BOTH_LIMITS, ) is None ) @@ -7381,14 +7390,85 @@ class TestEnforceRpmTpmOnModelAdd: (LiteLLM_Params(model="azure/gpt-5.2", rpm=10, tpm=-1), "tpm"), ], ) - def test_raises_when_enabled_and_missing(self, params, expected_missing): - from litellm.proxy._types import ProxyException - + def test_raises_when_enabled_and_missing(self, params: LiteLLM_Params, expected_missing: str): with pytest.raises(ProxyException) as exc_info: - _raise_if_rate_limits_required_but_missing(litellm_params=params, enforced=True) - assert expected_missing in str(exc_info.value.message) + _raise_if_rate_limits_required_but_missing( + model_name="gpt-5.2", litellm_params=params, policy=REQUIRE_BOTH_LIMITS + ) + assert exc_info.value.message.startswith(f"{expected_missing} must be set") assert exc_info.value.code == "400" + @pytest.mark.parametrize( + ("model_name", "params"), + [ + ("runway-gen4", LiteLLM_Params(model="runwayml/gen4_turbo")), + ("my-runway", LiteLLM_Params(model="gen4_turbo", custom_llm_provider="runwayml")), + ("jev-latest", LiteLLM_Params(model="openai/jev-latest")), + ("veo-3.1-generate-001", LiteLLM_Params(model="vertex_ai/veo-3.1-generate-001", rpm=10)), + ("gemini-3-pro", LiteLLM_Params(model="vertex_ai/gemini-3-pro", rpm=10, tpm=1000)), + ], + ) + def test_policy_asks_each_model_only_for_the_limits_its_entry_requires( + self, model_name: str, params: LiteLLM_Params + ): + assert ( + _raise_if_rate_limits_required_but_missing( + model_name=model_name, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY + ) + is None + ) + + @pytest.mark.parametrize( + ("model_name", "params", "expected_missing"), + [ + ("veo-3.1-generate-001", LiteLLM_Params(model="vertex_ai/veo-3.1-generate-001", tpm=1000), "rpm"), + ("gemini-3-pro", LiteLLM_Params(model="vertex_ai/gemini-3-pro", rpm=10), "tpm"), + ("openai-gpt", LiteLLM_Params(model="openai/gpt-5.2"), "rpm and tpm"), + ("runway-gen4", LiteLLM_Params(model="runwayml/gen4_turbo", rpm=0), "rpm"), + ], + ) + def test_policy_rejects_a_model_missing_a_required_or_carrying_a_non_positive_limit( + self, model_name: str, params: LiteLLM_Params, expected_missing: str + ): + with pytest.raises(ProxyException) as exc_info: + _raise_if_rate_limits_required_but_missing( + model_name=model_name, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY + ) + assert exc_info.value.message.startswith(f"{expected_missing} must be set to a positive value for '{model_name}'") + assert exc_info.value.code == "400" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("setting", "expected_code", "expected_message"), + [ + ({"default": [], "models": {"veo-*": ["rpm"]}}, "400", "rpm must be set to a positive value for 'veo-3.1'"), + ("yes", "400", "rpm and tpm must be set to a positive value for 'veo-3.1'"), + ( + {"providers": {"vertex_ai": ["qps"]}}, + "500", + "general_settings.enforce_rpm_tpm_on_model_add is invalid", + ), + ], +) +async def test_model_add_resolves_the_quota_policy_for_the_public_model_name( + monkeypatch: pytest.MonkeyPatch, setting: object, expected_code: str, expected_message: str +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + prisma: Final = MagicMock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "general_settings", {"enforce_rpm_tpm_on_model_add": setting}) + with pytest.raises(ProxyException) as error: + await add_new_model( + Deployment(model_name="veo-3.1", litellm_params={"model": "vertex_ai/veo-3.1"}), + UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert error.value.code == expected_code + assert error.value.message.startswith(expected_message) + prisma.db.litellm_proxymodeltable.create.assert_not_called() + class TestBlockModelResponseSerialization: @pytest.mark.parametrize( From acacc486a87e626e535be0f6ab22ebe8a69366c9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:29:40 -0700 Subject: [PATCH 5/6] fix(proxy): keep team model names off quota model entries, read the provider behind api_base, match patterns like the router A team admin picks a team model's public name, so a name matching an exempt model entry no longer skips its provider's limits. A mapping with no policy key keeps requiring both limits as before, the project create permission check runs before the setting is read, and an unprefixed model behind a known OpenAI-compatible api_base resolves to that provider --- .../management_endpoints/project_endpoints.py | 34 +++--- .../project_model_quota.py | 6 +- .../model_management_endpoints.py | 18 ++- .../management_helpers/model_quota_policy.py | 100 +++++++++++----- .../test_project_endpoints_prisma.py | 110 +++++++++++++++--- .../test_model_management_endpoints.py | 69 ++++++++++- 6 files changed, 265 insertions(+), 72 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index 883e7b8ae8e..dea8e1b196f 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -35,10 +35,10 @@ from litellm.proxy.management_endpoints.team_admin_field_permissions import team from litellm.proxy.management_helpers.model_quota_policy import ( REQUIRE_BOTH_LIMITS, ModelQuotaPolicy, - ProvidersOf, - no_providers, + RouteOf, + no_route, parse_model_quota_setting, - router_deployment_providers, + router_model_routes, ) from litellm.proxy.management_helpers.utils import ( management_endpoint_wrapper, @@ -323,7 +323,7 @@ def _raise_on_missing_project_model_quota( data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset(), policy: ModelQuotaPolicy = REQUIRE_BOTH_LIMITS, - providers_of: ProvidersOf = no_providers, + route_of: RouteOf = no_route, ) -> None: """Require the positive `rpm`/`tpm` quotas the policy asks of each model on project CREATE. @@ -333,7 +333,7 @@ def _raise_on_missing_project_model_quota( _raise_on_project_models_expanding_at_request_time(data.models, access_group_names) metadata = data.metadata or {} missing: Final = missing_project_model_quota( - data.models or (), metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit"), policy, providers_of + data.models or (), metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit"), policy, route_of ) if not missing: return @@ -350,7 +350,7 @@ def _raise_on_missing_project_model_quota_on_update( existing_project: object, access_group_names: frozenset[str] = frozenset(), policy: ModelQuotaPolicy = REQUIRE_BOTH_LIMITS, - providers_of: ProvidersOf = no_providers, + route_of: RouteOf = no_route, ) -> None: """Require the positive `rpm`/`tpm` quotas the policy asks of each model over the RESULTING state on project UPDATE. @@ -369,7 +369,7 @@ def _raise_on_missing_project_model_quota_on_update( resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit"), policy, - providers_of, + route_of, ) if not missing: return @@ -576,15 +576,6 @@ async def new_project( data=data, ) - quota_policy: Final = _project_model_quota_policy(general_settings_view()) - if quota_policy is not None: - _raise_on_missing_project_model_quota( - data, - _router_access_group_names(llm_router), - quota_policy, - router_deployment_providers(llm_router, data.team_id), - ) - # Check if user has permission to create projects for this team # only team admins can create projects for their team has_permission = await _check_user_permission_for_project( @@ -603,6 +594,15 @@ async def new_project( }, ) + quota_policy: Final = _project_model_quota_policy(general_settings_view()) + if quota_policy is not None: + _raise_on_missing_project_model_quota( + data, + _router_access_group_names(llm_router), + quota_policy, + router_model_routes(llm_router, data.team_id), + ) + # Generate project_id if not provided if data.project_id is None: data.project_id = str(uuid.uuid4()) @@ -839,7 +839,7 @@ async def update_project( existing_project, _router_access_group_names(llm_router), quota_policy, - router_deployment_providers(llm_router, target_team_id), + router_model_routes(llm_router, target_team_id), ) # Prepare update data diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py index e427e37f10b..e7078e0bf30 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py @@ -5,8 +5,8 @@ from typing import Final from litellm.proxy.management_helpers.model_quota_policy import ( ModelQuotaPolicy, - ProvidersOf, QuotaLimit, + RouteOf, required_limits, unmet_limits, ) @@ -32,14 +32,14 @@ def missing_project_model_quota( rpm_limits: Mapping[str, object] | None, tpm_limits: Mapping[str, object] | None, policy: ModelQuotaPolicy, - providers_of: ProvidersOf, + route_of: RouteOf, ) -> tuple[MissingModelQuota, ...]: rpm_set: Final = rpm_limits or _NO_LIMITS_SET tpm_set: Final = tpm_limits or _NO_LIMITS_SET gaps: Final = tuple( MissingModelQuota( model, - unmet_limits(required_limits(policy, model, providers_of(model)), _limits_set_for(model, rpm_set, tpm_set)), + unmet_limits(required_limits(policy, model, route_of(model)), _limits_set_for(model, rpm_set, tpm_set)), ) for model in models ) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 504cc9afcbc..ef00db71eeb 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -104,6 +104,7 @@ from litellm.proxy.management_helpers.auto_router_permissions import ( from litellm.proxy.management_helpers.model_allowlist_rename_sync import sync_model_allowlists_for_renamed_model from litellm.proxy.management_helpers.model_quota_policy import ( ModelQuotaPolicy, + ModelRoute, deployment_provider, parse_model_quota_setting, required_limits, @@ -781,7 +782,7 @@ def _model_add_quota_policy(general_settings: Mapping[str, object]) -> ModelQuot def _raise_if_rate_limits_required_but_missing( - *, model_name: str, litellm_params: LiteLLM_Params, policy: ModelQuotaPolicy | None + *, model_name: str, litellm_params: LiteLLM_Params, team_model: bool, policy: ModelQuotaPolicy | None ) -> None: """Off by default, so deployments keep adding models without limits. Once the operator opts in under general_settings, the policy decides per model name and provider which of rpm and tpm a new model needs, @@ -790,10 +791,20 @@ def _raise_if_rate_limits_required_but_missing( if policy is None: return provider: Final = deployment_provider( - MappingProxyType({"model": litellm_params.model, "custom_llm_provider": litellm_params.custom_llm_provider}) + MappingProxyType( + { + "model": litellm_params.model, + "custom_llm_provider": litellm_params.custom_llm_provider, + "api_base": litellm_params.api_base, + } + ) ) missing: Final = unmet_limits( - required_limits(policy, model_name, frozenset() if provider is None else frozenset((provider,))), + required_limits( + policy, + model_name, + ModelRoute(providers=frozenset() if provider is None else frozenset((provider,)), team_named=team_model), + ), MappingProxyType({"rpm": litellm_params.rpm, "tpm": litellm_params.tpm}), ) if not missing: @@ -2740,6 +2751,7 @@ async def add_new_model( _raise_if_rate_limits_required_but_missing( model_name=model_params.model_name, litellm_params=model_params.litellm_params, + team_model=model_params.model_info.team_id is not None, policy=_model_add_quota_policy(general_settings), ) diff --git a/litellm/proxy/management_helpers/model_quota_policy.py b/litellm/proxy/management_helpers/model_quota_policy.py index e2467e32997..28372b2aa2f 100644 --- a/litellm/proxy/management_helpers/model_quota_policy.py +++ b/litellm/proxy/management_helpers/model_quota_policy.py @@ -1,9 +1,13 @@ import re from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field +import litellm +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider, is_openai_compatible_endpoint from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.common_utils import provider_for_generic_call from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils @@ -12,7 +16,6 @@ if TYPE_CHECKING: from litellm import Router QuotaLimit: TypeAlias = Literal["rpm", "tpm"] -ProvidersOf: TypeAlias = Callable[[str], frozenset[str]] _LIMITS: Final[tuple[QuotaLimit, ...]] = ("rpm", "tpm") _ROUTER_PATTERNS: Final = PatternMatchRouter() @@ -27,20 +30,31 @@ class ModelQuotaPolicy(BaseModel): REQUIRE_BOTH_LIMITS: Final = ModelQuotaPolicy() +_POLICY_KEYS: Final = frozenset(ModelQuotaPolicy.model_fields) + + +@dataclass(frozen=True, slots=True) +class ModelRoute: + providers: frozenset[str] = frozenset() + team_named: bool = False + + +RouteOf: TypeAlias = Callable[[str], ModelRoute] +NO_ROUTE: Final = ModelRoute() def parse_model_quota_setting(setting: object) -> ModelQuotaPolicy | None: - """Any falsy value is off and any other non-mapping value requires both limits, as the boolean - setting always behaved, so only a mapping is read as a per-provider and per-model policy.""" + """Any falsy value is off and any other value requires both limits, as the boolean setting always + behaved, so only a mapping that names a policy key is read as a per-provider and per-model policy.""" if not setting: return None - if is_str_keyed_mapping(setting): + if is_str_keyed_mapping(setting) and not _POLICY_KEYS.isdisjoint(setting): return ModelQuotaPolicy.model_validate(setting) return REQUIRE_BOTH_LIMITS def _wildcard_matches(pattern: str, model: str) -> bool: - return re.fullmatch(_ROUTER_PATTERNS.pattern_to_regex(pattern), model) is not None + return re.match(_ROUTER_PATTERNS.pattern_to_regex(pattern), model) is not None def _router_specificity(pattern: str) -> tuple[int, int]: @@ -52,19 +66,26 @@ def _most_specific_model_pattern(model: str, entries: Mapping[str, frozenset[Quo return max(matching, key=_router_specificity, default=None) -def required_limits(policy: ModelQuotaPolicy, model: str, providers: frozenset[str]) -> frozenset[QuotaLimit]: - """A model entry (exact name, then the most specific `*` pattern) wins over the providers of the - model's deployments, which win over the default. A model served by several providers needs every - limit any of them requires.""" - exact: Final = policy.models.get(model) +def _model_entry(entries: Mapping[str, frozenset[QuotaLimit]], model: str) -> frozenset[QuotaLimit] | None: + exact: Final = entries.get(model) if exact is not None: return exact - pattern: Final = _most_specific_model_pattern(model, policy.models) - if pattern is not None: - return policy.models[pattern] - if not providers: + pattern: Final = _most_specific_model_pattern(model, entries) + return None if pattern is None else entries[pattern] + + +def required_limits(policy: ModelQuotaPolicy, model: str, route: ModelRoute) -> frozenset[QuotaLimit]: + """A model entry (exact name, then the most specific `*` pattern) wins over the providers of the + model's deployments, which win over the default. A model served by several providers needs every + limit any of them requires. A team model's name is picked by its team, so model entries never match it.""" + entry: Final = None if route.team_named else _model_entry(policy.models, model) + if entry is not None: + return entry + if not route.providers: return policy.default - return frozenset[QuotaLimit]().union(*(policy.providers.get(provider, policy.default) for provider in providers)) + return frozenset[QuotaLimit]().union( + *(policy.providers.get(provider, policy.default) for provider in route.providers) + ) def _is_positive(value: object) -> bool: @@ -79,12 +100,28 @@ def unmet_limits(required: frozenset[QuotaLimit], values: Mapping[QuotaLimit, ob ) -def no_providers(_model: str) -> frozenset[str]: - return frozenset() +def no_route(_model: str) -> ModelRoute: + return NO_ROUTE + + +def _provider_behind_api_base(litellm_params: Mapping[str, object]) -> str | None: + model: Final = litellm_params.get("model") + api_base: Final = litellm_params.get("api_base") + if litellm_params.get("custom_llm_provider") or not isinstance(model, str) or not isinstance(api_base, str): + return None + if model.split("/", 1)[0] in litellm.provider_list or not is_openai_compatible_endpoint(api_base): + return None + try: + _, provider, _, _ = get_llm_provider(model=model, api_base=api_base) + except BadRequestError: + return None + return provider def deployment_provider(litellm_params: object) -> str | None: - return provider_for_generic_call(litellm_params) if is_str_keyed_mapping(litellm_params) else None + if not is_str_keyed_mapping(litellm_params): + return None + return _provider_behind_api_base(litellm_params) or provider_for_generic_call(litellm_params) def _routed_deployments(llm_router: "Router", model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: @@ -96,15 +133,24 @@ def _routed_deployments(llm_router: "Router", model: str, team_id: str | None) - return named or llm_router.wildcard_route_deployments(model, team_id) -def router_deployment_providers(llm_router: "Router | None", team_id: str | None) -> ProvidersOf: - if llm_router is None: - return no_providers +def _is_team_deployment(deployment: Mapping[str, object]) -> bool: + model_info: Final = deployment.get("model_info") + return is_str_keyed_mapping(model_info) and model_info.get("team_id") is not None - def providers_of(model: str) -> frozenset[str]: - return frozenset( - provider - for deployment in _routed_deployments(llm_router, model, team_id) - if (provider := deployment_provider(deployment.get("litellm_params"))) is not None + +def router_model_routes(llm_router: "Router | None", team_id: str | None) -> RouteOf: + if llm_router is None: + return no_route + + def route_of(model: str) -> ModelRoute: + deployments: Final = _routed_deployments(llm_router, model, team_id) + return ModelRoute( + providers=frozenset( + provider + for deployment in deployments + if (provider := deployment_provider(deployment.get("litellm_params"))) is not None + ), + team_named=any(_is_team_deployment(deployment) for deployment in deployments), ) - return providers_of + return route_of diff --git a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index c1e3c035b57..d430e1d3df0 100644 --- a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -30,6 +30,7 @@ from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( delete_project, project_info, ) +from litellm.proxy.management_helpers.model_quota_policy import ModelRoute, router_model_routes from litellm.proxy.proxy_server import ( LitellmUserRoles, ) @@ -1618,8 +1619,8 @@ _CUSTOMER_MODEL_PROVIDERS: Final = { } -def _customer_providers_of(model: str) -> frozenset[str]: - return _CUSTOMER_MODEL_PROVIDERS.get(model, frozenset()) +def _customer_route_of(model: str) -> ModelRoute: + return ModelRoute(providers=_CUSTOMER_MODEL_PROVIDERS.get(model, frozenset())) def _quota_error( @@ -1627,7 +1628,7 @@ def _quota_error( rpm: dict[str, int] | None = None, tpm: dict[str, int] | None = None, setting: object = _CUSTOMER_QUOTA_POLICY, - providers_of: Callable[[str], frozenset[str]] = _customer_providers_of, + route_of: Callable[[str], ModelRoute] = _customer_route_of, ) -> str | None: from litellm_enterprise.proxy.management_endpoints.project_endpoints import ( _raise_on_missing_project_model_quota, @@ -1638,7 +1639,7 @@ def _quota_error( assert policy is not None data: Final = NewProjectRequest(team_id="test-team", models=models, model_rpm_limit=rpm, model_tpm_limit=tpm) try: - _raise_on_missing_project_model_quota(data, frozenset(), policy, providers_of) + _raise_on_missing_project_model_quota(data, frozenset(), policy, route_of) except HTTPException as e: rejection: Final = e else: @@ -1722,16 +1723,32 @@ def test_quota_policy_model_entry_matches_names_literally_apart_from_star(): def test_quota_policy_model_on_several_providers_needs_every_required_limit(): - def mixed_providers(_model: str) -> frozenset[str]: - return frozenset({"runwayml", "vertex_ai"}) + def mixed_providers(_model: str) -> ModelRoute: + return ModelRoute(providers=frozenset({"runwayml", "vertex_ai"})) - error = _quota_error(["video-pool"], rpm={"video-pool": 10}, providers_of=mixed_providers) + error = _quota_error(["video-pool"], rpm={"video-pool": 10}, route_of=mixed_providers) assert error is not None assert "video-pool (model_tpm_limit)" in error -@pytest.mark.parametrize("setting", [True, "true", "false", 1, ["rpm"]]) -def test_quota_setting_truthy_non_mapping_still_requires_both_limits(setting: object): +def test_quota_policy_model_pattern_matches_from_the_start_like_the_router(): + setting: Final = {"providers": {"vertex_ai": ["rpm", "tpm"]}, "models": {"*-lite": []}} + assert _quota_error(["veo-3.1-lite-generate-001"], setting=setting) is None + assert _quota_error(["gemini-3.8-flash"], setting=setting) is not None + + +def test_quota_policy_model_entries_never_match_a_team_model_name(): + def team_vertex_route(_model: str) -> ModelRoute: + return ModelRoute(providers=frozenset({"vertex_ai"}), team_named=True) + + exact: Final = _quota_error(["jev-latest"], route_of=team_vertex_route) + pattern: Final = _quota_error(["veo-team"], rpm={"veo-team": 5}, route_of=team_vertex_route) + assert exact is not None and "jev-latest (model_rpm_limit and model_tpm_limit)" in exact + assert pattern is not None and "veo-team (model_tpm_limit)" in pattern + + +@pytest.mark.parametrize("setting", [True, "true", "false", 1, ["rpm"], {"enabled": True}, {"provider": {"runwayml": []}}]) +def test_quota_setting_truthy_non_policy_value_still_requires_both_limits(setting: object): error: Final = _quota_error(["runway-gen4-turbo"], setting=setting) assert error is not None assert "runway-gen4-turbo (model_rpm_limit and model_tpm_limit)" in error @@ -1761,7 +1778,7 @@ def test_update_quota_policy_resolves_providers_for_the_resulting_models(): existing, frozenset(), policy, - _customer_providers_of, + _customer_route_of, ) is None ) @@ -1771,7 +1788,7 @@ def test_update_quota_policy_resolves_providers_for_the_resulting_models(): existing, frozenset(), policy, - _customer_providers_of, + _customer_route_of, ) assert "gemini-3.8-flash (model_rpm_limit and model_tpm_limit)" in str(exc_info.value.detail) @@ -1796,8 +1813,6 @@ def _customer_router() -> "litellm.Router": def test_quota_providers_come_from_the_one_wildcard_route_requests_use(): - from litellm.proxy.management_helpers.model_quota_policy import router_deployment_providers - router: Final = litellm.Router( model_list=[ {"model_name": "gen4*", "litellm_params": {"model": "runwayml/gen4*", "api_key": "qa-runway-key"}}, @@ -1808,10 +1823,30 @@ def test_quota_providers_come_from_the_one_wildcard_route_requests_use(): }, ] ) - providers_of: Final = router_deployment_providers(router, "media-team") + route_of: Final = router_model_routes(router, "media-team") - assert providers_of("gen4_turbo") == frozenset({"runwayml"}) - assert providers_of("gen3a_turbo") == frozenset({"vertex_ai"}) + assert route_of("gen4_turbo") == ModelRoute(providers=frozenset({"runwayml"})) + assert route_of("gen3a_turbo") == ModelRoute(providers=frozenset({"vertex_ai"}), team_named=True) + + +def test_quota_providers_read_the_provider_behind_a_known_api_base(): + groq_api_base: Final = "https://api.groq.com/openai/v1" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qa-llama", + "litellm_params": {"model": "qa-llama", "api_base": groq_api_base, "api_key": "qa-groq-key"}, + }, + { + "model_name": "qa-runway", + "litellm_params": {"model": "gen4_turbo", "custom_llm_provider": "runwayml", "api_base": groq_api_base}, + }, + ] + ) + route_of: Final = router_model_routes(router, None) + + assert route_of("qa-llama").providers == frozenset({"groq"}) + assert route_of("qa-runway").providers == frozenset({"runwayml"}) @pytest.mark.asyncio @@ -1839,7 +1874,7 @@ async def test_new_project_rejects_malformed_quota_setting_instead_of_skipping_i monkeypatch, team_models=["runway-gen4-turbo"], llm_router=_customer_router(), - quota_setting={"provider": {"runwayml": []}}, + quota_setting={"providers": {"runwayml": []}, "model": {"jev-latest": []}}, ) with pytest.raises(ProxyException) as exc_info: @@ -1849,6 +1884,47 @@ async def test_new_project_rejects_malformed_quota_setting_instead_of_skipping_i assert "general_settings.enforce_project_model_quota is invalid" in str(exc_info.value.message) +@pytest.mark.asyncio +async def test_new_project_checks_permission_before_reading_the_quota_setting(monkeypatch): + from litellm_enterprise.proxy.management_endpoints import project_endpoints as pe + + _enforced_new_project_mocks( + monkeypatch, + team_models=["runway-gen4-turbo"], + llm_router=_customer_router(), + quota_setting={"providers": {"runwayml": []}, "model": {"jev-latest": []}}, + ) + monkeypatch.setattr(pe, "_check_user_permission_for_project", mock.AsyncMock(return_value=False)) + + with pytest.raises(ProxyException) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=["runway-gen4-turbo"])) + + assert exc_info.value.code == "403" + assert "enforce_project_model_quota" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_new_project_quota_policy_ignores_model_entries_for_a_team_model_name(monkeypatch): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "model_name_test-team_qa", + "litellm_params": {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"}, + "model_info": {"team_id": "test-team", "team_public_model_name": "jev-latest"}, + }, + ] + ) + _enforced_new_project_mocks( + monkeypatch, team_models=["jev-latest"], llm_router=router, quota_setting=_CUSTOMER_QUOTA_POLICY + ) + + with pytest.raises(ProxyException) as exc_info: + await _run_new_project(NewProjectRequest(team_id="test-team", models=["jev-latest"])) + + assert exc_info.value.code == "400" + assert "jev-latest (model_rpm_limit and model_tpm_limit)" in str(exc_info.value.message) + + _ACCESS_NOW: Final = datetime.now(timezone.utc) diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 5870e6fad9a..aebbaf28446 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -7365,7 +7365,7 @@ class TestEnforceRpmTpmOnModelAdd: def test_passes_when_disabled_even_without_limits(self): assert ( _raise_if_rate_limits_required_but_missing( - model_name="gpt-5.2", litellm_params=LiteLLM_Params(model="azure/gpt-5.2"), policy=None + model_name="gpt-5.2", team_model=False, litellm_params=LiteLLM_Params(model="azure/gpt-5.2"), policy=None ) is None ) @@ -7373,7 +7373,7 @@ class TestEnforceRpmTpmOnModelAdd: def test_passes_when_enabled_and_both_set(self): assert ( _raise_if_rate_limits_required_but_missing( - model_name="gpt-5.2", + model_name="gpt-5.2", team_model=False, litellm_params=LiteLLM_Params(model="azure/gpt-5.2", rpm=10, tpm=1000), policy=REQUIRE_BOTH_LIMITS, ) @@ -7393,7 +7393,7 @@ class TestEnforceRpmTpmOnModelAdd: def test_raises_when_enabled_and_missing(self, params: LiteLLM_Params, expected_missing: str): with pytest.raises(ProxyException) as exc_info: _raise_if_rate_limits_required_but_missing( - model_name="gpt-5.2", litellm_params=params, policy=REQUIRE_BOTH_LIMITS + model_name="gpt-5.2", team_model=False, litellm_params=params, policy=REQUIRE_BOTH_LIMITS ) assert exc_info.value.message.startswith(f"{expected_missing} must be set") assert exc_info.value.code == "400" @@ -7413,7 +7413,7 @@ class TestEnforceRpmTpmOnModelAdd: ): assert ( _raise_if_rate_limits_required_but_missing( - model_name=model_name, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY + model_name=model_name, team_model=False, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY ) is None ) @@ -7432,11 +7432,43 @@ class TestEnforceRpmTpmOnModelAdd: ): with pytest.raises(ProxyException) as exc_info: _raise_if_rate_limits_required_but_missing( - model_name=model_name, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY + model_name=model_name, team_model=False, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY ) assert exc_info.value.message.startswith(f"{expected_missing} must be set to a positive value for '{model_name}'") assert exc_info.value.code == "400" + def test_policy_reads_the_provider_behind_a_known_api_base(self): + policy: Final = ModelQuotaPolicy.model_validate({"default": [], "providers": {"groq": ["rpm", "tpm"]}}) + with pytest.raises(ProxyException) as exc_info: + _raise_if_rate_limits_required_but_missing( + model_name="qa-llama", team_model=False, + litellm_params=LiteLLM_Params(model="qa-llama", api_base="https://api.groq.com/openai/v1"), + policy=policy, + ) + assert exc_info.value.message.startswith("rpm and tpm must be set to a positive value for 'qa-llama'") + + @pytest.mark.parametrize( + ("model_name", "params", "expected_missing"), + [ + ("jev-latest", LiteLLM_Params(model="vertex_ai/gemini-3.8-flash"), "rpm and tpm"), + ("veo-team", LiteLLM_Params(model="vertex_ai/veo-3.1-generate-001", rpm=5), "tpm"), + ], + ) + def test_policy_model_entries_never_match_a_team_model_name( + self, model_name: str, params: LiteLLM_Params, expected_missing: str + ): + assert ( + _raise_if_rate_limits_required_but_missing( + model_name=model_name, team_model=False, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY + ) + is None + ) + with pytest.raises(ProxyException) as exc_info: + _raise_if_rate_limits_required_but_missing( + model_name=model_name, team_model=True, litellm_params=params, policy=_CUSTOMER_MODEL_ADD_POLICY + ) + assert exc_info.value.message.startswith(f"{expected_missing} must be set to a positive value for '{model_name}'") + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -7444,6 +7476,7 @@ class TestEnforceRpmTpmOnModelAdd: [ ({"default": [], "models": {"veo-*": ["rpm"]}}, "400", "rpm must be set to a positive value for 'veo-3.1'"), ("yes", "400", "rpm and tpm must be set to a positive value for 'veo-3.1'"), + ({"enabled": True}, "400", "rpm and tpm must be set to a positive value for 'veo-3.1'"), ( {"providers": {"vertex_ai": ["qps"]}}, "500", @@ -7470,6 +7503,32 @@ async def test_model_add_resolves_the_quota_policy_for_the_public_model_name( prisma.db.litellm_proxymodeltable.create.assert_not_called() +@pytest.mark.asyncio +async def test_team_model_add_cannot_borrow_a_model_entry_by_its_public_name(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + prisma: Final = MagicMock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr( + proxy_server, + "general_settings", + {"enforce_rpm_tpm_on_model_add": {"providers": {"vertex_ai": ["rpm", "tpm"]}, "models": {"jev-latest": []}}}, + ) + with pytest.raises(ProxyException) as error: + await add_new_model( + Deployment( + model_name="jev-latest", + litellm_params={"model": "vertex_ai/gemini-3.8-flash"}, + model_info={"team_id": "media-team"}, + ), + UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert error.value.code == "400" + assert error.value.message.startswith("rpm and tpm must be set to a positive value for 'jev-latest'") + prisma.db.litellm_proxymodeltable.create.assert_not_called() + + class TestBlockModelResponseSerialization: @pytest.mark.parametrize( ("route", "blocked"), [("/model/block", True), ("/model/unblock", False)] From 39cb563098818b5259f78ad4ef6721f7b59678a3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:31:55 -0700 Subject: [PATCH 6/6] fix(proxy): re-check team projects on team model writes and validate quota provider ids --- .../management_endpoints/project_endpoints.py | 2 +- .../project_model_quota.py | 1 - .../model_management_endpoints.py | 165 ++++++++++++++-- .../management_helpers/model_quota_policy.py | 59 +++++- .../test_project_endpoints_prisma.py | 125 ++++++++++++ .../test_model_management_endpoints.py | 179 ++++++++++++++++++ 6 files changed, 515 insertions(+), 16 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index dea8e1b196f..a4a56a634de 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, Final, cast from fastapi import APIRouter, Depends, HTTPException, Request from litellm_enterprise.proxy.management_endpoints.project_model_quota import ( - ENFORCE_PROJECT_MODEL_QUOTA_SETTING, describe_missing_quota, missing_project_model_quota, ) @@ -33,6 +32,7 @@ from litellm.proxy.management.users.service import org_admin_org_ids from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects from litellm.proxy.management_helpers.model_quota_policy import ( + ENFORCE_PROJECT_MODEL_QUOTA_SETTING, REQUIRE_BOTH_LIMITS, ModelQuotaPolicy, RouteOf, diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py index e7078e0bf30..e36ff9b53dc 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_model_quota.py @@ -11,7 +11,6 @@ from litellm.proxy.management_helpers.model_quota_policy import ( unmet_limits, ) -ENFORCE_PROJECT_MODEL_QUOTA_SETTING: Final = "enforce_project_model_quota" _NO_LIMITS_SET: Final[Mapping[str, object]] = MappingProxyType({}) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index ef00db71eeb..0ce827d214d 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.ptu_pricing import ( ptu_config_error, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import ( BlockModelRequest, CommonProxyErrors, @@ -103,9 +104,11 @@ from litellm.proxy.management_helpers.auto_router_permissions import ( ) from litellm.proxy.management_helpers.model_allowlist_rename_sync import sync_model_allowlists_for_renamed_model from litellm.proxy.management_helpers.model_quota_policy import ( + ENFORCE_PROJECT_MODEL_QUOTA_SETTING, ModelQuotaPolicy, ModelRoute, deployment_provider, + limits_added_by_team_write, parse_model_quota_setting, required_limits, unmet_limits, @@ -119,6 +122,7 @@ from litellm.repositories.chunked_in import find_many_in from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.model_repository import ModelRepository from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ModelTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.router import Router @@ -769,12 +773,24 @@ async def _auto_router_capability_slot( ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add" -def _model_add_quota_policy(general_settings: Mapping[str, object]) -> ModelQuotaPolicy | None: +_ROUTING_KEYS: Final = ("model", "custom_llm_provider", "api_base") +_NOTHING_SENT: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _written_routing_params( + incoming: GenericLiteLLMParams | None, existing: GenericLiteLLMParams | None +) -> Mapping[str, object]: + sent: Final = _NOTHING_SENT if incoming is None else _dump_without_none(incoming) + stored: Final = _NOTHING_SENT if existing is None else _decrypted_litellm_params(existing) + return MappingProxyType({key: sent.get(key, stored.get(key)) for key in _ROUTING_KEYS}) + + +def _quota_policy(general_settings: Mapping[str, object], setting: str) -> ModelQuotaPolicy | None: try: - return parse_model_quota_setting(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING)) + return parse_model_quota_setting(general_settings.get(setting)) except ValidationError as e: raise ProxyException( - message=f"general_settings.{ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING} is invalid: {e}", + message=f"general_settings.{setting} is invalid: {e}", type=ProxyErrorTypes.internal_server_error.value, code=status.HTTP_500_INTERNAL_SERVER_ERROR, param=None, @@ -790,15 +806,7 @@ def _raise_if_rate_limits_required_but_missing( (or effectively excluded from routing).""" if policy is None: return - provider: Final = deployment_provider( - MappingProxyType( - { - "model": litellm_params.model, - "custom_llm_provider": litellm_params.custom_llm_provider, - "api_base": litellm_params.api_base, - } - ) - ) + provider: Final = deployment_provider(_written_routing_params(litellm_params, None)) missing: Final = unmet_limits( required_limits( policy, @@ -820,6 +828,94 @@ def _raise_if_rate_limits_required_but_missing( ) +def _project_limit(metadata: object, limits_key: str, model: str) -> object: + limits: Final = metadata.get(limits_key) if is_str_keyed_mapping(metadata) else None + return limits.get(model) if is_str_keyed_mapping(limits) else None + + +async def _raise_if_team_write_leaves_projects_short( + *, + prisma_client: PrismaClient, + general_settings: Mapping[str, object], + llm_router: Router | None, + team_id: str | None, + public_name: str, + replaced_id: str | None, + written_params: Mapping[str, object] | None, +) -> None: + if team_id is None: + return + policy: Final = _quota_policy(general_settings, ENFORCE_PROJECT_MODEL_QUOTA_SETTING) + if policy is None: + return + added: Final = limits_added_by_team_write(policy, llm_router, team_id, public_name, replaced_id, written_params) + if not added: + return + projects: Final = await ProjectRepository(prisma_client).table.find_many( + where={"team_id": team_id, "models": {"has": public_name}} + ) + short: Final = tuple( + project.project_alias or project.project_id + for project in projects + if unmet_limits( + added, + MappingProxyType( + { + "rpm": _project_limit(project.metadata, "model_rpm_limit", public_name), + "tpm": _project_limit(project.metadata, "model_tpm_limit", public_name), + } + ), + ) + ) + if not short: + return + raise ProxyException( + message=( + f"projects {', '.join(short)} list '{public_name}' without a positive {' and '.join(sorted(added))} " + f"quota, which '{ENFORCE_PROJECT_MODEL_QUOTA_SETTING}' requires once this team model change applies. " + "Set those limits on the projects first" + ), + type=ProxyErrorTypes.validation_error.value, + code=status.HTTP_400_BAD_REQUEST, + param=None, + ) + + +async def _raise_if_team_update_leaves_projects_short( + *, + prisma_client: PrismaClient, + general_settings: Mapping[str, object], + llm_router: Router | None, + existing: Deployment, + written_team_id: str | None, + written_name: str, + incoming_params: GenericLiteLLMParams | None, +) -> None: + if written_team_id is None and existing.model_info.team_id is None: + return + stored_name: Final = existing.model_info.team_public_model_name or existing.model_name + await _raise_if_team_write_leaves_projects_short( + prisma_client=prisma_client, + general_settings=general_settings, + llm_router=llm_router, + team_id=written_team_id, + public_name=written_name, + replaced_id=existing.model_info.id, + written_params=_written_routing_params(incoming_params, existing.litellm_params), + ) + if (written_team_id, written_name) == (existing.model_info.team_id, stored_name): + return + await _raise_if_team_write_leaves_projects_short( + prisma_client=prisma_client, + general_settings=general_settings, + llm_router=llm_router, + team_id=existing.model_info.team_id, + public_name=stored_name, + replaced_id=existing.model_info.id, + written_params=None, + ) + + _PTU_PRICED_PAIR: Final = frozenset({"ptu_count", "cost_per_ptu_per_hour"}) @@ -1308,6 +1404,7 @@ async def patch_model( ProxyException: For various error conditions including authentication and database errors """ from litellm.proxy.proxy_server import ( + general_settings_view, litellm_proxy_admin_name, llm_router, premium_user, @@ -1417,6 +1514,17 @@ async def patch_model( existing_params=db_model.litellm_params, ) + await _raise_if_team_update_leaves_projects_short( + prisma_client=prisma_client, + general_settings=general_settings_view(), + llm_router=llm_router, + existing=db_model, + written_team_id=(patch_data.model_info.team_id if patch_data.model_info else None) + or db_model.model_info.team_id, + written_name=_get_public_model_name(effective_patch, db_model), + incoming_params=patch_data.litellm_params, + ) + effective_params: Final = _effective_complexity_router_params( patch_data.litellm_params, db_model.litellm_params ) @@ -2492,6 +2600,7 @@ async def delete_model( from litellm.proxy.proxy_server import ( MODEL_RECONCILE_LOCK, + general_settings_view, llm_router, premium_user, prisma_client, @@ -2525,6 +2634,15 @@ async def delete_model( incoming_params=None, allow_missing_team=True, ) + await _raise_if_team_write_leaves_projects_short( + prisma_client=prisma_client, + general_settings=general_settings_view(), + llm_router=llm_router, + team_id=model_params.model_info.team_id, + public_name=model_params.model_info.team_public_model_name or model_params.model_name, + replaced_id=model_params.model_info.id, + written_params=None, + ) # update DB if store_model_in_db is True: @@ -2691,6 +2809,8 @@ async def add_new_model( """ from litellm.proxy.proxy_server import ( general_settings, + general_settings_view, + llm_router, premium_user, prisma_client, proxy_config, @@ -2752,7 +2872,16 @@ async def add_new_model( model_name=model_params.model_name, litellm_params=model_params.litellm_params, team_model=model_params.model_info.team_id is not None, - policy=_model_add_quota_policy(general_settings), + policy=_quota_policy(general_settings_view(), ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING), + ) + await _raise_if_team_write_leaves_projects_short( + prisma_client=prisma_client, + general_settings=general_settings_view(), + llm_router=llm_router, + team_id=model_params.model_info.team_id, + public_name=model_params.model_name, + replaced_id=None, + written_params=_written_routing_params(model_params.litellm_params, None), ) clean_model_info: Final = ModelInfo.model_validate( @@ -2891,6 +3020,7 @@ async def update_model( """ from litellm.proxy.proxy_server import ( LITELLM_PROXY_ADMIN_NAME, + general_settings_view, llm_router, premium_user, prisma_client, @@ -2955,6 +3085,15 @@ async def update_model( incoming_params=model_params.litellm_params, existing_params=deployment.litellm_params, ) + await _raise_if_team_update_leaves_projects_short( + prisma_client=prisma_client, + general_settings=general_settings_view(), + llm_router=llm_router, + existing=deployment, + written_team_id=deployment.model_info.team_id, + written_name=deployment.model_info.team_public_model_name or deployment.model_name, + incoming_params=model_params.litellm_params, + ) effective_params: Final = _effective_complexity_router_params( model_params.litellm_params, deployment.litellm_params ) diff --git a/litellm/proxy/management_helpers/model_quota_policy.py b/litellm/proxy/management_helpers/model_quota_policy.py index 28372b2aa2f..57bb83f2fda 100644 --- a/litellm/proxy/management_helpers/model_quota_policy.py +++ b/litellm/proxy/management_helpers/model_quota_policy.py @@ -3,12 +3,13 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, TypeAlias -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator import litellm from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider, is_openai_compatible_endpoint from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping +from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.common_utils import provider_for_generic_call from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils @@ -17,10 +18,16 @@ if TYPE_CHECKING: QuotaLimit: TypeAlias = Literal["rpm", "tpm"] +ENFORCE_PROJECT_MODEL_QUOTA_SETTING: Final = "enforce_project_model_quota" + _LIMITS: Final[tuple[QuotaLimit, ...]] = ("rpm", "tpm") _ROUTER_PATTERNS: Final = PatternMatchRouter() +def _is_known_provider(provider: str) -> bool: + return provider in litellm.provider_list or JSONProviderRegistry.exists(provider) + + class ModelQuotaPolicy(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -28,6 +35,14 @@ class ModelQuotaPolicy(BaseModel): providers: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) models: Mapping[str, frozenset[QuotaLimit]] = Field(default_factory=dict) + @field_validator("providers") + @classmethod + def _known_providers(cls, providers: Mapping[str, frozenset[QuotaLimit]]) -> Mapping[str, frozenset[QuotaLimit]]: + unknown: Final = sorted(provider for provider in providers if not _is_known_provider(provider)) + if unknown: + raise ValueError(f"unknown provider ids {unknown}") + return providers + REQUIRE_BOTH_LIMITS: Final = ModelQuotaPolicy() _POLICY_KEYS: Final = frozenset(ModelQuotaPolicy.model_fields) @@ -154,3 +169,45 @@ def router_model_routes(llm_router: "Router | None", team_id: str | None) -> Rou ) return route_of + + +def _deployment_id(deployment: Mapping[str, object]) -> object: + model_info: Final = deployment.get("model_info") + return model_info.get("id") if is_str_keyed_mapping(model_info) else None + + +def team_route_after_write( + llm_router: "Router | None", + team_id: str, + public_name: str, + replaced_id: str | None, + written_params: Mapping[str, object] | None, +) -> ModelRoute: + deployments: Final = () if llm_router is None else llm_router.get_all_deployments(public_name, team_id=team_id) + kept: Final = tuple( + deployment.get("litellm_params") + for deployment in deployments + if _is_team_deployment(deployment) and _deployment_id(deployment) != replaced_id + ) + team_params: Final = kept if written_params is None else (*kept, written_params) + if not team_params: + return router_model_routes(llm_router, None)(public_name) + return ModelRoute( + providers=frozenset( + provider for litellm_params in team_params if (provider := deployment_provider(litellm_params)) is not None + ), + team_named=True, + ) + + +def limits_added_by_team_write( + policy: ModelQuotaPolicy, + llm_router: "Router | None", + team_id: str, + public_name: str, + replaced_id: str | None, + written_params: Mapping[str, object] | None, +) -> frozenset[QuotaLimit]: + after: Final = team_route_after_write(llm_router, team_id, public_name, replaced_id, written_params) + now: Final = router_model_routes(llm_router, team_id)(public_name) + return required_limits(policy, public_name, after) - required_limits(policy, public_name, now) diff --git a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index d430e1d3df0..97bb36dc6c3 100644 --- a/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -1849,6 +1849,131 @@ def test_quota_providers_read_the_provider_behind_a_known_api_base(): assert route_of("qa-runway").providers == frozenset({"runwayml"}) +def test_quota_policy_rejects_an_unknown_provider_id(): + from pydantic import ValidationError + + from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting + + with pytest.raises(ValidationError, match=r"unknown provider ids \['runway'\]"): + parse_model_quota_setting({"providers": {"vertex_ai": ["rpm"], "runway": []}}) + + +def test_quota_policy_accepts_a_provider_registered_at_runtime(monkeypatch): + from litellm.proxy.management_helpers.model_quota_policy import parse_model_quota_setting + + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "qa-custom-llm"]) + policy: Final = parse_model_quota_setting({"providers": {"qa-custom-llm": []}}) + + assert policy is not None + assert policy.providers["qa-custom-llm"] == frozenset() + + +_TEAM_WRITE_POLICY: Final = {"providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []}, "models": {"jev-latest": []}} +_VERTEX_PARAMS: Final = {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"} +_RUNWAY_PARAMS: Final = {"model": "gen4_turbo", "custom_llm_provider": "runwayml"} + + +def _team_deployment(public_name: str, model_id: str, litellm_params: Mapping[str, object]) -> dict[str, object]: + return { + "model_name": f"model_name_media-team_{model_id}", + "litellm_params": dict(litellm_params), + "model_info": {"id": model_id, "team_id": "media-team", "team_public_model_name": public_name}, + } + + +@pytest.mark.parametrize( + ("setting", "public_name", "proxy_params", "team_routes", "replaced_id", "written_params", "added"), + [ + pytest.param( + _TEAM_WRITE_POLICY, + "jev-latest", + {"model": "openai/jev-latest"}, + (), + None, + _VERTEX_PARAMS, + {"rpm", "tpm"}, + id="team-add-on-vertex-behind-an-exempt-proxy-name", + ), + pytest.param( + _TEAM_WRITE_POLICY, + "jev-latest", + {"model": "openai/jev-latest"}, + (), + None, + _RUNWAY_PARAMS, + set(), + id="team-add-on-an-exempt-provider", + ), + pytest.param( + _TEAM_WRITE_POLICY, + "media-latest", + _VERTEX_PARAMS, + (("t1", _RUNWAY_PARAMS),), + "t1", + None, + {"rpm", "tpm"}, + id="team-delete-falls-back-to-the-proxy-vertex-route", + ), + pytest.param( + _TEAM_WRITE_POLICY, + "media-latest", + _VERTEX_PARAMS, + (("t1", _RUNWAY_PARAMS), ("t2", _RUNWAY_PARAMS)), + "t1", + None, + set(), + id="team-delete-keeps-a-sibling-team-route", + ), + pytest.param( + _TEAM_WRITE_POLICY, + "media-latest", + _VERTEX_PARAMS, + (("t1", _RUNWAY_PARAMS),), + "t1", + _VERTEX_PARAMS, + {"rpm", "tpm"}, + id="team-update-moves-the-route-to-vertex", + ), + pytest.param( + True, + "jev-latest", + {"model": "openai/jev-latest"}, + (), + None, + _VERTEX_PARAMS, + set(), + id="true-already-requires-both-limits", + ), + ], +) +def test_team_model_write_adds_only_the_limits_its_new_route_requires( + setting: object, + public_name: str, + proxy_params: Mapping[str, object], + team_routes: tuple[tuple[str, Mapping[str, object]], ...], + replaced_id: str | None, + written_params: Mapping[str, object] | None, + added: set[str], +): + from litellm.proxy.management_helpers.model_quota_policy import ( + limits_added_by_team_write, + parse_model_quota_setting, + ) + + policy: Final = parse_model_quota_setting(setting) + router: Final = litellm.Router( + model_list=[ + {"model_name": public_name, "litellm_params": dict(proxy_params)}, + *(_team_deployment(public_name, model_id, params) for model_id, params in team_routes), + ] + ) + + assert policy is not None + assert limits_added_by_team_write( + policy, router, "media-team", public_name, replaced_id, written_params + ) == frozenset(added) + + @pytest.mark.asyncio async def test_new_project_quota_policy_reads_providers_from_router_deployments(monkeypatch): team_models = ["gemini-3.8-flash", "veo-3.1-lite-generate-001", "runway-gen4-turbo", "jev-latest"] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index aebbaf28446..e78b975f62b 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -15,6 +15,7 @@ from fastapi.testclient import TestClient from litellm._uuid import uuid from litellm.models.credentials import CredentialItem +from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( LiteLLM_ModelTable, @@ -7509,7 +7510,9 @@ async def test_team_model_add_cannot_borrow_a_model_entry_by_its_public_name(mon from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model prisma: Final = MagicMock() + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=LiteLLM_TeamTable(team_id="media-team")) monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "premium_user", True) monkeypatch.setattr( proxy_server, "general_settings", @@ -7529,6 +7532,182 @@ async def test_team_model_add_cannot_borrow_a_model_entry_by_its_public_name(mon prisma.db.litellm_proxymodeltable.create.assert_not_called() +_PROJECT_QUOTA_SETTINGS: Final = { + "enforce_project_model_quota": { + "providers": {"vertex_ai": ["rpm", "tpm"], "runwayml": []}, + "models": {"jev-latest": []}, + } +} +_TEAM_VERTEX: Final = {"model": "vertex_ai/gemini-3.8-flash", "vertex_project": "qa-project"} +_TEAM_RUNWAY: Final = {"model": "gen4_turbo", "custom_llm_provider": "runwayml"} + + +def _media_team_db(public_name: str, project_metadata: Mapping[str, object]) -> MagicMock: + prisma: Final = MagicMock() + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=LiteLLM_TeamTable(team_id="media-team")) + prisma.db.litellm_projecttable.find_many = AsyncMock( + return_value=[ + LiteLLM_ProjectTable( + project_id="p-1", + project_alias="video-gen", + team_id="media-team", + models=[public_name], + metadata=dict(project_metadata), + ) + ] + ) + return prisma + + +def _team_model_row(public_name: str, litellm_params: Mapping[str, object]) -> LiteLLM_ProxyModelTable: + return LiteLLM_ProxyModelTable( + model_id="team-row", + model_name="model_name_media-team_team-row", + litellm_params=dict(litellm_params), + model_info={"id": "team-row", "team_id": "media-team", "team_public_model_name": public_name}, + created_by="admin", + updated_by="admin", + ) + + +def _quota_router(public_name: str, proxy_params: Mapping[str, object], *team_params: Mapping[str, object]) -> Router: + return Router( + model_list=[ + {"model_name": public_name, "litellm_params": dict(proxy_params)}, + *( + { + "model_name": "model_name_media-team_team-row", + "litellm_params": dict(params), + "model_info": {"id": "team-row", "team_id": "media-team", "team_public_model_name": public_name}, + } + for params in team_params + ), + ] + ) + + +def _use_quota_proxy(monkeypatch: pytest.MonkeyPatch, prisma: MagicMock, router: Router) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "general_settings", _PROJECT_QUOTA_SETTINGS) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + monkeypatch.setattr(proxy_server, "premium_user", True) + + +_ADMIN: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + + +@pytest.mark.asyncio +async def test_team_model_add_rejects_a_route_its_projects_lack_the_quota_for(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + prisma: Final = _media_team_db("jev-latest", {}) + _use_quota_proxy(monkeypatch, prisma, _quota_router("jev-latest", {"model": "openai/jev-latest"})) + + with pytest.raises(ProxyException) as error: + await add_new_model( + Deployment( + model_name="jev-latest", + litellm_params={**_TEAM_VERTEX, "rpm": 10, "tpm": 1000}, + model_info={"team_id": "media-team"}, + ), + _ADMIN, + ) + + assert error.value.code == "400" + assert error.value.message.startswith( + "projects video-gen list 'jev-latest' without a positive rpm and tpm quota, " + "which 'enforce_project_model_quota' requires once this team model change applies" + ) + prisma.db.litellm_projecttable.find_many.assert_awaited_once_with( + where={"team_id": "media-team", "models": {"has": "jev-latest"}} + ) + + +@pytest.mark.parametrize( + ("litellm_params", "project_metadata"), + [ + pytest.param( + _TEAM_VERTEX, + {"model_rpm_limit": {"jev-latest": 5}, "model_tpm_limit": {"jev-latest": 500}}, + id="projects-carry-the-added-limits", + ), + pytest.param(_TEAM_RUNWAY, {}, id="new-route-is-exempt"), + ], +) +@pytest.mark.asyncio +async def test_team_model_add_goes_through_when_its_projects_meet_the_new_route( + monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object], project_metadata: Mapping[str, object] +) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + _use_quota_proxy( + monkeypatch, _media_team_db("jev-latest", project_metadata), _quota_router("jev-latest", {"model": "openai/jev-latest"}) + ) + + with pytest.raises(ProxyException) as error: + await add_new_model( + Deployment(model_name="jev-latest", litellm_params=dict(litellm_params), model_info={"team_id": "media-team"}), + _ADMIN, + ) + + assert "STORE_MODEL_IN_DB" in error.value.message + + +@pytest.mark.parametrize("endpoint", ["patch", "put"]) +@pytest.mark.asyncio +async def test_team_model_update_rejects_moving_its_route_where_its_projects_lack_quota( + monkeypatch: pytest.MonkeyPatch, endpoint: str +) -> None: + from litellm.proxy import proxy_server + + runway_by_prefix: Final = {"model": "runwayml/gen4_turbo"} + prisma: Final = _media_team_db("jev-latest", {}) + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock( + return_value=_team_model_row("jev-latest", runway_by_prefix) + ) + _use_quota_proxy(monkeypatch, prisma, _quota_router("jev-latest", {"model": "openai/jev-latest"}, runway_by_prefix)) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + moved_to_vertex: Final = updateLiteLLMParams(model="vertex_ai/gemini-3.8-flash", vertex_project="qa-project") + + write: Final = ( + patch_model( + model_id="team-row", patch_data=updateDeployment(litellm_params=moved_to_vertex), user_api_key_dict=_ADMIN + ) + if endpoint == "patch" + else update_model( + model_params=updateDeployment(litellm_params=moved_to_vertex, model_info=ModelInfo(id="team-row")), + user_api_key_dict=_ADMIN, + ) + ) + + with pytest.raises(ProxyException) as error: + await write + + assert error.value.code == "400" + assert error.value.message.startswith("projects video-gen list 'jev-latest' without a positive rpm and tpm quota") + + +@pytest.mark.asyncio +async def test_team_model_delete_rejects_falling_back_to_a_route_its_projects_lack_quota_for( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete, delete_model + + prisma: Final = _media_team_db("media-latest", {}) + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=_team_model_row("media-latest", _TEAM_RUNWAY)) + _use_quota_proxy(monkeypatch, prisma, _quota_router("media-latest", _TEAM_VERTEX, _TEAM_RUNWAY)) + + with pytest.raises(ProxyException) as error: + await delete_model(model_info=ModelInfoDelete(id="team-row"), user_api_key_dict=_ADMIN) + + assert error.value.code == "400" + assert error.value.message.startswith("projects video-gen list 'media-latest' without a positive rpm and tpm quota") + prisma.db.litellm_proxymodeltable.delete.assert_not_called() + + class TestBlockModelResponseSerialization: @pytest.mark.parametrize( ("route", "blocked"), [("/model/block", True), ("/model/unblock", False)]