diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql new file mode 100644 index 00000000000..298fbb5c241 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT; + +-- AlterTable +ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT; + +-- AlterTable +ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT; + +-- AlterTable +ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT; diff --git a/litellm/constants.py b/litellm/constants.py index 5751e6e46af..5b3b07e91d9 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1556,6 +1556,8 @@ BASE_MCP_ROUTE: Final = "/mcp" BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24)) # for 24 hours +BATCH_TPD_WINDOW_SECONDS: Final = 86400 +BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd" HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)) # 60 seconds _background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS") diff --git a/litellm/models/budget.py b/litellm/models/budget.py index 335800a49a8..125ce739d6a 100644 --- a/litellm/models/budget.py +++ b/litellm/models/budget.py @@ -26,6 +26,7 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase): max_parallel_requests: int | None = None tpm_limit: int | None = None rpm_limit: int | None = None + tpd_limit: int | None = None model_max_budget: dict | None = None budget_duration: str | None = None allowed_models: list[str] | None = None # per-member model scope; empty = inherit team models diff --git a/litellm/models/team.py b/litellm/models/team.py index da526515e6e..8edf10703b1 100644 --- a/litellm/models/team.py +++ b/litellm/models/team.py @@ -71,6 +71,7 @@ class TeamBase(LiteLLMPydanticObjectBase): metadata: dict | None = None tpm_limit: int | None = None rpm_limit: int | None = None + tpd_limit: int | None = None max_budget: float | None = None soft_budget: float | None = None budget_duration: str | None = None diff --git a/litellm/models/verification_token.py b/litellm/models/verification_token.py index fec3caec457..06ff877a41a 100644 --- a/litellm/models/verification_token.py +++ b/litellm/models/verification_token.py @@ -31,6 +31,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase): metadata: dict = {} tpm_limit: int | None = None rpm_limit: int | None = None + tpd_limit: int | None = None budget_duration: str | None = None budget_reset_at: datetime | None = None allowed_cache_controls: list | None = [] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b315a2beac9..f5e0565ca0f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1197,6 +1197,7 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): class KeyRequestBase(GenerateRequestBase): key: str | None = None + tpd_limit: int | None = None default_estimated_output_tokens: PositiveInt | None = None default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None budget_id: str | None = None @@ -1882,6 +1883,9 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase): ) tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.") rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.") + tpd_limit: int | None = Field( + default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id." + ) budget_duration: str | None = Field( default=None, description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')", @@ -2052,6 +2056,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): metadata: dict | None = None tpm_limit: int | None = None rpm_limit: int | None = None + tpd_limit: int | None = None max_budget: float | None = None soft_budget: float | None = None models: list | None = None @@ -3003,6 +3008,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): team_alias: str | None = None team_tpm_limit: int | None = None team_rpm_limit: int | None = None + team_tpd_limit: int | None = None team_max_budget: float | None = None team_soft_budget: float | None = None team_models: list = [] @@ -3022,6 +3028,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): end_user_id: str | None = None end_user_tpm_limit: int | None = None end_user_rpm_limit: int | None = None + end_user_tpd_limit: int | None = None end_user_max_budget: float | None = None end_user_model_max_budget: dict | None = None diff --git a/litellm/proxy/auth/team_grants.py b/litellm/proxy/auth/team_grants.py index 1196011dcdd..0421659c331 100644 --- a/litellm/proxy/auth/team_grants.py +++ b/litellm/proxy/auth/team_grants.py @@ -56,6 +56,7 @@ class TeamGrants(TypedDict, total=False): team_alias: ReadOnly[str | None] team_tpm_limit: ReadOnly[int | None] team_rpm_limit: ReadOnly[int | None] + team_tpd_limit: ReadOnly[int | None] team_max_budget: ReadOnly[float | None] team_soft_budget: ReadOnly[float | None] team_spend: ReadOnly[float | None] @@ -97,6 +98,7 @@ def team_grants( team_alias=team_object.team_alias, team_tpm_limit=team_object.tpm_limit, team_rpm_limit=team_object.rpm_limit, + team_tpd_limit=team_object.tpd_limit, team_max_budget=team_object.max_budget, team_soft_budget=team_object.soft_budget, team_spend=team_object.spend, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 687f36bbe8b..4b72d90e427 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -535,6 +535,9 @@ def _apply_budget_limits_to_end_user_params( if budget_info.rpm_limit is not None: end_user_params["end_user_rpm_limit"] = budget_info.rpm_limit + if budget_info.tpd_limit is not None: + end_user_params["end_user_tpd_limit"] = budget_info.tpd_limit + if budget_info.max_budget is not None: end_user_params["end_user_max_budget"] = budget_info.max_budget @@ -619,6 +622,8 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use valid_token.end_user_tpm_limit = end_user_params["end_user_tpm_limit"] if end_user_params.get("end_user_rpm_limit") is not None: valid_token.end_user_rpm_limit = end_user_params["end_user_rpm_limit"] + if end_user_params.get("end_user_tpd_limit") is not None: + valid_token.end_user_tpd_limit = end_user_params["end_user_tpd_limit"] if end_user_params.get("allowed_model_region") is not None: valid_token.allowed_model_region = end_user_params["allowed_model_region"] if end_user_params.get("end_user_model_max_budget") is not None: @@ -2010,6 +2015,7 @@ async def _user_api_key_auth_builder( valid_token.end_user_id = end_user_params.get("end_user_id") valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit") valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit") + valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit") valid_token.allowed_model_region = end_user_params.get("allowed_model_region") if valid_token is not None: @@ -2283,6 +2289,7 @@ async def _user_api_key_auth_builder( spend=valid_token.team_spend, tpm_limit=valid_token.team_tpm_limit, rpm_limit=valid_token.team_rpm_limit, + tpd_limit=valid_token.team_tpd_limit, blocked=valid_token.team_blocked, models=token_team_models, metadata=valid_token.team_metadata, @@ -2436,6 +2443,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached spend=valid_token.team_spend, tpm_limit=valid_token.team_tpm_limit, rpm_limit=valid_token.team_rpm_limit, + tpd_limit=valid_token.team_tpd_limit, blocked=valid_token.team_blocked, models=token_team_models, metadata=valid_token.team_metadata, diff --git a/litellm/proxy/db/create_views.py b/litellm/proxy/db/create_views.py index 10daeee4e7b..d3f3de730ab 100644 --- a/litellm/proxy/db/create_views.py +++ b/litellm/proxy/db/create_views.py @@ -80,6 +80,7 @@ async def create_missing_views(db: SupportsRawQueries) -> None: t.max_budget AS team_max_budget, t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit, + t.tpd_limit AS team_tpd_limit, p.project_alias AS project_alias FROM "LiteLLM_VerificationToken" v LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index dcd34a1d9cb..fffbf24753e 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -33,6 +33,7 @@ from litellm.batches.batch_utils import ( _extract_file_access_credentials, _iter_batch_input_lines, ) +from litellm.constants import BATCH_TPD_DESCRIPTOR_SUFFIX, BATCH_TPD_WINDOW_SECONDS from litellm.exceptions import RateLimitErrorCategory from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ( @@ -236,14 +237,48 @@ class _PROXY_BatchRateLimiter(CustomLogger): file-bound/top-level routing model this function resolves. Charging project quotas here would let a caller bind the file to a model without a quota while rows execute against a quota-limited model. + + Scopes with a ``tpd_limit`` (key, team, end user) are charged against a + daily token descriptor instead of their per-minute RPM/TPM descriptor, + because a batch's rows are scheduled by the provider and never share a + minute with the submission. The daily descriptor uses its own key so + its 24h window never collides with the online limiter's counters. """ - return self.parallel_request_limiter._create_rate_limit_descriptors( + descriptors: Final = self.parallel_request_limiter._create_rate_limit_descriptors( user_api_key_dict=user_api_key_dict, data=data, rpm_limit_type=None, tpm_limit_type=None, model_has_failures=False, ) + tpd_limits: Final[Mapping[str, tuple[str, int]]] = MappingProxyType( + { + key: (value, limit) + for key, value, limit in ( + ("api_key", user_api_key_dict.api_key, user_api_key_dict.tpd_limit), + ("team", user_api_key_dict.team_id, user_api_key_dict.team_tpd_limit), + ("end_user", user_api_key_dict.end_user_id, user_api_key_dict.end_user_tpd_limit), + ) + if value and limit is not None + } + ) + if not tpd_limits: + return descriptors + return [ + *(d for d in descriptors if d["key"] not in tpd_limits), + *( + RateLimitDescriptor( + key=f"{key}{BATCH_TPD_DESCRIPTOR_SUFFIX}", + value=value, + rate_limit={ + "requests_per_unit": None, + "tokens_per_unit": limit, + "window_size": BATCH_TPD_WINDOW_SECONDS, + }, + ) + for key, (value, limit) in tpd_limits.items() + ), + ] @staticmethod def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool: @@ -610,7 +645,9 @@ class _PROXY_BatchRateLimiter(CustomLogger): ) now: Final = datetime.now().timestamp() - window_size: Final = self.parallel_request_limiter.window_size + window_size: Final = (descriptor.get("rate_limit") or {}).get( + "window_size" + ) or self.parallel_request_limiter.window_size reset_time: Final = now + window_size reset_time_formatted: Final = datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S UTC") @@ -643,10 +680,13 @@ class _PROXY_BatchRateLimiter(CustomLogger): if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY else batch_usage.total_tokens ) + token_limit_label: Final = ( + "TPD" if descriptor.get("key", "").endswith(BATCH_TPD_DESCRIPTOR_SUFFIX) else "TPM" + ) detail = ( f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. " f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining " - f"out of {current_limit} TPM limit. " + f"out of {current_limit} {token_limit_label} limit. " f"Limit resets at: {reset_time_formatted}" ) diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py index 81a607aaa43..e16ea4a812e 100644 --- a/litellm/proxy/management_endpoints/budget_management_endpoints.py +++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py @@ -52,6 +52,7 @@ async def new_budget( - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget. - tpm_limit: Optional[int] - The tokens per minute limit for the budget. - rpm_limit: Optional[int] - The requests per minute limit for the budget. + - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit. - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}} - budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now. """ @@ -135,6 +136,7 @@ async def update_budget( - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget. - tpm_limit: Optional[int] - The tokens per minute limit for the budget. - rpm_limit: Optional[int] - The requests per minute limit for the budget. + - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit. - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}} - budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset. """ @@ -272,6 +274,7 @@ async def budget_settings( "max_parallel_requests": {"type": "Integer"}, "tpm_limit": {"type": "Integer"}, "rpm_limit": {"type": "Integer"}, + "tpd_limit": {"type": "Integer"}, "budget_duration": {"type": "String"}, "max_budget": {"type": "Float"}, "soft_budget": {"type": "Float"}, diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index d2d87331d55..b35bc01b4d0 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -335,6 +335,7 @@ async def new_end_user( - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) + - tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}} - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer. - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 95ccb7bbe0b..50324cee835 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -916,7 +916,9 @@ async def validate_team_id_used_in_service_account_request( return True -_BUDGET_NUMERIC_KEYS = frozenset(["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit"]) +_BUDGET_NUMERIC_KEYS = frozenset( + ["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit", "tpd_limit"] +) def _enforce_upperbound_key_params( @@ -1784,6 +1786,7 @@ async def generate_key_fn( - blocked: Optional[bool] - Whether the key is blocked. - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute) - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute) + - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit. - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached. - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). - prompts: Optional[List[str]] - List of prompts that the key is allowed to use. @@ -1990,6 +1993,7 @@ async def generate_service_account_key_fn( - blocked: Optional[bool] - Whether the key is blocked. - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute) - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute) + - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit. - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached. - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests) @@ -2989,6 +2993,7 @@ async def update_key_fn( - metadata: Optional[dict] - Metadata for key. Example {"team": "core-infra", "app": "app2"} - tpm_limit: Optional[int] - Tokens per minute limit - rpm_limit: Optional[int] - Requests per minute limit + - tpd_limit: Optional[int] - Tokens per day limit, charged by batch submissions instead of tpm_limit/rpm_limit - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200} - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200} - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit. @@ -4109,6 +4114,7 @@ async def generate_key_helper_fn( metadata: dict | None = {}, tpm_limit: int | None = None, rpm_limit: int | None = None, + tpd_limit: int | None = None, query_type: Literal["insert_data", "update_data"] = "insert_data", update_key_values: dict | None = None, key_alias: str | None = None, @@ -4263,6 +4269,7 @@ async def generate_key_helper_fn( "metadata": metadata_json, "tpm_limit": tpm_limit, "rpm_limit": rpm_limit, + "tpd_limit": tpd_limit, "budget_duration": key_budget_duration, "budget_reset_at": key_reset_at, "allowed_cache_controls": allowed_cache_controls, diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index cc2fefc426f..ea13e4547bd 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -58,6 +58,7 @@ class BudgetListItem(BaseModel): soft_budget: float | None = None tpm_limit: int | None = None rpm_limit: int | None = None + tpd_limit: int | None = None budget_duration: str | None = None budget_reset_at: datetime | None = None created_at: datetime @@ -123,7 +124,7 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec( resource="budgets", - sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at")), + sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")), searchable=frozenset(("budget_id",)), filters=BUDGET_FILTERS, default_sort=(SortKey(field="created_at", descending=True),), @@ -154,7 +155,7 @@ async def list_budgets( way to page, sort or filter it. `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, - `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending, + `rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending, and defaults to `-created_at`. `budget_id` is appended to every sort as the tiebreaker. `q` is a case-insensitive substring match on `budget_id`. `page_size` defaults to 50 and is capped at 100. Filters are diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0b7f69bbb7f..d7a4dcfdc0d 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1215,6 +1215,7 @@ async def new_team( - mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team. - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit + - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit - rpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of RPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating RPM, or "best_effort_throughput" for best effort enforcement. - tpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of TPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating TPM, or "best_effort_throughput" for best effort enforcement. - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget @@ -1959,6 +1960,7 @@ async def update_team( - metadata: Optional[dict] - Metadata for team, store information for team. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit + - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget - soft_budget: Optional[float] - The soft budget threshold for the team. If max_budget is set (either in the request or existing), soft_budget must be strictly lower than max_budget. Can be set independently if max_budget is not set. - budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index dd7967aafe3..f4ff9113d76 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -17,6 +17,7 @@ model LiteLLM_BudgetTable { max_parallel_requests Int? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? model_max_budget Json? budget_duration String? budget_reset_at DateTime? @@ -133,6 +134,7 @@ model LiteLLM_TeamTable { max_parallel_requests Int? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? budget_duration String? budget_reset_at DateTime? blocked Boolean @default(false) @@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable { max_parallel_requests Int? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? budget_duration String? budget_reset_at DateTime? blocked Boolean @default(false) @@ -438,6 +441,7 @@ model LiteLLM_VerificationToken { blocked Boolean? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? max_budget Float? budget_duration String? budget_reset_at DateTime? diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c095586b6c9..e560d352603 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4287,7 +4287,8 @@ class PrismaClient: t.spend AS team_spend, t.max_budget AS team_max_budget, t.tpm_limit AS team_tpm_limit, - t.rpm_limit AS team_rpm_limit + t.rpm_limit AS team_rpm_limit, + t.tpd_limit AS team_tpd_limit FROM "LiteLLM_VerificationToken" v LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id; """, @@ -4726,6 +4727,7 @@ class PrismaClient: t.soft_budget AS team_soft_budget, t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit, + t.tpd_limit AS team_tpd_limit, t.models AS team_models, t.metadata AS team_metadata, t.blocked AS team_blocked, @@ -4743,6 +4745,7 @@ class PrismaClient: b.max_budget AS litellm_budget_table_max_budget, b.tpm_limit AS litellm_budget_table_tpm_limit, b.rpm_limit AS litellm_budget_table_rpm_limit, + b.tpd_limit AS litellm_budget_table_tpd_limit, b.model_max_budget as litellm_budget_table_model_max_budget, b.soft_budget as litellm_budget_table_soft_budget, o.metadata as organization_metadata, diff --git a/schema.prisma b/schema.prisma index dd7967aafe3..f4ff9113d76 100644 --- a/schema.prisma +++ b/schema.prisma @@ -17,6 +17,7 @@ model LiteLLM_BudgetTable { max_parallel_requests Int? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? model_max_budget Json? budget_duration String? budget_reset_at DateTime? @@ -133,6 +134,7 @@ model LiteLLM_TeamTable { max_parallel_requests Int? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? budget_duration String? budget_reset_at DateTime? blocked Boolean @default(false) @@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable { max_parallel_requests Int? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? budget_duration String? budget_reset_at DateTime? blocked Boolean @default(false) @@ -438,6 +441,7 @@ model LiteLLM_VerificationToken { blocked Boolean? tpm_limit BigInt? rpm_limit BigInt? + tpd_limit BigInt? max_budget Float? budget_duration String? budget_reset_at DateTime? diff --git a/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py b/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py index cf1f665ad21..5263cf2774c 100644 --- a/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py +++ b/tests/test_litellm/proxy/auth/test_custom_auth_end_user_budget.py @@ -277,3 +277,18 @@ def test_update_valid_token_db_values_override_custom_auth_when_set(): # DB values should win assert result.end_user_tpm_limit == 500 assert result.end_user_model_max_budget == db_budget + + +def test_end_user_budget_tpd_limit_reaches_the_token(): + from litellm.proxy.auth.user_api_key_auth import _apply_budget_limits_to_end_user_params + + end_user_params = {"end_user_id": "user_1"} + _apply_budget_limits_to_end_user_params( + end_user_params=end_user_params, + budget_info=LiteLLM_BudgetTable(rpm_limit=5, tpd_limit=750000), + end_user_id="user_1", + ) + result = update_valid_token_with_end_user_params(UserAPIKeyAuth(token="test_token"), end_user_params) + + assert result.end_user_rpm_limit == 5 + assert result.end_user_tpd_limit == 750000 diff --git a/tests/test_litellm/proxy/auth/test_team_grants.py b/tests/test_litellm/proxy/auth/test_team_grants.py index 447fc1c93a1..7b6717f804f 100644 --- a/tests/test_litellm/proxy/auth/test_team_grants.py +++ b/tests/test_litellm/proxy/auth/test_team_grants.py @@ -27,6 +27,7 @@ def _full_team(model_aliases=ALIASES) -> LiteLLM_TeamTable: team_alias="grants-team", tpm_limit=1000, rpm_limit=10, + tpd_limit=200000, max_budget=50.0, soft_budget=25.0, spend=12.5, @@ -67,6 +68,7 @@ def test_team_grants_cover_every_team_field_the_key_path_gets(): assert token.team_alias == "grants-team" assert token.team_tpm_limit == 1000 assert token.team_rpm_limit == 10 + assert token.team_tpd_limit == 200000 assert token.team_max_budget == 50.0 assert token.team_soft_budget == 25.0 assert token.team_spend == 12.5 diff --git a/tests/test_litellm/proxy/hooks/test_batch_rate_limiter.py b/tests/test_litellm/proxy/hooks/test_batch_rate_limiter.py new file mode 100644 index 00000000000..3a8b2de44bf --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_batch_rate_limiter.py @@ -0,0 +1,171 @@ +""" +Tests for `tpd_limit` (tokens per day) enforcement on batch submissions. + +A batch's rows are scheduled by the provider, so a caller cannot keep a large +batch under a per-minute RPM/TPM budget. Scopes that configure `tpd_limit` +are charged against a 24h token window instead of their minute counters. +""" + +import pytest +from fastapi import HTTPException + +from litellm import DualCache +from litellm.constants import BATCH_TPD_WINDOW_SECONDS +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.hooks.batch_rate_limiter import BatchFileUsage +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + _PROXY_MaxParallelRequestsHandler_v3, +) +from litellm.proxy.utils import InternalUsageCache, hash_token + + +def _make_limiters(): + internal_usage_cache = InternalUsageCache(dual_cache=DualCache()) + rate_limiter = _PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=internal_usage_cache) + batch_limiter = rate_limiter._get_batch_rate_limiter() + assert batch_limiter is not None + return internal_usage_cache, rate_limiter, batch_limiter + + +async def _counter(internal_usage_cache, rate_limiter, descriptor_key, value, rate_limit_type): + cache_key = rate_limiter.create_rate_limit_keys(descriptor_key, value, rate_limit_type) + raw = await internal_usage_cache.async_get_cache(key=cache_key, litellm_parent_otel_span=None, local_only=True) + return int(raw or 0) + + +@pytest.mark.asyncio +async def test_batch_over_rpm_and_tpm_but_under_tpd_is_accepted(): + internal_usage_cache, rate_limiter, batch_limiter = _make_limiters() + api_key = hash_token("tpd-key") + user_api_key_dict = UserAPIKeyAuth(api_key=api_key, rpm_limit=1, tpm_limit=10, tpd_limit=1000) + + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=user_api_key_dict, + data={}, + batch_usage=BatchFileUsage(total_tokens=500, request_count=50), + ) + + assert await _counter(internal_usage_cache, rate_limiter, "api_key_tpd", api_key, "tokens") == 500 + assert await _counter(internal_usage_cache, rate_limiter, "api_key", api_key, "requests") == 0 + assert await _counter(internal_usage_cache, rate_limiter, "api_key", api_key, "tokens") == 0 + + +@pytest.mark.asyncio +async def test_cumulative_batch_tokens_over_tpd_returns_429_with_daily_reset(): + _internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters() + user_api_key_dict = UserAPIKeyAuth(api_key=hash_token("tpd-key-2"), rpm_limit=1, tpd_limit=1000) + + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=user_api_key_dict, + data={}, + batch_usage=BatchFileUsage(total_tokens=600, request_count=6), + ) + with pytest.raises(HTTPException) as exc: + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=user_api_key_dict, + data={}, + batch_usage=BatchFileUsage(total_tokens=600, request_count=6), + ) + + assert exc.value.status_code == 429 + assert "api_key_tpd" in str(exc.value.detail) + assert "600 tokens but only 400 tokens remaining out of 1000 TPD limit" in str(exc.value.detail) + assert exc.value.headers["retry-after"] == str(BATCH_TPD_WINDOW_SECONDS) + + +@pytest.mark.asyncio +async def test_batch_without_tpd_still_enforces_minute_rpm(): + _internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters() + user_api_key_dict = UserAPIKeyAuth(api_key=hash_token("rpm-only-key"), rpm_limit=1, tpm_limit=1000) + + with pytest.raises(HTTPException) as exc: + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=user_api_key_dict, + data={}, + batch_usage=BatchFileUsage(total_tokens=50, request_count=5), + ) + + assert exc.value.status_code == 429 + assert "RPM limit" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_team_tpd_replaces_team_minute_limits_but_key_minute_limits_still_apply(): + internal_usage_cache, rate_limiter, batch_limiter = _make_limiters() + team_key = UserAPIKeyAuth( + api_key=hash_token("team-key"), + team_id="team-1", + team_rpm_limit=1, + team_tpm_limit=10, + team_tpd_limit=5000, + ) + + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=team_key, + data={}, + batch_usage=BatchFileUsage(total_tokens=800, request_count=8), + ) + assert await _counter(internal_usage_cache, rate_limiter, "team_tpd", "team-1", "tokens") == 800 + assert await _counter(internal_usage_cache, rate_limiter, "team", "team-1", "requests") == 0 + + key_rpm_in_team_with_tpd = UserAPIKeyAuth( + api_key=hash_token("team-key-2"), + rpm_limit=1, + team_id="team-1", + team_tpd_limit=5000, + ) + with pytest.raises(HTTPException) as exc: + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=key_rpm_in_team_with_tpd, + data={}, + batch_usage=BatchFileUsage(total_tokens=10, request_count=2), + ) + assert exc.value.status_code == 429 + assert "api_key:" in str(exc.value.detail) + assert "RPM limit" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_end_user_tpd_is_enforced_per_end_user(): + _internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters() + first_customer = UserAPIKeyAuth( + api_key=hash_token("shared-key"), end_user_id="customer-a", end_user_rpm_limit=1, end_user_tpd_limit=100 + ) + second_customer = UserAPIKeyAuth( + api_key=hash_token("shared-key"), end_user_id="customer-b", end_user_rpm_limit=1, end_user_tpd_limit=100 + ) + + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=first_customer, data={}, batch_usage=BatchFileUsage(total_tokens=90, request_count=9) + ) + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=second_customer, data={}, batch_usage=BatchFileUsage(total_tokens=90, request_count=9) + ) + with pytest.raises(HTTPException) as exc: + await batch_limiter._check_and_increment_batch_counters( + user_api_key_dict=first_customer, data={}, batch_usage=BatchFileUsage(total_tokens=20, request_count=2) + ) + assert exc.value.status_code == 429 + assert "end_user_tpd: customer-a" in str(exc.value.detail) + + +def test_tpd_only_key_is_not_skipped_as_having_no_limits(): + _internal_usage_cache, _rate_limiter, batch_limiter = _make_limiters() + descriptors = batch_limiter._create_batch_rate_limit_descriptors( + user_api_key_dict=UserAPIKeyAuth(api_key=hash_token("tpd-only"), tpd_limit=100), + data={}, + ) + assert batch_limiter._has_applicable_batch_rate_limits(descriptors) is True + + +def test_online_descriptors_ignore_tpd_limit(): + _internal_usage_cache, rate_limiter, _batch_limiter = _make_limiters() + api_key = hash_token("online-key") + descriptors = rate_limiter._create_rate_limit_descriptors( + user_api_key_dict=UserAPIKeyAuth(api_key=api_key, rpm_limit=5, tpd_limit=100, team_id="t", team_tpd_limit=9), + data={"model": "gpt-4o"}, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + assert [(d["key"], d["rate_limit"]["window_size"]) for d in descriptors] == [("api_key", rate_limiter.window_size)] diff --git a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py index add2126ac7b..2b438a9d370 100644 --- a/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py +++ b/tests/test_litellm/proxy/management_endpoints/management_v1/test_budgets.py @@ -52,7 +52,7 @@ app.include_router(router) client = TestClient(app) BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets" -SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpm_limit"] +SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"] def _row(budget_id: str, **overrides: Any) -> dict[str, Any]: @@ -62,6 +62,7 @@ def _row(budget_id: str, **overrides: Any) -> dict[str, Any]: "soft_budget": None, "tpm_limit": None, "rpm_limit": None, + "tpd_limit": None, "budget_duration": "30d", "budget_reset_at": None, "created_at": "2026-07-20T12:00:00+00:00", @@ -123,7 +124,7 @@ def test_returns_flat_rows_in_the_control_plane_envelope(query_raw, as_proxy_adm def test_serves_the_columns_the_budgets_page_renders(query_raw, as_proxy_admin): - _serve(query_raw, [_row("b-1", soft_budget=5.0, budget_reset_at="2026-08-01T00:00:00+00:00")]) + _serve(query_raw, [_row("b-1", soft_budget=5.0, tpd_limit=250000, budget_reset_at="2026-08-01T00:00:00+00:00")]) row = _get().json()["data"][0] @@ -133,12 +134,14 @@ def test_serves_the_columns_the_budgets_page_renders(query_raw, as_proxy_admin): "soft_budget", "tpm_limit", "rpm_limit", + "tpd_limit", "budget_duration", "budget_reset_at", "created_at", "updated_at", } assert row["soft_budget"] == 5.0 + assert row["tpd_limit"] == 250000 assert row["budget_reset_at"].startswith("2026-08-01T00:00:00") diff --git a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py index 4b6815d7552..2f3be61d00f 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_budget_endpoints.py @@ -136,6 +136,21 @@ async def test_update_budget_success(client_and_mocks, monkeypatch): assert body["updated_by"] == "test_user" +@pytest.mark.asyncio +async def test_new_and_update_budget_persist_tpd_limit(client_and_mocks): + client, _, mock_table = client_and_mocks + + resp = client.post("/budget/new", json={"budget_id": "budget_tpd", "tpd_limit": 250000}) + assert resp.status_code == 200, resp.text + assert resp.json()["tpd_limit"] == 250000 + assert mock_table.create.await_args.kwargs["data"]["tpd_limit"] == 250000 + + resp = client.post("/budget/update", json={"budget_id": "budget_tpd", "tpd_limit": 500000}) + assert resp.status_code == 200, resp.text + assert resp.json()["tpd_limit"] == 500000 + assert mock_table.update.await_args.kwargs["data"]["tpd_limit"] == 500000 + + @pytest.mark.asyncio async def test_update_budget_missing_id(client_and_mocks, monkeypatch): client, mock_prisma, mock_table = client_and_mocks diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 2ac52da57df..f919e5919bf 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -461,6 +461,28 @@ async def test_key_expiration_exact_duration_hours(monkeypatch): ), f"Expected expiration to be approximately 12 hours from creation, got {hours_diff} hours" +@pytest.mark.asyncio +async def test_generate_key_persists_tpd_limit(monkeypatch): + mock_prisma_client = AsyncMock() + mock_prisma_client.insert_data = AsyncMock( + return_value=MagicMock(token="hashed_token_123", litellm_budget_table=None) + ) + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_verificationtoken.count = AsyncMock(return_value=0) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + data_json = GenerateKeyRequest(tpd_limit=250000, rpm_limit=5).model_dump(exclude_none=True) + response = await generate_key_helper_fn(request_type="key", **data_json, table_name="key") + + assert response["tpd_limit"] == 250000 + key_insert = mock_prisma_client.insert_data.await_args_list[-1].kwargs + assert key_insert["table_name"] == "key" + assert key_insert["data"]["tpd_limit"] == 250000 + assert key_insert["data"]["rpm_limit"] == 5 + + @pytest.mark.asyncio async def test_key_generation_with_object_permission(monkeypatch): """Ensure /key/generate correctly handles `object_permission` input by @@ -1813,6 +1835,18 @@ async def test_update_key_enable_prompt_caching_folds_into_metadata(flag_value): assert "enable_prompt_caching" not in {k for k in updated if k != "metadata"} +@pytest.mark.asyncio +@pytest.mark.parametrize("tpd_limit", [250000, None]) +async def test_update_key_writes_tpd_limit_as_a_column(tpd_limit): + data = UpdateKeyRequest(key="sk-1", tpd_limit=tpd_limit) + existing_key = LiteLLM_VerificationToken(token="hashed", tpd_limit=1) + + updated = await prepare_key_update_data(data=data, existing_key_row=existing_key) + + assert updated["tpd_limit"] == tpd_limit + assert "rpm_limit" not in updated + + @pytest.mark.asyncio async def test_update_preserves_service_account_id_when_metadata_replaced(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 0e1831614ac..c4100c45aa2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -626,6 +626,42 @@ async def test_new_team_with_object_permission(mock_db_client, mock_admin_auth): assert "object_permission" not in team_data +@pytest.mark.asyncio +async def test_new_team_persists_tpd_limit(mock_db_client, mock_admin_auth): + mock_db_client.jsonify_team_object = lambda db_data: db_data + mock_db_client.get_data = AsyncMock(return_value=None) + mock_db_client.update_data = AsyncMock(return_value=MagicMock()) + mock_db_client.db = MagicMock() + mock_db_client.db.litellm_modeltable = MagicMock() + mock_db_client.db.litellm_modeltable.create = AsyncMock(return_value=MagicMock(id="model123")) + + team_create_result = MagicMock(team_id="team-tpd") + team_create_result.model_dump.return_value = {"team_id": "team-tpd", "tpd_limit": 250000} + mock_team_create = AsyncMock(return_value=team_create_result) + mock_db_client.db.litellm_teamtable = MagicMock() + mock_db_client.db.litellm_teamtable.create = mock_team_create + _wire_team_create_tx(mock_db_client) + mock_db_client.db.litellm_teamtable.count = AsyncMock(return_value=0) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=team_create_result) + mock_db_client.db.litellm_usertable = MagicMock() + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + from fastapi import Request + + from litellm.proxy._types import NewTeamRequest + from litellm.proxy.management_endpoints.team_endpoints import new_team + + await new_team( + data=NewTeamRequest(team_alias="tpd-team", rpm_limit=5, tpd_limit=250000), + http_request=MagicMock(spec=Request), + user_api_key_dict=mock_admin_auth, + ) + + team_data = mock_team_create.call_args.kwargs["data"] + assert team_data["tpd_limit"] == 250000 + assert team_data["rpm_limit"] == 5 + + @pytest.mark.asyncio async def test_new_team_with_mcp_tool_permissions(mock_db_client, mock_admin_auth): """ @@ -7338,6 +7374,48 @@ async def test_update_team_rpm_limit_not_gated_by_user_limit( assert result is not None +@pytest.mark.asyncio +async def test_update_team_persists_tpd_limit(disable_audit_logging_for_mocked_team): + from fastapi import Request + + from litellm.proxy._types import UpdateTeamRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import update_team + + with ( + patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point + "litellm.proxy.proxy_server.prisma_client" + ) as mock_prisma, + patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point + "litellm.proxy.proxy_server.user_api_key_cache" + ) as mock_cache, + patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point + "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin" + ), + patch( # test-quality-ok: stubs the audit write so the test observes only the team column written + "litellm.proxy.proxy_server.create_audit_log_for_update", new=AsyncMock() + ), + ): + existing_team = MagicMock(team_id="team-tpd", organization_id=None, model_id=None, tpd_limit=None) + existing_team.model_dump.return_value = {"team_id": "team-tpd", "organization_id": None} + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing_team) + mock_prisma.jsonify_team_object = lambda db_data: db_data + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_set_cache = AsyncMock() + updated_team = MagicMock(team_id="team-tpd", organization_id=None, litellm_model_table=None) + updated_team.model_dump.return_value = {"team_id": "team-tpd", "organization_id": None, "tpd_limit": 250000} + mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=updated_team) + + await update_team( + data=UpdateTeamRequest(team_id="team-tpd", tpd_limit=250000), + http_request=MagicMock(spec=Request), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), + ) + + written = mock_prisma.db.litellm_teamtable.update.call_args.kwargs["data"] + assert written["tpd_limit"] == 250000 + assert "rpm_limit" not in written + + @pytest.mark.asyncio async def test_new_team_org_scoped_tpm_exceeds_org_limit(): """ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx index c425e766f2d..15fef13f23b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx @@ -129,7 +129,7 @@ describe("BudgetTable", () => { const user = userEvent.setup(); renderWithProviders(); await showColumn(user, "created_at"); - for (const field of ["budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at"]) { + for (const field of ["budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at"]) { expect(screen.getByTestId(`sort-header-${field}`)).toBeInTheDocument(); } }); @@ -152,9 +152,11 @@ describe("BudgetTable", () => { }); it("should show n/a for missing rate limits and Unlimited for a missing max budget", () => { - const list = makeList({ rows: [makeBudget({ max_budget: null, tpm_limit: null, rpm_limit: null })] }); + const list = makeList({ + rows: [makeBudget({ max_budget: null, tpm_limit: null, rpm_limit: null, tpd_limit: null })], + }); renderWithProviders(); - expect(screen.getAllByText("n/a")).toHaveLength(2); + expect(screen.getAllByText("n/a")).toHaveLength(3); expect(screen.getByText("Unlimited")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index e5cd9043492..fb894322208 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -126,6 +126,14 @@ export const getBudgetTableColumns = ({ size: 100, cell: ({ row }) => , }, + { + id: "tpd_limit", + accessorKey: "tpd_limit", + meta: { title: "TPD (batch)", numeric: true }, + header: ({ column }) => , + size: 110, + cell: ({ row }) => , + }, { id: "budget_duration", accessorKey: "budget_duration", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx index 492a6b5c630..5068cbed453 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx @@ -17,6 +17,7 @@ const budgetShape = { budget_id: z.string().min(1, "Please input a human-friendly name for the budget"), tpm_limit: z.number().nullish(), rpm_limit: z.number().nullish(), + tpd_limit: z.number().nullish(), max_budget: z.number().nullish(), budget_duration: z.string().nullish(), }; @@ -112,6 +113,23 @@ const BudgetModal: React.FC = ({ isModalVisible, setIsModalVis /> )} + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx index 25344c52847..7455c252e26 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx @@ -133,6 +133,7 @@ const BudgetPanel: React.FC = ({ accessToken }) => { { label: "Max Budget", value: selectedBudget?.max_budget }, { label: "TPM", value: selectedBudget?.tpm_limit }, { label: "RPM", value: selectedBudget?.rpm_limit }, + { label: "TPD (batch)", value: selectedBudget?.tpd_limit }, ]} onCancel={handleDeleteCancel} onOk={handleDeleteConfirm} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx index 71fce2de836..1931a88f096 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx @@ -15,13 +15,14 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u type EditBudgetFormValues = Pick< budgetItem, - "budget_id" | "tpm_limit" | "rpm_limit" | "max_budget" | "budget_duration" + "budget_id" | "tpm_limit" | "rpm_limit" | "tpd_limit" | "max_budget" | "budget_duration" >; const toFormValues = (budget: budgetItem): EditBudgetFormValues => ({ budget_id: budget.budget_id, tpm_limit: budget.tpm_limit, rpm_limit: budget.rpm_limit, + tpd_limit: budget.tpd_limit, max_budget: budget.max_budget, budget_duration: budget.budget_duration, }); @@ -118,6 +119,23 @@ const EditBudgetModal: React.FC = ({ isModalVisible, setIs /> )} + + {({ ref, value, onChange, ...field }) => ( + onChange(event.target.value === "" ? null : event.target.valueAsNumber)} + /> + )} + diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index 70c30596c75..851c9e6d487 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -1187,6 +1187,7 @@ describe("Teams - which fields reach the create payload depends on the open sect "organization_id", "rpm_limit", "team_alias", + "tpd_limit", "tpm_limit", ]); expect(payload.team_alias).toBe("Closed Sections Team"); @@ -1314,6 +1315,7 @@ describe("Teams - the exact bytes the create call sends", () => { budget_duration: undefined, tpm_limit: undefined, rpm_limit: undefined, + tpd_limit: undefined, metadata: undefined, }); expect(wireBody(payload)).toStrictEqual({ @@ -1341,6 +1343,7 @@ describe("Teams - the exact bytes the create call sends", () => { budget_duration: undefined, tpm_limit: undefined, rpm_limit: undefined, + tpd_limit: undefined, metadata: undefined, team_id: undefined, team_member_budget: undefined, @@ -1513,6 +1516,7 @@ describe("Teams - the exact bytes the create call sends", () => { budget_duration: undefined, tpm_limit: undefined, rpm_limit: undefined, + tpd_limit: undefined, metadata: undefined, team_id: undefined, team_member_budget: undefined, diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index dc531ea5dad..4f3367d8b98 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -77,6 +77,7 @@ const teamCreateFieldsSchema = z.object({ budget_duration: z.string().nullish(), tpm_limit: numericInputSchema, rpm_limit: numericInputSchema, + tpd_limit: numericInputSchema, metadata: metadataPairsSchema.optional(), team_id: z.string().optional(), team_member_budget: z.number().optional(), @@ -113,6 +114,7 @@ const EMPTY_TEAM_CREATE_VALUES: TeamCreateFormValues = { budget_duration: undefined, tpm_limit: undefined, rpm_limit: undefined, + tpd_limit: undefined, metadata: [], team_id: undefined, team_member_budget: undefined, @@ -821,6 +823,18 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser )} + + {({ ref, value, ...field }) => ( + + )} + Metadata | null; budget_reset_at?: string | null; @@ -47,6 +48,7 @@ export interface KeyResponse { metadata: Record; tpm_limit: number; rpm_limit: number; + tpd_limit?: number | null; duration: string; budget_duration: string; budget_reset_at: string; diff --git a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts index fef94cc3c2b..7c6d5def8da 100644 --- a/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts +++ b/ui/litellm-dashboard/src/components/organisms/createKeyPayload.test.ts @@ -45,6 +45,7 @@ const DROPPED_AT_SERIALISATION = [ "rpm_limit", "tags", "throttle_on_budget_exceeded", + "tpd_limit", "tpm_limit", ]; @@ -64,6 +65,7 @@ const OPTIONAL_SETTINGS_VALUES = { tpm_limit_type: "key", rpm_limit: undefined, rpm_limit_type: "key", + tpd_limit: undefined, throttle_on_budget_exceeded: undefined, enable_prompt_caching: undefined, guardrails: undefined, @@ -456,6 +458,18 @@ describe("budget duration", () => { }); }); +describe("tpd_limit", () => { + it("forwards the daily batch token budget alongside the minute limits", () => { + expect(payloadOf(build({ key_alias: "my-key", tpd_limit: 250000, rpm_limit: 5 }))).toStrictEqual( + aliasOnly({ tpd_limit: 250000, rpm_limit: 5 }), + ); + }); + + it("keeps a zero tpd_limit rather than treating it as unset", () => { + expect(payloadOf(build({ key_alias: "my-key", tpd_limit: 0 }))).toStrictEqual(aliasOnly({ tpd_limit: 0 })); + }); +}); + describe("purity", () => { it("leaves the submitted form values untouched", () => { const values = { @@ -499,9 +513,9 @@ describe("serialised wire shape", () => { expect(payloadOf(build({ ...CLOSED_SECTIONS_VALUES, team_id: "team-1" })).team_id).toBe("team-1"); }); - it("adds fifteen keys to the object and only the two limit types to the wire when Optional Settings opens", () => { + it("adds sixteen keys to the object and only the two limit types to the wire when Optional Settings opens", () => { const payload = payloadOf(build(OPTIONAL_SETTINGS_VALUES)); - expect(Object.keys(payload)).toHaveLength(23); + expect(Object.keys(payload)).toHaveLength(24); expect(wireKeys(payload)).toStrictEqual([ "team_id", "key_alias", diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 0d5d9f5ec8d..3e3c29e330d 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -143,6 +143,7 @@ const OPTIONAL_OPEN_PAYLOAD = { tpm_limit_type: null, rpm_limit: undefined, rpm_limit_type: null, + tpd_limit: undefined, throttle_on_budget_exceeded: undefined, enable_prompt_caching: undefined, guardrails: undefined, @@ -395,6 +396,7 @@ describe("CreateKey", () => { it.each([ ["Tokens per minute Limit (TPM)", "tpm_limit"], ["Requests per minute Limit (RPM)", "rpm_limit"], + ["Tokens per day Limit (TPD)", "tpd_limit"], ])("routes a typed %s into the %s payload key", async (label, key) => { await openModal(); await nameTheKey(); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index b5789101f77..b8ea8de7f59 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1150,6 +1150,32 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp /> )} + + Tokens per day Limit (TPD){" "} + + + + + } + name="tpd_limit" + help={`TPD cannot exceed team TPD limit: ${team?.tpd_limit !== null && team?.tpd_limit !== undefined ? team?.tpd_limit : "unlimited"}`} + rules={ceilingRule( + team?.tpd_limit, + (limit) => `TPD limit cannot exceed team TPD limit: ${limit}`, + )} + > + {(control) => ( + + )} + @@ -1760,6 +1786,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp "budget_duration", "tpm_limit", "rpm_limit", + "tpd_limit", ...(disableCustomApiKeys ? ["key"] : []), ]} /> diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index a25ca28651a..eb912ffa3cc 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -1968,6 +1968,7 @@ describe("TeamInfoView - the exact bytes the update call sends", () => { models: ["gpt-4"], tpm_limit: 1000, rpm_limit: 1000, + tpd_limit: null, model_tpm_limit: {}, model_rpm_limit: {}, max_budget: 100, diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index ffc83d0165e..ce705008678 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -264,6 +264,7 @@ export interface TeamData { metadata: Record; tpm_limit: number | null; rpm_limit: number | null; + tpd_limit?: number | null; max_budget: number | null; soft_budget?: number | null; budget_duration: string | null; @@ -330,6 +331,7 @@ const teamUpdateFieldsSchema = z.object({ budget_duration: z.string().nullish(), tpm_limit: numericInputSchema, rpm_limit: numericInputSchema, + tpd_limit: numericInputSchema, modelLimits: z .array( z.object({ @@ -411,6 +413,7 @@ const EMPTY_TEAM_UPDATE_VALUES: TeamUpdateFormValues = { budget_duration: undefined, tpm_limit: undefined, rpm_limit: undefined, + tpd_limit: undefined, modelLimits: [], default_estimated_output_tokens: undefined, default_estimated_output_tokens_per_model: "", @@ -460,6 +463,7 @@ const toTeamFormValues = (info: TeamInfoRecord, effectiveGuardrails: string[]): budget_duration: info.budget_duration, tpm_limit: info.tpm_limit, rpm_limit: info.rpm_limit, + tpd_limit: info.tpd_limit, modelLimits: Array.from( new Set([ ...Object.keys(info.metadata?.model_tpm_limit ?? {}), @@ -918,6 +922,7 @@ const TeamInfoView: React.FC = ({ models: normalizeTeamModelSelection(values.models), tpm_limit: sanitizeNumeric(values.tpm_limit), rpm_limit: sanitizeNumeric(values.rpm_limit), + tpd_limit: sanitizeNumeric(values.tpd_limit), model_tpm_limit: modelTpmLimit, model_rpm_limit: modelRpmLimit, max_budget: values.max_budget, @@ -1168,6 +1173,7 @@ const TeamInfoView: React.FC = ({

TPM: {info.tpm_limit ?? "Unlimited"}

RPM: {info.rpm_limit ?? "Unlimited"}

+

TPD (batch): {info.tpd_limit ?? "Unlimited"}

{info.max_parallel_requests &&

Max Parallel Requests: {info.max_parallel_requests}

} {(() => { const modelTpm = (info.metadata?.model_tpm_limit ?? {}) as Record; @@ -1538,6 +1544,17 @@ const TeamInfoView: React.FC = ({ {({ ref, value, ...field }) => } + + {({ ref, value, ...field }) => } + + Metadata = ({

Rate Limits

TPM: {info.tpm_limit ?? "Unlimited"}
RPM: {info.rpm_limit ?? "Unlimited"}
+
TPD (batch): {info.tpd_limit ?? "Unlimited"}
{(() => { const modelTpm = (info.metadata?.model_tpm_limit ?? {}) as Record; const modelRpm = (info.metadata?.model_rpm_limit ?? {}) as Record; diff --git a/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx b/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx index f5d5562ccfe..312ba6a5398 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx @@ -58,6 +58,9 @@ export const KeyTypeSelect = ({ const SKILLS_HINT = "Enabled skills are visible to every key. Grant disabled (private) Claude Code plugins to this key here."; +export const TPD_HINT = + "Daily token budget for batch submissions (/v1/batches). When set, batch input files are charged against this 24h window instead of the key's TPM/RPM limits. Online requests keep using TPM/RPM."; + export const KeyAgentAndSkillFields = ({ control, accessToken, diff --git a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.test.ts b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.test.ts index 948ed659cd5..f12088bea19 100644 --- a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.test.ts +++ b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.test.ts @@ -1,8 +1,30 @@ import { describe, expect, it } from "vitest"; -import { keyEditFormSchema } from "./keyEditFormValues"; +import type { KeyResponse } from "../key_team_helpers/key_list"; +import { keyEditFormSchema, toKeyEditFormValues, toSubmittedValues } from "./keyEditFormValues"; const parse = (values: Record) => keyEditFormSchema.safeParse(values); +describe("tpd_limit round trip", () => { + const keyData = { token: "tok", models: [], rpm_limit: 5, tpd_limit: 250000 } as unknown as KeyResponse; + + it("hydrates the stored daily batch budget into the edit form", () => { + expect(toKeyEditFormValues(keyData)).toMatchObject({ rpm_limit: 5, tpd_limit: 250000 }); + }); + + it("submits tpd_limit next to the minute limits", () => { + const submitted = toSubmittedValues(toKeyEditFormValues(keyData), { canViewPolicies: true, canViewPrompts: true }); + expect(submitted).toMatchObject({ rpm_limit: 5, tpd_limit: 250000 }); + }); + + it("submits null when the operator cleared tpd_limit", () => { + const submitted = toSubmittedValues( + { ...toKeyEditFormValues(keyData), tpd_limit: null }, + { canViewPolicies: true, canViewPrompts: true }, + ); + expect(submitted.tpd_limit).toBeNull(); + }); +}); + describe("keyEditFormSchema", () => { it("accepts an empty form", () => { expect(parse({}).success).toBe(true); diff --git a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts index 233b58b48ab..7436380d6ee 100644 --- a/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts +++ b/ui/litellm-dashboard/src/components/templates/keyEditFormValues.ts @@ -28,6 +28,7 @@ export interface KeyEditFormValues { tpm_limit_type?: string | null; rpm_limit?: number | string | null; rpm_limit_type?: string | null; + tpd_limit?: number | string | null; throttle_on_budget_exceeded?: boolean; enable_prompt_caching?: boolean; max_parallel_requests?: number | string | null; @@ -77,6 +78,7 @@ export const toKeyEditFormValues = (keyData: KeyResponse): KeyEditFormValues => tpm_limit_type: (keyData as { tpm_limit_type?: string | null }).tpm_limit_type ?? null, rpm_limit: keyData.rpm_limit, rpm_limit_type: (keyData as { rpm_limit_type?: string | null }).rpm_limit_type ?? null, + tpd_limit: keyData.tpd_limit, throttle_on_budget_exceeded: Boolean(readMetadata(keyData, "throttle_on_budget_exceeded")), enable_prompt_caching: Boolean(readMetadata(keyData, "enable_prompt_caching")), max_parallel_requests: keyData.max_parallel_requests, @@ -130,6 +132,7 @@ export const keyEditFormSchema = z.object({ tpm_limit_type: z.custom(), rpm_limit: z.custom(), rpm_limit_type: z.custom(), + tpd_limit: z.custom(), throttle_on_budget_exceeded: z.custom(), enable_prompt_caching: z.custom(), max_parallel_requests: z.custom(), @@ -184,6 +187,7 @@ export const toSubmittedValues = ( tpm_limit_type: values.tpm_limit_type, rpm_limit: values.rpm_limit, rpm_limit_type: values.rpm_limit_type, + tpd_limit: values.tpd_limit, throttle_on_budget_exceeded: values.throttle_on_budget_exceeded, enable_prompt_caching: values.enable_prompt_caching, max_parallel_requests: values.max_parallel_requests, diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx index cbe17b67865..1a1736dc78c 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.integration.test.tsx @@ -188,6 +188,7 @@ describe("KeyEditView", () => { }, tpm_limit: 10, rpm_limit: 10, + tpd_limit: 250000, duration: "30d", budget_duration: "30d", budget_reset_at: "never", @@ -1986,6 +1987,7 @@ describe("KeyEditView", () => { tpm_limit_type: null, rpm_limit: 10, rpm_limit_type: null, + tpd_limit: 250000, throttle_on_budget_exceeded: false, enable_prompt_caching: false, max_parallel_requests: 10, diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index d327db21a9e..ad29dafd13e 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -31,7 +31,13 @@ import { modelSentinelOptions, parseAllowedRoutes, } from "./keyEditFieldNormalizers"; -import { KeyAgentAndSkillFields, KeyBudgetNumberField, KeyTypeSelect, labelWithHint } from "./KeyEditViewControls"; +import { + KeyAgentAndSkillFields, + KeyBudgetNumberField, + KeyTypeSelect, + labelWithHint, + TPD_HINT, +} from "./KeyEditViewControls"; import { KeyEditFormValues, keyEditFormSchema, @@ -508,6 +514,10 @@ export function KeyEditView({ )} + + {({ ref: _ref, ...field }) => } + + RPM: {currentKeyData.rpm_limit !== null ? currentKeyData.rpm_limit : "Unlimited"}

+

TPD (batch): {currentKeyData.tpd_limit ?? "Unlimited"}

{Boolean(currentKeyData.metadata?.throttle_on_budget_exceeded) && (

Throttle on budget exceeded: Yes

)} @@ -1064,6 +1066,7 @@ export default function KeyInfoView({

RPM: {currentKeyData.rpm_limit !== null ? currentKeyData.rpm_limit : "Unlimited"}

+

TPD (batch): {currentKeyData.tpd_limit ?? "Unlimited"}

Max Parallel Requests:{" "} {currentKeyData.max_parallel_requests !== null diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..5c0d8bc6363 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1843,6 +1843,7 @@ export interface paths { * - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget. * - tpm_limit: Optional[int] - The tokens per minute limit for the budget. * - rpm_limit: Optional[int] - The requests per minute limit for the budget. + * - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit. * - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}} * - budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now. */ @@ -1899,6 +1900,7 @@ export interface paths { * - max_parallel_requests: Optional[int] - The max number of parallel requests for the budget. * - tpm_limit: Optional[int] - The tokens per minute limit for the budget. * - rpm_limit: Optional[int] - The requests per minute limit for the budget. + * - tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit. * - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}} * - budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset. */ @@ -3951,6 +3953,7 @@ export interface paths { * - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). * - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) * - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) + * - tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit * - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}} * - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer. * - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. @@ -4485,6 +4488,7 @@ export interface paths { * - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). * - tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute) * - rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute) + * - tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit * - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}} * - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer. * - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. @@ -7707,6 +7711,7 @@ export interface paths { * - blocked: Optional[bool] - Whether the key is blocked. * - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute) * - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute) + * - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit. * - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached. * - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). * - prompts: Optional[List[str]] - List of prompts that the key is allowed to use. @@ -8020,6 +8025,7 @@ export interface paths { * - blocked: Optional[bool] - Whether the key is blocked. * - rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute) * - tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute) + * - tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit. * - soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached. * - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing). * - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests) @@ -8146,6 +8152,7 @@ export interface paths { * - metadata: Optional[dict] - Metadata for key. Example {"team": "core-infra", "app": "app2"} * - tpm_limit: Optional[int] - Tokens per minute limit * - rpm_limit: Optional[int] - Requests per minute limit + * - tpd_limit: Optional[int] - Tokens per day limit, charged by batch submissions instead of tpm_limit/rpm_limit * - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200} * - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200} * - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit. @@ -8395,7 +8402,7 @@ export interface paths { * way to page, sort or filter it. * * `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, - * `rpm_limit` or `created_at`, each optionally prefixed with `-` for descending, + * `rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending, * and defaults to `-created_at`. `budget_id` is appended to every sort as the * tiebreaker. `q` is a case-insensitive substring match on `budget_id`. * `page_size` defaults to 50 and is capped at 100. Filters are @@ -15463,6 +15470,7 @@ export interface paths { * - mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team. * - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit * - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit + * - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit * - rpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of RPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating RPM, or "best_effort_throughput" for best effort enforcement. * - tpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of TPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating TPM, or "best_effort_throughput" for best effort enforcement. * - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget @@ -15691,6 +15699,7 @@ export interface paths { * - metadata: Optional[dict] - Metadata for team, store information for team. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit * - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit + * - tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit * - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget * - soft_budget: Optional[float] - The soft budget threshold for the team. If max_budget is set (either in the request or existing), soft_budget must be strictly lower than max_budget. Can be set independently if max_budget is not set. * - budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets) @@ -16781,7 +16790,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16895,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -24442,6 +24449,8 @@ export interface components { rpm_limit?: number | null; /** Soft Budget */ soft_budget?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** @@ -24494,6 +24503,11 @@ export interface components { * @description Requests will NOT fail if this is exceeded. Will fire alerting though. */ soft_budget?: number | null; + /** + * Tpd Limit + * @description Max tokens per day, charged by batch submissions, allowed for this budget id. + */ + tpd_limit?: number | null; /** * Tpm Limit * @description Max tokens per minute, allowed for this budget id. @@ -27856,6 +27870,8 @@ export interface components { team_id?: string | null; /** Throttle On Budget Exceeded */ throttle_on_budget_exceeded?: boolean | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Type */ @@ -28016,6 +28032,8 @@ export interface components { token?: string | null; /** Token Id */ token_id?: string | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Type */ @@ -28746,6 +28764,8 @@ export interface components { rpm_limit?: number | null; /** Soft Budget */ soft_budget?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -28779,6 +28799,8 @@ export interface components { rpm_limit?: number | null; /** Soft Budget */ soft_budget?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -28887,6 +28909,8 @@ export interface components { team_id: string; /** Team Member Permissions */ team_member_permissions?: string[] | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Updated At */ @@ -29054,6 +29078,8 @@ export interface components { team_id?: string | null; /** Token */ token?: string | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Updated At */ @@ -30241,6 +30267,8 @@ export interface components { team_id: string; /** Team Member Permissions */ team_member_permissions?: string[] | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Updated At */ @@ -30617,6 +30645,8 @@ export interface components { team_id?: string | null; /** Token */ token?: string | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Updated At */ @@ -32260,6 +32290,11 @@ export interface components { soft_budget?: number | null; /** Spend */ spend?: number | null; + /** + * Tpd Limit + * @description Max tokens per day, charged by batch submissions, allowed for this budget id. + */ + tpd_limit?: number | null; /** * Tpm Limit * @description Max tokens per minute, allowed for this budget id. @@ -32475,6 +32510,8 @@ export interface components { rpm_limit?: number | null; /** Soft Budget */ soft_budget?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -32596,6 +32633,8 @@ export interface components { tags?: string[] | null; /** Team Id */ team_id: string; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -32782,6 +32821,8 @@ export interface components { team_member_rpm_limit?: number | null; /** Team Member Tpm Limit */ team_member_tpm_limit?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Type */ @@ -33075,6 +33116,8 @@ export interface components { token?: string | null; /** Token Id */ token_id?: string | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Type */ @@ -33531,6 +33574,8 @@ export interface components { team_member_rpm_limit?: number | null; /** Team Member Tpm Limit */ team_member_tpm_limit?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -34940,6 +34985,8 @@ export interface components { team_id?: string | null; /** Throttle On Budget Exceeded */ throttle_on_budget_exceeded?: boolean | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Type */ @@ -36917,6 +36964,8 @@ export interface components { team_id: string; /** Team Member Permissions */ team_member_permissions?: string[] | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Updated At */ @@ -37057,6 +37106,8 @@ export interface components { team_id: string; /** Team Member Permissions */ team_member_permissions?: string[] | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Updated At */ @@ -38124,6 +38175,8 @@ export interface components { temp_budget_increase?: number | null; /** Throttle On Budget Exceeded */ throttle_on_budget_exceeded?: boolean | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Type */ @@ -38388,6 +38441,8 @@ export interface components { tags?: string[] | null; /** Team Id */ team_id?: string | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -38583,6 +38638,8 @@ export interface components { team_member_rpm_limit?: number | null; /** Team Member Tpm Limit */ team_member_tpm_limit?: number | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; @@ -39093,6 +39150,8 @@ export interface components { end_user_object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null; /** End User Rpm Limit */ end_user_rpm_limit?: number | null; + /** End User Tpd Limit */ + end_user_tpd_limit?: number | null; /** End User Tpm Limit */ end_user_tpm_limit?: number | null; /** Expires */ @@ -39261,10 +39320,14 @@ export interface components { team_soft_budget?: number | null; /** Team Spend */ team_spend?: number | null; + /** Team Tpd Limit */ + team_tpd_limit?: number | null; /** Team Tpm Limit */ team_tpm_limit?: number | null; /** Token */ token?: string | null; + /** Tpd Limit */ + tpd_limit?: number | null; /** Tpm Limit */ tpm_limit?: number | null; /** Tpm Limit Per Model */